Added config properties for max req/s and max concurrent requests

This commit is contained in:
grimsi
2022-08-10 21:01:24 +02:00
parent 6d0f0d4c01
commit c196fd2cbc
8 changed files with 53 additions and 32 deletions
+2 -1
View File
@@ -7,7 +7,7 @@
<parent>
<artifactId>gameyfin</artifactId>
<groupId>de.grimsi</groupId>
<version>1.0.0</version>
<version>1.0.1</version>
</parent>
<artifactId>gameyfin-backend</artifactId>
@@ -131,6 +131,7 @@
</dependencies>
<build>
<finalName>gameyfin-${project.parent.version}</finalName>
<resources>
<resource>
<directory>${basedir}/src/main/resources</directory>
@@ -5,7 +5,9 @@ import io.github.resilience4j.bulkhead.BulkheadConfig;
import io.github.resilience4j.ratelimiter.RateLimiter;
import io.github.resilience4j.ratelimiter.RateLimiterConfig;
import io.netty.handler.logging.LogLevel;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.web.reactive.function.client.WebClientCustomizer;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.HttpHeaders;
@@ -20,24 +22,32 @@ import reactor.netty.transport.logging.AdvancedByteBufFormat;
import java.time.Duration;
@Slf4j
@Getter
@Configuration
public class WebClientConfig implements WebClientCustomizer {
// The IGDB API has a rate limit of 4 req/s
public static final RateLimiter IGDB_RATE_LIMITER = RateLimiter.of("igdb-rate-limiter",
RateLimiterConfig.custom()
.limitForPeriod(4)
.limitRefreshPeriod(Duration.ofSeconds(1))
.timeoutDuration(Duration.ofMinutes(1))
.build());
private final RateLimiter igdbRateLimiter;
private final Bulkhead igdbConcurrencyLimiter;
// According to the docs, there is a maximum of 8 concurrent requests, but in my tests the actual limit was 4
// and even then it sometimes failed, so I set it to 3 to be sure
public static final Bulkhead IGDB_CONCURRENCY_LIMITER = Bulkhead.of("igdb-concurrency-limiter",
BulkheadConfig.custom()
.maxConcurrentCalls(2)
.maxWaitDuration(Duration.ofMinutes(1))
.build());
public WebClientConfig(@Value("${gameyfin.igdb.api.max-concurrent-requests}") int maxConcurrentRequestsToIgdb,
@Value("${gameyfin.igdb.api.max-requests-per-second}") int maxRequestsPerSecondToIgdb) {
log.info("IGDB API connection properties: max. {} req/s, max. {} concurrent requests", maxRequestsPerSecondToIgdb, maxConcurrentRequestsToIgdb);
igdbRateLimiter = RateLimiter.of("igdb-rate-limiter",
RateLimiterConfig.custom()
.limitForPeriod(maxRequestsPerSecondToIgdb)
.limitRefreshPeriod(Duration.ofSeconds(1))
.timeoutDuration(Duration.ofMinutes(1))
.build());
igdbConcurrencyLimiter = Bulkhead.of("igdb-concurrency-limiter",
BulkheadConfig.custom()
.maxConcurrentCalls(maxConcurrentRequestsToIgdb)
.maxWaitDuration(Duration.ofMinutes(1))
.build());
}
@Override
public void customize(WebClient.Builder webClientBuilder) {
@@ -38,6 +38,9 @@ public class IgdbWrapper {
@Autowired
private WebClient.Builder webclientBuilder;
@Autowired
private WebClientConfig webClientConfig;
private WebClient twitchApiClient;
private WebClient igdbApiClient;
@@ -173,8 +176,8 @@ public class IgdbWrapper {
.bodyValue(query)
.retrieve()
.bodyToMono(responseClass)
.transformDeferred(BulkheadOperator.of(WebClientConfig.IGDB_CONCURRENCY_LIMITER))
.transformDeferred(RateLimiterOperator.of(WebClientConfig.IGDB_RATE_LIMITER))
.transformDeferred(BulkheadOperator.of(webClientConfig.getIgdbConcurrencyLimiter()))
.transformDeferred(RateLimiterOperator.of(webClientConfig.getIgdbRateLimiter()))
.block();
}
}
@@ -14,8 +14,15 @@ gameyfin.db=${gameyfin.root}/.gameyfin/db
# File extensions which gameyfin will recognize as game files
gameyfin.file-extensions=iso, zip, rar, 7z, exe
# List of IGDB platform enums to limit search results. FOr possible values see: https://api-docs.igdb.com/#platform
# List of IGDB platform enums to limit search results. For possible values see: https://api-docs.igdb.com/#platform
gameyfin.igdb.config.preferred-platforms=6
# Twitch Client ID and Client Secret
gameyfin.igdb.api.client-id=
gameyfin.igdb.api.client-secret=
gameyfin.igdb.api.client-secret=
# The IGDB API has a rate limit of 4 req/s
gameyfin.igdb.api.max-requests-per-second=4
# According to the docs, there is a maximum of 8 concurrent requests, but in my tests the actual limit was 4 and even then it sometimes failed, so I set it to 2 to be sure
gameyfin.igdb.api.max-concurrent-requests=2
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "frontend",
"version": "0.0.1-SNAPSHOT",
"version": "@project.version@",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "frontend",
"version": "0.0.1-SNAPSHOT",
"version": "@project.version@",
"dependencies": {
"@angular/animations": "^14.0.0",
"@angular/cdk": "^14.1.0",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "frontend",
"version": "0.0.1-SNAPSHOT",
"version": "@project.version@",
"scripts": {
"ng": "ng",
"start": "ng serve",
+9 -9
View File
@@ -5,7 +5,7 @@
<parent>
<artifactId>gameyfin</artifactId>
<groupId>de.grimsi</groupId>
<version>1.0.0</version>
<version>1.0.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -18,6 +18,13 @@
</properties>
<build>
<resources>
<resource>
<directory>./dist/frontend</directory>
<targetPath>static</targetPath>
</resource>
</resources>
<plugins>
<!-- clean the dist directory used by Angular -->
<plugin>
@@ -63,19 +70,12 @@
<goal>npm</goal>
</goals>
<configuration>
<arguments>run build --prod</arguments>
<arguments>run build --omit=dev</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<resources>
<resource>
<directory>./dist/frontend</directory>
<targetPath>static</targetPath>
</resource>
</resources>
</build>
</project>
+1 -1
View File
@@ -5,7 +5,7 @@
<groupId>de.grimsi</groupId>
<artifactId>gameyfin</artifactId>
<version>1.0.0</version>
<version>1.0.1</version>
<name>gameyfin</name>
<description>gameyfin</description>