Brendon/gameyfin
1
0
Fork 0
mirror of https://github.com/BrenBroZAYT/gameyfin.git synced 2026-06-13 16:40:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement rate limiter for IGDB API

Browse Source 
Rename BlacklistEntry to UnmappableFile
This commit is contained in:
Simon Grimme
2022-07-16 02:07:46 +02:00
parent 183f58c64c
commit 132d4d3694
9 changed files with 59 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pom.xml
+13
View File
@@ -36,6 +36,19 @@
<artifactId>spring-boot-starter-webflux</artifactId>
</dependency>
<!-- Webclient Rate limiter -->
<dependency>
<groupId>io.github.resilience4j</groupId>
<artifactId>resilience4j-reactor</artifactId>
<version>1.7.1</version>
</dependency>
<dependency>
<groupId>io.github.resilience4j</groupId>
<artifactId>resilience4j-ratelimiter</artifactId>
<version>1.7.1</version>
</dependency>
<!-- Persistence -->
<dependency>
<groupId>com.h2database</groupId>
src/main/java/de/grimsi/gameyfin/config/WebClientConfig.java
+11
View File
@@ -1,5 +1,7 @@
package de.grimsi.gameyfin.config;
import io.github.resilience4j.ratelimiter.RateLimiter;
import io.github.resilience4j.ratelimiter.RateLimiterConfig;
import io.netty.handler.logging.LogLevel;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.web.reactive.function.client.WebClientCustomizer;
@@ -13,10 +15,19 @@ import reactor.core.publisher.Mono;
import reactor.netty.http.client.HttpClient;
import reactor.netty.transport.logging.AdvancedByteBufFormat;
import java.time.Duration;
@Slf4j
@Configuration
public class WebClientConfig implements WebClientCustomizer {
public static final RateLimiter IGDB_RATE_LIMITER = RateLimiter.of("igdb-rate-limiter",
RateLimiterConfig.custom()
.limitRefreshPeriod(Duration.ofSeconds(1))
.limitForPeriod(4)
.timeoutDuration(Duration.ofMinutes(1)) // max wait time for a request, if reached then error
.build());
@Override
public void customize(WebClient.Builder webClientBuilder) {
HttpClient httpClient = HttpClient.create()
src/main/java/de/grimsi/gameyfin/entities/BlacklistEntry.java → src/main/java/de/grimsi/gameyfin/entities/UnmappableFile.java
+11 -6
View File
@@ -3,9 +3,7 @@ package de.grimsi.gameyfin.entities;
import lombok.*;
import org.hibernate.Hibernate;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
import javax.persistence.*;
import java.util.Objects;
@Entity
@@ -14,16 +12,23 @@ import java.util.Objects;
@Setter
@ToString
@NoArgsConstructor
@AllArgsConstructor
public class BlacklistEntry {
public class UnmappableFile {
public UnmappableFile(String path) {
this.path = path;
}
@Id
@GeneratedValue(strategy = GenerationType.SEQUENCE)
private Long id;
private String path;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
BlacklistEntry that = (BlacklistEntry) o;
UnmappableFile that = (UnmappableFile) o;
return path != null && Objects.equals(path, that.path);
}
src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java
+3
View File
@@ -3,6 +3,7 @@ package de.grimsi.gameyfin.igdb;
import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.config.WebClientConfig;
import de.grimsi.gameyfin.igdb.dto.TwitchOAuthTokenDto;
import io.github.resilience4j.reactor.ratelimiter.operator.RateLimiterOperator;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@@ -72,6 +73,7 @@ public class IgdbWrapper {
.bodyValue("fields *; where id = %d & category = %d; limit 1;".formatted(id, MAIN_GAME_CATEGORY_VALUE))
.retrieve()
.bodyToMono(Igdb.GameResult.class)
.transformDeferred(RateLimiterOperator.of(WebClientConfig.IGDB_RATE_LIMITER))
.block();
if (gameResult == null) return Optional.empty();
@@ -85,6 +87,7 @@ public class IgdbWrapper {
.bodyValue("fields *; search \"%s\"; where platforms = (%s) & category = %d;".formatted(searchTerm, preferredPlatforms, MAIN_GAME_CATEGORY_VALUE))
.retrieve()
.bodyToMono(Igdb.GameResult.class)
.transformDeferred(RateLimiterOperator.of(WebClientConfig.IGDB_RATE_LIMITER))
.block();
if (gameResult == null) {
src/main/java/de/grimsi/gameyfin/repositories/BlacklistRepository.java → src/main/java/de/grimsi/gameyfin/repositories/UnmappableFileRepository.java
+2 -2
View File
@@ -1,9 +1,9 @@
package de.grimsi.gameyfin.repositories;
import de.grimsi.gameyfin.entities.BlacklistEntry;
import de.grimsi.gameyfin.entities.UnmappableFile;
import org.springframework.data.jpa.repository.JpaRepository;
public interface BlacklistRepository extends JpaRepository<BlacklistEntry, String> {
public interface UnmappableFileRepository extends JpaRepository<UnmappableFile, String> {
boolean existsByPath(String path);
}
src/main/java/de/grimsi/gameyfin/rest/GameyfinDevController.java
+6
View File
@@ -3,6 +3,7 @@ package de.grimsi.gameyfin.rest;
import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.dto.GameDto;
import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.entities.UnmappableFile;
import de.grimsi.gameyfin.igdb.IgdbWrapper;
import de.grimsi.gameyfin.service.FilesystemService;
import de.grimsi.gameyfin.service.GameService;
@@ -68,4 +69,9 @@ public class GameyfinDevController {
filesystemService.scanGameLibrary();
}
@GetMapping(value = "/dev/unmappedFiles", produces = MediaType.APPLICATION_JSON_VALUE)
public List<UnmappableFile> getUnmappedFiles() {
return gameService.getAllUnmappedFiles();
}
}
src/main/java/de/grimsi/gameyfin/service/FilesystemService.java
+5 -5
View File
@@ -1,11 +1,11 @@
package de.grimsi.gameyfin.service;
import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.entities.BlacklistEntry;
import de.grimsi.gameyfin.entities.UnmappableFile;
import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.igdb.IgdbWrapper;
import de.grimsi.gameyfin.mapper.GameMapper;
import de.grimsi.gameyfin.repositories.BlacklistRepository;
import de.grimsi.gameyfin.repositories.UnmappableFileRepository;
import de.grimsi.gameyfin.repositories.DetectedGameRepository;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FilenameUtils;
@@ -38,7 +38,7 @@ public class FilesystemService {
private DetectedGameRepository detectedGameRepository;
@Autowired
private BlacklistRepository blacklistRepository;
private UnmappableFileRepository unmappableFileRepository;
public List<Path> getGameFiles() {
@@ -68,7 +68,7 @@ public class FilesystemService {
// Filter out the games we already know and the ones we already tried to map to a game without success
gameFiles = gameFiles.stream()
.filter(g -> !detectedGameRepository.existsByPath(g.toString()))
.filter(g -> !blacklistRepository.existsByPath(g.toString()))
.filter(g -> !unmappableFileRepository.existsByPath(g.toString()))
.peek(p -> log.info("Found new potential game: {}", p))
.toList();
@@ -78,7 +78,7 @@ public class FilesystemService {
.map(p -> {
Optional<Igdb.Game> optionalGame = igdbWrapper.searchForGameByTitle(getFilename(p));
return optionalGame.map(game -> Map.entry(p, game)).or(() -> {
blacklistRepository.save(new BlacklistEntry(p.toString()));
unmappableFileRepository.save(new UnmappableFile(p.toString()));
newBlacklistCounter.getAndIncrement();
log.info("Added path '{}' to blacklist", p);
return Optional.empty();
src/main/java/de/grimsi/gameyfin/service/GameService.java
+7 -2
View File
@@ -1,7 +1,8 @@
package de.grimsi.gameyfin.service;
import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.repositories.BlacklistRepository;
import de.grimsi.gameyfin.entities.UnmappableFile;
import de.grimsi.gameyfin.repositories.UnmappableFileRepository;
import de.grimsi.gameyfin.repositories.DetectedGameRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -15,9 +16,13 @@ public class GameService {
private DetectedGameRepository detectedGameRepository;
@Autowired
private BlacklistRepository blacklistRepository;
private UnmappableFileRepository unmappableFileRepository;
public List<DetectedGame> getAllDetectedGames() {
return detectedGameRepository.findAll();
}
public List<UnmappableFile> getAllUnmappedFiles() {
return unmappableFileRepository.findAll();
}
}
src/main/resources/application-dev.yml
+1 -1
View File
@@ -1,5 +1,5 @@
gameyfin:
root: D:\Games
root: \\NAS-Simon\Öffentlich\Spiele
cache: C:\Projects\privat\gameyfin-library\.gameyfin
db: C:\Projects\privat\gameyfin-library\.gameyfin
igdb: