From 132d4d36946ba22de0a8cae38c29a088e2b621f4 Mon Sep 17 00:00:00 2001 From: Simon Grimme <9295182+grimsi@users.noreply.github.com> Date: Sat, 16 Jul 2022 02:07:46 +0200 Subject: [PATCH] Implement rate limiter for IGDB API Rename BlacklistEntry to UnmappableFile --- pom.xml | 13 +++++++++++++ .../grimsi/gameyfin/config/WebClientConfig.java | 11 +++++++++++ ...{BlacklistEntry.java => UnmappableFile.java} | 17 +++++++++++------ .../de/grimsi/gameyfin/igdb/IgdbWrapper.java | 3 +++ ...itory.java => UnmappableFileRepository.java} | 4 ++-- .../gameyfin/rest/GameyfinDevController.java | 6 ++++++ .../gameyfin/service/FilesystemService.java | 10 +++++----- .../de/grimsi/gameyfin/service/GameService.java | 9 +++++++-- src/main/resources/application-dev.yml | 2 +- 9 files changed, 59 insertions(+), 16 deletions(-) rename src/main/java/de/grimsi/gameyfin/entities/{BlacklistEntry.java => UnmappableFile.java} (68%) rename src/main/java/de/grimsi/gameyfin/repositories/{BlacklistRepository.java => UnmappableFileRepository.java} (51%) diff --git a/pom.xml b/pom.xml index 173f962..249fa9a 100644 --- a/pom.xml +++ b/pom.xml @@ -36,6 +36,19 @@ spring-boot-starter-webflux + + + io.github.resilience4j + resilience4j-reactor + 1.7.1 + + + io.github.resilience4j + resilience4j-ratelimiter + 1.7.1 + + + com.h2database diff --git a/src/main/java/de/grimsi/gameyfin/config/WebClientConfig.java b/src/main/java/de/grimsi/gameyfin/config/WebClientConfig.java index fa1f625..41a8429 100644 --- a/src/main/java/de/grimsi/gameyfin/config/WebClientConfig.java +++ b/src/main/java/de/grimsi/gameyfin/config/WebClientConfig.java @@ -1,5 +1,7 @@ package de.grimsi.gameyfin.config; +import io.github.resilience4j.ratelimiter.RateLimiter; +import io.github.resilience4j.ratelimiter.RateLimiterConfig; import io.netty.handler.logging.LogLevel; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.web.reactive.function.client.WebClientCustomizer; @@ -13,10 +15,19 @@ import reactor.core.publisher.Mono; import reactor.netty.http.client.HttpClient; import reactor.netty.transport.logging.AdvancedByteBufFormat; +import java.time.Duration; + @Slf4j @Configuration public class WebClientConfig implements WebClientCustomizer { + public static final RateLimiter IGDB_RATE_LIMITER = RateLimiter.of("igdb-rate-limiter", + RateLimiterConfig.custom() + .limitRefreshPeriod(Duration.ofSeconds(1)) + .limitForPeriod(4) + .timeoutDuration(Duration.ofMinutes(1)) // max wait time for a request, if reached then error + .build()); + @Override public void customize(WebClient.Builder webClientBuilder) { HttpClient httpClient = HttpClient.create() diff --git a/src/main/java/de/grimsi/gameyfin/entities/BlacklistEntry.java b/src/main/java/de/grimsi/gameyfin/entities/UnmappableFile.java similarity index 68% rename from src/main/java/de/grimsi/gameyfin/entities/BlacklistEntry.java rename to src/main/java/de/grimsi/gameyfin/entities/UnmappableFile.java index ee3ad31..13330f2 100644 --- a/src/main/java/de/grimsi/gameyfin/entities/BlacklistEntry.java +++ b/src/main/java/de/grimsi/gameyfin/entities/UnmappableFile.java @@ -3,9 +3,7 @@ package de.grimsi.gameyfin.entities; import lombok.*; import org.hibernate.Hibernate; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.persistence.Table; +import javax.persistence.*; import java.util.Objects; @Entity @@ -14,16 +12,23 @@ import java.util.Objects; @Setter @ToString @NoArgsConstructor -@AllArgsConstructor -public class BlacklistEntry { +public class UnmappableFile { + + public UnmappableFile(String path) { + this.path = path; + } + @Id + @GeneratedValue(strategy = GenerationType.SEQUENCE) + private Long id; + private String path; @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; - BlacklistEntry that = (BlacklistEntry) o; + UnmappableFile that = (UnmappableFile) o; return path != null && Objects.equals(path, that.path); } diff --git a/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java b/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java index 2f4a089..73b9f7c 100644 --- a/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java +++ b/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java @@ -3,6 +3,7 @@ package de.grimsi.gameyfin.igdb; import com.igdb.proto.Igdb; import de.grimsi.gameyfin.config.WebClientConfig; import de.grimsi.gameyfin.igdb.dto.TwitchOAuthTokenDto; +import io.github.resilience4j.reactor.ratelimiter.operator.RateLimiterOperator; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -72,6 +73,7 @@ public class IgdbWrapper { .bodyValue("fields *; where id = %d & category = %d; limit 1;".formatted(id, MAIN_GAME_CATEGORY_VALUE)) .retrieve() .bodyToMono(Igdb.GameResult.class) + .transformDeferred(RateLimiterOperator.of(WebClientConfig.IGDB_RATE_LIMITER)) .block(); if (gameResult == null) return Optional.empty(); @@ -85,6 +87,7 @@ public class IgdbWrapper { .bodyValue("fields *; search \"%s\"; where platforms = (%s) & category = %d;".formatted(searchTerm, preferredPlatforms, MAIN_GAME_CATEGORY_VALUE)) .retrieve() .bodyToMono(Igdb.GameResult.class) + .transformDeferred(RateLimiterOperator.of(WebClientConfig.IGDB_RATE_LIMITER)) .block(); if (gameResult == null) { diff --git a/src/main/java/de/grimsi/gameyfin/repositories/BlacklistRepository.java b/src/main/java/de/grimsi/gameyfin/repositories/UnmappableFileRepository.java similarity index 51% rename from src/main/java/de/grimsi/gameyfin/repositories/BlacklistRepository.java rename to src/main/java/de/grimsi/gameyfin/repositories/UnmappableFileRepository.java index 8c8407a..9ad79e8 100644 --- a/src/main/java/de/grimsi/gameyfin/repositories/BlacklistRepository.java +++ b/src/main/java/de/grimsi/gameyfin/repositories/UnmappableFileRepository.java @@ -1,9 +1,9 @@ package de.grimsi.gameyfin.repositories; -import de.grimsi.gameyfin.entities.BlacklistEntry; +import de.grimsi.gameyfin.entities.UnmappableFile; import org.springframework.data.jpa.repository.JpaRepository; -public interface BlacklistRepository extends JpaRepository { +public interface UnmappableFileRepository extends JpaRepository { boolean existsByPath(String path); } diff --git a/src/main/java/de/grimsi/gameyfin/rest/GameyfinDevController.java b/src/main/java/de/grimsi/gameyfin/rest/GameyfinDevController.java index a59e27f..e87d881 100644 --- a/src/main/java/de/grimsi/gameyfin/rest/GameyfinDevController.java +++ b/src/main/java/de/grimsi/gameyfin/rest/GameyfinDevController.java @@ -3,6 +3,7 @@ package de.grimsi.gameyfin.rest; import com.igdb.proto.Igdb; import de.grimsi.gameyfin.dto.GameDto; import de.grimsi.gameyfin.entities.DetectedGame; +import de.grimsi.gameyfin.entities.UnmappableFile; import de.grimsi.gameyfin.igdb.IgdbWrapper; import de.grimsi.gameyfin.service.FilesystemService; import de.grimsi.gameyfin.service.GameService; @@ -68,4 +69,9 @@ public class GameyfinDevController { filesystemService.scanGameLibrary(); } + @GetMapping(value = "/dev/unmappedFiles", produces = MediaType.APPLICATION_JSON_VALUE) + public List getUnmappedFiles() { + return gameService.getAllUnmappedFiles(); + } + } diff --git a/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java b/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java index 53fae2f..ed28781 100644 --- a/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java +++ b/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java @@ -1,11 +1,11 @@ package de.grimsi.gameyfin.service; import com.igdb.proto.Igdb; -import de.grimsi.gameyfin.entities.BlacklistEntry; +import de.grimsi.gameyfin.entities.UnmappableFile; import de.grimsi.gameyfin.entities.DetectedGame; import de.grimsi.gameyfin.igdb.IgdbWrapper; import de.grimsi.gameyfin.mapper.GameMapper; -import de.grimsi.gameyfin.repositories.BlacklistRepository; +import de.grimsi.gameyfin.repositories.UnmappableFileRepository; import de.grimsi.gameyfin.repositories.DetectedGameRepository; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FilenameUtils; @@ -38,7 +38,7 @@ public class FilesystemService { private DetectedGameRepository detectedGameRepository; @Autowired - private BlacklistRepository blacklistRepository; + private UnmappableFileRepository unmappableFileRepository; public List getGameFiles() { @@ -68,7 +68,7 @@ public class FilesystemService { // Filter out the games we already know and the ones we already tried to map to a game without success gameFiles = gameFiles.stream() .filter(g -> !detectedGameRepository.existsByPath(g.toString())) - .filter(g -> !blacklistRepository.existsByPath(g.toString())) + .filter(g -> !unmappableFileRepository.existsByPath(g.toString())) .peek(p -> log.info("Found new potential game: {}", p)) .toList(); @@ -78,7 +78,7 @@ public class FilesystemService { .map(p -> { Optional optionalGame = igdbWrapper.searchForGameByTitle(getFilename(p)); return optionalGame.map(game -> Map.entry(p, game)).or(() -> { - blacklistRepository.save(new BlacklistEntry(p.toString())); + unmappableFileRepository.save(new UnmappableFile(p.toString())); newBlacklistCounter.getAndIncrement(); log.info("Added path '{}' to blacklist", p); return Optional.empty(); diff --git a/src/main/java/de/grimsi/gameyfin/service/GameService.java b/src/main/java/de/grimsi/gameyfin/service/GameService.java index 5d1789f..1e2cf0c 100644 --- a/src/main/java/de/grimsi/gameyfin/service/GameService.java +++ b/src/main/java/de/grimsi/gameyfin/service/GameService.java @@ -1,7 +1,8 @@ package de.grimsi.gameyfin.service; import de.grimsi.gameyfin.entities.DetectedGame; -import de.grimsi.gameyfin.repositories.BlacklistRepository; +import de.grimsi.gameyfin.entities.UnmappableFile; +import de.grimsi.gameyfin.repositories.UnmappableFileRepository; import de.grimsi.gameyfin.repositories.DetectedGameRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -15,9 +16,13 @@ public class GameService { private DetectedGameRepository detectedGameRepository; @Autowired - private BlacklistRepository blacklistRepository; + private UnmappableFileRepository unmappableFileRepository; public List getAllDetectedGames() { return detectedGameRepository.findAll(); } + + public List getAllUnmappedFiles() { + return unmappableFileRepository.findAll(); + } } diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 8ec0e05..5d47901 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -1,5 +1,5 @@ gameyfin: - root: D:\Games + root: \\NAS-Simon\Öffentlich\Spiele cache: C:\Projects\privat\gameyfin-library\.gameyfin db: C:\Projects\privat\gameyfin-library\.gameyfin igdb: