Implement rate limiter for IGDB API

Rename BlacklistEntry to UnmappableFile

pom.xml

@@ -36,6 +36,19 @@
 			<artifactId>spring-boot-starter-webflux</artifactId>
 		</dependency>
 
+		<!-- Webclient Rate limiter -->
+		<dependency>
+			<groupId>io.github.resilience4j</groupId>
+			<artifactId>resilience4j-reactor</artifactId>
+			<version>1.7.1</version>
+		</dependency>
+		<dependency>
+			<groupId>io.github.resilience4j</groupId>
+			<artifactId>resilience4j-ratelimiter</artifactId>
+			<version>1.7.1</version>
+		</dependency>
+
+
 		<!-- Persistence -->
 		<dependency>
 			<groupId>com.h2database</groupId>

src/main/java/de/grimsi/gameyfin/config/WebClientConfig.java

@@ -1,5 +1,7 @@
 package de.grimsi.gameyfin.config;
 
+import io.github.resilience4j.ratelimiter.RateLimiter;
+import io.github.resilience4j.ratelimiter.RateLimiterConfig;
 import io.netty.handler.logging.LogLevel;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.boot.web.reactive.function.client.WebClientCustomizer;
@@ -13,10 +15,19 @@ import reactor.core.publisher.Mono;
 import reactor.netty.http.client.HttpClient;
 import reactor.netty.transport.logging.AdvancedByteBufFormat;
 
+import java.time.Duration;
+
 @Slf4j
 @Configuration
 public class WebClientConfig implements WebClientCustomizer {
 
+    public static final RateLimiter IGDB_RATE_LIMITER = RateLimiter.of("igdb-rate-limiter",
+            RateLimiterConfig.custom()
+                    .limitRefreshPeriod(Duration.ofSeconds(1))
+                    .limitForPeriod(4)
+                    .timeoutDuration(Duration.ofMinutes(1)) // max wait time for a request, if reached then error
+            .build());
+
     @Override
     public void customize(WebClient.Builder webClientBuilder) {
         HttpClient httpClient = HttpClient.create()

src/main/java/de/grimsi/gameyfin/entities/UnmappableFile.java

@@ -3,9 +3,7 @@ package de.grimsi.gameyfin.entities;
 import lombok.*;
 import org.hibernate.Hibernate;
 
-import javax.persistence.Entity;
+import javax.persistence.*;
-import javax.persistence.Id;
-import javax.persistence.Table;
 import java.util.Objects;
 
 @Entity
@@ -14,16 +12,23 @@ import java.util.Objects;
 @Setter
 @ToString
 @NoArgsConstructor
-@AllArgsConstructor
+public class UnmappableFile {
-public class BlacklistEntry {
+
+    public UnmappableFile(String path) {
+        this.path = path;
+    }
+
     @Id
+    @GeneratedValue(strategy = GenerationType.SEQUENCE)
+    private Long id;
+
     private String path;
 
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
-        BlacklistEntry that = (BlacklistEntry) o;
+        UnmappableFile that = (UnmappableFile) o;
         return path != null && Objects.equals(path, that.path);
     }
 

src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java

@@ -3,6 +3,7 @@ package de.grimsi.gameyfin.igdb;
 import com.igdb.proto.Igdb;
 import de.grimsi.gameyfin.config.WebClientConfig;
 import de.grimsi.gameyfin.igdb.dto.TwitchOAuthTokenDto;
+import io.github.resilience4j.reactor.ratelimiter.operator.RateLimiterOperator;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -72,6 +73,7 @@ public class IgdbWrapper {
                 .bodyValue("fields *; where id = %d & category = %d; limit 1;".formatted(id, MAIN_GAME_CATEGORY_VALUE))
                 .retrieve()
                 .bodyToMono(Igdb.GameResult.class)
+                .transformDeferred(RateLimiterOperator.of(WebClientConfig.IGDB_RATE_LIMITER))
                 .block();
 
         if (gameResult == null) return Optional.empty();
@@ -85,6 +87,7 @@ public class IgdbWrapper {
                 .bodyValue("fields *; search \"%s\"; where platforms = (%s) & category = %d;".formatted(searchTerm, preferredPlatforms, MAIN_GAME_CATEGORY_VALUE))
                 .retrieve()
                 .bodyToMono(Igdb.GameResult.class)
+                .transformDeferred(RateLimiterOperator.of(WebClientConfig.IGDB_RATE_LIMITER))
                 .block();
 
         if (gameResult == null) {

src/main/java/de/grimsi/gameyfin/repositories/UnmappableFileRepository.java

@@ -1,9 +1,9 @@
 package de.grimsi.gameyfin.repositories;
 
-import de.grimsi.gameyfin.entities.BlacklistEntry;
+import de.grimsi.gameyfin.entities.UnmappableFile;
 import org.springframework.data.jpa.repository.JpaRepository;
 
-public interface BlacklistRepository extends JpaRepository<BlacklistEntry, String> {
+public interface UnmappableFileRepository extends JpaRepository<UnmappableFile, String> {
 
     boolean existsByPath(String path);
 }

src/main/java/de/grimsi/gameyfin/rest/GameyfinDevController.java

@@ -3,6 +3,7 @@ package de.grimsi.gameyfin.rest;
 import com.igdb.proto.Igdb;
 import de.grimsi.gameyfin.dto.GameDto;
 import de.grimsi.gameyfin.entities.DetectedGame;
+import de.grimsi.gameyfin.entities.UnmappableFile;
 import de.grimsi.gameyfin.igdb.IgdbWrapper;
 import de.grimsi.gameyfin.service.FilesystemService;
 import de.grimsi.gameyfin.service.GameService;
@@ -68,4 +69,9 @@ public class GameyfinDevController {
         filesystemService.scanGameLibrary();
     }
 
+    @GetMapping(value = "/dev/unmappedFiles", produces = MediaType.APPLICATION_JSON_VALUE)
+    public List<UnmappableFile> getUnmappedFiles() {
+        return gameService.getAllUnmappedFiles();
+    }
+
 }

src/main/java/de/grimsi/gameyfin/service/FilesystemService.java

@@ -1,11 +1,11 @@
 package de.grimsi.gameyfin.service;
 
 import com.igdb.proto.Igdb;
-import de.grimsi.gameyfin.entities.BlacklistEntry;
+import de.grimsi.gameyfin.entities.UnmappableFile;
 import de.grimsi.gameyfin.entities.DetectedGame;
 import de.grimsi.gameyfin.igdb.IgdbWrapper;
 import de.grimsi.gameyfin.mapper.GameMapper;
-import de.grimsi.gameyfin.repositories.BlacklistRepository;
+import de.grimsi.gameyfin.repositories.UnmappableFileRepository;
 import de.grimsi.gameyfin.repositories.DetectedGameRepository;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FilenameUtils;
@@ -38,7 +38,7 @@ public class FilesystemService {
     private DetectedGameRepository detectedGameRepository;
 
     @Autowired
-    private BlacklistRepository blacklistRepository;
+    private UnmappableFileRepository unmappableFileRepository;
 
     public List<Path> getGameFiles() {
 
@@ -68,7 +68,7 @@ public class FilesystemService {
         // Filter out the games we already know and the ones we already tried to map to a game without success
         gameFiles = gameFiles.stream()
                 .filter(g -> !detectedGameRepository.existsByPath(g.toString()))
-                .filter(g -> !blacklistRepository.existsByPath(g.toString()))
+                .filter(g -> !unmappableFileRepository.existsByPath(g.toString()))
                 .peek(p -> log.info("Found new potential game: {}", p))
                 .toList();
 
@@ -78,7 +78,7 @@ public class FilesystemService {
                 .map(p -> {
                     Optional<Igdb.Game> optionalGame = igdbWrapper.searchForGameByTitle(getFilename(p));
                     return optionalGame.map(game -> Map.entry(p, game)).or(() -> {
-                        blacklistRepository.save(new BlacklistEntry(p.toString()));
+                        unmappableFileRepository.save(new UnmappableFile(p.toString()));
                         newBlacklistCounter.getAndIncrement();
                         log.info("Added path '{}' to blacklist", p);
                         return Optional.empty();

src/main/java/de/grimsi/gameyfin/service/GameService.java

@@ -1,7 +1,8 @@
 package de.grimsi.gameyfin.service;
 
 import de.grimsi.gameyfin.entities.DetectedGame;
-import de.grimsi.gameyfin.repositories.BlacklistRepository;
+import de.grimsi.gameyfin.entities.UnmappableFile;
+import de.grimsi.gameyfin.repositories.UnmappableFileRepository;
 import de.grimsi.gameyfin.repositories.DetectedGameRepository;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -15,9 +16,13 @@ public class GameService {
     private DetectedGameRepository detectedGameRepository;
 
     @Autowired
-    private BlacklistRepository blacklistRepository;
+    private UnmappableFileRepository unmappableFileRepository;
 
     public List<DetectedGame> getAllDetectedGames() {
         return detectedGameRepository.findAll();
     }
+
+    public List<UnmappableFile> getAllUnmappedFiles() {
+        return unmappableFileRepository.findAll();
+    }
 }

src/main/resources/application-dev.yml

@@ -1,5 +1,5 @@
 gameyfin:
-  root: D:\Games
+  root: \\NAS-Simon\Öffentlich\Spiele
   cache: C:\Projects\privat\gameyfin-library\.gameyfin
   db: C:\Projects\privat\gameyfin-library\.gameyfin
   igdb: