Optimized query to IGDB API

Implemented mapping of nested fields
This commit is contained in:
grimsi
2022-07-16 15:47:20 +02:00
parent 937c440cdf
commit 64485bf3f0
27 changed files with 277 additions and 102 deletions
+5
View File
@@ -47,6 +47,11 @@
<artifactId>resilience4j-ratelimiter</artifactId>
<version>1.7.1</version>
</dependency>
<dependency>
<groupId>io.github.resilience4j</groupId>
<artifactId>resilience4j-bulkhead</artifactId>
<version>1.7.1</version>
</dependency>
<!-- Persistence -->
@@ -1,5 +1,7 @@
package de.grimsi.gameyfin.config;
import io.github.resilience4j.bulkhead.Bulkhead;
import io.github.resilience4j.bulkhead.BulkheadConfig;
import io.github.resilience4j.ratelimiter.RateLimiter;
import io.github.resilience4j.ratelimiter.RateLimiterConfig;
import io.netty.handler.logging.LogLevel;
@@ -21,12 +23,21 @@ import java.time.Duration;
@Configuration
public class WebClientConfig implements WebClientCustomizer {
// The IGDB API has a rate limit of 4 req/s
public static final RateLimiter IGDB_RATE_LIMITER = RateLimiter.of("igdb-rate-limiter",
RateLimiterConfig.custom()
.limitRefreshPeriod(Duration.ofSeconds(1))
.limitForPeriod(4)
.timeoutDuration(Duration.ofMinutes(1)) // max wait time for a request, if reached then error
.build());
.limitRefreshPeriod(Duration.ofSeconds(1))
.timeoutDuration(Duration.ofMinutes(1))
.build());
// According to the docs, there is a maximum of 8 concurrent requests, but in my tests the actual limit was 4
// and even then it sometimes failed, so I set it to 3 to be sure
public static final Bulkhead IGDB_CONCURRENCY_LIMITER = Bulkhead.of("igdb-concurrency-limiter",
BulkheadConfig.custom()
.maxConcurrentCalls(2)
.maxWaitDuration(Duration.ofMinutes(1))
.build());
@Override
public void customize(WebClient.Builder webClientBuilder) {
@@ -1,33 +0,0 @@
package de.grimsi.gameyfin.entities;
import lombok.*;
import org.hibernate.Hibernate;
import javax.persistence.Entity;
import javax.persistence.Id;
import java.util.Objects;
@Entity
@Getter
@Setter
@ToString
@RequiredArgsConstructor
public class Category {
@Id
private String slug;
private String name;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
Category category = (Category) o;
return slug != null && Objects.equals(slug, category.slug);
}
@Override
public int hashCode() {
return getClass().hashCode();
}
}
@@ -1,20 +1,21 @@
package de.grimsi.gameyfin.entities;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.*;
import org.hibernate.Hibernate;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Version;
import java.util.Objects;
@Entity
@Builder
@Getter
@Setter
@ToString
@AllArgsConstructor
@RequiredArgsConstructor
public class Company {
@Id
@@ -1,6 +1,7 @@
package de.grimsi.gameyfin.entities;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.*;
import org.hibernate.Hibernate;
@@ -37,8 +38,7 @@ public class DetectedGame {
private Integer totalRating;
@ManyToOne
private Category category;
private String category;
private boolean offlineCoop;
@@ -57,23 +57,23 @@ public class DetectedGame {
@ElementCollection
private List<Long> videoIds;
@ManyToMany
@ManyToMany(cascade = CascadeType.MERGE)
@ToString.Exclude
private List<Company> companies;
@ManyToMany
@ManyToMany(cascade = CascadeType.MERGE)
@ToString.Exclude
private List<Genre> genres;
@ManyToMany
@ManyToMany(cascade = CascadeType.MERGE)
@ToString.Exclude
private List<Keyword> keywords;
@ManyToMany
@ManyToMany(cascade = CascadeType.MERGE)
@ToString.Exclude
private List<Theme> themes;
@ManyToMany
@ManyToMany(cascade = CascadeType.MERGE)
@ToString.Exclude
private List<PlayerPerspective> playerPerspectives;
@@ -1,19 +1,20 @@
package de.grimsi.gameyfin.entities;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.*;
import org.hibernate.Hibernate;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Version;
import java.util.Objects;
@Entity
@Builder
@Getter
@Setter
@ToString
@AllArgsConstructor
@RequiredArgsConstructor
public class Genre {
@Id
@@ -1,16 +1,20 @@
package de.grimsi.gameyfin.entities;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.*;
import org.hibernate.Hibernate;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Version;
import java.util.Objects;
@Entity
@Builder
@Getter
@Setter
@ToString
@AllArgsConstructor
@RequiredArgsConstructor
public class Keyword {
@Id
@@ -1,16 +1,20 @@
package de.grimsi.gameyfin.entities;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.*;
import org.hibernate.Hibernate;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Version;
import java.util.Objects;
@Entity
@Builder
@Getter
@Setter
@ToString
@AllArgsConstructor
@RequiredArgsConstructor
public class PlayerPerspective {
@Id
@@ -1,16 +1,20 @@
package de.grimsi.gameyfin.entities;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.*;
import org.hibernate.Hibernate;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Version;
import java.util.Objects;
@Entity
@Builder
@Getter
@Setter
@ToString
@AllArgsConstructor
@RequiredArgsConstructor
public class Theme {
@Id
@@ -1,5 +1,6 @@
package de.grimsi.gameyfin.entities;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.*;
import org.hibernate.Hibernate;
@@ -7,7 +8,6 @@ import javax.persistence.*;
import java.util.Objects;
@Entity
@Table(name = "blacklist")
@Getter
@Setter
@ToString
@@ -0,0 +1,17 @@
package de.grimsi.gameyfin.igdb;
import java.util.List;
public class IgdbApiProperties {
public static final String IGDB_ENPOINT_GAMES_PROTOBUF = "games.pb";
public static final List<String> GAME_QUERY_FIELDS = List.of(
"slug", "name", "summary", "first_release_date", "rating", "aggregated_rating", "total_rating", "category", "multiplayer_modes", "cover", "screenshots", "videos", // All top-level fields
"involved_companies.company.slug", "involved_companies.company.name", "involved_companies.company.logo.id",
"genres.slug", "genres.name",
"keywords.slug", "keywords.name",
"themes.slug", "themes.name",
"player_perspectives.slug", "player_perspectives.name"
);
}
@@ -3,6 +3,7 @@ package de.grimsi.gameyfin.igdb;
import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.config.WebClientConfig;
import de.grimsi.gameyfin.igdb.dto.TwitchOAuthTokenDto;
import io.github.resilience4j.reactor.bulkhead.operator.BulkheadOperator;
import io.github.resilience4j.reactor.ratelimiter.operator.RateLimiterOperator;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@@ -20,7 +21,6 @@ import java.util.Optional;
@Slf4j
@Service
public class IgdbWrapper {
@Value("${gameyfin.igdb.api.client-id}")
private String clientId;
@@ -66,13 +66,11 @@ public class IgdbWrapper {
}
public Optional<Igdb.Game> getGameById(Long id) {
Igdb.GameResult gameResult = igdbApiClient.post()
.uri("games.pb")
.bodyValue("fields *; where id = %d; limit 1;".formatted(id))
.retrieve()
.bodyToMono(Igdb.GameResult.class)
.transformDeferred(RateLimiterOperator.of(WebClientConfig.IGDB_RATE_LIMITER))
.block();
Igdb.GameResult gameResult = queryIgdbApi(
IgdbApiProperties.IGDB_ENPOINT_GAMES_PROTOBUF,
"fields *; where id = %d; limit 1;".formatted(id),
Igdb.GameResult.class
);
if (gameResult == null) return Optional.empty();
@@ -80,13 +78,11 @@ public class IgdbWrapper {
}
public Optional<Igdb.Game> getGameBySlug(String slug) {
Igdb.GameResult gameResult = igdbApiClient.post()
.uri("games.pb")
.bodyValue("fields *; where slug = \"%s\"; limit 1;".formatted(slug))
.retrieve()
.bodyToMono(Igdb.GameResult.class)
.transformDeferred(RateLimiterOperator.of(WebClientConfig.IGDB_RATE_LIMITER))
.block();
Igdb.GameResult gameResult = queryIgdbApi(
IgdbApiProperties.IGDB_ENPOINT_GAMES_PROTOBUF,
"fields *; where slug = \"%s\"; limit 1;".formatted(slug),
Igdb.GameResult.class
);
if (gameResult == null) return Optional.empty();
@@ -94,13 +90,12 @@ public class IgdbWrapper {
}
public Optional<Igdb.Game> searchForGameByTitle(String searchTerm) {
Igdb.GameResult gameResult = igdbApiClient.post()
.uri("games.pb")
.bodyValue("fields *; search \"%s\"; where platforms = (%s);".formatted(searchTerm, preferredPlatforms))
.retrieve()
.bodyToMono(Igdb.GameResult.class)
.transformDeferred(RateLimiterOperator.of(WebClientConfig.IGDB_RATE_LIMITER))
.block();
Igdb.GameResult gameResult = queryIgdbApi(
IgdbApiProperties.IGDB_ENPOINT_GAMES_PROTOBUF,
"search \"%s\"; fields %s; where platforms = (%s);"
.formatted(searchTerm, String.join(",", IgdbApiProperties.GAME_QUERY_FIELDS), preferredPlatforms),
Igdb.GameResult.class
);
if (gameResult == null) {
log.warn("Could not find game for title '{}'", searchTerm);
@@ -110,7 +105,7 @@ public class IgdbWrapper {
List<Igdb.Game> games = gameResult.getGamesList();
// If we only get one game, we don't have to check for exact matches, so return it directly
if(games.size() == 1) return Optional.ofNullable(games.get(0));
if (games.size() == 1) return Optional.ofNullable(games.get(0));
// First check if there are any matches with the exact search term
// If no exact match has been found, check if there are matches where the name ends with the search term
@@ -141,4 +136,15 @@ public class IgdbWrapper {
.filter(WebClientConfig.fixProtobufContentTypeInterceptor())
.build();
}
private <T> T queryIgdbApi(String endpoint, String query, Class<T> responseClass) {
return igdbApiClient.post()
.uri(endpoint)
.bodyValue(query)
.retrieve()
.bodyToMono(responseClass)
.transformDeferred(BulkheadOperator.of(WebClientConfig.IGDB_CONCURRENCY_LIMITER))
.transformDeferred(RateLimiterOperator.of(WebClientConfig.IGDB_RATE_LIMITER))
.block();
}
}
@@ -0,0 +1,21 @@
package de.grimsi.gameyfin.mapper;
import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.entities.Company;
import java.util.List;
public class CompanyMapper {
public static Company toCompany(Igdb.InvolvedCompany c) {
return Company.builder()
.slug(c.getCompany().getSlug())
.name(c.getCompany().getName())
.logoId(c.getCompany().getLogo().getId())
.build();
}
public static List<Company> toCompanies(List<Igdb.InvolvedCompany> c) {
return c.stream().map(CompanyMapper::toCompany).toList();
}
}
@@ -1,12 +1,11 @@
package de.grimsi.gameyfin.mapper;
import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.entities.Category;
import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.entities.PlayerPerspective;
import de.grimsi.gameyfin.util.ProtobufUtils;
import java.nio.file.Path;
import java.util.Comparator;
import java.util.List;
public class GameMapper {
@@ -24,7 +23,7 @@ public class GameMapper {
.userRating((int) g.getRating())
.criticsRating((int) g.getAggregatedRating())
.totalRating((int) g.getTotalRating())
//.category()
.category(g.getCategory().name())
.offlineCoop(hasOfflineCoop(multiplayerModes))
.onlineCoop(hasOnlineCoop(multiplayerModes))
.lanSupport(hasLanSupport(multiplayerModes))
@@ -32,11 +31,11 @@ public class GameMapper {
.coverId(g.getCover().getId())
.screenshotIds(screenshotIds)
.videoIds(videoIds)
//.companies()
//.genres()
//.keywords()
//.themes()
//.playerPerspectives()
.companies(CompanyMapper.toCompanies(g.getInvolvedCompaniesList()))
.genres(GenreMapper.toGenres(g.getGenresList()))
.keywords(KeywordMapper.toKeywords(g.getKeywordsList()))
.themes(ThemeMapper.toThemes(g.getThemesList()))
.playerPerspectives(PlayerPerspectiveMapper.toPlayerPerspectives(g.getPlayerPerspectivesList()))
.path(path.toString())
.isFolder(path.toFile().isDirectory())
.build();
@@ -0,0 +1,20 @@
package de.grimsi.gameyfin.mapper;
import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.entities.Genre;
import java.util.List;
public class GenreMapper {
public static Genre toGenre(Igdb.Genre g) {
return Genre.builder()
.slug(g.getSlug())
.name(g.getName())
.build();
}
public static List<Genre> toGenres(List<Igdb.Genre> g) {
return g.stream().map(GenreMapper::toGenre).toList();
}
}
@@ -0,0 +1,19 @@
package de.grimsi.gameyfin.mapper;
import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.entities.Keyword;
import java.util.List;
public class KeywordMapper {
public static Keyword toKeyword(Igdb.Keyword g) {
return Keyword.builder()
.slug(g.getSlug())
.name(g.getName())
.build();
}
public static List<Keyword> toKeywords(List<Igdb.Keyword> g) {
return g.stream().map(KeywordMapper::toKeyword).toList();
}
}
@@ -0,0 +1,20 @@
package de.grimsi.gameyfin.mapper;
import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.entities.PlayerPerspective;
import java.util.List;
public class PlayerPerspectiveMapper {
public static PlayerPerspective toPlayerPerspective(Igdb.PlayerPerspective g) {
return PlayerPerspective.builder()
.slug(g.getSlug())
.name(g.getName())
.build();
}
public static List<PlayerPerspective> toPlayerPerspectives(List<Igdb.PlayerPerspective> g) {
return g.stream().map(PlayerPerspectiveMapper::toPlayerPerspective).toList();
}
}
@@ -0,0 +1,19 @@
package de.grimsi.gameyfin.mapper;
import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.entities.Theme;
import java.util.List;
public class ThemeMapper {
public static Theme toTheme(Igdb.Theme g) {
return Theme.builder()
.slug(g.getSlug())
.name(g.getName())
.build();
}
public static List<Theme> toThemes(List<Igdb.Theme> g) {
return g.stream().map(ThemeMapper::toTheme).toList();
}
}
@@ -0,0 +1,7 @@
package de.grimsi.gameyfin.repositories;
import de.grimsi.gameyfin.entities.Company;
import org.springframework.data.jpa.repository.JpaRepository;
public interface CompanyRepository extends JpaRepository<Company, String> {
}
@@ -3,8 +3,21 @@ package de.grimsi.gameyfin.repositories;
import de.grimsi.gameyfin.entities.DetectedGame;
import org.springframework.data.jpa.repository.JpaRepository;
import javax.transaction.Transactional;
import java.nio.file.Path;
import java.util.Collection;
import java.util.List;
public interface DetectedGameRepository extends JpaRepository<DetectedGame, String> {
boolean existsByPath(String path);
boolean existsBySlug(String slug);
List<DetectedGame> getAllByPathNotIn(Collection<String> paths);
default List<DetectedGame> getAllByPathNotIn(List<Path> paths) {
List<String> pathStrings = paths.stream().map(Path::toString).toList();
return getAllByPathNotIn(pathStrings);
}
}
@@ -0,0 +1,7 @@
package de.grimsi.gameyfin.repositories;
import de.grimsi.gameyfin.entities.Genre;
import org.springframework.data.jpa.repository.JpaRepository;
public interface GenreRepository extends JpaRepository<Genre, String> {
}
@@ -0,0 +1,7 @@
package de.grimsi.gameyfin.repositories;
import de.grimsi.gameyfin.entities.Keyword;
import org.springframework.data.jpa.repository.JpaRepository;
public interface KeywordRepository extends JpaRepository<Keyword, String> {
}
@@ -0,0 +1,7 @@
package de.grimsi.gameyfin.repositories;
import de.grimsi.gameyfin.entities.PlayerPerspective;
import org.springframework.data.jpa.repository.JpaRepository;
public interface PlayerPerspectiveRepository extends JpaRepository<PlayerPerspective, String> {
}
@@ -0,0 +1,7 @@
package de.grimsi.gameyfin.repositories;
import de.grimsi.gameyfin.entities.Keyword;
import org.springframework.data.jpa.repository.JpaRepository;
public interface ThemeRepository extends JpaRepository<Keyword, String> {
}
@@ -12,6 +12,7 @@ import org.apache.commons.io.FilenameUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.StopWatch;
import java.io.IOException;
import java.nio.file.Files;
@@ -19,6 +20,7 @@ import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
@@ -59,12 +61,20 @@ public class FilesystemService {
}
public void scanGameLibrary() {
log.info("Starting scan...");
StopWatch stopWatch = new StopWatch();
AtomicInteger newBlacklistCounter = new AtomicInteger();
log.info("Starting scan...");
stopWatch.start();
AtomicInteger newUnmappedFilesCounter = new AtomicInteger();
List<Path> gameFiles = getGameFiles();
// Check if any games that are in the library have been removed from the file system
// This would include renamed files, but they will be re-detected by the next step
List<DetectedGame> deletedGames = detectedGameRepository.getAllByPathNotIn(gameFiles);
detectedGameRepository.deleteAll(deletedGames);
// Filter out the games we already know and the ones we already tried to map to a game without success
gameFiles = gameFiles.stream()
.filter(g -> !detectedGameRepository.existsByPath(g.toString()))
@@ -74,12 +84,13 @@ public class FilesystemService {
// For each new game, load the info from IGDB
// If a game is not found on IGDB, add it to the list of unmapped files so we won't query the API later on for the same path
// If a game is not found on IGDB, blacklist the path, so we won't query the API later for the same path
List<DetectedGame> newDetectedGames = gameFiles.parallelStream()
.map(p -> {
Optional<Igdb.Game> optionalGame = igdbWrapper.searchForGameByTitle(getFilename(p));
return optionalGame.map(game -> Map.entry(p, game)).or(() -> {
unmappableFileRepository.save(new UnmappableFile(p.toString()));
newBlacklistCounter.getAndIncrement();
newUnmappedFilesCounter.getAndIncrement();
log.info("Added path '{}' to blacklist", p);
return Optional.empty();
});
@@ -91,7 +102,15 @@ public class FilesystemService {
newDetectedGames = detectedGameRepository.saveAll(newDetectedGames);
log.info("Scan finished: Found {} new games, deleted {} games, could not map {} files/folders, {} games total.", newDetectedGames.size(), "NOT_IMPLEMENTED_YET", newBlacklistCounter.get(), detectedGameRepository.count());
stopWatch.stop();
String scanDuration = "%dmin : %ds".formatted(
TimeUnit.MILLISECONDS.toMinutes(stopWatch.getLastTaskTimeMillis()),
TimeUnit.MILLISECONDS.toSeconds(stopWatch.getLastTaskTimeMillis()) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(stopWatch.getLastTaskTimeMillis()))
);
log.info("Scan finished in {}: Found {} new games, deleted {} games, could not map {} files/folders, {} games total.",
scanDuration, newDetectedGames.size(), deletedGames.size(), newUnmappedFilesCounter.get(), detectedGameRepository.count());
}
private String getFilename(Path p) {
+5 -2
View File
@@ -1,5 +1,5 @@
gameyfin:
root: \\NAS-Simon\Öffentlich\Spiele
root: D:\Games
cache: C:\Projects\privat\gameyfin-library\.gameyfin
db: C:\Projects\privat\gameyfin-library\.gameyfin
igdb:
@@ -9,4 +9,7 @@ gameyfin:
logging:
level:
de.grimsi: debug
de.grimsi: debug
org.springframework.web.reactive.function.client.ExchangeFunctions: debug
spring.mvc.log-request-details: true
@@ -1,13 +0,0 @@
package de.grimsi.gameyfin;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
@SpringBootTest
class GameyfinApplicationTests {
@Test
void contextLoads() {
}
}