From 64485bf3f01b83a53d50c37b3f300663e70e7a78 Mon Sep 17 00:00:00 2001 From: grimsi <9295182+grimsi@users.noreply.github.com> Date: Sat, 16 Jul 2022 15:47:20 +0200 Subject: [PATCH] Optimized query to IGDB API Implemented mapping of nested fields --- pom.xml | 5 ++ .../gameyfin/config/WebClientConfig.java | 17 ++++-- .../de/grimsi/gameyfin/entities/Category.java | 33 ------------ .../de/grimsi/gameyfin/entities/Company.java | 9 ++-- .../gameyfin/entities/DetectedGame.java | 14 ++--- .../de/grimsi/gameyfin/entities/Genre.java | 9 ++-- .../de/grimsi/gameyfin/entities/Keyword.java | 4 ++ .../gameyfin/entities/PlayerPerspective.java | 4 ++ .../de/grimsi/gameyfin/entities/Theme.java | 4 ++ .../gameyfin/entities/UnmappableFile.java | 2 +- .../gameyfin/igdb/IgdbApiProperties.java | 17 ++++++ .../de/grimsi/gameyfin/igdb/IgdbWrapper.java | 52 +++++++++++-------- .../grimsi/gameyfin/mapper/CompanyMapper.java | 21 ++++++++ .../de/grimsi/gameyfin/mapper/GameMapper.java | 15 +++--- .../grimsi/gameyfin/mapper/GenreMapper.java | 20 +++++++ .../grimsi/gameyfin/mapper/KeywordMapper.java | 19 +++++++ .../mapper/PlayerPerspectiveMapper.java | 20 +++++++ .../grimsi/gameyfin/mapper/ThemeMapper.java | 19 +++++++ .../repositories/CompanyRepository.java | 7 +++ .../repositories/DetectedGameRepository.java | 13 +++++ .../repositories/GenreRepository.java | 7 +++ .../repositories/KeywordRepository.java | 7 +++ .../PlayerPerspectiveRepository.java | 7 +++ .../repositories/ThemeRepository.java | 7 +++ .../gameyfin/service/FilesystemService.java | 27 ++++++++-- src/main/resources/application-dev.yml | 7 ++- .../gameyfin/GameyfinApplicationTests.java | 13 ----- 27 files changed, 277 insertions(+), 102 deletions(-) delete mode 100644 src/main/java/de/grimsi/gameyfin/entities/Category.java create mode 100644 src/main/java/de/grimsi/gameyfin/igdb/IgdbApiProperties.java create mode 100644 src/main/java/de/grimsi/gameyfin/mapper/CompanyMapper.java create mode 100644 src/main/java/de/grimsi/gameyfin/mapper/GenreMapper.java create mode 100644 src/main/java/de/grimsi/gameyfin/mapper/KeywordMapper.java create mode 100644 src/main/java/de/grimsi/gameyfin/mapper/PlayerPerspectiveMapper.java create mode 100644 src/main/java/de/grimsi/gameyfin/mapper/ThemeMapper.java create mode 100644 src/main/java/de/grimsi/gameyfin/repositories/CompanyRepository.java create mode 100644 src/main/java/de/grimsi/gameyfin/repositories/GenreRepository.java create mode 100644 src/main/java/de/grimsi/gameyfin/repositories/KeywordRepository.java create mode 100644 src/main/java/de/grimsi/gameyfin/repositories/PlayerPerspectiveRepository.java create mode 100644 src/main/java/de/grimsi/gameyfin/repositories/ThemeRepository.java delete mode 100644 src/test/java/de/grimsi/gameyfin/GameyfinApplicationTests.java diff --git a/pom.xml b/pom.xml index 249fa9a..8930416 100644 --- a/pom.xml +++ b/pom.xml @@ -47,6 +47,11 @@ resilience4j-ratelimiter 1.7.1 + + io.github.resilience4j + resilience4j-bulkhead + 1.7.1 + diff --git a/src/main/java/de/grimsi/gameyfin/config/WebClientConfig.java b/src/main/java/de/grimsi/gameyfin/config/WebClientConfig.java index 41a8429..cdc1b09 100644 --- a/src/main/java/de/grimsi/gameyfin/config/WebClientConfig.java +++ b/src/main/java/de/grimsi/gameyfin/config/WebClientConfig.java @@ -1,5 +1,7 @@ package de.grimsi.gameyfin.config; +import io.github.resilience4j.bulkhead.Bulkhead; +import io.github.resilience4j.bulkhead.BulkheadConfig; import io.github.resilience4j.ratelimiter.RateLimiter; import io.github.resilience4j.ratelimiter.RateLimiterConfig; import io.netty.handler.logging.LogLevel; @@ -21,12 +23,21 @@ import java.time.Duration; @Configuration public class WebClientConfig implements WebClientCustomizer { + // The IGDB API has a rate limit of 4 req/s public static final RateLimiter IGDB_RATE_LIMITER = RateLimiter.of("igdb-rate-limiter", RateLimiterConfig.custom() - .limitRefreshPeriod(Duration.ofSeconds(1)) .limitForPeriod(4) - .timeoutDuration(Duration.ofMinutes(1)) // max wait time for a request, if reached then error - .build()); + .limitRefreshPeriod(Duration.ofSeconds(1)) + .timeoutDuration(Duration.ofMinutes(1)) + .build()); + + // According to the docs, there is a maximum of 8 concurrent requests, but in my tests the actual limit was 4 + // and even then it sometimes failed, so I set it to 3 to be sure + public static final Bulkhead IGDB_CONCURRENCY_LIMITER = Bulkhead.of("igdb-concurrency-limiter", + BulkheadConfig.custom() + .maxConcurrentCalls(2) + .maxWaitDuration(Duration.ofMinutes(1)) + .build()); @Override public void customize(WebClient.Builder webClientBuilder) { diff --git a/src/main/java/de/grimsi/gameyfin/entities/Category.java b/src/main/java/de/grimsi/gameyfin/entities/Category.java deleted file mode 100644 index d292123..0000000 --- a/src/main/java/de/grimsi/gameyfin/entities/Category.java +++ /dev/null @@ -1,33 +0,0 @@ -package de.grimsi.gameyfin.entities; - -import lombok.*; -import org.hibernate.Hibernate; - -import javax.persistence.Entity; -import javax.persistence.Id; -import java.util.Objects; - -@Entity -@Getter -@Setter -@ToString -@RequiredArgsConstructor -public class Category { - @Id - private String slug; - - private String name; - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; - Category category = (Category) o; - return slug != null && Objects.equals(slug, category.slug); - } - - @Override - public int hashCode() { - return getClass().hashCode(); - } -} diff --git a/src/main/java/de/grimsi/gameyfin/entities/Company.java b/src/main/java/de/grimsi/gameyfin/entities/Company.java index ca8a541..a0d88f0 100644 --- a/src/main/java/de/grimsi/gameyfin/entities/Company.java +++ b/src/main/java/de/grimsi/gameyfin/entities/Company.java @@ -1,20 +1,21 @@ package de.grimsi.gameyfin.entities; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.Setter; -import lombok.ToString; +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.*; import org.hibernate.Hibernate; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; +import javax.persistence.Version; import java.util.Objects; @Entity +@Builder @Getter @Setter @ToString +@AllArgsConstructor @RequiredArgsConstructor public class Company { @Id diff --git a/src/main/java/de/grimsi/gameyfin/entities/DetectedGame.java b/src/main/java/de/grimsi/gameyfin/entities/DetectedGame.java index 9ddb7db..ea4e853 100644 --- a/src/main/java/de/grimsi/gameyfin/entities/DetectedGame.java +++ b/src/main/java/de/grimsi/gameyfin/entities/DetectedGame.java @@ -1,6 +1,7 @@ package de.grimsi.gameyfin.entities; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.*; import org.hibernate.Hibernate; @@ -37,8 +38,7 @@ public class DetectedGame { private Integer totalRating; - @ManyToOne - private Category category; + private String category; private boolean offlineCoop; @@ -57,23 +57,23 @@ public class DetectedGame { @ElementCollection private List videoIds; - @ManyToMany + @ManyToMany(cascade = CascadeType.MERGE) @ToString.Exclude private List companies; - @ManyToMany + @ManyToMany(cascade = CascadeType.MERGE) @ToString.Exclude private List genres; - @ManyToMany + @ManyToMany(cascade = CascadeType.MERGE) @ToString.Exclude private List keywords; - @ManyToMany + @ManyToMany(cascade = CascadeType.MERGE) @ToString.Exclude private List themes; - @ManyToMany + @ManyToMany(cascade = CascadeType.MERGE) @ToString.Exclude private List playerPerspectives; diff --git a/src/main/java/de/grimsi/gameyfin/entities/Genre.java b/src/main/java/de/grimsi/gameyfin/entities/Genre.java index 68e6987..0a7cb32 100644 --- a/src/main/java/de/grimsi/gameyfin/entities/Genre.java +++ b/src/main/java/de/grimsi/gameyfin/entities/Genre.java @@ -1,19 +1,20 @@ package de.grimsi.gameyfin.entities; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.Setter; -import lombok.ToString; +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.*; import org.hibernate.Hibernate; import javax.persistence.Entity; import javax.persistence.Id; +import javax.persistence.Version; import java.util.Objects; @Entity +@Builder @Getter @Setter @ToString +@AllArgsConstructor @RequiredArgsConstructor public class Genre { @Id diff --git a/src/main/java/de/grimsi/gameyfin/entities/Keyword.java b/src/main/java/de/grimsi/gameyfin/entities/Keyword.java index 1586862..2483adf 100644 --- a/src/main/java/de/grimsi/gameyfin/entities/Keyword.java +++ b/src/main/java/de/grimsi/gameyfin/entities/Keyword.java @@ -1,16 +1,20 @@ package de.grimsi.gameyfin.entities; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.*; import org.hibernate.Hibernate; import javax.persistence.Entity; import javax.persistence.Id; +import javax.persistence.Version; import java.util.Objects; @Entity +@Builder @Getter @Setter @ToString +@AllArgsConstructor @RequiredArgsConstructor public class Keyword { @Id diff --git a/src/main/java/de/grimsi/gameyfin/entities/PlayerPerspective.java b/src/main/java/de/grimsi/gameyfin/entities/PlayerPerspective.java index 876109b..51c49a1 100644 --- a/src/main/java/de/grimsi/gameyfin/entities/PlayerPerspective.java +++ b/src/main/java/de/grimsi/gameyfin/entities/PlayerPerspective.java @@ -1,16 +1,20 @@ package de.grimsi.gameyfin.entities; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.*; import org.hibernate.Hibernate; import javax.persistence.Entity; import javax.persistence.Id; +import javax.persistence.Version; import java.util.Objects; @Entity +@Builder @Getter @Setter @ToString +@AllArgsConstructor @RequiredArgsConstructor public class PlayerPerspective { @Id diff --git a/src/main/java/de/grimsi/gameyfin/entities/Theme.java b/src/main/java/de/grimsi/gameyfin/entities/Theme.java index aadb122..d1d5e80 100644 --- a/src/main/java/de/grimsi/gameyfin/entities/Theme.java +++ b/src/main/java/de/grimsi/gameyfin/entities/Theme.java @@ -1,16 +1,20 @@ package de.grimsi.gameyfin.entities; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.*; import org.hibernate.Hibernate; import javax.persistence.Entity; import javax.persistence.Id; +import javax.persistence.Version; import java.util.Objects; @Entity +@Builder @Getter @Setter @ToString +@AllArgsConstructor @RequiredArgsConstructor public class Theme { @Id diff --git a/src/main/java/de/grimsi/gameyfin/entities/UnmappableFile.java b/src/main/java/de/grimsi/gameyfin/entities/UnmappableFile.java index 13330f2..a75a985 100644 --- a/src/main/java/de/grimsi/gameyfin/entities/UnmappableFile.java +++ b/src/main/java/de/grimsi/gameyfin/entities/UnmappableFile.java @@ -1,5 +1,6 @@ package de.grimsi.gameyfin.entities; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.*; import org.hibernate.Hibernate; @@ -7,7 +8,6 @@ import javax.persistence.*; import java.util.Objects; @Entity -@Table(name = "blacklist") @Getter @Setter @ToString diff --git a/src/main/java/de/grimsi/gameyfin/igdb/IgdbApiProperties.java b/src/main/java/de/grimsi/gameyfin/igdb/IgdbApiProperties.java new file mode 100644 index 0000000..4dcec99 --- /dev/null +++ b/src/main/java/de/grimsi/gameyfin/igdb/IgdbApiProperties.java @@ -0,0 +1,17 @@ +package de.grimsi.gameyfin.igdb; + +import java.util.List; + +public class IgdbApiProperties { + public static final String IGDB_ENPOINT_GAMES_PROTOBUF = "games.pb"; + + public static final List GAME_QUERY_FIELDS = List.of( + "slug", "name", "summary", "first_release_date", "rating", "aggregated_rating", "total_rating", "category", "multiplayer_modes", "cover", "screenshots", "videos", // All top-level fields + "involved_companies.company.slug", "involved_companies.company.name", "involved_companies.company.logo.id", + "genres.slug", "genres.name", + "keywords.slug", "keywords.name", + "themes.slug", "themes.name", + "player_perspectives.slug", "player_perspectives.name" + ); + +} diff --git a/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java b/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java index 3d62ded..1961172 100644 --- a/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java +++ b/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java @@ -3,6 +3,7 @@ package de.grimsi.gameyfin.igdb; import com.igdb.proto.Igdb; import de.grimsi.gameyfin.config.WebClientConfig; import de.grimsi.gameyfin.igdb.dto.TwitchOAuthTokenDto; +import io.github.resilience4j.reactor.bulkhead.operator.BulkheadOperator; import io.github.resilience4j.reactor.ratelimiter.operator.RateLimiterOperator; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -20,7 +21,6 @@ import java.util.Optional; @Slf4j @Service public class IgdbWrapper { - @Value("${gameyfin.igdb.api.client-id}") private String clientId; @@ -66,13 +66,11 @@ public class IgdbWrapper { } public Optional getGameById(Long id) { - Igdb.GameResult gameResult = igdbApiClient.post() - .uri("games.pb") - .bodyValue("fields *; where id = %d; limit 1;".formatted(id)) - .retrieve() - .bodyToMono(Igdb.GameResult.class) - .transformDeferred(RateLimiterOperator.of(WebClientConfig.IGDB_RATE_LIMITER)) - .block(); + Igdb.GameResult gameResult = queryIgdbApi( + IgdbApiProperties.IGDB_ENPOINT_GAMES_PROTOBUF, + "fields *; where id = %d; limit 1;".formatted(id), + Igdb.GameResult.class + ); if (gameResult == null) return Optional.empty(); @@ -80,13 +78,11 @@ public class IgdbWrapper { } public Optional getGameBySlug(String slug) { - Igdb.GameResult gameResult = igdbApiClient.post() - .uri("games.pb") - .bodyValue("fields *; where slug = \"%s\"; limit 1;".formatted(slug)) - .retrieve() - .bodyToMono(Igdb.GameResult.class) - .transformDeferred(RateLimiterOperator.of(WebClientConfig.IGDB_RATE_LIMITER)) - .block(); + Igdb.GameResult gameResult = queryIgdbApi( + IgdbApiProperties.IGDB_ENPOINT_GAMES_PROTOBUF, + "fields *; where slug = \"%s\"; limit 1;".formatted(slug), + Igdb.GameResult.class + ); if (gameResult == null) return Optional.empty(); @@ -94,13 +90,12 @@ public class IgdbWrapper { } public Optional searchForGameByTitle(String searchTerm) { - Igdb.GameResult gameResult = igdbApiClient.post() - .uri("games.pb") - .bodyValue("fields *; search \"%s\"; where platforms = (%s);".formatted(searchTerm, preferredPlatforms)) - .retrieve() - .bodyToMono(Igdb.GameResult.class) - .transformDeferred(RateLimiterOperator.of(WebClientConfig.IGDB_RATE_LIMITER)) - .block(); + Igdb.GameResult gameResult = queryIgdbApi( + IgdbApiProperties.IGDB_ENPOINT_GAMES_PROTOBUF, + "search \"%s\"; fields %s; where platforms = (%s);" + .formatted(searchTerm, String.join(",", IgdbApiProperties.GAME_QUERY_FIELDS), preferredPlatforms), + Igdb.GameResult.class + ); if (gameResult == null) { log.warn("Could not find game for title '{}'", searchTerm); @@ -110,7 +105,7 @@ public class IgdbWrapper { List games = gameResult.getGamesList(); // If we only get one game, we don't have to check for exact matches, so return it directly - if(games.size() == 1) return Optional.ofNullable(games.get(0)); + if (games.size() == 1) return Optional.ofNullable(games.get(0)); // First check if there are any matches with the exact search term // If no exact match has been found, check if there are matches where the name ends with the search term @@ -141,4 +136,15 @@ public class IgdbWrapper { .filter(WebClientConfig.fixProtobufContentTypeInterceptor()) .build(); } + + private T queryIgdbApi(String endpoint, String query, Class responseClass) { + return igdbApiClient.post() + .uri(endpoint) + .bodyValue(query) + .retrieve() + .bodyToMono(responseClass) + .transformDeferred(BulkheadOperator.of(WebClientConfig.IGDB_CONCURRENCY_LIMITER)) + .transformDeferred(RateLimiterOperator.of(WebClientConfig.IGDB_RATE_LIMITER)) + .block(); + } } diff --git a/src/main/java/de/grimsi/gameyfin/mapper/CompanyMapper.java b/src/main/java/de/grimsi/gameyfin/mapper/CompanyMapper.java new file mode 100644 index 0000000..29a1ce4 --- /dev/null +++ b/src/main/java/de/grimsi/gameyfin/mapper/CompanyMapper.java @@ -0,0 +1,21 @@ +package de.grimsi.gameyfin.mapper; + +import com.igdb.proto.Igdb; +import de.grimsi.gameyfin.entities.Company; + +import java.util.List; + +public class CompanyMapper { + + public static Company toCompany(Igdb.InvolvedCompany c) { + return Company.builder() + .slug(c.getCompany().getSlug()) + .name(c.getCompany().getName()) + .logoId(c.getCompany().getLogo().getId()) + .build(); + } + + public static List toCompanies(List c) { + return c.stream().map(CompanyMapper::toCompany).toList(); + } +} diff --git a/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java b/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java index 23ce997..0bb7bee 100644 --- a/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java +++ b/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java @@ -1,12 +1,11 @@ package de.grimsi.gameyfin.mapper; import com.igdb.proto.Igdb; -import de.grimsi.gameyfin.entities.Category; import de.grimsi.gameyfin.entities.DetectedGame; +import de.grimsi.gameyfin.entities.PlayerPerspective; import de.grimsi.gameyfin.util.ProtobufUtils; import java.nio.file.Path; -import java.util.Comparator; import java.util.List; public class GameMapper { @@ -24,7 +23,7 @@ public class GameMapper { .userRating((int) g.getRating()) .criticsRating((int) g.getAggregatedRating()) .totalRating((int) g.getTotalRating()) - //.category() + .category(g.getCategory().name()) .offlineCoop(hasOfflineCoop(multiplayerModes)) .onlineCoop(hasOnlineCoop(multiplayerModes)) .lanSupport(hasLanSupport(multiplayerModes)) @@ -32,11 +31,11 @@ public class GameMapper { .coverId(g.getCover().getId()) .screenshotIds(screenshotIds) .videoIds(videoIds) - //.companies() - //.genres() - //.keywords() - //.themes() - //.playerPerspectives() + .companies(CompanyMapper.toCompanies(g.getInvolvedCompaniesList())) + .genres(GenreMapper.toGenres(g.getGenresList())) + .keywords(KeywordMapper.toKeywords(g.getKeywordsList())) + .themes(ThemeMapper.toThemes(g.getThemesList())) + .playerPerspectives(PlayerPerspectiveMapper.toPlayerPerspectives(g.getPlayerPerspectivesList())) .path(path.toString()) .isFolder(path.toFile().isDirectory()) .build(); diff --git a/src/main/java/de/grimsi/gameyfin/mapper/GenreMapper.java b/src/main/java/de/grimsi/gameyfin/mapper/GenreMapper.java new file mode 100644 index 0000000..eb9208f --- /dev/null +++ b/src/main/java/de/grimsi/gameyfin/mapper/GenreMapper.java @@ -0,0 +1,20 @@ +package de.grimsi.gameyfin.mapper; + +import com.igdb.proto.Igdb; +import de.grimsi.gameyfin.entities.Genre; + +import java.util.List; + +public class GenreMapper { + + public static Genre toGenre(Igdb.Genre g) { + return Genre.builder() + .slug(g.getSlug()) + .name(g.getName()) + .build(); + } + + public static List toGenres(List g) { + return g.stream().map(GenreMapper::toGenre).toList(); + } +} diff --git a/src/main/java/de/grimsi/gameyfin/mapper/KeywordMapper.java b/src/main/java/de/grimsi/gameyfin/mapper/KeywordMapper.java new file mode 100644 index 0000000..fbaca16 --- /dev/null +++ b/src/main/java/de/grimsi/gameyfin/mapper/KeywordMapper.java @@ -0,0 +1,19 @@ +package de.grimsi.gameyfin.mapper; + +import com.igdb.proto.Igdb; +import de.grimsi.gameyfin.entities.Keyword; + +import java.util.List; + +public class KeywordMapper { + public static Keyword toKeyword(Igdb.Keyword g) { + return Keyword.builder() + .slug(g.getSlug()) + .name(g.getName()) + .build(); + } + + public static List toKeywords(List g) { + return g.stream().map(KeywordMapper::toKeyword).toList(); + } +} diff --git a/src/main/java/de/grimsi/gameyfin/mapper/PlayerPerspectiveMapper.java b/src/main/java/de/grimsi/gameyfin/mapper/PlayerPerspectiveMapper.java new file mode 100644 index 0000000..67a5529 --- /dev/null +++ b/src/main/java/de/grimsi/gameyfin/mapper/PlayerPerspectiveMapper.java @@ -0,0 +1,20 @@ +package de.grimsi.gameyfin.mapper; + +import com.igdb.proto.Igdb; +import de.grimsi.gameyfin.entities.PlayerPerspective; + +import java.util.List; + +public class PlayerPerspectiveMapper { + + public static PlayerPerspective toPlayerPerspective(Igdb.PlayerPerspective g) { + return PlayerPerspective.builder() + .slug(g.getSlug()) + .name(g.getName()) + .build(); +} + + public static List toPlayerPerspectives(List g) { + return g.stream().map(PlayerPerspectiveMapper::toPlayerPerspective).toList(); + } +} diff --git a/src/main/java/de/grimsi/gameyfin/mapper/ThemeMapper.java b/src/main/java/de/grimsi/gameyfin/mapper/ThemeMapper.java new file mode 100644 index 0000000..5102b5c --- /dev/null +++ b/src/main/java/de/grimsi/gameyfin/mapper/ThemeMapper.java @@ -0,0 +1,19 @@ +package de.grimsi.gameyfin.mapper; + +import com.igdb.proto.Igdb; +import de.grimsi.gameyfin.entities.Theme; + +import java.util.List; + +public class ThemeMapper { + public static Theme toTheme(Igdb.Theme g) { + return Theme.builder() + .slug(g.getSlug()) + .name(g.getName()) + .build(); + } + + public static List toThemes(List g) { + return g.stream().map(ThemeMapper::toTheme).toList(); + } +} diff --git a/src/main/java/de/grimsi/gameyfin/repositories/CompanyRepository.java b/src/main/java/de/grimsi/gameyfin/repositories/CompanyRepository.java new file mode 100644 index 0000000..9e2f7cc --- /dev/null +++ b/src/main/java/de/grimsi/gameyfin/repositories/CompanyRepository.java @@ -0,0 +1,7 @@ +package de.grimsi.gameyfin.repositories; + +import de.grimsi.gameyfin.entities.Company; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface CompanyRepository extends JpaRepository { +} diff --git a/src/main/java/de/grimsi/gameyfin/repositories/DetectedGameRepository.java b/src/main/java/de/grimsi/gameyfin/repositories/DetectedGameRepository.java index 78ac2d0..ea2eb02 100644 --- a/src/main/java/de/grimsi/gameyfin/repositories/DetectedGameRepository.java +++ b/src/main/java/de/grimsi/gameyfin/repositories/DetectedGameRepository.java @@ -3,8 +3,21 @@ package de.grimsi.gameyfin.repositories; import de.grimsi.gameyfin.entities.DetectedGame; import org.springframework.data.jpa.repository.JpaRepository; +import javax.transaction.Transactional; +import java.nio.file.Path; +import java.util.Collection; +import java.util.List; + public interface DetectedGameRepository extends JpaRepository { boolean existsByPath(String path); + boolean existsBySlug(String slug); + + List getAllByPathNotIn(Collection paths); + + default List getAllByPathNotIn(List paths) { + List pathStrings = paths.stream().map(Path::toString).toList(); + return getAllByPathNotIn(pathStrings); + } } diff --git a/src/main/java/de/grimsi/gameyfin/repositories/GenreRepository.java b/src/main/java/de/grimsi/gameyfin/repositories/GenreRepository.java new file mode 100644 index 0000000..08b49aa --- /dev/null +++ b/src/main/java/de/grimsi/gameyfin/repositories/GenreRepository.java @@ -0,0 +1,7 @@ +package de.grimsi.gameyfin.repositories; + +import de.grimsi.gameyfin.entities.Genre; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface GenreRepository extends JpaRepository { +} diff --git a/src/main/java/de/grimsi/gameyfin/repositories/KeywordRepository.java b/src/main/java/de/grimsi/gameyfin/repositories/KeywordRepository.java new file mode 100644 index 0000000..7d8ecaf --- /dev/null +++ b/src/main/java/de/grimsi/gameyfin/repositories/KeywordRepository.java @@ -0,0 +1,7 @@ +package de.grimsi.gameyfin.repositories; + +import de.grimsi.gameyfin.entities.Keyword; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface KeywordRepository extends JpaRepository { +} diff --git a/src/main/java/de/grimsi/gameyfin/repositories/PlayerPerspectiveRepository.java b/src/main/java/de/grimsi/gameyfin/repositories/PlayerPerspectiveRepository.java new file mode 100644 index 0000000..8762c53 --- /dev/null +++ b/src/main/java/de/grimsi/gameyfin/repositories/PlayerPerspectiveRepository.java @@ -0,0 +1,7 @@ +package de.grimsi.gameyfin.repositories; + +import de.grimsi.gameyfin.entities.PlayerPerspective; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface PlayerPerspectiveRepository extends JpaRepository { +} diff --git a/src/main/java/de/grimsi/gameyfin/repositories/ThemeRepository.java b/src/main/java/de/grimsi/gameyfin/repositories/ThemeRepository.java new file mode 100644 index 0000000..d29c7ff --- /dev/null +++ b/src/main/java/de/grimsi/gameyfin/repositories/ThemeRepository.java @@ -0,0 +1,7 @@ +package de.grimsi.gameyfin.repositories; + +import de.grimsi.gameyfin.entities.Keyword; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface ThemeRepository extends JpaRepository { +} diff --git a/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java b/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java index 70527e7..a5ecb59 100644 --- a/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java +++ b/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java @@ -12,6 +12,7 @@ import org.apache.commons.io.FilenameUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; import java.io.IOException; import java.nio.file.Files; @@ -19,6 +20,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; @@ -59,12 +61,20 @@ public class FilesystemService { } public void scanGameLibrary() { - log.info("Starting scan..."); + StopWatch stopWatch = new StopWatch(); - AtomicInteger newBlacklistCounter = new AtomicInteger(); + log.info("Starting scan..."); + stopWatch.start(); + + AtomicInteger newUnmappedFilesCounter = new AtomicInteger(); List gameFiles = getGameFiles(); + // Check if any games that are in the library have been removed from the file system + // This would include renamed files, but they will be re-detected by the next step + List deletedGames = detectedGameRepository.getAllByPathNotIn(gameFiles); + detectedGameRepository.deleteAll(deletedGames); + // Filter out the games we already know and the ones we already tried to map to a game without success gameFiles = gameFiles.stream() .filter(g -> !detectedGameRepository.existsByPath(g.toString())) @@ -74,12 +84,13 @@ public class FilesystemService { // For each new game, load the info from IGDB // If a game is not found on IGDB, add it to the list of unmapped files so we won't query the API later on for the same path + // If a game is not found on IGDB, blacklist the path, so we won't query the API later for the same path List newDetectedGames = gameFiles.parallelStream() .map(p -> { Optional optionalGame = igdbWrapper.searchForGameByTitle(getFilename(p)); return optionalGame.map(game -> Map.entry(p, game)).or(() -> { unmappableFileRepository.save(new UnmappableFile(p.toString())); - newBlacklistCounter.getAndIncrement(); + newUnmappedFilesCounter.getAndIncrement(); log.info("Added path '{}' to blacklist", p); return Optional.empty(); }); @@ -91,7 +102,15 @@ public class FilesystemService { newDetectedGames = detectedGameRepository.saveAll(newDetectedGames); - log.info("Scan finished: Found {} new games, deleted {} games, could not map {} files/folders, {} games total.", newDetectedGames.size(), "NOT_IMPLEMENTED_YET", newBlacklistCounter.get(), detectedGameRepository.count()); + stopWatch.stop(); + + String scanDuration = "%dmin : %ds".formatted( + TimeUnit.MILLISECONDS.toMinutes(stopWatch.getLastTaskTimeMillis()), + TimeUnit.MILLISECONDS.toSeconds(stopWatch.getLastTaskTimeMillis()) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(stopWatch.getLastTaskTimeMillis())) + ); + + log.info("Scan finished in {}: Found {} new games, deleted {} games, could not map {} files/folders, {} games total.", + scanDuration, newDetectedGames.size(), deletedGames.size(), newUnmappedFilesCounter.get(), detectedGameRepository.count()); } private String getFilename(Path p) { diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 5d47901..2572ec2 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -1,5 +1,5 @@ gameyfin: - root: \\NAS-Simon\Öffentlich\Spiele + root: D:\Games cache: C:\Projects\privat\gameyfin-library\.gameyfin db: C:\Projects\privat\gameyfin-library\.gameyfin igdb: @@ -9,4 +9,7 @@ gameyfin: logging: level: - de.grimsi: debug \ No newline at end of file + de.grimsi: debug + org.springframework.web.reactive.function.client.ExchangeFunctions: debug + +spring.mvc.log-request-details: true \ No newline at end of file diff --git a/src/test/java/de/grimsi/gameyfin/GameyfinApplicationTests.java b/src/test/java/de/grimsi/gameyfin/GameyfinApplicationTests.java deleted file mode 100644 index 26b2b88..0000000 --- a/src/test/java/de/grimsi/gameyfin/GameyfinApplicationTests.java +++ /dev/null @@ -1,13 +0,0 @@ -package de.grimsi.gameyfin; - -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; - -@SpringBootTest -class GameyfinApplicationTests { - - @Test - void contextLoads() { - } - -}