diff --git a/src/main/java/de/grimsi/gameyfin/config/WebClientConfig.java b/src/main/java/de/grimsi/gameyfin/config/WebClientConfig.java index 8efabac..fa1f625 100644 --- a/src/main/java/de/grimsi/gameyfin/config/WebClientConfig.java +++ b/src/main/java/de/grimsi/gameyfin/config/WebClientConfig.java @@ -20,7 +20,7 @@ public class WebClientConfig implements WebClientCustomizer { @Override public void customize(WebClient.Builder webClientBuilder) { HttpClient httpClient = HttpClient.create() - .wiretap(this.getClass().getCanonicalName(), LogLevel.DEBUG, AdvancedByteBufFormat.TEXTUAL) // Enable full request / response logging (active only in DEV profile) + .wiretap(this.getClass().getCanonicalName(), LogLevel.TRACE, AdvancedByteBufFormat.TEXTUAL) // Enable full request / response logging in TRACE .proxyWithSystemProperties(); // Enable use of system proxy webClientBuilder.clientConnector(new ReactorClientHttpConnector(httpClient)); diff --git a/src/main/java/de/grimsi/gameyfin/dto/GameDto.java b/src/main/java/de/grimsi/gameyfin/dto/GameDto.java index 8c83568..8f72c3a 100644 --- a/src/main/java/de/grimsi/gameyfin/dto/GameDto.java +++ b/src/main/java/de/grimsi/gameyfin/dto/GameDto.java @@ -16,7 +16,7 @@ import java.util.List; public class GameDto { private String name; private String publisher; - private Long igdbGameId; + private String slug; private Instant releaseDate; private List files; diff --git a/src/main/java/de/grimsi/gameyfin/entities/BlacklistEntry.java b/src/main/java/de/grimsi/gameyfin/entities/BlacklistEntry.java new file mode 100644 index 0000000..ee3ad31 --- /dev/null +++ b/src/main/java/de/grimsi/gameyfin/entities/BlacklistEntry.java @@ -0,0 +1,34 @@ +package de.grimsi.gameyfin.entities; + +import lombok.*; +import org.hibernate.Hibernate; + +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import java.util.Objects; + +@Entity +@Table(name = "blacklist") +@Getter +@Setter +@ToString +@NoArgsConstructor +@AllArgsConstructor +public class BlacklistEntry { + @Id + private String path; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + BlacklistEntry that = (BlacklistEntry) o; + return path != null && Objects.equals(path, that.path); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/src/main/java/de/grimsi/gameyfin/entities/DetectedGame.java b/src/main/java/de/grimsi/gameyfin/entities/DetectedGame.java index 751913e..9ddb7db 100644 --- a/src/main/java/de/grimsi/gameyfin/entities/DetectedGame.java +++ b/src/main/java/de/grimsi/gameyfin/entities/DetectedGame.java @@ -14,6 +14,7 @@ import java.util.Objects; @Getter @Setter @ToString +@AllArgsConstructor @RequiredArgsConstructor public class DetectedGame { @@ -24,6 +25,8 @@ public class DetectedGame { @Column(nullable = false) private String title; + @Lob + @Column(columnDefinition="CLOB") private String summary; private Instant releaseDate; diff --git a/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java b/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java index 84e46f9..2f4a089 100644 --- a/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java +++ b/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java @@ -87,7 +87,10 @@ public class IgdbWrapper { .bodyToMono(Igdb.GameResult.class) .block(); - if (gameResult == null) return Optional.empty(); + if (gameResult == null) { + log.warn("Could not find game for title '{}'", searchTerm); + return Optional.empty(); + } List games = gameResult.getGamesList(); diff --git a/src/main/java/de/grimsi/gameyfin/repositories/BlacklistRepository.java b/src/main/java/de/grimsi/gameyfin/repositories/BlacklistRepository.java new file mode 100644 index 0000000..8c8407a --- /dev/null +++ b/src/main/java/de/grimsi/gameyfin/repositories/BlacklistRepository.java @@ -0,0 +1,9 @@ +package de.grimsi.gameyfin.repositories; + +import de.grimsi.gameyfin.entities.BlacklistEntry; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface BlacklistRepository extends JpaRepository { + + boolean existsByPath(String path); +} diff --git a/src/main/java/de/grimsi/gameyfin/repositories/DetectedGameRepository.java b/src/main/java/de/grimsi/gameyfin/repositories/DetectedGameRepository.java new file mode 100644 index 0000000..0122f07 --- /dev/null +++ b/src/main/java/de/grimsi/gameyfin/repositories/DetectedGameRepository.java @@ -0,0 +1,9 @@ +package de.grimsi.gameyfin.repositories; + +import de.grimsi.gameyfin.entities.DetectedGame; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface DetectedGameRepository extends JpaRepository { + + boolean existsByPath(String path); +} diff --git a/src/main/java/de/grimsi/gameyfin/rest/GameyfinDevController.java b/src/main/java/de/grimsi/gameyfin/rest/GameyfinDevController.java index 2e380e5..a59e27f 100644 --- a/src/main/java/de/grimsi/gameyfin/rest/GameyfinDevController.java +++ b/src/main/java/de/grimsi/gameyfin/rest/GameyfinDevController.java @@ -2,8 +2,10 @@ package de.grimsi.gameyfin.rest; import com.igdb.proto.Igdb; import de.grimsi.gameyfin.dto.GameDto; +import de.grimsi.gameyfin.entities.DetectedGame; import de.grimsi.gameyfin.igdb.IgdbWrapper; import de.grimsi.gameyfin.service.FilesystemService; +import de.grimsi.gameyfin.service.GameService; import de.grimsi.gameyfin.util.ProtobufUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; @@ -26,6 +28,9 @@ public class GameyfinDevController { @Autowired private FilesystemService filesystemService; + @Autowired + private GameService gameService; + @GetMapping(value = "/dev/findGameByTitle/{title}", produces = MediaType.APPLICATION_JSON_VALUE) public GameDto findGameByTitle(@PathVariable("title") String title) { Igdb.Game game = igdbWrapper.searchForGameByTitle(title) @@ -54,14 +59,13 @@ public class GameyfinDevController { } @GetMapping(value = "/dev/games", produces = MediaType.APPLICATION_JSON_VALUE) - public List getAllGames() { - return filesystemService.getGameFileNames().parallelStream() - .map(t -> igdbWrapper.searchForGameByTitle(t).orElse(null)) - .filter(Objects::nonNull) - .map(g -> GameDto.builder() - .name(g.getName()) - .releaseDate(ProtobufUtils.toInstant(g.getFirstReleaseDate())) - .build()) - .toList(); + public List getAllGames() { + return gameService.getAllDetectedGames(); } + + @GetMapping(value = "/dev/startScan", produces = MediaType.APPLICATION_JSON_VALUE) + public void scanLibrary() { + filesystemService.scanGameLibrary(); + } + } diff --git a/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java b/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java index 18b9b51..53fae2f 100644 --- a/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java +++ b/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java @@ -1,6 +1,15 @@ package de.grimsi.gameyfin.service; +import com.igdb.proto.Igdb; +import de.grimsi.gameyfin.entities.BlacklistEntry; +import de.grimsi.gameyfin.entities.DetectedGame; +import de.grimsi.gameyfin.igdb.IgdbWrapper; +import de.grimsi.gameyfin.mapper.GameMapper; +import de.grimsi.gameyfin.repositories.BlacklistRepository; +import de.grimsi.gameyfin.repositories.DetectedGameRepository; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FilenameUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -8,9 +17,12 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import java.util.stream.Collectors; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; +@Slf4j @Service public class FilesystemService { @Value("${gameyfin.root}") @@ -19,18 +31,70 @@ public class FilesystemService { @Value("${gameyfin.file-extensions}") private List possibleGameFileExtensions; + @Autowired + private IgdbWrapper igdbWrapper; + + @Autowired + private DetectedGameRepository detectedGameRepository; + + @Autowired + private BlacklistRepository blacklistRepository; + public List getGameFiles() { + Path rootFolder = Path.of(rootFolderPath); - try(Stream stream = Files.list(rootFolder)) { + try (Stream stream = Files.list(rootFolder)) { // return all sub-folders (non-recursive) and files that have an extension that indicates that they are a downloadable file - return stream.filter(p -> Files.isDirectory(p) || possibleGameFileExtensions.contains(FilenameUtils.getExtension(p.getFileName().toString()))).toList(); + return stream + .filter(p -> Files.isDirectory(p) || possibleGameFileExtensions.contains(FilenameUtils.getExtension(p.getFileName().toString()))) + .toList(); } catch (IOException e) { throw new RuntimeException("Error while opening root folder", e); } } public List getGameFileNames() { - return this.getGameFiles().stream().map(p -> FilenameUtils.getBaseName(p.toString())).toList(); + return this.getGameFiles().stream().map(this::getFilename).toList(); + } + + public void scanGameLibrary() { + log.info("Starting scan..."); + + AtomicInteger newBlacklistCounter = new AtomicInteger(); + + List gameFiles = getGameFiles(); + + // Filter out the games we already know and the ones we already tried to map to a game without success + gameFiles = gameFiles.stream() + .filter(g -> !detectedGameRepository.existsByPath(g.toString())) + .filter(g -> !blacklistRepository.existsByPath(g.toString())) + .peek(p -> log.info("Found new potential game: {}", p)) + .toList(); + + // For each new game, load the info from IGDB + // If a game is not found on IGDB, blacklist the path so we won't query the API later on for the same path + List newDetectedGames = gameFiles.parallelStream() + .map(p -> { + Optional optionalGame = igdbWrapper.searchForGameByTitle(getFilename(p)); + return optionalGame.map(game -> Map.entry(p, game)).or(() -> { + blacklistRepository.save(new BlacklistEntry(p.toString())); + newBlacklistCounter.getAndIncrement(); + log.info("Added path '{}' to blacklist", p); + return Optional.empty(); + }); + }) + .filter(Optional::isPresent) + .map(Optional::get) + .map(e -> GameMapper.toDetectedGame(e.getValue(), e.getKey())) + .toList(); + + newDetectedGames = detectedGameRepository.saveAll(newDetectedGames); + + log.info("Scan finished: Found {} new games, deleted {} games, backlisted {} files/folders, {} games total.", newDetectedGames.size(), "NOT_IMPLEMENTED_YET", newBlacklistCounter.get(), detectedGameRepository.count()); + } + + private String getFilename(Path p) { + return FilenameUtils.getBaseName(p.toString()); } } diff --git a/src/main/java/de/grimsi/gameyfin/service/GameService.java b/src/main/java/de/grimsi/gameyfin/service/GameService.java new file mode 100644 index 0000000..5d1789f --- /dev/null +++ b/src/main/java/de/grimsi/gameyfin/service/GameService.java @@ -0,0 +1,23 @@ +package de.grimsi.gameyfin.service; + +import de.grimsi.gameyfin.entities.DetectedGame; +import de.grimsi.gameyfin.repositories.BlacklistRepository; +import de.grimsi.gameyfin.repositories.DetectedGameRepository; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class GameService { + + @Autowired + private DetectedGameRepository detectedGameRepository; + + @Autowired + private BlacklistRepository blacklistRepository; + + public List getAllDetectedGames() { + return detectedGameRepository.findAll(); + } +} diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index b735051..8ec0e05 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -1,5 +1,5 @@ gameyfin: - root: C:\Projects\privat\gameyfin-library + root: D:\Games cache: C:\Projects\privat\gameyfin-library\.gameyfin db: C:\Projects\privat\gameyfin-library\.gameyfin igdb: