diff --git a/backend/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java b/backend/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java index 436e106..3a319bc 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java +++ b/backend/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java @@ -3,7 +3,7 @@ package de.grimsi.gameyfin.mapper; import com.igdb.proto.Igdb; import de.grimsi.gameyfin.dto.GameOverviewDto; import de.grimsi.gameyfin.entities.DetectedGame; -import de.grimsi.gameyfin.util.ProtobufUtils; +import de.grimsi.gameyfin.util.ProtobufUtil; import java.nio.file.Path; import java.util.List; @@ -19,7 +19,7 @@ public class GameMapper { .slug(g.getSlug()) .title(g.getName()) .summary(g.getSummary()) - .releaseDate(ProtobufUtils.toInstant(g.getFirstReleaseDate())) + .releaseDate(ProtobufUtil.toInstant(g.getFirstReleaseDate())) .userRating((int) g.getRating()) .criticsRating((int) g.getAggregatedRating()) .totalRating((int) g.getTotalRating()) diff --git a/backend/src/main/java/de/grimsi/gameyfin/rest/GamesController.java b/backend/src/main/java/de/grimsi/gameyfin/rest/GamesController.java index bec791f..fdcc7ba 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/rest/GamesController.java +++ b/backend/src/main/java/de/grimsi/gameyfin/rest/GamesController.java @@ -2,7 +2,7 @@ package de.grimsi.gameyfin.rest; import de.grimsi.gameyfin.dto.GameOverviewDto; import de.grimsi.gameyfin.entities.DetectedGame; -import de.grimsi.gameyfin.service.FilesystemService; +import de.grimsi.gameyfin.service.DownloadService; import de.grimsi.gameyfin.service.GameService; import lombok.RequiredArgsConstructor; import org.springframework.http.MediaType; @@ -22,8 +22,7 @@ import java.util.Map; public class GamesController { private final GameService gameService; - - private final FilesystemService filesystemService; + private final DownloadService downloadService; @GetMapping(produces = MediaType.APPLICATION_JSON_VALUE) public List getAllGames() { @@ -50,12 +49,12 @@ public class GamesController { DetectedGame game = gameService.getDetectedGame(slug); - String downloadFileName = filesystemService.getDownloadFileName(game); + String downloadFileName = downloadService.getDownloadFileName(game); return ResponseEntity .ok() .header("Content-Disposition", "attachment; filename=\"%s\"".formatted(downloadFileName)) - .body(out -> filesystemService.downloadGameFiles(game, out)); + .body(out -> downloadService.downloadGameFiles(game, out)); } } diff --git a/backend/src/main/java/de/grimsi/gameyfin/rest/ImageController.java b/backend/src/main/java/de/grimsi/gameyfin/rest/ImageController.java index 7316753..5f30154 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/rest/ImageController.java +++ b/backend/src/main/java/de/grimsi/gameyfin/rest/ImageController.java @@ -1,6 +1,6 @@ package de.grimsi.gameyfin.rest; -import de.grimsi.gameyfin.service.FilesystemService; +import de.grimsi.gameyfin.service.DownloadService; import lombok.RequiredArgsConstructor; import org.springframework.core.io.Resource; import org.springframework.http.MediaType; @@ -17,10 +17,10 @@ import org.springframework.web.bind.annotation.RestController; @RequiredArgsConstructor public class ImageController { - private final FilesystemService filesystemService; + private final DownloadService downloadService; @GetMapping(value = "/{imageId}", produces = MediaType.IMAGE_PNG_VALUE) public Resource getCoverImageForGame(@PathVariable String imageId) { - return filesystemService.getImage(imageId); + return downloadService.downloadImage(imageId); } } diff --git a/backend/src/main/java/de/grimsi/gameyfin/rest/LibraryController.java b/backend/src/main/java/de/grimsi/gameyfin/rest/LibraryController.java index 5a83cfa..7df7eaf 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/rest/LibraryController.java +++ b/backend/src/main/java/de/grimsi/gameyfin/rest/LibraryController.java @@ -1,6 +1,7 @@ package de.grimsi.gameyfin.rest; -import de.grimsi.gameyfin.service.FilesystemService; +import de.grimsi.gameyfin.service.DownloadService; +import de.grimsi.gameyfin.service.LibraryService; import lombok.RequiredArgsConstructor; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.GetMapping; @@ -19,24 +20,25 @@ import java.util.List; @RequiredArgsConstructor public class LibraryController { - private final FilesystemService filesystemService; + private final LibraryService libraryService; + private final DownloadService downloadService; @GetMapping(value = "/scan", produces = MediaType.APPLICATION_JSON_VALUE) public void scanLibrary(@RequestParam(value = "download_images", defaultValue = "true") boolean downloadImages) { - filesystemService.scanGameLibrary(); + libraryService.scanGameLibrary(); if(downloadImages) downloadImages(); } @GetMapping(value = "/download-images") public void downloadImages() { - filesystemService.downloadGameCovers(); - filesystemService.downloadGameScreenshots(); - filesystemService.downloadCompanyLogos(); + downloadService.downloadGameCoversFromIgdb(); + downloadService.downloadGameScreenshotsFromIgdb(); + downloadService.downloadCompanyLogosFromIgdb(); } @GetMapping(value = "/files", produces = MediaType.APPLICATION_JSON_VALUE) public List getAllFiles() { - return filesystemService.getGameFiles().stream().map(Path::toString).toList(); + return libraryService.getGameFiles().stream().map(Path::toString).toList(); } } diff --git a/backend/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java b/backend/src/main/java/de/grimsi/gameyfin/service/DownloadService.java similarity index 60% rename from backend/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java rename to backend/src/main/java/de/grimsi/gameyfin/service/DownloadService.java index 452f975..490e97e 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java +++ b/backend/src/main/java/de/grimsi/gameyfin/service/DownloadService.java @@ -1,22 +1,12 @@ package de.grimsi.gameyfin.service; -import com.igdb.proto.Igdb; import de.grimsi.gameyfin.entities.Company; import de.grimsi.gameyfin.entities.DetectedGame; -import de.grimsi.gameyfin.entities.UnmappableFile; import de.grimsi.gameyfin.igdb.IgdbApiProperties; -import de.grimsi.gameyfin.igdb.IgdbWrapper; -import de.grimsi.gameyfin.mapper.GameMapper; import de.grimsi.gameyfin.repositories.DetectedGameRepository; -import de.grimsi.gameyfin.repositories.UnmappableFileRepository; +import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.compress.archivers.zip.ParallelScatterZipCreator; -import org.apache.commons.compress.archivers.zip.Zip64Mode; -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; -import org.apache.commons.compress.parallel.InputStreamSupplier; -import org.apache.commons.io.FilenameUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.ByteArrayResource; @@ -36,40 +26,29 @@ import reactor.core.publisher.Flux; import javax.annotation.PostConstruct; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; -import java.util.*; -import java.util.concurrent.ExecutionException; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import java.util.stream.Stream; import java.util.zip.Deflater; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import static de.grimsi.gameyfin.util.FilenameUtil.getFilenameWithExtension; + @Slf4j @Service -public class FilesystemService { - - @Value("${gameyfin.root}") - private String rootFolderPath; +@RequiredArgsConstructor +public class DownloadService { @Value("${gameyfin.cache}") private String cacheFolderPath; - @Value("${gameyfin.file-extensions}") - private List possibleGameFileExtensions; - - @Autowired - private IgdbWrapper igdbWrapper; - - @Autowired - private DetectedGameRepository detectedGameRepository; - - @Autowired - private UnmappableFileRepository unmappableFileRepository; + private final DetectedGameRepository detectedGameRepository; @Autowired private WebClient.Builder webclientBuilder; @@ -80,20 +59,6 @@ public class FilesystemService { igdbImageClient = webclientBuilder.baseUrl(IgdbApiProperties.IMAGES_BASE_URL).build(); } - public List getGameFiles() { - - Path rootFolder = Path.of(rootFolderPath); - - try (Stream stream = Files.list(rootFolder)) { - // return all sub-folders (non-recursive) and files that have an extension that indicates that they are a downloadable file - return stream - .filter(p -> Files.isDirectory(p) || hasGameArchiveExtension(p)) - .toList(); - } catch (IOException e) { - throw new RuntimeException("Error while opening root folder", e); - } - } - public String getDownloadFileName(DetectedGame g) { Path path = Path.of(g.getPath()); @@ -101,62 +66,40 @@ public class FilesystemService { return getFilenameWithExtension(path) + ".zip"; } - public void scanGameLibrary() { + public Resource downloadImage(String imageId) { + String filename = "%s.png".formatted(imageId); + + try { + return new ByteArrayResource(Files.readAllBytes(Paths.get("%s/%s".formatted(cacheFolderPath, filename)))); + } catch (IOException e) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Could not find image file %s".formatted(filename)); + } + } + + public void downloadGameFiles(DetectedGame game, OutputStream outputStream) { + StopWatch stopWatch = new StopWatch(); - log.info("Starting scan..."); + log.info("Starting game file download for {}...", game.getTitle()); + stopWatch.start(); - AtomicInteger newUnmappedFilesCounter = new AtomicInteger(); + Path path = Path.of(game.getPath()); - List gameFiles = getGameFiles(); - - // Check if any games that are in the library have been removed from the file system - // This would include renamed files, but they will be re-detected by the next step - List deletedGames = detectedGameRepository.getAllByPathNotIn(gameFiles); - detectedGameRepository.deleteAll(deletedGames); - deletedGames.forEach(g -> log.info("Game '{}' has been moved or deleted.", g.getPath())); - - // Now check if there are any unmapped files that have been removed from the file system - List deletedUnmappableFiles = unmappableFileRepository.getAllByPathNotIn(gameFiles); - unmappableFileRepository.deleteAll(deletedUnmappableFiles); - deletedUnmappableFiles.forEach(g -> log.info("Unmapped file '{}' has been moved or deleted.", g.getPath())); - - // Filter out the games we already know and the ones we already tried to map to a game without success - gameFiles = gameFiles.stream() - .filter(g -> !detectedGameRepository.existsByPath(g.toString())) - .filter(g -> !unmappableFileRepository.existsByPath(g.toString())) - .peek(p -> log.info("Found new potential game: {}", p)) - .toList(); - - // For each new game, load the info from IGDB - // If a game is not found on IGDB, add it to the list of unmapped files so we won't query the API later on for the same path - // If a game is not found on IGDB, blacklist the path, so we won't query the API later for the same path - List newDetectedGames = gameFiles.parallelStream() - .map(p -> { - Optional optionalGame = igdbWrapper.searchForGameByTitle(getFilenameWithoutExtension(p)); - return optionalGame.map(game -> Map.entry(p, game)).or(() -> { - unmappableFileRepository.save(new UnmappableFile(p.toString())); - newUnmappedFilesCounter.getAndIncrement(); - log.info("Added path '{}' to list of unmapped files", p); - return Optional.empty(); - }); - }) - .filter(Optional::isPresent) - .map(Optional::get) - .peek(e -> log.info("Mapped file '{}' to game '{}' (slug: {})", e.getKey(), e.getValue().getName(), e.getValue().getSlug())) - .map(e -> GameMapper.toDetectedGame(e.getValue(), e.getKey())) - .toList(); - - newDetectedGames = detectedGameRepository.saveAll(newDetectedGames); + if(path.toFile().isDirectory()) { + downloadFilesAsZip(path, outputStream); + } else { + downloadFile(path, outputStream); + } stopWatch.stop(); - log.info("Scan finished in {} seconds: Found {} new games, deleted {} games, could not map {} files/folders, {} games total.", - (int) stopWatch.getTotalTimeSeconds(), newDetectedGames.size(), deletedGames.size() + deletedUnmappableFiles.size(), newUnmappedFilesCounter.get(), detectedGameRepository.count()); + log.info("Downloaded game files of {} in {} seconds.", game.getTitle(), (int) stopWatch.getTotalTimeSeconds()); } - public void downloadGameCovers() { + + + public void downloadGameCoversFromIgdb() { StopWatch stopWatch = new StopWatch(); log.info("Starting game cover download..."); @@ -173,7 +116,7 @@ public class FilesystemService { log.info("Downloaded {} covers in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds()); } - public void downloadGameScreenshots() { + public void downloadGameScreenshotsFromIgdb() { StopWatch stopWatch = new StopWatch(); log.info("Starting game screenshot download..."); @@ -190,7 +133,7 @@ public class FilesystemService { log.info("Downloaded {} screenshots in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds()); } - public void downloadCompanyLogos() { + public void downloadCompanyLogosFromIgdb() { StopWatch stopWatch = new StopWatch(); log.info("Starting company logo download..."); @@ -209,36 +152,6 @@ public class FilesystemService { log.info("Downloaded {} company logos in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds()); } - public Resource getImage(String imageId) { - String filename = "%s.png".formatted(imageId); - - try { - return new ByteArrayResource(Files.readAllBytes(Paths.get("%s/%s".formatted(cacheFolderPath, filename)))); - } catch (IOException e) { - throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Could not find image file %s".formatted(filename)); - } - } - - public void downloadGameFiles(DetectedGame game, OutputStream outputStream) { - - StopWatch stopWatch = new StopWatch(); - - log.info("Starting game file download..."); - stopWatch.start(); - - Path path = Path.of(game.getPath()); - - if(path.toFile().isDirectory()) { - downloadFilesAsZip(path, outputStream); - } else { - downloadFile(path, outputStream); - } - - stopWatch.stop(); - - log.info("Downloaded game files of {} in {} seconds.", game.getTitle(), (int) stopWatch.getTotalTimeSeconds()); - } - private void downloadFile(Path path, OutputStream outputStream) { try { Files.copy(path, outputStream); @@ -270,18 +183,6 @@ public class FilesystemService { } } - private String getFilenameWithoutExtension(Path p) { - return FilenameUtils.getBaseName(p.toString()); - } - - private String getFilenameWithExtension(Path p) { - return FilenameUtils.getName(p.toString()); - } - - private boolean hasGameArchiveExtension(Path p) { - return possibleGameFileExtensions.contains(FilenameUtils.getExtension(p.getFileName().toString())); - } - private int downloadImagesIntoCache(MultiValueMap entityToImageIds, String imageSize, String imageType, String entityType) { AtomicInteger downloadCounter = new AtomicInteger(); Path cacheFolder = Path.of(cacheFolderPath); @@ -329,4 +230,5 @@ public class FilesystemService { return downloadCounter.get(); } + } diff --git a/backend/src/main/java/de/grimsi/gameyfin/service/LibraryService.java b/backend/src/main/java/de/grimsi/gameyfin/service/LibraryService.java new file mode 100644 index 0000000..cd0d8d1 --- /dev/null +++ b/backend/src/main/java/de/grimsi/gameyfin/service/LibraryService.java @@ -0,0 +1,111 @@ +package de.grimsi.gameyfin.service; + +import com.igdb.proto.Igdb; +import de.grimsi.gameyfin.entities.DetectedGame; +import de.grimsi.gameyfin.entities.UnmappableFile; +import de.grimsi.gameyfin.igdb.IgdbWrapper; +import de.grimsi.gameyfin.mapper.GameMapper; +import de.grimsi.gameyfin.repositories.DetectedGameRepository; +import de.grimsi.gameyfin.repositories.UnmappableFileRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; + +import java.io.IOException; +import java.nio.file.*; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +import static de.grimsi.gameyfin.util.FilenameUtil.getFilenameWithoutExtension; +import static de.grimsi.gameyfin.util.FilenameUtil.hasGameArchiveExtension; + +@Slf4j +@Service +public class LibraryService { + + @Value("${gameyfin.root}") + private String rootFolderPath; + + @Value("${gameyfin.cache}") + private String cacheFolderPath; + @Autowired + private IgdbWrapper igdbWrapper; + + @Autowired + private DetectedGameRepository detectedGameRepository; + + @Autowired + private UnmappableFileRepository unmappableFileRepository; + + public List getGameFiles() { + + Path rootFolder = Path.of(rootFolderPath); + + try (Stream stream = Files.list(rootFolder)) { + // return all sub-folders (non-recursive) and files that have an extension that indicates that they are a downloadable file + return stream + .filter(p -> Files.isDirectory(p) || hasGameArchiveExtension(p)) + .toList(); + } catch (IOException e) { + throw new RuntimeException("Error while opening root folder", e); + } + } + + public void scanGameLibrary() { + StopWatch stopWatch = new StopWatch(); + + log.info("Starting scan..."); + stopWatch.start(); + + AtomicInteger newUnmappedFilesCounter = new AtomicInteger(); + + List gameFiles = getGameFiles(); + + // Check if any games that are in the library have been removed from the file system + // This would include renamed files, but they will be re-detected by the next step + List deletedGames = detectedGameRepository.getAllByPathNotIn(gameFiles); + detectedGameRepository.deleteAll(deletedGames); + deletedGames.forEach(g -> log.info("Game '{}' has been moved or deleted.", g.getPath())); + + // Now check if there are any unmapped files that have been removed from the file system + List deletedUnmappableFiles = unmappableFileRepository.getAllByPathNotIn(gameFiles); + unmappableFileRepository.deleteAll(deletedUnmappableFiles); + deletedUnmappableFiles.forEach(g -> log.info("Unmapped file '{}' has been moved or deleted.", g.getPath())); + + // Filter out the games we already know and the ones we already tried to map to a game without success + gameFiles = gameFiles.stream() + .filter(g -> !detectedGameRepository.existsByPath(g.toString())) + .filter(g -> !unmappableFileRepository.existsByPath(g.toString())) + .peek(p -> log.info("Found new potential game: {}", p)) + .toList(); + + // For each new game, load the info from IGDB + // If a game is not found on IGDB, add it to the list of unmapped files so we won't query the API later on for the same path + // If a game is not found on IGDB, blacklist the path, so we won't query the API later for the same path + List newDetectedGames = gameFiles.parallelStream() + .map(p -> { + Optional optionalGame = igdbWrapper.searchForGameByTitle(getFilenameWithoutExtension(p)); + return optionalGame.map(game -> Map.entry(p, game)).or(() -> { + unmappableFileRepository.save(new UnmappableFile(p.toString())); + newUnmappedFilesCounter.getAndIncrement(); + log.info("Added path '{}' to list of unmapped files", p); + return Optional.empty(); + }); + }) + .filter(Optional::isPresent) + .map(Optional::get) + .peek(e -> log.info("Mapped file '{}' to game '{}' (slug: {})", e.getKey(), e.getValue().getName(), e.getValue().getSlug())) + .map(e -> GameMapper.toDetectedGame(e.getValue(), e.getKey())) + .toList(); + + newDetectedGames = detectedGameRepository.saveAll(newDetectedGames); + + stopWatch.stop(); + + log.info("Scan finished in {} seconds: Found {} new games, deleted {} games, could not map {} files/folders, {} games total.", + (int) stopWatch.getTotalTimeSeconds(), newDetectedGames.size(), deletedGames.size() + deletedUnmappableFiles.size(), newUnmappedFilesCounter.get(), detectedGameRepository.count()); + } +} diff --git a/backend/src/main/java/de/grimsi/gameyfin/util/FilenameUtil.java b/backend/src/main/java/de/grimsi/gameyfin/util/FilenameUtil.java new file mode 100644 index 0000000..fda1eae --- /dev/null +++ b/backend/src/main/java/de/grimsi/gameyfin/util/FilenameUtil.java @@ -0,0 +1,32 @@ +package de.grimsi.gameyfin.util; + +import org.apache.commons.io.FilenameUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.nio.file.Path; +import java.util.List; + +@Service +public class FilenameUtil { + + private static List possibleGameFileExtensions; + + @Value("${gameyfin.file-extensions}") + public void setPossibleGameFileExtensions(List possibleGameFileExtensions) { + FilenameUtil.possibleGameFileExtensions = possibleGameFileExtensions; + } + + public static String getFilenameWithoutExtension(Path p) { + return FilenameUtils.getBaseName(p.toString()); + } + + public static String getFilenameWithExtension(Path p) { + return FilenameUtils.getName(p.toString()); + } + + public static boolean hasGameArchiveExtension(Path p) { + return possibleGameFileExtensions.contains(FilenameUtils.getExtension(p.getFileName().toString())); + } + +} diff --git a/backend/src/main/java/de/grimsi/gameyfin/util/ProtobufUtils.java b/backend/src/main/java/de/grimsi/gameyfin/util/ProtobufUtil.java similarity index 88% rename from backend/src/main/java/de/grimsi/gameyfin/util/ProtobufUtils.java rename to backend/src/main/java/de/grimsi/gameyfin/util/ProtobufUtil.java index faac3b9..68439f2 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/util/ProtobufUtils.java +++ b/backend/src/main/java/de/grimsi/gameyfin/util/ProtobufUtil.java @@ -4,7 +4,7 @@ import com.google.protobuf.Timestamp; import java.time.Instant; -public class ProtobufUtils { +public class ProtobufUtil { public static Instant toInstant(Timestamp t) { return Instant.ofEpochSecond(t.getSeconds(), t.getNanos()); }