diff --git a/backend/pom.xml b/backend/pom.xml index 0bfaaf9..66bbe98 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -7,7 +7,7 @@ gameyfin de.grimsi - 1.1.4 + 1.1.4-RC1 gameyfin-backend diff --git a/backend/src/main/java/de/grimsi/gameyfin/config/FilesystemConfig.java b/backend/src/main/java/de/grimsi/gameyfin/config/FilesystemConfig.java index 41078ba..10df1fc 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/config/FilesystemConfig.java +++ b/backend/src/main/java/de/grimsi/gameyfin/config/FilesystemConfig.java @@ -39,7 +39,7 @@ public class FilesystemConfig { props.setProperty("gameyfin.db", "%s/.gameyfin/db".formatted(firstLibraryPath)); } - if(StringUtils.hasText(cachePath)) { + if(!StringUtils.hasText(cachePath)) { props.setProperty("gameyfin.cache", "%s/.gameyfin/cache".formatted(firstLibraryPath)); } diff --git a/backend/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java b/backend/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java index 26fe5ad..c4bc180 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java +++ b/backend/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java @@ -7,6 +7,7 @@ import de.grimsi.gameyfin.igdb.dto.TwitchOAuthTokenDto; import de.grimsi.gameyfin.mapper.GameMapper; import io.github.resilience4j.reactor.bulkhead.operator.BulkheadOperator; import io.github.resilience4j.reactor.ratelimiter.operator.RateLimiterOperator; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -24,6 +25,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; @Slf4j +@RequiredArgsConstructor @Service public class IgdbWrapper { @Value("${gameyfin.igdb.api.client-id}") @@ -35,11 +37,9 @@ public class IgdbWrapper { @Value("${gameyfin.igdb.config.preferred-platforms:6}") private String preferredPlatforms; - @Autowired - private WebClient.Builder webclientBuilder; - - @Autowired - private WebClientConfig webClientConfig; + private final WebClient.Builder webclientBuilder; + private final WebClientConfig webClientConfig; + private final GameMapper gameMapper; private WebClient twitchApiClient; @@ -106,7 +106,7 @@ public class IgdbWrapper { if(gameResult == null) return Collections.emptyList(); - return gameResult.getGamesList().stream().map(GameMapper::toAutocompleteSuggestionDto).toList(); + return gameResult.getGamesList().stream().map(gameMapper::toAutocompleteSuggestionDto).toList(); } public Optional searchForGameByTitle(String searchTerm) { diff --git a/backend/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java b/backend/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java index fd63e13..bc4ddb3 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java +++ b/backend/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java @@ -4,6 +4,7 @@ import com.igdb.proto.Igdb; import de.grimsi.gameyfin.dto.AutocompleteSuggestionDto; import de.grimsi.gameyfin.dto.GameOverviewDto; import de.grimsi.gameyfin.entities.DetectedGame; +import de.grimsi.gameyfin.service.FilesystemService; import de.grimsi.gameyfin.service.LibraryService; import de.grimsi.gameyfin.util.ProtobufUtil; import lombok.RequiredArgsConstructor; @@ -24,9 +25,13 @@ import java.util.List; import java.util.stream.Stream; @Slf4j +@RequiredArgsConstructor +@Component public class GameMapper { - public static DetectedGame toDetectedGame(Igdb.Game g, Path path) { + private final FilesystemService filesystemService; + + public DetectedGame toDetectedGame(Igdb.Game g, Path path) { List multiplayerModes = g.getMultiplayerModesList(); List screenshotIds = g.getScreenshotsList().stream().map(Igdb.Screenshot::getImageId).toList(); List videoIds = g.getVideosList().stream().map(Igdb.GameVideo::getVideoId).toList(); @@ -58,7 +63,7 @@ public class GameMapper { .build(); } - public static GameOverviewDto toGameOverviewDto(DetectedGame game) { + public GameOverviewDto toGameOverviewDto(DetectedGame game) { return GameOverviewDto.builder() .slug(game.getSlug()) .title(game.getTitle()) @@ -66,7 +71,7 @@ public class GameMapper { .build(); } - public static AutocompleteSuggestionDto toAutocompleteSuggestionDto(Igdb.Game game) { + public AutocompleteSuggestionDto toAutocompleteSuggestionDto(Igdb.Game game) { return AutocompleteSuggestionDto.builder() .slug(game.getSlug()) .title(game.getName()) @@ -74,7 +79,7 @@ public class GameMapper { .build(); } - private static String getCoverId(Igdb.Game g) { + private String getCoverId(Igdb.Game g) { String coverId = g.getCover().getImageId(); if(StringUtils.hasText(coverId)) return coverId; @@ -82,23 +87,23 @@ public class GameMapper { return "nocover"; } - private static boolean hasOfflineCoop(List modes) { + private boolean hasOfflineCoop(List modes) { return modes.stream().anyMatch(Igdb.MultiplayerMode::getOfflinecoop); } - private static boolean hasLanSupport(List modes) { + private boolean hasLanSupport(List modes) { return modes.stream().anyMatch(Igdb.MultiplayerMode::getLancoop); } - private static boolean hasOnlineCoop(List modes) { + private boolean hasOnlineCoop(List modes) { return modes.stream().anyMatch(Igdb.MultiplayerMode::getOnlinecoop); } - private static int getMaxPlayers(List modes) { + private int getMaxPlayers(List modes) { return modes.stream().mapToInt(Igdb.MultiplayerMode::getOnlinecoopmax).max().orElse(0); } - private static long calculateDiskSize(Igdb.Game g, Path path) { + private long calculateDiskSize(Igdb.Game g, Path path) { StopWatch stopWatch = new StopWatch(); log.info("Calculating disk size for game '{}'...", g.getName()); @@ -106,16 +111,11 @@ public class GameMapper { long fileSize; - if(Files.isDirectory(path)) { - // Some benchmarks I did have shown that trying to parallelize this process makes it slower instead of faster - fileSize = FileUtils.sizeOfDirectory(path.toFile()); - } else { - try{ - fileSize = Files.size(path); - } catch (IOException e) { - log.error("Error while calculating size of file '{}'.", path); - fileSize = -1L; - } + try { + fileSize = filesystemService.getSizeOnDisk(path); + } catch(IOException e) { + log.error("Error while calculating disk size for game '{}'", g.getName()); + fileSize = -1L; } stopWatch.stop(); diff --git a/backend/src/main/java/de/grimsi/gameyfin/rest/GamesController.java b/backend/src/main/java/de/grimsi/gameyfin/rest/GamesController.java index fdcc7ba..156988a 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/rest/GamesController.java +++ b/backend/src/main/java/de/grimsi/gameyfin/rest/GamesController.java @@ -54,7 +54,7 @@ public class GamesController { return ResponseEntity .ok() .header("Content-Disposition", "attachment; filename=\"%s\"".formatted(downloadFileName)) - .body(out -> downloadService.downloadGameFiles(game, out)); + .body(out -> downloadService.sendGamefilesToClient(game, out)); } } diff --git a/backend/src/main/java/de/grimsi/gameyfin/rest/ImageController.java b/backend/src/main/java/de/grimsi/gameyfin/rest/ImageController.java index 9d2bd79..d4f535a 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/rest/ImageController.java +++ b/backend/src/main/java/de/grimsi/gameyfin/rest/ImageController.java @@ -27,6 +27,6 @@ public class ImageController { public ResponseEntity getCoverImageForGame(@PathVariable String imageId) { return ResponseEntity.ok() .cacheControl(CacheControl.maxAge(365, TimeUnit.DAYS).cachePublic()) - .body(downloadService.downloadImage(imageId)); + .body(downloadService.sendImageToClient(imageId)); } } diff --git a/backend/src/main/java/de/grimsi/gameyfin/rest/LibraryController.java b/backend/src/main/java/de/grimsi/gameyfin/rest/LibraryController.java index 8e3090b..a0a14b0 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/rest/LibraryController.java +++ b/backend/src/main/java/de/grimsi/gameyfin/rest/LibraryController.java @@ -1,6 +1,7 @@ package de.grimsi.gameyfin.rest; import de.grimsi.gameyfin.service.DownloadService; +import de.grimsi.gameyfin.service.ImageService; import de.grimsi.gameyfin.service.LibraryService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -25,7 +26,7 @@ import java.util.List; public class LibraryController { private final LibraryService libraryService; - private final DownloadService downloadService; + private final ImageService imageService; @GetMapping(value = "/scan", produces = MediaType.APPLICATION_JSON_VALUE) public void scanLibrary(@RequestParam(value = "download_images", defaultValue = "true") boolean downloadImages) { @@ -36,9 +37,9 @@ public class LibraryController { @GetMapping(value = "/download-images") public void downloadImages() { - downloadService.downloadGameCoversFromIgdb(); - downloadService.downloadGameScreenshotsFromIgdb(); - downloadService.downloadCompanyLogosFromIgdb(); + imageService.downloadGameCoversFromIgdb(); + imageService.downloadGameScreenshotsFromIgdb(); + imageService.downloadCompanyLogosFromIgdb(); log.info("Downloading images completed."); } diff --git a/backend/src/main/java/de/grimsi/gameyfin/rest/LibraryManagementController.java b/backend/src/main/java/de/grimsi/gameyfin/rest/LibraryManagementController.java index d5d4ce6..05d9da0 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/rest/LibraryManagementController.java +++ b/backend/src/main/java/de/grimsi/gameyfin/rest/LibraryManagementController.java @@ -8,6 +8,7 @@ import de.grimsi.gameyfin.entities.UnmappableFile; import de.grimsi.gameyfin.repositories.DetectedGameRepository; import de.grimsi.gameyfin.service.DownloadService; import de.grimsi.gameyfin.service.GameService; +import de.grimsi.gameyfin.service.ImageService; import de.grimsi.gameyfin.service.LibraryService; import lombok.RequiredArgsConstructor; import org.springframework.http.MediaType; @@ -23,8 +24,7 @@ import java.util.List; public class LibraryManagementController { private final GameService gameService; - private final DownloadService downloadService; - + private final ImageService imageService; private final LibraryService libraryService; @DeleteMapping(value = "/delete-game/{slug}", produces = MediaType.APPLICATION_JSON_VALUE) @@ -45,10 +45,10 @@ public class LibraryManagementController { @PostMapping(value = "/map-path", produces = MediaType.APPLICATION_JSON_VALUE) public DetectedGame manuallyMapPathToSlug(@RequestBody PathToSlugDto pathToSlugDto) { DetectedGame game = gameService.mapPathToGame(pathToSlugDto.getPath(), pathToSlugDto.getSlug()); - - downloadService.downloadGameCoversFromIgdb(); - downloadService.downloadGameScreenshotsFromIgdb(); - downloadService.downloadCompanyLogosFromIgdb(); + + imageService.downloadGameCoversFromIgdb(); + imageService.downloadGameScreenshotsFromIgdb(); + imageService.downloadCompanyLogosFromIgdb(); return game; } diff --git a/backend/src/main/java/de/grimsi/gameyfin/service/DownloadService.java b/backend/src/main/java/de/grimsi/gameyfin/service/DownloadService.java index ecf6dea..563e522 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/service/DownloadService.java +++ b/backend/src/main/java/de/grimsi/gameyfin/service/DownloadService.java @@ -1,41 +1,19 @@ package de.grimsi.gameyfin.service; -import de.grimsi.gameyfin.entities.Company; import de.grimsi.gameyfin.entities.DetectedGame; import de.grimsi.gameyfin.exceptions.DownloadAbortedException; -import de.grimsi.gameyfin.igdb.IgdbApiProperties; -import de.grimsi.gameyfin.repositories.DetectedGameRepository; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.catalina.connector.ClientAbortException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.ByteArrayResource; import org.springframework.core.io.Resource; -import org.springframework.core.io.buffer.DataBuffer; -import org.springframework.core.io.buffer.DataBufferUtils; -import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; import org.springframework.util.StopWatch; -import org.springframework.util.StringUtils; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.reactive.function.client.WebClientResponseException; -import org.springframework.web.server.ResponseStatusException; -import reactor.core.publisher.Flux; -import javax.annotation.PostConstruct; import java.io.IOException; import java.io.OutputStream; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; import java.util.zip.Deflater; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -47,19 +25,7 @@ import static de.grimsi.gameyfin.util.FilenameUtil.getFilenameWithExtension; @RequiredArgsConstructor public class DownloadService { - @Value("${gameyfin.cache}") - private String cacheFolderPath; - - private final DetectedGameRepository detectedGameRepository; - - @Autowired - private WebClient.Builder webclientBuilder; - private WebClient igdbImageClient; - - @PostConstruct - public void init() { - igdbImageClient = webclientBuilder.baseUrl(IgdbApiProperties.IMAGES_BASE_URL).build(); - } + private final FilesystemService filesystemService; public String getDownloadFileName(DetectedGame g) { Path path = Path.of(g.getPath()); @@ -68,17 +34,12 @@ public class DownloadService { return getFilenameWithExtension(path) + ".zip"; } - public Resource downloadImage(String imageId) { + public Resource sendImageToClient(String imageId) { String filename = "%s.png".formatted(imageId); - - try { - return new ByteArrayResource(Files.readAllBytes(Paths.get("%s/%s".formatted(cacheFolderPath, filename)))); - } catch (IOException e) { - throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Could not find image file %s".formatted(filename)); - } + return filesystemService.getFileFromCache(filename); } - public void downloadGameFiles(DetectedGame game, OutputStream outputStream) { + public void sendGamefilesToClient(DetectedGame game, OutputStream outputStream) { StopWatch stopWatch = new StopWatch(); @@ -88,18 +49,16 @@ public class DownloadService { Path path = Path.of(game.getPath()); - if (path.toFile().isDirectory()) { - - try { - downloadFilesAsZip(path, outputStream); - } catch(DownloadAbortedException e) { - stopWatch.stop(); - log.info("Download of game {} was aborted by client after {} seconds", game.getTitle(), (int) stopWatch.getTotalTimeSeconds()); - return; + try { + if (path.toFile().isDirectory()) { + sendGamefilesAsZipToClient(path, outputStream); + } else { + sendGamefileToClient(path, outputStream); } - - } else { - downloadFile(path, outputStream); + } catch (DownloadAbortedException e) { + stopWatch.stop(); + log.info("Download of game {} was aborted by client after {} seconds", game.getTitle(), (int) stopWatch.getTotalTimeSeconds()); + return; } stopWatch.stop(); @@ -107,70 +66,18 @@ public class DownloadService { log.info("Downloaded game files of {} in {} seconds.", game.getTitle(), (int) stopWatch.getTotalTimeSeconds()); } - - public void downloadGameCoversFromIgdb() { - StopWatch stopWatch = new StopWatch(); - - log.info("Starting game cover download..."); - stopWatch.start(); - - MultiValueMap gameToImageIds = new LinkedMultiValueMap<>( - detectedGameRepository.findAll().stream() - .collect(Collectors.toMap(DetectedGame::getSlug, g -> Collections.singletonList(g.getCoverId())))); - - int downloadCount = downloadImagesIntoCache(gameToImageIds, IgdbApiProperties.COVER_IMAGE_SIZE, "cover", "game"); - - stopWatch.stop(); - - log.info("Downloaded {} covers in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds()); - } - - public void downloadGameScreenshotsFromIgdb() { - StopWatch stopWatch = new StopWatch(); - - log.info("Starting game screenshot download..."); - stopWatch.start(); - - MultiValueMap gamesToImageIds = new LinkedMultiValueMap<>( - detectedGameRepository.findAll().stream() - .collect(Collectors.toMap(DetectedGame::getSlug, DetectedGame::getScreenshotIds))); - - int downloadCount = downloadImagesIntoCache(gamesToImageIds, IgdbApiProperties.SCREENSHOT_IMAGE_SIZE, "screenshot", "game"); - - stopWatch.stop(); - - log.info("Downloaded {} screenshots in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds()); - } - - public void downloadCompanyLogosFromIgdb() { - StopWatch stopWatch = new StopWatch(); - - log.info("Starting company logo download..."); - stopWatch.start(); - - Map> companyToLogoIdMap = detectedGameRepository.findAll().stream() - .flatMap(g -> g.getCompanies().stream()) - .collect(Collectors.toMap(Company::getSlug, c -> Collections.singletonList(c.getLogoId()), (c1, c2) -> c1)); - - MultiValueMap companiesToLogoIds = new LinkedMultiValueMap<>(companyToLogoIdMap); - - int downloadCount = downloadImagesIntoCache(companiesToLogoIds, IgdbApiProperties.LOGO_IMAGE_SIZE, "logo", "company"); - - stopWatch.stop(); - - log.info("Downloaded {} company logos in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds()); - } - - private void downloadFile(Path path, OutputStream outputStream) { + private void sendGamefileToClient(Path path, OutputStream outputStream) { try { Files.copy(path, outputStream); + } catch (ClientAbortException e) { + // Aborted downloads will be handled gracefully + throw new DownloadAbortedException(); } catch (IOException e) { log.error("Error while downloading file:", e); - throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Could not load file '%s'.".formatted(path)); } } - private void downloadFilesAsZip(Path path, OutputStream outputStream) { + private void sendGamefilesAsZipToClient(Path path, OutputStream outputStream) { ZipOutputStream zos = new ZipOutputStream(outputStream) {{ def.setLevel(Deflater.NO_COMPRESSION); }}; @@ -195,65 +102,4 @@ public class DownloadService { log.error("Error while zipping files:", e); } } - - private int downloadImagesIntoCache(MultiValueMap entityToImageIds, String imageSize, String imageType, String entityType) { - AtomicInteger downloadCounter = new AtomicInteger(); - Path cacheFolder = Path.of(cacheFolderPath); - - try { - Files.createDirectories(cacheFolder); - } catch (IOException e) { - throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Could not create cache folder."); - } - - entityToImageIds.entrySet().parallelStream().forEach(entry -> - entry.getValue().forEach(imageId -> { - - if (!StringUtils.hasText(imageId)) return; - - String imgFileName = "%s.png".formatted(imageId); - String imgUrl = "t_%s/%s".formatted(imageSize, imgFileName); - - if (Files.exists(Path.of(cacheFolderPath, imgFileName))) { - - Path existingImageFile = Path.of(cacheFolderPath, imgFileName); - - try { - if(Files.size(existingImageFile) == 0L) { - log.info("File '{}' is corrupt, retrying download...", imgFileName); - Files.delete(existingImageFile); - } else { - log.debug("{} for {} '{}' already downloaded ({}), skipping.", - imageType.substring(0, 1).toUpperCase() + imageType.substring(1).toLowerCase(), - entityType, - entry.getKey(), - imgFileName); - return; - } - } catch (IOException e) { - log.error("Error while checking file '{}'.", existingImageFile); - } - } - - Flux dataBuffer = igdbImageClient.get() - .uri(imgUrl) - .retrieve() - .bodyToFlux(DataBuffer.class); - - try { - DataBufferUtils.write(dataBuffer, cacheFolder.resolve(imgFileName), StandardOpenOption.CREATE) - .share().block(); - } catch (WebClientResponseException e) { - if (e.getStatusCode().is4xxClientError()) { - log.error("Could not download {} for {} '{}' from {}: {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl, e.getStatusCode()); - } - } - - downloadCounter.getAndIncrement(); - log.info("Downloaded {} for {} '{}' from {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl); - })); - - return downloadCounter.get(); - } - } diff --git a/backend/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java b/backend/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java new file mode 100644 index 0000000..25e551a --- /dev/null +++ b/backend/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java @@ -0,0 +1,84 @@ +package de.grimsi.gameyfin.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.core.io.buffer.DataBufferUtils; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; +import reactor.core.publisher.Flux; + +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; + +@Slf4j +@Service +public class FilesystemService { + + @Value("${gameyfin.cache}") + private String cacheFolderPath; + + @PostConstruct + public void createCacheFolder() throws IOException { + Files.createDirectories(Path.of(cacheFolderPath)); + } + + public void saveFileToCache(Flux dataBuffer, String filename) { + DataBufferUtils.write(dataBuffer, Path.of(cacheFolderPath).resolve(filename), StandardOpenOption.CREATE) + .share().block(); + } + + public ByteArrayResource getFileFromCache(String filename) { + try { + return new ByteArrayResource(Files.readAllBytes(Paths.get("%s/%s".formatted(cacheFolderPath, filename)))); + } catch (IOException e) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Could not find image file %s".formatted(filename)); + } + } + + public void deleteFileFromCache(String filename) { + try { + Files.delete(getPathFromFilename(filename)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public boolean isCachedFileCorrupt(String filename) { + try { + return Files.size(getPathFromFilename(filename)) == 0L; + } catch (IOException e) { + log.error("Could not determine file size of '{}'", filename); + return true; + } + } + + public boolean doesCachedFileExist(String filename) { + return Files.exists(getPathFromFilename(filename)); + } + + public long getSizeOnDisk(Path path) throws IOException { + if (Files.isDirectory(path)) { + // Some benchmarks I did have shown that trying to parallelize this process makes it slower instead of faster + return FileUtils.sizeOfDirectory(path.toFile()); + } else { + try { + return Files.size(path); + } catch (IOException e) { + log.error("Error while calculating size of file '{}'.", path); + throw e; + } + } + } + + private Path getPathFromFilename(String filename) { + return Path.of(cacheFolderPath, filename); + } +} diff --git a/backend/src/main/java/de/grimsi/gameyfin/service/GameService.java b/backend/src/main/java/de/grimsi/gameyfin/service/GameService.java index 2e2f338..3fd2a82 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/service/GameService.java +++ b/backend/src/main/java/de/grimsi/gameyfin/service/GameService.java @@ -8,7 +8,7 @@ import de.grimsi.gameyfin.igdb.IgdbWrapper; import de.grimsi.gameyfin.mapper.GameMapper; import de.grimsi.gameyfin.repositories.DetectedGameRepository; import de.grimsi.gameyfin.repositories.UnmappableFileRepository; -import org.springframework.beans.factory.annotation.Autowired; +import lombok.RequiredArgsConstructor; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; import org.springframework.web.server.ResponseStatusException; @@ -19,17 +19,14 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +@RequiredArgsConstructor @Service public class GameService { - @Autowired - private IgdbWrapper igdbWrapper; - - @Autowired - private DetectedGameRepository detectedGameRepository; - - @Autowired - private UnmappableFileRepository unmappableFileRepository; + private final IgdbWrapper igdbWrapper; + private final GameMapper gameMapper; + private final DetectedGameRepository detectedGameRepository; + private final UnmappableFileRepository unmappableFileRepository; public List getAllDetectedGames() { return detectedGameRepository.findAll(); @@ -48,13 +45,13 @@ public class GameService { } public List getGameOverviews() { - return detectedGameRepository.findAll().stream().map(GameMapper::toGameOverviewDto).toList(); + return detectedGameRepository.findAll().stream().map(gameMapper::toGameOverviewDto).toList(); } public void deleteGame(String slug) { DetectedGame gameToBeDeleted = getDetectedGame(slug); - // Add the path of the game to be deleted to the unmappable files + // Add the path of the game to be deleted to the unmappable files, // so it doesn't get re-indexed on the next library scan unmappableFileRepository.save(new UnmappableFile(gameToBeDeleted.getPath())); @@ -94,7 +91,7 @@ public class GameService { Igdb.Game igdbGame = igdbWrapper.getGameBySlug(slug) .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Game with slug '%s' does not exist on IGDB.".formatted(slug))); - DetectedGame game = GameMapper.toDetectedGame(igdbGame, Path.of(unmappableFile.getPath())); + DetectedGame game = gameMapper.toDetectedGame(igdbGame, Path.of(unmappableFile.getPath())); game = detectedGameRepository.save(game); unmappableFileRepository.delete(unmappableFile); @@ -106,7 +103,7 @@ public class GameService { Igdb.Game igdbGame = igdbWrapper.getGameBySlug(slug) .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Game with slug '%s' does not exist on IGDB.".formatted(slug))); - DetectedGame game = GameMapper.toDetectedGame(igdbGame, Path.of(existingGame.getPath())); + DetectedGame game = gameMapper.toDetectedGame(igdbGame, Path.of(existingGame.getPath())); game = detectedGameRepository.save(game); detectedGameRepository.deleteById(existingGame.getSlug()); diff --git a/backend/src/main/java/de/grimsi/gameyfin/service/ImageService.java b/backend/src/main/java/de/grimsi/gameyfin/service/ImageService.java new file mode 100644 index 0000000..292f764 --- /dev/null +++ b/backend/src/main/java/de/grimsi/gameyfin/service/ImageService.java @@ -0,0 +1,142 @@ +package de.grimsi.gameyfin.service; + +import de.grimsi.gameyfin.entities.Company; +import de.grimsi.gameyfin.entities.DetectedGame; +import de.grimsi.gameyfin.igdb.IgdbApiProperties; +import de.grimsi.gameyfin.repositories.DetectedGameRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.util.StopWatch; +import org.springframework.util.StringUtils; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.reactive.function.client.WebClientResponseException; +import reactor.core.publisher.Flux; + +import javax.annotation.PostConstruct; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +@Slf4j +@RequiredArgsConstructor +@Service +public class ImageService { + + @Value("${gameyfin.cache}") + private String cacheFolderPath; + + private final FilesystemService filesystemService; + private final DetectedGameRepository detectedGameRepository; + private final WebClient.Builder webclientBuilder; + private WebClient igdbImageClient; + + @PostConstruct + public void init() { + igdbImageClient = webclientBuilder.baseUrl(IgdbApiProperties.IMAGES_BASE_URL).build(); + } + + public void downloadGameCoversFromIgdb() { + StopWatch stopWatch = new StopWatch(); + + log.info("Starting game cover download..."); + stopWatch.start(); + + MultiValueMap gameToImageIds = new LinkedMultiValueMap<>( + detectedGameRepository.findAll().stream() + .collect(Collectors.toMap(DetectedGame::getSlug, g -> Collections.singletonList(g.getCoverId())))); + + int downloadCount = saveImagesIntoCache(gameToImageIds, IgdbApiProperties.COVER_IMAGE_SIZE, "cover", "game"); + + stopWatch.stop(); + + log.info("Downloaded {} covers in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds()); + } + + public void downloadGameScreenshotsFromIgdb() { + StopWatch stopWatch = new StopWatch(); + + log.info("Starting game screenshot download..."); + stopWatch.start(); + + MultiValueMap gamesToImageIds = new LinkedMultiValueMap<>( + detectedGameRepository.findAll().stream() + .collect(Collectors.toMap(DetectedGame::getSlug, DetectedGame::getScreenshotIds))); + + int downloadCount = saveImagesIntoCache(gamesToImageIds, IgdbApiProperties.SCREENSHOT_IMAGE_SIZE, "screenshot", "game"); + + stopWatch.stop(); + + log.info("Downloaded {} screenshots in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds()); + } + + public void downloadCompanyLogosFromIgdb() { + StopWatch stopWatch = new StopWatch(); + + log.info("Starting company logo download..."); + stopWatch.start(); + + Map> companyToLogoIdMap = detectedGameRepository.findAll().stream() + .flatMap(g -> g.getCompanies().stream()) + .collect(Collectors.toMap(Company::getSlug, c -> Collections.singletonList(c.getLogoId()), (c1, c2) -> c1)); + + MultiValueMap companiesToLogoIds = new LinkedMultiValueMap<>(companyToLogoIdMap); + + int downloadCount = saveImagesIntoCache(companiesToLogoIds, IgdbApiProperties.LOGO_IMAGE_SIZE, "logo", "company"); + + stopWatch.stop(); + + log.info("Downloaded {} company logos in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds()); + } + + private int saveImagesIntoCache(MultiValueMap entityToImageIds, String imageSize, String imageType, String entityType) { + AtomicInteger downloadCounter = new AtomicInteger(); + + entityToImageIds.entrySet().parallelStream().forEach(entry -> + entry.getValue().forEach(imageId -> { + + if (!StringUtils.hasText(imageId)) return; + + String imgFileName = "%s.png".formatted(imageId); + String imgUrl = "t_%s/%s".formatted(imageSize, imgFileName); + + if (filesystemService.doesCachedFileExist(imgFileName)) { + if (filesystemService.isCachedFileCorrupt(imgFileName)) { + log.info("File '{}' is corrupt, retrying download...", imgFileName); + filesystemService.deleteFileFromCache(imgFileName); + } else { + log.debug("{} for {} '{}' already downloaded ({}), skipping.", + imageType.substring(0, 1).toUpperCase() + imageType.substring(1).toLowerCase(), + entityType, + entry.getKey(), + imgFileName); + return; + } + } + + Flux dataBuffer = igdbImageClient.get() + .uri(imgUrl) + .retrieve() + .bodyToFlux(DataBuffer.class); + + try { + filesystemService.saveFileToCache(dataBuffer, imgFileName); + } catch (WebClientResponseException e) { + if (e.getStatusCode().is4xxClientError()) { + log.error("Could not download {} for {} '{}' from {}: {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl, e.getStatusCode()); + } + } + + downloadCounter.getAndIncrement(); + log.info("Downloaded {} for {} '{}' from {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl); + })); + + return downloadCounter.get(); + } +} diff --git a/backend/src/main/java/de/grimsi/gameyfin/service/LibraryService.java b/backend/src/main/java/de/grimsi/gameyfin/service/LibraryService.java index 7e5dd67..9930bc3 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/service/LibraryService.java +++ b/backend/src/main/java/de/grimsi/gameyfin/service/LibraryService.java @@ -32,7 +32,9 @@ public class LibraryService { @Value("${gameyfin.sources}") private List libraryFolders; + private final IgdbWrapper igdbWrapper; + private final GameMapper gameMapper; private final DetectedGameRepository detectedGameRepository; private final UnmappableFileRepository unmappableFileRepository; @@ -105,7 +107,7 @@ public class LibraryService { .filter(Optional::isPresent) .map(Optional::get) .peek(e -> log.info("Mapped file '{}' to game '{}' (slug: {})", e.getKey(), e.getValue().getName(), e.getValue().getSlug())) - .map(e -> GameMapper.toDetectedGame(e.getValue(), e.getKey())) + .map(e -> gameMapper.toDetectedGame(e.getValue(), e.getKey())) .collect(Collectors.toList()); List duplicateGames = getDuplicates(newDetectedGames); diff --git a/backend/src/main/java/de/grimsi/gameyfin/util/FilenameUtil.java b/backend/src/main/java/de/grimsi/gameyfin/util/FilenameUtil.java index 2c0b8b9..e8ec6a1 100644 --- a/backend/src/main/java/de/grimsi/gameyfin/util/FilenameUtil.java +++ b/backend/src/main/java/de/grimsi/gameyfin/util/FilenameUtil.java @@ -4,6 +4,7 @@ import org.apache.commons.io.FilenameUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -21,7 +22,7 @@ public class FilenameUtil { // If the path points to a folder, return the folder name // Folders like "Counter Strike 1.6" would otherwise be returned as "Counter Strike 1" - if(p.toFile().isDirectory()) return FilenameUtils.getName(p.toString()); + if(Files.isDirectory(p)) return FilenameUtils.getName(p.toString()); return FilenameUtils.getBaseName(p.toString()); } diff --git a/backend/src/main/resources/application.yml b/backend/src/main/resources/application.yml index bf05140..708facf 100644 --- a/backend/src/main/resources/application.yml +++ b/backend/src/main/resources/application.yml @@ -1,3 +1,5 @@ # General logging.level: - root: info \ No newline at end of file + root: info + # Hides an error log on the first aborted download + org.apache.catalina.core.ContainerBase: off diff --git a/backend/src/main/resources/config/gameyfin.yml b/backend/src/main/resources/config/gameyfin.yml index f70079e..1d7eef8 100644 --- a/backend/src/main/resources/config/gameyfin.yml +++ b/backend/src/main/resources/config/gameyfin.yml @@ -1,4 +1,6 @@ gameyfin: + db: "" + cache: "" file-extensions: iso, zip, rar, 7z, exe igdb: api: @@ -7,4 +9,4 @@ gameyfin: max-concurrent-requests: 2 max-requests-per-second: 4 config: - preferred-platforms: 6 \ No newline at end of file + preferred-platforms: 6 diff --git a/frontend/pom.xml b/frontend/pom.xml index 6203568..01da9ac 100644 --- a/frontend/pom.xml +++ b/frontend/pom.xml @@ -5,7 +5,7 @@ gameyfin de.grimsi - 1.1.4 + 1.1.4-RC1 4.0.0 diff --git a/pom.xml b/pom.xml index 5d265dc..1959753 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ de.grimsi gameyfin - 1.1.4 + 1.1.4-RC1 gameyfin gameyfin