diff --git a/src/main/java/de/grimsi/gameyfin/entities/Company.java b/src/main/java/de/grimsi/gameyfin/entities/Company.java index a0d88f0..0c26bbc 100644 --- a/src/main/java/de/grimsi/gameyfin/entities/Company.java +++ b/src/main/java/de/grimsi/gameyfin/entities/Company.java @@ -24,7 +24,7 @@ public class Company { @Column(nullable = false) private String name; - private Long logoId; + private String logoId; @Override public boolean equals(Object o) { diff --git a/src/main/java/de/grimsi/gameyfin/entities/DetectedGame.java b/src/main/java/de/grimsi/gameyfin/entities/DetectedGame.java index ea4e853..17f9d18 100644 --- a/src/main/java/de/grimsi/gameyfin/entities/DetectedGame.java +++ b/src/main/java/de/grimsi/gameyfin/entities/DetectedGame.java @@ -49,13 +49,13 @@ public class DetectedGame { private int maxPlayers; @Column(nullable = false) - private Long coverId; + private String coverId; @ElementCollection - private List screenshotIds; + private List screenshotIds; @ElementCollection - private List videoIds; + private List videoIds; @ManyToMany(cascade = CascadeType.MERGE) @ToString.Exclude diff --git a/src/main/java/de/grimsi/gameyfin/igdb/IgdbApiProperties.java b/src/main/java/de/grimsi/gameyfin/igdb/IgdbApiProperties.java index b2cd8fc..ed497dd 100644 --- a/src/main/java/de/grimsi/gameyfin/igdb/IgdbApiProperties.java +++ b/src/main/java/de/grimsi/gameyfin/igdb/IgdbApiProperties.java @@ -3,11 +3,13 @@ package de.grimsi.gameyfin.igdb; import java.util.List; public class IgdbApiProperties { - public static final String IGDB_ENPOINT_GAMES_PROTOBUF = "games.pb"; + public static final String ENPOINT_GAMES_PROTOBUF = "games.pb"; private static final List GAME_QUERY_FIELDS = List.of( - "slug", "name", "summary", "first_release_date", "rating", "aggregated_rating", "total_rating", "category", "multiplayer_modes", "cover", "screenshots", "videos", - "involved_companies.company.slug", "involved_companies.company.name", "involved_companies.company.logo.id", + "slug", "name", "summary", "first_release_date", "rating", "aggregated_rating", "total_rating", "category", + "multiplayer_modes.lancoop", "multiplayer_modes.onlinecoop", "multiplayer_modes.offlinecoop", "multiplayer_modes.onlinemax", + "cover.image_id", "screenshots.image_id", "videos.video_id", + "involved_companies.company.slug", "involved_companies.company.name", "involved_companies.company.logo.image_id", "genres.slug", "genres.name", "keywords.slug", "keywords.name", "themes.slug", "themes.name", @@ -16,4 +18,10 @@ public class IgdbApiProperties { public static final String GAME_QUERY_FIELDS_STRING = String.join(",", GAME_QUERY_FIELDS); + public static final String IMAGES_BASE_URL = "https://images.igdb.com/igdb/image/upload/"; + + public static final String COVER_IMAGE_SIZE = "cover_big"; + public static final String SCREENSHOT_IMAGE_SIZE = "screenshot_med"; + public static final String LOGO_IMAGE_SIZE = "logo_med"; + } diff --git a/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java b/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java index e3fa551..d71755d 100644 --- a/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java +++ b/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java @@ -67,7 +67,7 @@ public class IgdbWrapper { public Optional getGameById(Long id) { Igdb.GameResult gameResult = queryIgdbApi( - IgdbApiProperties.IGDB_ENPOINT_GAMES_PROTOBUF, + IgdbApiProperties.ENPOINT_GAMES_PROTOBUF, "fields %s; where id = %d; limit 1;".formatted(IgdbApiProperties.GAME_QUERY_FIELDS_STRING, id), Igdb.GameResult.class ); @@ -79,7 +79,7 @@ public class IgdbWrapper { public Optional getGameBySlug(String slug) { Igdb.GameResult gameResult = queryIgdbApi( - IgdbApiProperties.IGDB_ENPOINT_GAMES_PROTOBUF, + IgdbApiProperties.ENPOINT_GAMES_PROTOBUF, "fields %s; where slug = \"%s\"; limit 1;".formatted(IgdbApiProperties.GAME_QUERY_FIELDS_STRING, slug), Igdb.GameResult.class ); @@ -91,7 +91,7 @@ public class IgdbWrapper { public Optional searchForGameByTitle(String searchTerm) { Igdb.GameResult gameResult = queryIgdbApi( - IgdbApiProperties.IGDB_ENPOINT_GAMES_PROTOBUF, + IgdbApiProperties.ENPOINT_GAMES_PROTOBUF, "search \"%s\"; fields %s; where platforms = (%s);" .formatted(searchTerm, IgdbApiProperties.GAME_QUERY_FIELDS_STRING, preferredPlatforms), Igdb.GameResult.class diff --git a/src/main/java/de/grimsi/gameyfin/mapper/CompanyMapper.java b/src/main/java/de/grimsi/gameyfin/mapper/CompanyMapper.java index 29a1ce4..8f0eb00 100644 --- a/src/main/java/de/grimsi/gameyfin/mapper/CompanyMapper.java +++ b/src/main/java/de/grimsi/gameyfin/mapper/CompanyMapper.java @@ -11,7 +11,7 @@ public class CompanyMapper { return Company.builder() .slug(c.getCompany().getSlug()) .name(c.getCompany().getName()) - .logoId(c.getCompany().getLogo().getId()) + .logoId(c.getCompany().getLogo().getImageId()) .build(); } diff --git a/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java b/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java index 0bb7bee..c4740f5 100644 --- a/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java +++ b/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java @@ -12,8 +12,8 @@ public class GameMapper { public static DetectedGame toDetectedGame(Igdb.Game g, Path path) { List multiplayerModes = g.getMultiplayerModesList(); - List screenshotIds = g.getScreenshotsList().stream().map(Igdb.Screenshot::getId).toList(); - List videoIds = g.getVideosList().stream().map(Igdb.GameVideo::getId).toList(); + List screenshotIds = g.getScreenshotsList().stream().map(Igdb.Screenshot::getImageId).toList(); + List videoIds = g.getVideosList().stream().map(Igdb.GameVideo::getVideoId).toList(); return DetectedGame.builder() .slug(g.getSlug()) @@ -28,7 +28,7 @@ public class GameMapper { .onlineCoop(hasOnlineCoop(multiplayerModes)) .lanSupport(hasLanSupport(multiplayerModes)) .maxPlayers(getMaxPlayers(multiplayerModes)) - .coverId(g.getCover().getId()) + .coverId(g.getCover().getImageId()) .screenshotIds(screenshotIds) .videoIds(videoIds) .companies(CompanyMapper.toCompanies(g.getInvolvedCompaniesList())) diff --git a/src/main/java/de/grimsi/gameyfin/rest/GameyfinDevController.java b/src/main/java/de/grimsi/gameyfin/rest/GameyfinDevController.java index 62f3f45..e58e5f5 100644 --- a/src/main/java/de/grimsi/gameyfin/rest/GameyfinDevController.java +++ b/src/main/java/de/grimsi/gameyfin/rest/GameyfinDevController.java @@ -11,12 +11,15 @@ import de.grimsi.gameyfin.util.ProtobufUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ResponseStatusException; import java.nio.file.Path; import java.util.List; -import java.util.Objects; +import java.util.Map; @RestController public class GameyfinDevController { @@ -67,15 +70,27 @@ public class GameyfinDevController { filesystemService.scanGameLibrary(); } + @GetMapping(value = "/dev/downloadCovers") + public void downloadCovers() { + filesystemService.downloadGameCovers(); + filesystemService.downloadGameScreenshots(); + filesystemService.downloadCompanyLogos(); + } + + @GetMapping(value = "/dev/unmappedFiles", produces = MediaType.APPLICATION_JSON_VALUE) public List getUnmappedFiles() { return gameService.getAllUnmappedFiles(); } + @GetMapping(value = "/dev/gameMappings", produces = MediaType.APPLICATION_JSON_VALUE) + public Map getGameMappings() { + return gameService.getAllMappings(); + } + @PostMapping(value = "/dev/unmappedFiles/{unmappedGameId}/mapTo/{igdbSlug}", produces = MediaType.APPLICATION_JSON_VALUE) public DetectedGame mapGameManually(@PathVariable Long unmappedGameId, @PathVariable String igdbSlug) { return gameService.mapUnmappedFile(unmappedGameId, igdbSlug); } - } diff --git a/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java b/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java index 9b80813..38ae040 100644 --- a/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java +++ b/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java @@ -1,35 +1,53 @@ package de.grimsi.gameyfin.service; import com.igdb.proto.Igdb; -import de.grimsi.gameyfin.entities.UnmappableFile; +import de.grimsi.gameyfin.entities.Company; import de.grimsi.gameyfin.entities.DetectedGame; +import de.grimsi.gameyfin.entities.UnmappableFile; +import de.grimsi.gameyfin.igdb.IgdbApiProperties; import de.grimsi.gameyfin.igdb.IgdbWrapper; import de.grimsi.gameyfin.mapper.GameMapper; -import de.grimsi.gameyfin.repositories.UnmappableFileRepository; import de.grimsi.gameyfin.repositories.DetectedGameRepository; +import de.grimsi.gameyfin.repositories.UnmappableFileRepository; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FilenameUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.core.io.buffer.DataBufferUtils; +import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; import org.springframework.util.StopWatch; +import org.springframework.util.StringUtils; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.reactive.function.client.WebClientResponseException; +import org.springframework.web.server.ResponseStatusException; +import reactor.core.publisher.Flux; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import java.util.stream.Stream; @Slf4j @Service public class FilesystemService { + @Value("${gameyfin.root}") private String rootFolderPath; + @Value("${gameyfin.cache}") + private String cacheFolderPath; + @Value("${gameyfin.file-extensions}") private List possibleGameFileExtensions; @@ -42,6 +60,8 @@ public class FilesystemService { @Autowired private UnmappableFileRepository unmappableFileRepository; + private WebClient igdbImageClient = WebClient.create(IgdbApiProperties.IMAGES_BASE_URL); + public List getGameFiles() { Path rootFolder = Path.of(rootFolderPath); @@ -56,10 +76,6 @@ public class FilesystemService { } } - public List getGameFileNames() { - return this.getGameFiles().stream().map(this::getFilename).toList(); - } - public void scanGameLibrary() { StopWatch stopWatch = new StopWatch(); @@ -97,12 +113,13 @@ public class FilesystemService { return optionalGame.map(game -> Map.entry(p, game)).or(() -> { unmappableFileRepository.save(new UnmappableFile(p.toString())); newUnmappedFilesCounter.getAndIncrement(); - log.info("Added path '{}' to blacklist", p); + log.info("Added path '{}' to list of unmapped files", p); return Optional.empty(); }); }) .filter(Optional::isPresent) .map(Optional::get) + .peek(e -> log.info("Mapped file '{}' to game '{}' (slug: {})", e.getKey(), e.getValue().getName(), e.getValue().getSlug())) .map(e -> GameMapper.toDetectedGame(e.getValue(), e.getKey())) .toList(); @@ -114,7 +131,108 @@ public class FilesystemService { (int) stopWatch.getTotalTimeSeconds(), newDetectedGames.size(), deletedGames.size() + deletedUnmappableFiles.size(), newUnmappedFilesCounter.get(), detectedGameRepository.count()); } + public void downloadGameCovers() { + StopWatch stopWatch = new StopWatch(); + + log.info("Starting game cover download..."); + stopWatch.start(); + + MultiValueMap multiValueMap = new LinkedMultiValueMap<>( + detectedGameRepository.findAll().stream() + .collect(Collectors.toMap(DetectedGame::getTitle, g -> Collections.singletonList(g.getCoverId())))); + + int downloadCount = downloadImagesIntoCache(multiValueMap, IgdbApiProperties.COVER_IMAGE_SIZE, "cover", "game"); + + stopWatch.stop(); + + log.info("Downloaded {} covers in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds()); + } + + public void downloadGameScreenshots() { + StopWatch stopWatch = new StopWatch(); + + log.info("Starting game screenshot download..."); + stopWatch.start(); + + MultiValueMap gamesToImageIds = new LinkedMultiValueMap<>( + detectedGameRepository.findAll().stream() + .collect(Collectors.toMap(DetectedGame::getTitle, DetectedGame::getScreenshotIds))); + + int downloadCount = downloadImagesIntoCache(gamesToImageIds, IgdbApiProperties.SCREENSHOT_IMAGE_SIZE, "screenshot", "game"); + + stopWatch.stop(); + + log.info("Downloaded {} screenshots in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds()); + } + + public void downloadCompanyLogos() { + StopWatch stopWatch = new StopWatch(); + + log.info("Starting company logo download..."); + stopWatch.start(); + + Map> test = detectedGameRepository.findAll().stream() + .flatMap(g -> g.getCompanies().stream()) + .collect(Collectors.toMap(Company::getName, c -> Collections.singletonList(c.getLogoId()), (c1, c2) -> c1)); + + MultiValueMap gamesToImageIds = new LinkedMultiValueMap<>(test); + + int downloadCount = downloadImagesIntoCache(gamesToImageIds, IgdbApiProperties.LOGO_IMAGE_SIZE, "logo", "company"); + + stopWatch.stop(); + + log.info("Downloaded {} company logos in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds()); + } + private String getFilename(Path p) { return FilenameUtils.getBaseName(p.toString()); } + + private int downloadImagesIntoCache(MultiValueMap entityToImageIds, String imageSize, String imageType, String entityType) { + AtomicInteger downloadCounter = new AtomicInteger(); + Path cacheFolder = Path.of(cacheFolderPath); + + try { + Files.createDirectories(cacheFolder); + } catch (IOException e) { + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Could not create cache folder."); + } + + entityToImageIds.entrySet().parallelStream().forEach(entry -> + entry.getValue().forEach(imageId -> { + + if(!StringUtils.hasText(imageId)) return; + + String imgFileName = "%s.jpg".formatted(imageId); + String imgUrl = "t_%s/%s".formatted(imageSize, imgFileName); + + if (Files.exists(Path.of(cacheFolderPath, imgFileName))) { + log.debug("{} for {} '{}' already downloaded ({}), skipping.", + imageType.substring(0, 1).toUpperCase() + imageType.substring(1).toLowerCase(), + entityType, + entry.getKey(), + imgFileName); + return; + } + + Flux dataBuffer = igdbImageClient.get() + .uri(imgUrl) + .retrieve() + .bodyToFlux(DataBuffer.class); + + try { + DataBufferUtils.write(dataBuffer, cacheFolder.resolve(imgFileName), StandardOpenOption.CREATE) + .share().block(); + } catch (WebClientResponseException e) { + if (e.getStatusCode().is4xxClientError()) { + log.error("Could not download {} for {} '{}' from {}: {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl, e.getStatusCode()); + } + } + + downloadCounter.getAndIncrement(); + log.info("Downloaded {} for {} '{}' from {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl); + })); + + return downloadCounter.get(); + } } diff --git a/src/main/java/de/grimsi/gameyfin/service/GameService.java b/src/main/java/de/grimsi/gameyfin/service/GameService.java index d9679ff..bea73b0 100644 --- a/src/main/java/de/grimsi/gameyfin/service/GameService.java +++ b/src/main/java/de/grimsi/gameyfin/service/GameService.java @@ -14,6 +14,8 @@ import org.springframework.web.server.ResponseStatusException; import java.nio.file.Path; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; @Service public class GameService { @@ -35,6 +37,10 @@ public class GameService { return unmappableFileRepository.findAll(); } + public Map getAllMappings() { + return detectedGameRepository.findAll().stream().collect(Collectors.toMap(DetectedGame::getPath, DetectedGame::getTitle)); + } + public DetectedGame mapUnmappedFile(Long unmappedGameId, String igdbSlug) { UnmappableFile unmappableFile = unmappableFileRepository.findById(unmappedGameId) diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 2572ec2..2026a71 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -1,7 +1,7 @@ gameyfin: - root: D:\Games - cache: C:\Projects\privat\gameyfin-library\.gameyfin - db: C:\Projects\privat\gameyfin-library\.gameyfin + root: \\NAS-Simon\Öffentlich\Spiele + cache: ${gameyfin.root}\.gameyfin\cache + db: ${gameyfin.root}\.gameyfin\db # Currently unused igdb: api: client-id: 23l3l5qshx4dwjuao6yb8jyf1qrd08 @@ -10,6 +10,6 @@ gameyfin: logging: level: de.grimsi: debug - org.springframework.web.reactive.function.client.ExchangeFunctions: debug + # org.springframework.web.reactive.function.client.ExchangeFunctions: debug spring.mvc.log-request-details: true \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index df8a107..35a3af0 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -9,9 +9,13 @@ spring: datasource.username: gfadmin datasource.password: gameyfin datasource.driverClassName: org.h2.Driver - jpa.database-platform: org.hibernate.dialect.H2Dialect - jpa.hibernate.ddl-auto: update - jpa.open-in-view: true + jpa: + database-platform: org.hibernate.dialect.H2Dialect + hibernate.ddl-auto: update + open-in-view: true + properties: + hibernate: + event.merge.entity_copy_observer: allow gameyfin: root: ""