Brendon/gameyfin
1
0
Fork 0
mirror of https://github.com/BrenBroZAYT/gameyfin.git synced 2026-06-13 16:40:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implemented screenshot and logo download

Browse Source 
Fixed a few bugs
This commit is contained in:
grimsi
2022-07-17 12:46:22 +02:00
parent 9803a9dc0a
commit f7f989e3c9
11 changed files with 183 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/de/grimsi/gameyfin/entities/Company.java
+1 -1
View File
@@ -24,7 +24,7 @@ public class Company {
@Column(nullable = false)
private String name;
private Long logoId;
private String logoId;
@Override
public boolean equals(Object o) {
src/main/java/de/grimsi/gameyfin/entities/DetectedGame.java
+3 -3
View File
@@ -49,13 +49,13 @@ public class DetectedGame {
private int maxPlayers;
@Column(nullable = false)
private Long coverId;
private String coverId;
@ElementCollection
private List<Long> screenshotIds;
private List<String> screenshotIds;
@ElementCollection
private List<Long> videoIds;
private List<String> videoIds;
@ManyToMany(cascade = CascadeType.MERGE)
@ToString.Exclude
src/main/java/de/grimsi/gameyfin/igdb/IgdbApiProperties.java
+11 -3
View File
@@ -3,11 +3,13 @@ package de.grimsi.gameyfin.igdb;
import java.util.List;
public class IgdbApiProperties {
public static final String IGDB_ENPOINT_GAMES_PROTOBUF = "games.pb";
public static final String ENPOINT_GAMES_PROTOBUF = "games.pb";
private static final List<String> GAME_QUERY_FIELDS = List.of(
"slug", "name", "summary", "first_release_date", "rating", "aggregated_rating", "total_rating", "category", "multiplayer_modes", "cover", "screenshots", "videos",
"involved_companies.company.slug", "involved_companies.company.name", "involved_companies.company.logo.id",
"slug", "name", "summary", "first_release_date", "rating", "aggregated_rating", "total_rating", "category",
"multiplayer_modes.lancoop", "multiplayer_modes.onlinecoop", "multiplayer_modes.offlinecoop", "multiplayer_modes.onlinemax",
"cover.image_id", "screenshots.image_id", "videos.video_id",
"involved_companies.company.slug", "involved_companies.company.name", "involved_companies.company.logo.image_id",
"genres.slug", "genres.name",
"keywords.slug", "keywords.name",
"themes.slug", "themes.name",
@@ -16,4 +18,10 @@ public class IgdbApiProperties {
public static final String GAME_QUERY_FIELDS_STRING = String.join(",", GAME_QUERY_FIELDS);
public static final String IMAGES_BASE_URL = "https://images.igdb.com/igdb/image/upload/";
public static final String COVER_IMAGE_SIZE = "cover_big";
public static final String SCREENSHOT_IMAGE_SIZE = "screenshot_med";
public static final String LOGO_IMAGE_SIZE = "logo_med";
}
src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java
+3 -3
View File
@@ -67,7 +67,7 @@ public class IgdbWrapper {
public Optional<Igdb.Game> getGameById(Long id) {
Igdb.GameResult gameResult = queryIgdbApi(
IgdbApiProperties.IGDB_ENPOINT_GAMES_PROTOBUF,
IgdbApiProperties.ENPOINT_GAMES_PROTOBUF,
"fields %s; where id = %d; limit 1;".formatted(IgdbApiProperties.GAME_QUERY_FIELDS_STRING, id),
Igdb.GameResult.class
);
@@ -79,7 +79,7 @@ public class IgdbWrapper {
public Optional<Igdb.Game> getGameBySlug(String slug) {
Igdb.GameResult gameResult = queryIgdbApi(
IgdbApiProperties.IGDB_ENPOINT_GAMES_PROTOBUF,
IgdbApiProperties.ENPOINT_GAMES_PROTOBUF,
"fields %s; where slug = \"%s\"; limit 1;".formatted(IgdbApiProperties.GAME_QUERY_FIELDS_STRING, slug),
Igdb.GameResult.class
);
@@ -91,7 +91,7 @@ public class IgdbWrapper {
public Optional<Igdb.Game> searchForGameByTitle(String searchTerm) {
Igdb.GameResult gameResult = queryIgdbApi(
IgdbApiProperties.IGDB_ENPOINT_GAMES_PROTOBUF,
IgdbApiProperties.ENPOINT_GAMES_PROTOBUF,
"search \"%s\"; fields %s; where platforms = (%s);"
.formatted(searchTerm, IgdbApiProperties.GAME_QUERY_FIELDS_STRING, preferredPlatforms),
Igdb.GameResult.class
src/main/java/de/grimsi/gameyfin/mapper/CompanyMapper.java
+1 -1
View File
@@ -11,7 +11,7 @@ public class CompanyMapper {
return Company.builder()
.slug(c.getCompany().getSlug())
.name(c.getCompany().getName())
.logoId(c.getCompany().getLogo().getId())
.logoId(c.getCompany().getLogo().getImageId())
.build();
}
src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java
+3 -3
View File
@@ -12,8 +12,8 @@ public class GameMapper {
public static DetectedGame toDetectedGame(Igdb.Game g, Path path) {
List<Igdb.MultiplayerMode> multiplayerModes = g.getMultiplayerModesList();
List<Long> screenshotIds = g.getScreenshotsList().stream().map(Igdb.Screenshot::getId).toList();
List<Long> videoIds = g.getVideosList().stream().map(Igdb.GameVideo::getId).toList();
List<String> screenshotIds = g.getScreenshotsList().stream().map(Igdb.Screenshot::getImageId).toList();
List<String> videoIds = g.getVideosList().stream().map(Igdb.GameVideo::getVideoId).toList();
return DetectedGame.builder()
.slug(g.getSlug())
@@ -28,7 +28,7 @@ public class GameMapper {
.onlineCoop(hasOnlineCoop(multiplayerModes))
.lanSupport(hasLanSupport(multiplayerModes))
.maxPlayers(getMaxPlayers(multiplayerModes))
.coverId(g.getCover().getId())
.coverId(g.getCover().getImageId())
.screenshotIds(screenshotIds)
.videoIds(videoIds)
.companies(CompanyMapper.toCompanies(g.getInvolvedCompaniesList()))
src/main/java/de/grimsi/gameyfin/rest/GameyfinDevController.java
+18 -3
View File
@@ -11,12 +11,15 @@ import de.grimsi.gameyfin.util.ProtobufUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.server.ResponseStatusException;
import java.nio.file.Path;
import java.util.List;
import java.util.Objects;
import java.util.Map;
@RestController
public class GameyfinDevController {
@@ -67,15 +70,27 @@ public class GameyfinDevController {
filesystemService.scanGameLibrary();
}
@GetMapping(value = "/dev/downloadCovers")
public void downloadCovers() {
filesystemService.downloadGameCovers();
filesystemService.downloadGameScreenshots();
filesystemService.downloadCompanyLogos();
}
@GetMapping(value = "/dev/unmappedFiles", produces = MediaType.APPLICATION_JSON_VALUE)
public List<UnmappableFile> getUnmappedFiles() {
return gameService.getAllUnmappedFiles();
}
@GetMapping(value = "/dev/gameMappings", produces = MediaType.APPLICATION_JSON_VALUE)
public Map<String, String> getGameMappings() {
return gameService.getAllMappings();
}
@PostMapping(value = "/dev/unmappedFiles/{unmappedGameId}/mapTo/{igdbSlug}", produces = MediaType.APPLICATION_JSON_VALUE)
public DetectedGame mapGameManually(@PathVariable Long unmappedGameId, @PathVariable String igdbSlug) {
return gameService.mapUnmappedFile(unmappedGameId, igdbSlug);
}
}
src/main/java/de/grimsi/gameyfin/service/FilesystemService.java
+126 -8
View File
@@ -1,35 +1,53 @@
package de.grimsi.gameyfin.service;
import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.entities.UnmappableFile;
import de.grimsi.gameyfin.entities.Company;
import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.entities.UnmappableFile;
import de.grimsi.gameyfin.igdb.IgdbApiProperties;
import de.grimsi.gameyfin.igdb.IgdbWrapper;
import de.grimsi.gameyfin.mapper.GameMapper;
import de.grimsi.gameyfin.repositories.UnmappableFileRepository;
import de.grimsi.gameyfin.repositories.DetectedGameRepository;
import de.grimsi.gameyfin.repositories.UnmappableFileRepository;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FilenameUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.util.StopWatch;
import org.springframework.util.StringUtils;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientResponseException;
import org.springframework.web.server.ResponseStatusException;
import reactor.core.publisher.Flux;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j
@Service
public class FilesystemService {
@Value("${gameyfin.root}")
private String rootFolderPath;
@Value("${gameyfin.cache}")
private String cacheFolderPath;
@Value("${gameyfin.file-extensions}")
private List<String> possibleGameFileExtensions;
@@ -42,6 +60,8 @@ public class FilesystemService {
@Autowired
private UnmappableFileRepository unmappableFileRepository;
private WebClient igdbImageClient = WebClient.create(IgdbApiProperties.IMAGES_BASE_URL);
public List<Path> getGameFiles() {
Path rootFolder = Path.of(rootFolderPath);
@@ -56,10 +76,6 @@ public class FilesystemService {
}
}
public List<String> getGameFileNames() {
return this.getGameFiles().stream().map(this::getFilename).toList();
}
public void scanGameLibrary() {
StopWatch stopWatch = new StopWatch();
@@ -97,12 +113,13 @@ public class FilesystemService {
return optionalGame.map(game -> Map.entry(p, game)).or(() -> {
unmappableFileRepository.save(new UnmappableFile(p.toString()));
newUnmappedFilesCounter.getAndIncrement();
log.info("Added path '{}' to blacklist", p);
log.info("Added path '{}' to list of unmapped files", p);
return Optional.empty();
});
})
.filter(Optional::isPresent)
.map(Optional::get)
.peek(e -> log.info("Mapped file '{}' to game '{}' (slug: {})", e.getKey(), e.getValue().getName(), e.getValue().getSlug()))
.map(e -> GameMapper.toDetectedGame(e.getValue(), e.getKey()))
.toList();
@@ -114,7 +131,108 @@ public class FilesystemService {
(int) stopWatch.getTotalTimeSeconds(), newDetectedGames.size(), deletedGames.size() + deletedUnmappableFiles.size(), newUnmappedFilesCounter.get(), detectedGameRepository.count());
}
public void downloadGameCovers() {
StopWatch stopWatch = new StopWatch();
log.info("Starting game cover download...");
stopWatch.start();
MultiValueMap<String, String> multiValueMap = new LinkedMultiValueMap<>(
detectedGameRepository.findAll().stream()
.collect(Collectors.toMap(DetectedGame::getTitle, g -> Collections.singletonList(g.getCoverId()))));
int downloadCount = downloadImagesIntoCache(multiValueMap, IgdbApiProperties.COVER_IMAGE_SIZE, "cover", "game");
stopWatch.stop();
log.info("Downloaded {} covers in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds());
}
public void downloadGameScreenshots() {
StopWatch stopWatch = new StopWatch();
log.info("Starting game screenshot download...");
stopWatch.start();
MultiValueMap<String, String> gamesToImageIds = new LinkedMultiValueMap<>(
detectedGameRepository.findAll().stream()
.collect(Collectors.toMap(DetectedGame::getTitle, DetectedGame::getScreenshotIds)));
int downloadCount = downloadImagesIntoCache(gamesToImageIds, IgdbApiProperties.SCREENSHOT_IMAGE_SIZE, "screenshot", "game");
stopWatch.stop();
log.info("Downloaded {} screenshots in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds());
}
public void downloadCompanyLogos() {
StopWatch stopWatch = new StopWatch();
log.info("Starting company logo download...");
stopWatch.start();
Map<String, List<String>> test = detectedGameRepository.findAll().stream()
.flatMap(g -> g.getCompanies().stream())
.collect(Collectors.toMap(Company::getName, c -> Collections.singletonList(c.getLogoId()), (c1, c2) -> c1));
MultiValueMap<String, String> gamesToImageIds = new LinkedMultiValueMap<>(test);
int downloadCount = downloadImagesIntoCache(gamesToImageIds, IgdbApiProperties.LOGO_IMAGE_SIZE, "logo", "company");
stopWatch.stop();
log.info("Downloaded {} company logos in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds());
}
private String getFilename(Path p) {
return FilenameUtils.getBaseName(p.toString());
}
private int downloadImagesIntoCache(MultiValueMap<String, String> entityToImageIds, String imageSize, String imageType, String entityType) {
AtomicInteger downloadCounter = new AtomicInteger();
Path cacheFolder = Path.of(cacheFolderPath);
try {
Files.createDirectories(cacheFolder);
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Could not create cache folder.");
}
entityToImageIds.entrySet().parallelStream().forEach(entry ->
entry.getValue().forEach(imageId -> {
if(!StringUtils.hasText(imageId)) return;
String imgFileName = "%s.jpg".formatted(imageId);
String imgUrl = "t_%s/%s".formatted(imageSize, imgFileName);
if (Files.exists(Path.of(cacheFolderPath, imgFileName))) {
log.debug("{} for {} '{}' already downloaded ({}), skipping.",
imageType.substring(0, 1).toUpperCase() + imageType.substring(1).toLowerCase(),
entityType,
entry.getKey(),
imgFileName);
return;
}
Flux<DataBuffer> dataBuffer = igdbImageClient.get()
.uri(imgUrl)
.retrieve()
.bodyToFlux(DataBuffer.class);
try {
DataBufferUtils.write(dataBuffer, cacheFolder.resolve(imgFileName), StandardOpenOption.CREATE)
.share().block();
} catch (WebClientResponseException e) {
if (e.getStatusCode().is4xxClientError()) {
log.error("Could not download {} for {} '{}' from {}: {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl, e.getStatusCode());
}
}
downloadCounter.getAndIncrement();
log.info("Downloaded {} for {} '{}' from {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl);
}));
return downloadCounter.get();
}
}
src/main/java/de/grimsi/gameyfin/service/GameService.java
+6
View File
@@ -14,6 +14,8 @@ import org.springframework.web.server.ResponseStatusException;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Service
public class GameService {
@@ -35,6 +37,10 @@ public class GameService {
return unmappableFileRepository.findAll();
}
public Map<String, String> getAllMappings() {
return detectedGameRepository.findAll().stream().collect(Collectors.toMap(DetectedGame::getPath, DetectedGame::getTitle));
}
public DetectedGame mapUnmappedFile(Long unmappedGameId, String igdbSlug) {
UnmappableFile unmappableFile = unmappableFileRepository.findById(unmappedGameId)
src/main/resources/application-dev.yml
+4 -4
View File
@@ -1,7 +1,7 @@
gameyfin:
root: D:\Games
cache: C:\Projects\privat\gameyfin-library\.gameyfin
db: C:\Projects\privat\gameyfin-library\.gameyfin
root: \\NAS-Simon\Öffentlich\Spiele
cache: ${gameyfin.root}\.gameyfin\cache
db: ${gameyfin.root}\.gameyfin\db # Currently unused
igdb:
api:
client-id: 23l3l5qshx4dwjuao6yb8jyf1qrd08
@@ -10,6 +10,6 @@ gameyfin:
logging:
level:
de.grimsi: debug
org.springframework.web.reactive.function.client.ExchangeFunctions: debug
# org.springframework.web.reactive.function.client.ExchangeFunctions: debug
spring.mvc.log-request-details: true
src/main/resources/application.yml
+7 -3
View File
@@ -9,9 +9,13 @@ spring:
datasource.username: gfadmin
datasource.password: gameyfin
datasource.driverClassName: org.h2.Driver
jpa.database-platform: org.hibernate.dialect.H2Dialect
jpa.hibernate.ddl-auto: update
jpa.open-in-view: true
jpa:
database-platform: org.hibernate.dialect.H2Dialect
hibernate.ddl-auto: update
open-in-view: true
properties:
hibernate:
event.merge.entity_copy_observer: allow
gameyfin:
root: ""