Brendon/gameyfin
1
0
Fork 0
mirror of https://github.com/BrenBroZAYT/gameyfin.git synced 2026-06-15 16:20:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactored file-system code

Browse Source 
Fixed logging when aborting download of single files
This commit is contained in:
grimsi
2022-08-18 20:26:09 +02:00
parent c097c6fdc0
commit 1fbfeb1c7e
18 changed files with 307 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/main/java/de/grimsi/gameyfin/config/FilesystemConfig.java
+1 -1
View File
@@ -39,7 +39,7 @@ public class FilesystemConfig {
props.setProperty("gameyfin.db", "%s/.gameyfin/db".formatted(firstLibraryPath));
}
if(StringUtils.hasText(cachePath)) {
if(!StringUtils.hasText(cachePath)) {
props.setProperty("gameyfin.cache", "%s/.gameyfin/cache".formatted(firstLibraryPath));
}
backend/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java
+6 -6
View File
@@ -7,6 +7,7 @@ import de.grimsi.gameyfin.igdb.dto.TwitchOAuthTokenDto;
import de.grimsi.gameyfin.mapper.GameMapper;
import io.github.resilience4j.reactor.bulkhead.operator.BulkheadOperator;
import io.github.resilience4j.reactor.ratelimiter.operator.RateLimiterOperator;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@@ -24,6 +25,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Slf4j
@RequiredArgsConstructor
@Service
public class IgdbWrapper {
@Value("${gameyfin.igdb.api.client-id}")
@@ -35,11 +37,9 @@ public class IgdbWrapper {
@Value("${gameyfin.igdb.config.preferred-platforms:6}")
private String preferredPlatforms;
@Autowired
private WebClient.Builder webclientBuilder;
@Autowired
private WebClientConfig webClientConfig;
private final WebClient.Builder webclientBuilder;
private final WebClientConfig webClientConfig;
private final GameMapper gameMapper;
private WebClient twitchApiClient;
@@ -106,7 +106,7 @@ public class IgdbWrapper {
if(gameResult == null) return Collections.emptyList();
return gameResult.getGamesList().stream().map(GameMapper::toAutocompleteSuggestionDto).toList();
return gameResult.getGamesList().stream().map(gameMapper::toAutocompleteSuggestionDto).toList();
}
public Optional<Igdb.Game> searchForGameByTitle(String searchTerm) {
backend/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java
+19 -19
View File
@@ -4,6 +4,7 @@ import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.dto.AutocompleteSuggestionDto;
import de.grimsi.gameyfin.dto.GameOverviewDto;
import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.service.FilesystemService;
import de.grimsi.gameyfin.service.LibraryService;
import de.grimsi.gameyfin.util.ProtobufUtil;
import lombok.RequiredArgsConstructor;
@@ -24,9 +25,13 @@ import java.util.List;
import java.util.stream.Stream;
@Slf4j
@RequiredArgsConstructor
@Component
public class GameMapper {
public static DetectedGame toDetectedGame(Igdb.Game g, Path path) {
private final FilesystemService filesystemService;
public DetectedGame toDetectedGame(Igdb.Game g, Path path) {
List<Igdb.MultiplayerMode> multiplayerModes = g.getMultiplayerModesList();
List<String> screenshotIds = g.getScreenshotsList().stream().map(Igdb.Screenshot::getImageId).toList();
List<String> videoIds = g.getVideosList().stream().map(Igdb.GameVideo::getVideoId).toList();
@@ -58,7 +63,7 @@ public class GameMapper {
.build();
}
public static GameOverviewDto toGameOverviewDto(DetectedGame game) {
public GameOverviewDto toGameOverviewDto(DetectedGame game) {
return GameOverviewDto.builder()
.slug(game.getSlug())
.title(game.getTitle())
@@ -66,7 +71,7 @@ public class GameMapper {
.build();
}
public static AutocompleteSuggestionDto toAutocompleteSuggestionDto(Igdb.Game game) {
public AutocompleteSuggestionDto toAutocompleteSuggestionDto(Igdb.Game game) {
return AutocompleteSuggestionDto.builder()
.slug(game.getSlug())
.title(game.getName())
@@ -74,7 +79,7 @@ public class GameMapper {
.build();
}
private static String getCoverId(Igdb.Game g) {
private String getCoverId(Igdb.Game g) {
String coverId = g.getCover().getImageId();
if(StringUtils.hasText(coverId)) return coverId;
@@ -82,23 +87,23 @@ public class GameMapper {
return "nocover";
}
private static boolean hasOfflineCoop(List<Igdb.MultiplayerMode> modes) {
private boolean hasOfflineCoop(List<Igdb.MultiplayerMode> modes) {
return modes.stream().anyMatch(Igdb.MultiplayerMode::getOfflinecoop);
}
private static boolean hasLanSupport(List<Igdb.MultiplayerMode> modes) {
private boolean hasLanSupport(List<Igdb.MultiplayerMode> modes) {
return modes.stream().anyMatch(Igdb.MultiplayerMode::getLancoop);
}
private static boolean hasOnlineCoop(List<Igdb.MultiplayerMode> modes) {
private boolean hasOnlineCoop(List<Igdb.MultiplayerMode> modes) {
return modes.stream().anyMatch(Igdb.MultiplayerMode::getOnlinecoop);
}
private static int getMaxPlayers(List<Igdb.MultiplayerMode> modes) {
private int getMaxPlayers(List<Igdb.MultiplayerMode> modes) {
return modes.stream().mapToInt(Igdb.MultiplayerMode::getOnlinecoopmax).max().orElse(0);
}
private static long calculateDiskSize(Igdb.Game g, Path path) {
private long calculateDiskSize(Igdb.Game g, Path path) {
StopWatch stopWatch = new StopWatch();
log.info("Calculating disk size for game '{}'...", g.getName());
@@ -106,16 +111,11 @@ public class GameMapper {
long fileSize;
if(Files.isDirectory(path)) {
// Some benchmarks I did have shown that trying to parallelize this process makes it slower instead of faster
fileSize = FileUtils.sizeOfDirectory(path.toFile());
} else {
try{
fileSize = Files.size(path);
} catch (IOException e) {
log.error("Error while calculating size of file '{}'.", path);
fileSize = -1L;
}
try {
fileSize = filesystemService.getSizeOnDisk(path);
} catch(IOException e) {
log.error("Error while calculating disk size for game '{}'", g.getName());
fileSize = -1L;
}
stopWatch.stop();
backend/src/main/java/de/grimsi/gameyfin/rest/GamesController.java
+1 -1
View File
@@ -54,7 +54,7 @@ public class GamesController {
return ResponseEntity
.ok()
.header("Content-Disposition", "attachment; filename=\"%s\"".formatted(downloadFileName))
.body(out -> downloadService.downloadGameFiles(game, out));
.body(out -> downloadService.sendGamefilesToClient(game, out));
}
}
backend/src/main/java/de/grimsi/gameyfin/rest/ImageController.java
+1 -1
View File
@@ -27,6 +27,6 @@ public class ImageController {
public ResponseEntity<Resource> getCoverImageForGame(@PathVariable String imageId) {
return ResponseEntity.ok()
.cacheControl(CacheControl.maxAge(365, TimeUnit.DAYS).cachePublic())
.body(downloadService.downloadImage(imageId));
.body(downloadService.sendImageToClient(imageId));
}
}
backend/src/main/java/de/grimsi/gameyfin/rest/LibraryController.java
+5 -4
View File
@@ -1,6 +1,7 @@
package de.grimsi.gameyfin.rest;
import de.grimsi.gameyfin.service.DownloadService;
import de.grimsi.gameyfin.service.ImageService;
import de.grimsi.gameyfin.service.LibraryService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -25,7 +26,7 @@ import java.util.List;
public class LibraryController {
private final LibraryService libraryService;
private final DownloadService downloadService;
private final ImageService imageService;
@GetMapping(value = "/scan", produces = MediaType.APPLICATION_JSON_VALUE)
public void scanLibrary(@RequestParam(value = "download_images", defaultValue = "true") boolean downloadImages) {
@@ -36,9 +37,9 @@ public class LibraryController {
@GetMapping(value = "/download-images")
public void downloadImages() {
downloadService.downloadGameCoversFromIgdb();
downloadService.downloadGameScreenshotsFromIgdb();
downloadService.downloadCompanyLogosFromIgdb();
imageService.downloadGameCoversFromIgdb();
imageService.downloadGameScreenshotsFromIgdb();
imageService.downloadCompanyLogosFromIgdb();
log.info("Downloading images completed.");
}
backend/src/main/java/de/grimsi/gameyfin/rest/LibraryManagementController.java
+6 -6
View File
@@ -8,6 +8,7 @@ import de.grimsi.gameyfin.entities.UnmappableFile;
import de.grimsi.gameyfin.repositories.DetectedGameRepository;
import de.grimsi.gameyfin.service.DownloadService;
import de.grimsi.gameyfin.service.GameService;
import de.grimsi.gameyfin.service.ImageService;
import de.grimsi.gameyfin.service.LibraryService;
import lombok.RequiredArgsConstructor;
import org.springframework.http.MediaType;
@@ -23,8 +24,7 @@ import java.util.List;
public class LibraryManagementController {
private final GameService gameService;
private final DownloadService downloadService;
private final ImageService imageService;
private final LibraryService libraryService;
@DeleteMapping(value = "/delete-game/{slug}", produces = MediaType.APPLICATION_JSON_VALUE)
@@ -45,10 +45,10 @@ public class LibraryManagementController {
@PostMapping(value = "/map-path", produces = MediaType.APPLICATION_JSON_VALUE)
public DetectedGame manuallyMapPathToSlug(@RequestBody PathToSlugDto pathToSlugDto) {
DetectedGame game = gameService.mapPathToGame(pathToSlugDto.getPath(), pathToSlugDto.getSlug());
downloadService.downloadGameCoversFromIgdb();
downloadService.downloadGameScreenshotsFromIgdb();
downloadService.downloadCompanyLogosFromIgdb();
imageService.downloadGameCoversFromIgdb();
imageService.downloadGameScreenshotsFromIgdb();
imageService.downloadCompanyLogosFromIgdb();
return game;
}
backend/src/main/java/de/grimsi/gameyfin/service/DownloadService.java
+18 -172
View File
@@ -1,41 +1,19 @@
package de.grimsi.gameyfin.service;
import de.grimsi.gameyfin.entities.Company;
import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.exceptions.DownloadAbortedException;
import de.grimsi.gameyfin.igdb.IgdbApiProperties;
import de.grimsi.gameyfin.repositories.DetectedGameRepository;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.catalina.connector.ClientAbortException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.Resource;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.util.StopWatch;
import org.springframework.util.StringUtils;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientResponseException;
import org.springframework.web.server.ResponseStatusException;
import reactor.core.publisher.Flux;
import javax.annotation.PostConstruct;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.zip.Deflater;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -47,19 +25,7 @@ import static de.grimsi.gameyfin.util.FilenameUtil.getFilenameWithExtension;
@RequiredArgsConstructor
public class DownloadService {
@Value("${gameyfin.cache}")
private String cacheFolderPath;
private final DetectedGameRepository detectedGameRepository;
@Autowired
private WebClient.Builder webclientBuilder;
private WebClient igdbImageClient;
@PostConstruct
public void init() {
igdbImageClient = webclientBuilder.baseUrl(IgdbApiProperties.IMAGES_BASE_URL).build();
}
private final FilesystemService filesystemService;
public String getDownloadFileName(DetectedGame g) {
Path path = Path.of(g.getPath());
@@ -68,17 +34,12 @@ public class DownloadService {
return getFilenameWithExtension(path) + ".zip";
}
public Resource downloadImage(String imageId) {
public Resource sendImageToClient(String imageId) {
String filename = "%s.png".formatted(imageId);
try {
return new ByteArrayResource(Files.readAllBytes(Paths.get("%s/%s".formatted(cacheFolderPath, filename))));
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Could not find image file %s".formatted(filename));
}
return filesystemService.getFileFromCache(filename);
}
public void downloadGameFiles(DetectedGame game, OutputStream outputStream) {
public void sendGamefilesToClient(DetectedGame game, OutputStream outputStream) {
StopWatch stopWatch = new StopWatch();
@@ -88,18 +49,16 @@ public class DownloadService {
Path path = Path.of(game.getPath());
if (path.toFile().isDirectory()) {
try {
downloadFilesAsZip(path, outputStream);
} catch(DownloadAbortedException e) {
stopWatch.stop();
log.info("Download of game {} was aborted by client after {} seconds", game.getTitle(), (int) stopWatch.getTotalTimeSeconds());
return;
try {
if (path.toFile().isDirectory()) {
sendGamefilesAsZipToClient(path, outputStream);
} else {
sendGamefileToClient(path, outputStream);
}
} else {
downloadFile(path, outputStream);
} catch (DownloadAbortedException e) {
stopWatch.stop();
log.info("Download of game {} was aborted by client after {} seconds", game.getTitle(), (int) stopWatch.getTotalTimeSeconds());
return;
}
stopWatch.stop();
@@ -107,70 +66,18 @@ public class DownloadService {
log.info("Downloaded game files of {} in {} seconds.", game.getTitle(), (int) stopWatch.getTotalTimeSeconds());
}
public void downloadGameCoversFromIgdb() {
StopWatch stopWatch = new StopWatch();
log.info("Starting game cover download...");
stopWatch.start();
MultiValueMap<String, String> gameToImageIds = new LinkedMultiValueMap<>(
detectedGameRepository.findAll().stream()
.collect(Collectors.toMap(DetectedGame::getSlug, g -> Collections.singletonList(g.getCoverId()))));
int downloadCount = downloadImagesIntoCache(gameToImageIds, IgdbApiProperties.COVER_IMAGE_SIZE, "cover", "game");
stopWatch.stop();
log.info("Downloaded {} covers in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds());
}
public void downloadGameScreenshotsFromIgdb() {
StopWatch stopWatch = new StopWatch();
log.info("Starting game screenshot download...");
stopWatch.start();
MultiValueMap<String, String> gamesToImageIds = new LinkedMultiValueMap<>(
detectedGameRepository.findAll().stream()
.collect(Collectors.toMap(DetectedGame::getSlug, DetectedGame::getScreenshotIds)));
int downloadCount = downloadImagesIntoCache(gamesToImageIds, IgdbApiProperties.SCREENSHOT_IMAGE_SIZE, "screenshot", "game");
stopWatch.stop();
log.info("Downloaded {} screenshots in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds());
}
public void downloadCompanyLogosFromIgdb() {
StopWatch stopWatch = new StopWatch();
log.info("Starting company logo download...");
stopWatch.start();
Map<String, List<String>> companyToLogoIdMap = detectedGameRepository.findAll().stream()
.flatMap(g -> g.getCompanies().stream())
.collect(Collectors.toMap(Company::getSlug, c -> Collections.singletonList(c.getLogoId()), (c1, c2) -> c1));
MultiValueMap<String, String> companiesToLogoIds = new LinkedMultiValueMap<>(companyToLogoIdMap);
int downloadCount = downloadImagesIntoCache(companiesToLogoIds, IgdbApiProperties.LOGO_IMAGE_SIZE, "logo", "company");
stopWatch.stop();
log.info("Downloaded {} company logos in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds());
}
private void downloadFile(Path path, OutputStream outputStream) {
private void sendGamefileToClient(Path path, OutputStream outputStream) {
try {
Files.copy(path, outputStream);
} catch (ClientAbortException e) {
// Aborted downloads will be handled gracefully
throw new DownloadAbortedException();
} catch (IOException e) {
log.error("Error while downloading file:", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Could not load file '%s'.".formatted(path));
}
}
private void downloadFilesAsZip(Path path, OutputStream outputStream) {
private void sendGamefilesAsZipToClient(Path path, OutputStream outputStream) {
ZipOutputStream zos = new ZipOutputStream(outputStream) {{
def.setLevel(Deflater.NO_COMPRESSION);
}};
@@ -195,65 +102,4 @@ public class DownloadService {
log.error("Error while zipping files:", e);
}
}
private int downloadImagesIntoCache(MultiValueMap<String, String> entityToImageIds, String imageSize, String imageType, String entityType) {
AtomicInteger downloadCounter = new AtomicInteger();
Path cacheFolder = Path.of(cacheFolderPath);
try {
Files.createDirectories(cacheFolder);
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Could not create cache folder.");
}
entityToImageIds.entrySet().parallelStream().forEach(entry ->
entry.getValue().forEach(imageId -> {
if (!StringUtils.hasText(imageId)) return;
String imgFileName = "%s.png".formatted(imageId);
String imgUrl = "t_%s/%s".formatted(imageSize, imgFileName);
if (Files.exists(Path.of(cacheFolderPath, imgFileName))) {
Path existingImageFile = Path.of(cacheFolderPath, imgFileName);
try {
if(Files.size(existingImageFile) == 0L) {
log.info("File '{}' is corrupt, retrying download...", imgFileName);
Files.delete(existingImageFile);
} else {
log.debug("{} for {} '{}' already downloaded ({}), skipping.",
imageType.substring(0, 1).toUpperCase() + imageType.substring(1).toLowerCase(),
entityType,
entry.getKey(),
imgFileName);
return;
}
} catch (IOException e) {
log.error("Error while checking file '{}'.", existingImageFile);
}
}
Flux<DataBuffer> dataBuffer = igdbImageClient.get()
.uri(imgUrl)
.retrieve()
.bodyToFlux(DataBuffer.class);
try {
DataBufferUtils.write(dataBuffer, cacheFolder.resolve(imgFileName), StandardOpenOption.CREATE)
.share().block();
} catch (WebClientResponseException e) {
if (e.getStatusCode().is4xxClientError()) {
log.error("Could not download {} for {} '{}' from {}: {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl, e.getStatusCode());
}
}
downloadCounter.getAndIncrement();
log.info("Downloaded {} for {} '{}' from {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl);
}));
return downloadCounter.get();
}
}
backend/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java
+84
View File
@@ -0,0 +1,84 @@
package de.grimsi.gameyfin.service;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.web.server.ResponseStatusException;
import reactor.core.publisher.Flux;
import javax.annotation.PostConstruct;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
@Slf4j
@Service
public class FilesystemService {
@Value("${gameyfin.cache}")
private String cacheFolderPath;
@PostConstruct
public void createCacheFolder() throws IOException {
Files.createDirectories(Path.of(cacheFolderPath));
}
public void saveFileToCache(Flux<DataBuffer> dataBuffer, String filename) {
DataBufferUtils.write(dataBuffer, Path.of(cacheFolderPath).resolve(filename), StandardOpenOption.CREATE)
.share().block();
}
public ByteArrayResource getFileFromCache(String filename) {
try {
return new ByteArrayResource(Files.readAllBytes(Paths.get("%s/%s".formatted(cacheFolderPath, filename))));
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Could not find image file %s".formatted(filename));
}
}
public void deleteFileFromCache(String filename) {
try {
Files.delete(getPathFromFilename(filename));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public boolean isCachedFileCorrupt(String filename) {
try {
return Files.size(getPathFromFilename(filename)) == 0L;
} catch (IOException e) {
log.error("Could not determine file size of '{}'", filename);
return true;
}
}
public boolean doesCachedFileExist(String filename) {
return Files.exists(getPathFromFilename(filename));
}
public long getSizeOnDisk(Path path) throws IOException {
if (Files.isDirectory(path)) {
// Some benchmarks I did have shown that trying to parallelize this process makes it slower instead of faster
return FileUtils.sizeOfDirectory(path.toFile());
} else {
try {
return Files.size(path);
} catch (IOException e) {
log.error("Error while calculating size of file '{}'.", path);
throw e;
}
}
}
private Path getPathFromFilename(String filename) {
return Path.of(cacheFolderPath, filename);
}
}
backend/src/main/java/de/grimsi/gameyfin/service/GameService.java
+10 -13
View File
@@ -8,7 +8,7 @@ import de.grimsi.gameyfin.igdb.IgdbWrapper;
import de.grimsi.gameyfin.mapper.GameMapper;
import de.grimsi.gameyfin.repositories.DetectedGameRepository;
import de.grimsi.gameyfin.repositories.UnmappableFileRepository;
import org.springframework.beans.factory.annotation.Autowired;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.web.server.ResponseStatusException;
@@ -19,17 +19,14 @@ import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
@RequiredArgsConstructor
@Service
public class GameService {
@Autowired
private IgdbWrapper igdbWrapper;
@Autowired
private DetectedGameRepository detectedGameRepository;
@Autowired
private UnmappableFileRepository unmappableFileRepository;
private final IgdbWrapper igdbWrapper;
private final GameMapper gameMapper;
private final DetectedGameRepository detectedGameRepository;
private final UnmappableFileRepository unmappableFileRepository;
public List<DetectedGame> getAllDetectedGames() {
return detectedGameRepository.findAll();
@@ -48,13 +45,13 @@ public class GameService {
}
public List<GameOverviewDto> getGameOverviews() {
return detectedGameRepository.findAll().stream().map(GameMapper::toGameOverviewDto).toList();
return detectedGameRepository.findAll().stream().map(gameMapper::toGameOverviewDto).toList();
}
public void deleteGame(String slug) {
DetectedGame gameToBeDeleted = getDetectedGame(slug);
// Add the path of the game to be deleted to the unmappable files
// Add the path of the game to be deleted to the unmappable files,
// so it doesn't get re-indexed on the next library scan
unmappableFileRepository.save(new UnmappableFile(gameToBeDeleted.getPath()));
@@ -94,7 +91,7 @@ public class GameService {
Igdb.Game igdbGame = igdbWrapper.getGameBySlug(slug)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Game with slug '%s' does not exist on IGDB.".formatted(slug)));
DetectedGame game = GameMapper.toDetectedGame(igdbGame, Path.of(unmappableFile.getPath()));
DetectedGame game = gameMapper.toDetectedGame(igdbGame, Path.of(unmappableFile.getPath()));
game = detectedGameRepository.save(game);
unmappableFileRepository.delete(unmappableFile);
@@ -106,7 +103,7 @@ public class GameService {
Igdb.Game igdbGame = igdbWrapper.getGameBySlug(slug)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Game with slug '%s' does not exist on IGDB.".formatted(slug)));
DetectedGame game = GameMapper.toDetectedGame(igdbGame, Path.of(existingGame.getPath()));
DetectedGame game = gameMapper.toDetectedGame(igdbGame, Path.of(existingGame.getPath()));
game = detectedGameRepository.save(game);
detectedGameRepository.deleteById(existingGame.getSlug());
backend/src/main/java/de/grimsi/gameyfin/service/ImageService.java
+142
View File
@@ -0,0 +1,142 @@
package de.grimsi.gameyfin.service;
import de.grimsi.gameyfin.entities.Company;
import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.igdb.IgdbApiProperties;
import de.grimsi.gameyfin.repositories.DetectedGameRepository;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.stereotype.Service;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.util.StopWatch;
import org.springframework.util.StringUtils;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientResponseException;
import reactor.core.publisher.Flux;
import javax.annotation.PostConstruct;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@Slf4j
@RequiredArgsConstructor
@Service
public class ImageService {
@Value("${gameyfin.cache}")
private String cacheFolderPath;
private final FilesystemService filesystemService;
private final DetectedGameRepository detectedGameRepository;
private final WebClient.Builder webclientBuilder;
private WebClient igdbImageClient;
@PostConstruct
public void init() {
igdbImageClient = webclientBuilder.baseUrl(IgdbApiProperties.IMAGES_BASE_URL).build();
}
public void downloadGameCoversFromIgdb() {
StopWatch stopWatch = new StopWatch();
log.info("Starting game cover download...");
stopWatch.start();
MultiValueMap<String, String> gameToImageIds = new LinkedMultiValueMap<>(
detectedGameRepository.findAll().stream()
.collect(Collectors.toMap(DetectedGame::getSlug, g -> Collections.singletonList(g.getCoverId()))));
int downloadCount = saveImagesIntoCache(gameToImageIds, IgdbApiProperties.COVER_IMAGE_SIZE, "cover", "game");
stopWatch.stop();
log.info("Downloaded {} covers in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds());
}
public void downloadGameScreenshotsFromIgdb() {
StopWatch stopWatch = new StopWatch();
log.info("Starting game screenshot download...");
stopWatch.start();
MultiValueMap<String, String> gamesToImageIds = new LinkedMultiValueMap<>(
detectedGameRepository.findAll().stream()
.collect(Collectors.toMap(DetectedGame::getSlug, DetectedGame::getScreenshotIds)));
int downloadCount = saveImagesIntoCache(gamesToImageIds, IgdbApiProperties.SCREENSHOT_IMAGE_SIZE, "screenshot", "game");
stopWatch.stop();
log.info("Downloaded {} screenshots in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds());
}
public void downloadCompanyLogosFromIgdb() {
StopWatch stopWatch = new StopWatch();
log.info("Starting company logo download...");
stopWatch.start();
Map<String, List<String>> companyToLogoIdMap = detectedGameRepository.findAll().stream()
.flatMap(g -> g.getCompanies().stream())
.collect(Collectors.toMap(Company::getSlug, c -> Collections.singletonList(c.getLogoId()), (c1, c2) -> c1));
MultiValueMap<String, String> companiesToLogoIds = new LinkedMultiValueMap<>(companyToLogoIdMap);
int downloadCount = saveImagesIntoCache(companiesToLogoIds, IgdbApiProperties.LOGO_IMAGE_SIZE, "logo", "company");
stopWatch.stop();
log.info("Downloaded {} company logos in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds());
}
private int saveImagesIntoCache(MultiValueMap<String, String> entityToImageIds, String imageSize, String imageType, String entityType) {
AtomicInteger downloadCounter = new AtomicInteger();
entityToImageIds.entrySet().parallelStream().forEach(entry ->
entry.getValue().forEach(imageId -> {
if (!StringUtils.hasText(imageId)) return;
String imgFileName = "%s.png".formatted(imageId);
String imgUrl = "t_%s/%s".formatted(imageSize, imgFileName);
if (filesystemService.doesCachedFileExist(imgFileName)) {
if (filesystemService.isCachedFileCorrupt(imgFileName)) {
log.info("File '{}' is corrupt, retrying download...", imgFileName);
filesystemService.deleteFileFromCache(imgFileName);
} else {
log.debug("{} for {} '{}' already downloaded ({}), skipping.",
imageType.substring(0, 1).toUpperCase() + imageType.substring(1).toLowerCase(),
entityType,
entry.getKey(),
imgFileName);
return;
}
}
Flux<DataBuffer> dataBuffer = igdbImageClient.get()
.uri(imgUrl)
.retrieve()
.bodyToFlux(DataBuffer.class);
try {
filesystemService.saveFileToCache(dataBuffer, imgFileName);
} catch (WebClientResponseException e) {
if (e.getStatusCode().is4xxClientError()) {
log.error("Could not download {} for {} '{}' from {}: {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl, e.getStatusCode());
}
}
downloadCounter.getAndIncrement();
log.info("Downloaded {} for {} '{}' from {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl);
}));
return downloadCounter.get();
}
}
backend/src/main/java/de/grimsi/gameyfin/service/LibraryService.java
+3 -1
View File
@@ -32,7 +32,9 @@ public class LibraryService {
@Value("${gameyfin.sources}")
private List<String> libraryFolders;
private final IgdbWrapper igdbWrapper;
private final GameMapper gameMapper;
private final DetectedGameRepository detectedGameRepository;
private final UnmappableFileRepository unmappableFileRepository;
@@ -105,7 +107,7 @@ public class LibraryService {
.filter(Optional::isPresent)
.map(Optional::get)
.peek(e -> log.info("Mapped file '{}' to game '{}' (slug: {})", e.getKey(), e.getValue().getName(), e.getValue().getSlug()))
.map(e -> GameMapper.toDetectedGame(e.getValue(), e.getKey()))
.map(e -> gameMapper.toDetectedGame(e.getValue(), e.getKey()))
.collect(Collectors.toList());
List<DetectedGame> duplicateGames = getDuplicates(newDetectedGames);
backend/src/main/java/de/grimsi/gameyfin/util/FilenameUtil.java
+2 -1
View File
@@ -4,6 +4,7 @@ import org.apache.commons.io.FilenameUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
@@ -21,7 +22,7 @@ public class FilenameUtil {
// If the path points to a folder, return the folder name
// Folders like "Counter Strike 1.6" would otherwise be returned as "Counter Strike 1"
if(p.toFile().isDirectory()) return FilenameUtils.getName(p.toString());
if(Files.isDirectory(p)) return FilenameUtils.getName(p.toString());
return FilenameUtils.getBaseName(p.toString());
}
backend/src/main/resources/application.yml
+3 -1
View File
@@ -1,3 +1,5 @@
# General
logging.level:
root: info
root: info
# Hides an error log on the first aborted download
org.apache.catalina.core.ContainerBase: off
backend/src/main/resources/config/gameyfin.yml
+3 -1
View File
@@ -1,4 +1,6 @@
gameyfin:
db: ""
cache: ""
file-extensions: iso, zip, rar, 7z, exe
igdb:
api:
@@ -7,4 +9,4 @@ gameyfin:
max-concurrent-requests: 2
max-requests-per-second: 4
config:
preferred-platforms: 6
preferred-platforms: 6