Brendon/gameyfin
1
0
Fork 0
mirror of https://github.com/BrenBroZAYT/gameyfin.git synced 2026-06-16 16:20:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactored file-system code

Browse Source 
Fixed logging when aborting download of single files
This commit is contained in:
grimsi
2022-08-18 20:26:09 +02:00
parent c097c6fdc0
commit 1fbfeb1c7e
18 changed files with 307 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/pom.xml
+1 -1
View File
@@ -7,7 +7,7 @@
<parent> <parent>
<artifactId>gameyfin</artifactId> <artifactId>gameyfin</artifactId>
<groupId>de.grimsi</groupId> <groupId>de.grimsi</groupId>
<version>1.1.4</version> <version>1.1.4-RC1</version>
</parent> </parent>
<artifactId>gameyfin-backend</artifactId> <artifactId>gameyfin-backend</artifactId>
backend/src/main/java/de/grimsi/gameyfin/config/FilesystemConfig.java
+1 -1
View File
@@ -39,7 +39,7 @@ public class FilesystemConfig {
props.setProperty("gameyfin.db", "%s/.gameyfin/db".formatted(firstLibraryPath)); props.setProperty("gameyfin.db", "%s/.gameyfin/db".formatted(firstLibraryPath));
} }
if(StringUtils.hasText(cachePath)) { if(!StringUtils.hasText(cachePath)) {
props.setProperty("gameyfin.cache", "%s/.gameyfin/cache".formatted(firstLibraryPath)); props.setProperty("gameyfin.cache", "%s/.gameyfin/cache".formatted(firstLibraryPath));
} }
backend/src/main/java/de/grimsi/gameyfin/igdb/IgdbWrapper.java
+6 -6
View File
@@ -7,6 +7,7 @@ import de.grimsi.gameyfin.igdb.dto.TwitchOAuthTokenDto;
import de.grimsi.gameyfin.mapper.GameMapper; import de.grimsi.gameyfin.mapper.GameMapper;
import io.github.resilience4j.reactor.bulkhead.operator.BulkheadOperator; import io.github.resilience4j.reactor.bulkhead.operator.BulkheadOperator;
import io.github.resilience4j.reactor.ratelimiter.operator.RateLimiterOperator; import io.github.resilience4j.reactor.ratelimiter.operator.RateLimiterOperator;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
@@ -24,6 +25,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@Slf4j @Slf4j
@RequiredArgsConstructor
@Service @Service
public class IgdbWrapper { public class IgdbWrapper {
@Value("${gameyfin.igdb.api.client-id}") @Value("${gameyfin.igdb.api.client-id}")
@@ -35,11 +37,9 @@ public class IgdbWrapper {
@Value("${gameyfin.igdb.config.preferred-platforms:6}") @Value("${gameyfin.igdb.config.preferred-platforms:6}")
private String preferredPlatforms; private String preferredPlatforms;
@Autowired private final WebClient.Builder webclientBuilder;
private WebClient.Builder webclientBuilder; private final WebClientConfig webClientConfig;
private final GameMapper gameMapper;
@Autowired
private WebClientConfig webClientConfig;
private WebClient twitchApiClient; private WebClient twitchApiClient;
@@ -106,7 +106,7 @@ public class IgdbWrapper {
if(gameResult == null) return Collections.emptyList(); if(gameResult == null) return Collections.emptyList();
return gameResult.getGamesList().stream().map(GameMapper::toAutocompleteSuggestionDto).toList(); return gameResult.getGamesList().stream().map(gameMapper::toAutocompleteSuggestionDto).toList();
} }
public Optional<Igdb.Game> searchForGameByTitle(String searchTerm) { public Optional<Igdb.Game> searchForGameByTitle(String searchTerm) {
backend/src/main/java/de/grimsi/gameyfin/mapper/GameMapper.java
+19 -19
View File
@@ -4,6 +4,7 @@ import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.dto.AutocompleteSuggestionDto; import de.grimsi.gameyfin.dto.AutocompleteSuggestionDto;
import de.grimsi.gameyfin.dto.GameOverviewDto; import de.grimsi.gameyfin.dto.GameOverviewDto;
import de.grimsi.gameyfin.entities.DetectedGame; import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.service.FilesystemService;
import de.grimsi.gameyfin.service.LibraryService; import de.grimsi.gameyfin.service.LibraryService;
import de.grimsi.gameyfin.util.ProtobufUtil; import de.grimsi.gameyfin.util.ProtobufUtil;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@@ -24,9 +25,13 @@ import java.util.List;
import java.util.stream.Stream; import java.util.stream.Stream;
@Slf4j @Slf4j
@RequiredArgsConstructor
@Component
public class GameMapper { public class GameMapper {
public static DetectedGame toDetectedGame(Igdb.Game g, Path path) { private final FilesystemService filesystemService;
public DetectedGame toDetectedGame(Igdb.Game g, Path path) {
List<Igdb.MultiplayerMode> multiplayerModes = g.getMultiplayerModesList(); List<Igdb.MultiplayerMode> multiplayerModes = g.getMultiplayerModesList();
List<String> screenshotIds = g.getScreenshotsList().stream().map(Igdb.Screenshot::getImageId).toList(); List<String> screenshotIds = g.getScreenshotsList().stream().map(Igdb.Screenshot::getImageId).toList();
List<String> videoIds = g.getVideosList().stream().map(Igdb.GameVideo::getVideoId).toList(); List<String> videoIds = g.getVideosList().stream().map(Igdb.GameVideo::getVideoId).toList();
@@ -58,7 +63,7 @@ public class GameMapper {
.build(); .build();
} }
public static GameOverviewDto toGameOverviewDto(DetectedGame game) { public GameOverviewDto toGameOverviewDto(DetectedGame game) {
return GameOverviewDto.builder() return GameOverviewDto.builder()
.slug(game.getSlug()) .slug(game.getSlug())
.title(game.getTitle()) .title(game.getTitle())
@@ -66,7 +71,7 @@ public class GameMapper {
.build(); .build();
} }
public static AutocompleteSuggestionDto toAutocompleteSuggestionDto(Igdb.Game game) { public AutocompleteSuggestionDto toAutocompleteSuggestionDto(Igdb.Game game) {
return AutocompleteSuggestionDto.builder() return AutocompleteSuggestionDto.builder()
.slug(game.getSlug()) .slug(game.getSlug())
.title(game.getName()) .title(game.getName())
@@ -74,7 +79,7 @@ public class GameMapper {
.build(); .build();
} }
private static String getCoverId(Igdb.Game g) { private String getCoverId(Igdb.Game g) {
String coverId = g.getCover().getImageId(); String coverId = g.getCover().getImageId();
if(StringUtils.hasText(coverId)) return coverId; if(StringUtils.hasText(coverId)) return coverId;
@@ -82,23 +87,23 @@ public class GameMapper {
return "nocover"; return "nocover";
} }
private static boolean hasOfflineCoop(List<Igdb.MultiplayerMode> modes) { private boolean hasOfflineCoop(List<Igdb.MultiplayerMode> modes) {
return modes.stream().anyMatch(Igdb.MultiplayerMode::getOfflinecoop); return modes.stream().anyMatch(Igdb.MultiplayerMode::getOfflinecoop);
} }
private static boolean hasLanSupport(List<Igdb.MultiplayerMode> modes) { private boolean hasLanSupport(List<Igdb.MultiplayerMode> modes) {
return modes.stream().anyMatch(Igdb.MultiplayerMode::getLancoop); return modes.stream().anyMatch(Igdb.MultiplayerMode::getLancoop);
} }
private static boolean hasOnlineCoop(List<Igdb.MultiplayerMode> modes) { private boolean hasOnlineCoop(List<Igdb.MultiplayerMode> modes) {
return modes.stream().anyMatch(Igdb.MultiplayerMode::getOnlinecoop); return modes.stream().anyMatch(Igdb.MultiplayerMode::getOnlinecoop);
} }
private static int getMaxPlayers(List<Igdb.MultiplayerMode> modes) { private int getMaxPlayers(List<Igdb.MultiplayerMode> modes) {
return modes.stream().mapToInt(Igdb.MultiplayerMode::getOnlinecoopmax).max().orElse(0); return modes.stream().mapToInt(Igdb.MultiplayerMode::getOnlinecoopmax).max().orElse(0);
} }
private static long calculateDiskSize(Igdb.Game g, Path path) { private long calculateDiskSize(Igdb.Game g, Path path) {
StopWatch stopWatch = new StopWatch(); StopWatch stopWatch = new StopWatch();
log.info("Calculating disk size for game '{}'...", g.getName()); log.info("Calculating disk size for game '{}'...", g.getName());
@@ -106,16 +111,11 @@ public class GameMapper {
long fileSize; long fileSize;
if(Files.isDirectory(path)) { try {
// Some benchmarks I did have shown that trying to parallelize this process makes it slower instead of faster fileSize = filesystemService.getSizeOnDisk(path);
fileSize = FileUtils.sizeOfDirectory(path.toFile()); } catch(IOException e) {
} else { log.error("Error while calculating disk size for game '{}'", g.getName());
try{ fileSize = -1L;
fileSize = Files.size(path);
} catch (IOException e) {
log.error("Error while calculating size of file '{}'.", path);
fileSize = -1L;
}
} }
stopWatch.stop(); stopWatch.stop();
backend/src/main/java/de/grimsi/gameyfin/rest/GamesController.java
+1 -1
View File
@@ -54,7 +54,7 @@ public class GamesController {
return ResponseEntity return ResponseEntity
.ok() .ok()
.header("Content-Disposition", "attachment; filename=\"%s\"".formatted(downloadFileName)) .header("Content-Disposition", "attachment; filename=\"%s\"".formatted(downloadFileName))
.body(out -> downloadService.downloadGameFiles(game, out)); .body(out -> downloadService.sendGamefilesToClient(game, out));
} }
} }
backend/src/main/java/de/grimsi/gameyfin/rest/ImageController.java
+1 -1
View File
@@ -27,6 +27,6 @@ public class ImageController {
public ResponseEntity<Resource> getCoverImageForGame(@PathVariable String imageId) { public ResponseEntity<Resource> getCoverImageForGame(@PathVariable String imageId) {
return ResponseEntity.ok() return ResponseEntity.ok()
.cacheControl(CacheControl.maxAge(365, TimeUnit.DAYS).cachePublic()) .cacheControl(CacheControl.maxAge(365, TimeUnit.DAYS).cachePublic())
.body(downloadService.downloadImage(imageId)); .body(downloadService.sendImageToClient(imageId));
} }
} }
backend/src/main/java/de/grimsi/gameyfin/rest/LibraryController.java
+5 -4
View File
@@ -1,6 +1,7 @@
package de.grimsi.gameyfin.rest; package de.grimsi.gameyfin.rest;
import de.grimsi.gameyfin.service.DownloadService; import de.grimsi.gameyfin.service.DownloadService;
import de.grimsi.gameyfin.service.ImageService;
import de.grimsi.gameyfin.service.LibraryService; import de.grimsi.gameyfin.service.LibraryService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -25,7 +26,7 @@ import java.util.List;
public class LibraryController { public class LibraryController {
private final LibraryService libraryService; private final LibraryService libraryService;
private final DownloadService downloadService; private final ImageService imageService;
@GetMapping(value = "/scan", produces = MediaType.APPLICATION_JSON_VALUE) @GetMapping(value = "/scan", produces = MediaType.APPLICATION_JSON_VALUE)
public void scanLibrary(@RequestParam(value = "download_images", defaultValue = "true") boolean downloadImages) { public void scanLibrary(@RequestParam(value = "download_images", defaultValue = "true") boolean downloadImages) {
@@ -36,9 +37,9 @@ public class LibraryController {
@GetMapping(value = "/download-images") @GetMapping(value = "/download-images")
public void downloadImages() { public void downloadImages() {
downloadService.downloadGameCoversFromIgdb(); imageService.downloadGameCoversFromIgdb();
downloadService.downloadGameScreenshotsFromIgdb(); imageService.downloadGameScreenshotsFromIgdb();
downloadService.downloadCompanyLogosFromIgdb(); imageService.downloadCompanyLogosFromIgdb();
log.info("Downloading images completed."); log.info("Downloading images completed.");
} }
backend/src/main/java/de/grimsi/gameyfin/rest/LibraryManagementController.java
+6 -6
View File
@@ -8,6 +8,7 @@ import de.grimsi.gameyfin.entities.UnmappableFile;
import de.grimsi.gameyfin.repositories.DetectedGameRepository; import de.grimsi.gameyfin.repositories.DetectedGameRepository;
import de.grimsi.gameyfin.service.DownloadService; import de.grimsi.gameyfin.service.DownloadService;
import de.grimsi.gameyfin.service.GameService; import de.grimsi.gameyfin.service.GameService;
import de.grimsi.gameyfin.service.ImageService;
import de.grimsi.gameyfin.service.LibraryService; import de.grimsi.gameyfin.service.LibraryService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
@@ -23,8 +24,7 @@ import java.util.List;
public class LibraryManagementController { public class LibraryManagementController {
private final GameService gameService; private final GameService gameService;
private final DownloadService downloadService; private final ImageService imageService;
private final LibraryService libraryService; private final LibraryService libraryService;
@DeleteMapping(value = "/delete-game/{slug}", produces = MediaType.APPLICATION_JSON_VALUE) @DeleteMapping(value = "/delete-game/{slug}", produces = MediaType.APPLICATION_JSON_VALUE)
@@ -45,10 +45,10 @@ public class LibraryManagementController {
@PostMapping(value = "/map-path", produces = MediaType.APPLICATION_JSON_VALUE) @PostMapping(value = "/map-path", produces = MediaType.APPLICATION_JSON_VALUE)
public DetectedGame manuallyMapPathToSlug(@RequestBody PathToSlugDto pathToSlugDto) { public DetectedGame manuallyMapPathToSlug(@RequestBody PathToSlugDto pathToSlugDto) {
DetectedGame game = gameService.mapPathToGame(pathToSlugDto.getPath(), pathToSlugDto.getSlug()); DetectedGame game = gameService.mapPathToGame(pathToSlugDto.getPath(), pathToSlugDto.getSlug());
downloadService.downloadGameCoversFromIgdb(); imageService.downloadGameCoversFromIgdb();
downloadService.downloadGameScreenshotsFromIgdb(); imageService.downloadGameScreenshotsFromIgdb();
downloadService.downloadCompanyLogosFromIgdb(); imageService.downloadCompanyLogosFromIgdb();
return game; return game;
} }
backend/src/main/java/de/grimsi/gameyfin/service/DownloadService.java
+18 -172
View File
@@ -1,41 +1,19 @@
package de.grimsi.gameyfin.service; package de.grimsi.gameyfin.service;
import de.grimsi.gameyfin.entities.Company;
import de.grimsi.gameyfin.entities.DetectedGame; import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.exceptions.DownloadAbortedException; import de.grimsi.gameyfin.exceptions.DownloadAbortedException;
import de.grimsi.gameyfin.igdb.IgdbApiProperties;
import de.grimsi.gameyfin.repositories.DetectedGameRepository;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.catalina.connector.ClientAbortException; import org.apache.catalina.connector.ClientAbortException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.Resource; import org.springframework.core.io.Resource;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.util.StopWatch; import org.springframework.util.StopWatch;
import org.springframework.util.StringUtils;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientResponseException;
import org.springframework.web.server.ResponseStatusException;
import reactor.core.publisher.Flux;
import javax.annotation.PostConstruct;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.nio.file.*; import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.BasicFileAttributes;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.zip.Deflater; import java.util.zip.Deflater;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream; import java.util.zip.ZipOutputStream;
@@ -47,19 +25,7 @@ import static de.grimsi.gameyfin.util.FilenameUtil.getFilenameWithExtension;
@RequiredArgsConstructor @RequiredArgsConstructor
public class DownloadService { public class DownloadService {
@Value("${gameyfin.cache}") private final FilesystemService filesystemService;
private String cacheFolderPath;
private final DetectedGameRepository detectedGameRepository;
@Autowired
private WebClient.Builder webclientBuilder;
private WebClient igdbImageClient;
@PostConstruct
public void init() {
igdbImageClient = webclientBuilder.baseUrl(IgdbApiProperties.IMAGES_BASE_URL).build();
}
public String getDownloadFileName(DetectedGame g) { public String getDownloadFileName(DetectedGame g) {
Path path = Path.of(g.getPath()); Path path = Path.of(g.getPath());
@@ -68,17 +34,12 @@ public class DownloadService {
return getFilenameWithExtension(path) + ".zip"; return getFilenameWithExtension(path) + ".zip";
} }
public Resource downloadImage(String imageId) { public Resource sendImageToClient(String imageId) {
String filename = "%s.png".formatted(imageId); String filename = "%s.png".formatted(imageId);
return filesystemService.getFileFromCache(filename);
try {
return new ByteArrayResource(Files.readAllBytes(Paths.get("%s/%s".formatted(cacheFolderPath, filename))));
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Could not find image file %s".formatted(filename));
}
} }
public void downloadGameFiles(DetectedGame game, OutputStream outputStream) { public void sendGamefilesToClient(DetectedGame game, OutputStream outputStream) {
StopWatch stopWatch = new StopWatch(); StopWatch stopWatch = new StopWatch();
@@ -88,18 +49,16 @@ public class DownloadService {
Path path = Path.of(game.getPath()); Path path = Path.of(game.getPath());
if (path.toFile().isDirectory()) { try {
if (path.toFile().isDirectory()) {
try { sendGamefilesAsZipToClient(path, outputStream);
downloadFilesAsZip(path, outputStream); } else {
} catch(DownloadAbortedException e) { sendGamefileToClient(path, outputStream);
stopWatch.stop();
log.info("Download of game {} was aborted by client after {} seconds", game.getTitle(), (int) stopWatch.getTotalTimeSeconds());
return;
} }
} catch (DownloadAbortedException e) {
} else { stopWatch.stop();
downloadFile(path, outputStream); log.info("Download of game {} was aborted by client after {} seconds", game.getTitle(), (int) stopWatch.getTotalTimeSeconds());
return;
} }
stopWatch.stop(); stopWatch.stop();
@@ -107,70 +66,18 @@ public class DownloadService {
log.info("Downloaded game files of {} in {} seconds.", game.getTitle(), (int) stopWatch.getTotalTimeSeconds()); log.info("Downloaded game files of {} in {} seconds.", game.getTitle(), (int) stopWatch.getTotalTimeSeconds());
} }
private void sendGamefileToClient(Path path, OutputStream outputStream) {
public void downloadGameCoversFromIgdb() {
StopWatch stopWatch = new StopWatch();
log.info("Starting game cover download...");
stopWatch.start();
MultiValueMap<String, String> gameToImageIds = new LinkedMultiValueMap<>(
detectedGameRepository.findAll().stream()
.collect(Collectors.toMap(DetectedGame::getSlug, g -> Collections.singletonList(g.getCoverId()))));
int downloadCount = downloadImagesIntoCache(gameToImageIds, IgdbApiProperties.COVER_IMAGE_SIZE, "cover", "game");
stopWatch.stop();
log.info("Downloaded {} covers in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds());
}
public void downloadGameScreenshotsFromIgdb() {
StopWatch stopWatch = new StopWatch();
log.info("Starting game screenshot download...");
stopWatch.start();
MultiValueMap<String, String> gamesToImageIds = new LinkedMultiValueMap<>(
detectedGameRepository.findAll().stream()
.collect(Collectors.toMap(DetectedGame::getSlug, DetectedGame::getScreenshotIds)));
int downloadCount = downloadImagesIntoCache(gamesToImageIds, IgdbApiProperties.SCREENSHOT_IMAGE_SIZE, "screenshot", "game");
stopWatch.stop();
log.info("Downloaded {} screenshots in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds());
}
public void downloadCompanyLogosFromIgdb() {
StopWatch stopWatch = new StopWatch();
log.info("Starting company logo download...");
stopWatch.start();
Map<String, List<String>> companyToLogoIdMap = detectedGameRepository.findAll().stream()
.flatMap(g -> g.getCompanies().stream())
.collect(Collectors.toMap(Company::getSlug, c -> Collections.singletonList(c.getLogoId()), (c1, c2) -> c1));
MultiValueMap<String, String> companiesToLogoIds = new LinkedMultiValueMap<>(companyToLogoIdMap);
int downloadCount = downloadImagesIntoCache(companiesToLogoIds, IgdbApiProperties.LOGO_IMAGE_SIZE, "logo", "company");
stopWatch.stop();
log.info("Downloaded {} company logos in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds());
}
private void downloadFile(Path path, OutputStream outputStream) {
try { try {
Files.copy(path, outputStream); Files.copy(path, outputStream);
} catch (ClientAbortException e) {
// Aborted downloads will be handled gracefully
throw new DownloadAbortedException();
} catch (IOException e) { } catch (IOException e) {
log.error("Error while downloading file:", e); log.error("Error while downloading file:", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Could not load file '%s'.".formatted(path));
} }
} }
private void downloadFilesAsZip(Path path, OutputStream outputStream) { private void sendGamefilesAsZipToClient(Path path, OutputStream outputStream) {
ZipOutputStream zos = new ZipOutputStream(outputStream) {{ ZipOutputStream zos = new ZipOutputStream(outputStream) {{
def.setLevel(Deflater.NO_COMPRESSION); def.setLevel(Deflater.NO_COMPRESSION);
}}; }};
@@ -195,65 +102,4 @@ public class DownloadService {
log.error("Error while zipping files:", e); log.error("Error while zipping files:", e);
} }
} }
private int downloadImagesIntoCache(MultiValueMap<String, String> entityToImageIds, String imageSize, String imageType, String entityType) {
AtomicInteger downloadCounter = new AtomicInteger();
Path cacheFolder = Path.of(cacheFolderPath);
try {
Files.createDirectories(cacheFolder);
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Could not create cache folder.");
}
entityToImageIds.entrySet().parallelStream().forEach(entry ->
entry.getValue().forEach(imageId -> {
if (!StringUtils.hasText(imageId)) return;
String imgFileName = "%s.png".formatted(imageId);
String imgUrl = "t_%s/%s".formatted(imageSize, imgFileName);
if (Files.exists(Path.of(cacheFolderPath, imgFileName))) {
Path existingImageFile = Path.of(cacheFolderPath, imgFileName);
try {
if(Files.size(existingImageFile) == 0L) {
log.info("File '{}' is corrupt, retrying download...", imgFileName);
Files.delete(existingImageFile);
} else {
log.debug("{} for {} '{}' already downloaded ({}), skipping.",
imageType.substring(0, 1).toUpperCase() + imageType.substring(1).toLowerCase(),
entityType,
entry.getKey(),
imgFileName);
return;
}
} catch (IOException e) {
log.error("Error while checking file '{}'.", existingImageFile);
}
}
Flux<DataBuffer> dataBuffer = igdbImageClient.get()
.uri(imgUrl)
.retrieve()
.bodyToFlux(DataBuffer.class);
try {
DataBufferUtils.write(dataBuffer, cacheFolder.resolve(imgFileName), StandardOpenOption.CREATE)
.share().block();
} catch (WebClientResponseException e) {
if (e.getStatusCode().is4xxClientError()) {
log.error("Could not download {} for {} '{}' from {}: {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl, e.getStatusCode());
}
}
downloadCounter.getAndIncrement();
log.info("Downloaded {} for {} '{}' from {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl);
}));
return downloadCounter.get();
}
} }
backend/src/main/java/de/grimsi/gameyfin/service/FilesystemService.java
+84
View File
@@ -0,0 +1,84 @@
package de.grimsi.gameyfin.service;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.web.server.ResponseStatusException;
import reactor.core.publisher.Flux;
import javax.annotation.PostConstruct;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
@Slf4j
@Service
public class FilesystemService {
@Value("${gameyfin.cache}")
private String cacheFolderPath;
@PostConstruct
public void createCacheFolder() throws IOException {
Files.createDirectories(Path.of(cacheFolderPath));
}
public void saveFileToCache(Flux<DataBuffer> dataBuffer, String filename) {
DataBufferUtils.write(dataBuffer, Path.of(cacheFolderPath).resolve(filename), StandardOpenOption.CREATE)
.share().block();
}
public ByteArrayResource getFileFromCache(String filename) {
try {
return new ByteArrayResource(Files.readAllBytes(Paths.get("%s/%s".formatted(cacheFolderPath, filename))));
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Could not find image file %s".formatted(filename));
}
}
public void deleteFileFromCache(String filename) {
try {
Files.delete(getPathFromFilename(filename));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public boolean isCachedFileCorrupt(String filename) {
try {
return Files.size(getPathFromFilename(filename)) == 0L;
} catch (IOException e) {
log.error("Could not determine file size of '{}'", filename);
return true;
}
}
public boolean doesCachedFileExist(String filename) {
return Files.exists(getPathFromFilename(filename));
}
public long getSizeOnDisk(Path path) throws IOException {
if (Files.isDirectory(path)) {
// Some benchmarks I did have shown that trying to parallelize this process makes it slower instead of faster
return FileUtils.sizeOfDirectory(path.toFile());
} else {
try {
return Files.size(path);
} catch (IOException e) {
log.error("Error while calculating size of file '{}'.", path);
throw e;
}
}
}
private Path getPathFromFilename(String filename) {
return Path.of(cacheFolderPath, filename);
}
}
backend/src/main/java/de/grimsi/gameyfin/service/GameService.java
+10 -13
View File
@@ -8,7 +8,7 @@ import de.grimsi.gameyfin.igdb.IgdbWrapper;
import de.grimsi.gameyfin.mapper.GameMapper; import de.grimsi.gameyfin.mapper.GameMapper;
import de.grimsi.gameyfin.repositories.DetectedGameRepository; import de.grimsi.gameyfin.repositories.DetectedGameRepository;
import de.grimsi.gameyfin.repositories.UnmappableFileRepository; import de.grimsi.gameyfin.repositories.UnmappableFileRepository;
import org.springframework.beans.factory.annotation.Autowired; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.web.server.ResponseStatusException; import org.springframework.web.server.ResponseStatusException;
@@ -19,17 +19,14 @@ import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@RequiredArgsConstructor
@Service @Service
public class GameService { public class GameService {
@Autowired private final IgdbWrapper igdbWrapper;
private IgdbWrapper igdbWrapper; private final GameMapper gameMapper;
private final DetectedGameRepository detectedGameRepository;
@Autowired private final UnmappableFileRepository unmappableFileRepository;
private DetectedGameRepository detectedGameRepository;
@Autowired
private UnmappableFileRepository unmappableFileRepository;
public List<DetectedGame> getAllDetectedGames() { public List<DetectedGame> getAllDetectedGames() {
return detectedGameRepository.findAll(); return detectedGameRepository.findAll();
@@ -48,13 +45,13 @@ public class GameService {
} }
public List<GameOverviewDto> getGameOverviews() { public List<GameOverviewDto> getGameOverviews() {
return detectedGameRepository.findAll().stream().map(GameMapper::toGameOverviewDto).toList(); return detectedGameRepository.findAll().stream().map(gameMapper::toGameOverviewDto).toList();
} }
public void deleteGame(String slug) { public void deleteGame(String slug) {
DetectedGame gameToBeDeleted = getDetectedGame(slug); DetectedGame gameToBeDeleted = getDetectedGame(slug);
// Add the path of the game to be deleted to the unmappable files // Add the path of the game to be deleted to the unmappable files,
// so it doesn't get re-indexed on the next library scan // so it doesn't get re-indexed on the next library scan
unmappableFileRepository.save(new UnmappableFile(gameToBeDeleted.getPath())); unmappableFileRepository.save(new UnmappableFile(gameToBeDeleted.getPath()));
@@ -94,7 +91,7 @@ public class GameService {
Igdb.Game igdbGame = igdbWrapper.getGameBySlug(slug) Igdb.Game igdbGame = igdbWrapper.getGameBySlug(slug)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Game with slug '%s' does not exist on IGDB.".formatted(slug))); .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Game with slug '%s' does not exist on IGDB.".formatted(slug)));
DetectedGame game = GameMapper.toDetectedGame(igdbGame, Path.of(unmappableFile.getPath())); DetectedGame game = gameMapper.toDetectedGame(igdbGame, Path.of(unmappableFile.getPath()));
game = detectedGameRepository.save(game); game = detectedGameRepository.save(game);
unmappableFileRepository.delete(unmappableFile); unmappableFileRepository.delete(unmappableFile);
@@ -106,7 +103,7 @@ public class GameService {
Igdb.Game igdbGame = igdbWrapper.getGameBySlug(slug) Igdb.Game igdbGame = igdbWrapper.getGameBySlug(slug)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Game with slug '%s' does not exist on IGDB.".formatted(slug))); .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Game with slug '%s' does not exist on IGDB.".formatted(slug)));
DetectedGame game = GameMapper.toDetectedGame(igdbGame, Path.of(existingGame.getPath())); DetectedGame game = gameMapper.toDetectedGame(igdbGame, Path.of(existingGame.getPath()));
game = detectedGameRepository.save(game); game = detectedGameRepository.save(game);
detectedGameRepository.deleteById(existingGame.getSlug()); detectedGameRepository.deleteById(existingGame.getSlug());
backend/src/main/java/de/grimsi/gameyfin/service/ImageService.java
+142
View File
@@ -0,0 +1,142 @@
package de.grimsi.gameyfin.service;
import de.grimsi.gameyfin.entities.Company;
import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.igdb.IgdbApiProperties;
import de.grimsi.gameyfin.repositories.DetectedGameRepository;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.stereotype.Service;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.util.StopWatch;
import org.springframework.util.StringUtils;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientResponseException;
import reactor.core.publisher.Flux;
import javax.annotation.PostConstruct;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@Slf4j
@RequiredArgsConstructor
@Service
public class ImageService {
@Value("${gameyfin.cache}")
private String cacheFolderPath;
private final FilesystemService filesystemService;
private final DetectedGameRepository detectedGameRepository;
private final WebClient.Builder webclientBuilder;
private WebClient igdbImageClient;
@PostConstruct
public void init() {
igdbImageClient = webclientBuilder.baseUrl(IgdbApiProperties.IMAGES_BASE_URL).build();
}
public void downloadGameCoversFromIgdb() {
StopWatch stopWatch = new StopWatch();
log.info("Starting game cover download...");
stopWatch.start();
MultiValueMap<String, String> gameToImageIds = new LinkedMultiValueMap<>(
detectedGameRepository.findAll().stream()
.collect(Collectors.toMap(DetectedGame::getSlug, g -> Collections.singletonList(g.getCoverId()))));
int downloadCount = saveImagesIntoCache(gameToImageIds, IgdbApiProperties.COVER_IMAGE_SIZE, "cover", "game");
stopWatch.stop();
log.info("Downloaded {} covers in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds());
}
public void downloadGameScreenshotsFromIgdb() {
StopWatch stopWatch = new StopWatch();
log.info("Starting game screenshot download...");
stopWatch.start();
MultiValueMap<String, String> gamesToImageIds = new LinkedMultiValueMap<>(
detectedGameRepository.findAll().stream()
.collect(Collectors.toMap(DetectedGame::getSlug, DetectedGame::getScreenshotIds)));
int downloadCount = saveImagesIntoCache(gamesToImageIds, IgdbApiProperties.SCREENSHOT_IMAGE_SIZE, "screenshot", "game");
stopWatch.stop();
log.info("Downloaded {} screenshots in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds());
}
public void downloadCompanyLogosFromIgdb() {
StopWatch stopWatch = new StopWatch();
log.info("Starting company logo download...");
stopWatch.start();
Map<String, List<String>> companyToLogoIdMap = detectedGameRepository.findAll().stream()
.flatMap(g -> g.getCompanies().stream())
.collect(Collectors.toMap(Company::getSlug, c -> Collections.singletonList(c.getLogoId()), (c1, c2) -> c1));
MultiValueMap<String, String> companiesToLogoIds = new LinkedMultiValueMap<>(companyToLogoIdMap);
int downloadCount = saveImagesIntoCache(companiesToLogoIds, IgdbApiProperties.LOGO_IMAGE_SIZE, "logo", "company");
stopWatch.stop();
log.info("Downloaded {} company logos in {} seconds.", downloadCount, (int) stopWatch.getTotalTimeSeconds());
}
private int saveImagesIntoCache(MultiValueMap<String, String> entityToImageIds, String imageSize, String imageType, String entityType) {
AtomicInteger downloadCounter = new AtomicInteger();
entityToImageIds.entrySet().parallelStream().forEach(entry ->
entry.getValue().forEach(imageId -> {
if (!StringUtils.hasText(imageId)) return;
String imgFileName = "%s.png".formatted(imageId);
String imgUrl = "t_%s/%s".formatted(imageSize, imgFileName);
if (filesystemService.doesCachedFileExist(imgFileName)) {
if (filesystemService.isCachedFileCorrupt(imgFileName)) {
log.info("File '{}' is corrupt, retrying download...", imgFileName);
filesystemService.deleteFileFromCache(imgFileName);
} else {
log.debug("{} for {} '{}' already downloaded ({}), skipping.",
imageType.substring(0, 1).toUpperCase() + imageType.substring(1).toLowerCase(),
entityType,
entry.getKey(),
imgFileName);
return;
}
}
Flux<DataBuffer> dataBuffer = igdbImageClient.get()
.uri(imgUrl)
.retrieve()
.bodyToFlux(DataBuffer.class);
try {
filesystemService.saveFileToCache(dataBuffer, imgFileName);
} catch (WebClientResponseException e) {
if (e.getStatusCode().is4xxClientError()) {
log.error("Could not download {} for {} '{}' from {}: {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl, e.getStatusCode());
}
}
downloadCounter.getAndIncrement();
log.info("Downloaded {} for {} '{}' from {}", imageType, entityType, entry.getKey(), IgdbApiProperties.IMAGES_BASE_URL + imgUrl);
}));
return downloadCounter.get();
}
}
backend/src/main/java/de/grimsi/gameyfin/service/LibraryService.java
+3 -1
View File
@@ -32,7 +32,9 @@ public class LibraryService {
@Value("${gameyfin.sources}") @Value("${gameyfin.sources}")
private List<String> libraryFolders; private List<String> libraryFolders;
private final IgdbWrapper igdbWrapper; private final IgdbWrapper igdbWrapper;
private final GameMapper gameMapper;
private final DetectedGameRepository detectedGameRepository; private final DetectedGameRepository detectedGameRepository;
private final UnmappableFileRepository unmappableFileRepository; private final UnmappableFileRepository unmappableFileRepository;
@@ -105,7 +107,7 @@ public class LibraryService {
.filter(Optional::isPresent) .filter(Optional::isPresent)
.map(Optional::get) .map(Optional::get)
.peek(e -> log.info("Mapped file '{}' to game '{}' (slug: {})", e.getKey(), e.getValue().getName(), e.getValue().getSlug())) .peek(e -> log.info("Mapped file '{}' to game '{}' (slug: {})", e.getKey(), e.getValue().getName(), e.getValue().getSlug()))
.map(e -> GameMapper.toDetectedGame(e.getValue(), e.getKey())) .map(e -> gameMapper.toDetectedGame(e.getValue(), e.getKey()))
.collect(Collectors.toList()); .collect(Collectors.toList());
List<DetectedGame> duplicateGames = getDuplicates(newDetectedGames); List<DetectedGame> duplicateGames = getDuplicates(newDetectedGames);
backend/src/main/java/de/grimsi/gameyfin/util/FilenameUtil.java
+2 -1
View File
@@ -4,6 +4,7 @@ import org.apache.commons.io.FilenameUtils;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.List; import java.util.List;
@@ -21,7 +22,7 @@ public class FilenameUtil {
// If the path points to a folder, return the folder name // If the path points to a folder, return the folder name
// Folders like "Counter Strike 1.6" would otherwise be returned as "Counter Strike 1" // Folders like "Counter Strike 1.6" would otherwise be returned as "Counter Strike 1"
if(p.toFile().isDirectory()) return FilenameUtils.getName(p.toString()); if(Files.isDirectory(p)) return FilenameUtils.getName(p.toString());
return FilenameUtils.getBaseName(p.toString()); return FilenameUtils.getBaseName(p.toString());
} }
backend/src/main/resources/application.yml
+3 -1
View File
@@ -1,3 +1,5 @@
# General # General
logging.level: logging.level:
root: info root: info
# Hides an error log on the first aborted download
org.apache.catalina.core.ContainerBase: off
backend/src/main/resources/config/gameyfin.yml
+3 -1
View File
@@ -1,4 +1,6 @@
gameyfin: gameyfin:
db: ""
cache: ""
file-extensions: iso, zip, rar, 7z, exe file-extensions: iso, zip, rar, 7z, exe
igdb: igdb:
api: api:
@@ -7,4 +9,4 @@ gameyfin:
max-concurrent-requests: 2 max-concurrent-requests: 2
max-requests-per-second: 4 max-requests-per-second: 4
config: config:
preferred-platforms: 6 preferred-platforms: 6
frontend/pom.xml
+1 -1
View File
@@ -5,7 +5,7 @@
<parent> <parent>
<artifactId>gameyfin</artifactId> <artifactId>gameyfin</artifactId>
<groupId>de.grimsi</groupId> <groupId>de.grimsi</groupId>
<version>1.1.4</version> <version>1.1.4-RC1</version>
</parent> </parent>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
pom.xml
+1 -1
View File
@@ -5,7 +5,7 @@
<groupId>de.grimsi</groupId> <groupId>de.grimsi</groupId>
<artifactId>gameyfin</artifactId> <artifactId>gameyfin</artifactId>
<version>1.1.4</version> <version>1.1.4-RC1</version>
<name>gameyfin</name> <name>gameyfin</name>
<description>gameyfin</description> <description>gameyfin</description>