From f91b289cee217ed3a6540c6afdb7235f08dd2bd5 Mon Sep 17 00:00:00 2001 From: grimsi <9295182+grimsi@users.noreply.github.com> Date: Thu, 15 May 2025 22:57:25 +0200 Subject: [PATCH] Implement file size calculation Implement compression level config for DirectDownloadPlugin Fix download of games hogging RAM --- .../frontend/endpoints/DownloadEndpoint.ts | 31 +----- gameyfin/src/main/frontend/util/utils.ts | 32 ++++++ gameyfin/src/main/frontend/views/GameView.tsx | 4 +- .../core/download/DownloadEndpoint.kt | 22 ++-- .../gameyfin/core/download/DownloadService.kt | 4 +- .../core/filesystem/FilesystemService.kt | 17 +++ .../de/grimsi/gameyfin/games/GameService.kt | 1 + .../de/grimsi/gameyfin/games/dto/GameDto.kt | 1 + .../de/grimsi/gameyfin/games/entities/Game.kt | 2 + .../gameyfin/libraries/LibraryService.kt | 23 +++- gameyfin/src/main/resources/application.yml | 4 + .../pluginapi/download/DownloadProvider.kt | 2 +- .../directdownload/DirectDownloadPlugin.kt | 103 +++++++++++------- 13 files changed, 157 insertions(+), 89 deletions(-) diff --git a/gameyfin/src/main/frontend/endpoints/DownloadEndpoint.ts b/gameyfin/src/main/frontend/endpoints/DownloadEndpoint.ts index 55a37a6..84bd65f 100644 --- a/gameyfin/src/main/frontend/endpoints/DownloadEndpoint.ts +++ b/gameyfin/src/main/frontend/endpoints/DownloadEndpoint.ts @@ -1,30 +1,3 @@ -export async function downloadGame(gameId: number, provider: string) { - try { - const response = await fetch(`/download/${gameId}?provider=${provider}`, { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - }, - }); - - if (!response.ok) { - throw new Error('Network response was not ok'); - } - - const blob = await response.blob(); - const url = window.URL.createObjectURL(blob); - const contentDisposition = response.headers.get('Content-Disposition'); - const filename = contentDisposition - ? contentDisposition.split('filename=')[1].replace(/"/g, '') - : 'downloaded_file'; - - const link = document.createElement('a'); - link.href = url; - link.setAttribute('download', filename); - document.body.appendChild(link); - link.click(); - link.remove(); - } catch (error) { - console.error('Error downloading the file', error); - } +export function downloadGame(gameId: number, provider: string) { + window.open(`/download/${gameId}?provider=${provider}`, '_top'); } \ No newline at end of file diff --git a/gameyfin/src/main/frontend/util/utils.ts b/gameyfin/src/main/frontend/util/utils.ts index 2c60e9a..44e3cb8 100644 --- a/gameyfin/src/main/frontend/util/utils.ts +++ b/gameyfin/src/main/frontend/util/utils.ts @@ -87,6 +87,38 @@ export function timeUntil(instantString: string, timeZone: string = moment.tz.gu return "just now"; } +/** + * Format bytes as human-readable text. + * + * @param bytes Number of bytes. + * @param si True to use metric (SI) units, aka powers of 1000. False to use + * binary (IEC), aka powers of 1024. + * @param dp Number of decimal places to display. + * + * @return Formatted string. + */ +export function humanFileSize(bytes: number, si: boolean = false, dp: number = 1) { + const thresh = si ? 1000 : 1024; + + if (Math.abs(bytes) < thresh) { + return bytes + ' B'; + } + + const units = si + ? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] + : ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']; + let u = -1; + const r = 10 ** dp; + + do { + bytes /= thresh; + ++u; + } while (Math.round(Math.abs(bytes) * r) / r >= thresh && u < units.length - 1); + + + return bytes.toFixed(dp) + ' ' + units[u]; +} + /** * Select a random number of games from the library based on the library ID. * @param library diff --git a/gameyfin/src/main/frontend/views/GameView.tsx b/gameyfin/src/main/frontend/views/GameView.tsx index 1a9e210..37ee1ae 100644 --- a/gameyfin/src/main/frontend/views/GameView.tsx +++ b/gameyfin/src/main/frontend/views/GameView.tsx @@ -6,7 +6,7 @@ import {GameCover} from "Frontend/components/general/covers/GameCover"; import ComboButton, {ComboButtonOption} from "Frontend/components/general/input/ComboButton"; import ImageCarousel from "Frontend/components/general/covers/ImageCarousel"; import {Chip} from "@heroui/react"; -import {toTitleCase} from "Frontend/util/utils"; +import {humanFileSize, toTitleCase} from "Frontend/util/utils"; import {DownloadEndpoint} from "Frontend/endpoints/endpoints"; export default function GameView() { @@ -61,7 +61,7 @@ export default function GameView() {

{game.release !== undefined ? new Date(game.release).getFullYear() : "unknown"}

- diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/download/DownloadEndpoint.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/download/DownloadEndpoint.kt index 878ffe6..596a22e 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/download/DownloadEndpoint.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/download/DownloadEndpoint.kt @@ -4,10 +4,9 @@ import de.grimsi.gameyfin.core.annotations.DynamicPublicAccess import de.grimsi.gameyfin.games.GameService import de.grimsi.gameyfin.pluginapi.download.FileDownload import de.grimsi.gameyfin.pluginapi.download.LinkDownload -import org.springframework.core.io.InputStreamResource -import org.springframework.core.io.Resource import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.* +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody @RestController @RequestMapping("/download") @@ -21,19 +20,20 @@ class DownloadEndpoint( } @GetMapping("/{gameId}") - fun downloadGame(@PathVariable gameId: Long, @RequestParam provider: String): ResponseEntity { + fun downloadGame( + @PathVariable gameId: Long, + @RequestParam provider: String + ): ResponseEntity { val game = gameService.getGame(gameId) - val downloadElement = downloadService.getDownloadElement(game.path, provider) + val download = downloadService.getDownload(game.path, provider) - return when (downloadElement) { + return when (download) { is FileDownload -> { - val resource = InputStreamResource(downloadElement.data) ResponseEntity.ok() - .header( - "Content-Disposition", - "attachment; filename=\"${game.title}.${downloadElement.fileExtension}\"" - ) - .body(resource) + .header("Content-Disposition", "attachment; filename=\"${game.title}.zip\"") + .body(StreamingResponseBody { outputStream -> + download.data.copyTo(outputStream) + }) } is LinkDownload -> { diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/download/DownloadService.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/download/DownloadService.kt index 829264d..d86fe4e 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/download/DownloadService.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/download/DownloadService.kt @@ -17,10 +17,10 @@ class DownloadService( return downloadPlugins.map { it.javaClass.name } } - fun getDownloadElement(path: String, provider: String): Download { + fun getDownload(path: String, provider: String): Download { val provider = downloadPlugins.firstOrNull { it.javaClass.name == provider } ?: throw IllegalArgumentException("Download provider $provider not found") - return provider.getDownloadSources(Path(path)) + return provider.download(Path(path)) } } \ No newline at end of file diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemService.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemService.kt index 01f7ceb..9389c2e 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemService.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemService.kt @@ -6,6 +6,7 @@ import de.grimsi.gameyfin.libraries.Library import io.github.oshai.kotlinlogging.KotlinLogging import org.apache.commons.io.FilenameUtils import org.springframework.stereotype.Service +import java.io.File import java.nio.file.FileSystems import java.nio.file.Path import kotlin.io.path.* @@ -125,6 +126,22 @@ class FilesystemService( ) } + fun calculateFileSize(path: String): Long { + return try { + val file = File(path) + if (file.isFile) { + file.length() + } else if (file.isDirectory) { + File(path).walkTopDown().filter { it.isFile }.map { it.length() }.sum() + } else { + 0L + } + } catch (e: Exception) { + log.warn { "Error calculating file size for $path: ${e.message}" } + 0L + } + } + private fun safeReadDirectoryContents(path: String): List { return safeReadDirectoryContents(Path(path)) .map { FileDto(it.name, if (it.isDirectory()) FileType.DIRECTORY else FileType.FILE, it.hashCode()) } diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt index a2ac3e5..04ca5c9 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt @@ -345,6 +345,7 @@ fun Game.toDto(): GameDto { imageIds = this.images.mapNotNull { it.id }, videoUrls = this.videoUrls.map { it.toString() }, path = this.path, + fileSize = this.fileSize ?: 0L, metadata = toDto(this.metadata), originalIds = this.originalIds.mapKeys { it.key.pluginId } ) diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/dto/GameDto.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/dto/GameDto.kt index df420ff..822cd03 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/dto/GameDto.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/dto/GameDto.kt @@ -24,6 +24,7 @@ class GameDto( val imageIds: List?, val videoUrls: List?, val path: String, + val fileSize: Long, val metadata: Map, val originalIds: Map ) \ No newline at end of file diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt index 9487907..31344bb 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt @@ -78,6 +78,8 @@ class Game( @Column(unique = true) val path: String, + var fileSize: Long? = null, + @OneToMany(cascade = [CascadeType.ALL], orphanRemoval = true) var metadata: Map = emptyMap(), diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt index 3c05838..1ac3630 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt @@ -170,6 +170,7 @@ class LibraryService( val totalPaths = gamePaths.size val completedMetadata = AtomicInteger(0) val completedImageDownload = AtomicInteger(0) + val calculatedFileSize = AtomicInteger(0) log.info { "Scanning library '${library.name}' with $totalPaths paths..." } @@ -238,14 +239,30 @@ class LibraryService( val gamesWithImages = executor.invokeAll(imageDownloadTasks).mapNotNull { it.get() } - // 3. Persist new games + // 3. Calculate game file sizes + val calculateFileSizeTask = matchedGames.map { game -> + Callable { + game.path.let { path -> + val fileSize = filesystemService.calculateFileSize(path) + game.fileSize = fileSize + val progress = calculatedFileSize.incrementAndGet() + log.debug { "${progress}/${totalPaths} file sizes calculated" } + game + } + } + } + + val gamesWithFileSizes = executor.invokeAll(calculateFileSizeTask).map { it.get() } + + + // 4. Persist new games val persistedGames = gameService.create(gamesWithImages) log.debug { "${persistedGames.size}/${totalPaths} saved to database" } - // 4. Add new games to library + // 5. Add new games to library addGamesToLibrary(persistedGames, library) - // 5. Persist library + // 6. Persist library libraryRepository.save(library) return LibraryScanResult( diff --git a/gameyfin/src/main/resources/application.yml b/gameyfin/src/main/resources/application.yml index 20b8341..f8d1901 100644 --- a/gameyfin/src/main/resources/application.yml +++ b/gameyfin/src/main/resources/application.yml @@ -39,6 +39,10 @@ spring: fs.filesystem-root: ./data/ application: name: Gameyfin + threads: + virtual.enabled: true + mvc: + async.request-timeout: 0 vaadin: # To improve the performance during development. diff --git a/plugin-api/src/main/kotlin/de/grimsi/gameyfin/pluginapi/download/DownloadProvider.kt b/plugin-api/src/main/kotlin/de/grimsi/gameyfin/pluginapi/download/DownloadProvider.kt index 41c9238..fea638c 100644 --- a/plugin-api/src/main/kotlin/de/grimsi/gameyfin/pluginapi/download/DownloadProvider.kt +++ b/plugin-api/src/main/kotlin/de/grimsi/gameyfin/pluginapi/download/DownloadProvider.kt @@ -5,5 +5,5 @@ import java.nio.file.Path interface DownloadProvider : ExtensionPoint { - fun getDownloadSources(path: Path): Download + fun download(path: Path): Download } \ No newline at end of file diff --git a/plugins/directdownload/src/main/kotlin/de/grimsi/gameyfin/plugins/directdownload/DirectDownloadPlugin.kt b/plugins/directdownload/src/main/kotlin/de/grimsi/gameyfin/plugins/directdownload/DirectDownloadPlugin.kt index d8da0e4..4fe4e55 100644 --- a/plugins/directdownload/src/main/kotlin/de/grimsi/gameyfin/plugins/directdownload/DirectDownloadPlugin.kt +++ b/plugins/directdownload/src/main/kotlin/de/grimsi/gameyfin/plugins/directdownload/DirectDownloadPlugin.kt @@ -10,13 +10,12 @@ import org.pf4j.Extension import org.pf4j.PluginWrapper import org.slf4j.Logger import org.slf4j.LoggerFactory +import java.io.IOException import java.io.InputStream import java.io.PipedInputStream import java.io.PipedOutputStream import java.nio.file.* import java.nio.file.attribute.BasicFileAttributes -import java.util.concurrent.BlockingQueue -import java.util.concurrent.LinkedBlockingQueue import java.util.zip.Deflater import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream @@ -26,6 +25,15 @@ import kotlin.io.path.fileSize import kotlin.io.path.isDirectory class DirectDownloadPlugin(wrapper: PluginWrapper) : GameyfinPlugin(wrapper), Configurable { + companion object { + lateinit var plugin: DirectDownloadPlugin + private set + } + + init { + plugin = this + } + val log: Logger = LoggerFactory.getLogger(javaClass) enum class CompressionMode { @@ -68,64 +76,77 @@ class DirectDownloadPlugin(wrapper: PluginWrapper) : GameyfinPlugin(wrapper), Co @Extension class DirectDownloadProvider : DownloadProvider { - companion object { - private val END_OF_QUEUE = Pair(ZipEntry("__END__"), Paths.get("")) - } - - override fun getDownloadSources(path: Path): Download { + override fun download(path: Path): Download { if (!path.exists()) throw IllegalArgumentException("Path $path does not exist") return FileDownload( - data = readContentAsSingleFile(path), + data = streamContentAsSingleFile(path), fileExtension = if (path.isDirectory()) "zip" else path.extension, size = path.fileSize() ) } - fun readContentAsSingleFile(path: Path): InputStream { - if (path.isDirectory()) return zipFilesInPath(path) - return Files.newInputStream(path, StandardOpenOption.READ) + fun streamContentAsSingleFile(path: Path): InputStream { + if (path.isDirectory()) return streamFolderAsZip(path) + return streamFile(path) } - private fun zipFilesInPath(path: Path): InputStream { - val pipedIn = PipedInputStream(64 * 1024) - val pipedOut = PipedOutputStream(pipedIn) - val queue: BlockingQueue?> = LinkedBlockingQueue() + fun streamFile(path: Path): InputStream { + val pipeIn = PipedInputStream(512 * 1024) + val pipeOut = PipedOutputStream(pipeIn) - // Producer: walks the file tree and enqueues files - Thread.startVirtualThread { + Thread.ofVirtual().start { try { - Files.walkFileTree(path, object : SimpleFileVisitor() { - override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult { - val entry = ZipEntry(path.relativize(file).toString()) - queue.put(entry to file) - return FileVisitResult.CONTINUE - } - }) + Files.newInputStream(path, StandardOpenOption.READ).use { input -> + input.copyTo(pipeOut, 512 * 1024) + } + } catch (_: IOException) { } finally { - queue.put(END_OF_QUEUE) // signal end + try { + pipeOut.close() + } catch (_: IOException) { + } } } - // Consumer: zips files in parallel, but writes entries in order - Thread { - ZipOutputStream(pipedOut).use { zos -> - zos.setLevel(Deflater.NO_COMPRESSION) - while (true) { - val item = queue.take() - if (item === END_OF_QUEUE || item == null) break - val (entry, file) = item - zos.putNextEntry(entry) - Files.newInputStream(file, StandardOpenOption.READ).use { input -> - input.copyTo(zos, 128 * 1024) - } - zos.closeEntry() + return pipeIn + } + + fun streamFolderAsZip(path: Path): InputStream { + val pipeIn = PipedInputStream(512 * 1024) // 512 KB buffer + val pipeOut = PipedOutputStream(pipeIn) + + Thread.ofVirtual().start { + try { + ZipOutputStream(pipeOut).use { zos -> + + zos.setLevel(CompressionMode.toDeflaterLevel(plugin.config["compressionMode"]?.let { + CompressionMode.valueOf(it.uppercase()) + } ?: CompressionMode.NONE)) + + Files.walkFileTree(path, object : SimpleFileVisitor() { + override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult { + val entry = ZipEntry(path.relativize(file).toString()) + zos.putNextEntry(entry) + Files.newInputStream(file, StandardOpenOption.READ).use { input -> + input.copyTo(zos, 512 * 1024) + } + zos.closeEntry() + return FileVisitResult.CONTINUE + } + }) + } + pipeOut.close() + } catch (_: IOException) { + } finally { + try { + pipeOut.close() + } catch (_: IOException) { } } - pipedOut.close() - }.start() + } - return pipedIn + return pipeIn } } } \ No newline at end of file