Implement file size calculation

Implement compression level config for DirectDownloadPlugin
Fix download of games hogging RAM

gameyfin/src/main/frontend/endpoints/DownloadEndpoint.ts

@@ -1,30 +1,3 @@
-export async function downloadGame(gameId: number, provider: string) {
+export function downloadGame(gameId: number, provider: string) {
-    try {
+    window.open(`/download/${gameId}?provider=${provider}`, '_top');
-        const response = await fetch(`/download/${gameId}?provider=${provider}`, {
-            method: 'GET',
-            headers: {
-                'Content-Type': 'application/json',
-            },
-        });
-
-        if (!response.ok) {
-            throw new Error('Network response was not ok');
-        }
-
-        const blob = await response.blob();
-        const url = window.URL.createObjectURL(blob);
-        const contentDisposition = response.headers.get('Content-Disposition');
-        const filename = contentDisposition
-            ? contentDisposition.split('filename=')[1].replace(/"/g, '')
-            : 'downloaded_file';
-
-        const link = document.createElement('a');
-        link.href = url;
-        link.setAttribute('download', filename);
-        document.body.appendChild(link);
-        link.click();
-        link.remove();
-    } catch (error) {
-        console.error('Error downloading the file', error);
-    }
 }

gameyfin/src/main/frontend/util/utils.ts

@@ -87,6 +87,38 @@ export function timeUntil(instantString: string, timeZone: string = moment.tz.gu
     return "just now";
 }
 
+/**
+ * Format bytes as human-readable text.
+ *
+ * @param bytes Number of bytes.
+ * @param si True to use metric (SI) units, aka powers of 1000. False to use
+ *           binary (IEC), aka powers of 1024.
+ * @param dp Number of decimal places to display.
+ *
+ * @return Formatted string.
+ */
+export function humanFileSize(bytes: number, si: boolean = false, dp: number = 1) {
+    const thresh = si ? 1000 : 1024;
+
+    if (Math.abs(bytes) < thresh) {
+        return bytes + ' B';
+    }
+
+    const units = si
+        ? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
+        : ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
+    let u = -1;
+    const r = 10 ** dp;
+
+    do {
+        bytes /= thresh;
+        ++u;
+    } while (Math.round(Math.abs(bytes) * r) / r >= thresh && u < units.length - 1);
+
+
+    return bytes.toFixed(dp) + ' ' + units[u];
+}
+
 /**
  * Select a random number of games from the library based on the library ID.
  * @param library

gameyfin/src/main/frontend/views/GameView.tsx

@@ -6,7 +6,7 @@ import {GameCover} from "Frontend/components/general/covers/GameCover";
 import ComboButton, {ComboButtonOption} from "Frontend/components/general/input/ComboButton";
 import ImageCarousel from "Frontend/components/general/covers/ImageCarousel";
 import {Chip} from "@heroui/react";
-import {toTitleCase} from "Frontend/util/utils";
+import {humanFileSize, toTitleCase} from "Frontend/util/utils";
 import {DownloadEndpoint} from "Frontend/endpoints/endpoints";
 
 export default function GameView() {
@@ -61,7 +61,7 @@ export default function GameView() {
                             <p className="text-foreground/60">{game.release !== undefined ? new Date(game.release).getFullYear() : "unknown"}</p>
                         </div>
                     </div>
-                    <ComboButton description="64 GiB"
+                    <ComboButton description={humanFileSize(game.fileSize)}
                                  options={downloadOptions}
                                  preferredOptionKey="preferred-download-method"
                     />

gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/download/DownloadEndpoint.kt

@@ -4,10 +4,9 @@ import de.grimsi.gameyfin.core.annotations.DynamicPublicAccess
 import de.grimsi.gameyfin.games.GameService
 import de.grimsi.gameyfin.pluginapi.download.FileDownload
 import de.grimsi.gameyfin.pluginapi.download.LinkDownload
-import org.springframework.core.io.InputStreamResource
-import org.springframework.core.io.Resource
 import org.springframework.http.ResponseEntity
 import org.springframework.web.bind.annotation.*
+import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody
 
 @RestController
 @RequestMapping("/download")
@@ -21,19 +20,20 @@ class DownloadEndpoint(
     }
 
     @GetMapping("/{gameId}")
-    fun downloadGame(@PathVariable gameId: Long, @RequestParam provider: String): ResponseEntity<Resource> {
+    fun downloadGame(
+        @PathVariable gameId: Long,
+        @RequestParam provider: String
+    ): ResponseEntity<StreamingResponseBody> {
         val game = gameService.getGame(gameId)
-        val downloadElement = downloadService.getDownloadElement(game.path, provider)
+        val download = downloadService.getDownload(game.path, provider)
 
-        return when (downloadElement) {
+        return when (download) {
             is FileDownload -> {
-                val resource = InputStreamResource(downloadElement.data)
                 ResponseEntity.ok()
-                    .header(
+                    .header("Content-Disposition", "attachment; filename=\"${game.title}.zip\"")
-                        "Content-Disposition",
+                    .body(StreamingResponseBody { outputStream ->
-                        "attachment; filename=\"${game.title}.${downloadElement.fileExtension}\""
+                        download.data.copyTo(outputStream)
-                    )
+                    })
-                    .body(resource)
             }
 
             is LinkDownload -> {

gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/download/DownloadService.kt

@@ -17,10 +17,10 @@ class DownloadService(
         return downloadPlugins.map { it.javaClass.name }
     }
 
-    fun getDownloadElement(path: String, provider: String): Download {
+    fun getDownload(path: String, provider: String): Download {
         val provider = downloadPlugins.firstOrNull { it.javaClass.name == provider }
             ?: throw IllegalArgumentException("Download provider $provider not found")
 
-        return provider.getDownloadSources(Path(path))
+        return provider.download(Path(path))
     }
 }

gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemService.kt

@@ -6,6 +6,7 @@ import de.grimsi.gameyfin.libraries.Library
 import io.github.oshai.kotlinlogging.KotlinLogging
 import org.apache.commons.io.FilenameUtils
 import org.springframework.stereotype.Service
+import java.io.File
 import java.nio.file.FileSystems
 import java.nio.file.Path
 import kotlin.io.path.*
@@ -125,6 +126,22 @@ class FilesystemService(
         )
     }
 
+    fun calculateFileSize(path: String): Long {
+        return try {
+            val file = File(path)
+            if (file.isFile) {
+                file.length()
+            } else if (file.isDirectory) {
+                File(path).walkTopDown().filter { it.isFile }.map { it.length() }.sum()
+            } else {
+                0L
+            }
+        } catch (e: Exception) {
+            log.warn { "Error calculating file size for $path: ${e.message}" }
+            0L
+        }
+    }
+
     private fun safeReadDirectoryContents(path: String): List<FileDto> {
         return safeReadDirectoryContents(Path(path))
             .map { FileDto(it.name, if (it.isDirectory()) FileType.DIRECTORY else FileType.FILE, it.hashCode()) }

gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt

@@ -345,6 +345,7 @@ fun Game.toDto(): GameDto {
         imageIds = this.images.mapNotNull { it.id },
         videoUrls = this.videoUrls.map { it.toString() },
         path = this.path,
+        fileSize = this.fileSize ?: 0L,
         metadata = toDto(this.metadata),
         originalIds = this.originalIds.mapKeys { it.key.pluginId }
     )

gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/dto/GameDto.kt

@@ -24,6 +24,7 @@ class GameDto(
     val imageIds: List<Long>?,
     val videoUrls: List<String>?,
     val path: String,
+    val fileSize: Long,
     val metadata: Map<String, GameMetadataDto>,
     val originalIds: Map<String, String>
 )

gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt

@@ -78,6 +78,8 @@ class Game(
     @Column(unique = true)
     val path: String,
 
+    var fileSize: Long? = null,
+
     @OneToMany(cascade = [CascadeType.ALL], orphanRemoval = true)
     var metadata: Map<String, FieldMetadata> = emptyMap(),
 

gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt

@@ -170,6 +170,7 @@ class LibraryService(
             val totalPaths = gamePaths.size
             val completedMetadata = AtomicInteger(0)
             val completedImageDownload = AtomicInteger(0)
+            val calculatedFileSize = AtomicInteger(0)
 
             log.info { "Scanning library '${library.name}' with $totalPaths paths..." }
 
@@ -238,14 +239,30 @@ class LibraryService(
 
             val gamesWithImages = executor.invokeAll(imageDownloadTasks).mapNotNull { it.get() }
 
-            // 3. Persist new games
+            // 3. Calculate game file sizes
+            val calculateFileSizeTask = matchedGames.map { game ->
+                Callable {
+                    game.path.let { path ->
+                        val fileSize = filesystemService.calculateFileSize(path)
+                        game.fileSize = fileSize
+                        val progress = calculatedFileSize.incrementAndGet()
+                        log.debug { "${progress}/${totalPaths} file sizes calculated" }
+                        game
+                    }
+                }
+            }
+
+            val gamesWithFileSizes = executor.invokeAll(calculateFileSizeTask).map { it.get() }
+
+
+            // 4. Persist new games
             val persistedGames = gameService.create(gamesWithImages)
             log.debug { "${persistedGames.size}/${totalPaths} saved to database" }
 
-            // 4. Add new games to library
+            // 5. Add new games to library
             addGamesToLibrary(persistedGames, library)
 
-            // 5. Persist library
+            // 6. Persist library
             libraryRepository.save(library)
 
             return LibraryScanResult(

gameyfin/src/main/resources/application.yml

@@ -39,6 +39,10 @@ spring:
     fs.filesystem-root: ./data/
   application:
     name: Gameyfin
+  threads:
+    virtual.enabled: true
+  mvc:
+    async.request-timeout: 0
 
 vaadin:
   # To improve the performance during development.

plugin-api/src/main/kotlin/de/grimsi/gameyfin/pluginapi/download/DownloadProvider.kt

@@ -5,5 +5,5 @@ import java.nio.file.Path
 
 interface DownloadProvider : ExtensionPoint {
 
-    fun getDownloadSources(path: Path): Download
+    fun download(path: Path): Download
 }

plugins/directdownload/src/main/kotlin/de/grimsi/gameyfin/plugins/directdownload/DirectDownloadPlugin.kt

@@ -10,13 +10,12 @@ import org.pf4j.Extension
 import org.pf4j.PluginWrapper
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
+import java.io.IOException
 import java.io.InputStream
 import java.io.PipedInputStream
 import java.io.PipedOutputStream
 import java.nio.file.*
 import java.nio.file.attribute.BasicFileAttributes
-import java.util.concurrent.BlockingQueue
-import java.util.concurrent.LinkedBlockingQueue
 import java.util.zip.Deflater
 import java.util.zip.ZipEntry
 import java.util.zip.ZipOutputStream
@@ -26,6 +25,15 @@ import kotlin.io.path.fileSize
 import kotlin.io.path.isDirectory
 
 class DirectDownloadPlugin(wrapper: PluginWrapper) : GameyfinPlugin(wrapper), Configurable {
+    companion object {
+        lateinit var plugin: DirectDownloadPlugin
+            private set
+    }
+
+    init {
+        plugin = this
+    }
+
     val log: Logger = LoggerFactory.getLogger(javaClass)
 
     enum class CompressionMode {
@@ -68,64 +76,77 @@ class DirectDownloadPlugin(wrapper: PluginWrapper) : GameyfinPlugin(wrapper), Co
 
     @Extension
     class DirectDownloadProvider : DownloadProvider {
-        companion object {
+        override fun download(path: Path): Download {
-            private val END_OF_QUEUE = Pair<ZipEntry, Path>(ZipEntry("__END__"), Paths.get(""))
-        }
-
-        override fun getDownloadSources(path: Path): Download {
             if (!path.exists()) throw IllegalArgumentException("Path $path does not exist")
 
             return FileDownload(
-                data = readContentAsSingleFile(path),
+                data = streamContentAsSingleFile(path),
                 fileExtension = if (path.isDirectory()) "zip" else path.extension,
                 size = path.fileSize()
             )
         }
 
-        fun readContentAsSingleFile(path: Path): InputStream {
+        fun streamContentAsSingleFile(path: Path): InputStream {
-            if (path.isDirectory()) return zipFilesInPath(path)
+            if (path.isDirectory()) return streamFolderAsZip(path)
-            return Files.newInputStream(path, StandardOpenOption.READ)
+            return streamFile(path)
         }
 
-        private fun zipFilesInPath(path: Path): InputStream {
+        fun streamFile(path: Path): InputStream {
-            val pipedIn = PipedInputStream(64 * 1024)
+            val pipeIn = PipedInputStream(512 * 1024)
-            val pipedOut = PipedOutputStream(pipedIn)
+            val pipeOut = PipedOutputStream(pipeIn)
-            val queue: BlockingQueue<Pair<ZipEntry, Path>?> = LinkedBlockingQueue()
 
-            // Producer: walks the file tree and enqueues files
+            Thread.ofVirtual().start {
-            Thread.startVirtualThread {
                 try {
-                    Files.walkFileTree(path, object : SimpleFileVisitor<Path>() {
+                    Files.newInputStream(path, StandardOpenOption.READ).use { input ->
-                        override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
+                        input.copyTo(pipeOut, 512 * 1024)
-                            val entry = ZipEntry(path.relativize(file).toString())
+                    }
-                            queue.put(entry to file)
+                } catch (_: IOException) {
-                            return FileVisitResult.CONTINUE
-                        }
-                    })
                 } finally {
-                    queue.put(END_OF_QUEUE) // signal end
+                    try {
+                        pipeOut.close()
+                    } catch (_: IOException) {
+                    }
                 }
             }
 
-            // Consumer: zips files in parallel, but writes entries in order
+            return pipeIn
-            Thread {
+        }
-                ZipOutputStream(pipedOut).use { zos ->
+
-                    zos.setLevel(Deflater.NO_COMPRESSION)
+        fun streamFolderAsZip(path: Path): InputStream {
-                    while (true) {
+            val pipeIn = PipedInputStream(512 * 1024) // 512 KB buffer
-                        val item = queue.take()
+            val pipeOut = PipedOutputStream(pipeIn)
-                        if (item === END_OF_QUEUE || item == null) break
+
-                        val (entry, file) = item
+            Thread.ofVirtual().start {
-                        zos.putNextEntry(entry)
+                try {
-                        Files.newInputStream(file, StandardOpenOption.READ).use { input ->
+                    ZipOutputStream(pipeOut).use { zos ->
-                            input.copyTo(zos, 128 * 1024)
+
-                        }
+                        zos.setLevel(CompressionMode.toDeflaterLevel(plugin.config["compressionMode"]?.let {
-                        zos.closeEntry()
+                            CompressionMode.valueOf(it.uppercase())
+                        } ?: CompressionMode.NONE))
+
+                        Files.walkFileTree(path, object : SimpleFileVisitor<Path>() {
+                            override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
+                                val entry = ZipEntry(path.relativize(file).toString())
+                                zos.putNextEntry(entry)
+                                Files.newInputStream(file, StandardOpenOption.READ).use { input ->
+                                    input.copyTo(zos, 512 * 1024)
+                                }
+                                zos.closeEntry()
+                                return FileVisitResult.CONTINUE
+                            }
+                        })
+                    }
+                    pipeOut.close()
+                } catch (_: IOException) {
+                } finally {
+                    try {
+                        pipeOut.close()
+                    } catch (_: IOException) {
                     }
                 }
-                pipedOut.close()
+            }
-            }.start()
 
-            return pipedIn
+            return pipeIn
         }
     }
 }