Implement detection of removed games & unmatched paths

Implement persisting unmatched paths
Various minor refactorings

gameyfin/src/main/frontend/components/general/cards/LibraryOverviewCard.tsx

@@ -23,6 +23,7 @@ import {
 } from "@phosphor-icons/react";
 import LibraryDetailsModal from "Frontend/components/general/modals/LibraryDetailsModal";
 import LibraryUpdateDto from "Frontend/generated/de/grimsi/gameyfin/libraries/dto/LibraryUpdateDto";
+import ScanType from "Frontend/generated/de/grimsi/gameyfin/libraries/enums/ScanType";
 
 export function LibraryOverviewCard({library, updateLibrary}: {
     library: LibraryDto,
@@ -85,7 +86,8 @@ export function LibraryOverviewCard({library, updateLibrary}: {
 
                     <div className="absolute right-0 top-0 flex flex-row">
                         <Tooltip content="Scan library" placement="bottom" color="foreground">
-                            <Button isIconOnly variant="light" onPress={() => LibraryEndpoint.triggerScan([library])}>
+                            <Button isIconOnly variant="light"
+                                    onPress={() => LibraryEndpoint.triggerScan(ScanType.QUICK, [library])}>
                                 <MagnifyingGlass/>
                             </Button>
                         </Tooltip>

gameyfin/src/main/kotlin/de/grimsi/gameyfin/config/entities/ConfigEntry.kt

@@ -2,17 +2,14 @@ package de.grimsi.gameyfin.config.entities
 
 import de.grimsi.gameyfin.core.security.EncryptionConverter
 import jakarta.persistence.*
-import jakarta.validation.constraints.NotNull
 
 @Entity
 @Table(name = "app_config")
 class ConfigEntry(
     @Id
-    @NotNull
     @Column(name = "`key`", unique = true)
     val key: String,
 
-    @NotNull
     @Column(name = "`value`")
     @Convert(converter = EncryptionConverter::class)
     var value: String

gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemScanResult.kt

@@ -0,0 +1,9 @@
+package de.grimsi.gameyfin.core.filesystem
+
+import java.nio.file.Path
+
+data class FilesystemScanResult(
+    val newPaths: Set<Path>,
+    val removedGamePaths: Set<Path>,
+    val removedUnmatchedPaths: Set<Path>
+)

gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemService.kt

@@ -75,7 +75,7 @@ class FilesystemService(
      * @param library The library to scan.
      * @return A list of paths representing game files and directories.
      */
-    fun scanLibraryForGamefiles(library: Library): List<Path> {
+    fun scanLibraryForGamefiles(library: Library): FilesystemScanResult {
         // Cache the game file extensions to avoid reading them multiple times in the same scan
         val gamefileExtensions = gameFileExtensions
 
@@ -90,11 +90,39 @@ class FilesystemService(
                 }
             }
 
-        // Return all paths that are directories or match the game file extensions
+        // Get all paths that are directories or match the game file extensions
-        return validDirectories.flatMap { validDirectory ->
+        val currentFilesystemPaths = validDirectories.flatMap { validDirectory ->
             safeReadDirectoryContents(validDirectory)
                 .filter { it.isDirectory() || it.extension.lowercase() in gamefileExtensions }
         }
+
+        // Get all paths already in the library as game files or as unmatched paths
+        val currentLibraryGamePaths = library.games.map { Path(it.path) }
+        val currentLibraryUnmatchedPaths = library.unmatchedPaths.map { Path(it) }
+        val allCurrentLibraryPaths = currentLibraryGamePaths + currentLibraryUnmatchedPaths
+
+        //Get all paths that are on the filesystem, but not in the library (either as game or as unmatched path)
+        val newPaths = currentFilesystemPaths.filter { path ->
+            val isInLibrary = allCurrentLibraryPaths.any { it == path }
+            !isInLibrary
+        }.toSet()
+
+        //Get all paths that are in the library (either as game or as unmatched path), but not on the filesystem
+        val removedGamePaths = currentLibraryGamePaths.filter { path ->
+            val isOnFilesystem = currentFilesystemPaths.any { it == path }
+            !isOnFilesystem
+        }.toSet()
+
+        val removedUnmatchedPaths = currentLibraryUnmatchedPaths.filter { path ->
+            val isOnFilesystem = currentFilesystemPaths.any { it == path }
+            !isOnFilesystem
+        }.toSet()
+
+        return FilesystemScanResult(
+            newPaths = newPaths,
+            removedGamePaths = removedGamePaths,
+            removedUnmatchedPaths = removedUnmatchedPaths
+        )
     }
 
     private fun safeReadDirectoryContents(path: String): List<FileDto> {

gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/plugins/config/PluginConfigEntry.kt

@@ -1,23 +1,15 @@
 package de.grimsi.gameyfin.core.plugins.config
 
 import de.grimsi.gameyfin.core.security.EncryptionConverter
-import jakarta.persistence.Column
+import jakarta.persistence.*
-import jakarta.persistence.Convert
-import jakarta.persistence.Embeddable
-import jakarta.persistence.EmbeddedId
-import jakarta.persistence.Entity
-import jakarta.persistence.Table
-import jakarta.validation.constraints.NotNull
 import java.io.Serializable
 
 @Entity
 @Table(name = "plugin_config")
 data class PluginConfigEntry(
-    @NotNull
     @EmbeddedId
     val id: PluginConfigEntryKey,
 
-    @NotNull
     @Column(name = "`value`")
     @Convert(converter = EncryptionConverter::class)
     val value: String

gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt

@@ -86,6 +86,10 @@ class GameService(
         return mergedGame
     }
 
+    fun getAllByPaths(paths: Collection<String>): Collection<Game> {
+        return gameRepository.findAllByPathIn(paths)
+    }
+
     fun getAllGames(): Collection<GameDto> {
         val entities = gameRepository.findAll()
         return entities.map { toDto(it) }

gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt

@@ -39,10 +39,10 @@ class Game(
 
     var criticRating: Int? = null,
 
-    @ManyToMany(cascade = [CascadeType.ALL])
+    @ManyToMany(cascade = [CascadeType.PERSIST, CascadeType.MERGE, CascadeType.REFRESH])
     var publishers: Set<Company> = emptySet(),
 
-    @ManyToMany(cascade = [CascadeType.ALL])
+    @ManyToMany(cascade = [CascadeType.PERSIST, CascadeType.MERGE, CascadeType.REFRESH])
     var developers: Set<Company> = emptySet(),
 
     @ElementCollection(targetClass = Genre::class)

gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Image.kt

@@ -1,6 +1,5 @@
 package de.grimsi.gameyfin.games.entities
 
-import jakarta.annotation.Nullable
 import jakarta.persistence.Entity
 import jakarta.persistence.GeneratedValue
 import jakarta.persistence.GenerationType
@@ -21,15 +20,12 @@ class Image(
     val type: ImageType,
 
     @ContentId
-    @Nullable
     var contentId: String? = null,
 
     @ContentLength
-    @Nullable
     var contentLength: Long? = null,
 
     @MimeType
-    @Nullable
     var mimeType: String? = null
 )
 

gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/repositories/GameRepository.kt

@@ -5,4 +5,5 @@ import org.springframework.data.jpa.repository.JpaRepository
 
 interface GameRepository : JpaRepository<Game, Long> {
     fun findByPath(path: String): Game?
+    fun findAllByPathIn(paths: Collection<String>): Collection<Game>
 }

gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/Library.kt

@@ -14,6 +14,9 @@ class Library(
     @ElementCollection(fetch = FetchType.EAGER)
     var directories: MutableSet<String> = HashSet<String>(),
 
-    @ManyToMany(fetch = FetchType.EAGER)
+    @OneToMany(fetch = FetchType.EAGER, orphanRemoval = true)
-    var games: MutableSet<Game> = HashSet<Game>()
+    var games: MutableSet<Game> = HashSet<Game>(),
+
+    @ElementCollection(fetch = FetchType.EAGER)
+    var unmatchedPaths: MutableSet<String> = HashSet<String>()
 )

gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryEndpoint.kt

@@ -6,6 +6,7 @@ import de.grimsi.gameyfin.games.GameService
 import de.grimsi.gameyfin.games.dto.GameDto
 import de.grimsi.gameyfin.libraries.dto.LibraryDto
 import de.grimsi.gameyfin.libraries.dto.LibraryUpdateDto
+import de.grimsi.gameyfin.libraries.enums.ScanType
 import jakarta.annotation.security.PermitAll
 import jakarta.annotation.security.RolesAllowed
 
@@ -24,8 +25,8 @@ class LibraryEndpoint(
     }
 
     @RolesAllowed(Role.Names.ADMIN)
-    fun triggerScan(libraries: Collection<LibraryDto>?) {
+    fun triggerScan(scanType: ScanType = ScanType.QUICK, libraries: Collection<LibraryDto>?) {
-        return libraryService.triggerScan(libraries)
+        return libraryService.triggerScan(scanType, libraries)
     }
 
     @RolesAllowed(Role.Names.ADMIN)

gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryScanResult.kt

@@ -0,0 +1,10 @@
+package de.grimsi.gameyfin.libraries
+
+import de.grimsi.gameyfin.games.entities.Game
+
+data class LibraryScanResult(
+    val libraries: Set<Library>,
+    val newGames: Set<Game>,
+    val removedGames: Set<Game>,
+    val newUnmatchedPaths: Set<String>
+)

gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt

@@ -1,30 +1,29 @@
 package de.grimsi.gameyfin.libraries
 
 import de.grimsi.gameyfin.core.filesystem.FilesystemService
-import de.grimsi.gameyfin.games.CompanyService
 import de.grimsi.gameyfin.games.GameService
 import de.grimsi.gameyfin.games.dto.GameDto
 import de.grimsi.gameyfin.games.entities.Game
 import de.grimsi.gameyfin.libraries.dto.LibraryDto
 import de.grimsi.gameyfin.libraries.dto.LibraryStatsDto
 import de.grimsi.gameyfin.libraries.dto.LibraryUpdateDto
+import de.grimsi.gameyfin.libraries.enums.ScanType
 import de.grimsi.gameyfin.media.ImageService
 import io.github.oshai.kotlinlogging.KotlinLogging
-import kotlinx.coroutines.runBlocking
 import org.springframework.data.repository.findByIdOrNull
 import org.springframework.stereotype.Service
-import org.springframework.transaction.annotation.Transactional
 import java.util.concurrent.Callable
+import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.Executors
 import java.util.concurrent.atomic.AtomicInteger
+import kotlin.time.measureTimedValue
 
 @Service
 class LibraryService(
     private val libraryRepository: LibraryRepository,
     private val filesystemService: FilesystemService,
     private val gameService: GameService,
-    private val imageService: ImageService,
+    private val imageService: ImageService
-    private val companyService: CompanyService
 ) {
 
     companion object {
@@ -124,29 +123,49 @@ class LibraryService(
      */
     fun addGamesToLibrary(games: Collection<Game>, library: Library): Library {
         val newGames = games.filter { game -> library.games.none { it.id == game.id } }
-        library.games = library.games.toMutableSet().apply { addAll(newGames) }
+        library.games.addAll(newGames)
-        return libraryRepository.save(library)
+        return library
     }
 
     /**
      * Wrapper function to trigger a scan for a list of libraries.
      */
-    fun triggerScan(libraryDtos: Collection<LibraryDto>?) = runBlocking {
+    fun triggerScan(scanType: ScanType, libraryDtos: Collection<LibraryDto>?) {
-        scan(libraryDtos)
+        val scanResult = measureTimedValue {
+            when (scanType) {
+                ScanType.QUICK -> quickScan(libraryDtos)
+                ScanType.FULL -> TODO()
+            }
+        }
+
+        log.info {
+            """
+            Scan completed in ${scanResult.duration}.
+            Libraries scanned: ${libraryDtos?.joinToString { it.name } ?: "all libraries"}
+            Scan type: ${scanType.toString().lowercase()}
+            New games added: ${scanResult.value.newGames.size}
+            Removed games: ${scanResult.value.removedGames.size}
+            New unmatched paths: ${scanResult.value.newUnmatchedPaths.size}
+        """.trimIndent()
+        }
     }
 
     /**
-     * Triggers a scan for a list of libraries.
+     * Triggers a quick scan for a list of libraries.
+     * A quick scan will only scan for new games and deleted games, but will not touch existing games.
      * If no list is provided, all libraries will be scanned.
      *
      * @param libraryDtos: List of LibraryDto objects to scan.
      */
-    @Transactional(timeout = Integer.MAX_VALUE)
+    fun quickScan(libraryDtos: Collection<LibraryDto>?): LibraryScanResult {
-    fun scan(libraryDtos: Collection<LibraryDto>?) {
         val libraries = libraryDtos?.map { toEntity(it) } ?: libraryRepository.findAll()
 
-        libraries.forEach { library ->
+        val scanResults: List<LibraryScanResult> = libraries.map { library ->
-            val gamePaths = filesystemService.scanLibraryForGamefiles(library)
+            val scanResult = filesystemService.scanLibraryForGamefiles(library)
+            val gamePaths = scanResult.newPaths
+            val removedGamePaths = scanResult.removedGamePaths.map { it.toString() }
+            val removedUnmatchedPaths = scanResult.removedUnmatchedPaths.map { it.toString() }
+
             val totalPaths = gamePaths.size
             val completedMetadata = AtomicInteger(0)
             val completedImageDownload = AtomicInteger(0)
@@ -154,22 +173,42 @@ class LibraryService(
             log.info { "Scanning library '${library.name}' with $totalPaths paths..." }
 
             // 1. Fetch metadata for each game
+            val newUnmatchedPaths = ConcurrentHashMap.newKeySet<String>()
+
             val metadataTasks = gamePaths.map { path ->
                 Callable<Game?> {
                     try {
                         val game = gameService.matchFromFile(path, library)
+
+                        if (game == null) {
+                            newUnmatchedPaths.add(path.toString())
+                            return@Callable null
+                        }
+
                         val progress = completedMetadata.incrementAndGet()
                         log.info { "${progress}/${totalPaths} metadata matched" }
-                        game
+
+                        return@Callable game
                     } catch (e: Exception) {
                         log.error(e) { "Error processing game: ${e.message}" }
-                        null
+                        newUnmatchedPaths.add(path.toString())
+
+                        return@Callable null
                     }
                 }
             }
 
+            // 1.1 Wait for all metadata tasks to complete
             val matchedGames = executor.invokeAll(metadataTasks).mapNotNull { it.get() }
 
+            // 1.2 Add unmatched paths to the library
+            library.unmatchedPaths.removeAll(removedUnmatchedPaths)
+            library.unmatchedPaths.addAll(newUnmatchedPaths)
+
+            // 1.3 Remove deleted games from the library
+            val removedGames = gameService.getAllByPaths(removedGamePaths)
+            library.games.removeAll(removedGames)
+
             // 2. Download all images
             val totalImages = matchedGames.count { it.coverImage != null } + matchedGames.sumOf { it.images.size }
 
@@ -198,13 +237,31 @@ class LibraryService(
 
             val gamesWithImages = executor.invokeAll(imageDownloadTasks).mapNotNull { it.get() }
 
-            // 3. Persist entities
+            // 3. Persist new games
             val persistedGames = gameService.create(gamesWithImages)
             log.info { "${persistedGames.size}/${totalPaths} saved to database" }
 
-            // 4. Add games to library
+            // 4. Add new games to library
             addGamesToLibrary(persistedGames, library)
-            log.info { "Scan finished, matched ${persistedGames.size} new games" }
+
+            // 5. Persist library
+            libraryRepository.save(library)
+
+            return LibraryScanResult(
+                libraries = setOf(library),
+                newGames = persistedGames.toSet(),
+                removedGames = removedGames.toSet(),
+                newUnmatchedPaths = newUnmatchedPaths
+            )
+        }
+
+        return scanResults.reduce { acc, scanResult ->
+            LibraryScanResult(
+                libraries = acc.libraries + scanResult.libraries,
+                newGames = acc.newGames + scanResult.newGames,
+                removedGames = acc.removedGames + scanResult.removedGames,
+                newUnmatchedPaths = acc.newUnmatchedPaths + scanResult.newUnmatchedPaths
+            )
         }
     }
 

gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/enums/ScanType.kt

@@ -0,0 +1,5 @@
+package de.grimsi.gameyfin.libraries.enums
+
+enum class ScanType {
+    QUICK, FULL
+}

gameyfin/src/main/kotlin/de/grimsi/gameyfin/users/preferences/UserPreference.kt

@@ -2,16 +2,13 @@ package de.grimsi.gameyfin.users.preferences
 
 import de.grimsi.gameyfin.core.security.EncryptionConverter
 import jakarta.persistence.*
-import jakarta.validation.constraints.NotNull
 import java.io.Serializable
 
 @Entity
 class UserPreference(
-    @NotNull
     @EmbeddedId
     val id: UserPreferenceKey,
 
-    @NotNull
     @Column(name = "`value`")
     @Convert(converter = EncryptionConverter::class)
     var value: String

plugins/steam/build.gradle.kts

@@ -1,4 +1,4 @@
-val ktor_version = "3.0.0"
+val ktor_version = "3.1.2"
 
 plugins {
     id("com.google.devtools.ksp")

plugins/steam/src/main/kotlin/de/grimsi/gameyfin/plugins/steam/SteamPlugin.kt

@@ -122,7 +122,7 @@ class SteamPlugin(wrapper: PluginWrapper) : GameyfinPlugin(wrapper) {
 
 
         /**
-         * Often titles on Steam copyright symbols which makes matching between different providers harder
+         * Often titles on Steam contain copyright symbols which makes matching between different providers harder
          * This method removes those symbols
          */
         private fun sanitizeTitle(originalTitle: String): String {