From 4e3b6f71525c67f10c57e09a68c03db8b977fef5 Mon Sep 17 00:00:00 2001 From: grimsi <9295182+grimsi@users.noreply.github.com> Date: Wed, 7 May 2025 12:55:01 +0200 Subject: [PATCH] Implement detection of removed games & unmatched paths Implement persisting unmatched paths Various minor refactorings --- .../general/cards/LibraryOverviewCard.tsx | 4 +- .../gameyfin/config/entities/ConfigEntry.kt | 3 - .../core/filesystem/FilesystemScanResult.kt | 9 ++ .../core/filesystem/FilesystemService.kt | 34 ++++++- .../core/plugins/config/PluginConfigEntry.kt | 10 +- .../de/grimsi/gameyfin/games/GameService.kt | 4 + .../de/grimsi/gameyfin/games/entities/Game.kt | 4 +- .../grimsi/gameyfin/games/entities/Image.kt | 4 - .../games/repositories/GameRepository.kt | 1 + .../de/grimsi/gameyfin/libraries/Library.kt | 7 +- .../gameyfin/libraries/LibraryEndpoint.kt | 5 +- .../gameyfin/libraries/LibraryScanResult.kt | 10 ++ .../gameyfin/libraries/LibraryService.kt | 95 +++++++++++++++---- .../gameyfin/libraries/enums/ScanType.kt | 5 + .../users/preferences/UserPreference.kt | 3 - plugins/steam/build.gradle.kts | 2 +- .../gameyfin/plugins/steam/SteamPlugin.kt | 2 +- 17 files changed, 152 insertions(+), 50 deletions(-) create mode 100644 gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemScanResult.kt create mode 100644 gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryScanResult.kt create mode 100644 gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/enums/ScanType.kt diff --git a/gameyfin/src/main/frontend/components/general/cards/LibraryOverviewCard.tsx b/gameyfin/src/main/frontend/components/general/cards/LibraryOverviewCard.tsx index f21c17d..a1eff51 100644 --- a/gameyfin/src/main/frontend/components/general/cards/LibraryOverviewCard.tsx +++ b/gameyfin/src/main/frontend/components/general/cards/LibraryOverviewCard.tsx @@ -23,6 +23,7 @@ import { } from "@phosphor-icons/react"; import LibraryDetailsModal from "Frontend/components/general/modals/LibraryDetailsModal"; import LibraryUpdateDto from "Frontend/generated/de/grimsi/gameyfin/libraries/dto/LibraryUpdateDto"; +import ScanType from "Frontend/generated/de/grimsi/gameyfin/libraries/enums/ScanType"; export function LibraryOverviewCard({library, updateLibrary}: { library: LibraryDto, @@ -85,7 +86,8 @@ export function LibraryOverviewCard({library, updateLibrary}: {
- diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/config/entities/ConfigEntry.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/config/entities/ConfigEntry.kt index af0f7f3..dcf4217 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/config/entities/ConfigEntry.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/config/entities/ConfigEntry.kt @@ -2,17 +2,14 @@ package de.grimsi.gameyfin.config.entities import de.grimsi.gameyfin.core.security.EncryptionConverter import jakarta.persistence.* -import jakarta.validation.constraints.NotNull @Entity @Table(name = "app_config") class ConfigEntry( @Id - @NotNull @Column(name = "`key`", unique = true) val key: String, - @NotNull @Column(name = "`value`") @Convert(converter = EncryptionConverter::class) var value: String diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemScanResult.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemScanResult.kt new file mode 100644 index 0000000..279f053 --- /dev/null +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemScanResult.kt @@ -0,0 +1,9 @@ +package de.grimsi.gameyfin.core.filesystem + +import java.nio.file.Path + +data class FilesystemScanResult( + val newPaths: Set, + val removedGamePaths: Set, + val removedUnmatchedPaths: Set +) diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemService.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemService.kt index 4d42bfc..01f7ceb 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemService.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemService.kt @@ -75,7 +75,7 @@ class FilesystemService( * @param library The library to scan. * @return A list of paths representing game files and directories. */ - fun scanLibraryForGamefiles(library: Library): List { + fun scanLibraryForGamefiles(library: Library): FilesystemScanResult { // Cache the game file extensions to avoid reading them multiple times in the same scan val gamefileExtensions = gameFileExtensions @@ -90,11 +90,39 @@ class FilesystemService( } } - // Return all paths that are directories or match the game file extensions - return validDirectories.flatMap { validDirectory -> + // Get all paths that are directories or match the game file extensions + val currentFilesystemPaths = validDirectories.flatMap { validDirectory -> safeReadDirectoryContents(validDirectory) .filter { it.isDirectory() || it.extension.lowercase() in gamefileExtensions } } + + // Get all paths already in the library as game files or as unmatched paths + val currentLibraryGamePaths = library.games.map { Path(it.path) } + val currentLibraryUnmatchedPaths = library.unmatchedPaths.map { Path(it) } + val allCurrentLibraryPaths = currentLibraryGamePaths + currentLibraryUnmatchedPaths + + //Get all paths that are on the filesystem, but not in the library (either as game or as unmatched path) + val newPaths = currentFilesystemPaths.filter { path -> + val isInLibrary = allCurrentLibraryPaths.any { it == path } + !isInLibrary + }.toSet() + + //Get all paths that are in the library (either as game or as unmatched path), but not on the filesystem + val removedGamePaths = currentLibraryGamePaths.filter { path -> + val isOnFilesystem = currentFilesystemPaths.any { it == path } + !isOnFilesystem + }.toSet() + + val removedUnmatchedPaths = currentLibraryUnmatchedPaths.filter { path -> + val isOnFilesystem = currentFilesystemPaths.any { it == path } + !isOnFilesystem + }.toSet() + + return FilesystemScanResult( + newPaths = newPaths, + removedGamePaths = removedGamePaths, + removedUnmatchedPaths = removedUnmatchedPaths + ) } private fun safeReadDirectoryContents(path: String): List { diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/plugins/config/PluginConfigEntry.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/plugins/config/PluginConfigEntry.kt index a48dea6..8b28a1f 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/plugins/config/PluginConfigEntry.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/plugins/config/PluginConfigEntry.kt @@ -1,23 +1,15 @@ package de.grimsi.gameyfin.core.plugins.config import de.grimsi.gameyfin.core.security.EncryptionConverter -import jakarta.persistence.Column -import jakarta.persistence.Convert -import jakarta.persistence.Embeddable -import jakarta.persistence.EmbeddedId -import jakarta.persistence.Entity -import jakarta.persistence.Table -import jakarta.validation.constraints.NotNull +import jakarta.persistence.* import java.io.Serializable @Entity @Table(name = "plugin_config") data class PluginConfigEntry( - @NotNull @EmbeddedId val id: PluginConfigEntryKey, - @NotNull @Column(name = "`value`") @Convert(converter = EncryptionConverter::class) val value: String diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt index f45851c..3824324 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt @@ -86,6 +86,10 @@ class GameService( return mergedGame } + fun getAllByPaths(paths: Collection): Collection { + return gameRepository.findAllByPathIn(paths) + } + fun getAllGames(): Collection { val entities = gameRepository.findAll() return entities.map { toDto(it) } diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt index 3ece2e9..b474e18 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt @@ -39,10 +39,10 @@ class Game( var criticRating: Int? = null, - @ManyToMany(cascade = [CascadeType.ALL]) + @ManyToMany(cascade = [CascadeType.PERSIST, CascadeType.MERGE, CascadeType.REFRESH]) var publishers: Set = emptySet(), - @ManyToMany(cascade = [CascadeType.ALL]) + @ManyToMany(cascade = [CascadeType.PERSIST, CascadeType.MERGE, CascadeType.REFRESH]) var developers: Set = emptySet(), @ElementCollection(targetClass = Genre::class) diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Image.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Image.kt index f1056b5..a383a51 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Image.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Image.kt @@ -1,6 +1,5 @@ package de.grimsi.gameyfin.games.entities -import jakarta.annotation.Nullable import jakarta.persistence.Entity import jakarta.persistence.GeneratedValue import jakarta.persistence.GenerationType @@ -21,15 +20,12 @@ class Image( val type: ImageType, @ContentId - @Nullable var contentId: String? = null, @ContentLength - @Nullable var contentLength: Long? = null, @MimeType - @Nullable var mimeType: String? = null ) diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/repositories/GameRepository.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/repositories/GameRepository.kt index 91dc88d..e907ea8 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/repositories/GameRepository.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/repositories/GameRepository.kt @@ -5,4 +5,5 @@ import org.springframework.data.jpa.repository.JpaRepository interface GameRepository : JpaRepository { fun findByPath(path: String): Game? + fun findAllByPathIn(paths: Collection): Collection } \ No newline at end of file diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/Library.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/Library.kt index 5bd016e..3c1c137 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/Library.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/Library.kt @@ -14,6 +14,9 @@ class Library( @ElementCollection(fetch = FetchType.EAGER) var directories: MutableSet = HashSet(), - @ManyToMany(fetch = FetchType.EAGER) - var games: MutableSet = HashSet() + @OneToMany(fetch = FetchType.EAGER, orphanRemoval = true) + var games: MutableSet = HashSet(), + + @ElementCollection(fetch = FetchType.EAGER) + var unmatchedPaths: MutableSet = HashSet() ) \ No newline at end of file diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryEndpoint.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryEndpoint.kt index e8214b1..d466f25 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryEndpoint.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryEndpoint.kt @@ -6,6 +6,7 @@ import de.grimsi.gameyfin.games.GameService import de.grimsi.gameyfin.games.dto.GameDto import de.grimsi.gameyfin.libraries.dto.LibraryDto import de.grimsi.gameyfin.libraries.dto.LibraryUpdateDto +import de.grimsi.gameyfin.libraries.enums.ScanType import jakarta.annotation.security.PermitAll import jakarta.annotation.security.RolesAllowed @@ -24,8 +25,8 @@ class LibraryEndpoint( } @RolesAllowed(Role.Names.ADMIN) - fun triggerScan(libraries: Collection?) { - return libraryService.triggerScan(libraries) + fun triggerScan(scanType: ScanType = ScanType.QUICK, libraries: Collection?) { + return libraryService.triggerScan(scanType, libraries) } @RolesAllowed(Role.Names.ADMIN) diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryScanResult.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryScanResult.kt new file mode 100644 index 0000000..f41689e --- /dev/null +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryScanResult.kt @@ -0,0 +1,10 @@ +package de.grimsi.gameyfin.libraries + +import de.grimsi.gameyfin.games.entities.Game + +data class LibraryScanResult( + val libraries: Set, + val newGames: Set, + val removedGames: Set, + val newUnmatchedPaths: Set +) diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt index 4b55d33..4ff388c 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt @@ -1,30 +1,29 @@ package de.grimsi.gameyfin.libraries import de.grimsi.gameyfin.core.filesystem.FilesystemService -import de.grimsi.gameyfin.games.CompanyService import de.grimsi.gameyfin.games.GameService import de.grimsi.gameyfin.games.dto.GameDto import de.grimsi.gameyfin.games.entities.Game import de.grimsi.gameyfin.libraries.dto.LibraryDto import de.grimsi.gameyfin.libraries.dto.LibraryStatsDto import de.grimsi.gameyfin.libraries.dto.LibraryUpdateDto +import de.grimsi.gameyfin.libraries.enums.ScanType import de.grimsi.gameyfin.media.ImageService import io.github.oshai.kotlinlogging.KotlinLogging -import kotlinx.coroutines.runBlocking import org.springframework.data.repository.findByIdOrNull import org.springframework.stereotype.Service -import org.springframework.transaction.annotation.Transactional import java.util.concurrent.Callable +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger +import kotlin.time.measureTimedValue @Service class LibraryService( private val libraryRepository: LibraryRepository, private val filesystemService: FilesystemService, private val gameService: GameService, - private val imageService: ImageService, - private val companyService: CompanyService + private val imageService: ImageService ) { companion object { @@ -124,29 +123,49 @@ class LibraryService( */ fun addGamesToLibrary(games: Collection, library: Library): Library { val newGames = games.filter { game -> library.games.none { it.id == game.id } } - library.games = library.games.toMutableSet().apply { addAll(newGames) } - return libraryRepository.save(library) + library.games.addAll(newGames) + return library } /** * Wrapper function to trigger a scan for a list of libraries. */ - fun triggerScan(libraryDtos: Collection?) = runBlocking { - scan(libraryDtos) + fun triggerScan(scanType: ScanType, libraryDtos: Collection?) { + val scanResult = measureTimedValue { + when (scanType) { + ScanType.QUICK -> quickScan(libraryDtos) + ScanType.FULL -> TODO() + } + } + + log.info { + """ + Scan completed in ${scanResult.duration}. + Libraries scanned: ${libraryDtos?.joinToString { it.name } ?: "all libraries"} + Scan type: ${scanType.toString().lowercase()} + New games added: ${scanResult.value.newGames.size} + Removed games: ${scanResult.value.removedGames.size} + New unmatched paths: ${scanResult.value.newUnmatchedPaths.size} + """.trimIndent() + } } /** - * Triggers a scan for a list of libraries. + * Triggers a quick scan for a list of libraries. + * A quick scan will only scan for new games and deleted games, but will not touch existing games. * If no list is provided, all libraries will be scanned. * * @param libraryDtos: List of LibraryDto objects to scan. */ - @Transactional(timeout = Integer.MAX_VALUE) - fun scan(libraryDtos: Collection?) { + fun quickScan(libraryDtos: Collection?): LibraryScanResult { val libraries = libraryDtos?.map { toEntity(it) } ?: libraryRepository.findAll() - libraries.forEach { library -> - val gamePaths = filesystemService.scanLibraryForGamefiles(library) + val scanResults: List = libraries.map { library -> + val scanResult = filesystemService.scanLibraryForGamefiles(library) + val gamePaths = scanResult.newPaths + val removedGamePaths = scanResult.removedGamePaths.map { it.toString() } + val removedUnmatchedPaths = scanResult.removedUnmatchedPaths.map { it.toString() } + val totalPaths = gamePaths.size val completedMetadata = AtomicInteger(0) val completedImageDownload = AtomicInteger(0) @@ -154,22 +173,42 @@ class LibraryService( log.info { "Scanning library '${library.name}' with $totalPaths paths..." } // 1. Fetch metadata for each game + val newUnmatchedPaths = ConcurrentHashMap.newKeySet() + val metadataTasks = gamePaths.map { path -> Callable { try { val game = gameService.matchFromFile(path, library) + + if (game == null) { + newUnmatchedPaths.add(path.toString()) + return@Callable null + } + val progress = completedMetadata.incrementAndGet() log.info { "${progress}/${totalPaths} metadata matched" } - game + + return@Callable game } catch (e: Exception) { log.error(e) { "Error processing game: ${e.message}" } - null + newUnmatchedPaths.add(path.toString()) + + return@Callable null } } } + // 1.1 Wait for all metadata tasks to complete val matchedGames = executor.invokeAll(metadataTasks).mapNotNull { it.get() } + // 1.2 Add unmatched paths to the library + library.unmatchedPaths.removeAll(removedUnmatchedPaths) + library.unmatchedPaths.addAll(newUnmatchedPaths) + + // 1.3 Remove deleted games from the library + val removedGames = gameService.getAllByPaths(removedGamePaths) + library.games.removeAll(removedGames) + // 2. Download all images val totalImages = matchedGames.count { it.coverImage != null } + matchedGames.sumOf { it.images.size } @@ -198,13 +237,31 @@ class LibraryService( val gamesWithImages = executor.invokeAll(imageDownloadTasks).mapNotNull { it.get() } - // 3. Persist entities + // 3. Persist new games val persistedGames = gameService.create(gamesWithImages) log.info { "${persistedGames.size}/${totalPaths} saved to database" } - // 4. Add games to library + // 4. Add new games to library addGamesToLibrary(persistedGames, library) - log.info { "Scan finished, matched ${persistedGames.size} new games" } + + // 5. Persist library + libraryRepository.save(library) + + return LibraryScanResult( + libraries = setOf(library), + newGames = persistedGames.toSet(), + removedGames = removedGames.toSet(), + newUnmatchedPaths = newUnmatchedPaths + ) + } + + return scanResults.reduce { acc, scanResult -> + LibraryScanResult( + libraries = acc.libraries + scanResult.libraries, + newGames = acc.newGames + scanResult.newGames, + removedGames = acc.removedGames + scanResult.removedGames, + newUnmatchedPaths = acc.newUnmatchedPaths + scanResult.newUnmatchedPaths + ) } } diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/enums/ScanType.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/enums/ScanType.kt new file mode 100644 index 0000000..4e6b7da --- /dev/null +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/enums/ScanType.kt @@ -0,0 +1,5 @@ +package de.grimsi.gameyfin.libraries.enums + +enum class ScanType { + QUICK, FULL +} \ No newline at end of file diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/users/preferences/UserPreference.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/users/preferences/UserPreference.kt index 3e734bf..e3daf2a 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/users/preferences/UserPreference.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/users/preferences/UserPreference.kt @@ -2,16 +2,13 @@ package de.grimsi.gameyfin.users.preferences import de.grimsi.gameyfin.core.security.EncryptionConverter import jakarta.persistence.* -import jakarta.validation.constraints.NotNull import java.io.Serializable @Entity class UserPreference( - @NotNull @EmbeddedId val id: UserPreferenceKey, - @NotNull @Column(name = "`value`") @Convert(converter = EncryptionConverter::class) var value: String diff --git a/plugins/steam/build.gradle.kts b/plugins/steam/build.gradle.kts index 2157a92..9c19c9c 100644 --- a/plugins/steam/build.gradle.kts +++ b/plugins/steam/build.gradle.kts @@ -1,4 +1,4 @@ -val ktor_version = "3.0.0" +val ktor_version = "3.1.2" plugins { id("com.google.devtools.ksp") diff --git a/plugins/steam/src/main/kotlin/de/grimsi/gameyfin/plugins/steam/SteamPlugin.kt b/plugins/steam/src/main/kotlin/de/grimsi/gameyfin/plugins/steam/SteamPlugin.kt index c903cc3..393da2b 100644 --- a/plugins/steam/src/main/kotlin/de/grimsi/gameyfin/plugins/steam/SteamPlugin.kt +++ b/plugins/steam/src/main/kotlin/de/grimsi/gameyfin/plugins/steam/SteamPlugin.kt @@ -122,7 +122,7 @@ class SteamPlugin(wrapper: PluginWrapper) : GameyfinPlugin(wrapper) { /** - * Often titles on Steam copyright symbols which makes matching between different providers harder + * Often titles on Steam contain copyright symbols which makes matching between different providers harder * This method removes those symbols */ private fun sanitizeTitle(originalTitle: String): String {