Implement "full" scan type (#633)

This commit is contained in:
Simon
2025-07-18 16:42:23 +02:00
committed by GitHub
parent 664d47d1d7
commit f759b0c947
20 changed files with 857 additions and 320 deletions
@@ -14,7 +14,7 @@ import {scanState} from "Frontend/state/ScanState";
import LibraryScanProgress from "Frontend/generated/org/gameyfin/app/libraries/dto/LibraryScanProgress"; import LibraryScanProgress from "Frontend/generated/org/gameyfin/app/libraries/dto/LibraryScanProgress";
import {libraryState} from "Frontend/state/LibraryState"; import {libraryState} from "Frontend/state/LibraryState";
import {Target} from "@phosphor-icons/react"; import {Target} from "@phosphor-icons/react";
import {timeBetween, timeUntil} from "Frontend/util/utils"; import {timeBetween, timeUntil, toTitleCase} from "Frontend/util/utils";
import LibraryScanStatus from "Frontend/generated/org/gameyfin/app/libraries/dto/LibraryScanStatus"; import LibraryScanStatus from "Frontend/generated/org/gameyfin/app/libraries/dto/LibraryScanStatus";
import {useEffect, useState} from "react"; import {useEffect, useState} from "react";
@@ -50,7 +50,7 @@ export default function ScanProgressPopover() {
</Button> </Button>
</PopoverTrigger> </PopoverTrigger>
<PopoverContent> <PopoverContent>
<div className="flex flex-col gap-2 m-2 w-96"> <div className="flex flex-col gap-2 m-2 min-w-96 w-fit">
{scans.length === 0 ? {scans.length === 0 ?
<p className="flex h-12 items-center justify-center text-sm text-default-500"> <p className="flex h-12 items-center justify-center text-sm text-default-500">
No scans in progress or in history. No scans in progress or in history.
@@ -60,7 +60,7 @@ export default function ScanProgressPopover() {
<div className="flex flex-col"> <div className="flex flex-col">
<div <div
className="flex flex-row justify-between items-center text-default-500 mb-1"> className="flex flex-row justify-between items-center text-default-500 mb-1">
<p>Scan for library&nbsp; <p>{toTitleCase(scan.type)} scan for library&nbsp;
<Link underline="always" <Link underline="always"
color="foreground" color="foreground"
size="sm" size="sm"
@@ -94,6 +94,7 @@ export default function ScanProgressPopover() {
: :
<p> <p>
{scan.result?.new} new /&nbsp; {scan.result?.new} new /&nbsp;
{(scan as any).result?.updated != null && `${(scan as any).result.updated} updated / `}
{scan.result?.removed} removed /&nbsp; {scan.result?.removed} removed /&nbsp;
{scan.result?.unmatched} unmatched&nbsp; {scan.result?.unmatched} unmatched&nbsp;
(in {timeBetween(scan.startedAt, scan.finishedAt!)}) (in {timeBetween(scan.startedAt, scan.finishedAt!)})
@@ -4,7 +4,7 @@ import GameDto from "Frontend/generated/org/gameyfin/app/games/dto/GameDto";
import React from "react"; import React from "react";
import {LibraryEndpoint} from "Frontend/generated/endpoints"; import {LibraryEndpoint} from "Frontend/generated/endpoints";
import {GameCover} from "Frontend/components/general/covers/GameCover"; import {GameCover} from "Frontend/components/general/covers/GameCover";
import {MagnifyingGlass, SlidersHorizontal} from "@phosphor-icons/react"; import {MagnifyingGlass, MagnifyingGlassPlus, SlidersHorizontal} from "@phosphor-icons/react";
import ScanType from "Frontend/generated/org/gameyfin/app/libraries/enums/ScanType"; import ScanType from "Frontend/generated/org/gameyfin/app/libraries/enums/ScanType";
import {useNavigate} from "react-router"; import {useNavigate} from "react-router";
import {useSnapshot} from "valtio/react"; import {useSnapshot} from "valtio/react";
@@ -27,8 +27,8 @@ export function LibraryOverviewCard({library}: LibraryOverviewCardProps) {
return games.slice(0, MAX_COVER_COUNT); return games.slice(0, MAX_COVER_COUNT);
} }
async function triggerScan() { async function triggerScan(scanType: ScanType) {
await LibraryEndpoint.triggerScan(ScanType.QUICK, [library]); await LibraryEndpoint.triggerScan(scanType, [library]);
} }
return ( return (
@@ -48,11 +48,16 @@ export function LibraryOverviewCard({library}: LibraryOverviewCardProps) {
<p className="absolute text-2xl font-bold">{library.name}</p> <p className="absolute text-2xl font-bold">{library.name}</p>
<div className="absolute right-0 top-0 flex flex-row"> <div className="absolute right-0 top-0 flex flex-row">
<Tooltip content="Scan library" placement="bottom" color="foreground"> <Tooltip content="Scan library (quick)" placement="bottom" color="foreground">
<Button isIconOnly variant="light" onPress={triggerScan}> <Button isIconOnly variant="light" onPress={() => triggerScan(ScanType.QUICK)}>
<MagnifyingGlass/> <MagnifyingGlass/>
</Button> </Button>
</Tooltip> </Tooltip>
<Tooltip content="Scan library (full)" placement="bottom" color="foreground">
<Button isIconOnly variant="light" onPress={() => triggerScan(ScanType.FULL)}>
<MagnifyingGlassPlus/>
</Button>
</Tooltip>
<Tooltip content="Configuration" placement="bottom" color="foreground"> <Tooltip content="Configuration" placement="bottom" color="foreground">
<Button isIconOnly variant="light" onPress={() => navigate('library/' + library.id)}> <Button isIconOnly variant="light" onPress={() => navigate('library/' + library.id)}>
<SlidersHorizontal/> <SlidersHorizontal/>
@@ -6,6 +6,7 @@ import jakarta.annotation.security.RolesAllowed
import org.gameyfin.app.core.Role import org.gameyfin.app.core.Role
import org.gameyfin.app.core.annotations.DynamicPublicAccess import org.gameyfin.app.core.annotations.DynamicPublicAccess
import org.gameyfin.app.games.dto.* import org.gameyfin.app.games.dto.*
import org.gameyfin.app.libraries.LibraryCoreService
import org.gameyfin.app.libraries.LibraryService import org.gameyfin.app.libraries.LibraryService
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import java.nio.file.Path import java.nio.file.Path
@@ -15,7 +16,8 @@ import java.nio.file.Path
@AnonymousAllowed @AnonymousAllowed
class GameEndpoint( class GameEndpoint(
private val gameService: GameService, private val gameService: GameService,
private val libraryService: LibraryService private val libraryService: LibraryService,
private val libraryCoreService: LibraryCoreService
) { ) {
fun subscribe(): Flux<List<GameEvent>> { fun subscribe(): Flux<List<GameEvent>> {
return GameService.subscribe() return GameService.subscribe()
@@ -24,11 +26,11 @@ class GameEndpoint(
fun getAll(): List<GameDto> = gameService.getAll() fun getAll(): List<GameDto> = gameService.getAll()
@RolesAllowed(Role.Names.ADMIN) @RolesAllowed(Role.Names.ADMIN)
fun updateGame(game: GameUpdateDto) = gameService.update(game) fun updateGame(game: GameUpdateDto) = gameService.edit(game)
@RolesAllowed(Role.Names.ADMIN) @RolesAllowed(Role.Names.ADMIN)
fun deleteGame(gameId: Long) { fun deleteGame(gameId: Long) {
libraryService.deleteGameFromLibrary(gameId) libraryCoreService.deleteGameFromLibrary(gameId)
gameService.delete(gameId) gameService.delete(gameId)
} }
@@ -42,7 +44,7 @@ class GameEndpoint(
val library = libraryService.getById(libraryId) val library = libraryService.getById(libraryId)
val game = gameService.matchManually(originalIds, Path.of(path), library, replaceGameId) val game = gameService.matchManually(originalIds, Path.of(path), library, replaceGameId)
if (game != null) { if (game != null) {
libraryService.addGamesToLibrary(listOf(game), library, true) libraryCoreService.addGamesToLibrary(listOf(game), library, true)
} }
} }
} }
@@ -126,7 +126,7 @@ class GameService(
return gameRepository.saveAll(gamesToBePersisted) return gameRepository.saveAll(gamesToBePersisted)
} }
fun update(gameUpdateDto: GameUpdateDto) { fun edit(gameUpdateDto: GameUpdateDto) {
val existingGame = gameRepository.findByIdOrNull(gameUpdateDto.id) val existingGame = gameRepository.findByIdOrNull(gameUpdateDto.id)
?: throw IllegalArgumentException("Game with ID $gameUpdateDto.id not found") ?: throw IllegalArgumentException("Game with ID $gameUpdateDto.id not found")
@@ -235,6 +235,209 @@ class GameService(
gameRepository.save(existingGame) gameRepository.save(existingGame)
} }
@Transactional
fun update(game: Game): Game? {
var wasGameUpdated = false
val game = getById(game.id!!)
val originalIds: Map<String, OriginalIdDto> = game.metadata.originalIds
.map { (provider, originalId) ->
val providerId = pluginManager.getExtensions(provider.pluginId).first()?.javaClass?.name ?: return null
val pluginId = provider.pluginId
val originalId = originalId
providerId to OriginalIdDto(pluginId, originalId)
}
.toMap()
val updatedGame = matchManually(
originalIds = originalIds,
path = Path.of(game.metadata.path),
library = game.library,
replaceGameId = game.id,
persist = false
)
if (updatedGame == null) {
log.warn { "Failed to update game with ID ${game.id}" }
return null
}
fun <T> updateField(
fieldName: String,
originalValue: T?,
updatedValue: T?,
setValue: (T?) -> Unit,
updatedFieldMetadata: GameFieldMetadata?
) {
// Hibernate collections are of type "PersistentBag" which does not implement equals() properly when comparing with ArrayList
fun areEqual(a: Any?, b: Any?): Boolean {
return when {
a is Collection<*> && b is Collection<*> -> a.toList() == b.toList()
else -> a == b
}
}
val fieldSource = game.metadata.fields[fieldName]?.source
if (updatedValue != null && fieldSource !is GameFieldUserSource && !areEqual(
originalValue,
updatedValue
) && updatedFieldMetadata != null
) {
setValue(updatedValue)
game.metadata.fields[fieldName] = updatedFieldMetadata
wasGameUpdated = true
}
}
// Title
updateField(
"title",
game.title,
updatedGame.title,
{ game.title = it },
updatedGame.metadata.fields["title"]
)
// Summary
updateField(
"summary",
game.summary,
updatedGame.summary,
{ game.summary = it },
updatedGame.metadata.fields["summary"]
)
// Release
updateField(
"release",
game.release,
updatedGame.release,
{ game.release = it },
updatedGame.metadata.fields["release"]
)
// User Rating
updateField(
"userRating",
game.userRating,
updatedGame.userRating,
{ game.userRating = it },
updatedGame.metadata.fields["userRating"]
)
// Critic Rating
updateField(
"criticRating",
game.criticRating,
updatedGame.criticRating,
{ game.criticRating = it },
updatedGame.metadata.fields["criticRating"]
)
// Cover Image
updateField(
"coverImage",
game.coverImage,
updatedGame.coverImage,
{ game.coverImage = it },
updatedGame.metadata.fields["coverImage"]
)
// Header Image
updateField(
"headerImage",
game.headerImage,
updatedGame.headerImage,
{ game.headerImage = it },
updatedGame.metadata.fields["headerImage"]
)
// Publishers
updateField(
"publishers",
game.publishers,
updatedGame.publishers,
{ game.publishers = it ?: emptyList() },
updatedGame.metadata.fields["publishers"]
)
// Developers
updateField(
"developers",
game.developers,
updatedGame.developers,
{ game.developers = it ?: emptyList() },
updatedGame.metadata.fields["developers"]
)
// Genres
updateField(
"genres",
game.genres,
updatedGame.genres,
{ game.genres = it ?: emptyList() },
updatedGame.metadata.fields["genres"]
)
// Themes
updateField(
"themes",
game.themes,
updatedGame.themes,
{ game.themes = it ?: emptyList() },
updatedGame.metadata.fields["themes"]
)
// Keywords
updateField(
"keywords",
game.keywords,
updatedGame.keywords,
{ game.keywords = it ?: emptyList() },
updatedGame.metadata.fields["keywords"]
)
// Features
updateField(
"features",
game.features,
updatedGame.features,
{ game.features = it ?: emptyList() },
updatedGame.metadata.fields["features"]
)
// Perspectives
updateField(
"perspectives",
game.perspectives,
updatedGame.perspectives,
{ game.perspectives = it ?: emptyList() },
updatedGame.metadata.fields["perspectives"]
)
// Images
updateField(
"images",
game.images,
updatedGame.images,
{ game.images = it ?: emptyList() },
updatedGame.metadata.fields["images"]
)
// Video URLs
updateField(
"videoUrls",
game.videoUrls,
updatedGame.videoUrls,
{ game.videoUrls = it ?: emptyList() },
updatedGame.metadata.fields["videoUrls"]
)
return if (wasGameUpdated) game else null
}
fun delete(gameId: Long) { fun delete(gameId: Long) {
gameRepository.deleteById(gameId) gameRepository.deleteById(gameId)
} }
@@ -346,7 +549,8 @@ class GameService(
originalIds: Map<String, OriginalIdDto>, originalIds: Map<String, OriginalIdDto>,
path: Path, path: Path,
library: Library, library: Library,
replaceGameId: Long? = null replaceGameId: Long? = null,
persist: Boolean = true
): Game? { ): Game? {
// Step 0: Query all metadata plugins for metadata on the provided originalIds // Step 0: Query all metadata plugins for metadata on the provided originalIds
val metadataResults = runBlocking { val metadataResults = runBlocking {
@@ -389,7 +593,7 @@ class GameService(
mergedGame.metadata.matchConfirmed = true mergedGame.metadata.matchConfirmed = true
// Step 6: Save the game // Step 6: Save the game
return create(mergedGame) return if (persist) create(mergedGame) else mergedGame
} }
fun matchFromFile(path: Path, library: Library): Game? { fun matchFromFile(path: Path, library: Library): Game? {
@@ -10,7 +10,13 @@ class Company(
var id: Long? = null, var id: Long? = null,
val name: String, val name: String,
val type: CompanyType val type: CompanyType
) ) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other !is Company) return false
return name == other.name && type == other.type
}
}
enum class CompanyType { enum class CompanyType {
DEVELOPER, DEVELOPER,
@@ -72,7 +72,7 @@ class Game(
var features: List<GameFeature> = emptyList(), var features: List<GameFeature> = emptyList(),
@ElementCollection(targetClass = PlayerPerspective::class) @ElementCollection(targetClass = PlayerPerspective::class)
var perspectives: List<PlayerPerspective>? = null, var perspectives: List<PlayerPerspective> = emptyList(),
@OneToMany(cascade = [CascadeType.ALL], orphanRemoval = true) @OneToMany(cascade = [CascadeType.ALL], orphanRemoval = true)
var images: List<Image> = emptyList(), var images: List<Image> = emptyList(),
@@ -11,9 +11,9 @@ class GameMetadata(
var fileSize: Long? = null, var fileSize: Long? = null,
@OneToMany(cascade = [CascadeType.ALL], orphanRemoval = true, fetch = FetchType.EAGER) @OneToMany(cascade = [CascadeType.ALL], orphanRemoval = true, fetch = FetchType.EAGER)
var fields: Map<String, GameFieldMetadata> = emptyMap(), var fields: MutableMap<String, GameFieldMetadata> = mutableMapOf(),
@ElementCollection @ElementCollection(fetch = FetchType.EAGER)
var originalIds: Map<PluginManagementEntry, String> = emptyMap(), var originalIds: Map<PluginManagementEntry, String> = emptyMap(),
var downloadCount: Int = 0, var downloadCount: Int = 0,
@@ -27,7 +27,13 @@ class Image(
@MimeType @MimeType
var mimeType: String? = null var mimeType: String? = null
) ) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other !is Image) return false
return originalUrl.toString() == other.originalUrl.toString()
}
}
enum class ImageType { enum class ImageType {
COVER, COVER,
@@ -0,0 +1,89 @@
package org.gameyfin.app.libraries
import org.gameyfin.app.games.GameService
import org.gameyfin.app.games.entities.Game
import org.gameyfin.app.libraries.dto.DirectoryMappingDto
import org.gameyfin.app.libraries.dto.LibraryDto
import org.gameyfin.app.libraries.dto.LibraryStatsDto
import org.springframework.data.repository.findByIdOrNull
import org.springframework.stereotype.Service
import java.time.Instant
/**
* Class for shared logic from LibraryService and LibraryScanService.
*/
@Service
class LibraryCoreService(
private val libraryRepository: LibraryRepository,
private val gameService: GameService,
) {
/**
* Adds a collection of games to the library.
*
* @param games: The collection of games to add.
* @param library: The library to add the games to.
* @return The updated library.
*/
fun addGamesToLibrary(games: Collection<Game>, library: Library, persist: Boolean = false): Library {
val newGames = games.filter { game -> library.games.none { it.id == game.id } }
library.games.addAll(newGames)
var removedAnyUnmatchedPaths = false
for (game in newGames) {
if (library.unmatchedPaths.contains(game.metadata.path)) {
library.unmatchedPaths.remove(game.metadata.path)
removedAnyUnmatchedPaths = true
}
}
if (removedAnyUnmatchedPaths || persist) {
library.updatedAt = Instant.now()
return libraryRepository.save(library)
}
return library
}
fun deleteGameFromLibrary(gameId: Long) {
val game = gameService.getById(gameId)
val library = game.library
library.games.removeIf { it.id == gameId }
library.unmatchedPaths.add(game.metadata.path)
library.updatedAt = Instant.now() // Force the EntityListener to trigger an update and update the timestamp
libraryRepository.save(library)
}
/**
* Converts a LibraryDto to a Library entity.
*
* @param library: The LibraryDto to convert.
* @return The converted Library entity.
*/
fun toEntity(library: LibraryDto): Library {
return libraryRepository.findByIdOrNull(library.id) ?: Library(
name = library.name,
directories = library.directories.distinctBy { it.internalPath }.map {
DirectoryMapping(internalPath = it.internalPath, externalPath = it.externalPath)
}.toMutableList(),
)
}
}
fun Library.toDto(): LibraryDto {
val statsDto = LibraryStatsDto(
gamesCount = this.games.size,
downloadedGamesCount = this.games.sumOf { it.metadata.downloadCount }
)
return LibraryDto(
id = this.id!!,
name = this.name,
directories = this.directories.map { DirectoryMappingDto(it.internalPath, it.externalPath) },
games = this.games.mapNotNull { it.id },
stats = statsDto,
unmatchedPaths = this.unmatchedPaths
)
}
@@ -20,22 +20,23 @@ import reactor.core.publisher.Flux
class LibraryEndpoint( class LibraryEndpoint(
private val libraryService: LibraryService, private val libraryService: LibraryService,
private val userService: UserService, private val userService: UserService,
private val libraryScanService: LibraryScanService,
) { ) {
fun subscribeToLibraryEvents(): Flux<List<LibraryEvent>> { fun subscribeToLibraryEvents(): Flux<List<LibraryEvent>> {
return LibraryService.subscribeToLibraryEvents() return LibraryService.subscribeToLibraryEvents()
} }
fun getAll() = libraryService.getAll() fun getAll() = libraryService.getAll()
fun subscribeToScanProgressEvents(): Flux<List<LibraryScanProgress>> { fun subscribeToScanProgressEvents(): Flux<List<LibraryScanProgress>> {
val user = userService.getCurrentUser() val user = userService.getCurrentUser()
return if (user.isAdmin()) LibraryService.subscribeToScanProgressEvents() return if (user.isAdmin()) LibraryScanService.subscribeToScanProgressEvents()
else Flux.empty() else Flux.empty()
} }
@RolesAllowed(Role.Names.ADMIN) @RolesAllowed(Role.Names.ADMIN)
fun triggerScan(scanType: ScanType = ScanType.QUICK, libraries: Collection<LibraryDto>?) = fun triggerScan(scanType: ScanType = ScanType.QUICK, libraries: Collection<LibraryDto>?) =
libraryService.triggerScan(scanType, libraries) libraryScanService.triggerScan(scanType, libraries)
@RolesAllowed(Role.Names.ADMIN) @RolesAllowed(Role.Names.ADMIN)
fun createLibrary(library: LibraryDto, scanAfterCreation: Boolean = true) = fun createLibrary(library: LibraryDto, scanAfterCreation: Boolean = true) =
@@ -1,7 +1,34 @@
package org.gameyfin.app.libraries package org.gameyfin.app.libraries
data class LibraryScanResult( interface LibraryScanResult {
val new: Int, /**
val removed: Int, * Number of new games found in the library.
*/
val new: Int
/**
* Number of games removed from the library.
*/
val removed: Int
/**
* Number of unmatched games that were not found in the library.
*/
val unmatched: Int val unmatched: Int
) }
data class QuickScanResult(
override val new: Int,
override val removed: Int,
override val unmatched: Int
) : LibraryScanResult
data class FullScanResult(
override val new: Int,
override val removed: Int,
override val unmatched: Int,
/**
* Number of games updated in the library.
*/
val updated: Int
) : LibraryScanResult
@@ -0,0 +1,437 @@
package org.gameyfin.app.libraries
import io.github.oshai.kotlinlogging.KotlinLogging
import org.gameyfin.app.core.filesystem.FilesystemService
import org.gameyfin.app.games.GameService
import org.gameyfin.app.games.entities.Game
import org.gameyfin.app.libraries.dto.LibraryDto
import org.gameyfin.app.libraries.dto.LibraryScanProgress
import org.gameyfin.app.libraries.dto.LibraryScanStatus
import org.gameyfin.app.libraries.dto.LibraryScanStep
import org.gameyfin.app.libraries.enums.ScanType
import org.gameyfin.app.libraries.scan.*
import org.gameyfin.app.media.ImageService
import org.springframework.stereotype.Service
import reactor.core.publisher.Flux
import reactor.core.publisher.Sinks
import java.nio.file.Path
import java.time.Instant
import java.util.concurrent.Callable
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import kotlin.time.Duration.Companion.hours
import kotlin.time.Duration.Companion.seconds
import kotlin.time.toJavaDuration
@Service
class LibraryScanService(
private val libraryRepository: LibraryRepository,
private val filesystemService: FilesystemService,
private val libraryCoreService: LibraryCoreService,
private val gameService: GameService,
private val imageService: ImageService,
) {
companion object {
private val log = KotlinLogging.logger {}
private val SCAN_RESULT_TTL = 24.hours.toJavaDuration()
private val scanProgressEvents = Sinks.many().replay().limit<LibraryScanProgress>(SCAN_RESULT_TTL)
fun subscribeToScanProgressEvents(): Flux<List<LibraryScanProgress>> {
log.debug { "New subscription for scanProgressEvents" }
return scanProgressEvents.asFlux()
.buffer(1.seconds.toJavaDuration())
.doOnSubscribe {
log.debug { "Subscriber added to scanProgressEvents [${scanProgressEvents.currentSubscriberCount()}]" }
}
.doFinally {
log.debug { "Subscriber removed from scanProgressEvents with signal type $it [${scanProgressEvents.currentSubscriberCount()}]" }
}
}
fun emit(scanProgressDto: LibraryScanProgress) {
scanProgressEvents.tryEmitNext(scanProgressDto)
}
private val executor = Executors.newVirtualThreadPerTaskExecutor()
private val scansInProgress = ConcurrentHashMap<Long, Boolean>()
}
/**
* Wrapper function to trigger a scan for a list of libraries.
*/
fun triggerScan(scanType: ScanType, libraryDtos: Collection<LibraryDto>?) {
val libraries = libraryDtos?.map { libraryCoreService.toEntity(it) } ?: libraryRepository.findAll()
libraries.forEach { library ->
val libraryId = library.id!!
if (scansInProgress.putIfAbsent(libraryId, true) == null) {
executor.submit {
try {
when (scanType) {
ScanType.QUICK -> quickScan(library)
ScanType.FULL -> fullScan(library)
}
} finally {
scansInProgress.remove(libraryId)
}
}
} else {
log.info { "Scan already in progress for library $libraryId, skipping." }
}
}
}
/**
* Triggers a quick scan for a list of libraries.
* A quick scan will only scan for new games and deleted games, but will not touch existing games.
* If no list is provided, all libraries will be scanned.
*
* @param libraryDtos: List of LibraryDto objects to scan.
*/
fun quickScan(libraryDtos: Collection<LibraryDto>?) {
val libraries = libraryDtos?.map { libraryCoreService.toEntity(it) } ?: libraryRepository.findAll()
libraries.forEach { executor.submit { quickScan(it) } }
}
/**
* Triggers a full scan for a list of libraries.
* A full scan will rescan all games in the library, including metadata and images.
* If no list is provided, all libraries will be scanned.
*
* @param libraryDtos: List of LibraryDto objects to scan.
*/
fun fullScan(libraryDtos: Collection<LibraryDto>?) {
val libraries = libraryDtos?.map { libraryCoreService.toEntity(it) } ?: libraryRepository.findAll()
libraries.forEach { executor.submit { fullScan(it) } }
}
private fun quickScan(library: Library) {
val progress = LibraryScanProgress(
libraryId = library.id!!,
type = ScanType.QUICK,
currentStep = LibraryScanStep(
description = "Scanning filesystem"
)
)
emit(progress)
val scanResult = filesystemService.scanLibraryForGamefiles(library)
val newPaths = scanResult.newPaths
val removedGamePaths = scanResult.removedGamePaths.map { it.toString() }
val removedUnmatchedPaths = scanResult.removedUnmatchedPaths.map { it.toString() }
progress.currentStep = LibraryScanStep(
description = "Matching new games",
current = 0,
total = newPaths.size
)
emit(progress)
// 1. Match new games
val (newUnmatchedPaths, matchedGames) = matchNewGames(library, newPaths, progress)
val (removedGames) = updateLibrary(
library,
removedUnmatchedPaths,
newUnmatchedPaths,
removedGamePaths
)
// 2. Download all images
val totalImages = matchedGames.count { it.coverImage != null } +
matchedGames.count { it.headerImage !== null } +
matchedGames.sumOf { it.images.size }
progress.currentStep = LibraryScanStep(
description = "Downloading images",
current = 0,
total = totalImages
)
emit(progress)
val (gamesWithImages) = downloadImages(matchedGames, progress)
// 3. Calculate game file sizes
progress.currentStep = LibraryScanStep(
description = "Calculating file sizes",
current = 0,
total = gamesWithImages.size
)
emit(progress)
val (gamesWithFileSizes) = calculateFileSizes(gamesWithImages, progress)
progress.currentStep = LibraryScanStep(
description = "Finishing up",
current = 0,
total = gamesWithFileSizes.size
)
emit(progress)
val (persistedGames) = finishScan(gamesWithFileSizes, library, progress)
progress.currentStep = LibraryScanStep(description = "Finished")
progress.finishedAt = Instant.now()
progress.status = LibraryScanStatus.COMPLETED
progress.result = QuickScanResult(
new = persistedGames.size,
removed = removedGames.size,
unmatched = newUnmatchedPaths.size
)
emit(progress)
}
private fun fullScan(library: Library) {
val progress = LibraryScanProgress(
libraryId = library.id!!,
type = ScanType.FULL,
currentStep = LibraryScanStep(
description = "Scanning filesystem"
)
)
emit(progress)
val scanResult = filesystemService.scanLibraryForGamefiles(library)
val newPaths = scanResult.newPaths
val removedGamePaths = scanResult.removedGamePaths.map { it.toString() }
val removedUnmatchedPaths = scanResult.removedUnmatchedPaths.map { it.toString() }
// 1. Update existing games
progress.currentStep = LibraryScanStep(
description = "Updating existing games",
current = 0,
total = library.games.size
)
emit(progress)
val (updatedGames) = updateExistingGames(library.games, progress)
// 2. Match new games
progress.currentStep = LibraryScanStep(
description = "Matching new games",
current = 0,
total = newPaths.size
)
emit(progress)
val (newUnmatchedPaths, newMatchedGames) = matchNewGames(library, newPaths, progress)
val (removedGames) = updateLibrary(
library,
removedUnmatchedPaths,
newUnmatchedPaths,
removedGamePaths
)
// 3. Download all images
val newAndUpdatedGames = newMatchedGames + updatedGames
val totalImages = newAndUpdatedGames.count { it.coverImage != null } +
newAndUpdatedGames.count { it.headerImage !== null } +
newAndUpdatedGames.sumOf { it.images.size }
progress.currentStep = LibraryScanStep(
description = "Downloading images",
current = 0,
total = totalImages
)
emit(progress)
val (gamesWithImages) = downloadImages(newAndUpdatedGames, progress)
// 4. Calculate game file sizes
progress.currentStep = LibraryScanStep(
description = "Calculating file sizes",
current = 0,
total = gamesWithImages.size
)
emit(progress)
val (gamesWithFileSizes) = calculateFileSizes(gamesWithImages, progress)
// 5. Finish scan
progress.currentStep = LibraryScanStep(
description = "Finishing up",
current = 0,
total = gamesWithFileSizes.size
)
emit(progress)
val (persistedGames) = finishScan(gamesWithFileSizes, library, progress)
// 6. Send final progress update
progress.currentStep = LibraryScanStep(description = "Finished")
progress.finishedAt = Instant.now()
progress.status = LibraryScanStatus.COMPLETED
progress.result = FullScanResult(
new = persistedGames.size,
removed = removedGames.size,
unmatched = newUnmatchedPaths.size,
updated = updatedGames.size
)
emit(progress)
}
private fun matchNewGames(
library: Library,
gamePaths: List<Path>,
progress: LibraryScanProgress
): MatchNewGamesResult {
val completedMetadata = AtomicInteger(0)
// 1. Fetch metadata for each game
val newUnmatchedPaths = ConcurrentHashMap.newKeySet<String>()
val metadataTasks = gamePaths.map { path ->
Callable<Game?> {
try {
val game = gameService.matchFromFile(path, library)
if (game == null) {
newUnmatchedPaths.add(path.toString())
return@Callable null
}
return@Callable game
} catch (e: Exception) {
log.error(e) { "Error processing game: ${e.message}" }
newUnmatchedPaths.add(path.toString())
return@Callable null
} finally {
progress.currentStep.current = completedMetadata.incrementAndGet()
emit(progress)
}
}
}
// 1.1 Wait for all metadata tasks to complete
val matchedGames = executor.invokeAll(metadataTasks).mapNotNull { it.get() }
return MatchNewGamesResult(
unmatchedPaths = newUnmatchedPaths.toList(),
matchedGames = matchedGames
)
}
private fun updateLibrary(
library: Library,
removedUnmatchedPaths: List<String>,
newUnmatchedPaths: List<String>,
removedGamePaths: List<String>
): UpdateLibraryResult {
// 1.2 Add unmatched paths to the library
library.unmatchedPaths.removeAll(removedUnmatchedPaths)
library.unmatchedPaths.addAll(newUnmatchedPaths)
// 1.3 Remove deleted games from the library
val removedGames = library.games.filter { removedGamePaths.contains(it.metadata.path) }
library.games.removeAll(removedGames)
return UpdateLibraryResult(removedGames = removedGames)
}
private fun downloadImages(games: List<Game>, progress: LibraryScanProgress): DownloadImagesResult {
val completedImageDownload = AtomicInteger(0)
val imageDownloadTasks = games.map { game ->
Callable<Game?> {
try {
game.coverImage?.let {
imageService.downloadIfNew(it)
completedImageDownload.andIncrement
}
game.headerImage?.let {
imageService.downloadIfNew(it)
completedImageDownload.andIncrement
}
game.images.map {
imageService.downloadIfNew(it)
completedImageDownload.andIncrement
}
game
} catch (e: Exception) {
log.error(e) { "Error downloading images for game: ${e.message}" }
null
} finally {
progress.currentStep.current = completedImageDownload.get()
emit(progress)
}
}
}
val gamesWithImages = executor.invokeAll(imageDownloadTasks).mapNotNull { it.get() }
return DownloadImagesResult(gamesWithImages = gamesWithImages)
}
private fun calculateFileSizes(games: List<Game>, progress: LibraryScanProgress): CalculateFilesizesResult {
val calculatedFileSize = AtomicInteger(0)
val calculateFileSizeTask = games.map { game ->
Callable {
game.metadata.path.let { path ->
val fileSize = filesystemService.calculateFileSize(path)
game.metadata.fileSize = fileSize
progress.currentStep.current = calculatedFileSize.incrementAndGet()
emit(progress)
game
}
}
}
val gamesWithFileSizes = executor.invokeAll(calculateFileSizeTask).map { it.get() }
return CalculateFilesizesResult(gamesWithFilesizes = gamesWithFileSizes)
}
private fun finishScan(games: List<Game>, library: Library, progress: LibraryScanProgress): FinishScanResult {
// 4. Persist new games
val persistedGames = gameService.create(games)
progress.currentStep.current = persistedGames.size
emit(progress)
// 5. Add new games to library
libraryCoreService.addGamesToLibrary(persistedGames, library)
// 6. Persist library
library.updatedAt = Instant.now() // Force the EntityListener to trigger an update and update the timestamp
libraryRepository.save(library)
return FinishScanResult(persistedGames = persistedGames)
}
private fun updateExistingGames(
games: List<Game>,
progress: LibraryScanProgress
): UpdateExistingGamesResult {
val completedUpdates = AtomicInteger(0)
val metadataTasks = games.map { game ->
Callable<Game?> {
try {
val game = gameService.update(game)
return@Callable game
} catch (e: Exception) {
log.error(e) { "Error updating game with id '${game.id}': ${e.message}" }
return@Callable null
} finally {
progress.currentStep.current = completedUpdates.incrementAndGet()
emit(progress)
}
}
}
val updatedGames = executor.invokeAll(metadataTasks).mapNotNull { it.get() }
return UpdateExistingGamesResult(updatedGames = updatedGames)
}
}
@@ -1,42 +1,32 @@
package org.gameyfin.app.libraries package org.gameyfin.app.libraries
import io.github.oshai.kotlinlogging.KotlinLogging import io.github.oshai.kotlinlogging.KotlinLogging
import org.gameyfin.app.core.filesystem.FilesystemService
import org.gameyfin.app.games.GameService import org.gameyfin.app.games.GameService
import org.gameyfin.app.games.entities.Game import org.gameyfin.app.libraries.dto.LibraryDto
import org.gameyfin.app.libraries.dto.* import org.gameyfin.app.libraries.dto.LibraryEvent
import org.gameyfin.app.libraries.dto.LibraryUpdateDto
import org.gameyfin.app.libraries.enums.ScanType import org.gameyfin.app.libraries.enums.ScanType
import org.gameyfin.app.media.ImageService
import org.springframework.data.repository.findByIdOrNull import org.springframework.data.repository.findByIdOrNull
import org.springframework.stereotype.Service import org.springframework.stereotype.Service
import reactor.core.publisher.Flux import reactor.core.publisher.Flux
import reactor.core.publisher.Sinks import reactor.core.publisher.Sinks
import java.time.Instant import java.time.Instant
import java.util.concurrent.Callable
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import kotlin.time.Duration.Companion.hours
import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.Duration.Companion.seconds
import kotlin.time.toJavaDuration import kotlin.time.toJavaDuration
@Service @Service
class LibraryService( class LibraryService(
private val libraryRepository: LibraryRepository, private val libraryRepository: LibraryRepository,
private val filesystemService: FilesystemService, private val libraryCoreService: LibraryCoreService,
private val libraryScanService: LibraryScanService,
private val gameService: GameService, private val gameService: GameService,
private val imageService: ImageService
) { ) {
companion object { companion object {
private val log = KotlinLogging.logger {} private val log = KotlinLogging.logger {}
private val executor = Executors.newVirtualThreadPerTaskExecutor()
private val SCAN_RESULT_TTL = 24.hours.toJavaDuration()
/* Websockets */ /* Websockets */
private val libraryEvents = Sinks.many().multicast().onBackpressureBuffer<LibraryEvent>(1024, false) private val libraryEvents = Sinks.many().multicast().onBackpressureBuffer<LibraryEvent>(1024, false)
private val scanProgressEvents = Sinks.many().replay().limit<LibraryScanProgress>(SCAN_RESULT_TTL)
fun subscribeToLibraryEvents(): Flux<List<LibraryEvent>> { fun subscribeToLibraryEvents(): Flux<List<LibraryEvent>> {
log.debug { "New subscription for libraryEvents" } log.debug { "New subscription for libraryEvents" }
@@ -50,25 +40,9 @@ class LibraryService(
} }
} }
fun subscribeToScanProgressEvents(): Flux<List<LibraryScanProgress>> {
log.debug { "New subscription for scanProgressEvents" }
return scanProgressEvents.asFlux()
.buffer(1.seconds.toJavaDuration())
.doOnSubscribe {
log.debug { "Subscriber added to scanProgressEvents [${scanProgressEvents.currentSubscriberCount()}]" }
}
.doFinally {
log.debug { "Subscriber removed from scanProgressEvents with signal type $it [${scanProgressEvents.currentSubscriberCount()}]" }
}
}
fun emit(event: LibraryEvent) { fun emit(event: LibraryEvent) {
libraryEvents.tryEmitNext(event) libraryEvents.tryEmitNext(event)
} }
fun emit(scanProgressDto: LibraryScanProgress) {
scanProgressEvents.tryEmitNext(scanProgressDto)
}
} }
@@ -96,10 +70,10 @@ class LibraryService(
* @return The created or updated LibraryDto object. * @return The created or updated LibraryDto object.
*/ */
fun create(library: LibraryDto, scanAfterCreation: Boolean) { fun create(library: LibraryDto, scanAfterCreation: Boolean) {
val newLibrary = libraryRepository.save(toEntity(library)) val newLibrary = libraryRepository.save(libraryCoreService.toEntity(library))
if (scanAfterCreation) { if (scanAfterCreation) {
triggerScanSingleLibrary(ScanType.QUICK, newLibrary) libraryScanService.triggerScan(ScanType.QUICK, listOf(newLibrary.toDto()))
} }
} }
@@ -139,264 +113,4 @@ class LibraryService(
fun delete(libraryId: Long) { fun delete(libraryId: Long) {
libraryRepository.deleteById(libraryId) libraryRepository.deleteById(libraryId)
} }
fun deleteGameFromLibrary(gameId: Long) {
val game = gameService.getById(gameId)
val library = game.library
library.games.removeIf { it.id == gameId }
library.unmatchedPaths.add(game.metadata.path)
library.updatedAt = Instant.now() // Force the EntityListener to trigger an update and update the timestamp
libraryRepository.save(library)
}
/**
* Wrapper function to trigger a scan for a list of libraries.
*/
fun triggerScan(scanType: ScanType, libraryDtos: Collection<LibraryDto>?) {
executor.submit {
when (scanType) {
ScanType.QUICK -> quickScan(libraryDtos)
ScanType.FULL -> TODO()
}
}
}
fun triggerScanSingleLibrary(scanType: ScanType, library: Library) {
triggerScan(scanType, listOf(library.toDto()))
}
/**
* Triggers a quick scan for a list of libraries.
* A quick scan will only scan for new games and deleted games, but will not touch existing games.
* If no list is provided, all libraries will be scanned.
*
* @param libraryDtos: List of LibraryDto objects to scan.
*/
fun quickScan(libraryDtos: Collection<LibraryDto>?) {
val libraries = libraryDtos?.map { toEntity(it) } ?: libraryRepository.findAll()
libraries.forEach { executor.submit { quickScan(it) } }
}
fun quickScan(library: Library) {
val progress = LibraryScanProgress(
libraryId = library.id!!,
currentStep = LibraryScanStep(
description = "Scanning filesystem"
)
)
emit(progress)
val scanResult = filesystemService.scanLibraryForGamefiles(library)
val gamePaths = scanResult.newPaths
val removedGamePaths = scanResult.removedGamePaths.map { it.toString() }
val removedUnmatchedPaths = scanResult.removedUnmatchedPaths.map { it.toString() }
val totalPaths = gamePaths.size
val completedMetadata = AtomicInteger(0)
val completedImageDownload = AtomicInteger(0)
val calculatedFileSize = AtomicInteger(0)
progress.currentStep = LibraryScanStep(
description = "Matching games",
current = 0,
total = totalPaths
)
emit(progress)
// 1. Fetch metadata for each game
val newUnmatchedPaths = ConcurrentHashMap.newKeySet<String>()
val metadataTasks = gamePaths.map { path ->
Callable<Game?> {
try {
val game = gameService.matchFromFile(path, library)
if (game == null) {
newUnmatchedPaths.add(path.toString())
return@Callable null
}
progress.currentStep.current = completedMetadata.incrementAndGet()
emit(progress)
return@Callable game
} catch (e: Exception) {
log.error(e) { "Error processing game: ${e.message}" }
newUnmatchedPaths.add(path.toString())
return@Callable null
}
}
}
// 1.1 Wait for all metadata tasks to complete
val matchedGames = executor.invokeAll(metadataTasks).mapNotNull { it.get() }
// 1.2 Add unmatched paths to the library
library.unmatchedPaths.removeAll(removedUnmatchedPaths)
library.unmatchedPaths.addAll(newUnmatchedPaths)
// 1.3 Remove deleted games from the library
val removedGames = library.games.filter { removedGamePaths.contains(it.metadata.path) }
library.games.removeAll(removedGames)
// 2. Download all images
val totalImages = matchedGames.count { it.coverImage != null } +
matchedGames.count { it.headerImage !== null } +
matchedGames.sumOf { it.images.size }
progress.currentStep = LibraryScanStep(
description = "Downloading images",
current = 0,
total = totalImages
)
emit(progress)
val imageDownloadTasks = matchedGames.map { game ->
Callable<Game?> {
try {
game.coverImage?.let {
imageService.downloadIfNew(it)
completedImageDownload.andIncrement
}
game.headerImage?.let {
imageService.downloadIfNew(it)
completedImageDownload.andIncrement
}
game.images.map {
imageService.downloadIfNew(it)
completedImageDownload.andIncrement
}
progress.currentStep.current = completedImageDownload.get()
emit(progress)
game
} catch (e: Exception) {
log.error(e) { "Error downloading images for game: ${e.message}" }
null
}
}
}
val gamesWithImages = executor.invokeAll(imageDownloadTasks).mapNotNull { it.get() }
// 3. Calculate game file sizes
progress.currentStep = LibraryScanStep(
description = "Calculating file sizes",
current = 0,
total = gamesWithImages.size
)
emit(progress)
val calculateFileSizeTask = gamesWithImages.map { game ->
Callable {
game.metadata.path.let { path ->
val fileSize = filesystemService.calculateFileSize(path)
game.metadata.fileSize = fileSize
progress.currentStep.current = calculatedFileSize.incrementAndGet()
emit(progress)
game
}
}
}
val gamesWithFileSizes = executor.invokeAll(calculateFileSizeTask).map { it.get() }
progress.currentStep = LibraryScanStep(
description = "Finishing up",
current = 0,
total = gamesWithFileSizes.size
)
emit(progress)
// 4. Persist new games
val persistedGames = gameService.create(gamesWithFileSizes)
progress.currentStep.current = persistedGames.size
emit(progress)
// 5. Add new games to library
addGamesToLibrary(persistedGames, library)
// 6. Persist library
library.updatedAt = Instant.now() // Force the EntityListener to trigger an update and update the timestamp
libraryRepository.save(library)
progress.currentStep = LibraryScanStep(description = "Finished")
progress.finishedAt = Instant.now()
progress.status = LibraryScanStatus.COMPLETED
progress.result = LibraryScanResult(
new = persistedGames.size,
removed = removedGames.size,
unmatched = newUnmatchedPaths.size
)
emit(progress)
}
/**
* Adds a collection of games to the library.
*
* @param games: The collection of games to add.
* @param library: The library to add the games to.
* @return The updated library.
*/
fun addGamesToLibrary(games: Collection<Game>, library: Library, persist: Boolean = false): Library {
val newGames = games.filter { game -> library.games.none { it.id == game.id } }
library.games.addAll(newGames)
var removedAnyUnmatchedPaths = false
for (game in newGames) {
if (library.unmatchedPaths.contains(game.metadata.path)) {
library.unmatchedPaths.remove(game.metadata.path)
removedAnyUnmatchedPaths = true
}
}
if (removedAnyUnmatchedPaths || persist) {
library.updatedAt = Instant.now()
return libraryRepository.save(library)
}
return library
}
/**
* Converts a LibraryDto to a Library entity.
*
* @param library: The LibraryDto to convert.
* @return The converted Library entity.
*/
private fun toEntity(library: LibraryDto): Library {
return libraryRepository.findByIdOrNull(library.id) ?: Library(
name = library.name,
directories = library.directories.distinctBy { it.internalPath }.map {
DirectoryMapping(internalPath = it.internalPath, externalPath = it.externalPath)
}.toMutableList(),
)
}
}
fun Library.toDto(): LibraryDto {
val statsDto = LibraryStatsDto(
gamesCount = this.games.size,
downloadedGamesCount = this.games.sumOf { it.metadata.downloadCount }
)
return LibraryDto(
id = this.id!!,
name = this.name,
directories = this.directories.map { DirectoryMappingDto(it.internalPath, it.externalPath) },
games = this.games.mapNotNull { it.id },
stats = statsDto,
unmatchedPaths = this.unmatchedPaths
)
} }
@@ -1,12 +1,14 @@
package org.gameyfin.app.libraries.dto package org.gameyfin.app.libraries.dto
import org.gameyfin.app.libraries.LibraryScanResult import org.gameyfin.app.libraries.LibraryScanResult
import org.gameyfin.app.libraries.enums.ScanType
import java.time.Instant import java.time.Instant
import java.util.* import java.util.*
data class LibraryScanProgress( data class LibraryScanProgress(
val scanId: UUID = UUID.randomUUID(), val scanId: UUID = UUID.randomUUID(),
val libraryId: Long, val libraryId: Long,
val type: ScanType,
var status: LibraryScanStatus = LibraryScanStatus.IN_PROGRESS, var status: LibraryScanStatus = LibraryScanStatus.IN_PROGRESS,
var currentStep: LibraryScanStep, var currentStep: LibraryScanStep,
val startedAt: Instant = Instant.now(), val startedAt: Instant = Instant.now(),
@@ -0,0 +1,7 @@
package org.gameyfin.app.libraries.scan
import org.gameyfin.app.games.entities.Game
data class CalculateFilesizesResult(
val gamesWithFilesizes: List<Game>,
)
@@ -0,0 +1,7 @@
package org.gameyfin.app.libraries.scan
import org.gameyfin.app.games.entities.Game
data class DownloadImagesResult(
val gamesWithImages: List<Game>
)
@@ -0,0 +1,7 @@
package org.gameyfin.app.libraries.scan
import org.gameyfin.app.games.entities.Game
data class FinishScanResult(
val persistedGames: List<Game>,
)
@@ -0,0 +1,8 @@
package org.gameyfin.app.libraries.scan
import org.gameyfin.app.games.entities.Game
data class MatchNewGamesResult(
val unmatchedPaths: List<String>,
val matchedGames: List<Game>
)
@@ -0,0 +1,7 @@
package org.gameyfin.app.libraries.scan
import org.gameyfin.app.games.entities.Game
data class UpdateExistingGamesResult(
val updatedGames: List<Game>
)
@@ -0,0 +1,7 @@
package org.gameyfin.app.libraries.scan
import org.gameyfin.app.games.entities.Game
data class UpdateLibraryResult(
val removedGames: List<Game>
)