From df7e76aaf8ccc762e4d29a6c8a322e547b2422f6 Mon Sep 17 00:00:00 2001 From: grimsi <9295182+grimsi@users.noreply.github.com> Date: Tue, 6 May 2025 13:11:46 +0200 Subject: [PATCH] Refactor library scanning, enable multithreading using virtual threads --- gameyfin/build.gradle.kts | 2 +- .../grimsi/gameyfin/games/CompanyService.kt | 17 ++++ .../de/grimsi/gameyfin/games/GameService.kt | 99 ++++++++++--------- .../de/grimsi/gameyfin/games/dto/GameDto.kt | 1 + .../de/grimsi/gameyfin/games/entities/Game.kt | 25 +++-- .../gameyfin/libraries/LibraryService.kt | 79 +++++++++++++-- .../de/grimsi/gameyfin/media/ImageService.kt | 20 +++- .../de/grimsi/gameyfin/users/entities/User.kt | 5 +- .../de/grimsi/gameyfin/plugins/igdb/Mapper.kt | 3 +- 9 files changed, 178 insertions(+), 73 deletions(-) create mode 100644 gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/CompanyService.kt diff --git a/gameyfin/build.gradle.kts b/gameyfin/build.gradle.kts index 2b26e7d..7407235 100644 --- a/gameyfin/build.gradle.kts +++ b/gameyfin/build.gradle.kts @@ -53,7 +53,7 @@ dependencies { // Persistence & I/O implementation("org.springframework.boot:spring-boot-starter-data-jpa") - implementation("com.github.paulcwarren:spring-content-fs-boot-starter:3.0.15") + implementation("com.github.paulcwarren:spring-content-fs-boot-starter:3.0.17") implementation("commons-io:commons-io:2.18.0") // SSO diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/CompanyService.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/CompanyService.kt new file mode 100644 index 0000000..14bd41d --- /dev/null +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/CompanyService.kt @@ -0,0 +1,17 @@ +package de.grimsi.gameyfin.games + +import de.grimsi.gameyfin.games.entities.Company +import de.grimsi.gameyfin.games.repositories.CompanyRepository +import org.springframework.stereotype.Service + +@Service +class CompanyService( + private val companyRepository: CompanyRepository +) { + fun createOrGet(company: Company): Company { + companyRepository.findByNameAndType(company.name, company.type)?.let { return it } + + val company = Company(name = company.name, type = company.type) + return companyRepository.save(company) + } +} \ No newline at end of file diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt index c263b47..f45851c 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt @@ -7,10 +7,9 @@ import de.grimsi.gameyfin.core.plugins.management.PluginManagementService import de.grimsi.gameyfin.games.dto.GameDto import de.grimsi.gameyfin.games.dto.GameMetadataDto import de.grimsi.gameyfin.games.entities.* -import de.grimsi.gameyfin.games.repositories.CompanyRepository import de.grimsi.gameyfin.games.repositories.GameRepository -import de.grimsi.gameyfin.games.repositories.ImageContentStore -import de.grimsi.gameyfin.games.repositories.ImageRepository +import de.grimsi.gameyfin.libraries.Library +import de.grimsi.gameyfin.media.ImageService import de.grimsi.gameyfin.pluginapi.gamemetadata.GameMetadata import de.grimsi.gameyfin.pluginapi.gamemetadata.GameMetadataProvider import io.github.oshai.kotlinlogging.KotlinLogging @@ -22,8 +21,7 @@ import org.apache.commons.io.FilenameUtils import org.pf4j.PluginManager import org.springframework.data.repository.findByIdOrNull import org.springframework.stereotype.Service -import java.net.URI -import java.net.URLConnection +import org.springframework.transaction.annotation.Transactional import java.nio.file.Path @Service @@ -31,11 +29,12 @@ class GameService( private val pluginManager: PluginManager, private val pluginManagementService: PluginManagementService, private val gameRepository: GameRepository, - private val companyRepository: CompanyRepository, - private val imageRepository: ImageRepository, - private val imageContentStore: ImageContentStore + private val companyService: CompanyService, + private val imageService: ImageService ) { - private val log = KotlinLogging.logger {} + companion object { + private val log = KotlinLogging.logger {} + } private val metadataPlugins: List get() = pluginManager.getExtensions(GameMetadataProvider::class.java) @@ -45,7 +44,21 @@ class GameService( return gameRepository.save(game) } - fun createFromFile(path: Path): Game? { + @Transactional + fun create(games: Collection): Collection { + val gamesToBePersisted = games.filter { it.id == null } + + gamesToBePersisted.forEach { game -> + game.publishers = game.publishers.map { companyService.createOrGet(it) }.toSet() + game.developers = game.developers.map { companyService.createOrGet(it) }.toSet() + game + } + + return gameRepository.saveAll(gamesToBePersisted) + } + + @Transactional + fun matchFromFile(path: Path, library: Library): Game? { val query = FilenameUtils.removeExtension(path.fileName.toString()) // Step 0: Query all metadata plugins for metadata on the provided game title @@ -67,10 +80,10 @@ class GameService( } // Step 4: Merge results into a single Game entity - val mergedGame = mergeResults(sortedResults, path) + val mergedGame = mergeResults(sortedResults, path, library) // Step 5: Save the new game - return createOrUpdate(mergedGame) + return mergedGame } fun getAllGames(): Collection { @@ -123,7 +136,7 @@ class GameService( val availableTitles = results.map { it.value.title } val bestMatchingTitle = FuzzySearch.extractOne(originalQuery, availableTitles).string - log.info { "Best matching title: '$bestMatchingTitle' for '$originalQuery' determined from $availableTitles" } + log.debug { "Best matching title: '$bestMatchingTitle' for '$originalQuery' determined from $availableTitles" } return results.filter { it.value.title.alphaNumeric() == bestMatchingTitle.alphaNumeric() } } @@ -133,8 +146,12 @@ class GameService( * The merging is done by taking the first non-null value for each field * The plugin with the highest possible priority is used as the source for each field */ - private fun mergeResults(results: List>, path: Path): Game { - val mergedGame = Game(path = path.toString()) + private fun mergeResults( + results: List>, + path: Path, + library: Library + ): Game { + val mergedGame = Game(path = path.toString(), library = library) val metadataMap = mutableMapOf() val originalIdsMap = mutableMapOf() @@ -163,7 +180,7 @@ class GameService( } metadata.coverUrl?.let { coverUrl -> if (!metadataMap.containsKey("coverImage")) { - mergedGame.coverImage = downloadAndPersist(coverUrl, ImageType.COVER) + mergedGame.coverImage = Image(originalUrl = coverUrl.toURL(), type = ImageType.COVER) metadataMap["coverImage"] = FieldMetadata(sourcePlugin) } } @@ -188,14 +205,14 @@ class GameService( metadata.publishedBy?.takeIf { it.isNotEmpty() }?.let { publishedBy -> if (!metadataMap.containsKey("publishers")) { mergedGame.publishers = - publishedBy.map { name -> toEntity(name, CompanyType.PUBLISHER) }.toSet() + publishedBy.map { Company(name = it, type = CompanyType.PUBLISHER) }.toSet() metadataMap["publishers"] = FieldMetadata(sourcePlugin) } } metadata.developedBy?.takeIf { it.isNotEmpty() }?.let { developedBy -> if (!metadataMap.containsKey("developers")) { mergedGame.developers = - developedBy.map { name -> toEntity(name, CompanyType.DEVELOPER) }.toSet() + developedBy.map { Company(name = it, type = CompanyType.DEVELOPER) }.toSet() metadataMap["developers"] = FieldMetadata(sourcePlugin) } } @@ -231,9 +248,9 @@ class GameService( } metadata.screenshotUrls?.takeIf { it.isNotEmpty() }?.let { screenshotUrls -> if (!metadataMap.containsKey("images")) { - mergedGame.images = - screenshotUrls.map { url -> downloadAndPersist(url, ImageType.SCREENSHOT) }.toSet() - metadataMap["images"] = FieldMetadata(sourcePlugin) + mergedGame.images = runBlocking { + screenshotUrls.map { Image(originalUrl = it.toURL(), type = ImageType.SCREENSHOT) }.toSet() + } } } metadata.videoUrls?.takeIf { it.isNotEmpty() }?.let { videoUrls -> @@ -247,30 +264,34 @@ class GameService( mergedGame.metadata = metadataMap mergedGame.originalIds = originalIdsMap + return mergedGame } fun toDto(game: Game): GameDto { val gameId = game.id ?: throw IllegalArgumentException("Game ID is null") + val gameLibraryId = game.library.id ?: throw IllegalArgumentException("Game library ID is null") + val gameTitle = game.title ?: throw IllegalArgumentException("Game title is null") return GameDto( id = gameId, - title = game.title!!, + libraryId = gameLibraryId, + title = gameTitle, coverId = game.coverImage?.id, comment = game.comment, summary = game.summary, release = game.release, userRating = game.userRating, criticRating = game.criticRating, - publishers = game.publishers?.map { it.name }, - developers = game.developers?.map { it.name }, - genres = game.genres?.map { it.name }, - themes = game.themes?.map { it.name }, - keywords = game.keywords?.toList(), - features = game.features?.map { it.name }, + publishers = game.publishers.map { it.name }, + developers = game.developers.map { it.name }, + genres = game.genres.map { it.name }, + themes = game.themes.map { it.name }, + keywords = game.keywords.toList(), + features = game.features.map { it.name }, perspectives = game.perspectives?.map { it.name }, - imageIds = game.images?.mapNotNull { it.id }, - videoUrls = game.videoUrls?.map { it.toString() }, + imageIds = game.images.mapNotNull { it.id }, + videoUrls = game.videoUrls.map { it.toString() }, path = game.path, metadata = toDto(game.metadata), originalIds = game.originalIds.mapKeys { it.key.pluginId } @@ -287,22 +308,4 @@ class GameService( lastUpdated = metadata.lastUpdated ) } - - private fun toEntity(companyName: String, companyType: CompanyType): Company { - companyRepository.findByNameAndType(companyName, companyType)?.let { return it } - val company = Company(name = companyName, type = companyType) - return companyRepository.save(company) - } - - private fun downloadAndPersist(imageUrl: URI, type: ImageType): Image { - val parsedUrl = imageUrl.toURL() - imageRepository.findByOriginalUrl(parsedUrl)?.let { return it } - - val image = Image(originalUrl = parsedUrl, type = type) - parsedUrl.openStream().use { input -> - image.mimeType = URLConnection.guessContentTypeFromName(parsedUrl.file) - imageContentStore.setContent(image, input) - } - return imageRepository.save(image) - } } \ No newline at end of file diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/dto/GameDto.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/dto/GameDto.kt index 08cd3c8..8c03036 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/dto/GameDto.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/dto/GameDto.kt @@ -4,6 +4,7 @@ import java.time.Instant class GameDto( val id: Long, + val libraryId: Long, val title: String, val coverId: Long?, val comment: String?, diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt index be87a3b..3ece2e9 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt @@ -1,6 +1,7 @@ package de.grimsi.gameyfin.games.entities import de.grimsi.gameyfin.core.plugins.management.PluginManagementEntry +import de.grimsi.gameyfin.libraries.Library import de.grimsi.gameyfin.pluginapi.gamemetadata.GameFeature import de.grimsi.gameyfin.pluginapi.gamemetadata.Genre import de.grimsi.gameyfin.pluginapi.gamemetadata.PlayerPerspective @@ -15,6 +16,10 @@ class Game( @GeneratedValue(strategy = GenerationType.AUTO) var id: Long? = null, + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "library_id") + val library: Library, + var title: String? = null, @OneToOne(cascade = [CascadeType.ALL], orphanRemoval = true) @@ -34,32 +39,32 @@ class Game( var criticRating: Int? = null, - @ManyToMany - var publishers: Set? = null, + @ManyToMany(cascade = [CascadeType.ALL]) + var publishers: Set = emptySet(), - @ManyToMany - var developers: Set? = null, + @ManyToMany(cascade = [CascadeType.ALL]) + var developers: Set = emptySet(), @ElementCollection(targetClass = Genre::class) - var genres: Set? = null, + var genres: Set = emptySet(), @ElementCollection(targetClass = Theme::class) - var themes: Set? = null, + var themes: Set = emptySet(), @ElementCollection - var keywords: Set? = null, + var keywords: Set = emptySet(), @ElementCollection(targetClass = GameFeature::class) - var features: Set? = null, + var features: Set = emptySet(), @ElementCollection(targetClass = PlayerPerspective::class) var perspectives: Set? = null, @OneToMany(cascade = [CascadeType.ALL], orphanRemoval = true) - var images: Set? = null, + var images: Set = emptySet(), @ElementCollection - var videoUrls: Set? = null, + var videoUrls: Set = emptySet(), @Column(unique = true) val path: String, diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt index 59ebe16..4b55d33 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt @@ -1,25 +1,36 @@ package de.grimsi.gameyfin.libraries import de.grimsi.gameyfin.core.filesystem.FilesystemService +import de.grimsi.gameyfin.games.CompanyService import de.grimsi.gameyfin.games.GameService import de.grimsi.gameyfin.games.dto.GameDto import de.grimsi.gameyfin.games.entities.Game import de.grimsi.gameyfin.libraries.dto.LibraryDto import de.grimsi.gameyfin.libraries.dto.LibraryStatsDto import de.grimsi.gameyfin.libraries.dto.LibraryUpdateDto +import de.grimsi.gameyfin.media.ImageService import io.github.oshai.kotlinlogging.KotlinLogging import kotlinx.coroutines.runBlocking import org.springframework.data.repository.findByIdOrNull import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Transactional +import java.util.concurrent.Callable +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger @Service class LibraryService( private val libraryRepository: LibraryRepository, + private val filesystemService: FilesystemService, private val gameService: GameService, - private val filesystemService: FilesystemService + private val imageService: ImageService, + private val companyService: CompanyService ) { - private val log = KotlinLogging.logger {} + companion object { + private val log = KotlinLogging.logger {} + private val executor = Executors.newVirtualThreadPerTaskExecutor() + } /** * Creates or updates a library in the repository. @@ -117,7 +128,6 @@ class LibraryService( return libraryRepository.save(library) } - /** * Wrapper function to trigger a scan for a list of libraries. */ @@ -131,15 +141,70 @@ class LibraryService( * * @param libraryDtos: List of LibraryDto objects to scan. */ - suspend fun scan(libraryDtos: Collection?) { + @Transactional(timeout = Integer.MAX_VALUE) + fun scan(libraryDtos: Collection?) { val libraries = libraryDtos?.map { toEntity(it) } ?: libraryRepository.findAll() + libraries.forEach { library -> val gamePaths = filesystemService.scanLibraryForGamefiles(library) - val newGames = gamePaths.mapNotNull { - gameService.createFromFile(it) + val totalPaths = gamePaths.size + val completedMetadata = AtomicInteger(0) + val completedImageDownload = AtomicInteger(0) + + log.info { "Scanning library '${library.name}' with $totalPaths paths..." } + + // 1. Fetch metadata for each game + val metadataTasks = gamePaths.map { path -> + Callable { + try { + val game = gameService.matchFromFile(path, library) + val progress = completedMetadata.incrementAndGet() + log.info { "${progress}/${totalPaths} metadata matched" } + game + } catch (e: Exception) { + log.error(e) { "Error processing game: ${e.message}" } + null + } + } } - addGamesToLibrary(newGames, library) + val matchedGames = executor.invokeAll(metadataTasks).mapNotNull { it.get() } + + // 2. Download all images + val totalImages = matchedGames.count { it.coverImage != null } + matchedGames.sumOf { it.images.size } + + val imageDownloadTasks = matchedGames.map { game -> + Callable { + try { + game.coverImage?.let { + imageService.downloadIfNew(it) + completedImageDownload.andIncrement + } + + game.images.map { + imageService.downloadIfNew(it) + completedImageDownload.andIncrement + } + + log.info { "${completedImageDownload}/${totalImages} images downloaded" } + + game + } catch (e: Exception) { + log.error(e) { "Error downloading images for game: ${e.message}" } + null + } + } + } + + val gamesWithImages = executor.invokeAll(imageDownloadTasks).mapNotNull { it.get() } + + // 3. Persist entities + val persistedGames = gameService.create(gamesWithImages) + log.info { "${persistedGames.size}/${totalPaths} saved to database" } + + // 4. Add games to library + addGamesToLibrary(persistedGames, library) + log.info { "Scan finished, matched ${persistedGames.size} new games" } } } diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/media/ImageService.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/media/ImageService.kt index c6977a9..d768f09 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/media/ImageService.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/media/ImageService.kt @@ -4,6 +4,8 @@ import de.grimsi.gameyfin.games.entities.Image import de.grimsi.gameyfin.games.entities.ImageType import de.grimsi.gameyfin.games.repositories.ImageContentStore import de.grimsi.gameyfin.games.repositories.ImageRepository +import org.apache.tika.Tika +import org.apache.tika.io.TikaInputStream import org.springframework.data.repository.findByIdOrNull import org.springframework.stereotype.Service import java.io.InputStream @@ -13,9 +15,19 @@ class ImageService( private val imageRepository: ImageRepository, private val imageContentStore: ImageContentStore ) { + companion object { + private val tika = Tika(); + } - fun getImage(id: Long): Image? { - return imageRepository.findByIdOrNull(id) + fun downloadIfNew(image: Image) { + if (image.originalUrl == null) throw IllegalArgumentException("Image must have an original URL") + + imageRepository.findByOriginalUrl(image.originalUrl)?.let { return } + + TikaInputStream.get { image.originalUrl.openStream() }.use { input -> + image.mimeType = tika.detect(input) + imageContentStore.setContent(image, input) + } } fun createFile(type: ImageType, content: InputStream, mimeType: String): Image { @@ -24,6 +36,10 @@ class ImageService( return imageContentStore.setContent(image, content) } + fun getImage(id: Long): Image? { + return imageRepository.findByIdOrNull(id) + } + fun getFileContent(id: Long): InputStream? { val image = getImage(id) ?: return null return getFileContent(image) diff --git a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/users/entities/User.kt b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/users/entities/User.kt index 952365f..7caba13 100644 --- a/gameyfin/src/main/kotlin/de/grimsi/gameyfin/users/entities/User.kt +++ b/gameyfin/src/main/kotlin/de/grimsi/gameyfin/users/entities/User.kt @@ -3,9 +3,7 @@ package de.grimsi.gameyfin.users.entities import de.grimsi.gameyfin.core.Role import de.grimsi.gameyfin.core.security.EncryptionConverter import de.grimsi.gameyfin.games.entities.Image -import jakarta.annotation.Nullable import jakarta.persistence.* -import jakarta.validation.constraints.NotNull import org.springframework.security.oauth2.core.oidc.user.OidcUser @@ -16,15 +14,14 @@ class User( @GeneratedValue(strategy = GenerationType.AUTO) var id: Long? = null, - @NotNull @Column(unique = true) + @Convert(converter = EncryptionConverter::class) var username: String, var password: String? = null, var oidcProviderId: String? = null, - @Nullable @Column(unique = true) @Convert(converter = EncryptionConverter::class) var email: String, diff --git a/plugins/igdb/src/main/kotlin/de/grimsi/gameyfin/plugins/igdb/Mapper.kt b/plugins/igdb/src/main/kotlin/de/grimsi/gameyfin/plugins/igdb/Mapper.kt index 7f2b446..baa71f0 100644 --- a/plugins/igdb/src/main/kotlin/de/grimsi/gameyfin/plugins/igdb/Mapper.kt +++ b/plugins/igdb/src/main/kotlin/de/grimsi/gameyfin/plugins/igdb/Mapper.kt @@ -109,7 +109,7 @@ class Mapper { } fun gameFeatures(game: proto.Game): Set { - var gameFeatures = mutableSetOf() + val gameFeatures = mutableSetOf() // Get LAN support from multiplayer modes if (game.multiplayerModesList.any { it.lancoop }) gameFeatures.add(GameFeature.LOCAL_MULTIPLAYER) @@ -119,6 +119,7 @@ class Mapper { "single-player" -> gameFeatures.add(GameFeature.SINGLEPLAYER) "multiplayer" -> gameFeatures.add(GameFeature.MULTIPLAYER) "massively-multiplayer-online-mmo" -> gameFeatures.add(GameFeature.MULTIPLAYER) + "battle-royale" -> gameFeatures.add(GameFeature.MULTIPLAYER) "co-operative" -> gameFeatures.add(GameFeature.CO_OP) "split-screen" -> gameFeatures.add(GameFeature.SPLITSCREEN) else -> {