Refactor library scanning, enable multithreading using virtual threads

gameyfin/build.gradle.kts

@@ -53,7 +53,7 @@ dependencies {
 
     // Persistence & I/O
     implementation("org.springframework.boot:spring-boot-starter-data-jpa")
-    implementation("com.github.paulcwarren:spring-content-fs-boot-starter:3.0.15")
+    implementation("com.github.paulcwarren:spring-content-fs-boot-starter:3.0.17")
     implementation("commons-io:commons-io:2.18.0")
 
     // SSO

gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/CompanyService.kt

@@ -0,0 +1,17 @@
+package de.grimsi.gameyfin.games
+
+import de.grimsi.gameyfin.games.entities.Company
+import de.grimsi.gameyfin.games.repositories.CompanyRepository
+import org.springframework.stereotype.Service
+
+@Service
+class CompanyService(
+    private val companyRepository: CompanyRepository
+) {
+    fun createOrGet(company: Company): Company {
+        companyRepository.findByNameAndType(company.name, company.type)?.let { return it }
+
+        val company = Company(name = company.name, type = company.type)
+        return companyRepository.save(company)
+    }
+}

gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt

@@ -7,10 +7,9 @@ import de.grimsi.gameyfin.core.plugins.management.PluginManagementService
 import de.grimsi.gameyfin.games.dto.GameDto
 import de.grimsi.gameyfin.games.dto.GameMetadataDto
 import de.grimsi.gameyfin.games.entities.*
-import de.grimsi.gameyfin.games.repositories.CompanyRepository
 import de.grimsi.gameyfin.games.repositories.GameRepository
-import de.grimsi.gameyfin.games.repositories.ImageContentStore
-import de.grimsi.gameyfin.games.repositories.ImageRepository
+import de.grimsi.gameyfin.libraries.Library
+import de.grimsi.gameyfin.media.ImageService
 import de.grimsi.gameyfin.pluginapi.gamemetadata.GameMetadata
 import de.grimsi.gameyfin.pluginapi.gamemetadata.GameMetadataProvider
 import io.github.oshai.kotlinlogging.KotlinLogging
@@ -22,8 +21,7 @@ import org.apache.commons.io.FilenameUtils
 import org.pf4j.PluginManager
 import org.springframework.data.repository.findByIdOrNull
 import org.springframework.stereotype.Service
-import java.net.URI
-import java.net.URLConnection
+import org.springframework.transaction.annotation.Transactional
 import java.nio.file.Path
 
 @Service
@@ -31,11 +29,12 @@ class GameService(
     private val pluginManager: PluginManager,
     private val pluginManagementService: PluginManagementService,
     private val gameRepository: GameRepository,
-    private val companyRepository: CompanyRepository,
-    private val imageRepository: ImageRepository,
-    private val imageContentStore: ImageContentStore
+    private val companyService: CompanyService,
+    private val imageService: ImageService
 ) {
-    private val log = KotlinLogging.logger {}
+    companion object {
+        private val log = KotlinLogging.logger {}
+    }
 
     private val metadataPlugins: List<GameMetadataProvider>
         get() = pluginManager.getExtensions(GameMetadataProvider::class.java)
@@ -45,7 +44,21 @@ class GameService(
         return gameRepository.save(game)
     }
 
-    fun createFromFile(path: Path): Game? {
+    @Transactional
+    fun create(games: Collection<Game>): Collection<Game> {
+        val gamesToBePersisted = games.filter { it.id == null }
+
+        gamesToBePersisted.forEach { game ->
+            game.publishers = game.publishers.map { companyService.createOrGet(it) }.toSet()
+            game.developers = game.developers.map { companyService.createOrGet(it) }.toSet()
+            game
+        }
+
+        return gameRepository.saveAll(gamesToBePersisted)
+    }
+
+    @Transactional
+    fun matchFromFile(path: Path, library: Library): Game? {
         val query = FilenameUtils.removeExtension(path.fileName.toString())
 
         // Step 0: Query all metadata plugins for metadata on the provided game title
@@ -67,10 +80,10 @@ class GameService(
         }
 
         // Step 4: Merge results into a single Game entity
-        val mergedGame = mergeResults(sortedResults, path)
+        val mergedGame = mergeResults(sortedResults, path, library)
 
         // Step 5: Save the new game
-        return createOrUpdate(mergedGame)
+        return mergedGame
     }
 
     fun getAllGames(): Collection<GameDto> {
@@ -123,7 +136,7 @@ class GameService(
         val availableTitles = results.map { it.value.title }
         val bestMatchingTitle = FuzzySearch.extractOne(originalQuery, availableTitles).string
 
-        log.info { "Best matching title: '$bestMatchingTitle' for '$originalQuery' determined from $availableTitles" }
+        log.debug { "Best matching title: '$bestMatchingTitle' for '$originalQuery' determined from $availableTitles" }
 
         return results.filter { it.value.title.alphaNumeric() == bestMatchingTitle.alphaNumeric() }
     }
@@ -133,8 +146,12 @@ class GameService(
      * The merging is done by taking the first non-null value for each field
      * The plugin with the highest possible priority is used as the source for each field
      */
-    private fun mergeResults(results: List<Map.Entry<GameMetadataProvider, GameMetadata?>>, path: Path): Game {
-        val mergedGame = Game(path = path.toString())
+    private fun mergeResults(
+        results: List<Map.Entry<GameMetadataProvider, GameMetadata?>>,
+        path: Path,
+        library: Library
+    ): Game {
+        val mergedGame = Game(path = path.toString(), library = library)
         val metadataMap = mutableMapOf<String, FieldMetadata>()
         val originalIdsMap = mutableMapOf<PluginManagementEntry, String>()
 
@@ -163,7 +180,7 @@ class GameService(
                 }
                 metadata.coverUrl?.let { coverUrl ->
                     if (!metadataMap.containsKey("coverImage")) {
-                        mergedGame.coverImage = downloadAndPersist(coverUrl, ImageType.COVER)
+                        mergedGame.coverImage = Image(originalUrl = coverUrl.toURL(), type = ImageType.COVER)
                         metadataMap["coverImage"] = FieldMetadata(sourcePlugin)
                     }
                 }
@@ -188,14 +205,14 @@ class GameService(
                 metadata.publishedBy?.takeIf { it.isNotEmpty() }?.let { publishedBy ->
                     if (!metadataMap.containsKey("publishers")) {
                         mergedGame.publishers =
-                            publishedBy.map { name -> toEntity(name, CompanyType.PUBLISHER) }.toSet()
+                            publishedBy.map { Company(name = it, type = CompanyType.PUBLISHER) }.toSet()
                         metadataMap["publishers"] = FieldMetadata(sourcePlugin)
                     }
                 }
                 metadata.developedBy?.takeIf { it.isNotEmpty() }?.let { developedBy ->
                     if (!metadataMap.containsKey("developers")) {
                         mergedGame.developers =
-                            developedBy.map { name -> toEntity(name, CompanyType.DEVELOPER) }.toSet()
+                            developedBy.map { Company(name = it, type = CompanyType.DEVELOPER) }.toSet()
                         metadataMap["developers"] = FieldMetadata(sourcePlugin)
                     }
                 }
@@ -231,9 +248,9 @@ class GameService(
                 }
                 metadata.screenshotUrls?.takeIf { it.isNotEmpty() }?.let { screenshotUrls ->
                     if (!metadataMap.containsKey("images")) {
-                        mergedGame.images =
-                            screenshotUrls.map { url -> downloadAndPersist(url, ImageType.SCREENSHOT) }.toSet()
-                        metadataMap["images"] = FieldMetadata(sourcePlugin)
+                        mergedGame.images = runBlocking {
+                            screenshotUrls.map { Image(originalUrl = it.toURL(), type = ImageType.SCREENSHOT) }.toSet()
+                        }
                     }
                 }
                 metadata.videoUrls?.takeIf { it.isNotEmpty() }?.let { videoUrls ->
@@ -247,30 +264,34 @@ class GameService(
 
         mergedGame.metadata = metadataMap
         mergedGame.originalIds = originalIdsMap
+
         return mergedGame
     }
 
     fun toDto(game: Game): GameDto {
         val gameId = game.id ?: throw IllegalArgumentException("Game ID is null")
+        val gameLibraryId = game.library.id ?: throw IllegalArgumentException("Game library ID is null")
+        val gameTitle = game.title ?: throw IllegalArgumentException("Game title is null")
 
         return GameDto(
             id = gameId,
-            title = game.title!!,
+            libraryId = gameLibraryId,
+            title = gameTitle,
             coverId = game.coverImage?.id,
             comment = game.comment,
             summary = game.summary,
             release = game.release,
             userRating = game.userRating,
             criticRating = game.criticRating,
-            publishers = game.publishers?.map { it.name },
-            developers = game.developers?.map { it.name },
-            genres = game.genres?.map { it.name },
-            themes = game.themes?.map { it.name },
-            keywords = game.keywords?.toList(),
-            features = game.features?.map { it.name },
+            publishers = game.publishers.map { it.name },
+            developers = game.developers.map { it.name },
+            genres = game.genres.map { it.name },
+            themes = game.themes.map { it.name },
+            keywords = game.keywords.toList(),
+            features = game.features.map { it.name },
             perspectives = game.perspectives?.map { it.name },
-            imageIds = game.images?.mapNotNull { it.id },
-            videoUrls = game.videoUrls?.map { it.toString() },
+            imageIds = game.images.mapNotNull { it.id },
+            videoUrls = game.videoUrls.map { it.toString() },
             path = game.path,
             metadata = toDto(game.metadata),
             originalIds = game.originalIds.mapKeys { it.key.pluginId }
@@ -287,22 +308,4 @@ class GameService(
             lastUpdated = metadata.lastUpdated
         )
     }
-
-    private fun toEntity(companyName: String, companyType: CompanyType): Company {
-        companyRepository.findByNameAndType(companyName, companyType)?.let { return it }
-        val company = Company(name = companyName, type = companyType)
-        return companyRepository.save(company)
-    }
-
-    private fun downloadAndPersist(imageUrl: URI, type: ImageType): Image {
-        val parsedUrl = imageUrl.toURL()
-        imageRepository.findByOriginalUrl(parsedUrl)?.let { return it }
-
-        val image = Image(originalUrl = parsedUrl, type = type)
-        parsedUrl.openStream().use { input ->
-            image.mimeType = URLConnection.guessContentTypeFromName(parsedUrl.file)
-            imageContentStore.setContent(image, input)
-        }
-        return imageRepository.save(image)
-    }
 }

gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/dto/GameDto.kt

@@ -4,6 +4,7 @@ import java.time.Instant
 
 class GameDto(
     val id: Long,
+    val libraryId: Long,
     val title: String,
     val coverId: Long?,
     val comment: String?,

gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt

@@ -1,6 +1,7 @@
 package de.grimsi.gameyfin.games.entities
 
 import de.grimsi.gameyfin.core.plugins.management.PluginManagementEntry
+import de.grimsi.gameyfin.libraries.Library
 import de.grimsi.gameyfin.pluginapi.gamemetadata.GameFeature
 import de.grimsi.gameyfin.pluginapi.gamemetadata.Genre
 import de.grimsi.gameyfin.pluginapi.gamemetadata.PlayerPerspective
@@ -15,6 +16,10 @@ class Game(
     @GeneratedValue(strategy = GenerationType.AUTO)
     var id: Long? = null,
 
+    @ManyToOne(fetch = FetchType.LAZY)
+    @JoinColumn(name = "library_id")
+    val library: Library,
+
     var title: String? = null,
 
     @OneToOne(cascade = [CascadeType.ALL], orphanRemoval = true)
@@ -34,32 +39,32 @@ class Game(
 
     var criticRating: Int? = null,
 
-    @ManyToMany
-    var publishers: Set<Company>? = null,
+    @ManyToMany(cascade = [CascadeType.ALL])
+    var publishers: Set<Company> = emptySet(),
 
-    @ManyToMany
-    var developers: Set<Company>? = null,
+    @ManyToMany(cascade = [CascadeType.ALL])
+    var developers: Set<Company> = emptySet(),
 
     @ElementCollection(targetClass = Genre::class)
-    var genres: Set<Genre>? = null,
+    var genres: Set<Genre> = emptySet(),
 
     @ElementCollection(targetClass = Theme::class)
-    var themes: Set<Theme>? = null,
+    var themes: Set<Theme> = emptySet(),
 
     @ElementCollection
-    var keywords: Set<String>? = null,
+    var keywords: Set<String> = emptySet(),
 
     @ElementCollection(targetClass = GameFeature::class)
-    var features: Set<GameFeature>? = null,
+    var features: Set<GameFeature> = emptySet(),
 
     @ElementCollection(targetClass = PlayerPerspective::class)
     var perspectives: Set<PlayerPerspective>? = null,
 
     @OneToMany(cascade = [CascadeType.ALL], orphanRemoval = true)
-    var images: Set<Image>? = null,
+    var images: Set<Image> = emptySet(),
 
     @ElementCollection
-    var videoUrls: Set<URI>? = null,
+    var videoUrls: Set<URI> = emptySet(),
 
     @Column(unique = true)
     val path: String,

gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt

@@ -1,25 +1,36 @@
 package de.grimsi.gameyfin.libraries
 
 import de.grimsi.gameyfin.core.filesystem.FilesystemService
+import de.grimsi.gameyfin.games.CompanyService
 import de.grimsi.gameyfin.games.GameService
 import de.grimsi.gameyfin.games.dto.GameDto
 import de.grimsi.gameyfin.games.entities.Game
 import de.grimsi.gameyfin.libraries.dto.LibraryDto
 import de.grimsi.gameyfin.libraries.dto.LibraryStatsDto
 import de.grimsi.gameyfin.libraries.dto.LibraryUpdateDto
+import de.grimsi.gameyfin.media.ImageService
 import io.github.oshai.kotlinlogging.KotlinLogging
 import kotlinx.coroutines.runBlocking
 import org.springframework.data.repository.findByIdOrNull
 import org.springframework.stereotype.Service
+import org.springframework.transaction.annotation.Transactional
+import java.util.concurrent.Callable
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicInteger
 
 @Service
 class LibraryService(
     private val libraryRepository: LibraryRepository,
+    private val filesystemService: FilesystemService,
     private val gameService: GameService,
-    private val filesystemService: FilesystemService
+    private val imageService: ImageService,
+    private val companyService: CompanyService
 ) {
 
-    private val log = KotlinLogging.logger {}
+    companion object {
+        private val log = KotlinLogging.logger {}
+        private val executor = Executors.newVirtualThreadPerTaskExecutor()
+    }
 
     /**
      * Creates or updates a library in the repository.
@@ -117,7 +128,6 @@ class LibraryService(
         return libraryRepository.save(library)
     }
 
-
     /**
      * Wrapper function to trigger a scan for a list of libraries.
      */
@@ -131,15 +141,70 @@ class LibraryService(
      *
      * @param libraryDtos: List of LibraryDto objects to scan.
      */
-    suspend fun scan(libraryDtos: Collection<LibraryDto>?) {
+    @Transactional(timeout = Integer.MAX_VALUE)
+    fun scan(libraryDtos: Collection<LibraryDto>?) {
         val libraries = libraryDtos?.map { toEntity(it) } ?: libraryRepository.findAll()
+
         libraries.forEach { library ->
             val gamePaths = filesystemService.scanLibraryForGamefiles(library)
-            val newGames = gamePaths.mapNotNull {
-                gameService.createFromFile(it)
+            val totalPaths = gamePaths.size
+            val completedMetadata = AtomicInteger(0)
+            val completedImageDownload = AtomicInteger(0)
+
+            log.info { "Scanning library '${library.name}' with $totalPaths paths..." }
+
+            // 1. Fetch metadata for each game
+            val metadataTasks = gamePaths.map { path ->
+                Callable<Game?> {
+                    try {
+                        val game = gameService.matchFromFile(path, library)
+                        val progress = completedMetadata.incrementAndGet()
+                        log.info { "${progress}/${totalPaths} metadata matched" }
+                        game
+                    } catch (e: Exception) {
+                        log.error(e) { "Error processing game: ${e.message}" }
+                        null
+                    }
+                }
             }
 
-            addGamesToLibrary(newGames, library)
+            val matchedGames = executor.invokeAll(metadataTasks).mapNotNull { it.get() }
+
+            // 2. Download all images
+            val totalImages = matchedGames.count { it.coverImage != null } + matchedGames.sumOf { it.images.size }
+
+            val imageDownloadTasks = matchedGames.map { game ->
+                Callable<Game?> {
+                    try {
+                        game.coverImage?.let {
+                            imageService.downloadIfNew(it)
+                            completedImageDownload.andIncrement
+                        }
+
+                        game.images.map {
+                            imageService.downloadIfNew(it)
+                            completedImageDownload.andIncrement
+                        }
+
+                        log.info { "${completedImageDownload}/${totalImages} images downloaded" }
+
+                        game
+                    } catch (e: Exception) {
+                        log.error(e) { "Error downloading images for game: ${e.message}" }
+                        null
+                    }
+                }
+            }
+
+            val gamesWithImages = executor.invokeAll(imageDownloadTasks).mapNotNull { it.get() }
+
+            // 3. Persist entities
+            val persistedGames = gameService.create(gamesWithImages)
+            log.info { "${persistedGames.size}/${totalPaths} saved to database" }
+
+            // 4. Add games to library
+            addGamesToLibrary(persistedGames, library)
+            log.info { "Scan finished, matched ${persistedGames.size} new games" }
         }
     }
 

gameyfin/src/main/kotlin/de/grimsi/gameyfin/media/ImageService.kt

@@ -4,6 +4,8 @@ import de.grimsi.gameyfin.games.entities.Image
 import de.grimsi.gameyfin.games.entities.ImageType
 import de.grimsi.gameyfin.games.repositories.ImageContentStore
 import de.grimsi.gameyfin.games.repositories.ImageRepository
+import org.apache.tika.Tika
+import org.apache.tika.io.TikaInputStream
 import org.springframework.data.repository.findByIdOrNull
 import org.springframework.stereotype.Service
 import java.io.InputStream
@@ -13,9 +15,19 @@ class ImageService(
     private val imageRepository: ImageRepository,
     private val imageContentStore: ImageContentStore
 ) {
+    companion object {
+        private val tika = Tika();
+    }
 
-    fun getImage(id: Long): Image? {
-        return imageRepository.findByIdOrNull(id)
+    fun downloadIfNew(image: Image) {
+        if (image.originalUrl == null) throw IllegalArgumentException("Image must have an original URL")
+
+        imageRepository.findByOriginalUrl(image.originalUrl)?.let { return }
+
+        TikaInputStream.get { image.originalUrl.openStream() }.use { input ->
+            image.mimeType = tika.detect(input)
+            imageContentStore.setContent(image, input)
+        }
     }
 
     fun createFile(type: ImageType, content: InputStream, mimeType: String): Image {
@@ -24,6 +36,10 @@ class ImageService(
         return imageContentStore.setContent(image, content)
     }
 
+    fun getImage(id: Long): Image? {
+        return imageRepository.findByIdOrNull(id)
+    }
+
     fun getFileContent(id: Long): InputStream? {
         val image = getImage(id) ?: return null
         return getFileContent(image)

gameyfin/src/main/kotlin/de/grimsi/gameyfin/users/entities/User.kt

@@ -3,9 +3,7 @@ package de.grimsi.gameyfin.users.entities
 import de.grimsi.gameyfin.core.Role
 import de.grimsi.gameyfin.core.security.EncryptionConverter
 import de.grimsi.gameyfin.games.entities.Image
-import jakarta.annotation.Nullable
 import jakarta.persistence.*
-import jakarta.validation.constraints.NotNull
 import org.springframework.security.oauth2.core.oidc.user.OidcUser
 
 
@@ -16,15 +14,14 @@ class User(
     @GeneratedValue(strategy = GenerationType.AUTO)
     var id: Long? = null,
 
-    @NotNull
     @Column(unique = true)
+    @Convert(converter = EncryptionConverter::class)
     var username: String,
 
     var password: String? = null,
 
     var oidcProviderId: String? = null,
 
-    @Nullable
     @Column(unique = true)
     @Convert(converter = EncryptionConverter::class)
     var email: String,

plugins/igdb/src/main/kotlin/de/grimsi/gameyfin/plugins/igdb/Mapper.kt

@@ -109,7 +109,7 @@ class Mapper {
         }
 
         fun gameFeatures(game: proto.Game): Set<GameFeature> {
-            var gameFeatures = mutableSetOf<GameFeature>()
+            val gameFeatures = mutableSetOf<GameFeature>()
 
             // Get LAN support from multiplayer modes
             if (game.multiplayerModesList.any { it.lancoop }) gameFeatures.add(GameFeature.LOCAL_MULTIPLAYER)
@@ -119,6 +119,7 @@ class Mapper {
                     "single-player" -> gameFeatures.add(GameFeature.SINGLEPLAYER)
                     "multiplayer" -> gameFeatures.add(GameFeature.MULTIPLAYER)
                     "massively-multiplayer-online-mmo" -> gameFeatures.add(GameFeature.MULTIPLAYER)
+                    "battle-royale" -> gameFeatures.add(GameFeature.MULTIPLAYER)
                     "co-operative" -> gameFeatures.add(GameFeature.CO_OP)
                     "split-screen" -> gameFeatures.add(GameFeature.SPLITSCREEN)
                     else -> {