Implement scheduled library scans (closes #609) (#638)

First implementation of job system

app/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "gameyfin",
-  "version": "2.0.0.beta4",
+  "version": "2.0.0.beta6",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "gameyfin",
-      "version": "2.0.0.beta4",
+      "version": "2.0.0.beta6",
       "dependencies": {
         "@heroui/react": "2.7.9",
         "@material-tailwind/react": "^2.1.10",
@@ -33,7 +33,6 @@
         "@vaadin/vaadin-usage-statistics": "2.1.3",
         "classnames": "^2.5.1",
         "construct-style-sheets-polyfill": "3.1.0",
-        "cron-validator": "^1.3.1",
         "date-fns": "2.29.3",
         "formik": "^2.4.6",
         "framer-motion": "^12.5.0",
@@ -10300,12 +10299,6 @@
         "url": "https://opencollective.com/core-js"
       }
     },
-    "node_modules/cron-validator": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/cron-validator/-/cron-validator-1.3.1.tgz",
-      "integrity": "sha512-C1HsxuPCY/5opR55G5/WNzyEGDWFVG+6GLrA+fW/sCTcP6A6NTjUP2AK7B8n2PyFs90kDG2qzwm8LMheADku6A==",
-      "license": "MIT"
-    },
     "node_modules/cross-spawn": {
       "version": "7.0.6",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",

app/package.json

@@ -28,7 +28,6 @@
     "@vaadin/vaadin-usage-statistics": "2.1.3",
     "classnames": "^2.5.1",
     "construct-style-sheets-polyfill": "3.1.0",
-    "cron-validator": "^1.3.1",
     "date-fns": "2.29.3",
     "formik": "^2.4.6",
     "framer-motion": "^12.5.0",
@@ -123,7 +122,6 @@
     "@vaadin/hilla-lit-form": "$@vaadin/hilla-lit-form",
     "@vaadin/hilla-react-form": "$@vaadin/hilla-react-form",
     "@vaadin/hilla-react-signals": "$@vaadin/hilla-react-signals",
-    "cron-validator": "$cron-validator",
     "moment": "$moment",
     "moment-timezone": "$moment-timezone",
     "react-confetti-boom": "$react-confetti-boom",
@@ -265,6 +263,6 @@
       "workbox-precaching": "7.3.0"
     },
     "disableUsageStatistics": true,
-    "hash": "e964f24aa5152284476d9e99245c9b92f8e6f94274818ee46eec2a3f91e29fc6"
+    "hash": "962eccc3fa0735d5234901be4f9e384096113c45bec22564a53688096d62aef4"
   }
 }

app/src/main/frontend/App.tsx

@@ -1,6 +1,5 @@
 import {Outlet, useHref, useNavigate} from 'react-router';
 import "./main.css";
-import "Frontend/util/custom-validators";
 import {HeroUIProvider} from "@heroui/react";
 import {ThemeProvider as NextThemesProvider} from "next-themes";
 import {themeNames} from "Frontend/theming/themes";

app/src/main/frontend/components/administration/LibraryManagement.tsx

@@ -3,6 +3,7 @@ import ConfigFormField from "Frontend/components/administration/ConfigFormField"
 import withConfigPage from "Frontend/components/administration/withConfigPage";
 import Section from "Frontend/components/general/Section";
 import * as Yup from 'yup';
+import "Frontend/util/yup-extensions";
 import {addToast, Button, Divider, Tooltip, useDisclosure} from "@heroui/react";
 import {Plus} from "@phosphor-icons/react";
 import {LibraryEndpoint} from "Frontend/generated/endpoints";
@@ -55,7 +56,7 @@ function LibraryManagementLayout({getConfig, formik}: any) {
 
             <Section title="Metadata"/>
             <div className="flex flex-row items-baseline">
-                <ConfigFormField configElement={getConfig("library.metadata.update.enabled")} isDisabled/>
+                <ConfigFormField configElement={getConfig("library.metadata.update.enabled")}/>
                 <ConfigFormField configElement={getConfig("library.metadata.update.schedule")}
                                  isDisabled={!formik.values.library.metadata.update.enabled}/>
             </div>
@@ -95,8 +96,11 @@ const validationSchema = Yup.object({
     library: Yup.object({
         metadata: Yup.object({
             update: Yup.object({
-                // @ts-ignore
+                enabled: Yup.boolean(),
-                schedule: Yup.string().cron()
+                schedule: Yup.string().when("enabled", {
+                    is: true,
+                    then: (schema) => schema.cron()
+                }),
             })
         }),
         scan: Yup.object({

app/src/main/frontend/util/custom-validators.ts

@@ -1,10 +0,0 @@
-import * as Yup from "yup";
-import {isValidCron} from "cron-validator";
-
-// Custom validator for cron expressions
-Yup.addMethod(Yup.string, 'cron', function (message) {
-    return this.test('cron', message, function (value) {
-        const {path, createError} = this;
-        return isValidCron(value as string) || createError({path, message: message || 'Invalid cron expression'});
-    });
-});

app/src/main/frontend/util/yup-extensions.ts

@@ -0,0 +1,22 @@
+import * as Yup from 'yup';
+import {ConfigEndpoint} from 'Frontend/generated/endpoints';
+
+Yup.addMethod(Yup.string, 'cron', function (message = 'Invalid cron expression') {
+    return this.test('cron', message, async function (value) {
+        const {path, createError} = this;
+        if (!value) return true;
+        try {
+            const isValid = await ConfigEndpoint.validateCronExpression(value);
+            return isValid || createError({path, message});
+        } catch (e) {
+            return createError({path, message: 'Error validating cron expression'});
+        }
+    });
+});
+
+// TypeScript: Extend Yup's type definitions
+declare module 'yup' {
+    interface StringSchema {
+        cron(message?: string): StringSchema;
+    }
+}

app/src/main/kotlin/org/gameyfin/app/config/ConfigEndpoint.kt

@@ -11,6 +11,7 @@ import org.gameyfin.app.core.Role
 import org.gameyfin.app.core.annotations.DynamicPublicAccess
 import org.gameyfin.app.users.UserService
 import org.gameyfin.app.users.util.isAdmin
+import org.springframework.scheduling.support.CronExpression
 import reactor.core.publisher.Flux
 
 @Endpoint
@@ -36,7 +37,15 @@ class ConfigEndpoint(
 
     fun update(update: ConfigUpdateDto) = configService.update(update)
 
-    /** Specific read-only endpoint for all users **/
+    /**
+     * Validates a cron expression because Spring has a custom syntax for cron expressions.
+     * @see: https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/scheduling/support/CronExpression.html#parse(java.lang.String)
+     */
+    fun validateCronExpression(cronExpression: String): Boolean {
+        return CronExpression.isValidExpression(cronExpression)
+    }
+
+    /** Specific read-only endpoints for all users **/
 
     @DynamicPublicAccess
     @AnonymousAllowed
@@ -49,5 +58,4 @@ class ConfigEndpoint(
     @DynamicPublicAccess
     @AnonymousAllowed
     fun isPublicAccessEnabled(): Boolean = configService.get(ConfigProperties.Libraries.AllowPublicAccess) == true
-
 }

app/src/main/kotlin/org/gameyfin/app/config/ConfigProperties.kt

@@ -90,15 +90,15 @@ sealed class ConfigProperties<T : Serializable>(
             data object UpdateEnabled : ConfigProperties<Boolean>(
                 Boolean::class,
                 "library.metadata.update.enabled",
-                "Enable periodic refresh of video game metadata (coming soon™)",
+                "Enable periodic refresh of video game metadata",
-                false
+                true
             )
 
             data object UpdateSchedule : ConfigProperties<String>(
                 String::class,
                 "library.metadata.update.schedule",
-                "Schedule for periodic metadata refresh in cron format",
+                "Schedule for periodic metadata refresh in Spring cron format",
-                "0 0 * * 0"
+                "@daily"
             )
         }
     }

app/src/main/kotlin/org/gameyfin/app/config/dto/CronExpressionVerificationResultDto.kt

@@ -0,0 +1,11 @@
+package org.gameyfin.app.config.dto
+
+data class CronExpressionVerificationResultDto(
+    val valid: Boolean,
+    val errorMessage: String? = null
+) {
+    companion object {
+        val valid = CronExpressionVerificationResultDto(true)
+        fun invalid(errorMessage: String) = CronExpressionVerificationResultDto(false, errorMessage)
+    }
+}

app/src/main/kotlin/org/gameyfin/app/config/entities/ConfigEntry.kt

@@ -4,6 +4,7 @@ import jakarta.persistence.*
 import org.gameyfin.app.core.security.EncryptionConverter
 
 @Entity
+@EntityListeners(ConfigEntryEntityListener::class)
 @Table(name = "app_config")
 class ConfigEntry(
     @Id

app/src/main/kotlin/org/gameyfin/app/config/entities/ConfigEntryEntityListener.kt

@@ -0,0 +1,26 @@
+package org.gameyfin.app.config.entities
+
+import jakarta.persistence.PostPersist
+import jakarta.persistence.PostRemove
+import jakarta.persistence.PostUpdate
+import org.gameyfin.app.config.ConfigProperties
+import org.gameyfin.app.core.events.LibraryScanScheduleUpdatedEvent
+import org.gameyfin.app.util.EventPublisherHolder
+
+class ConfigEntryEntityListener {
+    @PostUpdate
+    @PostPersist
+    @PostRemove
+    fun process(configEntry: ConfigEntry) {
+        when (configEntry.key) {
+            in ConfigProperties.Libraries.Metadata.UpdateEnabled.key,
+            ConfigProperties.Libraries.Metadata.UpdateSchedule.key -> {
+                EventPublisherHolder.publish(LibraryScanScheduleUpdatedEvent(this))
+            }
+
+            ConfigProperties.Libraries.Scan.EnableFilesystemWatcher.key -> {
+                TODO()
+            }
+        }
+    }
+}

app/src/main/kotlin/org/gameyfin/app/core/events/Events.kt

@@ -23,6 +23,4 @@ class PasswordResetRequestEvent(source: Any, val token: Token<TokenType.Password
 
 class AccountDeletedEvent(source: Any, val user: User, val baseUrl: String) : ApplicationEvent(source)
 
-class GameRequestEvent(source: Any) : ApplicationEvent(source)
+class LibraryScanScheduleUpdatedEvent(source: Any) : ApplicationEvent(source)
-
-class GameRequestApprovalEvent(source: Any) : ApplicationEvent(source)

app/src/main/kotlin/org/gameyfin/app/core/jobs/Job.kt

@@ -0,0 +1,7 @@
+package org.gameyfin.app.core.jobs
+
+interface Job {
+    val name: String
+    fun run(): JobRunResult
+}
+

app/src/main/kotlin/org/gameyfin/app/core/jobs/JobRunResult.kt

@@ -0,0 +1,17 @@
+package org.gameyfin.app.core.jobs
+
+import jakarta.persistence.*
+import java.time.LocalDateTime
+
+@Entity
+data class JobRunResult(
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    val id: Long? = null,
+    val jobName: String,
+    val startedAt: LocalDateTime,
+    val finishedAt: LocalDateTime?,
+    @Enumerated(EnumType.STRING)
+    val status: JobRunStatus,
+    val message: String?
+)

app/src/main/kotlin/org/gameyfin/app/core/jobs/JobRunResultRepository.kt

@@ -0,0 +1,8 @@
+package org.gameyfin.app.core.jobs
+
+import org.springframework.data.jpa.repository.JpaRepository
+import org.springframework.stereotype.Repository
+
+@Repository
+interface JobRunResultRepository : JpaRepository<JobRunResult, Long>
+

app/src/main/kotlin/org/gameyfin/app/core/jobs/JobRunStatus.kt

@@ -0,0 +1,9 @@
+package org.gameyfin.app.core.jobs
+
+enum class JobRunStatus {
+    IN_PROGRESS,
+    SUCCESS,
+    FAILED,
+    CANCELLED
+}
+

app/src/main/kotlin/org/gameyfin/app/core/jobs/JobService.kt

@@ -0,0 +1,74 @@
+package org.gameyfin.app.core.jobs
+
+import com.vaadin.hilla.exception.EndpointException
+import io.github.oshai.kotlinlogging.KotlinLogging
+import jakarta.annotation.PostConstruct
+import org.gameyfin.app.config.ConfigProperties
+import org.gameyfin.app.config.ConfigService
+import org.gameyfin.app.core.events.LibraryScanScheduleUpdatedEvent
+import org.gameyfin.app.libraries.LibraryScanService
+import org.springframework.context.event.EventListener
+import org.springframework.scheduling.TaskScheduler
+import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler
+import org.springframework.scheduling.support.CronExpression
+import org.springframework.scheduling.support.CronTrigger
+import org.springframework.stereotype.Service
+import java.time.LocalDateTime
+import java.util.concurrent.ScheduledFuture
+
+@Service
+class JobService(
+    private val config: ConfigService,
+    libraryScanService: LibraryScanService,
+    private val jobRunResultRepository: JobRunResultRepository
+) {
+    private val scheduler: TaskScheduler = ThreadPoolTaskScheduler().apply { initialize() }
+    private var libraryScanFuture: ScheduledFuture<*>? = null
+
+    private val libraryScanJob: Job = LibraryScanJob(libraryScanService)
+
+    companion object {
+        private val log = KotlinLogging.logger { }
+    }
+
+    @PostConstruct
+    fun init() {
+        scheduleLibraryScanJob()
+    }
+
+    @EventListener(LibraryScanScheduleUpdatedEvent::class)
+    fun onLibraryScanScheduleUpdated() {
+        scheduleLibraryScanJob()
+    }
+
+    private fun scheduleLibraryScanJob() {
+        libraryScanFuture?.cancel(false)
+
+        if (config.get(ConfigProperties.Libraries.Metadata.UpdateEnabled) != true) {
+            log.debug { "Disabled scheduled library scans" }
+            return
+        }
+
+        val cronExpressionString = config.get(ConfigProperties.Libraries.Metadata.UpdateSchedule) ?: return
+
+        try {
+            val cronTrigger = CronTrigger(cronExpressionString)
+            libraryScanFuture = (scheduler as ThreadPoolTaskScheduler).schedule({
+                runAndPersistJob(libraryScanJob)
+            }, cronTrigger)
+            log.debug {
+                "Library scan job scheduled, next run will be @ " +
+                        "${CronExpression.parse(cronExpressionString).next(LocalDateTime.now())}"
+            }
+        } catch (e: Exception) {
+            log.error { "Failed to schedule library scan job: ${e.message}" }
+            log.debug(e) { }
+            throw EndpointException("Failed to schedule library scan job: ${e.message}")
+        }
+    }
+
+    private fun runAndPersistJob(job: Job) {
+        val result = job.run()
+        jobRunResultRepository.save(result)
+    }
+}

app/src/main/kotlin/org/gameyfin/app/core/jobs/LibraryScanJob.kt

@@ -0,0 +1,33 @@
+package org.gameyfin.app.core.jobs
+
+import org.gameyfin.app.libraries.LibraryScanService
+import org.gameyfin.app.libraries.enums.ScanType
+import java.time.LocalDateTime
+
+class LibraryScanJob(private val libraryScanService: LibraryScanService) : Job {
+    override val name: String = "LibraryScan"
+    override fun run(): JobRunResult {
+        val startedAt = LocalDateTime.now()
+        var finishedAt: LocalDateTime?
+        var status: JobRunStatus
+        var message: String?
+        try {
+            libraryScanService.triggerScan(ScanType.SCHEDULED, null)
+            message = "Library scan completed successfully"
+            status = JobRunStatus.SUCCESS
+        } catch (ex: Exception) {
+            message = ex.message
+            status = JobRunStatus.FAILED
+        } finally {
+            finishedAt = LocalDateTime.now()
+        }
+        return JobRunResult(
+            jobName = name,
+            startedAt = startedAt,
+            finishedAt = finishedAt,
+            status = status,
+            message = message
+        )
+    }
+}
+

app/src/main/kotlin/org/gameyfin/app/libraries/LibraryScanService.kt

@@ -71,7 +71,8 @@ class LibraryScanService(
                     try {
                         when (scanType) {
                             ScanType.QUICK -> quickScan(library)
-                            ScanType.FULL -> fullScan(library)
+                            ScanType.FULL -> fullScan(library, false)
+                            ScanType.SCHEDULED -> fullScan(library, true)
                         }
                     } finally {
                         scansInProgress.remove(libraryId)
@@ -104,7 +105,7 @@ class LibraryScanService(
      */
     fun fullScan(libraryDtos: Collection<LibraryDto>?) {
         val libraries = libraryDtos?.map { libraryCoreService.toEntity(it) } ?: libraryRepository.findAll()
-        libraries.forEach { executor.submit { fullScan(it) } }
+        libraries.forEach { executor.submit { fullScan(it, false) } }
     }
 
     private fun quickScan(library: Library) {
@@ -191,10 +192,10 @@ class LibraryScanService(
         }
     }
 
-    private fun fullScan(library: Library) {
+    private fun fullScan(library: Library, triggeredBySchedule: Boolean) {
         val progress = LibraryScanProgress(
             libraryId = library.id!!,
-            type = ScanType.FULL,
+            type = if (triggeredBySchedule) ScanType.SCHEDULED else ScanType.FULL,
             currentStep = LibraryScanStep(
                 description = "Scanning filesystem"
             )

app/src/main/kotlin/org/gameyfin/app/libraries/enums/ScanType.kt

@@ -1,5 +1,7 @@
 package org.gameyfin.app.libraries.enums
 
 enum class ScanType {
-    QUICK, FULL
+    QUICK,
+    FULL,
+    SCHEDULED,
 }

app/src/main/kotlin/org/gameyfin/app/util/EventPublisherHolder.kt

@@ -0,0 +1,21 @@
+package org.gameyfin.app.util
+
+import org.springframework.context.ApplicationContext
+import org.springframework.context.ApplicationContextAware
+import org.springframework.context.ApplicationEvent
+import org.springframework.context.ApplicationEventPublisher
+import org.springframework.stereotype.Component
+
+@Component
+object EventPublisherHolder : ApplicationContextAware {
+    private var publisher: ApplicationEventPublisher? = null
+
+    override fun setApplicationContext(context: ApplicationContext) {
+        publisher = context
+    }
+
+    fun publish(event: ApplicationEvent) {
+        publisher?.publishEvent(event)
+    }
+}
+