From 3cecdd558aea7611049ffb2351b380f82b95b011 Mon Sep 17 00:00:00 2001 From: Simon <9295182+grimsi@users.noreply.github.com> Date: Sun, 20 Jul 2025 13:57:54 +0200 Subject: [PATCH] Implement scheduled library scans (closes #609) (#638) First implementation of job system --- app/package-lock.json | 11 +-- app/package.json | 6 +- app/src/main/frontend/App.tsx | 1 - .../administration/LibraryManagement.tsx | 10 ++- .../main/frontend/util/custom-validators.ts | 10 --- app/src/main/frontend/util/yup-extensions.ts | 22 ++++++ .../org/gameyfin/app/config/ConfigEndpoint.kt | 12 ++- .../gameyfin/app/config/ConfigProperties.kt | 8 +- .../CronExpressionVerificationResultDto.kt | 11 +++ .../app/config/entities/ConfigEntry.kt | 1 + .../entities/ConfigEntryEntityListener.kt | 26 +++++++ .../org/gameyfin/app/core/events/Events.kt | 4 +- .../kotlin/org/gameyfin/app/core/jobs/Job.kt | 7 ++ .../gameyfin/app/core/jobs/JobRunResult.kt | 17 +++++ .../app/core/jobs/JobRunResultRepository.kt | 8 ++ .../gameyfin/app/core/jobs/JobRunStatus.kt | 9 +++ .../org/gameyfin/app/core/jobs/JobService.kt | 74 +++++++++++++++++++ .../gameyfin/app/core/jobs/LibraryScanJob.kt | 33 +++++++++ .../app/libraries/LibraryScanService.kt | 9 ++- .../gameyfin/app/libraries/enums/ScanType.kt | 4 +- .../gameyfin/app/util/EventPublisherHolder.kt | 21 ++++++ 21 files changed, 263 insertions(+), 41 deletions(-) delete mode 100644 app/src/main/frontend/util/custom-validators.ts create mode 100644 app/src/main/frontend/util/yup-extensions.ts create mode 100644 app/src/main/kotlin/org/gameyfin/app/config/dto/CronExpressionVerificationResultDto.kt create mode 100644 app/src/main/kotlin/org/gameyfin/app/config/entities/ConfigEntryEntityListener.kt create mode 100644 app/src/main/kotlin/org/gameyfin/app/core/jobs/Job.kt create mode 100644 app/src/main/kotlin/org/gameyfin/app/core/jobs/JobRunResult.kt create mode 100644 app/src/main/kotlin/org/gameyfin/app/core/jobs/JobRunResultRepository.kt create mode 100644 app/src/main/kotlin/org/gameyfin/app/core/jobs/JobRunStatus.kt create mode 100644 app/src/main/kotlin/org/gameyfin/app/core/jobs/JobService.kt create mode 100644 app/src/main/kotlin/org/gameyfin/app/core/jobs/LibraryScanJob.kt create mode 100644 app/src/main/kotlin/org/gameyfin/app/util/EventPublisherHolder.kt diff --git a/app/package-lock.json b/app/package-lock.json index 65e5b10..ea0abeb 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -1,12 +1,12 @@ { "name": "gameyfin", - "version": "2.0.0.beta4", + "version": "2.0.0.beta6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "gameyfin", - "version": "2.0.0.beta4", + "version": "2.0.0.beta6", "dependencies": { "@heroui/react": "2.7.9", "@material-tailwind/react": "^2.1.10", @@ -33,7 +33,6 @@ "@vaadin/vaadin-usage-statistics": "2.1.3", "classnames": "^2.5.1", "construct-style-sheets-polyfill": "3.1.0", - "cron-validator": "^1.3.1", "date-fns": "2.29.3", "formik": "^2.4.6", "framer-motion": "^12.5.0", @@ -10300,12 +10299,6 @@ "url": "https://opencollective.com/core-js" } }, - "node_modules/cron-validator": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/cron-validator/-/cron-validator-1.3.1.tgz", - "integrity": "sha512-C1HsxuPCY/5opR55G5/WNzyEGDWFVG+6GLrA+fW/sCTcP6A6NTjUP2AK7B8n2PyFs90kDG2qzwm8LMheADku6A==", - "license": "MIT" - }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", diff --git a/app/package.json b/app/package.json index 6cedd03..783ecad 100644 --- a/app/package.json +++ b/app/package.json @@ -28,7 +28,6 @@ "@vaadin/vaadin-usage-statistics": "2.1.3", "classnames": "^2.5.1", "construct-style-sheets-polyfill": "3.1.0", - "cron-validator": "^1.3.1", "date-fns": "2.29.3", "formik": "^2.4.6", "framer-motion": "^12.5.0", @@ -123,7 +122,6 @@ "@vaadin/hilla-lit-form": "$@vaadin/hilla-lit-form", "@vaadin/hilla-react-form": "$@vaadin/hilla-react-form", "@vaadin/hilla-react-signals": "$@vaadin/hilla-react-signals", - "cron-validator": "$cron-validator", "moment": "$moment", "moment-timezone": "$moment-timezone", "react-confetti-boom": "$react-confetti-boom", @@ -265,6 +263,6 @@ "workbox-precaching": "7.3.0" }, "disableUsageStatistics": true, - "hash": "e964f24aa5152284476d9e99245c9b92f8e6f94274818ee46eec2a3f91e29fc6" + "hash": "962eccc3fa0735d5234901be4f9e384096113c45bec22564a53688096d62aef4" } -} +} \ No newline at end of file diff --git a/app/src/main/frontend/App.tsx b/app/src/main/frontend/App.tsx index f145d6d..1a7abf7 100644 --- a/app/src/main/frontend/App.tsx +++ b/app/src/main/frontend/App.tsx @@ -1,6 +1,5 @@ import {Outlet, useHref, useNavigate} from 'react-router'; import "./main.css"; -import "Frontend/util/custom-validators"; import {HeroUIProvider} from "@heroui/react"; import {ThemeProvider as NextThemesProvider} from "next-themes"; import {themeNames} from "Frontend/theming/themes"; diff --git a/app/src/main/frontend/components/administration/LibraryManagement.tsx b/app/src/main/frontend/components/administration/LibraryManagement.tsx index ee6485d..8cb7d66 100644 --- a/app/src/main/frontend/components/administration/LibraryManagement.tsx +++ b/app/src/main/frontend/components/administration/LibraryManagement.tsx @@ -3,6 +3,7 @@ import ConfigFormField from "Frontend/components/administration/ConfigFormField" import withConfigPage from "Frontend/components/administration/withConfigPage"; import Section from "Frontend/components/general/Section"; import * as Yup from 'yup'; +import "Frontend/util/yup-extensions"; import {addToast, Button, Divider, Tooltip, useDisclosure} from "@heroui/react"; import {Plus} from "@phosphor-icons/react"; import {LibraryEndpoint} from "Frontend/generated/endpoints"; @@ -55,7 +56,7 @@ function LibraryManagementLayout({getConfig, formik}: any) {
- +
@@ -95,8 +96,11 @@ const validationSchema = Yup.object({ library: Yup.object({ metadata: Yup.object({ update: Yup.object({ - // @ts-ignore - schedule: Yup.string().cron() + enabled: Yup.boolean(), + schedule: Yup.string().when("enabled", { + is: true, + then: (schema) => schema.cron() + }), }) }), scan: Yup.object({ diff --git a/app/src/main/frontend/util/custom-validators.ts b/app/src/main/frontend/util/custom-validators.ts deleted file mode 100644 index d6963f6..0000000 --- a/app/src/main/frontend/util/custom-validators.ts +++ /dev/null @@ -1,10 +0,0 @@ -import * as Yup from "yup"; -import {isValidCron} from "cron-validator"; - -// Custom validator for cron expressions -Yup.addMethod(Yup.string, 'cron', function (message) { - return this.test('cron', message, function (value) { - const {path, createError} = this; - return isValidCron(value as string) || createError({path, message: message || 'Invalid cron expression'}); - }); -}); \ No newline at end of file diff --git a/app/src/main/frontend/util/yup-extensions.ts b/app/src/main/frontend/util/yup-extensions.ts new file mode 100644 index 0000000..e3d0702 --- /dev/null +++ b/app/src/main/frontend/util/yup-extensions.ts @@ -0,0 +1,22 @@ +import * as Yup from 'yup'; +import {ConfigEndpoint} from 'Frontend/generated/endpoints'; + +Yup.addMethod(Yup.string, 'cron', function (message = 'Invalid cron expression') { + return this.test('cron', message, async function (value) { + const {path, createError} = this; + if (!value) return true; + try { + const isValid = await ConfigEndpoint.validateCronExpression(value); + return isValid || createError({path, message}); + } catch (e) { + return createError({path, message: 'Error validating cron expression'}); + } + }); +}); + +// TypeScript: Extend Yup's type definitions +declare module 'yup' { + interface StringSchema { + cron(message?: string): StringSchema; + } +} \ No newline at end of file diff --git a/app/src/main/kotlin/org/gameyfin/app/config/ConfigEndpoint.kt b/app/src/main/kotlin/org/gameyfin/app/config/ConfigEndpoint.kt index 77281ba..7666814 100644 --- a/app/src/main/kotlin/org/gameyfin/app/config/ConfigEndpoint.kt +++ b/app/src/main/kotlin/org/gameyfin/app/config/ConfigEndpoint.kt @@ -11,6 +11,7 @@ import org.gameyfin.app.core.Role import org.gameyfin.app.core.annotations.DynamicPublicAccess import org.gameyfin.app.users.UserService import org.gameyfin.app.users.util.isAdmin +import org.springframework.scheduling.support.CronExpression import reactor.core.publisher.Flux @Endpoint @@ -36,7 +37,15 @@ class ConfigEndpoint( fun update(update: ConfigUpdateDto) = configService.update(update) - /** Specific read-only endpoint for all users **/ + /** + * Validates a cron expression because Spring has a custom syntax for cron expressions. + * @see: https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/scheduling/support/CronExpression.html#parse(java.lang.String) + */ + fun validateCronExpression(cronExpression: String): Boolean { + return CronExpression.isValidExpression(cronExpression) + } + + /** Specific read-only endpoints for all users **/ @DynamicPublicAccess @AnonymousAllowed @@ -49,5 +58,4 @@ class ConfigEndpoint( @DynamicPublicAccess @AnonymousAllowed fun isPublicAccessEnabled(): Boolean = configService.get(ConfigProperties.Libraries.AllowPublicAccess) == true - } diff --git a/app/src/main/kotlin/org/gameyfin/app/config/ConfigProperties.kt b/app/src/main/kotlin/org/gameyfin/app/config/ConfigProperties.kt index c5caccb..23e928e 100644 --- a/app/src/main/kotlin/org/gameyfin/app/config/ConfigProperties.kt +++ b/app/src/main/kotlin/org/gameyfin/app/config/ConfigProperties.kt @@ -90,15 +90,15 @@ sealed class ConfigProperties( data object UpdateEnabled : ConfigProperties( Boolean::class, "library.metadata.update.enabled", - "Enable periodic refresh of video game metadata (coming soon™)", - false + "Enable periodic refresh of video game metadata", + true ) data object UpdateSchedule : ConfigProperties( String::class, "library.metadata.update.schedule", - "Schedule for periodic metadata refresh in cron format", - "0 0 * * 0" + "Schedule for periodic metadata refresh in Spring cron format", + "@daily" ) } } diff --git a/app/src/main/kotlin/org/gameyfin/app/config/dto/CronExpressionVerificationResultDto.kt b/app/src/main/kotlin/org/gameyfin/app/config/dto/CronExpressionVerificationResultDto.kt new file mode 100644 index 0000000..486800c --- /dev/null +++ b/app/src/main/kotlin/org/gameyfin/app/config/dto/CronExpressionVerificationResultDto.kt @@ -0,0 +1,11 @@ +package org.gameyfin.app.config.dto + +data class CronExpressionVerificationResultDto( + val valid: Boolean, + val errorMessage: String? = null +) { + companion object { + val valid = CronExpressionVerificationResultDto(true) + fun invalid(errorMessage: String) = CronExpressionVerificationResultDto(false, errorMessage) + } +} \ No newline at end of file diff --git a/app/src/main/kotlin/org/gameyfin/app/config/entities/ConfigEntry.kt b/app/src/main/kotlin/org/gameyfin/app/config/entities/ConfigEntry.kt index 64fe3cf..d3a13dc 100644 --- a/app/src/main/kotlin/org/gameyfin/app/config/entities/ConfigEntry.kt +++ b/app/src/main/kotlin/org/gameyfin/app/config/entities/ConfigEntry.kt @@ -4,6 +4,7 @@ import jakarta.persistence.* import org.gameyfin.app.core.security.EncryptionConverter @Entity +@EntityListeners(ConfigEntryEntityListener::class) @Table(name = "app_config") class ConfigEntry( @Id diff --git a/app/src/main/kotlin/org/gameyfin/app/config/entities/ConfigEntryEntityListener.kt b/app/src/main/kotlin/org/gameyfin/app/config/entities/ConfigEntryEntityListener.kt new file mode 100644 index 0000000..a35958e --- /dev/null +++ b/app/src/main/kotlin/org/gameyfin/app/config/entities/ConfigEntryEntityListener.kt @@ -0,0 +1,26 @@ +package org.gameyfin.app.config.entities + +import jakarta.persistence.PostPersist +import jakarta.persistence.PostRemove +import jakarta.persistence.PostUpdate +import org.gameyfin.app.config.ConfigProperties +import org.gameyfin.app.core.events.LibraryScanScheduleUpdatedEvent +import org.gameyfin.app.util.EventPublisherHolder + +class ConfigEntryEntityListener { + @PostUpdate + @PostPersist + @PostRemove + fun process(configEntry: ConfigEntry) { + when (configEntry.key) { + in ConfigProperties.Libraries.Metadata.UpdateEnabled.key, + ConfigProperties.Libraries.Metadata.UpdateSchedule.key -> { + EventPublisherHolder.publish(LibraryScanScheduleUpdatedEvent(this)) + } + + ConfigProperties.Libraries.Scan.EnableFilesystemWatcher.key -> { + TODO() + } + } + } +} \ No newline at end of file diff --git a/app/src/main/kotlin/org/gameyfin/app/core/events/Events.kt b/app/src/main/kotlin/org/gameyfin/app/core/events/Events.kt index db5131b..1e30da0 100644 --- a/app/src/main/kotlin/org/gameyfin/app/core/events/Events.kt +++ b/app/src/main/kotlin/org/gameyfin/app/core/events/Events.kt @@ -23,6 +23,4 @@ class PasswordResetRequestEvent(source: Any, val token: Token + diff --git a/app/src/main/kotlin/org/gameyfin/app/core/jobs/JobRunStatus.kt b/app/src/main/kotlin/org/gameyfin/app/core/jobs/JobRunStatus.kt new file mode 100644 index 0000000..2a765e5 --- /dev/null +++ b/app/src/main/kotlin/org/gameyfin/app/core/jobs/JobRunStatus.kt @@ -0,0 +1,9 @@ +package org.gameyfin.app.core.jobs + +enum class JobRunStatus { + IN_PROGRESS, + SUCCESS, + FAILED, + CANCELLED +} + diff --git a/app/src/main/kotlin/org/gameyfin/app/core/jobs/JobService.kt b/app/src/main/kotlin/org/gameyfin/app/core/jobs/JobService.kt new file mode 100644 index 0000000..3b9f83a --- /dev/null +++ b/app/src/main/kotlin/org/gameyfin/app/core/jobs/JobService.kt @@ -0,0 +1,74 @@ +package org.gameyfin.app.core.jobs + +import com.vaadin.hilla.exception.EndpointException +import io.github.oshai.kotlinlogging.KotlinLogging +import jakarta.annotation.PostConstruct +import org.gameyfin.app.config.ConfigProperties +import org.gameyfin.app.config.ConfigService +import org.gameyfin.app.core.events.LibraryScanScheduleUpdatedEvent +import org.gameyfin.app.libraries.LibraryScanService +import org.springframework.context.event.EventListener +import org.springframework.scheduling.TaskScheduler +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.scheduling.support.CronExpression +import org.springframework.scheduling.support.CronTrigger +import org.springframework.stereotype.Service +import java.time.LocalDateTime +import java.util.concurrent.ScheduledFuture + +@Service +class JobService( + private val config: ConfigService, + libraryScanService: LibraryScanService, + private val jobRunResultRepository: JobRunResultRepository +) { + private val scheduler: TaskScheduler = ThreadPoolTaskScheduler().apply { initialize() } + private var libraryScanFuture: ScheduledFuture<*>? = null + + private val libraryScanJob: Job = LibraryScanJob(libraryScanService) + + companion object { + private val log = KotlinLogging.logger { } + } + + @PostConstruct + fun init() { + scheduleLibraryScanJob() + } + + @EventListener(LibraryScanScheduleUpdatedEvent::class) + fun onLibraryScanScheduleUpdated() { + scheduleLibraryScanJob() + } + + private fun scheduleLibraryScanJob() { + libraryScanFuture?.cancel(false) + + if (config.get(ConfigProperties.Libraries.Metadata.UpdateEnabled) != true) { + log.debug { "Disabled scheduled library scans" } + return + } + + val cronExpressionString = config.get(ConfigProperties.Libraries.Metadata.UpdateSchedule) ?: return + + try { + val cronTrigger = CronTrigger(cronExpressionString) + libraryScanFuture = (scheduler as ThreadPoolTaskScheduler).schedule({ + runAndPersistJob(libraryScanJob) + }, cronTrigger) + log.debug { + "Library scan job scheduled, next run will be @ " + + "${CronExpression.parse(cronExpressionString).next(LocalDateTime.now())}" + } + } catch (e: Exception) { + log.error { "Failed to schedule library scan job: ${e.message}" } + log.debug(e) { } + throw EndpointException("Failed to schedule library scan job: ${e.message}") + } + } + + private fun runAndPersistJob(job: Job) { + val result = job.run() + jobRunResultRepository.save(result) + } +} \ No newline at end of file diff --git a/app/src/main/kotlin/org/gameyfin/app/core/jobs/LibraryScanJob.kt b/app/src/main/kotlin/org/gameyfin/app/core/jobs/LibraryScanJob.kt new file mode 100644 index 0000000..04d171d --- /dev/null +++ b/app/src/main/kotlin/org/gameyfin/app/core/jobs/LibraryScanJob.kt @@ -0,0 +1,33 @@ +package org.gameyfin.app.core.jobs + +import org.gameyfin.app.libraries.LibraryScanService +import org.gameyfin.app.libraries.enums.ScanType +import java.time.LocalDateTime + +class LibraryScanJob(private val libraryScanService: LibraryScanService) : Job { + override val name: String = "LibraryScan" + override fun run(): JobRunResult { + val startedAt = LocalDateTime.now() + var finishedAt: LocalDateTime? + var status: JobRunStatus + var message: String? + try { + libraryScanService.triggerScan(ScanType.SCHEDULED, null) + message = "Library scan completed successfully" + status = JobRunStatus.SUCCESS + } catch (ex: Exception) { + message = ex.message + status = JobRunStatus.FAILED + } finally { + finishedAt = LocalDateTime.now() + } + return JobRunResult( + jobName = name, + startedAt = startedAt, + finishedAt = finishedAt, + status = status, + message = message + ) + } +} + diff --git a/app/src/main/kotlin/org/gameyfin/app/libraries/LibraryScanService.kt b/app/src/main/kotlin/org/gameyfin/app/libraries/LibraryScanService.kt index 2f0e382..1963d34 100644 --- a/app/src/main/kotlin/org/gameyfin/app/libraries/LibraryScanService.kt +++ b/app/src/main/kotlin/org/gameyfin/app/libraries/LibraryScanService.kt @@ -71,7 +71,8 @@ class LibraryScanService( try { when (scanType) { ScanType.QUICK -> quickScan(library) - ScanType.FULL -> fullScan(library) + ScanType.FULL -> fullScan(library, false) + ScanType.SCHEDULED -> fullScan(library, true) } } finally { scansInProgress.remove(libraryId) @@ -104,7 +105,7 @@ class LibraryScanService( */ fun fullScan(libraryDtos: Collection?) { val libraries = libraryDtos?.map { libraryCoreService.toEntity(it) } ?: libraryRepository.findAll() - libraries.forEach { executor.submit { fullScan(it) } } + libraries.forEach { executor.submit { fullScan(it, false) } } } private fun quickScan(library: Library) { @@ -191,10 +192,10 @@ class LibraryScanService( } } - private fun fullScan(library: Library) { + private fun fullScan(library: Library, triggeredBySchedule: Boolean) { val progress = LibraryScanProgress( libraryId = library.id!!, - type = ScanType.FULL, + type = if (triggeredBySchedule) ScanType.SCHEDULED else ScanType.FULL, currentStep = LibraryScanStep( description = "Scanning filesystem" ) diff --git a/app/src/main/kotlin/org/gameyfin/app/libraries/enums/ScanType.kt b/app/src/main/kotlin/org/gameyfin/app/libraries/enums/ScanType.kt index 34a44ce..1353967 100644 --- a/app/src/main/kotlin/org/gameyfin/app/libraries/enums/ScanType.kt +++ b/app/src/main/kotlin/org/gameyfin/app/libraries/enums/ScanType.kt @@ -1,5 +1,7 @@ package org.gameyfin.app.libraries.enums enum class ScanType { - QUICK, FULL + QUICK, + FULL, + SCHEDULED, } \ No newline at end of file diff --git a/app/src/main/kotlin/org/gameyfin/app/util/EventPublisherHolder.kt b/app/src/main/kotlin/org/gameyfin/app/util/EventPublisherHolder.kt new file mode 100644 index 0000000..f409d3c --- /dev/null +++ b/app/src/main/kotlin/org/gameyfin/app/util/EventPublisherHolder.kt @@ -0,0 +1,21 @@ +package org.gameyfin.app.util + +import org.springframework.context.ApplicationContext +import org.springframework.context.ApplicationContextAware +import org.springframework.context.ApplicationEvent +import org.springframework.context.ApplicationEventPublisher +import org.springframework.stereotype.Component + +@Component +object EventPublisherHolder : ApplicationContextAware { + private var publisher: ApplicationEventPublisher? = null + + override fun setApplicationContext(context: ApplicationContext) { + publisher = context + } + + fun publish(event: ApplicationEvent) { + publisher?.publishEvent(event) + } +} +