Brendon/gameyfin
1
0
Fork 0
mirror of https://github.com/BrenBroZAYT/gameyfin.git synced 2026-06-13 16:40:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement scheduled library scans (closes #609) (#638)

Browse Source 
First implementation of job system
This commit is contained in:
Simon
2025-07-20 13:57:54 +02:00
committed by GitHub
parent 357c68ffe1
commit 3cecdd558a
21 changed files with 263 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/package-lock.json
Generated
+2 -9
View File
@@ -1,12 +1,12 @@
{
"name": "gameyfin",
"version": "2.0.0.beta4",
"version": "2.0.0.beta6",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "gameyfin",
"version": "2.0.0.beta4",
"version": "2.0.0.beta6",
"dependencies": {
"@heroui/react": "2.7.9",
"@material-tailwind/react": "^2.1.10",
@@ -33,7 +33,6 @@
"@vaadin/vaadin-usage-statistics": "2.1.3",
"classnames": "^2.5.1",
"construct-style-sheets-polyfill": "3.1.0",
"cron-validator": "^1.3.1",
"date-fns": "2.29.3",
"formik": "^2.4.6",
"framer-motion": "^12.5.0",
@@ -10300,12 +10299,6 @@
"url": "https://opencollective.com/core-js"
}
},
"node_modules/cron-validator": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/cron-validator/-/cron-validator-1.3.1.tgz",
"integrity": "sha512-C1HsxuPCY/5opR55G5/WNzyEGDWFVG+6GLrA+fW/sCTcP6A6NTjUP2AK7B8n2PyFs90kDG2qzwm8LMheADku6A==",
"license": "MIT"
},
"node_modules/cross-spawn": {
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
app/package.json
+2 -4
View File
@@ -28,7 +28,6 @@
"@vaadin/vaadin-usage-statistics": "2.1.3",
"classnames": "^2.5.1",
"construct-style-sheets-polyfill": "3.1.0",
"cron-validator": "^1.3.1",
"date-fns": "2.29.3",
"formik": "^2.4.6",
"framer-motion": "^12.5.0",
@@ -123,7 +122,6 @@
"@vaadin/hilla-lit-form": "$@vaadin/hilla-lit-form",
"@vaadin/hilla-react-form": "$@vaadin/hilla-react-form",
"@vaadin/hilla-react-signals": "$@vaadin/hilla-react-signals",
"cron-validator": "$cron-validator",
"moment": "$moment",
"moment-timezone": "$moment-timezone",
"react-confetti-boom": "$react-confetti-boom",
@@ -265,6 +263,6 @@
"workbox-precaching": "7.3.0"
},
"disableUsageStatistics": true,
"hash": "e964f24aa5152284476d9e99245c9b92f8e6f94274818ee46eec2a3f91e29fc6"
"hash": "962eccc3fa0735d5234901be4f9e384096113c45bec22564a53688096d62aef4"
}
}
}
app/src/main/frontend/App.tsx
-1
View File
@@ -1,6 +1,5 @@
import {Outlet, useHref, useNavigate} from 'react-router';
import "./main.css";
import "Frontend/util/custom-validators";
import {HeroUIProvider} from "@heroui/react";
import {ThemeProvider as NextThemesProvider} from "next-themes";
import {themeNames} from "Frontend/theming/themes";
app/src/main/frontend/components/administration/LibraryManagement.tsx
+7 -3
View File
@@ -3,6 +3,7 @@ import ConfigFormField from "Frontend/components/administration/ConfigFormField"
import withConfigPage from "Frontend/components/administration/withConfigPage";
import Section from "Frontend/components/general/Section";
import * as Yup from 'yup';
import "Frontend/util/yup-extensions";
import {addToast, Button, Divider, Tooltip, useDisclosure} from "@heroui/react";
import {Plus} from "@phosphor-icons/react";
import {LibraryEndpoint} from "Frontend/generated/endpoints";
@@ -55,7 +56,7 @@ function LibraryManagementLayout({getConfig, formik}: any) {
<Section title="Metadata"/>
<div className="flex flex-row items-baseline">
<ConfigFormField configElement={getConfig("library.metadata.update.enabled")} isDisabled/>
<ConfigFormField configElement={getConfig("library.metadata.update.enabled")}/>
<ConfigFormField configElement={getConfig("library.metadata.update.schedule")}
isDisabled={!formik.values.library.metadata.update.enabled}/>
</div>
@@ -95,8 +96,11 @@ const validationSchema = Yup.object({
library: Yup.object({
metadata: Yup.object({
update: Yup.object({
// @ts-ignore
schedule: Yup.string().cron()
enabled: Yup.boolean(),
schedule: Yup.string().when("enabled", {
is: true,
then: (schema) => schema.cron()
}),
})
}),
scan: Yup.object({
app/src/main/frontend/util/custom-validators.ts
-10
View File
@@ -1,10 +0,0 @@
import * as Yup from "yup";
import {isValidCron} from "cron-validator";
// Custom validator for cron expressions
Yup.addMethod(Yup.string, 'cron', function (message) {
return this.test('cron', message, function (value) {
const {path, createError} = this;
return isValidCron(value as string) || createError({path, message: message || 'Invalid cron expression'});
});
});
app/src/main/frontend/util/yup-extensions.ts
+22
View File
@@ -0,0 +1,22 @@
import * as Yup from 'yup';
import {ConfigEndpoint} from 'Frontend/generated/endpoints';
Yup.addMethod(Yup.string, 'cron', function (message = 'Invalid cron expression') {
return this.test('cron', message, async function (value) {
const {path, createError} = this;
if (!value) return true;
try {
const isValid = await ConfigEndpoint.validateCronExpression(value);
return isValid || createError({path, message});
} catch (e) {
return createError({path, message: 'Error validating cron expression'});
}
});
});
// TypeScript: Extend Yup's type definitions
declare module 'yup' {
interface StringSchema {
cron(message?: string): StringSchema;
}
}
app/src/main/kotlin/org/gameyfin/app/config/ConfigEndpoint.kt
+10 -2
View File
@@ -11,6 +11,7 @@ import org.gameyfin.app.core.Role
import org.gameyfin.app.core.annotations.DynamicPublicAccess
import org.gameyfin.app.users.UserService
import org.gameyfin.app.users.util.isAdmin
import org.springframework.scheduling.support.CronExpression
import reactor.core.publisher.Flux
@Endpoint
@@ -36,7 +37,15 @@ class ConfigEndpoint(
fun update(update: ConfigUpdateDto) = configService.update(update)
/** Specific read-only endpoint for all users **/
/**
* Validates a cron expression because Spring has a custom syntax for cron expressions.
* @see: https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/scheduling/support/CronExpression.html#parse(java.lang.String)
*/
fun validateCronExpression(cronExpression: String): Boolean {
return CronExpression.isValidExpression(cronExpression)
}
/** Specific read-only endpoints for all users **/
@DynamicPublicAccess
@AnonymousAllowed
@@ -49,5 +58,4 @@ class ConfigEndpoint(
@DynamicPublicAccess
@AnonymousAllowed
fun isPublicAccessEnabled(): Boolean = configService.get(ConfigProperties.Libraries.AllowPublicAccess) == true
}
app/src/main/kotlin/org/gameyfin/app/config/ConfigProperties.kt
+4 -4
View File
@@ -90,15 +90,15 @@ sealed class ConfigProperties<T : Serializable>(
data object UpdateEnabled : ConfigProperties<Boolean>(
Boolean::class,
"library.metadata.update.enabled",
"Enable periodic refresh of video game metadata (coming soon™)",
false
"Enable periodic refresh of video game metadata",
true
)
data object UpdateSchedule : ConfigProperties<String>(
String::class,
"library.metadata.update.schedule",
"Schedule for periodic metadata refresh in cron format",
"0 0 * * 0"
"Schedule for periodic metadata refresh in Spring cron format",
"@daily"
)
}
}
app/src/main/kotlin/org/gameyfin/app/config/dto/CronExpressionVerificationResultDto.kt
+11
View File
@@ -0,0 +1,11 @@
package org.gameyfin.app.config.dto
data class CronExpressionVerificationResultDto(
val valid: Boolean,
val errorMessage: String? = null
) {
companion object {
val valid = CronExpressionVerificationResultDto(true)
fun invalid(errorMessage: String) = CronExpressionVerificationResultDto(false, errorMessage)
}
}
app/src/main/kotlin/org/gameyfin/app/config/entities/ConfigEntry.kt
+1
View File
@@ -4,6 +4,7 @@ import jakarta.persistence.*
import org.gameyfin.app.core.security.EncryptionConverter
@Entity
@EntityListeners(ConfigEntryEntityListener::class)
@Table(name = "app_config")
class ConfigEntry(
@Id
app/src/main/kotlin/org/gameyfin/app/config/entities/ConfigEntryEntityListener.kt
+26
View File
@@ -0,0 +1,26 @@
package org.gameyfin.app.config.entities
import jakarta.persistence.PostPersist
import jakarta.persistence.PostRemove
import jakarta.persistence.PostUpdate
import org.gameyfin.app.config.ConfigProperties
import org.gameyfin.app.core.events.LibraryScanScheduleUpdatedEvent
import org.gameyfin.app.util.EventPublisherHolder
class ConfigEntryEntityListener {
@PostUpdate
@PostPersist
@PostRemove
fun process(configEntry: ConfigEntry) {
when (configEntry.key) {
in ConfigProperties.Libraries.Metadata.UpdateEnabled.key,
ConfigProperties.Libraries.Metadata.UpdateSchedule.key -> {
EventPublisherHolder.publish(LibraryScanScheduleUpdatedEvent(this))
}
ConfigProperties.Libraries.Scan.EnableFilesystemWatcher.key -> {
TODO()
}
}
}
}
app/src/main/kotlin/org/gameyfin/app/core/events/Events.kt
+1 -3
View File
@@ -23,6 +23,4 @@ class PasswordResetRequestEvent(source: Any, val token: Token<TokenType.Password
class AccountDeletedEvent(source: Any, val user: User, val baseUrl: String) : ApplicationEvent(source)
class GameRequestEvent(source: Any) : ApplicationEvent(source)
class GameRequestApprovalEvent(source: Any) : ApplicationEvent(source)
class LibraryScanScheduleUpdatedEvent(source: Any) : ApplicationEvent(source)
app/src/main/kotlin/org/gameyfin/app/core/jobs/Job.kt
+7
View File
@@ -0,0 +1,7 @@
package org.gameyfin.app.core.jobs
interface Job {
val name: String
fun run(): JobRunResult
}
app/src/main/kotlin/org/gameyfin/app/core/jobs/JobRunResult.kt
+17
View File
@@ -0,0 +1,17 @@
package org.gameyfin.app.core.jobs
import jakarta.persistence.*
import java.time.LocalDateTime
@Entity
data class JobRunResult(
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
val id: Long? = null,
val jobName: String,
val startedAt: LocalDateTime,
val finishedAt: LocalDateTime?,
@Enumerated(EnumType.STRING)
val status: JobRunStatus,
val message: String?
)
app/src/main/kotlin/org/gameyfin/app/core/jobs/JobRunResultRepository.kt
+8
View File
@@ -0,0 +1,8 @@
package org.gameyfin.app.core.jobs
import org.springframework.data.jpa.repository.JpaRepository
import org.springframework.stereotype.Repository
@Repository
interface JobRunResultRepository : JpaRepository<JobRunResult, Long>
app/src/main/kotlin/org/gameyfin/app/core/jobs/JobRunStatus.kt
+9
View File
@@ -0,0 +1,9 @@
package org.gameyfin.app.core.jobs
enum class JobRunStatus {
IN_PROGRESS,
SUCCESS,
FAILED,
CANCELLED
}
app/src/main/kotlin/org/gameyfin/app/core/jobs/JobService.kt
+74
View File
@@ -0,0 +1,74 @@
package org.gameyfin.app.core.jobs
import com.vaadin.hilla.exception.EndpointException
import io.github.oshai.kotlinlogging.KotlinLogging
import jakarta.annotation.PostConstruct
import org.gameyfin.app.config.ConfigProperties
import org.gameyfin.app.config.ConfigService
import org.gameyfin.app.core.events.LibraryScanScheduleUpdatedEvent
import org.gameyfin.app.libraries.LibraryScanService
import org.springframework.context.event.EventListener
import org.springframework.scheduling.TaskScheduler
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler
import org.springframework.scheduling.support.CronExpression
import org.springframework.scheduling.support.CronTrigger
import org.springframework.stereotype.Service
import java.time.LocalDateTime
import java.util.concurrent.ScheduledFuture
@Service
class JobService(
private val config: ConfigService,
libraryScanService: LibraryScanService,
private val jobRunResultRepository: JobRunResultRepository
) {
private val scheduler: TaskScheduler = ThreadPoolTaskScheduler().apply { initialize() }
private var libraryScanFuture: ScheduledFuture<*>? = null
private val libraryScanJob: Job = LibraryScanJob(libraryScanService)
companion object {
private val log = KotlinLogging.logger { }
}
@PostConstruct
fun init() {
scheduleLibraryScanJob()
}
@EventListener(LibraryScanScheduleUpdatedEvent::class)
fun onLibraryScanScheduleUpdated() {
scheduleLibraryScanJob()
}
private fun scheduleLibraryScanJob() {
libraryScanFuture?.cancel(false)
if (config.get(ConfigProperties.Libraries.Metadata.UpdateEnabled) != true) {
log.debug { "Disabled scheduled library scans" }
return
}
val cronExpressionString = config.get(ConfigProperties.Libraries.Metadata.UpdateSchedule) ?: return
try {
val cronTrigger = CronTrigger(cronExpressionString)
libraryScanFuture = (scheduler as ThreadPoolTaskScheduler).schedule({
runAndPersistJob(libraryScanJob)
}, cronTrigger)
log.debug {
"Library scan job scheduled, next run will be @ " +
"${CronExpression.parse(cronExpressionString).next(LocalDateTime.now())}"
}
} catch (e: Exception) {
log.error { "Failed to schedule library scan job: ${e.message}" }
log.debug(e) { }
throw EndpointException("Failed to schedule library scan job: ${e.message}")
}
}
private fun runAndPersistJob(job: Job) {
val result = job.run()
jobRunResultRepository.save(result)
}
}
app/src/main/kotlin/org/gameyfin/app/core/jobs/LibraryScanJob.kt
+33
View File
@@ -0,0 +1,33 @@
package org.gameyfin.app.core.jobs
import org.gameyfin.app.libraries.LibraryScanService
import org.gameyfin.app.libraries.enums.ScanType
import java.time.LocalDateTime
class LibraryScanJob(private val libraryScanService: LibraryScanService) : Job {
override val name: String = "LibraryScan"
override fun run(): JobRunResult {
val startedAt = LocalDateTime.now()
var finishedAt: LocalDateTime?
var status: JobRunStatus
var message: String?
try {
libraryScanService.triggerScan(ScanType.SCHEDULED, null)
message = "Library scan completed successfully"
status = JobRunStatus.SUCCESS
} catch (ex: Exception) {
message = ex.message
status = JobRunStatus.FAILED
} finally {
finishedAt = LocalDateTime.now()
}
return JobRunResult(
jobName = name,
startedAt = startedAt,
finishedAt = finishedAt,
status = status,
message = message
)
}
}
app/src/main/kotlin/org/gameyfin/app/libraries/LibraryScanService.kt
+5 -4
View File
@@ -71,7 +71,8 @@ class LibraryScanService(
try {
when (scanType) {
ScanType.QUICK -> quickScan(library)
ScanType.FULL -> fullScan(library)
ScanType.FULL -> fullScan(library, false)
ScanType.SCHEDULED -> fullScan(library, true)
}
} finally {
scansInProgress.remove(libraryId)
@@ -104,7 +105,7 @@ class LibraryScanService(
*/
fun fullScan(libraryDtos: Collection<LibraryDto>?) {
val libraries = libraryDtos?.map { libraryCoreService.toEntity(it) } ?: libraryRepository.findAll()
libraries.forEach { executor.submit { fullScan(it) } }
libraries.forEach { executor.submit { fullScan(it, false) } }
}
private fun quickScan(library: Library) {
@@ -191,10 +192,10 @@ class LibraryScanService(
}
}
private fun fullScan(library: Library) {
private fun fullScan(library: Library, triggeredBySchedule: Boolean) {
val progress = LibraryScanProgress(
libraryId = library.id!!,
type = ScanType.FULL,
type = if (triggeredBySchedule) ScanType.SCHEDULED else ScanType.FULL,
currentStep = LibraryScanStep(
description = "Scanning filesystem"
)
app/src/main/kotlin/org/gameyfin/app/libraries/enums/ScanType.kt
+3 -1
View File
@@ -1,5 +1,7 @@
package org.gameyfin.app.libraries.enums
enum class ScanType {
QUICK, FULL
QUICK,
FULL,
SCHEDULED,
}
app/src/main/kotlin/org/gameyfin/app/util/EventPublisherHolder.kt
+21
View File
@@ -0,0 +1,21 @@
package org.gameyfin.app.util
import org.springframework.context.ApplicationContext
import org.springframework.context.ApplicationContextAware
import org.springframework.context.ApplicationEvent
import org.springframework.context.ApplicationEventPublisher
import org.springframework.stereotype.Component
@Component
object EventPublisherHolder : ApplicationContextAware {
private var publisher: ApplicationEventPublisher? = null
override fun setApplicationContext(context: ApplicationContext) {
publisher = context
}
fun publish(event: ApplicationEvent) {
publisher?.publishEvent(event)
}
}