Merge pull request #10 from grimsi/gh-8_MultipleLibraryFolders

Add support for multiple library folders

.gitignore

@@ -37,3 +37,4 @@ build/
 /data/
 /backend/src/main/resources/static/
 /docker/docker-compose.yml
+/.gameyfin/

backend/pom.xml

@@ -16,12 +16,16 @@
 
     <properties>
         <java.version>18</java.version>
+
         <springdoc-openapi-ui.version>1.6.9</springdoc-openapi-ui.version>
         <resilience4j.version>1.7.1</resilience4j.version>
         <commons-io.version>2.11.0</commons-io.version>
         <commons-compress.version>1.21</commons-compress.version>
         <protoc.plugin.version>3.11.4</protoc.plugin.version>
         <protobuf-java.version>3.21.3</protobuf-java.version>
+
+        <!-- Use older version because the newer versions have problems with non-ASCII characters in properties files (umlauts etc.) -->
+        <maven-resources-plugin.version>3.1.0</maven-resources-plugin.version>
     </properties>
 
     <dependencies>
@@ -140,6 +144,7 @@
                     <include>**/*.properties</include>
                     <include>**/*.yml</include>
                     <include>**/*.yaml</include>
+                    <include>**/*.json</include>
                     <include>**/*.txt</include>
                     <include>**/*.js</include>
                     <include>**/*.css</include>
@@ -166,6 +171,7 @@
             <!-- Import the compiled frontend -->
             <plugin>
                 <artifactId>maven-resources-plugin</artifactId>
+                <version>${maven-resources-plugin.version}</version>
                 <executions>
                     <execution>
                         <id>copy-resources</id>

backend/src/main/java/de/grimsi/gameyfin/GameyfinApplication.java

@@ -8,7 +8,7 @@ public class GameyfinApplication {
 
     public static void main(String[] args) {
         new SpringApplicationBuilder(GameyfinApplication.class)
-                .properties("spring.config.name=application,gameyfin,database,secure")
+                .properties( "file.encoding=UTF-8", "spring.config.name=application,gameyfin,database,secure")
                 .build()
                 .run(args);
     }

backend/src/main/java/de/grimsi/gameyfin/config/FilesystemConfig.java

@@ -0,0 +1,66 @@
+package de.grimsi.gameyfin.config;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.jdbc.DataSourceBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+import org.springframework.core.env.*;
+import org.springframework.util.PropertyPlaceholderHelper;
+
+import javax.sql.DataSource;
+import java.util.Arrays;
+import java.util.Properties;
+import java.util.stream.StreamSupport;
+
+@Configuration
+public class FilesystemConfig {
+
+    @Value("#{'${gameyfin.sources}'.split(',')[0]}")
+    private String firstLibraryPath;
+
+    @Autowired
+    Environment env;
+
+    @Autowired
+    public void setConfigurableEnvironment(ConfigurableEnvironment env) {
+        Properties props = new Properties();
+        props.setProperty("gameyfin.db", "%s/.gameyfin/db".formatted(firstLibraryPath));
+        props.setProperty("gameyfin.cache", "%s/.gameyfin/cache".formatted(firstLibraryPath));
+        env.getPropertySources().addFirst(new PropertiesPropertySource("gameyfinFilesystemProperties", props));
+    }
+
+    /**
+     * This bean is needed so Spring initializes the data source after we are done messing with the configuration environment
+     * @return DataSource
+     */
+    @ConfigurationProperties(prefix = "spring.datasource")
+    @Bean
+    @Primary
+    public DataSource getDataSource() {
+
+        Properties properties = loadAllProperties();
+
+        return DataSourceBuilder
+                .create()
+                .url(properties.getProperty("spring.datasource.url"))
+                .build();
+    }
+
+    private Properties loadAllProperties() {
+        Properties props = new Properties();
+
+        MutablePropertySources propSrcs = ((AbstractEnvironment) env).getPropertySources();
+
+        StreamSupport.stream(propSrcs.spliterator(), false)
+                .filter(ps -> ps instanceof EnumerablePropertySource)
+                .map(ps -> ((EnumerablePropertySource<?>) ps).getPropertyNames())
+                .flatMap(Arrays::stream)
+                .forEach(propName -> props.setProperty(propName, env.getProperty(propName)));
+
+        return props;
+    }
+
+}

backend/src/main/java/de/grimsi/gameyfin/config/SecureProperties.java

@@ -16,7 +16,7 @@ public class SecureProperties {
     @Autowired
     public void setConfigurableEnvironment(ConfigurableEnvironment env) {
         try {
-            Resource resource = new ClassPathResource("/config/secure.properties");
+            Resource resource = new ClassPathResource("/config/secure.yml");
             env.getPropertySources().addFirst(new PropertiesPropertySource(Objects.requireNonNull(resource.getFilename()), PropertiesLoaderUtils.loadProperties(resource)));
         } catch (Exception ex) {
             throw new RuntimeException(ex.getMessage(), ex);

backend/src/main/java/de/grimsi/gameyfin/service/LibraryService.java

@@ -10,13 +10,13 @@ import de.grimsi.gameyfin.repositories.DetectedGameRepository;
 import de.grimsi.gameyfin.repositories.UnmappableFileRepository;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.springframework.util.StopWatch;
 
 import java.io.IOException;
-import java.nio.file.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Stream;
@@ -29,25 +29,29 @@ import static de.grimsi.gameyfin.util.FilenameUtil.hasGameArchiveExtension;
 @RequiredArgsConstructor
 public class LibraryService {
 
-    @Value("${gameyfin.root}")
+    @Value("${gameyfin.sources}")
-    private String rootFolderPath;
+    private List<String> libraryFolders;
-
     private final IgdbWrapper igdbWrapper;
     private final DetectedGameRepository detectedGameRepository;
     private final UnmappableFileRepository unmappableFileRepository;
 
     public List<Path> getGameFiles() {
+        List<Path> gamefiles = new ArrayList<>();
 
-        Path rootFolder = Path.of(rootFolderPath);
+        libraryFolders.stream().map(Path::of).forEach(
+                folder -> {
+                    try (Stream<Path> stream = Files.list(folder)) {
+                        // return all sub-folders (non-recursive) and files that have an extension that indicates that they are a downloadable file
+                        List<Path> gameFilesFromThisFolder = stream.filter(p -> Files.isDirectory(p) || hasGameArchiveExtension(p)).toList();
+                        gamefiles.addAll(gameFilesFromThisFolder);
 
-        try (Stream<Path> stream = Files.list(rootFolder)) {
+                    } catch (IOException e) {
-            // return all sub-folders (non-recursive) and files that have an extension that indicates that they are a downloadable file
+                        throw new RuntimeException("Error while opening library folder '%s'".formatted(folder), e);
-            return stream
+                    }
-                    .filter(p -> Files.isDirectory(p) || hasGameArchiveExtension(p))
+                }
-                    .toList();
+        );
-        } catch (IOException e) {
+
-            throw new RuntimeException("Error while opening root folder", e);
+        return gamefiles;
-        }
     }
 
     public void scanGameLibrary() {

backend/src/main/resources/META-INF/additional-spring-configuration-metadata.json

@@ -0,0 +1,9 @@
+{
+  "properties": [
+    {
+      "name": "gameyfin.sources",
+      "type": "java.lang.String[]",
+      "description": "List of directories Gameyfin should scan for games."
+    }
+  ]
+}

backend/src/main/resources/application-dev.yml

@@ -1,8 +1,6 @@
 gameyfin:
   user: admin
   password: password
-  cache: ${gameyfin.root}\.gameyfin\cache
-  db: ${gameyfin.root}\.gameyfin\db
 
 logging:
   level:

backend/src/main/resources/config/database.properties

@@ -1,13 +0,0 @@
-# This file contains properties related to the database configuration
-#
-#
-spring.jpa.properties.hibernate.enable_lazy_load_no_trans=true
-spring.datasource.db-name=gameyfin_db
-spring.datasource.url=jdbc:h2:file:${gameyfin.db}/${spring.datasource.db-name}
-spring.datasource.username=gfadmin
-spring.datasource.password=gameyfin
-spring.datasource.driverClassName=org.h2.Driver
-spring.jpa.database-platform=org.hibernate.dialect.H2Dialect
-spring.jpa.hibernate.ddl-auto=update
-spring.jpa.open-in-view=true
-spring.jpa.properties.hibernate.event.merge.entity_copy_observer=allow

backend/src/main/resources/config/database.yml

@@ -0,0 +1,18 @@
+spring:
+  jpa:
+    open-in-view: true
+    properties:
+      hibernate:
+        enable_lazy_load_no_trans: true
+        event:
+          merge:
+            entity_copy_observer: allow
+    database-platform: org.hibernate.dialect.H2Dialect
+    hibernate:
+      ddl-auto: update
+  datasource:
+    username: gfadmin
+    password: gameyfin
+    db-name: gameyfin_db
+    url: jdbc:h2:file:${gameyfin.db}/${spring.datasource.db-name}
+    driverClassName: org.h2.Driver

backend/src/main/resources/config/gameyfin.properties

@@ -1,28 +0,0 @@
-# This file contains properties related to the configuration of Gameyfin
-#
-#
-# Username and password for the web interface
-gameyfin.user=
-gameyfin.password=
-
-# Root folder of your game library
-gameyfin.root=
-# Folders where gameyfin will store cached images and the database
-gameyfin.cache=${gameyfin.root}/.gameyfin/cache
-gameyfin.db=${gameyfin.root}/.gameyfin/db
-
-# File extensions which gameyfin will recognize as game files
-gameyfin.file-extensions=iso, zip, rar, 7z, exe
-
-# List of IGDB platform enums to limit search results. For possible values see: https://api-docs.igdb.com/#platform
-gameyfin.igdb.config.preferred-platforms=6
-
-# Twitch Client ID and Client Secret
-gameyfin.igdb.api.client-id=
-gameyfin.igdb.api.client-secret=
-
-# The IGDB API has a rate limit of 4 req/s
-gameyfin.igdb.api.max-requests-per-second=4
-
-# According to the docs, there is a maximum of 8 concurrent requests, but in my tests the actual limit was 4 and even then it sometimes failed, so I set it to 2 to be sure
-gameyfin.igdb.api.max-concurrent-requests=2

backend/src/main/resources/config/gameyfin.yml

@@ -0,0 +1,10 @@
+gameyfin:
+  file-extensions: iso, zip, rar, 7z, exe
+  igdb:
+    api:
+      client-id:
+      client-secret:
+      max-concurrent-requests: 2
+      max-requests-per-second: 4
+    config:
+      preferred-platforms: 6

backend/src/main/resources/config/secure.properties

@@ -1,19 +0,0 @@
-# This file contains properties that are *NOT* safe to override by the user
-# In theory a user should not be able to override them since they will be loaded from the classpath at launch, overriding existing user properties with the same key
-#
-#
-# System Info
-application.name=Gameyfin
-application.version=@project.version@
-# API
-server.servlet.context-path=/
-# Spring Actuator
-management.endpoints.enabled-by-default=false
-management.endpoint.health.enabled=true
-# Server
-server.error.include-stacktrace=never
-spring.mvc.async.request-timeout=-1
-# Jackson JSON Mapping
-spring.jackson.default-property-inclusion=non_null
-spring.jackson.mapper.accept-case-insensitive-enums=true
-spring.jackson.deserialization.fail-on-unknown-properties=false

backend/src/main/resources/config/secure.yml

@@ -0,0 +1,27 @@
+application:
+  version: '@project.version@'
+  name: Gameyfin
+
+server:
+  servlet:
+    context-path: /
+  error:
+    include-stacktrace: never
+
+spring:
+  jackson:
+    deserialization:
+      fail-on-unknown-properties: false
+    default-property-inclusion: non_null
+    mapper:
+      accept-case-insensitive-enums: true
+  mvc:
+    async:
+      request-timeout: -1
+
+management:
+  endpoint:
+    health:
+      enabled: true
+  endpoints:
+    enabled-by-default: false

docker/Dockerfile

@@ -1,10 +1,10 @@
 FROM openjdk:18
 
-ENV GAMEYFIN_ROOT=/opt/gameyfin-library
+ENV GAMEYFIN_SOURCES=/opt/gameyfin-library
 
 RUN groupadd gameyfin && useradd gameyfin -g gameyfin && \
-    mkdir -p /opt/gameyfin ${GAMEYFIN_ROOT} && \
+    mkdir -p /opt/gameyfin ${GAMEYFIN_SOURCES} && \
-    chown -R gameyfin:gameyfin /opt/gameyfin ${GAMEYFIN_ROOT}
+    chown -R gameyfin:gameyfin /opt/gameyfin ${GAMEYFIN_SOURCES}
 
 USER gameyfin:gameyfin