feat(platforms): added platform support (#67)

Now libraries can be assigned to platforms in the admin section.
Games will be assigned to libraries on scanning.

Resolves grimsi/gameyfin#31

Co-authored-by: shawly <shawlyde@gmail.com>
This commit is contained in:
Simon
2022-10-25 21:55:35 +03:00
committed by GitHub
parent 7504cd3500
commit 8e23549336
54 changed files with 1426 additions and 113 deletions
@@ -6,6 +6,7 @@ import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.Instant;
import java.util.List;
@Data
@Builder
@@ -15,4 +16,5 @@ public class AutocompleteSuggestionDto {
private String slug;
private String title;
private Instant releaseDate;
private List<String> platforms;
}
@@ -0,0 +1,9 @@
package de.grimsi.gameyfin.dto;
import lombok.Data;
@Data
public class LibraryScanRequestDto {
private String path;
private boolean downloadImages;
}
@@ -77,6 +77,14 @@ public class DetectedGame {
@ToString.Exclude
private List<PlayerPerspective> playerPerspectives;
@ManyToMany(cascade = CascadeType.MERGE)
@ToString.Exclude
private List<Platform> platforms;
@ManyToOne
@JoinColumn(name = "library")
private Library library;
// Technical properties
@Column(nullable = false)
private String path;
@@ -0,0 +1,38 @@
package de.grimsi.gameyfin.entities;
import lombok.*;
import org.hibernate.Hibernate;
import javax.persistence.*;
import java.util.List;
import java.util.Objects;
@Entity
@Builder
@Getter
@Setter
@ToString
@AllArgsConstructor
@RequiredArgsConstructor
public class Library {
@Id
private String path;
@ManyToMany(cascade = CascadeType.MERGE)
@ToString.Exclude
private List<Platform> platforms;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
Library library = (Library) o;
return path != null && Objects.equals(path, library.path);
}
@Override
public int hashCode() {
return getClass().hashCode();
}
}
@@ -0,0 +1,39 @@
package de.grimsi.gameyfin.entities;
import lombok.*;
import org.hibernate.Hibernate;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import java.util.Objects;
@Entity
@Builder
@Getter
@Setter
@ToString
@AllArgsConstructor
@RequiredArgsConstructor
public class Platform {
@Id
private String slug;
@Column(nullable = false)
private String name;
private String logoId;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
Platform platform = (Platform) o;
return slug != null && Objects.equals(slug, platform.slug);
}
@Override
public int hashCode() {
return getClass().hashCode();
}
}
@@ -3,7 +3,8 @@ package de.grimsi.gameyfin.igdb;
import java.util.List;
public class IgdbApiProperties {
public static final String ENPOINT_GAMES_PROTOBUF = "games.pb";
public static final String ENDPOINT_GAMES_PROTOBUF = "games.pb";
public static final String ENDPOINT_PLATFORMS_PROTOBUF = "platforms.pb";
private static final List<String> GAME_QUERY_FIELDS = List.of(
"slug", "name", "summary", "first_release_date", "rating", "aggregated_rating", "total_rating", "category",
@@ -13,7 +14,8 @@ public class IgdbApiProperties {
"genres.slug", "genres.name",
"keywords.slug", "keywords.name",
"themes.slug", "themes.name",
"player_perspectives.slug", "player_perspectives.name"
"player_perspectives.slug", "player_perspectives.name",
"platforms.slug", "platforms.name", "platforms.platform_logo.image_id"
);
public static final String GAME_QUERY_FIELDS_STRING = String.join(",", GAME_QUERY_FIELDS);
@@ -0,0 +1,448 @@
package de.grimsi.gameyfin.igdb;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
/**
* Builder for fluent building of igdb api queries.
*/
public class IgdbApiQueryBuilder {
private final StringBuilder stringBuilder;
private String search;
private String fields;
private String limit;
private String where;
private String sort;
public IgdbApiQueryBuilder() {
this.stringBuilder = new StringBuilder();
this.fields = "fields *;";
this.search = "";
this.limit = "";
this.where = "";
this.sort = "";
}
/**
* Creates a {@link Condition} that are concatenated through the `&` operator.
* This condition produces `(condition & condition & condition)`.
*
* @param conditions multiple conditions
* @return an {@link AndCondition}
*/
public static Condition and(Condition... conditions) {
return new AndCondition(conditions);
}
/**
* Creates a {@link Condition} that are concatenated through the `|` operator.
* This condition produces `(condition | condition | condition)`.
*
* @param conditions multiple conditions
* @return an {@link OrCondition}
*/
public static Condition or(Condition... conditions) {
return new OrCondition(conditions);
}
/**
* Creates a {@link Condition} to look for string values in a list.
* This condition produces `field = ("val1","val2")`.
*
* @param field a field to search through
* @param values the string values
* @return an {@link InCondition}
*/
public static Condition in(String field, String... values) {
return new InCondition(field, values);
}
/**
* Creates a {@link Condition} to look for number values in a list.
* This condition produces `field = (1,2,3)`.
*
* @param field a field to search through
* @param values the number values
* @return an {@link InCondition}
*/
public static Condition in(String field, Number... values) {
return new InCondition(field, values);
}
/**
* Creates a {@link Condition} to look for values in a collection.
* This condition produces `field = ("val1","val2")` if a collection of strings is passed.
* This condition produces `field = (1,2,3)` if a collection of numbers is passed.
*
* @param field a field to search through.
* @param values a collection of values.
* @return an {@link InCondition}.
*/
public static Condition in(String field, Collection<?> values) {
return new InCondition(field, values.toArray(new Object[0]));
}
/**
* Creates a {@link Condition} to filter for matching string values.
* This condition produces `field = "value"`.
*
* @param field a field to search through.
* @param value a value for comparison.
* @return an {@link InCondition}.
*/
public static Condition equal(String field, String value) {
return new EqualsCondition(field, value);
}
/**
* Creates a {@link Condition} to filter for matching number values.
* This condition produces `field = 123`.
*
* @param field a field to search through.
* @param value a value for comparison.
* @return an {@link InCondition}.
*/
public static Condition equal(String field, Number value) {
return new EqualsCondition(field, value);
}
/**
* Creates a {@link Condition} to filter for non-matching string values.
* This condition produces `field != "value"`.
*
* @param field a field to search through.
* @param value a value for comparison.
* @return an {@link InCondition}.
*/
public static Condition not(String field, String value) {
return new NotCondition(field, value);
}
/**
* Creates a {@link Condition} to filter for non-matching number values.
* This condition produces `field != 123`.
*
* @param field a field to search through.
* @param value a value for comparison.
* @return an {@link InCondition}.
*/
public static Condition not(String field, Number value) {
return new NotCondition(field, value);
}
/**
* Creates a {@link Condition} to check if a value is bigger than the given value.
* This condition produces `field > 123`.
*
* @param field a field to search through.
* @param value a value for comparison.
* @return an {@link InCondition}.
*/
public static Condition greater(String field, Number value) {
return new GreaterThanCondition(field, value);
}
/**
* Creates a {@link Condition} to check if a value is bigger or equal to the given value.
* This condition produces `field >= 123`.
*
* @param field a field to search through.
* @param value a value for comparison.
* @return an {@link InCondition}.
*/
public static Condition greaterEquals(String field, Number value) {
return new GreaterEqualsCondition(field, value);
}
/**
* Creates a {@link Condition} to check if a value is smaller than the given value.
* This condition produces `field < 123`.
*
* @param field a field to search through.
* @param value a value for comparison.
* @return an {@link InCondition}.
*/
public static Condition lesser(String field, Number value) {
return new LessThanCondition(field, value);
}
/**
* Creates a {@link Condition} to check if a value is smaller or equal to the given value.
* This condition produces `field <= 123`.
*
* @param field a field to search through.
* @param value a value for comparison.
* @return an {@link InCondition}.
*/
public static Condition lesserEquals(String field, Number value) {
return new LesserEqualsCondition(field, value);
}
/**
* Builds the query string.
*
* @return an igdb compatible query
*/
public String build() {
stringBuilder.append(search);
stringBuilder.append(fields);
stringBuilder.append(limit);
stringBuilder.append(where);
stringBuilder.append(sort);
return stringBuilder.toString();
}
/**
* Adds the `search "xyz";` query param.
*
* @param searchTerm a term to search for.
* @return the builder
*/
public IgdbApiQueryBuilder search(String searchTerm) {
this.search = "search \"%s\";".formatted(searchTerm);
return this;
}
/**
* Adds the `fields abc,xyz;` query param.
*
* @param fields fields that should be returned (defaults to *).
* @return the builder
*/
public IgdbApiQueryBuilder fields(String fields) {
this.fields = "fields %s;".formatted(fields);
return this;
}
/**
* Adds the `limit 1234;` query param.
*
* @param limit how many results should be returned.
* @return the builder
*/
public IgdbApiQueryBuilder limit(int limit) {
this.limit = "limit %d;".formatted(limit);
return this;
}
/**
* Adds the `where xyz;` query param.
*
* @param condition a {@link Condition} object containing all conditions to filter the igdb db.
* @return the builder
*/
public IgdbApiQueryBuilder where(Condition condition) {
this.where = "where %s;".formatted(condition.build());
return this;
}
/**
* Adds the `sort xyz asc;` query param.
*
* @param field a term to search for.
* @param order the {@link SortOrder} (either ASC or DESC).
* @return the builder
*/
public IgdbApiQueryBuilder sort(String field, SortOrder order) {
this.sort = "sort %s %s;".formatted(field, order.value);
return this;
}
/**
* Sort order enum for sorting query result.
*/
public enum SortOrder {
ASC("asc"), DESC("desc");
public final String value;
SortOrder(String value) {
this.value = value;
}
}
/**
* Abstract condition object.
*/
public abstract static class Condition {
protected static String wrap(String conditions) {
return "(%s)".formatted(conditions);
}
public abstract String build();
}
/**
* InCondition
*/
public static class InCondition extends Condition {
private static final String PATTERN = "%s = (%s)";
private final String field;
private final String in;
public InCondition(String field, Object[] values) {
this.field = field;
if (Arrays.stream(values).anyMatch(String.class::isInstance))
this.in = Arrays.stream(values).map("\"%s\""::formatted).collect(Collectors.joining(","));
else if (Arrays.stream(values).anyMatch(Number.class::isInstance))
this.in = Arrays.stream(values).map("%d"::formatted).collect(Collectors.joining(","));
else this.in = null;
}
@Override
public String build() {
return PATTERN.formatted(field, in);
}
}
/**
* NotCondition
*/
public static class NotCondition extends OperatorCondition {
private static final String OPERATOR = "!=";
public NotCondition(String field, String value) {
super(field, OPERATOR, value);
}
public NotCondition(String field, Number value) {
super(field, OPERATOR, value);
}
}
/**
* EqualsCondition
*/
public static class EqualsCondition extends OperatorCondition {
private static final String OPERATOR = "=";
public EqualsCondition(String field, String value) {
super(field, OPERATOR, value);
}
public EqualsCondition(String field, Number value) {
super(field, OPERATOR, value);
}
}
/**
* GreaterThanCondition
*/
public static class GreaterThanCondition extends OperatorCondition {
private static final String OPERATOR = ">";
public GreaterThanCondition(String field, Number value) {
super(field, OPERATOR, value);
}
}
/**
* GreaterEqualsCondition
*/
public static class GreaterEqualsCondition extends OperatorCondition {
private static final String OPERATOR = ">=";
public GreaterEqualsCondition(String field, Number value) {
super(field, OPERATOR, value);
}
}
/**
* LessThanCondition
*/
public static class LessThanCondition extends OperatorCondition {
private static final String OPERATOR = "<";
public LessThanCondition(String field, Number value) {
super(field, OPERATOR, value);
}
}
/**
* LesserEqualsCondition
*/
public static class LesserEqualsCondition extends OperatorCondition {
private static final String OPERATOR = "<=";
public LesserEqualsCondition(String field, Number value) {
super(field, OPERATOR, value);
}
}
/**
* OperatorCondition for inheritance
*/
public static class OperatorCondition extends Condition {
private static final String PATTERN = "%s %s %s";
private static final String ESCAPED_STRING = "\"%s\"";
private static final String DIGITS = "%s";
private final String field;
private final String value;
private final String operator;
public OperatorCondition(String field, String operator, String value) {
this.field = field;
this.operator = operator;
this.value = value != null ? ESCAPED_STRING.formatted(value) : null;
}
public OperatorCondition(String field, String operator, Number value) {
this.field = field;
this.operator = operator;
this.value = value != null ? DIGITS.formatted(value) : null;
}
public OperatorCondition() {
throw new UnsupportedOperationException();
}
@Override
public String build() {
return PATTERN.formatted(field, operator, value);
}
}
/**
* AndCondition
*/
public static class AndCondition extends Condition {
private static final String AND = " & ";
private final Condition[] conditions;
public AndCondition(Condition[] conditions) {
this.conditions = conditions;
}
@Override
public String build() {
return wrap(Arrays.stream(conditions).map(Condition::build).collect(Collectors.joining(AND)));
}
}
/**
* OrCondition
*/
public static class OrCondition extends Condition {
private static final String OR = " | ";
private final Condition[] conditions;
public OrCondition(Condition[] conditions) {
this.conditions = conditions;
}
@Override
public String build() {
return wrap(Arrays.stream(conditions).map(Condition::build).collect(Collectors.joining(OR)));
}
}
}
@@ -3,8 +3,11 @@ package de.grimsi.gameyfin.igdb;
import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.config.WebClientConfig;
import de.grimsi.gameyfin.dto.AutocompleteSuggestionDto;
import de.grimsi.gameyfin.entities.Platform;
import de.grimsi.gameyfin.igdb.IgdbApiQueryBuilder.Condition;
import de.grimsi.gameyfin.igdb.dto.TwitchOAuthTokenDto;
import de.grimsi.gameyfin.mapper.GameMapper;
import de.grimsi.gameyfin.mapper.PlatformMapper;
import io.github.resilience4j.reactor.bulkhead.operator.BulkheadOperator;
import io.github.resilience4j.reactor.ratelimiter.operator.RateLimiterOperator;
import lombok.RequiredArgsConstructor;
@@ -17,35 +20,34 @@ import org.springframework.web.util.UriComponentsBuilder;
import javax.annotation.PostConstruct;
import java.net.URI;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static de.grimsi.gameyfin.igdb.IgdbApiProperties.GAME_QUERY_FIELDS_STRING;
import static de.grimsi.gameyfin.igdb.IgdbApiQueryBuilder.*;
import static org.apache.commons.lang3.ObjectUtils.isNotEmpty;
@Slf4j
@RequiredArgsConstructor
@Service
public class IgdbWrapper {
@Value("${gameyfin.igdb.api.client-id}")
private String clientId;
@Value("${gameyfin.igdb.api.client-secret}")
private String clientSecret;
@Value("${gameyfin.igdb.config.preferred-platforms:6}")
private String preferredPlatforms;
@Value("${gameyfin.igdb.api.endpoints.base}")
private String igdbApiBaseUrl;
@Value("${gameyfin.igdb.api.endpoints.auth}")
private String twitchAuthUrl;
private final WebClient.Builder webclientBuilder;
private final WebClientConfig webClientConfig;
private final GameMapper gameMapper;
@Value("${gameyfin.igdb.api.client-id}")
private String clientId;
@Value("${gameyfin.igdb.api.client-secret}")
private String clientSecret;
@Value("${gameyfin.igdb.config.preferred-platforms:6}")
private List<Integer> preferredPlatforms;
@Value("${gameyfin.igdb.api.endpoints.base}")
private String igdbApiBaseUrl;
@Value("${gameyfin.igdb.api.endpoints.auth}")
private String twitchAuthUrl;
private WebClient twitchApiClient;
private WebClient igdbApiClient;
@@ -80,9 +82,13 @@ public class IgdbWrapper {
}
public Optional<Igdb.Game> getGameById(Long id) {
IgdbApiQueryBuilder queryBuilder = new IgdbApiQueryBuilder();
Igdb.GameResult gameResult = queryIgdbApi(
IgdbApiProperties.ENPOINT_GAMES_PROTOBUF,
"fields %s; where id = %d; limit 1;".formatted(IgdbApiProperties.GAME_QUERY_FIELDS_STRING, id),
IgdbApiProperties.ENDPOINT_GAMES_PROTOBUF,
queryBuilder.fields(GAME_QUERY_FIELDS_STRING)
.where(equal("id", id))
.limit(1)
.build(),
Igdb.GameResult.class
);
@@ -92,9 +98,13 @@ public class IgdbWrapper {
}
public Optional<Igdb.Game> getGameBySlug(String slug) {
IgdbApiQueryBuilder queryBuilder = new IgdbApiQueryBuilder();
Igdb.GameResult gameResult = queryIgdbApi(
IgdbApiProperties.ENPOINT_GAMES_PROTOBUF,
"fields %s; where slug = \"%s\"; limit 1;".formatted(IgdbApiProperties.GAME_QUERY_FIELDS_STRING, slug),
IgdbApiProperties.ENDPOINT_GAMES_PROTOBUF,
queryBuilder.fields(GAME_QUERY_FIELDS_STRING)
.where(equal("slug", slug))
.limit(1)
.build(),
Igdb.GameResult.class
);
@@ -104,9 +114,14 @@ public class IgdbWrapper {
}
public List<AutocompleteSuggestionDto> findPossibleMatchingTitles(String searchTerm, int limit) {
IgdbApiQueryBuilder queryBuilder = new IgdbApiQueryBuilder();
Igdb.GameResult gameResult = queryIgdbApi(
IgdbApiProperties.ENPOINT_GAMES_PROTOBUF,
"search \"%s\"; fields slug,name,first_release_date; where platforms = (%s); limit %d;".formatted(searchTerm, preferredPlatforms, limit),
IgdbApiProperties.ENDPOINT_GAMES_PROTOBUF,
queryBuilder.search(searchTerm)
.fields("slug,name,first_release_date,platforms.name")
.where(in("platforms", preferredPlatforms))
.limit(limit)
.build(),
Igdb.GameResult.class
);
@@ -116,10 +131,21 @@ public class IgdbWrapper {
}
public Optional<Igdb.Game> searchForGameByTitle(String searchTerm) {
return searchForGameByTitle(searchTerm, List.of());
}
public Optional<Igdb.Game> searchForGameByTitle(String searchTerm, Collection<String> platformSlugs) {
IgdbApiQueryBuilder queryBuilder = new IgdbApiQueryBuilder();
Condition platforms = isNotEmpty(platformSlugs) ?
and(in("platforms", preferredPlatforms), in("platforms.slug", platformSlugs)) :
in("platforms", preferredPlatforms);
Igdb.GameResult gameResult = queryIgdbApi(
IgdbApiProperties.ENPOINT_GAMES_PROTOBUF,
"search \"%s\"; fields %s; where platforms = (%s);"
.formatted(searchTerm, IgdbApiProperties.GAME_QUERY_FIELDS_STRING, preferredPlatforms),
IgdbApiProperties.ENDPOINT_GAMES_PROTOBUF,
queryBuilder.search(searchTerm)
.fields(GAME_QUERY_FIELDS_STRING)
.where(platforms)
.build(),
Igdb.GameResult.class
);
@@ -134,7 +160,7 @@ public class IgdbWrapper {
if (hasBrackets.find()) {
String searchTermWithoutBrackets = searchTerm.split(brackets.pattern())[0].trim();
log.warn("Trying again with search term '{}'", searchTermWithoutBrackets);
return searchForGameByTitle(searchTermWithoutBrackets);
return searchForGameByTitle(searchTermWithoutBrackets, platformSlugs);
}
return Optional.empty();
@@ -186,4 +212,35 @@ public class IgdbWrapper {
.transformDeferred(RateLimiterOperator.of(webClientConfig.getIgdbRateLimiter()))
.block();
}
public List<Platform> findPlatforms(String searchTerm, int limit) {
IgdbApiQueryBuilder queryBuilder = new IgdbApiQueryBuilder();
Igdb.PlatformResult platformResult = queryIgdbApi(
IgdbApiProperties.ENDPOINT_PLATFORMS_PROTOBUF,
queryBuilder.search(searchTerm)
.fields("slug,name")
.limit(limit)
.build(),
Igdb.PlatformResult.class
);
if (platformResult == null) return Collections.emptyList();
return platformResult.getPlatformsList().stream().map(PlatformMapper::toPlatform).toList();
}
public Platform getPlatformBySlug(String slug) {
IgdbApiQueryBuilder queryBuilder = new IgdbApiQueryBuilder();
Igdb.PlatformResult platformResult = queryIgdbApi(
IgdbApiProperties.ENDPOINT_PLATFORMS_PROTOBUF,
queryBuilder.fields("slug,name,platform_logo")
.where(equal("slug", slug))
.build(),
Igdb.PlatformResult.class
);
if (platformResult == null) return null;
return platformResult.getPlatformsList().stream().map(PlatformMapper::toPlatform).findFirst().orElse(null);
}
}
@@ -4,25 +4,23 @@ import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.dto.AutocompleteSuggestionDto;
import de.grimsi.gameyfin.dto.GameOverviewDto;
import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.entities.Library;
import de.grimsi.gameyfin.service.FilesystemService;
import de.grimsi.gameyfin.service.LibraryService;
import de.grimsi.gameyfin.util.ProtobufUtil;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.StopWatch;
import org.springframework.util.StringUtils;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.List;
import java.util.stream.Stream;
import static java.util.stream.Collectors.toList;
@Slf4j
@RequiredArgsConstructor
@@ -31,7 +29,7 @@ public class GameMapper {
private final FilesystemService filesystemService;
public DetectedGame toDetectedGame(Igdb.Game g, Path path) {
public DetectedGame toDetectedGame(Igdb.Game g, Path path, Library library) {
List<Igdb.MultiplayerMode> multiplayerModes = g.getMultiplayerModesList();
List<String> screenshotIds = g.getScreenshotsList().stream().map(Igdb.Screenshot::getImageId).toList();
List<String> videoIds = g.getVideosList().stream().map(Igdb.GameVideo::getVideoId).toList();
@@ -57,7 +55,9 @@ public class GameMapper {
.keywords(KeywordMapper.toKeywords(g.getKeywordsList()))
.themes(ThemeMapper.toThemes(g.getThemesList()))
.playerPerspectives(PlayerPerspectiveMapper.toPlayerPerspectives(g.getPlayerPerspectivesList()))
.platforms(PlatformMapper.toPlatforms(g.getPlatformsList()))
.path(path.toString())
.library(library)
.diskSize(calculateDiskSize(g, path))
.addedToLibrary(Instant.now())
.build();
@@ -76,13 +76,14 @@ public class GameMapper {
.slug(game.getSlug())
.title(game.getName())
.releaseDate(ProtobufUtil.toInstant(game.getFirstReleaseDate()))
.platforms(game.getPlatformsList().stream().map(Igdb.Platform::getName).toList())
.build();
}
private String getCoverId(Igdb.Game g) {
String coverId = g.getCover().getImageId();
if(StringUtils.hasText(coverId)) return coverId;
if (StringUtils.hasText(coverId)) return coverId;
return "nocover";
}
@@ -113,7 +114,7 @@ public class GameMapper {
try {
fileSize = filesystemService.getSizeOnDisk(path);
} catch(IOException e) {
} catch (IOException e) {
log.error("Error while calculating disk size for game '{}'", g.getName());
fileSize = -1L;
}
@@ -0,0 +1,21 @@
package de.grimsi.gameyfin.mapper;
import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.entities.Platform;
import java.util.List;
public class PlatformMapper {
public static Platform toPlatform(Igdb.Platform c) {
return Platform.builder()
.slug(c.getSlug())
.name(c.getName())
.logoId(c.getPlatformLogo().getImageId())
.build();
}
public static List<Platform> toPlatforms(List<Igdb.Platform> c) {
return c.stream().map(PlatformMapper::toPlatform).toList();
}
}
@@ -1,12 +1,17 @@
package de.grimsi.gameyfin.repositories;
import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.entities.UnmappableFile;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import java.nio.file.Path;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.function.Predicate;
import static org.apache.commons.lang3.StringUtils.isBlank;
public interface DetectedGameRepository extends JpaRepository<DetectedGame, String> {
@@ -16,10 +21,17 @@ public interface DetectedGameRepository extends JpaRepository<DetectedGame, Stri
Optional<DetectedGame> findByPath(String path);
List<DetectedGame> getAllByPathNotIn(Collection<String> paths);
List<DetectedGame> findByPathStartsWithAndLibraryIsNull(String path);
default List<DetectedGame> getAllByPathNotIn(List<Path> paths) {
List<DetectedGame> getAllByPathNotIn(Collection<String> paths);
List<DetectedGame> getAllByPathNotInAndPathStartsWith(Collection<String> paths, String libraryPath);
default List<DetectedGame> getAllByPathNotInAndPathStartsWith(List<Path> paths, String libraryPath) {
List<String> pathStrings = paths.stream().map(Path::toString).toList();
return getAllByPathNotIn(pathStrings);
// get games that are not in the paths list but are starting with libraryPath if libraryPath is not empty
return isBlank(libraryPath) ? getAllByPathNotIn(pathStrings) : getAllByPathNotInAndPathStartsWith(pathStrings, libraryPath);
}
}
@@ -0,0 +1,13 @@
package de.grimsi.gameyfin.repositories;
import de.grimsi.gameyfin.entities.Library;
import org.springframework.data.jpa.repository.JpaRepository;
import java.util.Optional;
public interface LibraryRepository extends JpaRepository<Library, String> {
boolean existsByPathIgnoreCase(String path);
Optional<Library> findByPath(String path);
}
@@ -0,0 +1,13 @@
package de.grimsi.gameyfin.repositories;
import de.grimsi.gameyfin.entities.Platform;
import org.springframework.data.jpa.repository.JpaRepository;
import java.util.Optional;
public interface PlatformRepository extends JpaRepository<Platform, String> {
boolean existsBySlug(String slug);
Optional<Platform> findBySlug(String slug);
}
@@ -7,6 +7,9 @@ import java.nio.file.Path;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.function.Predicate;
import static org.apache.commons.lang3.StringUtils.isBlank;
public interface UnmappableFileRepository extends JpaRepository<UnmappableFile, Long> {
@@ -14,10 +17,14 @@ public interface UnmappableFileRepository extends JpaRepository<UnmappableFile,
List<UnmappableFile> getAllByPathNotIn(Collection<String> paths);
List<UnmappableFile> getAllByPathNotInAndPathStartsWith(Collection<String> paths, String libraryPath);
Optional<UnmappableFile> findByPath(String path);
default List<UnmappableFile> getAllByPathNotIn(List<Path> paths) {
default List<UnmappableFile> getAllByPathNotInAndPathStartsWith(List<Path> paths, String libraryPath) {
List<String> pathStrings = paths.stream().map(Path::toString).toList();
return getAllByPathNotIn(pathStrings);
// get unmapped files that are not in the paths list but are starting with libraryPath if libraryPath is not empty
return isBlank(libraryPath) ? getAllByPathNotIn(pathStrings) : getAllByPathNotInAndPathStartsWith(pathStrings, libraryPath);
}
}
@@ -1,9 +1,7 @@
package de.grimsi.gameyfin.rest;
import de.grimsi.gameyfin.dto.ImageDownloadResultDto;
import de.grimsi.gameyfin.dto.LibraryScanResult;
import de.grimsi.gameyfin.dto.LibraryScanResultDto;
import de.grimsi.gameyfin.service.DownloadService;
import de.grimsi.gameyfin.dto.*;
import de.grimsi.gameyfin.entities.Library;
import de.grimsi.gameyfin.service.ImageService;
import de.grimsi.gameyfin.service.LibraryService;
import lombok.RequiredArgsConstructor;
@@ -11,14 +9,13 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.http.MediaType;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.util.StopWatch;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import java.nio.file.Path;
import java.util.List;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
/**
* This controller handles functionality of the library.
*/
@@ -32,20 +29,23 @@ public class LibraryController {
private final LibraryService libraryService;
private final ImageService imageService;
@GetMapping(value = "/scan", produces = MediaType.APPLICATION_JSON_VALUE)
public LibraryScanResultDto scanLibrary(@RequestParam(value = "download_images", defaultValue = "true") boolean downloadImages) {
@PostMapping(value = "/scan", produces = MediaType.APPLICATION_JSON_VALUE)
public LibraryScanResultDto scanLibraries(@RequestBody LibraryScanRequestDto libraryScanRequest) {
StopWatch stopWatch = new StopWatch();
stopWatch.start();
LibraryScanResultDto lscDto = new LibraryScanResultDto();
LibraryScanResult lsc = libraryService.scanGameLibrary();
lscDto.setNewGames(lsc.getNewGames());
lscDto.setDeletedGames(lsc.getDeletedGames());
lscDto.setNewUnmappableFiles(lsc.getNewUnmappableFiles());
lscDto.setTotalGames(lsc.getTotalGames());
String path = libraryScanRequest.getPath();
List<Library> libraries = isNotBlank(path) ? List.of(libraryService.getLibrary(path)) : libraryService.getLibraries();
List<LibraryScanResult> libraryScanResults = libraries.stream().map(libraryService::scanGameLibrary).toList();
if(downloadImages) {
lscDto.setNewGames(libraryScanResults.stream().map(LibraryScanResult::getNewGames).reduce(0, Integer::sum));
lscDto.setDeletedGames(libraryScanResults.stream().map(LibraryScanResult::getDeletedGames).reduce(0, Integer::sum));
lscDto.setNewUnmappableFiles(libraryScanResults.stream().map(LibraryScanResult::getNewUnmappableFiles).reduce(0, Integer::sum));
lscDto.setTotalGames(libraryScanResults.stream().map(LibraryScanResult::getTotalGames).reduce(0, Integer::sum));
if (libraryScanRequest.isDownloadImages()) {
ImageDownloadResultDto idrDto = downloadImages();
lscDto.setCoverDownloads(idrDto.getCoverDownloads());
@@ -4,9 +4,9 @@ package de.grimsi.gameyfin.rest;
import de.grimsi.gameyfin.dto.AutocompleteSuggestionDto;
import de.grimsi.gameyfin.dto.PathToSlugDto;
import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.entities.Library;
import de.grimsi.gameyfin.entities.Platform;
import de.grimsi.gameyfin.entities.UnmappableFile;
import de.grimsi.gameyfin.repositories.DetectedGameRepository;
import de.grimsi.gameyfin.service.DownloadService;
import de.grimsi.gameyfin.service.GameService;
import de.grimsi.gameyfin.service.ImageService;
import de.grimsi.gameyfin.service.LibraryService;
@@ -62,4 +62,20 @@ public class LibraryManagementController {
public List<AutocompleteSuggestionDto> getAutocompleteSuggestions(@RequestParam String searchTerm, @RequestParam(required = false, defaultValue = "10") int limit) {
return libraryService.getAutocompleteSuggestions(searchTerm, limit);
}
@GetMapping(value = "/platforms", produces = MediaType.APPLICATION_JSON_VALUE)
public List<Platform> getPlatforms(@RequestParam String searchTerm, @RequestParam(required = false, defaultValue = "10") int limit) {
return libraryService.getPlatforms(searchTerm, limit);
}
@GetMapping(value = "/libraries", produces = MediaType.APPLICATION_JSON_VALUE)
public List<Library> getLibraries() {
return libraryService.getOrCreateLibraries();
}
@PostMapping(value = "/map-library", produces = MediaType.APPLICATION_JSON_VALUE)
public Library mapPathToPlatform(@RequestBody PathToSlugDto pathToSlugDto) {
return libraryService.mapPlatformsToLibrary(pathToSlugDto.getPath(), pathToSlugDto.getSlug());
}
}
@@ -3,10 +3,12 @@ package de.grimsi.gameyfin.service;
import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.dto.GameOverviewDto;
import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.entities.Library;
import de.grimsi.gameyfin.entities.UnmappableFile;
import de.grimsi.gameyfin.igdb.IgdbWrapper;
import de.grimsi.gameyfin.mapper.GameMapper;
import de.grimsi.gameyfin.repositories.DetectedGameRepository;
import de.grimsi.gameyfin.repositories.LibraryRepository;
import de.grimsi.gameyfin.repositories.UnmappableFileRepository;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -14,6 +16,7 @@ import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.web.server.ResponseStatusException;
import java.io.File;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
@@ -30,6 +33,8 @@ public class GameService {
private final DetectedGameRepository detectedGameRepository;
private final UnmappableFileRepository unmappableFileRepository;
private final LibraryRepository libraryRepository;
public List<DetectedGame> getAllDetectedGames() {
return detectedGameRepository.findAll();
}
@@ -104,7 +109,11 @@ public class GameService {
Igdb.Game igdbGame = igdbWrapper.getGameBySlug(slug)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Game with slug '%s' does not exist on IGDB.".formatted(slug)));
DetectedGame game = gameMapper.toDetectedGame(igdbGame, Path.of(unmappableFile.getPath()));
Path path = Path.of(unmappableFile.getPath());
// parent file should equal to the library
File libraryPath = path.toFile().getParentFile();
Library library = libraryRepository.findByPath(libraryPath.toString()).orElse(null);
DetectedGame game = gameMapper.toDetectedGame(igdbGame, path, library);
game.setConfirmedMatch(true);
game = detectedGameRepository.save(game);
@@ -118,13 +127,13 @@ public class GameService {
Igdb.Game igdbGame = igdbWrapper.getGameBySlug(slug)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Game with slug '%s' does not exist on IGDB.".formatted(slug)));
DetectedGame game = gameMapper.toDetectedGame(igdbGame, Path.of(existingGame.getPath()));
Path path = Path.of(existingGame.getPath());
// parent file should equal to the library
File libraryPath = path.toFile().getParentFile();
Library library = libraryRepository.findByPath(libraryPath.toString()).orElse(null);
DetectedGame game = gameMapper.toDetectedGame(igdbGame, path, library);
game.setConfirmedMatch(true);
game = detectedGameRepository.save(game);
detectedGameRepository.delete(existingGame);
return game;
return detectedGameRepository.save(game);
}
}
@@ -4,29 +4,38 @@ import com.igdb.proto.Igdb;
import de.grimsi.gameyfin.dto.AutocompleteSuggestionDto;
import de.grimsi.gameyfin.dto.LibraryScanResult;
import de.grimsi.gameyfin.entities.DetectedGame;
import de.grimsi.gameyfin.entities.Library;
import de.grimsi.gameyfin.entities.Platform;
import de.grimsi.gameyfin.entities.UnmappableFile;
import de.grimsi.gameyfin.igdb.IgdbWrapper;
import de.grimsi.gameyfin.mapper.GameMapper;
import de.grimsi.gameyfin.repositories.DetectedGameRepository;
import de.grimsi.gameyfin.repositories.LibraryRepository;
import de.grimsi.gameyfin.repositories.PlatformRepository;
import de.grimsi.gameyfin.repositories.UnmappableFileRepository;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.util.StopWatch;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static de.grimsi.gameyfin.util.FilenameUtil.getFilenameWithoutExtension;
import static de.grimsi.gameyfin.util.FilenameUtil.hasGameArchiveExtension;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
import static org.apache.commons.lang3.StringUtils.isBlank;
@Slf4j
@Service
@@ -40,11 +49,17 @@ public class LibraryService {
private final GameMapper gameMapper;
private final DetectedGameRepository detectedGameRepository;
private final UnmappableFileRepository unmappableFileRepository;
private final LibraryRepository libraryRepository;
private final PlatformRepository platformRepository;
public List<Path> getGameFiles() {
return getGameFiles(null);
}
public List<Path> getGameFiles(String path) {
List<Path> gamefiles = new ArrayList<>();
libraryFolders.stream().map(Path::of).forEach(
libraryFolders.stream().map(Path::of).filter(allPathsOrSpecific(path)).forEach(
folder -> {
try (Stream<Path> stream = Files.list(folder)) {
// return all sub-folders (non-recursive) and files that have an extension that indicates that they are a downloadable file
@@ -60,11 +75,11 @@ public class LibraryService {
})
// filter out all empty directories
.filter(p -> {
if(!Files.isDirectory(p)) return true;
if (!Files.isDirectory(p)) return true;
try(DirectoryStream<Path> s = Files.newDirectoryStream(p)) {
try (DirectoryStream<Path> s = Files.newDirectoryStream(p)) {
return s.iterator().hasNext();
} catch(IOException e) {
} catch (IOException e) {
throw new RuntimeException("Error while checking if folder '%s' is empty.".formatted(p), e);
}
})
@@ -81,7 +96,11 @@ public class LibraryService {
return gamefiles;
}
public LibraryScanResult scanGameLibrary() {
private static Predicate<Path> allPathsOrSpecific(String path) {
return p -> isBlank(path) || p.equals(Path.of(path));
}
public LibraryScanResult scanGameLibrary(Library library) {
StopWatch stopWatch = new StopWatch();
log.info("Starting scan...");
@@ -89,16 +108,17 @@ public class LibraryService {
AtomicInteger newUnmappedFilesCounter = new AtomicInteger();
List<Path> gameFiles = getGameFiles();
String libraryPath = library.getPath();
List<Path> gameFiles = getGameFiles(libraryPath);
// Check if any games that are in the library have been removed from the file system
// This would include renamed files, but they will be re-detected by the next step
List<DetectedGame> deletedGames = detectedGameRepository.getAllByPathNotIn(gameFiles);
List<DetectedGame> deletedGames = detectedGameRepository.getAllByPathNotInAndPathStartsWith(gameFiles, libraryPath);
detectedGameRepository.deleteAll(deletedGames);
deletedGames.forEach(g -> log.info("Game '{}' has been moved or deleted.", g.getPath()));
// Now check if there are any unmapped files that have been removed from the file system
List<UnmappableFile> deletedUnmappableFiles = unmappableFileRepository.getAllByPathNotIn(gameFiles);
List<UnmappableFile> deletedUnmappableFiles = unmappableFileRepository.getAllByPathNotInAndPathStartsWith(gameFiles, libraryPath);
unmappableFileRepository.deleteAll(deletedUnmappableFiles);
deletedUnmappableFiles.forEach(g -> log.info("Unmapped file '{}' has been moved or deleted.", g.getPath()));
@@ -109,14 +129,21 @@ public class LibraryService {
.peek(p -> log.info("Found new potential game: {}", p))
.toList();
// Check if library has assigned platforms, so we can search for matching games by specific platforms to get a more accurate match
Set<String> platformsFilter = libraryRepository.findByPath(libraryPath).map(Library::getPlatforms)
.map(platforms -> platforms.stream()
.map(Platform::getSlug)
.collect(Collectors.toSet()))
.orElse(Set.of());
// For each new game, load the info from IGDB
// If a game is not found on IGDB, add it to the list of unmapped files, so we won't query the API later on for the same path
// If a game is not found on IGDB, blacklist the path, so we won't query the API later for the same path
List<DetectedGame> newDetectedGames = gameFiles.parallelStream()
.map(p -> {
Optional<Igdb.Game> optionalGame = igdbWrapper.searchForGameByTitle(getFilenameWithoutExtension(p));
Optional<Igdb.Game> optionalGame = igdbWrapper.searchForGameByTitle(getFilenameWithoutExtension(p), platformsFilter);
if(optionalGame.isPresent() && detectedGameRepository.existsBySlug(optionalGame.get().getSlug())) {
if (optionalGame.isPresent() && detectedGameRepository.existsBySlug(optionalGame.get().getSlug())) {
log.warn("Game with slug '{}' already exists in database", optionalGame.get().getSlug());
optionalGame = Optional.empty();
}
@@ -131,14 +158,21 @@ public class LibraryService {
.filter(Optional::isPresent)
.map(Optional::get)
.peek(e -> log.info("Mapped file '{}' to game '{}' (slug: {})", e.getKey(), e.getValue().getName(), e.getValue().getSlug()))
.map(e -> gameMapper.toDetectedGame(e.getValue(), e.getKey()))
.collect(Collectors.toList());
.map(e -> gameMapper.toDetectedGame(e.getValue(), e.getKey(), library))
.toList();
List<DetectedGame> duplicateGames = getDuplicates(newDetectedGames);
newUnmappedFilesCounter.getAndAdd(duplicateGames.size());
newDetectedGames.removeAll(duplicateGames);
newDetectedGames = detectedGameRepository.saveAll(newDetectedGames);
try {
newDetectedGames = detectedGameRepository.saveAll(newDetectedGames);
} catch (Exception e) {
log.error("Could not save {} detected games!", newDetectedGames.size());
List<UnmappableFile> unmappableFiles = newDetectedGames.stream()
.map(game -> new UnmappableFile(game.getPath())).toList();
unmappableFileRepository.saveAll(unmappableFiles);
}
stopWatch.stop();
@@ -158,11 +192,60 @@ public class LibraryService {
}
private List<DetectedGame> getDuplicates(List<DetectedGame> gamesToFilter) {
return gamesToFilter.stream().filter(g -> Collections.frequency(gamesToFilter, g) >1)
return gamesToFilter.stream().filter(g -> Collections.frequency(gamesToFilter, g) > 1)
.peek(d -> {
log.warn("Found duplicate for game '{}' under path '{}'. Mapping must be done manually.", d.getTitle(), d.getPath());
unmappableFileRepository.save(new UnmappableFile(d.getPath()));
})
.toList();
}
public List<Library> getLibraries() {
return libraryRepository.findAll();
}
public Library getLibrary(String path) {
return libraryRepository.findByPath(path).orElse(null);
}
public List<Library> getOrCreateLibraries() {
libraryFolders.stream().map(Path::of)
.filter(path -> path.toFile().isDirectory()) // check if path is a valid directory
.filter(path -> !libraryRepository.existsByPathIgnoreCase(path.toString()))
.forEach(path -> {
// save new paths as library without platforms
Library library = new Library(path.toString(), List.of());
libraryRepository.save(library);
});
List<Library> libraries = libraryRepository.findAll();
libraries.forEach(library -> {
// remap existing games to this library as well
List<DetectedGame> gamesWithoutLibraryAssignment =
detectedGameRepository.findByPathStartsWithAndLibraryIsNull(library.getPath());
gamesWithoutLibraryAssignment.forEach(game -> game.setLibrary(library));
detectedGameRepository.saveAll(gamesWithoutLibraryAssignment);
});
return libraries;
}
public List<Platform> getPlatforms(String searchTerm, int limit) {
return igdbWrapper.findPlatforms(searchTerm, limit);
}
public Library mapPlatformsToLibrary(String path, String slugs) {
Library library = libraryRepository.findByPath(path)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Could not find library for path %s".formatted(path)));
Set<String> platformSlugs = Arrays.stream(slugs.split(",")).collect(toSet());
List<Platform> platforms = platformSlugs.stream()
.map(slug -> platformRepository.findBySlug(slug).
orElseGet(() -> igdbWrapper.getPlatformBySlug(slug)))
.filter(Objects::nonNull)
.toList();
library.setPlatforms(platforms);
libraryRepository.save(library);
return library;
}
}
@@ -0,0 +1,45 @@
-- Add platforms
-- Platforms
CREATE TABLE platform
(
slug VARCHAR(255) NOT NULL,
name VARCHAR(255),
logo_id VARCHAR(255),
PRIMARY KEY (slug)
);
-- Game <-> Platforms
CREATE TABLE detected_game_platforms
(
detected_game_slug VARCHAR(255) NOT NULL,
platforms_slug VARCHAR(255) NOT NULL
);
ALTER TABLE detected_game_platforms
ADD CONSTRAINT platforms_platform_slug FOREIGN KEY (platforms_slug) REFERENCES platform;
ALTER TABLE detected_game_platforms
ADD CONSTRAINT platforms_detected_game_slug FOREIGN KEY (detected_game_slug) REFERENCES detected_game;
-- Add libraries
-- Libraries
CREATE TABLE library
(
path VARCHAR(255) NOT NULL,
PRIMARY KEY (path)
);
-- Library <-> Platforms
CREATE TABLE library_platforms
(
library_path VARCHAR(255) NOT NULL,
platforms_slug VARCHAR(255) NOT NULL
);
ALTER TABLE library_platforms
ADD CONSTRAINT libraries_platform_slug FOREIGN KEY (platforms_slug) REFERENCES platform;
ALTER TABLE library_platforms
ADD CONSTRAINT libraries_library_path FOREIGN KEY (library_path) REFERENCES library;
-- Library <-> Game
ALTER TABLE detected_game
ADD library VARCHAR(255);