moparisthebest/filebot
1
0
Fork 0
mirror of https://github.com/mitb-archive/filebot synced 2024-12-21 15:28:52 -05:00
Code Issues Releases Wiki Activity

Refactor AnimeList client

Browse Source
This commit is contained in:
Reinhard Pointner 2019-06-11 01:49:38 +07:00
parent b8d9f75de8
commit 9d0f7c9220
7 changed files with 75 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
source/net/filebot/WebServices.java
View File

@ -27,7 +27,6 @@ import javax.swing.Icon;
import net.filebot.media.LocalDatasource;
import net.filebot.similarity.MetricAvg;
import net.filebot.web.AcoustIDClient;
import net.filebot.web.AnidbClient;
import net.filebot.web.Artwork;
import net.filebot.web.Datasource;
import net.filebot.web.EpisodeListProvider;
@ -39,15 +38,12 @@ import net.filebot.web.Movie;
import net.filebot.web.MovieIdentificationService;
import net.filebot.web.MusicIdentificationService;
import net.filebot.web.OMDbClient;
import net.filebot.web.OpenSubtitlesClient;
import net.filebot.web.SearchResult;
import net.filebot.web.ShooterSubtitles;
import net.filebot.web.SubtitleProvider;
import net.filebot.web.SubtitleSearchResult;
import net.filebot.web.TMDbClient;
import net.filebot.web.TMDbTVClient;
import net.filebot.web.TVMazeClient;
import net.filebot.web.TheTVDBClient;
import net.filebot.web.TheTVDBSearchResult;
import net.filebot.web.ThumbnailProvider;
import net.filebot.web.VideoHashSubtitleService;
@ -59,27 +55,28 @@ public final class WebServices {
// movie sources
public static final OMDbClient OMDb = new OMDbClient(getApiKey("omdb"));
public static final TMDbClient TheMovieDB = new TMDbClientWithLocalSearch(getApiKey("themoviedb"));
public static final net.filebot.web.TMDbClient TheMovieDB = new TMDbClient(getApiKey("themoviedb"));
// episode sources
public static final TVMazeClient TVmaze = new TVMazeClient();
public static final AnidbClient AniDB = new AnidbClientWithLocalSearch(getApiKey("anidb"), 7);
public static final net.filebot.web.AnidbClient AniDB = new AnidbClient(getApiKey("anidb"), 7);
// extended TheTVDB module with local search
public static final TheTVDBClientWithLocalSearch TheTVDB = new TheTVDBClientWithLocalSearch(getApiKey("thetvdb"));
public static final net.filebot.web.TheTVDBClient TheTVDB = new TheTVDBClient(getApiKey("thetvdb"));
public static final TMDbTVClient TheMovieDB_TV = new TMDbTVClient(TheMovieDB);
// subtitle sources
public static final OpenSubtitlesClient OpenSubtitles = new OpenSubtitlesClientWithLocalSearch(getApiKey("opensubtitles"), getApplicationVersion());
public static final net.filebot.web.OpenSubtitlesClient OpenSubtitles = new OpenSubtitlesClient(getApiKey("opensubtitles"), getApplicationVersion());
public static final ShooterSubtitles Shooter = new ShooterSubtitles();
// other sources
public static final net.filebot.web.AnimeLists AnimeList = new AnimeLists();
public static final FanartTVClient FanartTV = new FanartTVClient(getApiKey("fanart.tv"));
public static final AcoustIDClient AcoustID = new AcoustIDClient(getApiKey("acoustid"));
public static final ID3Lookup MediaInfoID3 = new ID3Lookup();
public static Datasource[] getServices() {
return new Datasource[] { TheMovieDB, OMDb, TheTVDB, AniDB, TheMovieDB_TV, TVmaze, AcoustID, MediaInfoID3, LocalDatasource.EXIF, LocalDatasource.XATTR, LocalDatasource.FILE, OpenSubtitles, Shooter, FanartTV };
return new Datasource[] { TheMovieDB, OMDb, TheTVDB, AniDB, TheMovieDB_TV, TVmaze, AcoustID, MediaInfoID3, LocalDatasource.EXIF, LocalDatasource.XATTR, LocalDatasource.FILE, OpenSubtitles, Shooter, AnimeList, FanartTV };
}
public static MovieIdentificationService[] getMovieIdentificationServices() {
@ -135,9 +132,9 @@ public final class WebServices {
public static final ExecutorService requestThreadPool = Executors.newCachedThreadPool();
public static class TMDbClientWithLocalSearch extends TMDbClient implements ThumbnailProvider {
private static class TMDbClient extends net.filebot.web.TMDbClient implements ThumbnailProvider {
public TMDbClientWithLocalSearch(String apikey) {
public TMDbClient(String apikey) {
super(apikey);
}
@ -172,7 +169,7 @@ public final class WebServices {
List<Callable<List<Movie>>> searches = new ArrayList<>();
// online API search first
searches.add(() -> TMDbClientWithLocalSearch.super.searchMovie(movieName, movieYear, locale, extendedInfo));
searches.add(() -> TMDbClient.super.searchMovie(movieName, movieYear, locale, extendedInfo));
if (movieYear > 0) {
// the year might be off by 1 so we also check movies from the previous year and the next year
@ -200,9 +197,9 @@ public final class WebServices {
}
}
public static class TheTVDBClientWithLocalSearch extends TheTVDBClient implements ThumbnailProvider {
private static class TheTVDBClient extends net.filebot.web.TheTVDBClient implements ThumbnailProvider {
public TheTVDBClientWithLocalSearch(String apikey) {
public TheTVDBClient(String apikey) {
super(apikey);
}
@ -227,7 +224,7 @@ public final class WebServices {
@Override
public List<SearchResult> fetchSearchResult(String query, Locale locale) throws Exception {
// run local search and API search in parallel
Future<List<SearchResult>> apiSearch = requestThreadPool.submit(() -> TheTVDBClientWithLocalSearch.super.fetchSearchResult(query, locale));
Future<List<SearchResult>> apiSearch = requestThreadPool.submit(() -> TheTVDBClient.super.fetchSearchResult(query, locale));
Future<List<SearchResult>> localSearch = requestThreadPool.submit(() -> localIndex.get().search(query));
// combine alias names into a single search results, and keep API search name as primary name
@ -242,9 +239,9 @@ public final class WebServices {
}
}
public static class AnidbClientWithLocalSearch extends AnidbClient implements ThumbnailProvider {
private static class AnidbClient extends net.filebot.web.AnidbClient implements ThumbnailProvider {
public AnidbClientWithLocalSearch(String client, int clientver) {
public AnidbClient(String client, int clientver) {
super(client, clientver);
}
@ -264,9 +261,17 @@ public final class WebServices {
}
}
public static class OpenSubtitlesClientWithLocalSearch extends OpenSubtitlesClient {
private static class AnimeLists extends net.filebot.web.AnimeLists {
public OpenSubtitlesClientWithLocalSearch(String name, String version) {
@Override
public net.filebot.web.AnimeLists.Model getModel() throws Exception {
return releaseInfo.getAnimeListModel();
}
}
private static class OpenSubtitlesClient extends net.filebot.web.OpenSubtitlesClient {
public OpenSubtitlesClient(String name, String version) {
super(name, version);
}

10
source/net/filebot/format/MediaBindingBean.java
View File

@ -65,7 +65,7 @@ import net.filebot.mediainfo.MediaInfoException;
import net.filebot.similarity.Normalization;
import net.filebot.similarity.SimilarityComparator;
import net.filebot.util.FileUtilities;
import net.filebot.web.AnimeList;
import net.filebot.web.AnimeLists;
import net.filebot.web.AudioTrack;
import net.filebot.web.Episode;
import net.filebot.web.EpisodeFormat;
@ -797,7 +797,7 @@ public class MediaBindingBean {
}
if (AniDB.getIdentifier().equals(e.getSeriesInfo().getDatabase())) {
return new AnimeList().map(e, AnimeList.getDB(e), AnimeList.DB.TheTVDB).map(this::createBindingObject).orElse(null); // map AniDB to TheTVDB bindings
return AnimeList.map(e, AnimeLists.getDB(e), AnimeLists.DB.TheTVDB).map(this::createBindingObject).orElse(null); // map AniDB to TheTVDB bindings
}
return createBindingObject(fetchEpisode(e, SortOrder.Airdate, null));
@ -1175,13 +1175,13 @@ public class MediaBindingBean {
@Define("AnimeList")
public DynamicBindings getAnimeLists() {
return new DynamicBindings(AnimeList.DB::names, k -> {
return new DynamicBindings(AnimeLists.DB::names, k -> {
if (infoObject instanceof Episode) {
Episode e = getEpisode();
if (AnimeList.getDB(e) == AnimeList.getDB(k)) {
if (AnimeLists.getDB(e) == AnimeLists.getDB(k)) {
return e;
}
return new AnimeList().map(e, AnimeList.getDB(e), AnimeList.getDB(k)).orElse(e);
return AnimeList.map(e, AnimeLists.getDB(e), AnimeLists.getDB(k)).orElse(e);
}
return undefined(k);
});

23
source/net/filebot/media/ReleaseInfo.java
View File

@ -36,7 +36,6 @@ import java.util.function.IntFunction;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.tukaani.xz.XZInputStream;
@ -47,6 +46,7 @@ import net.filebot.Resource;
import net.filebot.util.FileUtilities.RegexFindFilter;
import net.filebot.util.FileUtilities.RegexMatchFilter;
import net.filebot.util.SystemProperty;
import net.filebot.web.AnimeLists;
import net.filebot.web.Movie;
import net.filebot.web.SearchResult;
import net.filebot.web.SubtitleSearchResult;
@ -385,6 +385,10 @@ public class ReleaseInfo {
return osdbIndex.get();
}
public AnimeLists.Model getAnimeListModel() throws Exception {
return animeListModel.get();
}
private static FolderEntryFilter diskFolderFilter;
public FileFilter getDiskFolderFilter() {
@ -449,6 +453,10 @@ public class ReleaseInfo {
return unmodifiableMap(map);
}).memoize();
private final Resource<AnimeLists.Model> animeListModel = resource("url.anime-list", Cache.ONE_WEEK, bytes -> {
return AnimeLists.unmarshal(bytes, AnimeLists.Model.class);
}).memoize();
private final Resource<String[]> releaseGroup = lines("url.release-groups", Cache.ONE_WEEK);
private final Resource<String[]> queryBlacklist = lines("url.query-blacklist", Cache.ONE_WEEK);
@ -495,14 +503,17 @@ public class ReleaseInfo {
}
protected <A> Resource<A[]> resource(String name, Duration expirationTime, Function<String, A> parse, IntFunction<A[]> generator) {
// all data files are UTF-8 encoded XZ compressed text files
return resource(name, expirationTime, bytes -> {
return NEWLINE.splitAsStream(UTF_8.decode(ByteBuffer.wrap(bytes))).filter(s -> s.length() > 0).map(parse).filter(Objects::nonNull).toArray(generator);
});
}
protected <A> Resource<A> resource(String name, Duration expirationTime, Function<byte[], A> parse) {
return () -> {
Cache cache = Cache.getCache("data", CacheType.Persistent);
byte[] bytes = cache.bytes(name, n -> new URL(getProperty(n)), XZInputStream::new).expire(refreshDuration.optional().orElse(expirationTime)).get();
// all data files are UTF-8 encoded XZ compressed text files
Stream<String> lines = NEWLINE.splitAsStream(UTF_8.decode(ByteBuffer.wrap(bytes)));
return lines.filter(s -> s.length() > 0).map(parse).filter(Objects::nonNull).toArray(generator);
return parse.apply(bytes);
};
}

3
source/net/filebot/media/ReleaseInfo.properties
View File

@ -46,6 +46,9 @@ url.anidb-index: @{url.data}/anidb.txt.xz
# OpenSubtitles index
url.osdb-index: @{url.data}/osdb.txt.xz
# Anime List mappings
url.anime-list: @{url.data}/anime-list.xml.xz
# disk folder matcher
pattern.diskfolder.entry: BDMV|HVDVD_TS|VIDEO_TS|AUDIO_TS|VCD|MovieObject.bdmv|VIDEO_TS.VOB|VTS_[0-9]+_[0-9]+.VOB

2
source/net/filebot/web/AnidbClient.java
View File

@ -223,7 +223,7 @@ public class AnidbClient extends AbstractEpisodeListProvider implements ArtworkP
}
/**
* This method is overridden in {@link net.filebot.WebServices.AnidbClientWithLocalSearch} to fetch the Anime Index from our own host and not anidb.net
* This method is overridden in {@link net.filebot.WebServices.AnidbClient} to fetch the Anime Index from our own host and not anidb.net
*/
public SearchResult[] getAnimeTitles() throws Exception {
// get data file (unzip and cache)

57
source/net/filebot/web/AnimeList.java → source/net/filebot/web/AnimeLists.java
View File

@ -2,7 +2,6 @@ package net.filebot.web;
import static java.util.Arrays.*;
import static java.util.stream.Collectors.*;
import static net.filebot.CachedResource.*;
import static net.filebot.Logging.*;
import static net.filebot.util.RegularExpressions.*;
import static net.filebot.util.StringUtilities.*;
@ -16,6 +15,7 @@ import java.util.Optional;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import javax.swing.Icon;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.Marshaller;
import javax.xml.bind.annotation.XmlAttribute;
@ -28,19 +28,18 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import net.filebot.Cache;
import net.filebot.CacheType;
import net.filebot.Resource;
import net.filebot.WebServices;
public class AnimeList {
public class AnimeLists implements Datasource {
private Model model;
public AnimeList() throws Exception {
this.model = MODEL.get();
@Override
public String getIdentifier() {
return "AnimeLists";
}
public Model getModel() {
return model;
@Override
public Icon getIcon() {
return null;
}
public Optional<Episode> map(Episode episode, DB source, DB destination) throws Exception {
@ -84,7 +83,7 @@ public class AnimeList {
}).findFirst();
}
private Episode derive(DB db, Entry a, Episode episode, int s, int e) {
protected Episode derive(DB db, Entry a, Episode episode, int s, int e) {
if (s == 0) {
// special
switch (db) {
@ -202,7 +201,7 @@ public class AnimeList {
}
public Stream<Entry> find(DB db, int id) throws Exception {
return stream(model.anime).filter(this::isValid).filter(a -> id == getId(db, a));
return stream(getModel().anime).filter(this::isValid).filter(a -> id == getId(db, a));
}
public Stream<Entry> find(DB db, int id, int s) throws Exception {
@ -214,21 +213,10 @@ public class AnimeList {
}
}
protected static Cache getCache() {
return Cache.getCache("animelists", CacheType.Persistent);
}
protected static final Resource<Model> MODEL = Resource.lazy(() -> unmarshal(request("anime-list.xml"), Model.class));
protected static byte[] request(String file) throws Exception {
// NOTE: GitHub only supports If-None-Match (If-Modified-Since is ignored)
Cache cache = getCache();
return cache.bytes(file, AnimeList::getResource).fetch(fetchIfNoneMatch(url -> file, cache)).expire(Cache.ONE_MONTH).get();
}
protected static URL getResource(String file) throws Exception {
return new URL("https://raw.githubusercontent.com/ScudLee/anime-lists/master/" + file);
public Model getModel() throws Exception {
return Cache.getCache(getIdentifier(), CacheType.Monthly).bytes("https://github.com/ScudLee/anime-lists/raw/master/anime-list.xml", URL::new).transform(bytes -> {
return unmarshal(bytes, Model.class);
}).get();
}
public static DB getDB(Episode e) {
@ -325,10 +313,9 @@ public class AnimeList {
public String toString() {
return marshal(this, Mapping.class);
}
}
private static class NumberAdapter extends XmlAdapter<String, Integer> {
protected static class NumberAdapter extends XmlAdapter<String, Integer> {
@Override
public Integer unmarshal(String s) throws Exception {
@ -341,7 +328,7 @@ public class AnimeList {
}
}
private static class NumberMapAdapter extends XmlAdapter<String, int[][]> {
protected static class NumberMapAdapter extends XmlAdapter<String, int[][]> {
@Override
public int[][] unmarshal(String s) throws Exception {
@ -354,15 +341,19 @@ public class AnimeList {
}
}
private static <T> T unmarshal(byte[] bytes, Class<T> type) throws Exception {
return (T) JAXBContext.newInstance(type).createUnmarshaller().unmarshal(new ByteArrayInputStream(bytes));
public static <T> T unmarshal(byte[] bytes, Class<T> type) {
try {
return (T) JAXBContext.newInstance(type).createUnmarshaller().unmarshal(new ByteArrayInputStream(bytes));
} catch (Exception e) {
throw new IllegalArgumentException(e);
}
}
private static <T> String marshal(T object, Class<T> type) {
public static <T> String marshal(T object, Class<T> type) {
try {
StringWriter buffer = new StringWriter();
Marshaller marshaller = JAXBContext.newInstance(type).createMarshaller();
marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.FALSE);
marshaller.setProperty(Marshaller.JAXB_FRAGMENT, Boolean.TRUE);
marshaller.marshal(object, buffer);
return buffer.toString();

2
source/net/filebot/web/EpisodeUtilities.java
View File

@ -55,7 +55,7 @@ public final class EpisodeUtilities {
if (episode.isAnime() && episode.isRegular()) {
return mapEpisode(episode, e -> {
try {
return new AnimeList().map(e, AnimeList.getDB(e), AnimeList.DB.TheTVDB).orElse(e);
return AnimeList.map(e, AnimeLists.getDB(e), AnimeLists.DB.TheTVDB).orElse(e);
} catch (Exception ioe) {
debug.warning(ioe::toString);
return e;