Refactor AnimeList client

source/net/filebot/WebServices.java

@@ -27,7 +27,6 @@ import javax.swing.Icon;
 import net.filebot.media.LocalDatasource;
 import net.filebot.similarity.MetricAvg;
 import net.filebot.web.AcoustIDClient;
-import net.filebot.web.AnidbClient;
 import net.filebot.web.Artwork;
 import net.filebot.web.Datasource;
 import net.filebot.web.EpisodeListProvider;
@@ -39,15 +38,12 @@ import net.filebot.web.Movie;
 import net.filebot.web.MovieIdentificationService;
 import net.filebot.web.MusicIdentificationService;
 import net.filebot.web.OMDbClient;
-import net.filebot.web.OpenSubtitlesClient;
 import net.filebot.web.SearchResult;
 import net.filebot.web.ShooterSubtitles;
 import net.filebot.web.SubtitleProvider;
 import net.filebot.web.SubtitleSearchResult;
-import net.filebot.web.TMDbClient;
 import net.filebot.web.TMDbTVClient;
 import net.filebot.web.TVMazeClient;
-import net.filebot.web.TheTVDBClient;
 import net.filebot.web.TheTVDBSearchResult;
 import net.filebot.web.ThumbnailProvider;
 import net.filebot.web.VideoHashSubtitleService;
@@ -59,27 +55,28 @@ public final class WebServices {
 
 	// movie sources
 	public static final OMDbClient OMDb = new OMDbClient(getApiKey("omdb"));
-	public static final TMDbClient TheMovieDB = new TMDbClientWithLocalSearch(getApiKey("themoviedb"));
+	public static final net.filebot.web.TMDbClient TheMovieDB = new TMDbClient(getApiKey("themoviedb"));
 
 	// episode sources
 	public static final TVMazeClient TVmaze = new TVMazeClient();
-	public static final AnidbClient AniDB = new AnidbClientWithLocalSearch(getApiKey("anidb"), 7);
+	public static final net.filebot.web.AnidbClient AniDB = new AnidbClient(getApiKey("anidb"), 7);
 
 	// extended TheTVDB module with local search
-	public static final TheTVDBClientWithLocalSearch TheTVDB = new TheTVDBClientWithLocalSearch(getApiKey("thetvdb"));
+	public static final net.filebot.web.TheTVDBClient TheTVDB = new TheTVDBClient(getApiKey("thetvdb"));
 	public static final TMDbTVClient TheMovieDB_TV = new TMDbTVClient(TheMovieDB);
 
 	// subtitle sources
-	public static final OpenSubtitlesClient OpenSubtitles = new OpenSubtitlesClientWithLocalSearch(getApiKey("opensubtitles"), getApplicationVersion());
+	public static final net.filebot.web.OpenSubtitlesClient OpenSubtitles = new OpenSubtitlesClient(getApiKey("opensubtitles"), getApplicationVersion());
 	public static final ShooterSubtitles Shooter = new ShooterSubtitles();
 
 	// other sources
+	public static final net.filebot.web.AnimeLists AnimeList = new AnimeLists();
 	public static final FanartTVClient FanartTV = new FanartTVClient(getApiKey("fanart.tv"));
 	public static final AcoustIDClient AcoustID = new AcoustIDClient(getApiKey("acoustid"));
 	public static final ID3Lookup MediaInfoID3 = new ID3Lookup();
 
 	public static Datasource[] getServices() {
-		return new Datasource[] { TheMovieDB, OMDb, TheTVDB, AniDB, TheMovieDB_TV, TVmaze, AcoustID, MediaInfoID3, LocalDatasource.EXIF, LocalDatasource.XATTR, LocalDatasource.FILE, OpenSubtitles, Shooter, FanartTV };
+		return new Datasource[] { TheMovieDB, OMDb, TheTVDB, AniDB, TheMovieDB_TV, TVmaze, AcoustID, MediaInfoID3, LocalDatasource.EXIF, LocalDatasource.XATTR, LocalDatasource.FILE, OpenSubtitles, Shooter, AnimeList, FanartTV };
 	}
 
 	public static MovieIdentificationService[] getMovieIdentificationServices() {
@@ -135,9 +132,9 @@ public final class WebServices {
 
 	public static final ExecutorService requestThreadPool = Executors.newCachedThreadPool();
 
-	public static class TMDbClientWithLocalSearch extends TMDbClient implements ThumbnailProvider {
+	private static class TMDbClient extends net.filebot.web.TMDbClient implements ThumbnailProvider {
 
-		public TMDbClientWithLocalSearch(String apikey) {
+		public TMDbClient(String apikey) {
 			super(apikey);
 		}
 
@@ -172,7 +169,7 @@ public final class WebServices {
 			List<Callable<List<Movie>>> searches = new ArrayList<>();
 
 			// online API search first
-			searches.add(() -> TMDbClientWithLocalSearch.super.searchMovie(movieName, movieYear, locale, extendedInfo));
+			searches.add(() -> TMDbClient.super.searchMovie(movieName, movieYear, locale, extendedInfo));
 
 			if (movieYear > 0) {
 				// the year might be off by 1 so we also check movies from the previous year and the next year
@@ -200,9 +197,9 @@ public final class WebServices {
 		}
 	}
 
-	public static class TheTVDBClientWithLocalSearch extends TheTVDBClient implements ThumbnailProvider {
+	private static class TheTVDBClient extends net.filebot.web.TheTVDBClient implements ThumbnailProvider {
 
-		public TheTVDBClientWithLocalSearch(String apikey) {
+		public TheTVDBClient(String apikey) {
 			super(apikey);
 		}
 
@@ -227,7 +224,7 @@ public final class WebServices {
 		@Override
 		public List<SearchResult> fetchSearchResult(String query, Locale locale) throws Exception {
 			// run local search and API search in parallel
-			Future<List<SearchResult>> apiSearch = requestThreadPool.submit(() -> TheTVDBClientWithLocalSearch.super.fetchSearchResult(query, locale));
+			Future<List<SearchResult>> apiSearch = requestThreadPool.submit(() -> TheTVDBClient.super.fetchSearchResult(query, locale));
 			Future<List<SearchResult>> localSearch = requestThreadPool.submit(() -> localIndex.get().search(query));
 
 			// combine alias names into a single search results, and keep API search name as primary name
@@ -242,9 +239,9 @@ public final class WebServices {
 		}
 	}
 
-	public static class AnidbClientWithLocalSearch extends AnidbClient implements ThumbnailProvider {
+	private static class AnidbClient extends net.filebot.web.AnidbClient implements ThumbnailProvider {
 
-		public AnidbClientWithLocalSearch(String client, int clientver) {
+		public AnidbClient(String client, int clientver) {
 			super(client, clientver);
 		}
 
@@ -264,9 +261,17 @@ public final class WebServices {
 		}
 	}
 
-	public static class OpenSubtitlesClientWithLocalSearch extends OpenSubtitlesClient {
+	private static class AnimeLists extends net.filebot.web.AnimeLists {
 
-		public OpenSubtitlesClientWithLocalSearch(String name, String version) {
+		@Override
+		public net.filebot.web.AnimeLists.Model getModel() throws Exception {
+			return releaseInfo.getAnimeListModel();
+		}
+	}
+
+	private static class OpenSubtitlesClient extends net.filebot.web.OpenSubtitlesClient {
+
+		public OpenSubtitlesClient(String name, String version) {
 			super(name, version);
 		}
 

source/net/filebot/format/MediaBindingBean.java

@@ -65,7 +65,7 @@ import net.filebot.mediainfo.MediaInfoException;
 import net.filebot.similarity.Normalization;
 import net.filebot.similarity.SimilarityComparator;
 import net.filebot.util.FileUtilities;
-import net.filebot.web.AnimeList;
+import net.filebot.web.AnimeLists;
 import net.filebot.web.AudioTrack;
 import net.filebot.web.Episode;
 import net.filebot.web.EpisodeFormat;
@@ -797,7 +797,7 @@ public class MediaBindingBean {
 		}
 
 		if (AniDB.getIdentifier().equals(e.getSeriesInfo().getDatabase())) {
-			return new AnimeList().map(e, AnimeList.getDB(e), AnimeList.DB.TheTVDB).map(this::createBindingObject).orElse(null); // map AniDB to TheTVDB bindings
+			return AnimeList.map(e, AnimeLists.getDB(e), AnimeLists.DB.TheTVDB).map(this::createBindingObject).orElse(null); // map AniDB to TheTVDB bindings
 		}
 
 		return createBindingObject(fetchEpisode(e, SortOrder.Airdate, null));
@@ -1175,13 +1175,13 @@ public class MediaBindingBean {
 
 	@Define("AnimeList")
 	public DynamicBindings getAnimeLists() {
-		return new DynamicBindings(AnimeList.DB::names, k -> {
+		return new DynamicBindings(AnimeLists.DB::names, k -> {
 			if (infoObject instanceof Episode) {
 				Episode e = getEpisode();
-				if (AnimeList.getDB(e) == AnimeList.getDB(k)) {
+				if (AnimeLists.getDB(e) == AnimeLists.getDB(k)) {
 					return e;
 				}
-				return new AnimeList().map(e, AnimeList.getDB(e), AnimeList.getDB(k)).orElse(e);
+				return AnimeList.map(e, AnimeLists.getDB(e), AnimeLists.getDB(k)).orElse(e);
 			}
 			return undefined(k);
 		});

source/net/filebot/media/ReleaseInfo.java

@@ -36,7 +36,6 @@ import java.util.function.IntFunction;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.IntStream;
-import java.util.stream.Stream;
 
 import org.tukaani.xz.XZInputStream;
 
@@ -47,6 +46,7 @@ import net.filebot.Resource;
 import net.filebot.util.FileUtilities.RegexFindFilter;
 import net.filebot.util.FileUtilities.RegexMatchFilter;
 import net.filebot.util.SystemProperty;
+import net.filebot.web.AnimeLists;
 import net.filebot.web.Movie;
 import net.filebot.web.SearchResult;
 import net.filebot.web.SubtitleSearchResult;
@@ -385,6 +385,10 @@ public class ReleaseInfo {
 		return osdbIndex.get();
 	}
 
+	public AnimeLists.Model getAnimeListModel() throws Exception {
+		return animeListModel.get();
+	}
+
 	private static FolderEntryFilter diskFolderFilter;
 
 	public FileFilter getDiskFolderFilter() {
@@ -449,6 +453,10 @@ public class ReleaseInfo {
 		return unmodifiableMap(map);
 	}).memoize();
 
+	private final Resource<AnimeLists.Model> animeListModel = resource("url.anime-list", Cache.ONE_WEEK, bytes -> {
+		return AnimeLists.unmarshal(bytes, AnimeLists.Model.class);
+	}).memoize();
+
 	private final Resource<String[]> releaseGroup = lines("url.release-groups", Cache.ONE_WEEK);
 	private final Resource<String[]> queryBlacklist = lines("url.query-blacklist", Cache.ONE_WEEK);
 
@@ -495,14 +503,17 @@ public class ReleaseInfo {
 	}
 
 	protected <A> Resource<A[]> resource(String name, Duration expirationTime, Function<String, A> parse, IntFunction<A[]> generator) {
+		// all data files are UTF-8 encoded XZ compressed text files
+		return resource(name, expirationTime, bytes -> {
+			return NEWLINE.splitAsStream(UTF_8.decode(ByteBuffer.wrap(bytes))).filter(s -> s.length() > 0).map(parse).filter(Objects::nonNull).toArray(generator);
+		});
+	}
+
+	protected <A> Resource<A> resource(String name, Duration expirationTime, Function<byte[], A> parse) {
 		return () -> {
 			Cache cache = Cache.getCache("data", CacheType.Persistent);
 			byte[] bytes = cache.bytes(name, n -> new URL(getProperty(n)), XZInputStream::new).expire(refreshDuration.optional().orElse(expirationTime)).get();
-
+			return parse.apply(bytes);
-			// all data files are UTF-8 encoded XZ compressed text files
-			Stream<String> lines = NEWLINE.splitAsStream(UTF_8.decode(ByteBuffer.wrap(bytes)));
-
-			return lines.filter(s -> s.length() > 0).map(parse).filter(Objects::nonNull).toArray(generator);
 		};
 	}
 

source/net/filebot/media/ReleaseInfo.properties

@@ -46,6 +46,9 @@ url.anidb-index: @{url.data}/anidb.txt.xz
 # OpenSubtitles index
 url.osdb-index: @{url.data}/osdb.txt.xz
 
+# Anime List mappings
+url.anime-list: @{url.data}/anime-list.xml.xz
+
 # disk folder matcher
 pattern.diskfolder.entry: BDMV|HVDVD_TS|VIDEO_TS|AUDIO_TS|VCD|MovieObject.bdmv|VIDEO_TS.VOB|VTS_[0-9]+_[0-9]+.VOB
 

source/net/filebot/web/AnidbClient.java

@@ -223,7 +223,7 @@ public class AnidbClient extends AbstractEpisodeListProvider implements ArtworkP
 	}
 
 	/**
-	 * This method is overridden in {@link net.filebot.WebServices.AnidbClientWithLocalSearch} to fetch the Anime Index from our own host and not anidb.net
+	 * This method is overridden in {@link net.filebot.WebServices.AnidbClient} to fetch the Anime Index from our own host and not anidb.net
 	 */
 	public SearchResult[] getAnimeTitles() throws Exception {
 		// get data file (unzip and cache)

source/net/filebot/web/AnimeLists.java

@@ -2,7 +2,6 @@ package net.filebot.web;
 
 import static java.util.Arrays.*;
 import static java.util.stream.Collectors.*;
-import static net.filebot.CachedResource.*;
 import static net.filebot.Logging.*;
 import static net.filebot.util.RegularExpressions.*;
 import static net.filebot.util.StringUtilities.*;
@@ -16,6 +15,7 @@ import java.util.Optional;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
+import javax.swing.Icon;
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.Marshaller;
 import javax.xml.bind.annotation.XmlAttribute;
@@ -28,19 +28,18 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 
 import net.filebot.Cache;
 import net.filebot.CacheType;
-import net.filebot.Resource;
 import net.filebot.WebServices;
 
-public class AnimeList {
+public class AnimeLists implements Datasource {
 
-	private Model model;
+	@Override
-
+	public String getIdentifier() {
-	public AnimeList() throws Exception {
+		return "AnimeLists";
-		this.model = MODEL.get();
 	}
 
-	public Model getModel() {
+	@Override
-		return model;
+	public Icon getIcon() {
+		return null;
 	}
 
 	public Optional<Episode> map(Episode episode, DB source, DB destination) throws Exception {
@@ -84,7 +83,7 @@ public class AnimeList {
 		}).findFirst();
 	}
 
-	private Episode derive(DB db, Entry a, Episode episode, int s, int e) {
+	protected Episode derive(DB db, Entry a, Episode episode, int s, int e) {
 		if (s == 0) {
 			// special
 			switch (db) {
@@ -202,7 +201,7 @@ public class AnimeList {
 	}
 
 	public Stream<Entry> find(DB db, int id) throws Exception {
-		return stream(model.anime).filter(this::isValid).filter(a -> id == getId(db, a));
+		return stream(getModel().anime).filter(this::isValid).filter(a -> id == getId(db, a));
 	}
 
 	public Stream<Entry> find(DB db, int id, int s) throws Exception {
@@ -214,21 +213,10 @@ public class AnimeList {
 		}
 	}
 
-	protected static Cache getCache() {
+	public Model getModel() throws Exception {
-		return Cache.getCache("animelists", CacheType.Persistent);
+		return Cache.getCache(getIdentifier(), CacheType.Monthly).bytes("https://github.com/ScudLee/anime-lists/raw/master/anime-list.xml", URL::new).transform(bytes -> {
-	}
+			return unmarshal(bytes, Model.class);
-
+		}).get();
-	protected static final Resource<Model> MODEL = Resource.lazy(() -> unmarshal(request("anime-list.xml"), Model.class));
-
-	protected static byte[] request(String file) throws Exception {
-		// NOTE: GitHub only supports If-None-Match (If-Modified-Since is ignored)
-		Cache cache = getCache();
-
-		return cache.bytes(file, AnimeList::getResource).fetch(fetchIfNoneMatch(url -> file, cache)).expire(Cache.ONE_MONTH).get();
-	}
-
-	protected static URL getResource(String file) throws Exception {
-		return new URL("https://raw.githubusercontent.com/ScudLee/anime-lists/master/" + file);
 	}
 
 	public static DB getDB(Episode e) {
@@ -325,10 +313,9 @@ public class AnimeList {
 		public String toString() {
 			return marshal(this, Mapping.class);
 		}
-
 	}
 
-	private static class NumberAdapter extends XmlAdapter<String, Integer> {
+	protected static class NumberAdapter extends XmlAdapter<String, Integer> {
 
 		@Override
 		public Integer unmarshal(String s) throws Exception {
@@ -341,7 +328,7 @@ public class AnimeList {
 		}
 	}
 
-	private static class NumberMapAdapter extends XmlAdapter<String, int[][]> {
+	protected static class NumberMapAdapter extends XmlAdapter<String, int[][]> {
 
 		@Override
 		public int[][] unmarshal(String s) throws Exception {
@@ -354,15 +341,19 @@ public class AnimeList {
 		}
 	}
 
-	private static <T> T unmarshal(byte[] bytes, Class<T> type) throws Exception {
+	public static <T> T unmarshal(byte[] bytes, Class<T> type) {
-		return (T) JAXBContext.newInstance(type).createUnmarshaller().unmarshal(new ByteArrayInputStream(bytes));
+		try {
+			return (T) JAXBContext.newInstance(type).createUnmarshaller().unmarshal(new ByteArrayInputStream(bytes));
+		} catch (Exception e) {
+			throw new IllegalArgumentException(e);
+		}
 	}
 
-	private static <T> String marshal(T object, Class<T> type) {
+	public static <T> String marshal(T object, Class<T> type) {
 		try {
 			StringWriter buffer = new StringWriter();
 			Marshaller marshaller = JAXBContext.newInstance(type).createMarshaller();
-			marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
+			marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.FALSE);
 			marshaller.setProperty(Marshaller.JAXB_FRAGMENT, Boolean.TRUE);
 			marshaller.marshal(object, buffer);
 			return buffer.toString();

source/net/filebot/web/EpisodeUtilities.java

@@ -55,7 +55,7 @@ public final class EpisodeUtilities {
 		if (episode.isAnime() && episode.isRegular()) {
 			return mapEpisode(episode, e -> {
 				try {
-					return new AnimeList().map(e, AnimeList.getDB(e), AnimeList.DB.TheTVDB).orElse(e);
+					return AnimeList.map(e, AnimeLists.getDB(e), AnimeLists.DB.TheTVDB).orElse(e);
 				} catch (Exception ioe) {
 					debug.warning(ioe::toString);
 					return e;