diff --git a/source/net/filebot/Cache.java b/source/net/filebot/Cache.java index 50aa9b0f..e67f6060 100644 --- a/source/net/filebot/Cache.java +++ b/source/net/filebot/Cache.java @@ -1,6 +1,8 @@ package net.filebot; import static java.nio.charset.StandardCharsets.*; +import static java.util.Arrays.*; +import static java.util.stream.Collectors.*; import static net.filebot.CachedResource.*; import static net.filebot.Logging.*; @@ -8,6 +10,8 @@ import java.io.Serializable; import java.net.URL; import java.time.Duration; import java.util.Arrays; +import java.util.List; +import java.util.function.Function; import java.util.function.Predicate; import net.filebot.CachedResource.Transform; @@ -49,12 +53,8 @@ public class Cache { public Object get(Object key) { try { - Element element = cache.get(key); - if (element != null) { - return element.getObjectValue(); - } + return getElementValue(cache.get(key)); } catch (Exception e) { - e.printStackTrace(); debug.warning(format("Cache get: %s => %s", key, e)); } return null; @@ -65,47 +65,35 @@ public class Cache { Element element = null; try { element = cache.get(key); - if (element != null && condition.test(element)) { - return element.getObjectValue(); + if (condition.test(element)) { + return getElementValue(element); } } catch (Exception e) { - debug.warning(format("Cache get: %s => %s", key, e)); + debug.warning(format("Cache computeIf: %s => %s", key, e)); } // compute if absent Object value = compute.apply(element); - try { - cache.put(new Element(key, value)); - } catch (Exception e) { - debug.warning(format("Cache put: %s => %s", key, e)); - } + put(key, value); return value; } - public Object computeIfAbsent(Object key, Compute compute) throws Exception { - return computeIf(key, isAbsent(), compute); - } - - public Object computeIfStale(Object key, Duration expirationTime, Compute compute) throws Exception { - return computeIf(key, isStale(expirationTime), compute); - } - - public Predicate isAbsent() { - return (element) -> element.getObjectValue() == null; - } - - public Predicate isStale(Duration expirationTime) { - return (element) -> System.currentTimeMillis() - element.getLatestOfCreationAndUpdateTime() < expirationTime.toMillis(); - } - public void put(Object key, Object value) { try { - cache.put(new Element(key, value)); + cache.put(createElement(key, value)); } catch (Exception e) { debug.warning(format("Cache put: %s => %s", key, e)); } } + protected Object getElementValue(Element element) { + return element == null ? null : element.getObjectValue(); + } + + protected Element createElement(Object key, Object value) { + return new Element(key, value); + } + public void remove(Object key) { try { cache.remove(key); @@ -122,11 +110,64 @@ public class Cache { } } + public static Predicate isAbsent() { + return (element) -> element == null; + } + + public static Predicate isStale(Duration expirationTime) { + return (element) -> element == null || element.getObjectValue() == null || System.currentTimeMillis() - element.getLatestOfCreationAndUpdateTime() < expirationTime.toMillis(); + } + @FunctionalInterface public interface Compute { R apply(Element element) throws Exception; } + public TypedCache typed(Function read, Function write) { + return new TypedCache(cache, read, write); + } + + public TypedCache cast(Class cls) { + return new TypedCache(cache, it -> cls.cast(it), it -> it); + } + + public TypedCache> castList(Class cls) { + return new TypedCache>(cache, it -> it == null ? null : stream((Object[]) it).map(cls::cast).collect(toList()), it -> it == null ? null : it.toArray()); + } + + @SuppressWarnings("unchecked") + public static class TypedCache extends Cache { + + private final Function read; + private final Function write; + + public TypedCache(net.sf.ehcache.Cache cache, Function read, Function write) { + super(cache); + this.read = read; + this.write = write; + } + + @Override + public V get(Object key) { + return (V) super.get(key); + } + + @Override + public V computeIf(Object key, Predicate condition, Compute compute) throws Exception { + return (V) super.computeIf(key, condition, compute); + } + + @Override + protected Object getElementValue(Element element) { + return read.apply(super.getElementValue(element)); + } + + @Override + protected Element createElement(Object key, Object value) { + return super.createElement(key, write.apply((V) value)); + } + } + @Deprecated public T get(Object key, Class type) { return type.cast(get(key)); diff --git a/source/net/filebot/CachedResource.java b/source/net/filebot/CachedResource.java index cf0b9f8e..72d52626 100644 --- a/source/net/filebot/CachedResource.java +++ b/source/net/filebot/CachedResource.java @@ -70,7 +70,7 @@ public class CachedResource implements Resource { @Override public synchronized R get() throws Exception { - Object value = cache.computeIfStale(key, expirationTime, element -> { + Object value = cache.computeIf(key, Cache.isStale(expirationTime), element -> { URL url = resource.transform(key); long lastModified = element == null ? 0 : element.getLatestOfCreationAndUpdateTime(); diff --git a/source/net/filebot/web/AbstractEpisodeListProvider.java b/source/net/filebot/web/AbstractEpisodeListProvider.java index 8be39705..676cd12e 100644 --- a/source/net/filebot/web/AbstractEpisodeListProvider.java +++ b/source/net/filebot/web/AbstractEpisodeListProvider.java @@ -5,11 +5,10 @@ import static java.util.Arrays.*; import java.io.Serializable; import java.util.List; import java.util.Locale; -import java.util.logging.Level; -import java.util.logging.Logger; import net.filebot.Cache; -import net.filebot.Cache.Key; +import net.filebot.Cache.TypedCache; +import net.filebot.CacheType; public abstract class AbstractEpisodeListProvider implements EpisodeListProvider { @@ -19,24 +18,15 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider protected abstract SearchResult createSearchResult(int id); - protected abstract ResultCache getCache(); - protected abstract SortOrder vetoRequestParameter(SortOrder order); protected abstract Locale vetoRequestParameter(Locale language); @Override public List search(String query, Locale language) throws Exception { - List results = getCache().getSearchResult(query, language); - if (results != null) { - return results; - } - - // perform actual search - results = fetchSearchResult(query, language); - - // cache results and return - return getCache().putSearchResult(query, language, results); + return getSearchCache(language).computeIf(query, Cache.isAbsent(), it -> { + return fetchSearchResult(query, language); + }); } @Override @@ -61,19 +51,24 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider protected SeriesData getSeriesData(SearchResult searchResult, SortOrder order, Locale language) throws Exception { // override preferences if requested parameters are not supported - order = vetoRequestParameter(order); - language = vetoRequestParameter(language); + SortOrder requestOrder = vetoRequestParameter(order); + Locale requestLanguage = vetoRequestParameter(language); - SeriesData data = getCache().getSeriesData(searchResult, order, language); - if (data != null) { - return data; - } + return getDataCache(requestOrder, requestLanguage).computeIf(searchResult.getId(), Cache.isAbsent(), it -> { + return fetchSeriesData(searchResult, requestOrder, requestLanguage); + }); + } - // perform actual lookup - data = fetchSeriesData(searchResult, order, language); + protected Cache getCache(String section) { + return Cache.getCache(getName() + "_" + section, CacheType.Daily); + } - // cache results and return - return getCache().putSeriesData(searchResult, order, language, data); + protected TypedCache> getSearchCache(Locale language) { + return getCache("search_" + language).castList(SearchResult.class); + } + + protected TypedCache getDataCache(SortOrder order, Locale language) { + return getCache("data_" + order.ordinal() + "_" + language).cast(SeriesData.class); } protected static class SeriesData implements Serializable { @@ -96,57 +91,4 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider } - protected static class ResultCache { - - private final String id; - private final Cache cache; - - public ResultCache(String id, Cache cache) { - this.id = id; - this.cache = cache; - } - - protected String normalize(String query) { - return query == null ? null : query.trim().toLowerCase(); - } - - public List putSearchResult(String query, Locale locale, List value) { - putData("SearchResult", normalize(query), locale, value.toArray(new SearchResult[value.size()])); - return value; - } - - public List getSearchResult(String query, Locale locale) { - SearchResult[] data = getData("SearchResult", normalize(query), locale, SearchResult[].class); - return data == null ? null : asList(data); - } - - public SeriesData putSeriesData(SearchResult key, SortOrder sortOrder, Locale locale, SeriesData seriesData) { - putData("SeriesData." + sortOrder.name(), key, locale, seriesData); - return seriesData; - } - - public SeriesData getSeriesData(SearchResult key, SortOrder sortOrder, Locale locale) { - return getData("SeriesData." + sortOrder.name(), key, locale, SeriesData.class); - } - - public T putData(Object category, Object key, Locale locale, T object) { - try { - cache.put(new Key(id, category, locale, key), object); - } catch (Exception e) { - Logger.getLogger(AbstractEpisodeListProvider.class.getName()).log(Level.WARNING, e.getMessage()); - } - return object; - } - - public T getData(Object category, Object key, Locale locale, Class type) { - try { - return cache.get(new Key(id, category, locale, key), type); - } catch (Exception e) { - Logger.getLogger(AbstractEpisodeListProvider.class.getName()).log(Level.WARNING, e.getMessage(), e); - } - return null; - } - - } - } diff --git a/source/net/filebot/web/AnidbClient.java b/source/net/filebot/web/AnidbClient.java index 736a43ce..de08f88c 100644 --- a/source/net/filebot/web/AnidbClient.java +++ b/source/net/filebot/web/AnidbClient.java @@ -6,6 +6,7 @@ import static net.filebot.util.XPathUtilities.*; import static net.filebot.web.EpisodeUtilities.*; import static net.filebot.web.WebRequest.*; +import java.io.ByteArrayInputStream; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -42,8 +43,6 @@ public class AnidbClient extends AbstractEpisodeListProvider { private static final FloodLimit REQUEST_LIMIT = new FloodLimit(2, 5, TimeUnit.SECONDS); // no more than 2 requests within a 5 second window - private final String host = "anidb.net"; - private final String client; private final int clientver; @@ -78,8 +77,8 @@ public class AnidbClient extends AbstractEpisodeListProvider { } @Override - public ResultCache getCache() { - return new ResultCache(getName(), Cache.getCache(getName(), CacheType.Weekly)); + protected Cache getCache(String section) { + return Cache.getCache(getName() + "_" + section, CacheType.Weekly); } @Override @@ -105,7 +104,7 @@ public class AnidbClient extends AbstractEpisodeListProvider { AnidbSearchResult anime = (AnidbSearchResult) searchResult; // e.g. http://api.anidb.net:9001/httpapi?request=anime&client=filebot&clientver=1&protover=1&aid=4521 - URL url = new URL("http", "api." + host, 9001, "/httpapi?request=anime&client=" + client + "&clientver=" + clientver + "&protover=1&aid=" + anime.getAnimeId()); + URL url = new URL("http://api.anidb.net:9001/httpapi?request=anime&client=" + client + "&clientver=" + clientver + "&protover=1&aid=" + anime.getAnimeId()); // respect flood protection limits REQUEST_LIMIT.acquirePermit(); @@ -190,7 +189,7 @@ public class AnidbClient extends AbstractEpisodeListProvider { @Override public URI getEpisodeListLink(SearchResult searchResult) { try { - return new URI("http", host, "/a" + ((AnidbSearchResult) searchResult).getAnimeId(), null); + return new URI("http://anidb.net/a" + searchResult.getId()); } catch (URISyntaxException e) { throw new RuntimeException(e); } @@ -200,14 +199,8 @@ public class AnidbClient extends AbstractEpisodeListProvider { * This method is (and must be!) overridden by WebServices.AnidbClientWithLocalSearch to use our own anime index from sourceforge (as to not abuse anidb servers) */ public synchronized List getAnimeTitles() throws Exception { - URL url = new URL("http", host, "/api/anime-titles.dat.gz"); - ResultCache cache = getCache(); - - @SuppressWarnings("unchecked") - List anime = (List) cache.getSearchResult(null, Locale.ROOT); - if (anime != null) { - return anime; - } + // get data file (cached) + byte[] bytes = getCache("root").bytes("anime-titles.dat.gz", n -> new URL("http://anidb.net/api/" + n)).get(); // ||| // type: 1=primary title (one per anime), 2=synonyms (multiple per anime), 3=shorttitles (multiple per anime), 4=official title (one per language) @@ -227,7 +220,7 @@ public class AnidbClient extends AbstractEpisodeListProvider { // fetch data Map<Integer, List<Object[]>> entriesByAnime = new HashMap<Integer, List<Object[]>>(65536); - Scanner scanner = new Scanner(new GZIPInputStream(url.openStream()), "UTF-8"); + Scanner scanner = new Scanner(new GZIPInputStream(new ByteArrayInputStream(bytes)), "UTF-8"); try { while (scanner.hasNextLine()) { Matcher matcher = pattern.matcher(scanner.nextLine()); @@ -260,7 +253,7 @@ public class AnidbClient extends AbstractEpisodeListProvider { } // build up a list of all possible AniDB search results - anime = new ArrayList<AnidbSearchResult>(entriesByAnime.size()); + List<AnidbSearchResult> anime = new ArrayList<AnidbSearchResult>(entriesByAnime.size()); for (Entry<Integer, List<Object[]>> entry : entriesByAnime.entrySet()) { int aid = entry.getKey(); @@ -289,7 +282,7 @@ public class AnidbClient extends AbstractEpisodeListProvider { anime.add(new AnidbSearchResult(aid, primaryTitle, aliasNames)); } - // populate cache - return cache.putSearchResult(null, Locale.ROOT, anime); + return anime; } + } diff --git a/source/net/filebot/web/TMDbClient.java b/source/net/filebot/web/TMDbClient.java index 1eee5720..b7b960fc 100644 --- a/source/net/filebot/web/TMDbClient.java +++ b/source/net/filebot/web/TMDbClient.java @@ -171,7 +171,7 @@ public class TMDbClient implements MovieIdentificationService { public MovieInfo getMovieInfo(String id, Locale locale, boolean extendedInfo) throws Exception { Object response = request("movie/" + id, extendedInfo ? singletonMap("append_to_response", "alternative_titles,releases,casts,trailers") : null, locale, REQUEST_LIMIT); - Map<MovieProperty, String> fields = mapStringValues(response, MovieProperty.class); + Map<MovieProperty, String> fields = getEnumMap(response, MovieProperty.class); try { Map<?, ?> collection = getMap(response, "belongs_to_collection"); @@ -241,7 +241,7 @@ public class TMDbClient implements MovieIdentificationService { List<Person> cast = new ArrayList<Person>(); try { Stream.of("cast", "crew").flatMap(section -> streamJsonObjects(getMap(response, "casts"), section)).map(it -> { - return mapStringValues(it, PersonProperty.class); + return getEnumMap(it, PersonProperty.class); }).map(Person::new).forEach(cast::add); } catch (Exception e) { debug.warning(format("Bad data: casts => %s", response)); diff --git a/source/net/filebot/web/TVMazeClient.java b/source/net/filebot/web/TVMazeClient.java index c4e65a25..251960e9 100644 --- a/source/net/filebot/web/TVMazeClient.java +++ b/source/net/filebot/web/TVMazeClient.java @@ -49,11 +49,6 @@ public class TVMazeClient extends AbstractEpisodeListProvider { return new TVMazeSearchResult(id, null); } - @Override - public ResultCache getCache() { - return new ResultCache(getName(), Cache.getCache(getName(), CacheType.Daily)); - } - @Override public List<SearchResult> fetchSearchResult(String query, Locale locale) throws Exception { // e.g. http://api.tvmaze.com/search/shows?q=girls diff --git a/source/net/filebot/web/TheTVDBClient.java b/source/net/filebot/web/TheTVDBClient.java index 95e876b2..ed8aab53 100644 --- a/source/net/filebot/web/TheTVDBClient.java +++ b/source/net/filebot/web/TheTVDBClient.java @@ -1,15 +1,14 @@ package net.filebot.web; -import static java.util.Arrays.*; import static java.util.Collections.*; import static java.util.stream.Collectors.*; +import static net.filebot.Logging.*; import static net.filebot.util.StringUtilities.*; import static net.filebot.util.XPathUtilities.*; import static net.filebot.web.EpisodeUtilities.*; import static net.filebot.web.WebRequest.*; import java.io.Serializable; -import java.net.MalformedURLException; import java.net.URI; import java.net.URL; import java.util.ArrayList; @@ -27,6 +26,7 @@ import java.util.logging.Logger; import javax.swing.Icon; import net.filebot.Cache; +import net.filebot.Cache.TypedCache; import net.filebot.CacheType; import net.filebot.ResourceManager; import net.filebot.util.FileUtilities; @@ -73,11 +73,6 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { return language != null ? language : Locale.ENGLISH; } - @Override - public ResultCache getCache() { - return new ResultCache(getName(), Cache.getCache(getName(), CacheType.Daily)); - } - public String getLanguageCode(Locale locale) { String code = locale.getLanguage(); @@ -225,41 +220,35 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { return new SeriesData(seriesInfo, episodes); } - public TheTVDBSearchResult lookupByID(int id, Locale locale) throws Exception { - TheTVDBSearchResult cachedItem = getCache().getData("lookupByID", id, locale, TheTVDBSearchResult.class); - if (cachedItem != null) { - return cachedItem; + public TheTVDBSearchResult lookupByID(int id, Locale language) throws Exception { + if (id <= 0) { + throw new IllegalArgumentException("Illegal TheTVDB ID: " + id); } - Document dom = getXmlResource(MirrorType.XML, "series/" + id + "/all/" + getLanguageCode(locale) + ".xml"); - String name = selectString("//SeriesName", dom); + return getLookupCache("id", language).computeIf(id, Cache.isAbsent(), it -> { + Document dom = getXmlResource(MirrorType.XML, "series/" + id + "/all/" + getLanguageCode(language) + ".xml"); + String name = selectString("//SeriesName", dom); - TheTVDBSearchResult series = new TheTVDBSearchResult(name, id); - getCache().putData("lookupByID", id, locale, series); - return series; + return new TheTVDBSearchResult(name, id); + }); } public TheTVDBSearchResult lookupByIMDbID(int imdbid, Locale locale) throws Exception { if (imdbid <= 0) { - throw new IllegalArgumentException("id must not be " + imdbid); + throw new IllegalArgumentException("Illegal IMDbID ID: " + imdbid); } - TheTVDBSearchResult cachedItem = getCache().getData("lookupByIMDbID", imdbid, locale, TheTVDBSearchResult.class); - if (cachedItem != null) { - return cachedItem; - } + return getLookupCache("imdbid", locale).computeIf(imdbid, Cache.isAbsent(), it -> { + Document dom = getXmlResource(MirrorType.SEARCH, "GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale)); - Document dom = getXmlResource(MirrorType.SEARCH, "GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale)); + String id = selectString("//seriesid", dom); + String name = selectString("//SeriesName", dom); - String id = selectString("//seriesid", dom); - String name = selectString("//SeriesName", dom); + if (id.isEmpty() || name.isEmpty()) + return null; - if (id == null || id.isEmpty() || name == null || name.isEmpty()) - return null; - - TheTVDBSearchResult series = new TheTVDBSearchResult(name, Integer.parseInt(id)); - getCache().putData("lookupByIMDbID", imdbid, locale, series); - return series; + return new TheTVDBSearchResult(name, Integer.parseInt(id)); + }); } protected String getMirror(MirrorType mirrorType) throws Exception { @@ -381,40 +370,28 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { } public List<BannerDescriptor> getBannerList(TheTVDBSearchResult series) throws Exception { - // check cache first - BannerDescriptor[] cachedList = getCache().getData("banners", series.getId(), null, BannerDescriptor[].class); - if (cachedList != null) { - return asList(cachedList); - } + return getBannerCache().computeIf(series.getId(), Cache.isAbsent(), it -> { + Document dom = getXmlResource(MirrorType.XML, "series/" + series.getId() + "/banners.xml"); - Document dom = getXmlResource(MirrorType.XML, "series/" + series.getId() + "/banners.xml"); + String bannerMirror = getResource(MirrorType.BANNER, "").toString(); - List<BannerDescriptor> banners = new ArrayList<BannerDescriptor>(); + return streamNodes("//Banner", dom).map(n -> { + Map<BannerProperty, String> map = getEnumMap(n, BannerProperty.class); + map.put(BannerProperty.BannerMirror, bannerMirror); - for (Node node : selectNodes("//Banner", dom)) { - try { - Map<BannerProperty, String> item = new EnumMap<BannerProperty, String>(BannerProperty.class); + return new BannerDescriptor(map); + }).filter(m -> m.getUrl() != null).collect(toList()); + }); + } - // insert banner mirror - item.put(BannerProperty.BannerMirror, getResource(MirrorType.BANNER, "").toString()); + protected TypedCache<TheTVDBSearchResult> getLookupCache(String type, Locale language) { + // lookup should always yield the same results so we can cache it for longer + return Cache.getCache(getName() + "_" + "lookup" + "_" + type + "_" + language, CacheType.Monthly).cast(TheTVDBSearchResult.class); + } - // copy values from xml - for (BannerProperty key : BannerProperty.values()) { - String value = getTextContent(key.name(), node); - if (value != null && value.length() > 0) { - item.put(key, value); - } - } - - banners.add(new BannerDescriptor(item)); - } catch (Exception e) { - // log and ignore - Logger.getLogger(getClass().getName()).log(Level.WARNING, "Invalid banner descriptor", e); - } - } - - getCache().putData("banners", series.getId(), null, banners.toArray(new BannerDescriptor[0])); - return banners; + protected TypedCache<List<BannerDescriptor>> getBannerCache() { + // banners do not change that often so we can cache them for longer + return Cache.getCache(getName() + "_" + "banner", CacheType.Weekly).castList(BannerDescriptor.class); } public static class BannerDescriptor implements Serializable { @@ -441,20 +418,17 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { return fields.get(key); } - public URL getBannerMirrorUrl() throws MalformedURLException { + public URL getBannerMirrorUrl(String path) { try { - return new URL(get(BannerProperty.BannerMirror)); + return new URL(new URL(get(BannerProperty.BannerMirror)), path); } catch (Exception e) { + debug.finest(format("Bad banner url: %s", e)); return null; } } - public URL getUrl() throws MalformedURLException { - try { - return new URL(getBannerMirrorUrl(), get(BannerProperty.BannerPath)); - } catch (Exception e) { - return null; - } + public URL getUrl() { + return getBannerMirrorUrl(get(BannerProperty.BannerPath)); } public String getExtension() { @@ -480,7 +454,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { public Integer getSeason() { try { return new Integer(get(BannerProperty.Season)); - } catch (NumberFormatException e) { + } catch (Exception e) { return null; } } @@ -517,26 +491,19 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { return Boolean.parseBoolean(get(BannerProperty.SeriesName)); } - public URL getThumbnailUrl() throws MalformedURLException { - try { - return new URL(getBannerMirrorUrl(), get(BannerProperty.ThumbnailPath)); - } catch (Exception e) { - return null; - } + public URL getThumbnailUrl() { + return getBannerMirrorUrl(get(BannerProperty.ThumbnailPath)); } - public URL getVignetteUrl() throws MalformedURLException { - try { - return new URL(getBannerMirrorUrl(), get(BannerProperty.VignettePath)); - } catch (Exception e) { - return null; - } + public URL getVignetteUrl() { + return getBannerMirrorUrl(get(BannerProperty.VignettePath)); } @Override public String toString() { return fields.toString(); } + } }