From 4e41d0dfd15f2fce69f5556767a95c9c6b0cd37f Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Mon, 7 Mar 2016 10:55:45 +0000 Subject: [PATCH] Experiment with new CachedResource framework --- source/net/filebot/Cache.java | 13 +- source/net/filebot/web/CachedResource2.java | 62 ++++++-- source/net/filebot/web/Resource.java | 8 + source/net/filebot/web/TheTVDBClient.java | 154 ++++++++------------ source/net/filebot/web/WebRequest.java | 22 ++- test/net/filebot/web/TheTVDBClientTest.java | 22 +-- 6 files changed, 153 insertions(+), 128 deletions(-) create mode 100644 source/net/filebot/web/Resource.java diff --git a/source/net/filebot/Cache.java b/source/net/filebot/Cache.java index 065c810d..9e2c9965 100644 --- a/source/net/filebot/Cache.java +++ b/source/net/filebot/Cache.java @@ -2,6 +2,7 @@ package net.filebot; import static java.nio.charset.StandardCharsets.*; import static net.filebot.Logging.*; +import static net.filebot.web.CachedResource2.*; import java.io.Serializable; import java.net.URL; @@ -10,9 +11,13 @@ import java.util.Arrays; import java.util.function.Predicate; import net.filebot.web.CachedResource2; +import net.filebot.web.CachedResource2.Source; import net.filebot.web.FloodLimit; +import net.filebot.web.Resource; import net.sf.ehcache.Element; +import org.w3c.dom.Document; + public class Cache { public static Cache getCache(String name, CacheType type) { @@ -131,8 +136,12 @@ public class Cache { } } - public CachedResource2 resource(String url, Duration expirationTime, FloodLimit limit) { - return new CachedResource2(url, URL::new, CachedResource2.fetchIfModified(limit), CachedResource2.decode(UTF_8), expirationTime, this); + public Resource xml(String key, Source source, Duration expirationTime) { + return new CachedResource2(key, source, fetchIfModified(), validateXml(getText(UTF_8)), getXml(String.class::cast), DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, this); + } + + public Resource resource(String url, Duration expirationTime, FloodLimit limit) { + return new CachedResource2(url, URL::new, withPermit(fetchIfModified(), r -> limit.acquirePermit() != null), getText(UTF_8), String.class::cast, DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, this); } } diff --git a/source/net/filebot/web/CachedResource2.java b/source/net/filebot/web/CachedResource2.java index 874340ce..7a8780a7 100644 --- a/source/net/filebot/web/CachedResource2.java +++ b/source/net/filebot/web/CachedResource2.java @@ -12,7 +12,9 @@ import java.util.concurrent.Callable; import net.filebot.Cache; -public class CachedResource2 { +import org.w3c.dom.Document; + +public class CachedResource2 implements Resource { public static final int DEFAULT_RETRY_LIMIT = 2; public static final Duration DEFAULT_RETRY_DELAY = Duration.ofSeconds(2); @@ -21,7 +23,8 @@ public class CachedResource2 { protected final Source source; protected final Fetch fetch; - protected final Parse parse; + protected final Transform parse; + protected final Transform cast; protected final Duration expirationTime; @@ -30,24 +33,21 @@ public class CachedResource2 { protected final Cache cache; - public CachedResource2(K key, Source source, Fetch fetch, Parse parse, Duration expirationTime, Cache cache) { - this(key, source, fetch, parse, DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, cache); - } - - public CachedResource2(K key, Source source, Fetch fetch, Parse parse, int retryCountLimit, Duration retryWaitTime, Duration expirationTime, Cache cache) { + public CachedResource2(K key, Source source, Fetch fetch, Transform parse, Transform cast, int retryCountLimit, Duration retryWaitTime, Duration expirationTime, Cache cache) { this.key = key; this.source = source; this.fetch = fetch; this.parse = parse; + this.cast = cast; this.expirationTime = expirationTime; this.retryCountLimit = retryCountLimit; this.retryWaitTime = retryWaitTime.toMillis(); this.cache = cache; } - @SuppressWarnings("unchecked") + @Override public synchronized R get() throws Exception { - return (R) cache.computeIfStale(key, expirationTime, element -> { + Object value = cache.computeIfStale(key, expirationTime, element -> { URL resource = source.source(key); long lastModified = element == null ? 0 : element.getLatestOfCreationAndUpdateTime(); @@ -61,7 +61,7 @@ public class CachedResource2 { return element.getObjectValue(); } - return parse.parse(data); + return parse.transform(data); } catch (IOException e) { debug.fine(format("Fetch failed => %s", e)); @@ -72,6 +72,8 @@ public class CachedResource2 { return element.getObjectKey(); } }); + + return cast.transform(value); } protected T retry(Callable callable, int retryCount, long retryWaitTime) throws Exception { @@ -101,23 +103,51 @@ public class CachedResource2 { } @FunctionalInterface - public interface Parse { - R parse(ByteBuffer bytes) throws Exception; + public interface Transform { + R transform(T object) throws Exception; } - public static Parse decode(Charset charset) { - return (bb) -> charset.decode(bb).toString(); + @FunctionalInterface + public interface Permit

{ + boolean acquirePermit(URL resource) throws Exception; } - public static Fetch fetchIfModified(FloodLimit limit) { + public static Transform getText(Charset charset) { + return (data) -> charset.decode(data).toString(); + } + + public static Transform validateXml(Transform parse) { + return (object) -> { + String xml = parse.transform(object); + WebRequest.validateXml(xml); + return xml; + }; + } + + public static Transform getXml(Transform parse) { + return (object) -> { + return WebRequest.getDocument(parse.transform(object)); + }; + } + + public static Fetch fetchIfModified() { return (url, lastModified) -> { try { - limit.acquirePermit(); return WebRequest.fetchIfModified(url, lastModified); } catch (FileNotFoundException e) { + debug.warning(format("Resource not found: %s => %s", url, e)); return ByteBuffer.allocate(0); } }; } + public static Fetch withPermit(Fetch fetch, Permit permit) { + return (url, lastModified) -> { + if (permit.acquirePermit(url)) { + return fetch.fetch(url, lastModified); + } + return null; + }; + } + } diff --git a/source/net/filebot/web/Resource.java b/source/net/filebot/web/Resource.java new file mode 100644 index 00000000..6e0bbcad --- /dev/null +++ b/source/net/filebot/web/Resource.java @@ -0,0 +1,8 @@ +package net.filebot.web; + +@FunctionalInterface +public interface Resource { + + R get() throws Exception; + +} diff --git a/source/net/filebot/web/TheTVDBClient.java b/source/net/filebot/web/TheTVDBClient.java index 7948cf80..a028dbb4 100644 --- a/source/net/filebot/web/TheTVDBClient.java +++ b/source/net/filebot/web/TheTVDBClient.java @@ -2,17 +2,17 @@ package net.filebot.web; import static java.util.Arrays.*; import static java.util.Collections.*; +import static java.util.stream.Collectors.*; import static net.filebot.util.StringUtilities.*; import static net.filebot.util.XPathUtilities.*; import static net.filebot.web.EpisodeUtilities.*; import static net.filebot.web.WebRequest.*; -import java.io.FileNotFoundException; -import java.io.IOException; import java.io.Serializable; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; +import java.time.Duration; import java.util.ArrayList; import java.util.EnumMap; import java.util.EnumSet; @@ -40,7 +40,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { private final String host = "www.thetvdb.com"; - private final Map mirrors = new EnumMap(MirrorType.class); + private final Map mirrors = MirrorType.newMap(); private final String apikey; @@ -106,7 +106,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { @Override public List fetchSearchResult(String query, Locale locale) throws Exception { // perform online search - Document dom = getXmlResource(MirrorType.SEARCH, "/api/GetSeries.php?seriesname=" + encode(query, true) + "&language=" + getLanguageCode(locale)); + Document dom = getXmlResource(MirrorType.SEARCH, "GetSeries.php?seriesname=" + encode(query, true) + "&language=" + getLanguageCode(locale)); List nodes = selectNodes("Data/Series", dom); Map resultSet = new LinkedHashMap(); @@ -140,7 +140,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { @Override protected SeriesData fetchSeriesData(SearchResult searchResult, SortOrder sortOrder, Locale locale) throws Exception { TheTVDBSearchResult series = (TheTVDBSearchResult) searchResult; - Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + series.getSeriesId() + "/all/" + getLanguageCode(locale) + ".xml"); + Document dom = getXmlResource(MirrorType.XML, "series/" + series.getSeriesId() + "/all/" + getLanguageCode(locale) + ".xml"); // parse series info Node seriesNode = selectNode("Data/Series", dom); @@ -163,9 +163,9 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { seriesInfo.setGenres(getListContent("Genre", "\\|", seriesNode)); seriesInfo.setStartDate(SimpleDate.parse(getTextContent("FirstAired", seriesNode))); - seriesInfo.setBannerUrl(getResourceURL(MirrorType.BANNER, "/banners/" + getTextContent("banner", seriesNode))); - seriesInfo.setFanartUrl(getResourceURL(MirrorType.BANNER, "/banners/" + getTextContent("fanart", seriesNode))); - seriesInfo.setPosterUrl(getResourceURL(MirrorType.BANNER, "/banners/" + getTextContent("poster", seriesNode))); + seriesInfo.setBannerUrl(getResource(MirrorType.BANNER, getTextContent("banner", seriesNode))); + seriesInfo.setFanartUrl(getResource(MirrorType.BANNER, getTextContent("fanart", seriesNode))); + seriesInfo.setPosterUrl(getResource(MirrorType.BANNER, getTextContent("poster", seriesNode))); // parse episode data List nodes = selectNodes("Data/Episode", dom); @@ -237,7 +237,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { return cachedItem; } - Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + id + "/all/" + getLanguageCode(locale) + ".xml"); + Document dom = getXmlResource(MirrorType.XML, "series/" + id + "/all/" + getLanguageCode(locale) + ".xml"); String name = selectString("//SeriesName", dom); TheTVDBSearchResult series = new TheTVDBSearchResult(name, id); @@ -255,7 +255,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { return cachedItem; } - Document dom = getXmlResource(null, "/api/GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale)); + Document dom = getXmlResource(MirrorType.SEARCH, "GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale)); String id = selectString("//seriesid", dom); String name = selectString("//SeriesName", dom); @@ -268,113 +268,87 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { return series; } - protected String getMirror(MirrorType mirrorType) throws IOException { + protected String getMirror(MirrorType mirrorType) throws Exception { + // use default server + if (mirrorType == MirrorType.NULL) { + return "http://thetvdb.com"; + } + synchronized (mirrors) { + // initialize mirrors if (mirrors.isEmpty()) { - // try cache first - try { - @SuppressWarnings("unchecked") - Map cachedMirrors = getCache().getData("mirrors", null, null, Map.class); - if (cachedMirrors != null) { - mirrors.putAll(cachedMirrors); - return mirrors.get(mirrorType); - } - } catch (Exception e) { - Logger.getLogger(getClass().getName()).log(Level.SEVERE, e.getMessage(), e); - } + Document dom = getXmlResource(MirrorType.NULL, "mirrors.xml"); - // initialize mirrors - Document dom = getXmlResource(null, "/api/" + apikey + "/mirrors.xml"); - - // all mirrors by type - Map> mirrorListMap = new EnumMap>(MirrorType.class); - - // initialize mirror list per type - for (MirrorType type : MirrorType.values()) { - mirrorListMap.put(type, new ArrayList(5)); - } - - // traverse all mirrors - for (Node node : selectNodes("Mirrors/Mirror", dom)) { - // mirror data + // collect all mirror data + Map> mirrorLists = selectNodes("Mirrors/Mirror", dom).stream().flatMap(node -> { String mirror = getTextContent("mirrorpath", node); int typeMask = Integer.parseInt(getTextContent("typemask", node)); - // add mirror to the according type lists - for (MirrorType type : MirrorType.fromTypeMask(typeMask)) { - mirrorListMap.get(type).add(mirror); - } - } + return MirrorType.fromTypeMask(typeMask).stream().collect(toMap(m -> m, m -> mirror)).entrySet().stream(); + }).collect(groupingBy(Entry::getKey, MirrorType::newMap, mapping(Entry::getValue, toList()))); - // put random entry from each type list into mirrors + // select random mirror for each type Random random = new Random(); - for (MirrorType type : MirrorType.values()) { - List list = mirrorListMap.get(type); - - if (!list.isEmpty()) { - mirrors.put(type, list.get(random.nextInt(list.size()))); - } - } - - getCache().putData("mirrors", null, null, mirrors); + mirrorLists.forEach((type, options) -> { + String selection = options.get(random.nextInt(options.size())); + mirrors.put(type, selection); + }); } + // return selected mirror return mirrors.get(mirrorType); } } - protected Document getXmlResource(final MirrorType mirrorType, final String path) throws IOException { - CachedXmlResource resource = new CachedXmlResource(path) { - - @Override - protected URL getResourceLocation(String path) throws IOException { - return getResourceURL(mirrorType, path); - }; - }; - - // fetch data or retrieve from cache - try { - return resource.getDocument(); - } catch (FileNotFoundException e) { - throw new FileNotFoundException("Resource not found: " + getResourceURL(mirrorType, path)); // simplify error message - } + protected Document getXmlResource(MirrorType mirror, String path) throws Exception { + Cache cache = Cache.getCache(getName(), CacheType.Monthly); + Duration expirationTime = Duration.ofDays(1); + Resource xml = cache.xml(path, s -> getResource(mirror, s), expirationTime); + return xml.get(); } - protected URL getResourceURL(MirrorType mirrorType, String path) throws IOException { - if (mirrorType != null) { - // use mirror - String mirror = getMirror(mirrorType); - if (mirror != null && mirror.length() > 0) { - return new URL(mirror + path); - } + protected URL getResource(MirrorType mirror, String path) throws Exception { + StringBuilder url = new StringBuilder(getMirror(mirror)).append('/').append(mirror.prefix()).append('/'); + if (mirror.keyRequired()) { + url.append(apikey).append('/'); } - - // use default server - return new URL("http", "thetvdb.com", path); + return new URL(url.append(path).toString()); } protected static enum MirrorType { - XML(1), BANNER(2), ZIP(4), SEARCH(1); - private final int bitMask; + NULL(0), SEARCH(1), XML(1), BANNER(2); + + final int bitMask; private MirrorType(int bitMask) { this.bitMask = bitMask; } - public static EnumSet fromTypeMask(int typeMask) { - // initialize enum set with all types - EnumSet enumSet = EnumSet.allOf(MirrorType.class); - for (MirrorType type : values()) { - if ((typeMask & type.bitMask) == 0) { - // remove types that are not set - enumSet.remove(type); - } - } - return enumSet; + public String prefix() { + return this != BANNER ? "api" : "banners"; + } + + public boolean keyRequired() { + return this != BANNER && this != SEARCH; + } + + public static EnumSet fromTypeMask(int mask) { + // convert bit mask to enumset + return EnumSet.of(SEARCH, XML, BANNER).stream().filter(m -> { + return (mask & m.bitMask) != 0; + }).collect(toCollection(MirrorType::newSet)); }; + public static EnumSet newSet() { + return EnumSet.noneOf(MirrorType.class); + } + + public static EnumMap newMap() { + return new EnumMap(MirrorType.class); + } + } public SeriesInfo getSeriesInfoByIMDbID(int imdbid, Locale locale) throws Exception { @@ -421,7 +395,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { return asList(cachedList); } - Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + series.getId() + "/banners.xml"); + Document dom = getXmlResource(MirrorType.XML, "series/" + series.getId() + "/banners.xml"); List nodes = selectNodes("//Banner", dom); List banners = new ArrayList(); @@ -431,7 +405,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider { Map item = new EnumMap(BannerProperty.class); // insert banner mirror - item.put(BannerProperty.BannerMirror, getResourceURL(MirrorType.BANNER, "/banners/").toString()); + item.put(BannerProperty.BannerMirror, getResource(MirrorType.BANNER, "").toString()); // copy values from xml for (BannerProperty key : BannerProperty.values()) { diff --git a/source/net/filebot/web/WebRequest.java b/source/net/filebot/web/WebRequest.java index 8f336ccb..30c22db2 100644 --- a/source/net/filebot/web/WebRequest.java +++ b/source/net/filebot/web/WebRequest.java @@ -31,6 +31,7 @@ import java.util.zip.InflaterInputStream; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParserFactory; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; @@ -43,6 +44,8 @@ import net.filebot.util.ByteBufferOutputStream; import org.w3c.dom.Document; import org.xml.sax.InputSource; import org.xml.sax.SAXException; +import org.xml.sax.XMLReader; +import org.xml.sax.helpers.DefaultHandler; public final class WebRequest { @@ -150,8 +153,9 @@ public final class WebRequest { } // no data, e.g. If-Modified-Since requests - if (contentLength < 0 && buffer.getByteBuffer().remaining() == 0) + if (contentLength < 0 && buffer.getByteBuffer().remaining() == 0) { return null; + } return buffer.getByteBuffer(); } @@ -285,10 +289,20 @@ public final class WebRequest { return buffer.toString(); } - /** - * Dummy constructor to prevent instantiation. - */ + public static void validateXml(String xml) throws SAXException, ParserConfigurationException, IOException { + SAXParserFactory sax = SAXParserFactory.newInstance(); + sax.setValidating(false); + sax.setNamespaceAware(false); + + XMLReader reader = sax.newSAXParser().getXMLReader(); + + // throw exception on error + reader.setErrorHandler(new DefaultHandler()); + reader.parse(new InputSource(new StringReader(xml))); + } + private WebRequest() { throw new UnsupportedOperationException(); } + } diff --git a/test/net/filebot/web/TheTVDBClientTest.java b/test/net/filebot/web/TheTVDBClientTest.java index 08327ec7..13d99709 100644 --- a/test/net/filebot/web/TheTVDBClientTest.java +++ b/test/net/filebot/web/TheTVDBClientTest.java @@ -103,23 +103,13 @@ public class TheTVDBClientTest { assertEquals("http://www.thetvdb.com/?tab=seasonall&id=78874", thetvdb.getEpisodeListLink(new TheTVDBSearchResult("Firefly", 78874)).toString()); } - @Test - public void getMirror() throws Exception { - assertNotNull(thetvdb.getMirror(MirrorType.XML)); - assertNotNull(thetvdb.getMirror(MirrorType.BANNER)); - assertNotNull(thetvdb.getMirror(MirrorType.ZIP)); - } - @Test public void resolveTypeMask() { // no flags set - assertEquals(EnumSet.noneOf(MirrorType.class), MirrorType.fromTypeMask(0)); - - // xml and zip flags set - assertEquals(EnumSet.of(MirrorType.ZIP, MirrorType.XML, MirrorType.SEARCH), MirrorType.fromTypeMask(5)); + assertEquals(MirrorType.newSet(), MirrorType.fromTypeMask(0)); // all flags set - assertEquals(EnumSet.allOf(MirrorType.class), MirrorType.fromTypeMask(7)); + assertEquals(EnumSet.of(MirrorType.SEARCH, MirrorType.XML, MirrorType.BANNER), MirrorType.fromTypeMask(7)); } @Test @@ -145,9 +135,9 @@ public class TheTVDBClientTest { assertEquals("2007-09-24", it.getFirstAired().toString()); assertEquals("Action", it.getGenres().get(0)); assertEquals("tt0934814", it.getImdbId()); - assertEquals("English", it.getLanguage()); - assertEquals(310, it.getOverview().length()); - assertEquals("60", it.getRuntime()); + assertEquals("en", it.getLanguage()); + assertEquals(987, it.getOverview().length()); + assertEquals("45", it.getRuntime().toString()); assertEquals("Chuck", it.getName()); } @@ -174,7 +164,7 @@ public class TheTVDBClientTest { assertEquals("fanart", banners.get(0).getBannerType()); assertEquals("1280x720", banners.get(0).getBannerType2()); - assertEquals(486993, WebRequest.fetch(banners.get(0).getUrl()).remaining(), 0); + assertEquals(460058, WebRequest.fetch(banners.get(0).getUrl()).remaining()); } }