diff --git a/source/net/sourceforge/filebot/web/CachedPage.java b/source/net/sourceforge/filebot/web/CachedPage.java new file mode 100644 index 00000000..4ae9ef17 --- /dev/null +++ b/source/net/sourceforge/filebot/web/CachedPage.java @@ -0,0 +1,47 @@ + +package net.sourceforge.filebot.web; + + +import static net.sourceforge.filebot.web.WebRequest.*; +import static net.sourceforge.tuned.FileUtilities.*; + +import java.io.IOException; +import java.io.Reader; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +import net.sf.ehcache.Cache; +import net.sf.ehcache.CacheManager; + + +public class CachedPage extends CachedResource { + + public CachedPage(URL url) { + super(url.toString(), String.class, 2 * 24 * 60 * 60 * 1000); // 48h update interval + } + + + @Override + protected Cache getCache() { + return CacheManager.getInstance().getCache("web-datasource"); + } + + + @Override + public String process(ByteBuffer data) throws Exception { + return Charset.forName("UTF-16BE").decode(data).toString(); + } + + + @Override + protected ByteBuffer fetchData(URL url, long lastModified) throws IOException { + return Charset.forName("UTF-16BE").encode(readAll(openConnection(url))); + } + + + protected Reader openConnection(URL url) throws IOException { + return getReader(url.openConnection()); + } + +} diff --git a/source/net/sourceforge/filebot/web/CachedResource.java b/source/net/sourceforge/filebot/web/CachedResource.java index d8611ecd..a6b43915 100644 --- a/source/net/sourceforge/filebot/web/CachedResource.java +++ b/source/net/sourceforge/filebot/web/CachedResource.java @@ -18,20 +18,28 @@ import net.sf.ehcache.Element; public abstract class CachedResource { - private Cache cache; private String resource; private Class type; private long expirationTime; public CachedResource(String resource, Class type, long expirationTime) { - this.cache = CacheManager.getInstance().getCache("web-persistent-datasource"); this.resource = resource; this.type = type; this.expirationTime = expirationTime; } + protected Cache getCache() { + return CacheManager.getInstance().getCache("web-persistent-datasource"); + } + + + protected ByteBuffer fetchData(URL url, long lastModified) throws IOException { + return fetchIfModified(url, lastModified); + } + + /** * Convert resource data into usable data */ @@ -44,7 +52,7 @@ public abstract class CachedResource { long lastUpdateTime = 0; try { - element = cache.get(cacheKey); + element = getCache().get(cacheKey); if (element != null) { lastUpdateTime = element.getLatestOfCreationAndUpdateTime(); } @@ -58,7 +66,7 @@ public abstract class CachedResource { } // fetch and process resource - ByteBuffer data = fetchIfModified(new URL(resource), element != null ? lastUpdateTime : 0); + ByteBuffer data = fetchData(new URL(resource), element != null ? lastUpdateTime : 0); if (data != null) { try { @@ -69,7 +77,7 @@ public abstract class CachedResource { } try { - cache.put(element); + getCache().put(element); } catch (Exception e) { Logger.getLogger(getClass().getName()).log(Level.WARNING, e.getMessage()); } @@ -77,4 +85,5 @@ public abstract class CachedResource { // update cached data and last-updated time return type.cast(element.getValue()); } + } diff --git a/source/net/sourceforge/filebot/web/IMDbClient.java b/source/net/sourceforge/filebot/web/IMDbClient.java index d4d541e4..553213b5 100644 --- a/source/net/sourceforge/filebot/web/IMDbClient.java +++ b/source/net/sourceforge/filebot/web/IMDbClient.java @@ -7,6 +7,7 @@ import static net.sourceforge.tuned.XPathUtilities.*; import java.io.File; import java.io.IOException; +import java.io.Reader; import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; @@ -111,12 +112,20 @@ public class IMDbClient implements MovieIdentificationService { protected Document parsePage(URL url) throws IOException, SAXException { - URLConnection connection = url.openConnection(); + CachedPage page = new CachedPage(url) { + + @Override + protected Reader openConnection(URL url) throws IOException { + URLConnection connection = url.openConnection(); + + // IMDb refuses default user agent (Java/1.6.0_12) + connection.addRequestProperty("User-Agent", "Mozilla"); + + return getReader(connection); + } + }; - // IMDb refuses default user agent (Java/1.6.0_12) - connection.addRequestProperty("User-Agent", "Mozilla"); - - return getHtmlDocument(connection); + return getHtmlDocument(page.get()); } diff --git a/source/net/sourceforge/filebot/web/TMDbClient.java b/source/net/sourceforge/filebot/web/TMDbClient.java index 8a879e2c..9b055d4a 100644 --- a/source/net/sourceforge/filebot/web/TMDbClient.java +++ b/source/net/sourceforge/filebot/web/TMDbClient.java @@ -5,7 +5,6 @@ package net.sourceforge.filebot.web; import static java.util.Arrays.*; import static java.util.Collections.*; import static net.sourceforge.filebot.web.WebRequest.*; -import static net.sourceforge.tuned.FileUtilities.*; import static net.sourceforge.tuned.XPathUtilities.*; import java.io.File; @@ -29,9 +28,6 @@ import org.w3c.dom.Document; import org.w3c.dom.Node; import org.xml.sax.SAXException; -import net.sf.ehcache.Cache; -import net.sf.ehcache.CacheManager; -import net.sf.ehcache.Element; import net.sourceforge.filebot.ResourceManager; import net.sourceforge.filebot.web.TMDbClient.Artwork.ArtworkProperty; import net.sourceforge.filebot.web.TMDbClient.MovieInfo.MovieProperty; @@ -132,19 +128,7 @@ public class TMDbClient implements MovieIdentificationService { protected Document fetchResource(String method, String parameter, Locale locale) throws IOException, SAXException { - URL url = getResourceLocation(method, parameter, locale); - - Cache cache = CacheManager.getInstance().getCache("web-persistent-datasource"); - Element element = cache.get(url.toString()); - if (element != null) { - return WebRequest.getDocument((String) element.getValue()); - } - - String xml = readAll(WebRequest.getReader(url.openConnection())); - Document dom = getDocument(xml); - - cache.put(new Element(url.toString(), xml)); - return dom; + return getDocument(new CachedPage(getResourceLocation(method, parameter, locale)).get()); } diff --git a/source/net/sourceforge/filebot/web/WebRequest.java b/source/net/sourceforge/filebot/web/WebRequest.java index 45afdfde..6f675a4c 100644 --- a/source/net/sourceforge/filebot/web/WebRequest.java +++ b/source/net/sourceforge/filebot/web/WebRequest.java @@ -87,6 +87,11 @@ public final class WebRequest { } + public static Document getHtmlDocument(String html) throws SAXException, IOException { + return getHtmlDocument(new StringReader(html)); + } + + public static Document getDocument(URL url) throws IOException, SAXException { return getDocument(url.openConnection()); }