moparisthebest
/
filebot
mirror of https://github.com/mitb-archive/filebot
1
0
Fork 0
Code Issues Releases Wiki Activity

* added caching for TMDB and IMDB

This commit is contained in:
Reinhard Pointner 2012-02-17 04:34:44 +00:00
parent 43ee82a9fa
commit f9c823581c
5 changed files with 81 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
source/net/sourceforge/filebot/web/CachedPage.java Normal file
View File

@ -0,0 +1,47 @@
package net.sourceforge.filebot.web;
import static net.sourceforge.filebot.web.WebRequest.*;
import static net.sourceforge.tuned.FileUtilities.*;
import java.io.IOException;
import java.io.Reader;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
public class CachedPage extends CachedResource<String> {
public CachedPage(URL url) {
super(url.toString(), String.class, 2 * 24 * 60 * 60 * 1000); // 48h update interval
}
@Override
protected Cache getCache() {
return CacheManager.getInstance().getCache("web-datasource");
}
@Override
public String process(ByteBuffer data) throws Exception {
return Charset.forName("UTF-16BE").decode(data).toString();
}
@Override
protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
return Charset.forName("UTF-16BE").encode(readAll(openConnection(url)));
}
protected Reader openConnection(URL url) throws IOException {
return getReader(url.openConnection());
}
}

19
source/net/sourceforge/filebot/web/CachedResource.java
View File

@ -18,20 +18,28 @@ import net.sf.ehcache.Element;
public abstract class CachedResource<T extends Serializable> {
private Cache cache;
private String resource;
private Class<T> type;
private long expirationTime;
public CachedResource(String resource, Class<T> type, long expirationTime) {
this.cache = CacheManager.getInstance().getCache("web-persistent-datasource");
this.resource = resource;
this.type = type;
this.expirationTime = expirationTime;
}
protected Cache getCache() {
return CacheManager.getInstance().getCache("web-persistent-datasource");
}
protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
return fetchIfModified(url, lastModified);
}
/**
* Convert resource data into usable data
*/
@ -44,7 +52,7 @@ public abstract class CachedResource<T extends Serializable> {
long lastUpdateTime = 0;
try {
element = cache.get(cacheKey);
element = getCache().get(cacheKey);
if (element != null) {
lastUpdateTime = element.getLatestOfCreationAndUpdateTime();
}
@ -58,7 +66,7 @@ public abstract class CachedResource<T extends Serializable> {
}
// fetch and process resource
ByteBuffer data = fetchIfModified(new URL(resource), element != null ? lastUpdateTime : 0);
ByteBuffer data = fetchData(new URL(resource), element != null ? lastUpdateTime : 0);
if (data != null) {
try {
@ -69,7 +77,7 @@ public abstract class CachedResource<T extends Serializable> {
}
try {
cache.put(element);
getCache().put(element);
} catch (Exception e) {
Logger.getLogger(getClass().getName()).log(Level.WARNING, e.getMessage());
}
@ -77,4 +85,5 @@ public abstract class CachedResource<T extends Serializable> {
// update cached data and last-updated time
return type.cast(element.getValue());
}
}

19
source/net/sourceforge/filebot/web/IMDbClient.java
View File

@ -7,6 +7,7 @@ import static net.sourceforge.tuned.XPathUtilities.*;
import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
@ -111,12 +112,20 @@ public class IMDbClient implements MovieIdentificationService {
protected Document parsePage(URL url) throws IOException, SAXException {
URLConnection connection = url.openConnection();
CachedPage page = new CachedPage(url) {
@Override
protected Reader openConnection(URL url) throws IOException {
URLConnection connection = url.openConnection();
// IMDb refuses default user agent (Java/1.6.0_12)
connection.addRequestProperty("User-Agent", "Mozilla");
return getReader(connection);
}
};
// IMDb refuses default user agent (Java/1.6.0_12)
connection.addRequestProperty("User-Agent", "Mozilla");
return getHtmlDocument(connection);
return getHtmlDocument(page.get());
}

18
source/net/sourceforge/filebot/web/TMDbClient.java
View File

@ -5,7 +5,6 @@ package net.sourceforge.filebot.web;
import static java.util.Arrays.*;
import static java.util.Collections.*;
import static net.sourceforge.filebot.web.WebRequest.*;
import static net.sourceforge.tuned.FileUtilities.*;
import static net.sourceforge.tuned.XPathUtilities.*;
import java.io.File;
@ -29,9 +28,6 @@ import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.xml.sax.SAXException;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
import net.sourceforge.filebot.ResourceManager;
import net.sourceforge.filebot.web.TMDbClient.Artwork.ArtworkProperty;
import net.sourceforge.filebot.web.TMDbClient.MovieInfo.MovieProperty;
@ -132,19 +128,7 @@ public class TMDbClient implements MovieIdentificationService {
protected Document fetchResource(String method, String parameter, Locale locale) throws IOException, SAXException {
URL url = getResourceLocation(method, parameter, locale);
Cache cache = CacheManager.getInstance().getCache("web-persistent-datasource");
Element element = cache.get(url.toString());
if (element != null) {
return WebRequest.getDocument((String) element.getValue());
}
String xml = readAll(WebRequest.getReader(url.openConnection()));
Document dom = getDocument(xml);
cache.put(new Element(url.toString(), xml));
return dom;
return getDocument(new CachedPage(getResourceLocation(method, parameter, locale)).get());
}

5
source/net/sourceforge/filebot/web/WebRequest.java
View File

@ -87,6 +87,11 @@ public final class WebRequest {
}
public static Document getHtmlDocument(String html) throws SAXException, IOException {
return getHtmlDocument(new StringReader(html));
}
public static Document getDocument(URL url) throws IOException, SAXException {
return getDocument(url.openConnection());
}