1
0
mirror of https://github.com/mitb-archive/filebot synced 2024-11-02 08:25:02 -04:00

Experiment with new CachedResource framework

This commit is contained in:
Reinhard Pointner 2016-03-07 10:55:45 +00:00
parent a0ebae1db2
commit 4e41d0dfd1
6 changed files with 153 additions and 128 deletions

View File

@ -2,6 +2,7 @@ package net.filebot;
import static java.nio.charset.StandardCharsets.*;
import static net.filebot.Logging.*;
import static net.filebot.web.CachedResource2.*;
import java.io.Serializable;
import java.net.URL;
@ -10,9 +11,13 @@ import java.util.Arrays;
import java.util.function.Predicate;
import net.filebot.web.CachedResource2;
import net.filebot.web.CachedResource2.Source;
import net.filebot.web.FloodLimit;
import net.filebot.web.Resource;
import net.sf.ehcache.Element;
import org.w3c.dom.Document;
public class Cache {
public static Cache getCache(String name, CacheType type) {
@ -131,8 +136,12 @@ public class Cache {
}
}
public CachedResource2<String, String> resource(String url, Duration expirationTime, FloodLimit limit) {
return new CachedResource2<String, String>(url, URL::new, CachedResource2.fetchIfModified(limit), CachedResource2.decode(UTF_8), expirationTime, this);
public Resource<Document> xml(String key, Source<String> source, Duration expirationTime) {
return new CachedResource2<String, Document>(key, source, fetchIfModified(), validateXml(getText(UTF_8)), getXml(String.class::cast), DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, this);
}
public Resource<String> resource(String url, Duration expirationTime, FloodLimit limit) {
return new CachedResource2<String, String>(url, URL::new, withPermit(fetchIfModified(), r -> limit.acquirePermit() != null), getText(UTF_8), String.class::cast, DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, this);
}
}

View File

@ -12,7 +12,9 @@ import java.util.concurrent.Callable;
import net.filebot.Cache;
public class CachedResource2<K, R> {
import org.w3c.dom.Document;
public class CachedResource2<K, R> implements Resource<R> {
public static final int DEFAULT_RETRY_LIMIT = 2;
public static final Duration DEFAULT_RETRY_DELAY = Duration.ofSeconds(2);
@ -21,7 +23,8 @@ public class CachedResource2<K, R> {
protected final Source<K> source;
protected final Fetch fetch;
protected final Parse<R> parse;
protected final Transform<ByteBuffer, ? extends Object> parse;
protected final Transform<? super Object, R> cast;
protected final Duration expirationTime;
@ -30,24 +33,21 @@ public class CachedResource2<K, R> {
protected final Cache cache;
public CachedResource2(K key, Source<K> source, Fetch fetch, Parse<R> parse, Duration expirationTime, Cache cache) {
this(key, source, fetch, parse, DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, cache);
}
public CachedResource2(K key, Source<K> source, Fetch fetch, Parse<R> parse, int retryCountLimit, Duration retryWaitTime, Duration expirationTime, Cache cache) {
public CachedResource2(K key, Source<K> source, Fetch fetch, Transform<ByteBuffer, ? extends Object> parse, Transform<? super Object, R> cast, int retryCountLimit, Duration retryWaitTime, Duration expirationTime, Cache cache) {
this.key = key;
this.source = source;
this.fetch = fetch;
this.parse = parse;
this.cast = cast;
this.expirationTime = expirationTime;
this.retryCountLimit = retryCountLimit;
this.retryWaitTime = retryWaitTime.toMillis();
this.cache = cache;
}
@SuppressWarnings("unchecked")
@Override
public synchronized R get() throws Exception {
return (R) cache.computeIfStale(key, expirationTime, element -> {
Object value = cache.computeIfStale(key, expirationTime, element -> {
URL resource = source.source(key);
long lastModified = element == null ? 0 : element.getLatestOfCreationAndUpdateTime();
@ -61,7 +61,7 @@ public class CachedResource2<K, R> {
return element.getObjectValue();
}
return parse.parse(data);
return parse.transform(data);
} catch (IOException e) {
debug.fine(format("Fetch failed => %s", e));
@ -72,6 +72,8 @@ public class CachedResource2<K, R> {
return element.getObjectKey();
}
});
return cast.transform(value);
}
protected <T> T retry(Callable<T> callable, int retryCount, long retryWaitTime) throws Exception {
@ -101,23 +103,51 @@ public class CachedResource2<K, R> {
}
@FunctionalInterface
public interface Parse<R> {
R parse(ByteBuffer bytes) throws Exception;
public interface Transform<T, R> {
R transform(T object) throws Exception;
}
public static Parse<String> decode(Charset charset) {
return (bb) -> charset.decode(bb).toString();
@FunctionalInterface
public interface Permit<P> {
boolean acquirePermit(URL resource) throws Exception;
}
public static Fetch fetchIfModified(FloodLimit limit) {
public static Transform<ByteBuffer, String> getText(Charset charset) {
return (data) -> charset.decode(data).toString();
}
public static <T> Transform<T, String> validateXml(Transform<T, String> parse) {
return (object) -> {
String xml = parse.transform(object);
WebRequest.validateXml(xml);
return xml;
};
}
public static <T> Transform<T, Document> getXml(Transform<T, String> parse) {
return (object) -> {
return WebRequest.getDocument(parse.transform(object));
};
}
public static Fetch fetchIfModified() {
return (url, lastModified) -> {
try {
limit.acquirePermit();
return WebRequest.fetchIfModified(url, lastModified);
} catch (FileNotFoundException e) {
debug.warning(format("Resource not found: %s => %s", url, e));
return ByteBuffer.allocate(0);
}
};
}
public static Fetch withPermit(Fetch fetch, Permit<?> permit) {
return (url, lastModified) -> {
if (permit.acquirePermit(url)) {
return fetch.fetch(url, lastModified);
}
return null;
};
}
}

View File

@ -0,0 +1,8 @@
package net.filebot.web;
@FunctionalInterface
public interface Resource<R> {
R get() throws Exception;
}

View File

@ -2,17 +2,17 @@ package net.filebot.web;
import static java.util.Arrays.*;
import static java.util.Collections.*;
import static java.util.stream.Collectors.*;
import static net.filebot.util.StringUtilities.*;
import static net.filebot.util.XPathUtilities.*;
import static net.filebot.web.EpisodeUtilities.*;
import static net.filebot.web.WebRequest.*;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Serializable;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.time.Duration;
import java.util.ArrayList;
import java.util.EnumMap;
import java.util.EnumSet;
@ -40,7 +40,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
private final String host = "www.thetvdb.com";
private final Map<MirrorType, String> mirrors = new EnumMap<MirrorType, String>(MirrorType.class);
private final Map<MirrorType, String> mirrors = MirrorType.newMap();
private final String apikey;
@ -106,7 +106,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
@Override
public List<SearchResult> fetchSearchResult(String query, Locale locale) throws Exception {
// perform online search
Document dom = getXmlResource(MirrorType.SEARCH, "/api/GetSeries.php?seriesname=" + encode(query, true) + "&language=" + getLanguageCode(locale));
Document dom = getXmlResource(MirrorType.SEARCH, "GetSeries.php?seriesname=" + encode(query, true) + "&language=" + getLanguageCode(locale));
List<Node> nodes = selectNodes("Data/Series", dom);
Map<Integer, TheTVDBSearchResult> resultSet = new LinkedHashMap<Integer, TheTVDBSearchResult>();
@ -140,7 +140,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
@Override
protected SeriesData fetchSeriesData(SearchResult searchResult, SortOrder sortOrder, Locale locale) throws Exception {
TheTVDBSearchResult series = (TheTVDBSearchResult) searchResult;
Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + series.getSeriesId() + "/all/" + getLanguageCode(locale) + ".xml");
Document dom = getXmlResource(MirrorType.XML, "series/" + series.getSeriesId() + "/all/" + getLanguageCode(locale) + ".xml");
// parse series info
Node seriesNode = selectNode("Data/Series", dom);
@ -163,9 +163,9 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
seriesInfo.setGenres(getListContent("Genre", "\\|", seriesNode));
seriesInfo.setStartDate(SimpleDate.parse(getTextContent("FirstAired", seriesNode)));
seriesInfo.setBannerUrl(getResourceURL(MirrorType.BANNER, "/banners/" + getTextContent("banner", seriesNode)));
seriesInfo.setFanartUrl(getResourceURL(MirrorType.BANNER, "/banners/" + getTextContent("fanart", seriesNode)));
seriesInfo.setPosterUrl(getResourceURL(MirrorType.BANNER, "/banners/" + getTextContent("poster", seriesNode)));
seriesInfo.setBannerUrl(getResource(MirrorType.BANNER, getTextContent("banner", seriesNode)));
seriesInfo.setFanartUrl(getResource(MirrorType.BANNER, getTextContent("fanart", seriesNode)));
seriesInfo.setPosterUrl(getResource(MirrorType.BANNER, getTextContent("poster", seriesNode)));
// parse episode data
List<Node> nodes = selectNodes("Data/Episode", dom);
@ -237,7 +237,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
return cachedItem;
}
Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + id + "/all/" + getLanguageCode(locale) + ".xml");
Document dom = getXmlResource(MirrorType.XML, "series/" + id + "/all/" + getLanguageCode(locale) + ".xml");
String name = selectString("//SeriesName", dom);
TheTVDBSearchResult series = new TheTVDBSearchResult(name, id);
@ -255,7 +255,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
return cachedItem;
}
Document dom = getXmlResource(null, "/api/GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale));
Document dom = getXmlResource(MirrorType.SEARCH, "GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale));
String id = selectString("//seriesid", dom);
String name = selectString("//SeriesName", dom);
@ -268,113 +268,87 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
return series;
}
protected String getMirror(MirrorType mirrorType) throws IOException {
protected String getMirror(MirrorType mirrorType) throws Exception {
// use default server
if (mirrorType == MirrorType.NULL) {
return "http://thetvdb.com";
}
synchronized (mirrors) {
if (mirrors.isEmpty()) {
// try cache first
try {
@SuppressWarnings("unchecked")
Map<MirrorType, String> cachedMirrors = getCache().getData("mirrors", null, null, Map.class);
if (cachedMirrors != null) {
mirrors.putAll(cachedMirrors);
return mirrors.get(mirrorType);
}
} catch (Exception e) {
Logger.getLogger(getClass().getName()).log(Level.SEVERE, e.getMessage(), e);
}
// initialize mirrors
Document dom = getXmlResource(null, "/api/" + apikey + "/mirrors.xml");
if (mirrors.isEmpty()) {
Document dom = getXmlResource(MirrorType.NULL, "mirrors.xml");
// all mirrors by type
Map<MirrorType, List<String>> mirrorListMap = new EnumMap<MirrorType, List<String>>(MirrorType.class);
// initialize mirror list per type
for (MirrorType type : MirrorType.values()) {
mirrorListMap.put(type, new ArrayList<String>(5));
}
// traverse all mirrors
for (Node node : selectNodes("Mirrors/Mirror", dom)) {
// mirror data
// collect all mirror data
Map<MirrorType, List<String>> mirrorLists = selectNodes("Mirrors/Mirror", dom).stream().flatMap(node -> {
String mirror = getTextContent("mirrorpath", node);
int typeMask = Integer.parseInt(getTextContent("typemask", node));
// add mirror to the according type lists
for (MirrorType type : MirrorType.fromTypeMask(typeMask)) {
mirrorListMap.get(type).add(mirror);
}
}
return MirrorType.fromTypeMask(typeMask).stream().collect(toMap(m -> m, m -> mirror)).entrySet().stream();
}).collect(groupingBy(Entry::getKey, MirrorType::newMap, mapping(Entry::getValue, toList())));
// put random entry from each type list into mirrors
// select random mirror for each type
Random random = new Random();
for (MirrorType type : MirrorType.values()) {
List<String> list = mirrorListMap.get(type);
if (!list.isEmpty()) {
mirrors.put(type, list.get(random.nextInt(list.size())));
}
}
getCache().putData("mirrors", null, null, mirrors);
mirrorLists.forEach((type, options) -> {
String selection = options.get(random.nextInt(options.size()));
mirrors.put(type, selection);
});
}
// return selected mirror
return mirrors.get(mirrorType);
}
}
protected Document getXmlResource(final MirrorType mirrorType, final String path) throws IOException {
CachedXmlResource resource = new CachedXmlResource(path) {
@Override
protected URL getResourceLocation(String path) throws IOException {
return getResourceURL(mirrorType, path);
};
};
// fetch data or retrieve from cache
try {
return resource.getDocument();
} catch (FileNotFoundException e) {
throw new FileNotFoundException("Resource not found: " + getResourceURL(mirrorType, path)); // simplify error message
}
protected Document getXmlResource(MirrorType mirror, String path) throws Exception {
Cache cache = Cache.getCache(getName(), CacheType.Monthly);
Duration expirationTime = Duration.ofDays(1);
Resource<Document> xml = cache.xml(path, s -> getResource(mirror, s), expirationTime);
return xml.get();
}
protected URL getResourceURL(MirrorType mirrorType, String path) throws IOException {
if (mirrorType != null) {
// use mirror
String mirror = getMirror(mirrorType);
if (mirror != null && mirror.length() > 0) {
return new URL(mirror + path);
protected URL getResource(MirrorType mirror, String path) throws Exception {
StringBuilder url = new StringBuilder(getMirror(mirror)).append('/').append(mirror.prefix()).append('/');
if (mirror.keyRequired()) {
url.append(apikey).append('/');
}
}
// use default server
return new URL("http", "thetvdb.com", path);
return new URL(url.append(path).toString());
}
protected static enum MirrorType {
XML(1), BANNER(2), ZIP(4), SEARCH(1);
private final int bitMask;
NULL(0), SEARCH(1), XML(1), BANNER(2);
final int bitMask;
private MirrorType(int bitMask) {
this.bitMask = bitMask;
}
public static EnumSet<MirrorType> fromTypeMask(int typeMask) {
// initialize enum set with all types
EnumSet<MirrorType> enumSet = EnumSet.allOf(MirrorType.class);
for (MirrorType type : values()) {
if ((typeMask & type.bitMask) == 0) {
// remove types that are not set
enumSet.remove(type);
public String prefix() {
return this != BANNER ? "api" : "banners";
}
public boolean keyRequired() {
return this != BANNER && this != SEARCH;
}
return enumSet;
public static EnumSet<MirrorType> fromTypeMask(int mask) {
// convert bit mask to enumset
return EnumSet.of(SEARCH, XML, BANNER).stream().filter(m -> {
return (mask & m.bitMask) != 0;
}).collect(toCollection(MirrorType::newSet));
};
public static EnumSet<MirrorType> newSet() {
return EnumSet.noneOf(MirrorType.class);
}
public static <T> EnumMap<MirrorType, T> newMap() {
return new EnumMap<MirrorType, T>(MirrorType.class);
}
}
public SeriesInfo getSeriesInfoByIMDbID(int imdbid, Locale locale) throws Exception {
@ -421,7 +395,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
return asList(cachedList);
}
Document dom = getXmlResource(MirrorType.XML, "/api/" + apikey + "/series/" + series.getId() + "/banners.xml");
Document dom = getXmlResource(MirrorType.XML, "series/" + series.getId() + "/banners.xml");
List<Node> nodes = selectNodes("//Banner", dom);
List<BannerDescriptor> banners = new ArrayList<BannerDescriptor>();
@ -431,7 +405,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
Map<BannerProperty, String> item = new EnumMap<BannerProperty, String>(BannerProperty.class);
// insert banner mirror
item.put(BannerProperty.BannerMirror, getResourceURL(MirrorType.BANNER, "/banners/").toString());
item.put(BannerProperty.BannerMirror, getResource(MirrorType.BANNER, "").toString());
// copy values from xml
for (BannerProperty key : BannerProperty.values()) {

View File

@ -31,6 +31,7 @@ import java.util.zip.InflaterInputStream;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParserFactory;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
@ -43,6 +44,8 @@ import net.filebot.util.ByteBufferOutputStream;
import org.w3c.dom.Document;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler;
public final class WebRequest {
@ -150,8 +153,9 @@ public final class WebRequest {
}
// no data, e.g. If-Modified-Since requests
if (contentLength < 0 && buffer.getByteBuffer().remaining() == 0)
if (contentLength < 0 && buffer.getByteBuffer().remaining() == 0) {
return null;
}
return buffer.getByteBuffer();
}
@ -285,10 +289,20 @@ public final class WebRequest {
return buffer.toString();
}
/**
* Dummy constructor to prevent instantiation.
*/
public static void validateXml(String xml) throws SAXException, ParserConfigurationException, IOException {
SAXParserFactory sax = SAXParserFactory.newInstance();
sax.setValidating(false);
sax.setNamespaceAware(false);
XMLReader reader = sax.newSAXParser().getXMLReader();
// throw exception on error
reader.setErrorHandler(new DefaultHandler());
reader.parse(new InputSource(new StringReader(xml)));
}
private WebRequest() {
throw new UnsupportedOperationException();
}
}

View File

@ -103,23 +103,13 @@ public class TheTVDBClientTest {
assertEquals("http://www.thetvdb.com/?tab=seasonall&id=78874", thetvdb.getEpisodeListLink(new TheTVDBSearchResult("Firefly", 78874)).toString());
}
@Test
public void getMirror() throws Exception {
assertNotNull(thetvdb.getMirror(MirrorType.XML));
assertNotNull(thetvdb.getMirror(MirrorType.BANNER));
assertNotNull(thetvdb.getMirror(MirrorType.ZIP));
}
@Test
public void resolveTypeMask() {
// no flags set
assertEquals(EnumSet.noneOf(MirrorType.class), MirrorType.fromTypeMask(0));
// xml and zip flags set
assertEquals(EnumSet.of(MirrorType.ZIP, MirrorType.XML, MirrorType.SEARCH), MirrorType.fromTypeMask(5));
assertEquals(MirrorType.newSet(), MirrorType.fromTypeMask(0));
// all flags set
assertEquals(EnumSet.allOf(MirrorType.class), MirrorType.fromTypeMask(7));
assertEquals(EnumSet.of(MirrorType.SEARCH, MirrorType.XML, MirrorType.BANNER), MirrorType.fromTypeMask(7));
}
@Test
@ -145,9 +135,9 @@ public class TheTVDBClientTest {
assertEquals("2007-09-24", it.getFirstAired().toString());
assertEquals("Action", it.getGenres().get(0));
assertEquals("tt0934814", it.getImdbId());
assertEquals("English", it.getLanguage());
assertEquals(310, it.getOverview().length());
assertEquals("60", it.getRuntime());
assertEquals("en", it.getLanguage());
assertEquals(987, it.getOverview().length());
assertEquals("45", it.getRuntime().toString());
assertEquals("Chuck", it.getName());
}
@ -174,7 +164,7 @@ public class TheTVDBClientTest {
assertEquals("fanart", banners.get(0).getBannerType());
assertEquals("1280x720", banners.get(0).getBannerType2());
assertEquals(486993, WebRequest.fetch(banners.get(0).getUrl()).remaining(), 0);
assertEquals(460058, WebRequest.fetch(banners.get(0).getUrl()).remaining());
}
}