1
0
mirror of https://github.com/mitb-archive/filebot synced 2024-12-23 16:28:51 -05:00

Experiment with new CachedResource framework

This commit is contained in:
Reinhard Pointner 2016-03-09 10:32:52 +00:00
parent bf2571f04f
commit 7b7d6b36a8
7 changed files with 152 additions and 214 deletions

View File

@ -1,6 +1,8 @@
package net.filebot;
import static java.nio.charset.StandardCharsets.*;
import static java.util.Arrays.*;
import static java.util.stream.Collectors.*;
import static net.filebot.CachedResource.*;
import static net.filebot.Logging.*;
@ -8,6 +10,8 @@ import java.io.Serializable;
import java.net.URL;
import java.time.Duration;
import java.util.Arrays;
import java.util.List;
import java.util.function.Function;
import java.util.function.Predicate;
import net.filebot.CachedResource.Transform;
@ -49,12 +53,8 @@ public class Cache {
public Object get(Object key) {
try {
Element element = cache.get(key);
if (element != null) {
return element.getObjectValue();
}
return getElementValue(cache.get(key));
} catch (Exception e) {
e.printStackTrace();
debug.warning(format("Cache get: %s => %s", key, e));
}
return null;
@ -65,47 +65,35 @@ public class Cache {
Element element = null;
try {
element = cache.get(key);
if (element != null && condition.test(element)) {
return element.getObjectValue();
if (condition.test(element)) {
return getElementValue(element);
}
} catch (Exception e) {
debug.warning(format("Cache get: %s => %s", key, e));
debug.warning(format("Cache computeIf: %s => %s", key, e));
}
// compute if absent
Object value = compute.apply(element);
try {
cache.put(new Element(key, value));
} catch (Exception e) {
debug.warning(format("Cache put: %s => %s", key, e));
}
put(key, value);
return value;
}
public Object computeIfAbsent(Object key, Compute<?> compute) throws Exception {
return computeIf(key, isAbsent(), compute);
}
public Object computeIfStale(Object key, Duration expirationTime, Compute<?> compute) throws Exception {
return computeIf(key, isStale(expirationTime), compute);
}
public Predicate<Element> isAbsent() {
return (element) -> element.getObjectValue() == null;
}
public Predicate<Element> isStale(Duration expirationTime) {
return (element) -> System.currentTimeMillis() - element.getLatestOfCreationAndUpdateTime() < expirationTime.toMillis();
}
public void put(Object key, Object value) {
try {
cache.put(new Element(key, value));
cache.put(createElement(key, value));
} catch (Exception e) {
debug.warning(format("Cache put: %s => %s", key, e));
}
}
protected Object getElementValue(Element element) {
return element == null ? null : element.getObjectValue();
}
protected Element createElement(Object key, Object value) {
return new Element(key, value);
}
public void remove(Object key) {
try {
cache.remove(key);
@ -122,11 +110,64 @@ public class Cache {
}
}
public static Predicate<Element> isAbsent() {
return (element) -> element == null;
}
public static Predicate<Element> isStale(Duration expirationTime) {
return (element) -> element == null || element.getObjectValue() == null || System.currentTimeMillis() - element.getLatestOfCreationAndUpdateTime() < expirationTime.toMillis();
}
@FunctionalInterface
public interface Compute<R> {
R apply(Element element) throws Exception;
}
public <V> TypedCache<V> typed(Function<Object, V> read, Function<V, Object> write) {
return new TypedCache<V>(cache, read, write);
}
public <V> TypedCache<V> cast(Class<V> cls) {
return new TypedCache<V>(cache, it -> cls.cast(it), it -> it);
}
public <V> TypedCache<List<V>> castList(Class<V> cls) {
return new TypedCache<List<V>>(cache, it -> it == null ? null : stream((Object[]) it).map(cls::cast).collect(toList()), it -> it == null ? null : it.toArray());
}
@SuppressWarnings("unchecked")
public static class TypedCache<V> extends Cache {
private final Function<Object, V> read;
private final Function<V, Object> write;
public TypedCache(net.sf.ehcache.Cache cache, Function<Object, V> read, Function<V, Object> write) {
super(cache);
this.read = read;
this.write = write;
}
@Override
public V get(Object key) {
return (V) super.get(key);
}
@Override
public V computeIf(Object key, Predicate<Element> condition, Compute<?> compute) throws Exception {
return (V) super.computeIf(key, condition, compute);
}
@Override
protected Object getElementValue(Element element) {
return read.apply(super.getElementValue(element));
}
@Override
protected Element createElement(Object key, Object value) {
return super.createElement(key, write.apply((V) value));
}
}
@Deprecated
public <T> T get(Object key, Class<T> type) {
return type.cast(get(key));

View File

@ -70,7 +70,7 @@ public class CachedResource<K, R> implements Resource<R> {
@Override
public synchronized R get() throws Exception {
Object value = cache.computeIfStale(key, expirationTime, element -> {
Object value = cache.computeIf(key, Cache.isStale(expirationTime), element -> {
URL url = resource.transform(key);
long lastModified = element == null ? 0 : element.getLatestOfCreationAndUpdateTime();

View File

@ -5,11 +5,10 @@ import static java.util.Arrays.*;
import java.io.Serializable;
import java.util.List;
import java.util.Locale;
import java.util.logging.Level;
import java.util.logging.Logger;
import net.filebot.Cache;
import net.filebot.Cache.Key;
import net.filebot.Cache.TypedCache;
import net.filebot.CacheType;
public abstract class AbstractEpisodeListProvider implements EpisodeListProvider {
@ -19,24 +18,15 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider
protected abstract SearchResult createSearchResult(int id);
protected abstract ResultCache getCache();
protected abstract SortOrder vetoRequestParameter(SortOrder order);
protected abstract Locale vetoRequestParameter(Locale language);
@Override
public List<SearchResult> search(String query, Locale language) throws Exception {
List<SearchResult> results = getCache().getSearchResult(query, language);
if (results != null) {
return results;
}
// perform actual search
results = fetchSearchResult(query, language);
// cache results and return
return getCache().putSearchResult(query, language, results);
return getSearchCache(language).computeIf(query, Cache.isAbsent(), it -> {
return fetchSearchResult(query, language);
});
}
@Override
@ -61,19 +51,24 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider
protected SeriesData getSeriesData(SearchResult searchResult, SortOrder order, Locale language) throws Exception {
// override preferences if requested parameters are not supported
order = vetoRequestParameter(order);
language = vetoRequestParameter(language);
SortOrder requestOrder = vetoRequestParameter(order);
Locale requestLanguage = vetoRequestParameter(language);
SeriesData data = getCache().getSeriesData(searchResult, order, language);
if (data != null) {
return data;
}
return getDataCache(requestOrder, requestLanguage).computeIf(searchResult.getId(), Cache.isAbsent(), it -> {
return fetchSeriesData(searchResult, requestOrder, requestLanguage);
});
}
// perform actual lookup
data = fetchSeriesData(searchResult, order, language);
protected Cache getCache(String section) {
return Cache.getCache(getName() + "_" + section, CacheType.Daily);
}
// cache results and return
return getCache().putSeriesData(searchResult, order, language, data);
protected TypedCache<List<SearchResult>> getSearchCache(Locale language) {
return getCache("search_" + language).castList(SearchResult.class);
}
protected TypedCache<SeriesData> getDataCache(SortOrder order, Locale language) {
return getCache("data_" + order.ordinal() + "_" + language).cast(SeriesData.class);
}
protected static class SeriesData implements Serializable {
@ -96,57 +91,4 @@ public abstract class AbstractEpisodeListProvider implements EpisodeListProvider
}
protected static class ResultCache {
private final String id;
private final Cache cache;
public ResultCache(String id, Cache cache) {
this.id = id;
this.cache = cache;
}
protected String normalize(String query) {
return query == null ? null : query.trim().toLowerCase();
}
public <T extends SearchResult> List<T> putSearchResult(String query, Locale locale, List<T> value) {
putData("SearchResult", normalize(query), locale, value.toArray(new SearchResult[value.size()]));
return value;
}
public List<SearchResult> getSearchResult(String query, Locale locale) {
SearchResult[] data = getData("SearchResult", normalize(query), locale, SearchResult[].class);
return data == null ? null : asList(data);
}
public SeriesData putSeriesData(SearchResult key, SortOrder sortOrder, Locale locale, SeriesData seriesData) {
putData("SeriesData." + sortOrder.name(), key, locale, seriesData);
return seriesData;
}
public SeriesData getSeriesData(SearchResult key, SortOrder sortOrder, Locale locale) {
return getData("SeriesData." + sortOrder.name(), key, locale, SeriesData.class);
}
public <T> T putData(Object category, Object key, Locale locale, T object) {
try {
cache.put(new Key(id, category, locale, key), object);
} catch (Exception e) {
Logger.getLogger(AbstractEpisodeListProvider.class.getName()).log(Level.WARNING, e.getMessage());
}
return object;
}
public <T> T getData(Object category, Object key, Locale locale, Class<T> type) {
try {
return cache.get(new Key(id, category, locale, key), type);
} catch (Exception e) {
Logger.getLogger(AbstractEpisodeListProvider.class.getName()).log(Level.WARNING, e.getMessage(), e);
}
return null;
}
}
}

View File

@ -6,6 +6,7 @@ import static net.filebot.util.XPathUtilities.*;
import static net.filebot.web.EpisodeUtilities.*;
import static net.filebot.web.WebRequest.*;
import java.io.ByteArrayInputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
@ -42,8 +43,6 @@ public class AnidbClient extends AbstractEpisodeListProvider {
private static final FloodLimit REQUEST_LIMIT = new FloodLimit(2, 5, TimeUnit.SECONDS); // no more than 2 requests within a 5 second window
private final String host = "anidb.net";
private final String client;
private final int clientver;
@ -78,8 +77,8 @@ public class AnidbClient extends AbstractEpisodeListProvider {
}
@Override
public ResultCache getCache() {
return new ResultCache(getName(), Cache.getCache(getName(), CacheType.Weekly));
protected Cache getCache(String section) {
return Cache.getCache(getName() + "_" + section, CacheType.Weekly);
}
@Override
@ -105,7 +104,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
AnidbSearchResult anime = (AnidbSearchResult) searchResult;
// e.g. http://api.anidb.net:9001/httpapi?request=anime&client=filebot&clientver=1&protover=1&aid=4521
URL url = new URL("http", "api." + host, 9001, "/httpapi?request=anime&client=" + client + "&clientver=" + clientver + "&protover=1&aid=" + anime.getAnimeId());
URL url = new URL("http://api.anidb.net:9001/httpapi?request=anime&client=" + client + "&clientver=" + clientver + "&protover=1&aid=" + anime.getAnimeId());
// respect flood protection limits
REQUEST_LIMIT.acquirePermit();
@ -190,7 +189,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
@Override
public URI getEpisodeListLink(SearchResult searchResult) {
try {
return new URI("http", host, "/a" + ((AnidbSearchResult) searchResult).getAnimeId(), null);
return new URI("http://anidb.net/a" + searchResult.getId());
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
@ -200,14 +199,8 @@ public class AnidbClient extends AbstractEpisodeListProvider {
* This method is (and must be!) overridden by WebServices.AnidbClientWithLocalSearch to use our own anime index from sourceforge (as to not abuse anidb servers)
*/
public synchronized List<AnidbSearchResult> getAnimeTitles() throws Exception {
URL url = new URL("http", host, "/api/anime-titles.dat.gz");
ResultCache cache = getCache();
@SuppressWarnings("unchecked")
List<AnidbSearchResult> anime = (List) cache.getSearchResult(null, Locale.ROOT);
if (anime != null) {
return anime;
}
// get data file (cached)
byte[] bytes = getCache("root").bytes("anime-titles.dat.gz", n -> new URL("http://anidb.net/api/" + n)).get();
// <aid>|<type>|<language>|<title>
// type: 1=primary title (one per anime), 2=synonyms (multiple per anime), 3=shorttitles (multiple per anime), 4=official title (one per language)
@ -227,7 +220,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
// fetch data
Map<Integer, List<Object[]>> entriesByAnime = new HashMap<Integer, List<Object[]>>(65536);
Scanner scanner = new Scanner(new GZIPInputStream(url.openStream()), "UTF-8");
Scanner scanner = new Scanner(new GZIPInputStream(new ByteArrayInputStream(bytes)), "UTF-8");
try {
while (scanner.hasNextLine()) {
Matcher matcher = pattern.matcher(scanner.nextLine());
@ -260,7 +253,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
}
// build up a list of all possible AniDB search results
anime = new ArrayList<AnidbSearchResult>(entriesByAnime.size());
List<AnidbSearchResult> anime = new ArrayList<AnidbSearchResult>(entriesByAnime.size());
for (Entry<Integer, List<Object[]>> entry : entriesByAnime.entrySet()) {
int aid = entry.getKey();
@ -289,7 +282,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
anime.add(new AnidbSearchResult(aid, primaryTitle, aliasNames));
}
// populate cache
return cache.putSearchResult(null, Locale.ROOT, anime);
return anime;
}
}

View File

@ -171,7 +171,7 @@ public class TMDbClient implements MovieIdentificationService {
public MovieInfo getMovieInfo(String id, Locale locale, boolean extendedInfo) throws Exception {
Object response = request("movie/" + id, extendedInfo ? singletonMap("append_to_response", "alternative_titles,releases,casts,trailers") : null, locale, REQUEST_LIMIT);
Map<MovieProperty, String> fields = mapStringValues(response, MovieProperty.class);
Map<MovieProperty, String> fields = getEnumMap(response, MovieProperty.class);
try {
Map<?, ?> collection = getMap(response, "belongs_to_collection");
@ -241,7 +241,7 @@ public class TMDbClient implements MovieIdentificationService {
List<Person> cast = new ArrayList<Person>();
try {
Stream.of("cast", "crew").flatMap(section -> streamJsonObjects(getMap(response, "casts"), section)).map(it -> {
return mapStringValues(it, PersonProperty.class);
return getEnumMap(it, PersonProperty.class);
}).map(Person::new).forEach(cast::add);
} catch (Exception e) {
debug.warning(format("Bad data: casts => %s", response));

View File

@ -49,11 +49,6 @@ public class TVMazeClient extends AbstractEpisodeListProvider {
return new TVMazeSearchResult(id, null);
}
@Override
public ResultCache getCache() {
return new ResultCache(getName(), Cache.getCache(getName(), CacheType.Daily));
}
@Override
public List<SearchResult> fetchSearchResult(String query, Locale locale) throws Exception {
// e.g. http://api.tvmaze.com/search/shows?q=girls

View File

@ -1,15 +1,14 @@
package net.filebot.web;
import static java.util.Arrays.*;
import static java.util.Collections.*;
import static java.util.stream.Collectors.*;
import static net.filebot.Logging.*;
import static net.filebot.util.StringUtilities.*;
import static net.filebot.util.XPathUtilities.*;
import static net.filebot.web.EpisodeUtilities.*;
import static net.filebot.web.WebRequest.*;
import java.io.Serializable;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
@ -27,6 +26,7 @@ import java.util.logging.Logger;
import javax.swing.Icon;
import net.filebot.Cache;
import net.filebot.Cache.TypedCache;
import net.filebot.CacheType;
import net.filebot.ResourceManager;
import net.filebot.util.FileUtilities;
@ -73,11 +73,6 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
return language != null ? language : Locale.ENGLISH;
}
@Override
public ResultCache getCache() {
return new ResultCache(getName(), Cache.getCache(getName(), CacheType.Daily));
}
public String getLanguageCode(Locale locale) {
String code = locale.getLanguage();
@ -225,41 +220,35 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
return new SeriesData(seriesInfo, episodes);
}
public TheTVDBSearchResult lookupByID(int id, Locale locale) throws Exception {
TheTVDBSearchResult cachedItem = getCache().getData("lookupByID", id, locale, TheTVDBSearchResult.class);
if (cachedItem != null) {
return cachedItem;
public TheTVDBSearchResult lookupByID(int id, Locale language) throws Exception {
if (id <= 0) {
throw new IllegalArgumentException("Illegal TheTVDB ID: " + id);
}
Document dom = getXmlResource(MirrorType.XML, "series/" + id + "/all/" + getLanguageCode(locale) + ".xml");
String name = selectString("//SeriesName", dom);
return getLookupCache("id", language).computeIf(id, Cache.isAbsent(), it -> {
Document dom = getXmlResource(MirrorType.XML, "series/" + id + "/all/" + getLanguageCode(language) + ".xml");
String name = selectString("//SeriesName", dom);
TheTVDBSearchResult series = new TheTVDBSearchResult(name, id);
getCache().putData("lookupByID", id, locale, series);
return series;
return new TheTVDBSearchResult(name, id);
});
}
public TheTVDBSearchResult lookupByIMDbID(int imdbid, Locale locale) throws Exception {
if (imdbid <= 0) {
throw new IllegalArgumentException("id must not be " + imdbid);
throw new IllegalArgumentException("Illegal IMDbID ID: " + imdbid);
}
TheTVDBSearchResult cachedItem = getCache().getData("lookupByIMDbID", imdbid, locale, TheTVDBSearchResult.class);
if (cachedItem != null) {
return cachedItem;
}
return getLookupCache("imdbid", locale).computeIf(imdbid, Cache.isAbsent(), it -> {
Document dom = getXmlResource(MirrorType.SEARCH, "GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale));
Document dom = getXmlResource(MirrorType.SEARCH, "GetSeriesByRemoteID.php?imdbid=" + imdbid + "&language=" + getLanguageCode(locale));
String id = selectString("//seriesid", dom);
String name = selectString("//SeriesName", dom);
String id = selectString("//seriesid", dom);
String name = selectString("//SeriesName", dom);
if (id.isEmpty() || name.isEmpty())
return null;
if (id == null || id.isEmpty() || name == null || name.isEmpty())
return null;
TheTVDBSearchResult series = new TheTVDBSearchResult(name, Integer.parseInt(id));
getCache().putData("lookupByIMDbID", imdbid, locale, series);
return series;
return new TheTVDBSearchResult(name, Integer.parseInt(id));
});
}
protected String getMirror(MirrorType mirrorType) throws Exception {
@ -381,40 +370,28 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
}
public List<BannerDescriptor> getBannerList(TheTVDBSearchResult series) throws Exception {
// check cache first
BannerDescriptor[] cachedList = getCache().getData("banners", series.getId(), null, BannerDescriptor[].class);
if (cachedList != null) {
return asList(cachedList);
}
return getBannerCache().computeIf(series.getId(), Cache.isAbsent(), it -> {
Document dom = getXmlResource(MirrorType.XML, "series/" + series.getId() + "/banners.xml");
Document dom = getXmlResource(MirrorType.XML, "series/" + series.getId() + "/banners.xml");
String bannerMirror = getResource(MirrorType.BANNER, "").toString();
List<BannerDescriptor> banners = new ArrayList<BannerDescriptor>();
return streamNodes("//Banner", dom).map(n -> {
Map<BannerProperty, String> map = getEnumMap(n, BannerProperty.class);
map.put(BannerProperty.BannerMirror, bannerMirror);
for (Node node : selectNodes("//Banner", dom)) {
try {
Map<BannerProperty, String> item = new EnumMap<BannerProperty, String>(BannerProperty.class);
return new BannerDescriptor(map);
}).filter(m -> m.getUrl() != null).collect(toList());
});
}
// insert banner mirror
item.put(BannerProperty.BannerMirror, getResource(MirrorType.BANNER, "").toString());
protected TypedCache<TheTVDBSearchResult> getLookupCache(String type, Locale language) {
// lookup should always yield the same results so we can cache it for longer
return Cache.getCache(getName() + "_" + "lookup" + "_" + type + "_" + language, CacheType.Monthly).cast(TheTVDBSearchResult.class);
}
// copy values from xml
for (BannerProperty key : BannerProperty.values()) {
String value = getTextContent(key.name(), node);
if (value != null && value.length() > 0) {
item.put(key, value);
}
}
banners.add(new BannerDescriptor(item));
} catch (Exception e) {
// log and ignore
Logger.getLogger(getClass().getName()).log(Level.WARNING, "Invalid banner descriptor", e);
}
}
getCache().putData("banners", series.getId(), null, banners.toArray(new BannerDescriptor[0]));
return banners;
protected TypedCache<List<BannerDescriptor>> getBannerCache() {
// banners do not change that often so we can cache them for longer
return Cache.getCache(getName() + "_" + "banner", CacheType.Weekly).castList(BannerDescriptor.class);
}
public static class BannerDescriptor implements Serializable {
@ -441,20 +418,17 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
return fields.get(key);
}
public URL getBannerMirrorUrl() throws MalformedURLException {
public URL getBannerMirrorUrl(String path) {
try {
return new URL(get(BannerProperty.BannerMirror));
return new URL(new URL(get(BannerProperty.BannerMirror)), path);
} catch (Exception e) {
debug.finest(format("Bad banner url: %s", e));
return null;
}
}
public URL getUrl() throws MalformedURLException {
try {
return new URL(getBannerMirrorUrl(), get(BannerProperty.BannerPath));
} catch (Exception e) {
return null;
}
public URL getUrl() {
return getBannerMirrorUrl(get(BannerProperty.BannerPath));
}
public String getExtension() {
@ -480,7 +454,7 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
public Integer getSeason() {
try {
return new Integer(get(BannerProperty.Season));
} catch (NumberFormatException e) {
} catch (Exception e) {
return null;
}
}
@ -517,26 +491,19 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
return Boolean.parseBoolean(get(BannerProperty.SeriesName));
}
public URL getThumbnailUrl() throws MalformedURLException {
try {
return new URL(getBannerMirrorUrl(), get(BannerProperty.ThumbnailPath));
} catch (Exception e) {
return null;
}
public URL getThumbnailUrl() {
return getBannerMirrorUrl(get(BannerProperty.ThumbnailPath));
}
public URL getVignetteUrl() throws MalformedURLException {
try {
return new URL(getBannerMirrorUrl(), get(BannerProperty.VignettePath));
} catch (Exception e) {
return null;
}
public URL getVignetteUrl() {
return getBannerMirrorUrl(get(BannerProperty.VignettePath));
}
@Override
public String toString() {
return fields.toString();
}
}
}