From 0d1264febf7ba52ced7f58799a379b988c81bf91 Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Sun, 25 Mar 2012 02:50:28 +0000 Subject: [PATCH] + support episode --filter CLI option --- .../sourceforge/filebot/cli/ArgumentBean.java | 5 +- .../filebot/cli/ArgumentProcessor.java | 2 +- .../filebot/cli/CmdlineInterface.java | 2 +- .../filebot/cli/CmdlineOperations.java | 27 ++++-- .../filebot/cli/ScriptShell.lib.groovy | 3 +- .../filebot/format/ExpressionFilter.java | 70 +++++++++++++++ .../filebot/format/ExpressionFormat.java | 80 +---------------- .../filebot/format/SecureCompiledScript.java | 90 +++++++++++++++++++ .../filebot/similarity/EpisodeMetrics.java | 2 +- .../filebot/similarity/Normalization.java | 3 +- 10 files changed, 193 insertions(+), 91 deletions(-) create mode 100644 source/net/sourceforge/filebot/format/ExpressionFilter.java create mode 100644 source/net/sourceforge/filebot/format/SecureCompiledScript.java diff --git a/source/net/sourceforge/filebot/cli/ArgumentBean.java b/source/net/sourceforge/filebot/cli/ArgumentBean.java index 3bb5dd4f..d8d2e983 100644 --- a/source/net/sourceforge/filebot/cli/ArgumentBean.java +++ b/source/net/sourceforge/filebot/cli/ArgumentBean.java @@ -37,7 +37,10 @@ public class ArgumentBean { @Option(name = "--conflict", usage = "Conflict resolution", metaVar = "[override, skip, fail]") public String conflict = "skip"; - @Option(name = "--format", usage = "Episode naming scheme", metaVar = "expression") + @Option(name = "--filter", usage = "Episode filter", metaVar = "expression") + public String filter = null; + + @Option(name = "--format", usage = "Episode/Movie naming scheme", metaVar = "expression") public String format; @Option(name = "-non-strict", usage = "Use less strict matching") diff --git a/source/net/sourceforge/filebot/cli/ArgumentProcessor.java b/source/net/sourceforge/filebot/cli/ArgumentProcessor.java index 326b35a5..3b2ada26 100644 --- a/source/net/sourceforge/filebot/cli/ArgumentProcessor.java +++ b/source/net/sourceforge/filebot/cli/ArgumentProcessor.java @@ -73,7 +73,7 @@ public class ArgumentProcessor { } if (args.rename) { - cli.rename(files, args.action, args.conflict, args.output, args.format, args.db, args.query, args.order, args.lang, !args.nonStrict); + cli.rename(files, args.action, args.conflict, args.output, args.format, args.db, args.query, args.order, args.filter, args.lang, !args.nonStrict); } if (args.check) { diff --git a/source/net/sourceforge/filebot/cli/CmdlineInterface.java b/source/net/sourceforge/filebot/cli/CmdlineInterface.java index b44b9743..f096b2a6 100644 --- a/source/net/sourceforge/filebot/cli/CmdlineInterface.java +++ b/source/net/sourceforge/filebot/cli/CmdlineInterface.java @@ -9,7 +9,7 @@ import java.util.List; public interface CmdlineInterface { - List rename(Collection files, String action, String conflict, String output, String format, String db, String query, String sortOrder, String lang, boolean strict) throws Exception; + List rename(Collection files, String action, String conflict, String output, String format, String db, String query, String sortOrder, String filter, String lang, boolean strict) throws Exception; List getSubtitles(Collection files, String query, String lang, String output, String encoding, boolean strict) throws Exception; diff --git a/source/net/sourceforge/filebot/cli/CmdlineOperations.java b/source/net/sourceforge/filebot/cli/CmdlineOperations.java index d579cecc..205ff090 100644 --- a/source/net/sourceforge/filebot/cli/CmdlineOperations.java +++ b/source/net/sourceforge/filebot/cli/CmdlineOperations.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -43,6 +44,7 @@ import net.sourceforge.filebot.MediaTypes; import net.sourceforge.filebot.WebServices; import net.sourceforge.filebot.archive.Archive; import net.sourceforge.filebot.archive.FileMapper; +import net.sourceforge.filebot.format.ExpressionFilter; import net.sourceforge.filebot.format.ExpressionFormat; import net.sourceforge.filebot.format.MediaBindingBean; import net.sourceforge.filebot.hash.HashType; @@ -79,8 +81,9 @@ import net.sourceforge.tuned.FileUtilities.FolderFilter; public class CmdlineOperations implements CmdlineInterface { @Override - public List rename(Collection files, String action, String conflict, String output, String expression, String db, String query, String sortOrder, String lang, boolean strict) throws Exception { - ExpressionFormat format = (expression != null) ? new ExpressionFormat(expression) : null; + public List rename(Collection files, String action, String conflict, String output, String formatExpression, String db, String query, String sortOrder, String filterExpression, String lang, boolean strict) throws Exception { + ExpressionFormat format = (formatExpression != null) ? new ExpressionFormat(formatExpression) : null; + ExpressionFilter filter = (filterExpression != null) ? new ExpressionFilter(filterExpression) : null; File outputDir = (output != null && output.length() > 0) ? new File(output) : null; Locale locale = getLanguage(lang).toLocale(); RenameAction renameAction = StandardRenameAction.forName(action); @@ -93,7 +96,7 @@ public class CmdlineOperations implements CmdlineInterface { if (getEpisodeListProvider(db) != null) { // tv series mode - return renameSeries(files, renameAction, conflictAction, outputDir, format, getEpisodeListProvider(db), query, SortOrder.forName(sortOrder), locale, strict); + return renameSeries(files, renameAction, conflictAction, outputDir, format, getEpisodeListProvider(db), query, SortOrder.forName(sortOrder), filter, locale, strict); } if (getMovieIdentificationService(db) != null) { @@ -129,15 +132,15 @@ public class CmdlineOperations implements CmdlineInterface { CLILogger.finest(format("Filename pattern: [%.02f] SxE, [%.02f] CWS", sxe / max, cws / max)); if (sxe >= (max * 0.65) || cws >= (max * 0.65)) { - return renameSeries(files, renameAction, conflictAction, outputDir, format, WebServices.TVRage, query, SortOrder.forName(sortOrder), locale, strict); // use default episode db + return renameSeries(files, renameAction, conflictAction, outputDir, format, WebServices.TVRage, query, SortOrder.forName(sortOrder), filter, locale, strict); // use default episode db } else { return renameMovie(files, renameAction, conflictAction, outputDir, format, WebServices.OpenSubtitles, query, locale, strict); // use default movie db } } - public List renameSeries(Collection files, RenameAction renameAction, ConflictAction conflictAction, File outputDir, ExpressionFormat format, EpisodeListProvider db, String query, SortOrder sortOrder, Locale locale, - boolean strict) throws Exception { + public List renameSeries(Collection files, RenameAction renameAction, ConflictAction conflictAction, File outputDir, ExpressionFormat format, EpisodeListProvider db, String query, SortOrder sortOrder, + ExpressionFilter filter, Locale locale, boolean strict) throws Exception { CLILogger.config(format("Rename episodes using [%s]", db.getName())); List mediaFiles = filter(files, VIDEO_FILES, SUBTITLE_FILES); @@ -167,6 +170,18 @@ public class CmdlineOperations implements CmdlineInterface { // fetch episode data Set episodes = fetchEpisodeSet(db, seriesNames, sortOrder, locale, strict); + // filter episodes + if (filter != null) { + CLILogger.fine(String.format("Apply Filter: {%s}", filter.getExpression())); + for (Iterator itr = episodes.iterator(); itr.hasNext();) { + Episode episode = itr.next(); + if (filter.matches(new MediaBindingBean(episode, null))) { + CLILogger.finest(String.format("Exclude [%s]", episode)); + itr.remove(); + } + } + } + if (episodes.size() > 0) { matches.addAll(matchEpisodes(filter(batch, VIDEO_FILES), episodes, strict)); matches.addAll(matchEpisodes(filter(batch, SUBTITLE_FILES), episodes, strict)); diff --git a/source/net/sourceforge/filebot/cli/ScriptShell.lib.groovy b/source/net/sourceforge/filebot/cli/ScriptShell.lib.groovy index 5d173385..f84a3442 100644 --- a/source/net/sourceforge/filebot/cli/ScriptShell.lib.groovy +++ b/source/net/sourceforge/filebot/cli/ScriptShell.lib.groovy @@ -187,7 +187,7 @@ List.metaClass.sortBySimilarity = { prime, Closure toStringFunction = { obj -> o // CLI bindings def rename(args) { args = _defaults(args) synchronized (_cli) { - _guarded { _cli.rename(_files(args), args.action as String, args.conflict as String, args.output as String, args.format as String, args.db as String, args.query as String, args.order as String, args.lang as String, args.strict as Boolean) } + _guarded { _cli.rename(_files(args), args.action as String, args.conflict as String, args.output as String, args.format as String, args.db as String, args.query as String, args.order as String, args.filter as String, args.lang as String, args.strict as Boolean) } } } @@ -259,6 +259,7 @@ def _defaults(args) { args.action = args.action ?: _args.action args.conflict = args.conflict ?: _args.conflict args.query = args.query ?: _args.query + args.filter = args.filter ?: _args.filter args.format = args.format ?: _args.format args.db = args.db ?: _args.db args.order = args.order ?: _args.order diff --git a/source/net/sourceforge/filebot/format/ExpressionFilter.java b/source/net/sourceforge/filebot/format/ExpressionFilter.java new file mode 100644 index 00000000..c4e46bed --- /dev/null +++ b/source/net/sourceforge/filebot/format/ExpressionFilter.java @@ -0,0 +1,70 @@ + +package net.sourceforge.filebot.format; + + +import java.io.InputStreamReader; +import java.security.AccessController; + +import javax.script.Bindings; +import javax.script.Compilable; +import javax.script.CompiledScript; +import javax.script.ScriptContext; +import javax.script.ScriptEngine; +import javax.script.ScriptException; +import javax.script.SimpleScriptContext; + +import org.codehaus.groovy.jsr223.GroovyScriptEngineFactory; + + +public class ExpressionFilter { + + private final String expression; + + private final CompiledScript script; + + + public ExpressionFilter(String expression) throws ScriptException { + this.expression = expression; + this.script = new SecureCompiledScript(((Compilable) initScriptEngine()).compile(expression)); // sandboxed script + } + + + public String getExpression() { + return expression; + } + + + protected ScriptEngine initScriptEngine() throws ScriptException { + // use Groovy script engine + ScriptEngine engine = new GroovyScriptEngineFactory().getScriptEngine(); + engine.eval(new InputStreamReader(ExpressionFormat.class.getResourceAsStream("ExpressionFormat.lib.groovy"))); + return engine; + } + + + public boolean matches(Object value) throws ScriptException { + return matches(new ExpressionBindings(value)); + } + + + public boolean matches(Bindings bindings) throws ScriptException { + // use privileged bindings so we are not restricted by the script sandbox + Bindings priviledgedBindings = PrivilegedInvocation.newProxy(Bindings.class, bindings, AccessController.getContext()); + + // initialize script context with the privileged bindings + ScriptContext context = new SimpleScriptContext(); + context.setBindings(priviledgedBindings, ScriptContext.GLOBAL_SCOPE); + + try { + Object value = script.eval(context); + if (value instanceof Boolean) { + return (Boolean) value; + } + } catch (Throwable e) { + // ignore any and all scripting exceptions + } + + return false; + } + +} diff --git a/source/net/sourceforge/filebot/format/ExpressionFormat.java b/source/net/sourceforge/filebot/format/ExpressionFormat.java index 60d91d23..4481c528 100644 --- a/source/net/sourceforge/filebot/format/ExpressionFormat.java +++ b/source/net/sourceforge/filebot/format/ExpressionFormat.java @@ -7,24 +7,13 @@ import static net.sourceforge.tuned.FileUtilities.*; import groovy.lang.GroovyRuntimeException; import groovy.lang.MissingPropertyException; -import java.io.File; -import java.io.FilePermission; import java.io.InputStreamReader; -import java.net.SocketPermission; -import java.security.AccessControlContext; -import java.security.AccessControlException; import java.security.AccessController; -import java.security.PermissionCollection; -import java.security.Permissions; -import java.security.PrivilegedActionException; -import java.security.PrivilegedExceptionAction; -import java.security.ProtectionDomain; import java.text.FieldPosition; import java.text.Format; import java.text.ParsePosition; import java.util.ArrayList; import java.util.List; -import java.util.PropertyPermission; import javax.script.Bindings; import javax.script.Compilable; @@ -37,8 +26,6 @@ import javax.script.SimpleScriptContext; import org.codehaus.groovy.control.MultipleCompilationErrorsException; import org.codehaus.groovy.jsr223.GroovyScriptEngineFactory; -import net.sourceforge.tuned.ExceptionUtilities; - public class ExpressionFormat extends Format { @@ -223,14 +210,11 @@ public class ExpressionFormat extends Format { private Object[] secure(Object[] compilation) { - // create sandbox AccessControlContext - AccessControlContext sandbox = new AccessControlContext(new ProtectionDomain[] { new ProtectionDomain(null, getSandboxPermissions()) }); - for (int i = 0; i < compilation.length; i++) { Object snipped = compilation[i]; if (snipped instanceof CompiledScript) { - compilation[i] = new SecureCompiledScript((CompiledScript) snipped, sandbox); + compilation[i] = new SecureCompiledScript((CompiledScript) snipped); } } @@ -238,68 +222,6 @@ public class ExpressionFormat extends Format { } - private PermissionCollection getSandboxPermissions() { - Permissions permissions = new Permissions(); - - permissions.add(new RuntimePermission("createClassLoader")); - permissions.add(new FilePermission("<>", "read")); - permissions.add(new SocketPermission("*", "connect")); - permissions.add(new PropertyPermission("*", "read")); - permissions.add(new RuntimePermission("getenv.*")); - - // write permissions for temp and cache folders - permissions.add(new FilePermission(new File(System.getProperty("ehcache.disk.store.dir")).getAbsolutePath() + File.separator + "-", "write, delete")); - permissions.add(new FilePermission(new File(System.getProperty("java.io.tmpdir")).getAbsolutePath() + File.separator + "-", "write, delete")); - - return permissions; - } - - - private static class SecureCompiledScript extends CompiledScript { - - private final CompiledScript compiledScript; - private final AccessControlContext sandbox; - - - private SecureCompiledScript(CompiledScript compiledScript, AccessControlContext sandbox) { - this.compiledScript = compiledScript; - this.sandbox = sandbox; - } - - - @Override - public Object eval(final ScriptContext context) throws ScriptException { - try { - return AccessController.doPrivileged(new PrivilegedExceptionAction() { - - @Override - public Object run() throws ScriptException { - return compiledScript.eval(context); - } - }, sandbox); - } catch (PrivilegedActionException e) { - AccessControlException accessException = ExceptionUtilities.findCause(e, AccessControlException.class); - - // try to unwrap AccessControlException - if (accessException != null) - throw new ExpressionException(accessException); - - // forward ScriptException - // e.getException() should be an instance of ScriptException, - // as only "checked" exceptions will be "wrapped" in a PrivilegedActionException - throw (ScriptException) e.getException(); - } - } - - - @Override - public ScriptEngine getEngine() { - return compiledScript.getEngine(); - } - - } - - @Override public Object parseObject(String source, ParsePosition pos) { throw new UnsupportedOperationException(); diff --git a/source/net/sourceforge/filebot/format/SecureCompiledScript.java b/source/net/sourceforge/filebot/format/SecureCompiledScript.java new file mode 100644 index 00000000..c1ae8ace --- /dev/null +++ b/source/net/sourceforge/filebot/format/SecureCompiledScript.java @@ -0,0 +1,90 @@ + +package net.sourceforge.filebot.format; + + +import java.io.File; +import java.io.FilePermission; +import java.net.SocketPermission; +import java.security.AccessControlContext; +import java.security.AccessControlException; +import java.security.AccessController; +import java.security.PermissionCollection; +import java.security.Permissions; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.security.ProtectionDomain; +import java.util.PropertyPermission; + +import javax.script.CompiledScript; +import javax.script.ScriptContext; +import javax.script.ScriptEngine; +import javax.script.ScriptException; + +import net.sourceforge.tuned.ExceptionUtilities; + + +public class SecureCompiledScript extends CompiledScript { + + public static PermissionCollection getDefaultSandboxPermissions() { + Permissions permissions = new Permissions(); + + permissions.add(new RuntimePermission("createClassLoader")); + permissions.add(new FilePermission("<>", "read")); + permissions.add(new SocketPermission("*", "connect")); + permissions.add(new PropertyPermission("*", "read")); + permissions.add(new RuntimePermission("getenv.*")); + + // write permissions for temp and cache folders + permissions.add(new FilePermission(new File(System.getProperty("ehcache.disk.store.dir")).getAbsolutePath() + File.separator + "-", "write, delete")); + permissions.add(new FilePermission(new File(System.getProperty("java.io.tmpdir")).getAbsolutePath() + File.separator + "-", "write, delete")); + + return permissions; + } + + + private final CompiledScript compiledScript; + private final AccessControlContext sandbox; + + + public SecureCompiledScript(CompiledScript compiledScript) { + this(compiledScript, new AccessControlContext(new ProtectionDomain[] { new ProtectionDomain(null, getDefaultSandboxPermissions()) })); + } + + + public SecureCompiledScript(CompiledScript compiledScript, AccessControlContext sandbox) { + this.compiledScript = compiledScript; + this.sandbox = sandbox; + } + + + @Override + public Object eval(final ScriptContext context) throws ScriptException { + try { + return AccessController.doPrivileged(new PrivilegedExceptionAction() { + + @Override + public Object run() throws ScriptException { + return compiledScript.eval(context); + } + }, sandbox); + } catch (PrivilegedActionException e) { + AccessControlException accessException = ExceptionUtilities.findCause(e, AccessControlException.class); + + // try to unwrap AccessControlException + if (accessException != null) + throw new ExpressionException(accessException); + + // forward ScriptException + // e.getException() should be an instance of ScriptException, + // as only "checked" exceptions will be "wrapped" in a PrivilegedActionException + throw (ScriptException) e.getException(); + } + } + + + @Override + public ScriptEngine getEngine() { + return compiledScript.getEngine(); + } + +} diff --git a/source/net/sourceforge/filebot/similarity/EpisodeMetrics.java b/source/net/sourceforge/filebot/similarity/EpisodeMetrics.java index 61ce073b..2b9b43d5 100644 --- a/source/net/sourceforge/filebot/similarity/EpisodeMetrics.java +++ b/source/net/sourceforge/filebot/similarity/EpisodeMetrics.java @@ -163,7 +163,7 @@ public enum EpisodeMetrics implements SimilarityMetric { protected Object[] fields(Object object) { if (object instanceof Episode) { Episode episode = (Episode) object; - return new Object[] { episode.getSeriesName(), episode.getTitle() }; + return new Object[] { removeTrailingBrackets(episode.getSeriesName()), episode.getTitle() }; } if (object instanceof File) { diff --git a/source/net/sourceforge/filebot/similarity/Normalization.java b/source/net/sourceforge/filebot/similarity/Normalization.java index a2d09536..ea7faf91 100644 --- a/source/net/sourceforge/filebot/similarity/Normalization.java +++ b/source/net/sourceforge/filebot/similarity/Normalization.java @@ -13,6 +13,7 @@ public class Normalization { private static final Pattern punctuation = compile("[\\p{Punct}\\p{Space}]+"); private static final Pattern[] brackets = new Pattern[] { compile("\\([^\\(]*\\)"), compile("\\[[^\\[]*\\]"), compile("\\{[^\\{]*\\}") }; + private static final Pattern trailingParentheses = compile("[(]([^)]*)[)]$"); private static final Pattern checksum = compile("[\\(\\[]\\p{XDigit}{8}[\\]\\)]"); @@ -43,7 +44,7 @@ public class Normalization { public static String removeTrailingBrackets(String name) { // remove trailing braces, e.g. Doctor Who (2005) -> Doctor Who - return name.replaceAll("[(]([^)]*)[)]$", "").trim(); + return trailingParentheses.matcher(name).replaceAll("").trim(); } }