From 28260e51d73822686428d9d28c48a79a8d202cfa Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Wed, 25 Mar 2015 22:38:15 +0000 Subject: [PATCH] * support for delegating 7z extract & list operation to the "7z" cmdline tool rathern than the native bindings which are default --- source/net/filebot/Settings.java | 5 + source/net/filebot/archive/Archive.java | 112 ++++----------- .../net/filebot/archive/ArchiveExtractor.java | 17 +++ source/net/filebot/archive/FileMapper.java | 22 ++- .../filebot/archive/SevenZipExecutable.java | 98 +++++++++++++ .../archive/SevenZipNativeBindings.java | 136 ++++++++++++++++++ source/net/filebot/cli/CmdlineOperations.java | 6 +- source/net/filebot/media/MediaDetection.java | 2 +- .../net/filebot/ui/analyze/ExtractTool.java | 8 +- 9 files changed, 300 insertions(+), 106 deletions(-) create mode 100644 source/net/filebot/archive/ArchiveExtractor.java create mode 100644 source/net/filebot/archive/SevenZipExecutable.java create mode 100644 source/net/filebot/archive/SevenZipNativeBindings.java diff --git a/source/net/filebot/Settings.java b/source/net/filebot/Settings.java index 2a26a599..791c016e 100644 --- a/source/net/filebot/Settings.java +++ b/source/net/filebot/Settings.java @@ -14,6 +14,7 @@ import java.util.prefs.BackingStoreException; import java.util.prefs.Preferences; import net.filebot.UserFiles.FileChooser; +import net.filebot.archive.Archive.Extractor; import net.filebot.cli.ArgumentBean; import net.filebot.util.ExceptionUtilities; import net.filebot.util.PreferencesList; @@ -112,6 +113,10 @@ public final class Settings { return FileChooser.valueOf(System.getProperty("net.filebot.UserFiles.fileChooser", "Swing")); } + public static Extractor getPreferredArchiveExtractor() { + return Extractor.valueOf(System.getProperty("net.filebot.Archive.extractor", "SevenZipNativeBindings")); + } + public static int getPreferredThreadPoolSize() { try { String threadPool = System.getProperty("threadPool"); diff --git a/source/net/filebot/archive/Archive.java b/source/net/filebot/archive/Archive.java index 9d60d737..64872e98 100644 --- a/source/net/filebot/archive/Archive.java +++ b/source/net/filebot/archive/Archive.java @@ -3,13 +3,8 @@ package net.filebot.archive; import java.io.Closeable; import java.io.File; import java.io.FileFilter; -import java.io.FileNotFoundException; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.EnumMap; import java.util.List; -import java.util.Map; import java.util.Scanner; import java.util.Set; import java.util.TreeSet; @@ -17,105 +12,52 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import net.filebot.MediaTypes; +import net.filebot.Settings; import net.filebot.util.FileUtilities.ExtensionFileFilter; import net.filebot.vfs.FileInfo; -import net.filebot.vfs.SimpleFileInfo; -import net.sf.sevenzipjbinding.ArchiveFormat; -import net.sf.sevenzipjbinding.ISevenZipInArchive; -import net.sf.sevenzipjbinding.PropID; -import net.sf.sevenzipjbinding.SevenZipException; public class Archive implements Closeable { - private ISevenZipInArchive inArchive; - private ArchiveOpenVolumeCallback openVolume; + public static enum Extractor { + SevenZipNativeBindings, SevenZipExecutable; - public Archive(File file) throws Exception { - // initialize 7-Zip-JBinding - if (!file.exists()) { - throw new FileNotFoundException(file.getAbsolutePath()); - } - - try { - openVolume = new ArchiveOpenVolumeCallback(); - if (!hasMultiPartIndex(file)) { - // single volume archives and multi-volume rar archives - inArchive = SevenZipLoader.open(openVolume.getStream(file.getAbsolutePath()), openVolume); - } else { - // raw multi-volume archives - inArchive = SevenZipLoader.open(new VolumedArchiveInStream(file.getAbsolutePath(), openVolume), null); + public ArchiveExtractor newInstance(File archive) throws Exception { + switch (this) { + case SevenZipNativeBindings: + return new SevenZipNativeBindings(archive); + case SevenZipExecutable: + return new SevenZipExecutable(archive); } - } catch (InvocationTargetException e) { - throw (Exception) e.getTargetException(); + return null; } } - public int itemCount() throws SevenZipException { - return inArchive.getNumberOfItems(); + public static Archive open(File archive) throws Exception { + return new Archive(Settings.getPreferredArchiveExtractor().newInstance(archive)); } - public Map getItem(int index) throws SevenZipException { - Map item = new EnumMap(PropID.class); + private final ArchiveExtractor extractor; - for (PropID prop : PropID.values()) { - Object value = inArchive.getProperty(index, prop); - if (value != null) { - item.put(prop, value); - } - } - - return item; + public Archive(ArchiveExtractor extractor) throws Exception { + this.extractor = extractor; } - public List listFiles() throws SevenZipException { - List paths = new ArrayList(); - - for (int i = 0; i < inArchive.getNumberOfItems(); i++) { - boolean isFolder = (Boolean) inArchive.getProperty(i, PropID.IS_FOLDER); - if (!isFolder) { - String path = (String) inArchive.getProperty(i, PropID.PATH); - Long length = (Long) inArchive.getProperty(i, PropID.SIZE); - if (path != null) { - paths.add(new SimpleFileInfo(path, length != null ? length : -1)); - } - } - } - - return paths; + public List listFiles() throws Exception { + return extractor.listFiles(); } - public void extract(ExtractOutProvider outputMapper) throws SevenZipException { - inArchive.extract(null, false, new ExtractCallback(inArchive, outputMapper)); + public void extract(File outputDir) throws Exception { + extractor.extract(outputDir); } - public void extract(ExtractOutProvider outputMapper, FileFilter filter) throws SevenZipException { - List selection = new ArrayList(); - - for (int i = 0; i < inArchive.getNumberOfItems(); i++) { - boolean isFolder = (Boolean) inArchive.getProperty(i, PropID.IS_FOLDER); - if (!isFolder) { - String path = (String) inArchive.getProperty(i, PropID.PATH); - if (path != null && filter.accept(new File(path))) { - selection.add(i); - } - } - } - - int[] indices = new int[selection.size()]; - for (int i = 0; i < indices.length; i++) { - indices[i] = selection.get(i); - } - inArchive.extract(indices, false, new ExtractCallback(inArchive, outputMapper)); + public void extract(File outputDir, FileFilter filter) throws Exception { + extractor.extract(outputDir, filter); } @Override public void close() throws IOException { - try { - inArchive.close(); - } catch (SevenZipException e) { - throw new IOException(e); - } finally { - openVolume.close(); + if (extractor instanceof Closeable) { + ((Closeable) extractor).close(); } } @@ -126,9 +68,7 @@ public class Archive implements Closeable { extensions.addAll(MediaTypes.getDefault().getExtensionList("archive")); // formats provided by the library - for (ArchiveFormat it : ArchiveFormat.values()) { - extensions.add(it.getMethodName()); - } + extensions.addAll(SevenZipNativeBindings.getArchiveTypes()); return extensions; } @@ -141,8 +81,8 @@ public class Archive implements Closeable { public static final FileFilter VOLUME_ONE_FILTER = new FileFilter() { - private Pattern volume = Pattern.compile("[.]r[0-9]+$|[.]part[0-9]+|[.][0-9]+$", Pattern.CASE_INSENSITIVE); - private FileFilter archives = new ExtensionFileFilter(getArchiveTypes()); + private final Pattern volume = Pattern.compile("[.]r[0-9]+$|[.]part[0-9]+|[.][0-9]+$", Pattern.CASE_INSENSITIVE); + private final FileFilter archives = new ExtensionFileFilter(getArchiveTypes()); @Override public boolean accept(File path) { diff --git a/source/net/filebot/archive/ArchiveExtractor.java b/source/net/filebot/archive/ArchiveExtractor.java new file mode 100644 index 00000000..0263ccc8 --- /dev/null +++ b/source/net/filebot/archive/ArchiveExtractor.java @@ -0,0 +1,17 @@ +package net.filebot.archive; + +import java.io.File; +import java.io.FileFilter; +import java.util.List; + +import net.filebot.vfs.FileInfo; + +public interface ArchiveExtractor { + + public List listFiles() throws Exception; + + public void extract(File outputDir) throws Exception; + + public void extract(File outputDir, FileFilter filter) throws Exception; + +} diff --git a/source/net/filebot/archive/FileMapper.java b/source/net/filebot/archive/FileMapper.java index 4e8e0040..1658e9ab 100644 --- a/source/net/filebot/archive/FileMapper.java +++ b/source/net/filebot/archive/FileMapper.java @@ -1,40 +1,38 @@ - package net.filebot.archive; - import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; - public class FileMapper implements ExtractOutProvider { - + private File outputDir; private boolean flatten; - - + public FileMapper(File outputDir, boolean flatten) { this.outputDir = outputDir; this.flatten = flatten; }; - - + + public File getOutputDir() { + return outputDir; + } + public File getOutputFile(File entry) { return new File(outputDir, flatten ? entry.getName() : entry.getPath()); } - - + @Override public OutputStream getStream(File entry) throws IOException { File outputFile = getOutputFile(entry); File outputFolder = outputFile.getParentFile(); - + // create parent folder if necessary if (!outputFolder.isDirectory() && !outputFolder.mkdirs()) { throw new IOException("Failed to create folder: " + outputFolder); } - + return new FileOutputStream(outputFile); } } diff --git a/source/net/filebot/archive/SevenZipExecutable.java b/source/net/filebot/archive/SevenZipExecutable.java new file mode 100644 index 00000000..e14d0be9 --- /dev/null +++ b/source/net/filebot/archive/SevenZipExecutable.java @@ -0,0 +1,98 @@ +package net.filebot.archive; + +import static java.nio.charset.StandardCharsets.*; + +import java.io.File; +import java.io.FileFilter; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.ProcessBuilder.Redirect; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import net.filebot.util.ByteBufferOutputStream; +import net.filebot.vfs.FileInfo; +import net.filebot.vfs.SimpleFileInfo; + +public class SevenZipExecutable implements ArchiveExtractor { + + // e.g. 2014-09-15 05:33:10 ....A 398536 625065 folder/file.txt + final Pattern listFilesLinePattern = Pattern.compile("^\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\s+\\S+\\s+(?\\d+)\\s+\\d*\\s+(?.+)$", Pattern.MULTILINE); + + final File archive; + + public SevenZipExecutable(File file) throws Exception { + if (!file.exists()) { + throw new FileNotFoundException(file.getAbsolutePath()); + } + + this.archive = file.getCanonicalFile(); + } + + protected String get7zCommand() { + // use 7z executable path as specified by the cmdline or default to "7z" and let the shell figure it out + return System.getProperty("net.filebot.Archive.7z", "7z"); + } + + protected CharSequence execute(String... command) throws IOException { + Process process = new ProcessBuilder(command).redirectError(Redirect.INHERIT).start(); + + ByteBufferOutputStream bb = new ByteBufferOutputStream(8 * 1024); + bb.transferFully(process.getInputStream()); + + try { + int returnCode = process.waitFor(); + CharSequence output = UTF_8.decode(bb.getByteBuffer()); + + // DEBUG + // System.out.println("Execute: " + Arrays.asList(command)); + // System.out.println(output); + + if (returnCode == 0) { + return output; + } else { + throw new IOException(String.format("%s failed with exit code %d: %s", get7zCommand(), returnCode, output.toString().replaceAll("\\s+", " ").trim())); + } + } catch (InterruptedException e) { + throw new IOException(String.format("%s timed out", get7zCommand()), e); + } + } + + public List listFiles() throws IOException { + List paths = new ArrayList(); + + // e.g. 7z l -y archive.7z + CharSequence output = execute(get7zCommand(), "l", "-y", archive.getPath()); + + Matcher m = listFilesLinePattern.matcher(output); + while (m.find()) { + String path = m.group("name").trim(); + long size = Long.parseLong(m.group("size")); + + // ignore folders, e.g. 2015-03-26 02:37:24 D.... 0 0 folder + if (size > 0 && path.length() > 0) { + paths.add(new SimpleFileInfo(path, size)); + } + } + + return paths; + } + + public void extract(File outputDir) throws IOException { + // e.g. 7z x -y -aos archive.7z + execute(get7zCommand(), "x", "-y", "-aos", archive.getPath(), "-o" + outputDir.getCanonicalPath()); + + } + + public void extract(File outputDir, FileFilter filter) throws IOException { + // e.g. 7z x -y -aos archive.7z file.txt image.png info.nfo + Stream command = Stream.of(get7zCommand(), "x", "-y", "-aos", archive.getPath(), "-o" + outputDir.getCanonicalPath()); + Stream selection = listFiles().stream().filter(f -> filter.accept(f.toFile())).map(f -> f.getPath()); + + execute(Stream.concat(command, selection).toArray(String[]::new)); + } + +} diff --git a/source/net/filebot/archive/SevenZipNativeBindings.java b/source/net/filebot/archive/SevenZipNativeBindings.java new file mode 100644 index 00000000..e74d7ee9 --- /dev/null +++ b/source/net/filebot/archive/SevenZipNativeBindings.java @@ -0,0 +1,136 @@ +package net.filebot.archive; + +import java.io.Closeable; +import java.io.File; +import java.io.FileFilter; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; + +import net.filebot.vfs.FileInfo; +import net.filebot.vfs.SimpleFileInfo; +import net.sf.sevenzipjbinding.ArchiveFormat; +import net.sf.sevenzipjbinding.ISevenZipInArchive; +import net.sf.sevenzipjbinding.PropID; +import net.sf.sevenzipjbinding.SevenZipException; + +public class SevenZipNativeBindings implements ArchiveExtractor, Closeable { + + private ISevenZipInArchive inArchive; + private ArchiveOpenVolumeCallback openVolume; + + public SevenZipNativeBindings(File file) throws Exception { + // initialize 7-Zip-JBinding + if (!file.exists()) { + throw new FileNotFoundException(file.getAbsolutePath()); + } + + try { + openVolume = new ArchiveOpenVolumeCallback(); + if (!Archive.hasMultiPartIndex(file)) { + // single volume archives and multi-volume rar archives + inArchive = SevenZipLoader.open(openVolume.getStream(file.getAbsolutePath()), openVolume); + } else { + // raw multi-volume archives + inArchive = SevenZipLoader.open(new VolumedArchiveInStream(file.getAbsolutePath(), openVolume), null); + } + } catch (InvocationTargetException e) { + throw (Exception) e.getTargetException(); + } + } + + public int itemCount() throws SevenZipException { + return inArchive.getNumberOfItems(); + } + + public Map getItem(int index) throws SevenZipException { + Map item = new EnumMap(PropID.class); + + for (PropID prop : PropID.values()) { + Object value = inArchive.getProperty(index, prop); + if (value != null) { + item.put(prop, value); + } + } + + return item; + } + + public List listFiles() throws SevenZipException { + List paths = new ArrayList(); + + for (int i = 0; i < inArchive.getNumberOfItems(); i++) { + boolean isFolder = (Boolean) inArchive.getProperty(i, PropID.IS_FOLDER); + if (!isFolder) { + String path = (String) inArchive.getProperty(i, PropID.PATH); + Long length = (Long) inArchive.getProperty(i, PropID.SIZE); + if (path != null) { + paths.add(new SimpleFileInfo(path, length != null ? length : -1)); + } + } + } + + return paths; + } + + @Override + public void extract(File outputDir) throws Exception { + extract(new FileMapper(outputDir, false)); + } + + @Override + public void extract(File outputDir, FileFilter filter) throws Exception { + extract(new FileMapper(outputDir, false), filter); + } + + public void extract(ExtractOutProvider outputMapper) throws SevenZipException { + inArchive.extract(null, false, new ExtractCallback(inArchive, outputMapper)); + } + + public void extract(ExtractOutProvider outputMapper, FileFilter filter) throws SevenZipException { + List selection = new ArrayList(); + + for (int i = 0; i < inArchive.getNumberOfItems(); i++) { + boolean isFolder = (Boolean) inArchive.getProperty(i, PropID.IS_FOLDER); + if (!isFolder) { + String path = (String) inArchive.getProperty(i, PropID.PATH); + if (path != null && filter.accept(new File(path))) { + selection.add(i); + } + } + } + + int[] indices = new int[selection.size()]; + for (int i = 0; i < indices.length; i++) { + indices[i] = selection.get(i); + } + inArchive.extract(indices, false, new ExtractCallback(inArchive, outputMapper)); + } + + @Override + public void close() throws IOException { + try { + inArchive.close(); + } catch (SevenZipException e) { + throw new IOException(e); + } finally { + openVolume.close(); + } + } + + public static List getArchiveTypes() { + List extensions = new ArrayList(); + + // formats provided by the library + for (ArchiveFormat it : ArchiveFormat.values()) { + extensions.add(it.getMethodName()); + } + + return extensions; + } + +} diff --git a/source/net/filebot/cli/CmdlineOperations.java b/source/net/filebot/cli/CmdlineOperations.java index 8b12eccf..5525618c 100644 --- a/source/net/filebot/cli/CmdlineOperations.java +++ b/source/net/filebot/cli/CmdlineOperations.java @@ -1127,7 +1127,7 @@ public class CmdlineOperations implements CmdlineInterface { List extractedFiles = new ArrayList(); for (File file : archiveFiles) { - Archive archive = new Archive(file); + Archive archive = Archive.open(file); try { File outputFolder = new File(output != null ? output : getName(file)); if (!outputFolder.isAbsolute()) { @@ -1169,7 +1169,7 @@ public class CmdlineOperations implements CmdlineInterface { CLILogger.finest("Extracting files " + outputMapping); // extract all files - archive.extract(outputMapper); + archive.extract(outputMapper.getOutputDir()); for (FileInfo it : outputMapping) { extractedFiles.add(it.toFile()); @@ -1178,7 +1178,7 @@ public class CmdlineOperations implements CmdlineInterface { CLILogger.finest("Extracting files " + selection); // extract files selected by the given filter - archive.extract(outputMapper, new FileFilter() { + archive.extract(outputMapper.getOutputDir(), new FileFilter() { @Override public boolean accept(File entry) { diff --git a/source/net/filebot/media/MediaDetection.java b/source/net/filebot/media/MediaDetection.java index 5bd41556..8a5dda55 100644 --- a/source/net/filebot/media/MediaDetection.java +++ b/source/net/filebot/media/MediaDetection.java @@ -100,7 +100,7 @@ public class MediaDetection { public static boolean isVideoDiskFile(File file) throws Exception { FileFilter diskFolderEntryFilter = releaseInfo.getDiskFolderEntryFilter(); - Archive iso = new Archive(file); + Archive iso = Archive.open(file); try { for (FileInfo it : iso.listFiles()) { for (File entry : listPath(it.toFile())) { diff --git a/source/net/filebot/ui/analyze/ExtractTool.java b/source/net/filebot/ui/analyze/ExtractTool.java index c0f15341..eb5ab649 100644 --- a/source/net/filebot/ui/analyze/ExtractTool.java +++ b/source/net/filebot/ui/analyze/ExtractTool.java @@ -82,7 +82,7 @@ class ExtractTool extends Tool { for (File file : files) { // ignore non-archives files and trailing multi-volume parts if (Archive.VOLUME_ONE_FILTER.accept(file)) { - Archive archive = new Archive(file); + Archive archive = Archive.open(file); try { for (FileInfo it : archive.listFiles()) { entries.add(new ArchiveEntry(file, it)); @@ -247,7 +247,7 @@ class ExtractTool extends Tool { // update progress dialog firePropertyChange("currentFile", null, file); - Archive archive = new Archive(file); + Archive archive = Archive.open(file); try { final FileMapper outputMapper = new FileMapper(outputFolder, false); @@ -281,10 +281,10 @@ class ExtractTool extends Tool { if (!skip || conflictAction == ConflictAction.OVERRIDE) { if (filter == null || forceExtractAll) { // extract all files - archive.extract(outputMapper); + archive.extract(outputMapper.getOutputDir()); } else { // extract files selected by the given filter - archive.extract(outputMapper, new FileFilter() { + archive.extract(outputMapper.getOutputDir(), new FileFilter() { @Override public boolean accept(File entry) {