filebot/source/net/filebot/util/FileUtilities.java

819 lines
23 KiB
Java

package net.filebot.util;
import static java.nio.charset.StandardCharsets.*;
import static java.util.Arrays.*;
import static java.util.Collections.*;
import static java.util.Comparator.*;
import static net.filebot.Logging.*;
import static net.filebot.util.RegularExpressions.*;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import org.apache.commons.io.FileUtils;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
public final class FileUtilities {
public static File moveRename(File source, File destination) throws IOException {
// resolve destination
destination = resolveDestination(source, destination);
// do nothing if source and destination path is the same
if (equalsCaseSensitive(source, destination)) {
return destination;
}
if (source.isDirectory()) {
// move folder
FileUtils.moveDirectory(source, destination);
return destination;
}
// on Windows, use ATOMIC_MOVE which allows us to rename files even if only lower/upper-case changes (without ATOMIC_MOVE the operation would be ignored)
// but ATOMIC_MOVE can only work for files on the same drive, if that is not the case there is no point trying move with ATOMIC_MOVE
if (source.equals(destination)) {
try {
return Files.move(source.toPath(), destination.toPath(), StandardCopyOption.ATOMIC_MOVE).toFile();
} catch (AtomicMoveNotSupportedException e) {
debug.warning(e::toString);
}
}
// Linux and Mac OS X
return Files.move(source.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING).toFile();
}
public static File copyAs(File source, File destination) throws IOException {
// resolve destination
destination = resolveDestination(source, destination);
if (source.isDirectory()) {
// copy folder
FileUtils.copyDirectory(source, destination);
return destination;
}
// copy file
return Files.copy(source.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING).toFile();
}
public static File resolve(File source, File destination) {
// resolve destination
if (!destination.isAbsolute()) {
// same folder, different name
destination = new File(source.getParentFile(), destination.getPath());
}
return destination;
}
public static File resolveDestination(File source, File destination) throws IOException {
// resolve destination
destination = resolve(source, destination);
// create parent folder if necessary and make sure that the folder structure is created, and throw exception if the folder structure can't be created
Files.createDirectories(destination.getParentFile().toPath());
return destination;
}
public static File createRelativeSymlink(File link, File target, boolean relativize) throws IOException {
if (relativize) {
// make sure we're working with the full path
link = link.getCanonicalFile();
target = target.getCanonicalFile();
try {
target = link.getParentFile().toPath().relativize(target.toPath()).toFile();
} catch (Throwable e) {
// unable to relativize link target
}
}
// create symlink via NIO.2
return Files.createSymbolicLink(link.toPath(), target.toPath()).toFile();
}
public static File createHardLinkStructure(File link, File target) throws IOException {
if (target.isFile()) {
return Files.createLink(link.toPath(), target.toPath()).toFile();
}
// if the target is a directory, recreate the structure and hardlink each file item
final Path source = target.getCanonicalFile().toPath();
final Path destination = link.getCanonicalFile().toPath();
Files.walkFileTree(source, EnumSet.of(FileVisitOption.FOLLOW_LINKS), FILE_WALK_MAX_DEPTH, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Path linkFile = destination.resolve(source.relativize(file));
Files.createDirectories(linkFile.getParent());
Files.createLink(linkFile, file);
return FileVisitResult.CONTINUE;
}
});
return destination.toFile();
}
public static void delete(File file) throws IOException {
if (file.isDirectory()) {
Files.walkFileTree(file.toPath(), new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path f, BasicFileAttributes attr) throws IOException {
Files.delete(f);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path f, IOException e) throws IOException {
Files.delete(f);
return FileVisitResult.CONTINUE;
}
});
} else {
Files.delete(file.toPath());
}
}
public static File createFolders(File folder) throws IOException {
return Files.createDirectories(folder.toPath()).toFile();
}
private static final String WIN_THUMBNAIL_STORE = "Thumbs.db";
private static final String MAC_THUMBNAIL_STORE = ".DS_Store";
public static boolean isThumbnailStore(File file) {
return MAC_THUMBNAIL_STORE.equals(file.getName()) || WIN_THUMBNAIL_STORE.equalsIgnoreCase(file.getName());
}
public static byte[] readFile(File file) throws IOException {
return Files.readAllBytes(file.toPath());
}
public static String readTextFile(File file) throws IOException {
byte[] bytes = readFile(file);
// check BOM
BOM bom = BOM.detect(bytes);
if (bom != null) {
return new String(bytes, bom.size(), bytes.length - bom.size(), bom.getCharset());
} else {
return new String(bytes, UTF_8);
}
}
public static List<String> readLines(File file) throws IOException {
return asList(NEWLINE.split(readTextFile(file)));
}
public static File writeFile(ByteBuffer data, File destination) throws IOException {
try (FileChannel channel = FileChannel.open(destination.toPath(), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) {
channel.write(data);
}
return destination;
}
public static Reader createTextReader(InputStream in, boolean guess, Charset declaredEncoding) throws IOException {
byte head[] = new byte[BOM.SIZE];
in.mark(head.length);
in.read(head);
in.reset(); // rewind
// check BOM
BOM bom = BOM.detect(head);
if (bom != null) {
in.skip(bom.size()); // skip BOM
return new InputStreamReader(in, bom.getCharset());
}
// auto-detect character encoding
if (guess) {
CharsetDetector detector = new CharsetDetector();
detector.setDeclaredEncoding(declaredEncoding.name());
detector.setText(in);
CharsetMatch match = detector.detect();
if (match != null) {
return match.getReader();
}
}
// default to declared encoding
return new InputStreamReader(in, declaredEncoding);
}
public static Reader createTextReader(File file) throws IOException {
return createTextReader(new BufferedInputStream(new FileInputStream(file), BUFFER_SIZE), true, UTF_8);
}
public static boolean equalsCaseSensitive(File a, File b) {
return a.getPath().equals(b.getPath());
}
/**
* Pattern used for matching file extensions.
*
* e.g. "file.txt" -> match "txt", ".hidden" -> no match
*/
public static final Pattern EXTENSION = Pattern.compile("(?<=.[.])\\p{Alnum}+$");
public static final String UNC_PREFIX = "\\\\";
public static String getExtension(File file) {
if (file.isDirectory())
return null;
return getExtension(file.getName());
}
public static String getExtension(String name) {
Matcher matcher = EXTENSION.matcher(name);
if (matcher.find()) {
// extension without leading '.'
return matcher.group();
}
// no extension
return null;
}
public static boolean hasExtension(File file, String... extensions) {
// avoid native call for speed, if possible
return hasExtension(file.getName(), extensions) && !file.isDirectory();
}
public static boolean hasExtension(String filename, String... extensions) {
for (String it : extensions) {
if (filename.length() - it.length() >= 2 && filename.charAt(filename.length() - it.length() - 1) == '.') {
String tail = filename.substring(filename.length() - it.length(), filename.length());
if (tail.equalsIgnoreCase(it)) {
return true;
}
}
}
return false;
}
public static String getNameWithoutExtension(String name) {
Matcher matcher = EXTENSION.matcher(name);
if (matcher.find()) {
return name.substring(0, matcher.start() - 1);
}
// no extension, return given name
return name;
}
public static String getName(File file) {
if (file == null)
return null;
if (file.getName().isEmpty() || UNC_PREFIX.equals(file.getParent()))
return getFolderName(file);
return getNameWithoutExtension(file.getName());
}
public static String getFolderName(File file) {
if (UNC_PREFIX.equals(file.getParent()))
return file.toString();
if (file.getName().length() > 0)
return file.getName();
// file might be a drive (only has a path, but no name)
return replacePathSeparators(file.toString(), "");
}
public static boolean isDerived(File derivate, File prime) {
String n = getName(prime).toLowerCase();
String s = getName(derivate).toLowerCase();
if (s.startsWith(n)) {
return s.length() == n.length() || !Character.isLetterOrDigit(s.charAt(n.length())); // e.g. x.z is not considered derived from xy.z
}
return false;
}
public static boolean isDerivedByExtension(File derivate, File prime) {
return isDerivedByExtension(getName(derivate), prime);
}
public static boolean isDerivedByExtension(String derivate, File prime) {
String base = getName(prime).trim().toLowerCase();
derivate = derivate.trim().toLowerCase();
if (derivate.equals(base))
return true;
while (derivate.length() > base.length() && getExtension(derivate) != null) {
derivate = getNameWithoutExtension(derivate);
if (derivate.equals(base))
return true;
}
return false;
}
public static boolean containsOnly(Collection<File> files, FileFilter filter) {
if (files.isEmpty()) {
return false;
}
for (File file : files) {
if (!filter.accept(file))
return false;
}
return true;
}
public static List<File> sortByUniquePath(Collection<File> files) {
TreeSet<File> sortedSet = new TreeSet<File>(CASE_INSENSITIVE_PATH_ORDER); // sort by unique lower-case paths
sortedSet.addAll(files);
return new ArrayList<File>(sortedSet);
}
public static List<File> filter(Iterable<File> files, FileFilter... filters) {
List<File> accepted = new ArrayList<File>();
for (File file : files) {
for (FileFilter filter : filters) {
if (filter.accept(file)) {
accepted.add(file);
break;
}
}
}
return accepted;
}
public static FileFilter not(FileFilter filter) {
return f -> !filter.accept(f);
}
public static FileFilter filter(FileFilter... filters) {
return f -> stream(filters).anyMatch(it -> it.accept(f));
}
public static List<File> listPath(File file) {
return listPathTail(file, Integer.MAX_VALUE, false);
}
public static List<File> listPathTail(File file, int tailSize, boolean reverse) {
LinkedList<File> nodes = new LinkedList<File>();
File node = file;
for (int i = 0; node != null && i < tailSize && !UNC_PREFIX.equals(node.toString()); i++, node = node.getParentFile()) {
if (reverse) {
nodes.addLast(node);
} else {
nodes.addFirst(node);
}
}
return nodes;
}
public static File getRelativePathTail(File file, int tailSize) {
File f = null;
for (File it : listPathTail(file, tailSize, false)) {
if (it.getParentFile() != null) {
f = new File(f, it.getName());
}
}
return f;
}
public static List<File> getFileSystemRoots() {
File[] roots = File.listRoots();
// roots array may be null if folder permissions do not allow listing of files
if (roots == null) {
roots = new File[0];
}
return asList(roots);
}
public static List<File> getChildren(File folder) {
return getChildren(folder, null, null);
}
public static List<File> getChildren(File folder, FileFilter filter) {
return getChildren(folder, filter, null);
}
public static List<File> getChildren(File folder, FileFilter filter, Comparator<File> order) {
File[] files = filter == null ? folder.listFiles() : folder.listFiles(filter);
// children array may be null if folder permissions do not allow listing of files
if (files == null) {
return emptyList();
}
if (order != null) {
sort(files, order);
}
return asList(files);
}
public static final int FILE_WALK_MAX_DEPTH = 32;
public static List<File> listFiles(File folder, FileFilter filter) {
return listFiles(new File[] { folder }, FILE_WALK_MAX_DEPTH, filter, null);
}
public static List<File> listFiles(File folder, FileFilter filter, Comparator<File> order) {
return listFiles(new File[] { folder }, FILE_WALK_MAX_DEPTH, filter, order);
}
public static List<File> listFiles(Collection<File> folders, FileFilter filter, Comparator<File> order) {
return listFiles(folders.toArray(new File[0]), FILE_WALK_MAX_DEPTH, filter, order);
}
public static List<File> listFiles(File[] files, int depth, FileFilter filter, Comparator<File> order) {
List<File> sink = new ArrayList<File>();
// traverse file tree recursively
streamFiles(files, FOLDERS, order).forEach(f -> listFiles(f, sink, depth, filter, order));
// add selected files in preferred order
streamFiles(files, filter, order).forEach(sink::add);
return sink;
}
private static void listFiles(File folder, List<File> sink, int depth, FileFilter filter, Comparator<File> order) {
if (depth < 0) {
return;
}
// children array may be null if folder permissions do not allow listing of files
File[] files = folder.listFiles(NOT_HIDDEN);
// traverse file tree recursively
streamFiles(files, FOLDERS, order).forEach(f -> listFiles(f, sink, depth - 1, filter, order));
// add selected files in preferred order
streamFiles(files, filter, order).forEach(sink::add);
}
private static Stream<File> streamFiles(File[] files, FileFilter filter, Comparator<File> order) {
if (files == null || files.length == 0) {
return Stream.empty();
}
if (order == null) {
return stream(files).filter(filter::accept);
} else {
return stream(files).filter(filter::accept).sorted(order);
}
}
public static SortedMap<File, List<File>> mapByFolder(Iterable<File> files) {
SortedMap<File, List<File>> map = new TreeMap<File, List<File>>();
for (File file : files) {
File key = file.getParentFile();
if (key == null) {
throw new IllegalArgumentException("Parent is null: " + file);
}
List<File> valueList = map.get(key);
if (valueList == null) {
valueList = new ArrayList<File>();
map.put(key, valueList);
}
valueList.add(file);
}
return map;
}
public static Map<String, List<File>> mapByExtension(Iterable<File> files) {
Map<String, List<File>> map = new HashMap<String, List<File>>();
for (File file : files) {
String key = getExtension(file);
if (key != null) {
key = key.toLowerCase();
}
List<File> valueList = map.get(key);
if (valueList == null) {
valueList = new ArrayList<File>();
map.put(key, valueList);
}
valueList.add(file);
}
return map;
}
/**
* Invalid file name characters: \, /, :, *, ?, ", <, >, |, \r, \n and excessive characters
*/
public static final Pattern ILLEGAL_CHARACTERS = Pattern.compile("[\\\\/:*?\"<>|\\r\\n]|\\p{Cntrl}|\\s+$|(?<=[^.])[.]+$|(?<=.{250})(.+)(?=[.]\\p{Alnum}{3}$)");
/**
* Strip file name of invalid characters
*
* @param filename
* original filename
* @return valid file name stripped of invalid characters
*/
public static String validateFileName(CharSequence filename) {
// strip invalid characters from file name
return SPACE.matcher(ILLEGAL_CHARACTERS.matcher(filename).replaceAll("")).replaceAll(" ").trim();
}
public static boolean isInvalidFileName(CharSequence filename) {
// check if file name contains any illegal characters
return ILLEGAL_CHARACTERS.matcher(filename).find();
}
public static File validateFileName(File file) {
// windows drives (e.g. c:, d:, etc.) are never invalid because name will be an empty string
if (!isInvalidFileName(file.getName()))
return file;
// validate file name only
return new File(file.getParentFile(), validateFileName(file.getName()));
}
public static File validateFilePath(File path) {
Iterator<File> nodes = listPath(path).iterator();
// initialize with root node, keep original root object if possible (so we don't loose the drive on windows)
File validatedPath = validateFileName(nodes.next());
// validate the rest of the path
while (nodes.hasNext()) {
validatedPath = new File(validatedPath, validateFileName(nodes.next().getName()));
}
return validatedPath;
}
public static boolean isInvalidFilePath(File path) {
// check if file name contains any illegal characters
for (File node = path; node != null; node = node.getParentFile()) {
if (isInvalidFileName(node.getName()))
return true;
}
return false;
}
public static String normalizePathSeparators(String path) {
// special handling for UNC paths
if (path.startsWith(UNC_PREFIX) && path.length() > 2) {
return UNC_PREFIX + path.substring(2).replace('\\', '/');
}
return path.replace('\\', '/');
}
public static String replacePathSeparators(CharSequence path) {
return replacePathSeparators(path, " ");
}
public static String replacePathSeparators(CharSequence path, String replacement) {
return SLASH.matcher(path).replaceAll(replacement);
}
public static String md5(String string) {
return md5(StandardCharsets.UTF_8.encode(string));
}
public static String md5(byte[] data) {
return md5(ByteBuffer.wrap(data));
}
public static String md5(ByteBuffer data) {
try {
MessageDigest hash = MessageDigest.getInstance("MD5");
hash.update(data);
return String.format("%032x", new BigInteger(1, hash.digest())); // as hex string
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
}
public static List<File> asFileList(Object... paths) {
List<File> files = new ArrayList<File>(paths.length);
for (Object it : paths) {
if (it instanceof CharSequence) {
files.add(new File(it.toString()));
} else if (it instanceof File) {
files.add((File) it);
} else if (it instanceof Path) {
files.add(((Path) it).toFile());
} else if (it instanceof Collection<?>) {
files.addAll(asFileList(((Collection<?>) it).toArray())); // flatten object structure
}
}
return files;
}
public static final int BUFFER_SIZE = 64 * 1024;
public static final long ONE_KILOBYTE = 1000;
public static final long ONE_MEGABYTE = 1000 * ONE_KILOBYTE;
public static final long ONE_GIGABYTE = 1000 * ONE_MEGABYTE;
public static String formatSize(long size) {
if (size >= 100 * ONE_GIGABYTE)
return String.format("%,d GB", size / ONE_GIGABYTE);
if (size >= 10 * ONE_GIGABYTE)
return String.format("%.1f GB", (double) size / ONE_GIGABYTE);
if (size >= ONE_GIGABYTE)
return String.format("%.2f GB", (double) size / ONE_GIGABYTE);
if (size >= 10 * ONE_MEGABYTE)
return String.format("%,d MB", size / ONE_MEGABYTE);
if (size >= ONE_MEGABYTE)
return String.format("%.1f MB", (double) size / ONE_MEGABYTE);
if (size >= ONE_KILOBYTE)
return String.format("%,d KB", size / ONE_KILOBYTE);
return String.format("%,d bytes", size);
}
public static final FileFilter FOLDERS = File::isDirectory;
public static final FileFilter FILES = File::isFile;
public static final FileFilter NOT_HIDDEN = not(File::isHidden);
public static final FileFilter TEMPORARY = new FileFilter() {
private final String tmpdir = System.getProperty("java.io.tmpdir");
@Override
public boolean accept(File file) {
return file.getPath().startsWith(tmpdir);
}
};
public static class ParentFilter implements FileFilter {
private final File folder;
public ParentFilter(File folder) {
this.folder = folder;
}
@Override
public boolean accept(File file) {
return listPath(file).contains(folder);
}
}
public static class ExtensionFileFilter implements FileFilter, FilenameFilter {
public static final List<String> WILDCARD = singletonList("*");
private final String[] extensions;
public ExtensionFileFilter(String... extensions) {
this.extensions = extensions.clone();
}
public ExtensionFileFilter(Collection<String> extensions) {
this.extensions = extensions.toArray(new String[0]);
}
@Override
public boolean accept(File dir, String name) {
return accept(name);
}
@Override
public boolean accept(File file) {
return accept(file.getName());
}
public boolean accept(String name) {
return acceptAny() || hasExtension(name, extensions);
}
public boolean acceptAny() {
return extensions.length == 1 && WILDCARD.get(0).equals(extensions[0]);
}
public boolean acceptExtension(String extension) {
if (acceptAny()) {
return true;
}
for (String other : extensions) {
if (other.equalsIgnoreCase(extension)) {
return true;
}
}
return false;
}
public String extension() {
return extensions[0];
}
public String[] extensions() {
return extensions.clone();
}
@Override
public String toString() {
StringBuilder s = new StringBuilder();
for (String it : extensions) {
if (s.length() > 0) {
s.append(", ");
}
s.append("*.").append(it);
}
return s.toString();
}
}
public static class RegexFileFilter implements FileFilter, FilenameFilter {
private final Pattern pattern;
public RegexFileFilter(Pattern pattern) {
this.pattern = pattern;
}
@Override
public boolean accept(File dir, String name) {
return pattern.matcher(name).matches();
}
@Override
public boolean accept(File file) {
return accept(file.getParentFile(), file.getName());
}
}
public static final Comparator<File> CASE_INSENSITIVE_PATH_ORDER = comparing(File::getPath, String.CASE_INSENSITIVE_ORDER);
public static final Comparator<File> HUMAN_NAME_ORDER = comparing(File::getName, new AlphanumComparator(Locale.ENGLISH));
/**
* Dummy constructor to prevent instantiation.
*/
private FileUtilities() {
throw new UnsupportedOperationException();
}
}