diff --git a/java/core/src/core/connector/ConnectorException.java b/java/core/src/core/connector/ConnectorException.java new file mode 100644 index 0000000..14bd28d --- /dev/null +++ b/java/core/src/core/connector/ConnectorException.java @@ -0,0 +1,19 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector; + +public class ConnectorException extends Exception +{ + public ConnectorException (Exception e) + { + super(e); + } + + public ConnectorException (String message) + { + super (message); + } +} diff --git a/java/core/src/core/connector/FileInfo.java b/java/core/src/core/connector/FileInfo.java new file mode 100644 index 0000000..fcbacb4 --- /dev/null +++ b/java/core/src/core/connector/FileInfo.java @@ -0,0 +1,53 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector; + +import java.util.Comparator; +import java.util.Date; + +public class FileInfo +{ + static public class SortByDateAscending implements Comparator + { + @Override + public int compare(FileInfo o1, FileInfo o2) + { + long time = o1.date.getTime() - o2.date.getTime(); + return time == 0 ? 0 : (time > 0 ? 1 : -1); + } + }; + + enum Type + { + Directory + } + + public String path; + public String relativePath; + public long size; + public Type type; + public Date date; + public String version; + public Object user; + + public FileInfo(String path, String relativePath, long size, Date date, String version) + { + this.path = path; + this.relativePath = relativePath; + this.size = size; + this.date = date; + this.version = version; + } + + public String getFileName() + { + int lastSlash = path.lastIndexOf('/'); + if (lastSlash == -1) + return path; + + return path.substring(lastSlash+1); + } +} \ No newline at end of file diff --git a/java/core/src/core/connector/async/AsyncStoreConnector.java b/java/core/src/core/connector/async/AsyncStoreConnector.java new file mode 100644 index 0000000..784132e --- /dev/null +++ b/java/core/src/core/connector/async/AsyncStoreConnector.java @@ -0,0 +1,26 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.async; + +import java.util.Date; + +import core.callback.Callback; + +public interface AsyncStoreConnector +{ + Callback list_ (String path); + Callback createDirectory_ (String path); + Callback ensureDirectories_ (String[] directories); + + Callback put_ (String path, byte[] bytes); + Callback get_ (String path); + + Callback put_ (String path); + Callback get_ (); + + Callback move_ (String from, String to); + Callback delete_ (String path); +} diff --git a/java/core/src/core/connector/async/AsyncStoreConnectorAdapter.java b/java/core/src/core/connector/async/AsyncStoreConnectorAdapter.java new file mode 100644 index 0000000..7530afa --- /dev/null +++ b/java/core/src/core/connector/async/AsyncStoreConnectorAdapter.java @@ -0,0 +1,84 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.async; + +import core.callback.Callback; +import core.callback.CallbackDefault; +import core.util.Base64; +import core.util.Zip; + +public abstract class AsyncStoreConnectorAdapter implements AsyncStoreConnector +{ + AsyncStoreConnector connector; + + AsyncStoreConnectorAdapter (AsyncStoreConnector connector) + { + this.connector = connector; + } + + @Override + public Callback list_(String path) + { + return connector.list_(path); + } + + @Override + public Callback createDirectory_(String path) + { + return connector.createDirectory_(path); + } + + @Override + public Callback ensureDirectories_(String[] directories) + { + return connector.ensureDirectories_(directories); + } + + public Callback get_(String path) + { + return + new CallbackDefault(path) { + public void onSuccess(Object... arguments) throws Exception { + String path = V(0); + get_().setReturn(callback).invoke((String)path); + } + }; + + } + public Callback get_() + { + return connector.get_(); + } + + public Callback put_(String path, byte[] bytes) + { + return + new CallbackDefault(path, bytes) { + public void onSuccess(Object... arguments) throws Exception { + String path = V(0); + byte[] bytes = V(1); + put_(path).setReturn(callback).invoke(bytes); + } + }; + } + + public Callback put_(String path) + { + return connector.put_(path); + } + + @Override + public Callback move_(String from, String to) + { + return connector.move_(from, to); + } + + @Override + public Callback delete_(String path) + { + return connector.delete_(path); + } +} diff --git a/java/core/src/core/connector/async/AsyncStoreConnectorBase64.java b/java/core/src/core/connector/async/AsyncStoreConnectorBase64.java new file mode 100644 index 0000000..123a8d5 --- /dev/null +++ b/java/core/src/core/connector/async/AsyncStoreConnectorBase64.java @@ -0,0 +1,36 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.async; + +import core.callback.Callback; +import core.callback.CallbackDefault; +import core.callback.CallbackWithVariables; +import core.callbacks.SaveArguments; +import core.util.Base64; + + +public class AsyncStoreConnectorBase64 extends AsyncStoreConnectorAdapter +{ + public AsyncStoreConnectorBase64(AsyncStoreConnector connector) + { + super(connector); + } + + public Callback get_() + { + SaveArguments save = new SaveArguments(); + + return super.get_() + .addCallback(save) + .addCallback(Base64.decodeBytes_()) + .addCallback(save.restore_(0,101)); + } + + public Callback put_(String path) + { + return Base64.encodeBytes_().addCallback(super.put_(path)); + } +} diff --git a/java/core/src/core/connector/async/AsyncStoreConnectorEncrypted.java b/java/core/src/core/connector/async/AsyncStoreConnectorEncrypted.java new file mode 100644 index 0000000..503777e --- /dev/null +++ b/java/core/src/core/connector/async/AsyncStoreConnectorEncrypted.java @@ -0,0 +1,41 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.async; + +import core.callback.Callback; +import core.callback.CallbackDefault; +import core.callbacks.SaveArguments; +import core.crypt.Cryptor; +import core.util.Base64; +import core.util.Zip; + +public class AsyncStoreConnectorEncrypted extends AsyncStoreConnectorAdapter +{ + Cryptor cryptor; + + public AsyncStoreConnectorEncrypted(Cryptor cryptor, AsyncStoreConnector connector) + { + super(connector); + this.cryptor = cryptor; + } + + public Callback get_() + { + SaveArguments save = new SaveArguments(); + + return super.get_() + .addCallback(save) + .addCallback(cryptor.decrypt_()) + .addCallback(Zip.inflate_()) + .addCallback(save.restore_(0,101)); + + } + + public Callback put_(String path) + { + return Zip.deflate_().addCallback(cryptor.encrypt_()).addCallback(super.put_(path)); + } +} diff --git a/java/core/src/core/connector/async/AsyncStoreConnectorHelper.java b/java/core/src/core/connector/async/AsyncStoreConnectorHelper.java new file mode 100644 index 0000000..b48b8b7 --- /dev/null +++ b/java/core/src/core/connector/async/AsyncStoreConnectorHelper.java @@ -0,0 +1,136 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.async; + +import java.util.Date; +import java.util.List; + +import core.callback.Callback; +import core.callback.CallbackChain; +import core.callback.CallbackDefault; +import core.callback.CallbackEmpty; +import core.callback.CallbackWithVariables; +import core.connector.FileInfo; + +public abstract class AsyncStoreConnectorHelper implements AsyncStoreConnector +{ + abstract public void list(String path, Callback callback); + abstract public void createDirectory(String path, Callback callback); + abstract public void get(String path, Callback callback); + abstract public void put(String path, byte[] bytes, Callback callback); + abstract public void delete(String path, Callback callback); +// abstract public void move(String from, String to, Callback callback); + + public void ensureDirectories(String[] folders, Callback callback) + { + CallbackChain chain = new CallbackChain(); + + for (String path : folders) + { + chain.addCallback(new CallbackWithVariables(path) { + + @Override + public void invoke(Object... arguments) + { + String path = V(0); + createDirectory(path, callback); + } + + }); + } + chain.setReturn(callback); + + chain.invoke(); + } + + @Override + public Callback createDirectory_(String path) + { + return new CallbackDefault(path) { + public void onSuccess(Object... arguments) throws Exception { + createDirectory((String)V(0), callback); + } + }; + } + + @Override + public Callback list_(String path) + { + return new CallbackDefault(path) { + public void onSuccess(Object... arguments) throws Exception { + list((String)V(0), callback); + } + }; + } + + @Override + public Callback get_() + { + return new CallbackDefault() { + public void onSuccess(Object... arguments) throws Exception { + get((String)arguments[0], callback); + } + }; + } + + public Callback get_(String path) + { + return + new CallbackDefault(path) { + public void onSuccess(Object... arguments) throws Exception { + get((String)V(0), callback); + } + }; + } + + @Override + public Callback put_(String path) + { + return new CallbackDefault(path) { + public void onSuccess(Object... arguments) throws Exception { + put((String)V(0), (byte[])arguments[0], callback); + } + }; + } + + public Callback put_(String path, byte[] bytes) + { + return + new CallbackDefault(path, bytes) { + public void onSuccess(Object... arguments) throws Exception { + put((String)V(0), (byte[])V(1), callback); + } + }; + } + + @Override + public Callback move_(String from, String to) + { + return new CallbackEmpty(); + } + + @Override + public Callback delete_(String path) { + return + new CallbackDefault(path) { + public void onSuccess(Object... arguments) throws Exception { + delete((String)V(0), callback); + } + }; + } + + @Override + public Callback ensureDirectories_(String[] folders) + { + return new CallbackDefault(new Object[] { folders }) { + + @Override + public void onSuccess(Object... arguments) throws Exception { + ensureDirectories((String[])V(0), callback); + } + }; + } +} diff --git a/java/core/src/core/connector/async/Lock.java b/java/core/src/core/connector/async/Lock.java new file mode 100644 index 0000000..2aef8c1 --- /dev/null +++ b/java/core/src/core/connector/async/Lock.java @@ -0,0 +1,244 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.async; + +import java.util.Date; +import java.util.List; + +import core.util.SecureRandom; + +import core.callback.Callback; +import core.callback.CallbackDefault; +import core.connector.FileInfo; +import core.util.LogNull; +import core.util.LogOut; + +public class Lock +{ + static LogNull log = new LogNull(Lock.class); + static SecureRandom random = new SecureRandom(); + + public AsyncStoreConnector connector; + String path; + int intervalSeconds; + int remainingBeforeRelockSecond; + Date expiration; + String version; + + public Lock(AsyncStoreConnector connector, String path, int intervalSeconds, int remainingBeforeRelockSecond) + { + this.connector = connector; + this.path = path; + this.intervalSeconds = intervalSeconds; + this.remainingBeforeRelockSecond = remainingBeforeRelockSecond; + } + + protected void reset () + { + version = null; + expiration = null; + } + + protected Date getExpirationFor (Date timeLocked) + { + return new Date(timeLocked.getTime() + intervalSeconds * 1000); + } + + protected boolean hasExpired (Date expiration) + { + Date now = new Date(); + return now.after(expiration); + } + + protected boolean closeToExpiration (Date expiration) + { + return getRemainingTimeInSeconds(expiration) < 1; + } + + protected long getRemainingTimeInSeconds (Date expiration) + { + Date now = new Date(); + return (expiration.getTime() - now.getTime())/1000; + } + + public Callback lock_() + { + return + connector.list_(path) + .addCallback(lockOnInfo_()) + .addCallback(possiblyLockIfNecessary_()); + } + + public Callback relock_() + { + return new CallbackDefault() + { + public void onSuccess(Object... arguments) throws Exception + { + if (expiration == null || closeToExpiration(expiration)) + { + log.debug(this, "lock close to expired or expired, going to fully lock"); + call(lock_()); + } + else + { + log.debug(this, "lock still active, relock only if necesary."); + call(possiblyLockIfNecessary_()); + } + } + } ; + } + + protected Callback possiblyLockIfNecessary_ () + { + return new CallbackDefault() { + public void onSuccess(Object... arguments) throws Exception + { + long remainingTime = 0; + + if (expiration != null) + { + remainingTime = getRemainingTimeInSeconds(expiration); + } + + if (remainingTime < remainingBeforeRelockSecond) + { + log.debug(this, "remainingTime",remainingTime,"<", remainingBeforeRelockSecond, "LOCKING!"); + + byte[] bytes = new byte[8]; + random.nextBytes(bytes); + + call( + connector.put_(path, bytes) + .addCallback(storeLock_()) + ); + } + else + { + log.debug(this, "remainingTime",remainingTime,">=", remainingBeforeRelockSecond); + next(arguments); + } + } + }; + } + + public Callback lockOnInfo_ () + { + return new CallbackDefault () { + @Override + public void onSuccess(Object... arguments) throws Exception { + List fileInfo = (List) arguments[0]; + + boolean locked = false; + if (!fileInfo.isEmpty()) + { + FileInfo info = fileInfo.get(0); + Date lockExpiration = getExpirationFor(info.date); + + // it's not our lock + if (!info.version.equals(version)) + { + locked = !hasExpired(lockExpiration); + reset(); + + log.debug(this, "file.date", info.date,"lockExpiration",lockExpiration,"locked",locked); + } + else + { + log.debug(this, "we have the lock!! setting expiration..", lockExpiration); + expiration = lockExpiration; + } + } + + if (locked) + throw new Exception("Someone else has the lock"); + + next(arguments); + } + }; + } + + public Callback storeLock_ () + { + return new CallbackDefault() { + public void onSuccess(Object... arguments) throws Exception { + expiration = getExpirationFor(new Date()); + version = (String) arguments[0]; + + next(arguments); + } + }; + } + + public void testLock (List fileInfo) throws Exception + { + boolean lockFound = false; + + for (FileInfo i : fileInfo) + { + log.trace("testLock", i.path, path); + if (i.path.equals(path)) + { + lockFound = true; + + if (i.version != version) + { + throw new Exception("Lock was not obtained"); + } + else + { + log.debug(this, "test lock found the lock, setting the expiration time to the file system.", i.date); + expiration = getExpirationFor(i.date); + } + + if (hasExpired(expiration)) + throw new Exception("Lock has already expired"); + } + } + + if (!lockFound) + throw new Exception("Lock not found."); + } + + public Callback testLock_ () + { + return new CallbackDefault () { + @Override + public void onSuccess(Object... arguments) throws Exception { + List fileInfo = (List) arguments[0]; + testLock(fileInfo); + next(arguments); + } + }; + } + + public Callback unlock_() + { + return new CallbackDefault () + { + public void onSuccess(Object... arguments) throws Exception + { + // never unlock, just let them expire + + /* + if (hasExpired(expiration)) + { + next(new Exception("Lock has already expired")); + } + else + { + connector.delete_(path).setReturn(callback).invoke(); + reset(); + } + */ + + next(arguments); + } + }; + } + + +}; diff --git a/java/core/src/core/connector/dropbox/ClientInfoDropbox.java b/java/core/src/core/connector/dropbox/ClientInfoDropbox.java new file mode 100644 index 0000000..47122d4 --- /dev/null +++ b/java/core/src/core/connector/dropbox/ClientInfoDropbox.java @@ -0,0 +1,53 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.dropbox; + +import core.constants.ConstantsDropbox; +import core.util.Environment; + + +public class ClientInfoDropbox +{ + private String userPrefix; + private String appKey; + private String appSecret; + private String tokenKey; + private String tokenSecret; + + public ClientInfoDropbox (Environment e) + { + userPrefix = e.get(ConstantsDropbox.DropboxUserPrefix); + appKey = e.checkGet(ConstantsDropbox.DropboxAppKey); + appSecret = e.checkGet(ConstantsDropbox.DropboxAppSecret); + tokenKey = e.checkGet(ConstantsDropbox.DropboxTokenKey); + tokenSecret = e.checkGet(ConstantsDropbox.DropboxTokenSecret); + } + + public String getUserPrefix () + { + return userPrefix; + } + + public String getAppKey () + { + return appKey; + } + + public String getAppSecret () + { + return appSecret; + } + + public String getTokenKey () + { + return tokenKey; + } + + public String getTokenSecret () + { + return tokenSecret; + } +} diff --git a/java/core/src/core/connector/dropbox/async/ConnectorDropbox.java b/java/core/src/core/connector/dropbox/async/ConnectorDropbox.java new file mode 100644 index 0000000..19811e9 --- /dev/null +++ b/java/core/src/core/connector/dropbox/async/ConnectorDropbox.java @@ -0,0 +1,375 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.dropbox.async; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import org.json.JSONArray; +import org.json.JSONObject; + +import core.callback.Callback; +import core.callback.CallbackDefault; +import core.callback.CallbackWithVariables; +import core.connector.FileInfo; +import core.connector.async.AsyncStoreConnectorHelper; +import core.connector.dropbox.ClientInfoDropbox; +import core.util.DateFormat; +import core.util.FastRandom; +import core.util.HttpDelegate; +import core.util.LogNull; + +public class ConnectorDropbox extends AsyncStoreConnectorHelper +{ + static LogNull log = new LogNull(ConnectorDropbox.class); + + ClientInfoDropbox info; + HttpDelegate httpDelegate; + FastRandom fastRandom; + + public ConnectorDropbox(ClientInfoDropbox clientInfo, HttpDelegate httpDelegate) + { + this.info = clientInfo; + this.httpDelegate = httpDelegate; + fastRandom = new FastRandom(); + } + + protected String getGlobalPath (String path) + { + if (info.getUserPrefix() != null) + return info.getUserPrefix() + "/" + path; + + return path; + } + + public void listDirectoryFinished (String containingPath, Callback callback, Object... arguments) + { + log.debug("listDirectoryFinished"); + try + { + if (arguments[0] instanceof Exception) + throw (Exception)arguments[0]; + + String result = (String)arguments[0]; + List fileInfos = new ArrayList(); + + JSONObject o = new JSONObject(result); + JSONArray contents = (JSONArray) o.get("contents"); + + DateFormat dateTimeFormat = new DateFormat("EEE, d MMM yyyy HH:mm:ss Z"); + + for (int i=0; i fileInfos = new ArrayList(); + + JSONArray contents = new JSONArray(result); + DateFormat dateTimeFormat = new DateFormat("EEE, d MMM yyyy HH:mm:ss Z"); + + for (int i=0; i db; + + public DropboxConnector (ClientInfoDropbox clientInfo) + { + this.clientInfo = clientInfo; + } + + public DropboxAPI createConnection (ClientInfoDropbox info) + { + AppKeyPair appKeyPair = new AppKeyPair(info.getAppKey(),info.getAppSecret()); + AccessTokenPair userTokenKeyPair = new AccessTokenPair(info.getTokenKey(), info.getTokenSecret()); + + WebAuthSession sourceSession = + new WebAuthSession(appKeyPair, Session.AccessType.APP_FOLDER, userTokenKeyPair); + + DropboxAPI sourceClient = new DropboxAPI(sourceSession); + + return sourceClient; + } + + + public void open () throws ConnectorException + { + try + { + db = createConnection(clientInfo); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + public void close () + { + db = null; + } + + protected String getGlobalPath (String path) + { + if (clientInfo.getUserPrefix() != null) + return "/" + clientInfo.getUserPrefix() + "/" + path; + + return "/" + path; + } + + protected String getUserPath (String path) + { + if (clientInfo.getUserPrefix() != null) + return path.substring(2 + clientInfo.getUserPrefix().length()); + + return path.substring(1); + } + + @Override + public List listDirectory(String path) throws ConnectorException + { + try + { + if (!path.endsWith("/")) + path = path + "/"; + + Entry directory = db.metadata(getGlobalPath(path), 10000, null, true, null); + List listing = new ArrayList(directory.contents.size()); + + SimpleDateFormat dateTimeFormat = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z"); + for (Entry file : directory.contents) + { + if (file.isDeleted) + continue; + + String fullPath = getUserPath(file.path); + String relativePath = fullPath.substring(path.length()); + + listing.add( + new FileInfo( + fullPath, + relativePath, + file.bytes, dateTimeFormat.parse(file.modified), + file.rev + ) + ); + } + + Collections.sort(listing, new FileInfo.SortByDateAscending()); + + return listing; + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public void createDirectory(String path) throws ConnectorException + { + try + { + db.createFolder(getGlobalPath(path)); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public byte[] get(String path) throws ConnectorException + { + try + { + Entry meta = db.metadata(getGlobalPath(path), 1, null, true, null); + if (meta.isDeleted) + throw new ConnectorException("File deleted"); + + return Streams.readFullyBytes(db.getFileStream(getGlobalPath(path), null)); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public byte[] get(String path, long size) throws ConnectorException + { + return get(path); + } + + @Override + public void put(String path, byte[] contents) throws ConnectorException + { + try + { + db.putFileOverwrite(getGlobalPath(path), new ByteArrayInputStream(contents), contents.length, null); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public void move(String from, String to) throws ConnectorException + { + try + { + db.move(getGlobalPath(from), getGlobalPath(to)); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public void delete(String path) throws ConnectorException + { + try + { + db.delete(getGlobalPath(path)); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + public boolean ensureDirectories (String ... folders) + { + for (String folder : folders) + { + String[] parts = folder.split("/"); + + String path = ""; + for (String part : parts) + { + if (!path.isEmpty()) + path += "/"; + + path += part; + String fullPath = getGlobalPath(path); + try + { + db.createFolder(fullPath); + } + catch (Exception e) + { + System.out.format("Folder[%s] already exists.\n", fullPath); + } + } + } + + return true; + } +} diff --git a/java/core/src/core/connector/dropbox/sync/DropboxSignup.java b/java/core/src/core/connector/dropbox/sync/DropboxSignup.java new file mode 100644 index 0000000..aae7f42 --- /dev/null +++ b/java/core/src/core/connector/dropbox/sync/DropboxSignup.java @@ -0,0 +1,82 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.dropbox.sync; + +import java.io.IOException; +import java.net.URL; +import java.net.URLConnection; +import java.util.Date; + +import com.dropbox.client2.session.AccessTokenPair; +import com.dropbox.client2.session.AppKeyPair; + +import core.util.HttpDelegate; +import core.util.LogNull; +import core.util.LogOut; +import core.util.Pair; +import core.util.Streams; + +public class DropboxSignup +{ + static LogNull log = new LogNull (DropboxSignup.class); + + static public AccessTokenPair getDropboxRequestToken (AppKeyPair appKeyPair) throws Exception + { + log.debug("getDropboxUserToken"); + URL url = new URL( + "https://api.dropbox.com/1/oauth/request_token" + + "?oauth_consumer_key=" + appKeyPair.key + + "&oauth_signature_method=PLAINTEXT" + + "&oauth_signature=" + appKeyPair.secret + "%26" + + "&oauth_nonce=\"" + (new Date()).getTime() + "\"" + ); + URLConnection c = url.openConnection(); + String response = Streams.readFullyString(c.getInputStream(), "UTF-8"); + + Pair token = parseAuthToken(response); + return new AccessTokenPair(token.first, token.second); + } + + static public AccessTokenPair getDropboxAccessToken (AppKeyPair appKeyPair, AccessTokenPair accessToken) throws Exception + { + URL url = new URL( + "https://api.dropbox.com/1/oauth/access_token" + + "?oauth_consumer_key=" + appKeyPair.key + + "&oauth_token=" + accessToken.key + "&" + + "&oauth_signature_method=PLAINTEXT" + + "&oauth_signature=" + appKeyPair.secret + "%26" + accessToken.secret + + "&oauth_nonce=\"" + (new Date()).getTime() + "\"" + ); + URLConnection c = url.openConnection(); + String response = Streams.readFullyString(c.getInputStream(), "UTF-8"); + + Pair token = parseAuthToken(response); + return new AccessTokenPair(token.first, token.second); + } + + static Pair parseAuthToken (String response) throws Exception + { + String userKey=null, userSecret=null; + String[] parts = response.split("&"); + for (String part : parts) + { + String[] keyValue = part.split("="); + String key = keyValue[0]; + String value = keyValue[1]; + + if (key.equalsIgnoreCase("oauth_token_secret")) + userSecret = value; + else + if (key.equalsIgnoreCase("oauth_token")) + userKey = value; + } + + if (userSecret == null || userKey == null) + throw new Exception ("Could parse authToken"); + + return new Pair(userKey, userSecret); + } +} diff --git a/java/core/src/core/connector/misc/sync/FileSystemConnector.java b/java/core/src/core/connector/misc/sync/FileSystemConnector.java new file mode 100644 index 0000000..733ba81 --- /dev/null +++ b/java/core/src/core/connector/misc/sync/FileSystemConnector.java @@ -0,0 +1,124 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.misc.sync; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.List; + +import core.connector.ConnectorException; +import core.connector.FileInfo; +import core.connector.sync.StoreConnector; +import core.util.FileSystem; +import core.util.Streams; + + +public class FileSystemConnector implements StoreConnector +{ + String prefix; + + public FileSystemConnector (String root) + { + this.prefix = root + "/"; + } + + @Override + public void open() throws ConnectorException + { + File directory = new File(prefix); + if (!directory.exists()) + throw new ConnectorException("Store directory does not exist"); + } + + @Override + public void close() + { + } + + @Override + public List listDirectory(String path) throws ConnectorException + { + return FileSystem.allFilesFor(new File(prefix + path)); + } + + @Override + public void createDirectory(String path) throws ConnectorException + { + File dir = new File (prefix + path); + if (!dir.mkdir()) + { + throw new ConnectorException("Unable to create directory"); + } + } + + @Override + public byte[] get(String path) throws ConnectorException + { + try + { + return Streams.readFullyBytes(new FileInputStream(prefix + path)); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public byte[] get(String path, long size) throws ConnectorException + { + return get(path); + } + + @Override + public void put(String path, byte[] contents) throws ConnectorException + { + try + { + if (path.contains("/")) + ensureDirectories(path.substring(0, path.lastIndexOf("/"))); + + FileOutputStream fos = new FileOutputStream(prefix + path); + fos.write(contents); + fos.close(); + } + catch (IOException e) + { + throw new ConnectorException(e); + } + } + + @Override + public void move(String from, String to) throws ConnectorException + { + File fromFile = new File(prefix + from); + File toFile = new File (prefix, to); + + if (!fromFile.renameTo(toFile)) + throw new ConnectorException("move file failed"); + } + + @Override + public void delete(String path) throws ConnectorException + { + File file = new File(prefix + path); + if (!file.delete()) + throw new ConnectorException("delete file failed"); + } + + public boolean ensureDirectories (String ... folders) + { + for (String folder : folders) + { + File path = new File (prefix + folder); + path.mkdirs(); + } + + return true; + } +} diff --git a/java/core/src/core/connector/s3/ClientInfoS3.java b/java/core/src/core/connector/s3/ClientInfoS3.java new file mode 100644 index 0000000..a539220 --- /dev/null +++ b/java/core/src/core/connector/s3/ClientInfoS3.java @@ -0,0 +1,58 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.s3; + +import core.constants.ConstantsS3; +import core.util.Environment; + + +public class ClientInfoS3 +{ + private String bucketName; + private String bucketRegion; + private String accessId; + private String secretKey; + + public ClientInfoS3 (Environment e) + { + bucketName = e.get(ConstantsS3.AWSBucketName); + bucketRegion = e.get(ConstantsS3.AWSBucketRegion); + accessId = e.checkGet(ConstantsS3.AWSAccessKeyId); + secretKey = e.checkGet(ConstantsS3.AWSSecretKey); + } + + public String getBucketEndpoint () + { + String bucketEndPoint = null; + + if (bucketRegion == null || bucketRegion.equals("")) + bucketEndPoint = "s3.amazonaws.com"; + else + bucketEndPoint = "s3-" + bucketRegion + ".amazonaws.com"; + + return bucketEndPoint; + } + + public String getBucketName() + { + return bucketName; + } + + public String getBucketRegion () + { + return bucketRegion; + } + + public String getAccessId() + { + return accessId; + } + + public String getSecretKey() + { + return secretKey; + } +} diff --git a/java/core/src/core/connector/s3/async/S3Connector.java b/java/core/src/core/connector/s3/async/S3Connector.java new file mode 100644 index 0000000..2e2cd68 --- /dev/null +++ b/java/core/src/core/connector/s3/async/S3Connector.java @@ -0,0 +1,351 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.s3.async; + +import java.util.ArrayList; + +import java.util.Collections; +import java.util.Date; +import java.util.List; +import core.util.Base64; +import core.callback.Callback; +import core.callback.CallbackDefault; +import core.callback.CallbackWithVariables; +import core.connector.FileInfo; +import core.connector.async.AsyncStoreConnectorHelper; +import core.connector.s3.ClientInfoS3; +import core.crypt.HashSha256; +import core.crypt.HmacSha1; +import core.util.DateFormat; +import core.util.FastRandom; +import core.util.HttpDelegate; +import core.util.LogNull; +import core.util.Strings; +import core.util.XML; + +public class S3Connector extends AsyncStoreConnectorHelper +{ + static LogNull log = new LogNull(S3Connector.class); + final int LOCK_INTERVAL = 10 * 1000; + + ClientInfoS3 info; + HttpDelegate httpDelegate; + + HmacSha1 mac; + static FastRandom fastRandom = new FastRandom(); + + + protected String createUrlPrefix () + { + return "https://" + info.getBucketEndpoint() + "/" + info.getBucketName() + "/"; + } + + protected String createRandomPostfix () + { + return "random=" + fastRandom.nextLong(); + } + + // This method converts AWSSecretKey into crypto instance. + protected void setKey(String AWSSecretKey) throws Exception + { + mac = new HmacSha1(Strings.toBytes(AWSSecretKey)); + } + + // This method creates S3 signature for a given String. + protected String sign(String data) throws Exception + { + // Signed String must be BASE64 encoded. + byte[] signBytes = mac.mac(Strings.toBytes(data)); + String signature = Base64.encode(signBytes); + return signature; + } + + protected String format(String format, Date date) + { + DateFormat df = new DateFormat(format); + String dateString = df.format(date, 0) + " GMT"; + return dateString; + } + + protected String[][] makeHeaders (String keyId, String method, String contentMD5, String contentType, int contentLength, Date date, String resource) throws Exception + { + String fmt = "EEE, dd MMM yyyy HH:mm:ss"; + String dateString = format(fmt, date); + + // Generate signature + StringBuffer buf = new StringBuffer(); + buf.append(method).append("\n"); + buf.append(contentMD5).append("\n"); + buf.append(contentType).append("\n"); + buf.append("\n"); // empty real date header + buf.append("x-amz-date:"); + buf.append(dateString).append("\n"); + buf.append(resource); + + log.debug("Signing:{" + buf.toString() + "}"); + String signature = sign(buf.toString()); + + String[][] headers; + if (method.equals("PUT")) + { + headers = new String[][] { + {"X-Amz-Date" , dateString }, + {"Content-Type", contentType }, + {"Content-Length", ""+contentLength }, + {"Authorization", "AWS " + keyId + ":" + signature } + }; + } + else + { + headers = new String[][] { + {"X-Amz-Date" , dateString }, + {"Authorization", "AWS " + keyId + ":" + signature } + }; + } + + return headers; + } + + public S3Connector(ClientInfoS3 clientInfo, HttpDelegate httpDelegate) throws Exception + { + this.info = clientInfo; + this.httpDelegate = httpDelegate; + + setKey (info.getSecretKey()); + } + + long toVersionFromString (String s) throws Exception + { + HashSha256 hash = new HashSha256(); + byte[] result = hash.hash(Strings.toBytes(s)); + + long l = + ((long)result[0]) | + ((long)result[1] << 8) | + ((long)result[2] << 16) | + ((long)result[3] << 24); + + return l; + } + + public void listDirectoryFinished (List files, Callback callback, String path, Object... arguments) + { + log.debug("listDirectoryFinished"); + try + { + if (arguments[0] instanceof Exception) + throw (Exception)arguments[0]; + + String result = (String)arguments[0]; + log.trace(result); + + DateFormat dateTimeFormat = new DateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z' Z"); + + Object doc = XML.parse(result); + Object[] nodes = XML.getElementsByTagName(doc, "Contents"); + for (Object currentNode : nodes) + { + if ( XML.getNodeType(currentNode) == XML.ELEMENT_NODE ) + { + Object keyNode = XML.getElementsByTagName(currentNode, "Key")[0]; + Object etagNode = XML.getElementsByTagName(currentNode, "ETag")[0]; + Object sizeNode = XML.getElementsByTagName(currentNode, "Size")[0]; + Object lastModifiedNode = XML.getElementsByTagName(currentNode, "LastModified")[0]; + + log.trace(XML.textOf(keyNode), XML.textOf(sizeNode), XML.textOf(etagNode), XML.textOf(lastModifiedNode)); + + String fullPath = XML.textOf(keyNode); + String relativePath = fullPath.substring(path.length()); + + FileInfo fi = new FileInfo( + fullPath, + relativePath, + Long.parseLong(XML.textOf(sizeNode)), + dateTimeFormat.parse(XML.textOf(lastModifiedNode) + " GMT"), + XML.textOf(etagNode) + ); + + files.add(fi); + } + } + + if (XML.textOf(XML.getElementsByTagName(doc, "IsTruncated")[0]).equals("true")) + { + log.debug("results were truncated, requesting more..."); + listIterative(files, path, callback); + } + else + { + log.debug("results were complete, invoking callback"); + + Collections.sort(files, new FileInfo.SortByDateAscending()); + for (FileInfo i : files) + log.trace ("path: ", i.path, " date:", i.date); + + callback.invoke(files); + } + } + catch (Throwable e) + { + e.printStackTrace(); + + callback.invoke(e); + } + } + + @Override + public void list(String path, Callback callback) + { + listIterative(new ArrayList(), path, callback); + } + + public void listIterative(List files, String path, Callback callback) + { + log.debug("listDirectory",path); + try + { + String url = + createUrlPrefix() + + "?prefix=" + path + "&max-keys=1000" + + (!files.isEmpty() ? ("&marker=" + files.get(files.size()-1).path) : "") + + "&" + createRandomPostfix(); + + log.debug(url); + + String[][] headers = makeHeaders ( + info.getAccessId(), "GET", "", "", 0, new Date(), "/" + info.getBucketName() + "/" + ); + + httpDelegate.execute (HttpDelegate.GET, url, headers, false, false, null, + new CallbackWithVariables(files, callback, path) { + @Override + public void invoke(Object... arguments) + { + List files = V(0); + Callback callback = V(1); + String path = V(2); + listDirectoryFinished(files, callback, path, arguments); + } + } + ); + } + catch (Throwable e) + { + e.printStackTrace(); + + callback.invoke(e); + } + } + + @Override + public void createDirectory(String path, Callback callback) + { + log.debug("createDirectory",path); + callback.invoke(path); + } + + @Override + public void get(String path, Callback callback) + { + log.debug("get",path); + try + { + String url = + createUrlPrefix() + + path + + "?" + createRandomPostfix(); + + log.debug(url); + + String[][] headers = makeHeaders ( + info.getAccessId(), "GET", "", "", 0, new Date(), "/" + info.getBucketName() + "/" + path + ); + + httpDelegate.execute(HttpDelegate.GET, url, headers, false, true, null, grabVersion_(true).setReturn(callback)); + } + catch (Throwable e) + { + callback.invoke(e); + } + } + + public Callback grabVersion_(boolean includeResponseData) + { + return new CallbackDefault(includeResponseData) { + public void onSuccess(Object... arguments) throws Exception { + + boolean includeResponseData = V(0); + String[][] headers = (String[][])arguments[1]; + + for (String[] pair : headers) + { + if (pair[0].equals("ETag")) + { + if (includeResponseData) + next(arguments[0], pair[1]); + else + next(pair[1]); + + return; + } + } + + throw new Exception("No ETag Response header"); + } + }; + } + + @Override + public void put(String path, byte[] contents, Callback callback) + { + log.debug("put",path); + try + { + String url = + createUrlPrefix() + + path + + "?" + createRandomPostfix(); + + log.debug(url); + + String[][] headers = makeHeaders ( + info.getAccessId(), "PUT", "", "application/octet-stream", contents.length, new Date(), "/" + info.getBucketName() + "/" + path + ); + + httpDelegate.execute(HttpDelegate.PUT, url, headers, true, false, contents, grabVersion_(false).setReturn(callback)); + } + catch (Throwable e) + { + callback.invoke(e); + } + } + + @Override + public void delete(String path, Callback callback) + { + log.debug("delete",path); + + try + { + String url = + createUrlPrefix() + + path + + "?" + createRandomPostfix(); + + log.debug(url); + + String[][] headers = makeHeaders ( + info.getAccessId(), "DELETE", "", "", 0, new Date(), "/" + info.getBucketName() + "/" + path + ); + + httpDelegate.execute(HttpDelegate.DELETE, url, headers, true, false, null, callback); + } + catch (Throwable e) + { + callback.invoke(e); + } + } +} diff --git a/java/core/src/core/connector/s3/sync/S3Connector.java b/java/core/src/core/connector/s3/sync/S3Connector.java new file mode 100644 index 0000000..4a1c821 --- /dev/null +++ b/java/core/src/core/connector/s3/sync/S3Connector.java @@ -0,0 +1,179 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.s3.sync; + +import java.io.ByteArrayInputStream; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.S3ObjectSummary; + +import core.connector.ConnectorException; +import core.connector.FileInfo; +import core.connector.s3.ClientInfoS3; +import core.connector.sync.StoreConnector; +import core.util.Streams; + + +public class S3Connector implements StoreConnector +{ + AmazonS3 s3; + ClientInfoS3 info; + + public S3Connector (ClientInfoS3 info) + { + this.info = info; + } + + public void open () throws ConnectorException + { + try + { + s3 = new AmazonS3Client(new SimpleAWSCredentials(info.getAccessId(), info.getSecretKey())); + s3.setEndpoint(info.getBucketEndpoint()); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + public void close () + { + s3 = null; + } + + @Override + public List listDirectory(String path) throws ConnectorException + { + try + { + ObjectListing bucketListing = + s3.listObjects( + new ListObjectsRequest() + .withBucketName(info.getBucketName()) + .withPrefix(path) + ); + + List listing = new ArrayList(bucketListing.getObjectSummaries().size()); + + boolean finished = false; + while (!finished) + { + for (S3ObjectSummary s3s : bucketListing.getObjectSummaries()) + { + String key = s3s.getKey(); + long size = s3s.getSize(); + Date date = s3s.getLastModified(); + + listing.add(new FileInfo(key, key.substring(path.length()+1), size, date, s3s.getETag())); + } + if (bucketListing.isTruncated()) + bucketListing = s3.listNextBatchOfObjects(bucketListing); + else + finished = true; + } + + return listing; + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public void createDirectory(String path) throws ConnectorException + { + // no need in s3 + } + + @Override + public byte[] get(String path, long size) throws ConnectorException + { + try + { + GetObjectRequest request = new GetObjectRequest(info.getBucketName(), path); + if (size >= 0) + request.withRange(0, size); + + return Streams.readFullyBytes(s3.getObject(request).getObjectContent()); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public byte[] get(String path) throws ConnectorException + { + return get(path, -1); + } + + @Override + public void put(String path, byte[] contents) throws ConnectorException + { + try + { + s3.putObject(info.getBucketName(), path, new ByteArrayInputStream(contents), null); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public void move(String from, String to) throws ConnectorException + { + try + { + s3.copyObject(info.getBucketName(), from, info.getBucketName(), to); + s3.deleteObject(info.getBucketName(), from); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public void delete(String path) throws ConnectorException + { + try + { + s3.deleteObject(info.getBucketName(), path); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + public boolean ensureDirectories (String ... folders) + { + for (String folder : folders) + { + try + { + createDirectory(folder); + } + catch (Exception e) + { + System.out.format("Folder[%s] already exists.\n", folder); + } + } + + return true; + } +} diff --git a/java/core/src/core/connector/s3/sync/SimpleAWSCredentials.java b/java/core/src/core/connector/s3/sync/SimpleAWSCredentials.java new file mode 100644 index 0000000..f9d152a --- /dev/null +++ b/java/core/src/core/connector/s3/sync/SimpleAWSCredentials.java @@ -0,0 +1,40 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.s3.sync; + + +import com.amazonaws.auth.AWSCredentials; + +import core.constants.ConstantsS3; +import core.util.Environment; + +public class SimpleAWSCredentials implements AWSCredentials +{ + private String a, s; + + public SimpleAWSCredentials(String a, String s) + { + this.a = a; + this.s = s; + } + + public SimpleAWSCredentials(Environment e) + { + this(e.checkGet(ConstantsS3.AWSAccessKeyId), e.checkGet(ConstantsS3.AWSSecretKey)); + } + + @Override + public String getAWSAccessKeyId() + { + return a; + } + + @Override + public String getAWSSecretKey() + { + return s; + } +} diff --git a/java/core/src/core/connector/sync/EncryptedStoreConnector.java b/java/core/src/core/connector/sync/EncryptedStoreConnector.java new file mode 100644 index 0000000..1d0b45a --- /dev/null +++ b/java/core/src/core/connector/sync/EncryptedStoreConnector.java @@ -0,0 +1,102 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.sync; + +import java.util.List; + +import core.connector.ConnectorException; +import core.connector.FileInfo; +import core.crypt.Cryptor; +import core.util.Streams; +import core.util.Zip; + + +public class EncryptedStoreConnector implements StoreConnector +{ + StoreConnector store; + Cryptor cryptor; + + public EncryptedStoreConnector (Cryptor cryptor, StoreConnector store) + { + this.store = store; + this.cryptor = cryptor; + } + + @Override + public void open() throws ConnectorException + { + store.open(); + } + + @Override + public void close() throws ConnectorException + { + store.close(); + } + + @Override + public List listDirectory(String path) throws ConnectorException + { + return store.listDirectory(path); + } + + @Override + public void createDirectory(String path) throws ConnectorException + { + store.createDirectory(path); + } + + @Override + public byte[] get(String path) throws ConnectorException + { + try + { + return Zip.inflate(cryptor.decrypt(store.get(path))); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public byte[] get(String path, long size) throws ConnectorException + { + return get(path); + } + + @Override + public void put(String path, byte[] contents) throws ConnectorException + { + try + { + store.put(path, cryptor.encrypt(Zip.deflate(contents))); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public void move(String from, String to) throws ConnectorException + { + store.move(from, to); + } + + @Override + public void delete(String path) throws ConnectorException + { + store.delete(path); + } + + @Override + public boolean ensureDirectories(String... folders) + { + return store.ensureDirectories(folders); + } + +} diff --git a/java/core/src/core/connector/sync/StoreConnector.java b/java/core/src/core/connector/sync/StoreConnector.java new file mode 100644 index 0000000..35e9caa --- /dev/null +++ b/java/core/src/core/connector/sync/StoreConnector.java @@ -0,0 +1,29 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ + +package core.connector.sync; + +import java.util.List; + +import core.connector.ConnectorException; +import core.connector.FileInfo; + +public interface StoreConnector +{ + public void open () throws ConnectorException; + public void close () throws ConnectorException; + + List listDirectory (String path) throws ConnectorException; + void createDirectory (String path) throws ConnectorException; + + byte[] get (String path) throws ConnectorException; + byte[] get (String path, long size) throws ConnectorException; + + void put (String path, byte[] contents) throws ConnectorException; + void move (String from, String to) throws ConnectorException; + void delete (String path) throws ConnectorException; + + public boolean ensureDirectories (String ... folders); +}