diff --git a/DEV_BRAIN_DUMP.txt b/DEV_BRAIN_DUMP.txt index 4678eb2..a5690a1 100644 --- a/DEV_BRAIN_DUMP.txt +++ b/DEV_BRAIN_DUMP.txt @@ -92,5 +92,15 @@ You are screwed. Totally screwed. Make backups to a safe place of the mysql se +* How should I have done the code to avoid all that nasty call back stuff + +I'm thinking that I should have done a master dispatcher. +I should have done all http requests sync not async. +And had like 20 threads to do request on and computation. + +Instead of creating all of these callback chains, I should have just dispatched to one of the workers. +* The StoreEnable and CreateBucket should happen behind the scenes. +The results should be encrypted with a user supplied key +also, CreateBucket could be hacked. diff --git a/build/compile-web b/build/compile-web index 980341e..24a5781 100755 --- a/build/compile-web +++ b/build/compile-web @@ -1,9 +1,7 @@ #!/bin/bash -if [ -z "$1" ]; then echo "Must supply prod or dev"; exit 0; fi - V=`date "+%Y%m%d_%H%M%S"` -M=$1 +M=prod rsync -avL --delete ../web/WebContent/ www/ rsync -avL --delete ../gwt/war/mailiverse_gwt/ www/mailiverse_gwt/ diff --git a/deploy/deploy-all b/deploy/deploy-all index 5a5d2dd..d1f01b8 100755 --- a/deploy/deploy-all +++ b/deploy/deploy-all @@ -1,7 +1,8 @@ #./server-shutdown tunnel + ./server-deploy tomcat #./server-deploy james -./server-deploy postfix +./server-deploy postfix-user ./server-deploy mail-user ./server-deploy mail-key ./server-deploy tools diff --git a/deploy/dev-deploy-all b/deploy/dev-deploy-all index 0292651..750a2a7 100755 --- a/deploy/dev-deploy-all +++ b/deploy/dev-deploy-all @@ -1,6 +1,6 @@ ./dev-server-deploy tomcat #./dev-server-deploy james -./dev-server-deploy postfix +./dev-server-deploy postfix-user ./dev-server-deploy mail-user ./dev-server-deploy mail-key ./dev-server-deploy tools diff --git a/deploy/postfix-user/config/main.cf b/deploy/postfix-user/config/main.cf index 371bb47..cfb41e4 100644 --- a/deploy/postfix-user/config/main.cf +++ b/deploy/postfix-user/config/main.cf @@ -33,8 +33,10 @@ inet_interfaces = all inet_protocols = all javapipe_destination_recipient_limit = 1 -virtual_mailbox_domains = hash:/etc/postfix/virtual_domains -virtual_mailbox_maps = mysql:/etc/postfix/virtual_mailbox_maps.cf +virtual_mailbox_domains = hash:/etc/postfix/config/virtual_domains +virtual_mailbox_maps = mysql:/etc/postfix/config/virtual_mailbox_maps.cf virtual_transport = javapipe +smtp_sasl_auth_enable = yes +smtp_sasl_password_maps = mysql:/etc/postfix/config/sasl_password.cf diff --git a/deploy/postfix-user/config/sasl_password.cf b/deploy/postfix-user/config/sasl_password.cf new file mode 100644 index 0000000..5fdf82b --- /dev/null +++ b/deploy/postfix-user/config/sasl_password.cf @@ -0,0 +1,16 @@ +# +# mysql config file for local(8) aliases(5) lookups +# + +# The user name and password to log into the mysql server. +hosts = 127.0.0.1 +user = postfix +password = postfix + +# The database name on the servers. +dbname = postfix + +# For Postfix 2.2 and later The SQL query template. +# See mysql_table(5) for details. +query = SELECT password FROM user WHERE name='%s' + diff --git a/deploy/postfix-user/resources/passwords/store b/deploy/postfix-user/resources/passwords/store new file mode 120000 index 0000000..b349416 --- /dev/null +++ b/deploy/postfix-user/resources/passwords/store @@ -0,0 +1 @@ +../../../../passwords/store \ No newline at end of file diff --git a/deploy/tomcat/resources/passwords/store b/deploy/tomcat/resources/passwords/store new file mode 120000 index 0000000..b349416 --- /dev/null +++ b/deploy/tomcat/resources/passwords/store @@ -0,0 +1 @@ +../../../../passwords/store \ No newline at end of file diff --git a/deploy/tomcat/restart b/deploy/tomcat/restart index 72028c8..656572e 100755 --- a/deploy/tomcat/restart +++ b/deploy/tomcat/restart @@ -1,3 +1,7 @@ +pushd ~/tomcat/webapps +rm -rf Mailiverse +popd + cd ~/tomcat/bin ./shutdown.sh diff --git a/gwt/src/core/connector/mv/ClientInfoMvStore.java b/gwt/src/core/connector/mv/ClientInfoMvStore.java new file mode 120000 index 0000000..5846eb8 --- /dev/null +++ b/gwt/src/core/connector/mv/ClientInfoMvStore.java @@ -0,0 +1 @@ +../../../../../java/core/src/core/connector/mv/ClientInfoMvStore.java \ No newline at end of file diff --git a/gwt/src/core/connector/mv/async b/gwt/src/core/connector/mv/async new file mode 120000 index 0000000..ee398c6 --- /dev/null +++ b/gwt/src/core/connector/mv/async @@ -0,0 +1 @@ +../../../../../java/core/src/core/connector/mv/async \ No newline at end of file diff --git a/gwt/src/core/constants/ConstantsMvStore.java b/gwt/src/core/constants/ConstantsMvStore.java new file mode 120000 index 0000000..48b2b77 --- /dev/null +++ b/gwt/src/core/constants/ConstantsMvStore.java @@ -0,0 +1 @@ +../../../../java/core/src/core/constants/ConstantsMvStore.java \ No newline at end of file diff --git a/gwt/src/core/util/HttpDelegateFactory.java b/gwt/src/core/util/HttpDelegateFactory.java index 04f2449..e085e1d 100644 --- a/gwt/src/core/util/HttpDelegateFactory.java +++ b/gwt/src/core/util/HttpDelegateFactory.java @@ -5,7 +5,7 @@ import app.service.Main; public class HttpDelegateFactory { - static HttpDelegate create () + static public HttpDelegate create () { return new JSHttpDelegate(Main.delegate); } diff --git a/gwt/src/mail/client/model/PublicKey.java b/gwt/src/mail/client/model/PublicKey.java new file mode 120000 index 0000000..ceb411e --- /dev/null +++ b/gwt/src/mail/client/model/PublicKey.java @@ -0,0 +1 @@ +../../../../../java/core/src/mail/client/model/PublicKey.java \ No newline at end of file diff --git a/gwt/src/mail/client/model/PublicKeyRing.java b/gwt/src/mail/client/model/PublicKeyRing.java new file mode 120000 index 0000000..82c64bf --- /dev/null +++ b/gwt/src/mail/client/model/PublicKeyRing.java @@ -0,0 +1 @@ +../../../../../java/core/src/mail/client/model/PublicKeyRing.java \ No newline at end of file diff --git a/install/requirements/server_root_account b/install/requirements/server_root_account index e9e5f7c..d8649da 100644 --- a/install/requirements/server_root_account +++ b/install/requirements/server_root_account @@ -1 +1 @@ -ubuntu +root diff --git a/install/setup-mysql b/install/setup-mysql index 2266509..86c037a 100755 --- a/install/setup-mysql +++ b/install/setup-mysql @@ -6,6 +6,7 @@ MAIL_PASSWORD=`cat ../passwords/mail` MAIL_EXTRA_PASSWORD=`cat ../passwords/mail_extra` JAMES_PASSWORD=`cat ../passwords/james` CAPTCHA_PASSWORD=`cat ../passwords/captcha` +STORE_PASSWORD=`cat ../passwords/store` M_HOST=$1 @@ -15,6 +16,7 @@ sed \ -e ''s/MAIL_EXTRA_PASSWORD/$MAIL_EXTRA_PASSWORD/'' \ -e ''s/JAMES_PASSWORD/$JAMES_PASSWORD/'' \ -e ''s/CAPTCHA_PASSWORD/$CAPTCHA_PASSWORD/'' \ + -e ''s/STORE_PASSWORD/$STORE_PASSWORD/'' \ setup-mysql.sql.template > setup-mysql.sql \ ` diff --git a/install/setup-mysql.sql.template b/install/setup-mysql.sql.template index a0c47fb..41fcf0b 100644 --- a/install/setup-mysql.sql.template +++ b/install/setup-mysql.sql.template @@ -1,17 +1,20 @@ CREATE DATABASE mail DEFAULT CHARACTER SET utf8 COLLATE utf8_bin; CREATE DATABASE mail_extra DEFAULT CHARACTER SET utf8 COLLATE utf8_bin; CREATE DATABASE captcha DEFAULT CHARACTER SET utf8 COLLATE utf8_bin; +CREATE DATABASE store DEFAULT CHARACTER SET utf8 COLLATE utf8_bin; CREATE DATABASE postfix DEFAULT CHARACTER SET utf8 COLLATE utf8_bin; CREATE DATABASE james CHARACTER SET utf8; CREATE USER 'mail'@'localhost' IDENTIFIED BY 'MAIL_PASSWORD'; CREATE USER 'mail_extra'@'localhost' IDENTIFIED BY 'MAIL_EXTRA_PASSWORD'; CREATE USER 'captcha'@'localhost' IDENTIFIED BY 'CAPTCHA_PASSWORD'; +CREATE USER 'store'@'localhost' IDENTIFIED BY 'STORE_PASSWORD'; CREATE USER 'james'@'localhost' IDENTIFIED BY 'JAMES_PASSWORD'; CREATE USER 'postfix'@'localhost' IDENTIFIED BY 'postfix'; GRANT ALL PRIVILEGES ON mail.* TO 'mail'@'localhost'; GRANT ALL PRIVILEGES ON mail_extra.* TO 'mail_extra'@'localhost'; GRANT ALL PRIVILEGES ON captcha.* TO 'captcha'@'localhost'; +GRANT ALL PRIVILEGES ON store.* TO 'store'@'localhost'; GRANT ALL PRIVILEGES ON james.* TO 'james'@'localhost'; GRANT ALL PRIVILEGES ON postfix.* TO 'postfix'@'localhost'; diff --git a/install/setup-postfix.remote b/install/setup-postfix.remote index 321bc1e..5fe8908 100755 --- a/install/setup-postfix.remote +++ b/install/setup-postfix.remote @@ -1,6 +1,8 @@ set -x +apt-get update echo "1" | apt-get install postfix postfix-mysql postfix-pcre --yes +apt-get install sasl2-bin --yes cp sudoers.d-postfix-user /etc/sudoers.d/postfix-user chmod 0440 /etc/sudoers.d/postfix-user @@ -11,7 +13,6 @@ rm master.cf ln -fs /home/postfix-user/config/main.cf ln -fs /home/postfix-user/config/master.cf -#ln -fs /home/postfix-user/config/virtual_domains.pcre -ln -fs /home/postfix-user/config/virtual_domains -ln -fs /home/postfix-user/config/virtual_mailbox_maps.cf +ln -fs /home/postfix-user/config + diff --git a/java/app/service/src/app/service/JSSignUp.java b/java/app/service/src/app/service/JSSignUp.java index 6bf55ff..a59d2b9 100644 --- a/java/app/service/src/app/service/JSSignUp.java +++ b/java/app/service/src/app/service/JSSignUp.java @@ -23,6 +23,7 @@ import core.callback.Callback; import core.callback.CallbackChain; import core.callback.CallbackDefault; import core.callback.CallbackWithVariables; +import core.constants.ConstantsMvStore; import core.constants.ConstantsS3; import core.constants.ConstantsEnvironmentKeys; import core.constants.ConstantsClient; @@ -125,7 +126,7 @@ public class JSSignUp implements Exportable, SRPClientListener } static class SignUpInfo { - public static enum Storage { Mailiverse, Dropbox }; + public static enum Storage { S3, Mailiverse, Dropbox }; String name, password, captchaToken; Environment serverEnvironment, clientEnvironment, completeEnvironment; @@ -144,6 +145,8 @@ public class JSSignUp implements Exportable, SRPClientListener String awsBucketName, awsBucketRegion; String awsWriteAccessKey, awsWriteSecretKey; String awsReadWriteAccessKey, awsReadWriteSecretKey; + + String mvAccessKey, mvSecretKey; String smtpPassword; byte[] rsaPublicKey; @@ -180,10 +183,15 @@ public class JSSignUp implements Exportable, SRPClientListener this.dropboxAuthSecret = authSecret; } - void initializeStorageMailiverse (String storageRegion) + void initializeStorageS3 (String storageRegion) + { + this.storage = Storage.S3; + this.storageRegion = storageRegion; + } + + void initializeStorageMailiverse () { this.storage = Storage.Mailiverse; - this.storageRegion = storageRegion; } public void setDropboxInfo (String userToken, String userSecret) @@ -205,6 +213,13 @@ public class JSSignUp implements Exportable, SRPClientListener this.awsReadWriteSecretKey = awsReadWriteSecretKey; } + public void setMailiverseInfo (String mvAccessKey, String mvSecretKey) + { + this.mvAccessKey = mvAccessKey; + this.mvSecretKey = mvSecretKey; + } + + public void calculateRSA (Callback callback) throws NoSuchAlgorithmException { new CryptorRSAFactory().generate(2048, new CallbackDefault() { @@ -347,13 +362,65 @@ public class JSSignUp implements Exportable, SRPClientListener completeEnvironment.addChildEnvironment(ConstantsEnvironmentKeys.CLIENT_ENVIRONMENT, clientEnvironment); } + public void calculateEnvironmentMailiverse () + { + String handler = ConstantsStorage.HANDLER_MV; + String prefix = handler + "/"; + + serverEnvironment = new Environment(); + serverEnvironment.put(ConstantsEnvironmentKeys.CONFIGURATION_VERSION, ConstantsVersion.CONFIGURATION); + serverEnvironment.put(ConstantsEnvironmentKeys.HANDLER, handler); + serverEnvironment.put(ConstantsEnvironmentKeys.SMTP_PASSWORD, smtpPassword); + serverEnvironment.put(prefix + ConstantsMvStore.AccessKeyId, mvAccessKey); + serverEnvironment.put(prefix + ConstantsMvStore.SecretKey, mvSecretKey); + serverEnvironment.put( + ConstantsEnvironmentKeys.PUBLIC_ENCRYPTION_KEY, + Base64.encode(rsaPublicKey) + ); + serverEnvironment.put( + ConstantsEnvironmentKeys.PGP_PUBLIC_KEY, + Base64.encode(pgpPublicKey) + ); + + clientEnvironment = new Environment(); + clientEnvironment.put(ConstantsEnvironmentKeys.CONFIGURATION_VERSION, ConstantsVersion.CONFIGURATION); + clientEnvironment.put(ConstantsEnvironmentKeys.HANDLER, handler); + clientEnvironment.put(ConstantsEnvironmentKeys.SMTP_PASSWORD, smtpPassword); + clientEnvironment.put(prefix + ConstantsMvStore.AccessKeyId, mvAccessKey); + clientEnvironment.put(prefix + ConstantsMvStore.SecretKey, mvSecretKey); + clientEnvironment.put( + ConstantsEnvironmentKeys.PUBLIC_ENCRYPTION_KEY, + Base64.encode(rsaPublicKey) + ); + clientEnvironment.put( + ConstantsEnvironmentKeys.PRIVATE_DECRYPTION_KEY, + Base64.encode(rsaPrivateKey) + ); + clientEnvironment.put( + ConstantsEnvironmentKeys.PGP_PUBLIC_KEY, + Base64.encode(pgpPublicKey) + ); + clientEnvironment.put( + ConstantsEnvironmentKeys.PGP_PRIVATE_KEY, + Base64.encode(pgpPrivateKey) + ); + + completeEnvironment = new Environment(); + completeEnvironment.put(ConstantsEnvironmentKeys.CONFIGURATION_VERSION, ConstantsVersion.CONFIGURATION); + completeEnvironment.addChildEnvironment(ConstantsEnvironmentKeys.SERVER_ENVIRONMENT, serverEnvironment); + completeEnvironment.addChildEnvironment(ConstantsEnvironmentKeys.CLIENT_ENVIRONMENT, clientEnvironment); + } + public void calculateEnvironment () { if (storage == Storage.Dropbox) calculateEnvironmentDropbox(); else - if (storage == Storage.Mailiverse) + if (storage == Storage.S3) calculateEnvironmentAWS(); + else + if (storage == Storage.Mailiverse) + calculateEnvironmentMailiverse(); } }; @@ -376,8 +443,11 @@ public class JSSignUp implements Exportable, SRPClientListener if (storage.equals("dropbox")) info.intializeStorageDropbox(ConstantsClient.DROPBOX_APPKEY, ConstantsClient.DROPBOX_APPSECRET, dropboxUserKey, dropboxUserSecret); else + if (storage.equals("s3")) + info.initializeStorageS3(JSON_.getString(JSON_.parse(storageInfo), "region")); + else if (storage.equals("mailiverse")) - info.initializeStorageMailiverse(JSON_.getString(JSON_.parse(storageInfo), "region")); + info.initializeStorageMailiverse(); CallbackChain signUpChain = new CallbackChain(); @@ -489,6 +559,9 @@ public class JSSignUp implements Exportable, SRPClientListener protected void signUp_step_requestAccess (SignUpInfo info, Callback callback) { + if (info.storage == SignUpInfo.Storage.S3) + signUp_step_requestS3Bucket(info, callback); + else if (info.storage == SignUpInfo.Storage.Mailiverse) signUp_step_requestMailiverseBucket(info, callback); else @@ -496,9 +569,9 @@ public class JSSignUp implements Exportable, SRPClientListener signUp_step_requestDropboxAccessToken(info, callback); } - protected void signUp_step_requestMailiverseBucket (SignUpInfo info, Callback callback) + protected void signUp_step_requestS3Bucket (SignUpInfo info, Callback callback) { - log.debug("signUp_step_requestMailiverseBucket"); + log.debug("signUp_step_requestS3Bucket"); String url = ConstantsClient.WEB_SERVER_TOMCAT + "CreateBucket" + @@ -509,7 +582,7 @@ public class JSSignUp implements Exportable, SRPClientListener new CallbackDefault(info) { public void onSuccess(Object... arguments) { - log.debug("signUp_step_requestMailiverseBucket callback"); + log.debug("signUp_step_requestS3Bucket callback"); SignUpInfo info = V(0); String response = (String)arguments[0]; String awsBucketName=null, awsBucketRegion=null, @@ -549,7 +622,46 @@ public class JSSignUp implements Exportable, SRPClientListener ); } - protected void signUp_step_requestDropboxAccessToken (SignUpInfo info, Callback callback) + protected void signUp_step_requestMailiverseBucket (SignUpInfo info, Callback callback) + { + log.debug("signUp_step_requestMailiverseBucket"); + + String url = + ConstantsClient.WEB_SERVER_TOMCAT + "StoreEnable" + + "?email=" + info.name + "&captcha=" + info.captchaToken; + + JSHttpDelegate http = new JSHttpDelegate(main.delegate); + http.execute(HttpDelegate.GET, url, null, false, false, null, + new CallbackDefault(info) { + public void onSuccess(Object... arguments) + { + log.debug("signUp_step_requestMailiverseBucket callback"); + SignUpInfo info = V(0); + String response = (String)arguments[0]; + String mvAccessKey=null, mvSecretKey=null; + + String[] parts = response.split("&"); + for (String part : parts) + { + String[] keyValue = part.split("!"); + String key = keyValue[0]; + String value = keyValue[1]; + + if (key.equalsIgnoreCase(ConstantsMvStore.AccessKeyId)) + mvAccessKey = value; + else + if (key.equalsIgnoreCase(ConstantsMvStore.SecretKey)) + mvSecretKey = value; + } + + info.setMailiverseInfo(mvAccessKey, mvSecretKey); + next(); + } + }.setReturn(callback) + ); + } + + protected void signUp_step_requestDropboxAccessToken (SignUpInfo info, Callback callback) { log.debug("signUp_step_requestAccessToken"); diff --git a/java/core/src/core/callback/CallbackSync.java b/java/core/src/core/callback/CallbackSync.java index 8720c00..be33d9a 100644 --- a/java/core/src/core/callback/CallbackSync.java +++ b/java/core/src/core/callback/CallbackSync.java @@ -79,4 +79,16 @@ public class CallbackSync return null; } + public void checkException () throws Exception + { + if (results != null && results.length > 0) + { + if (results[0] instanceof Exception) + { + Exception e = (Exception)results[0]; + throw e; + } + } + } + } diff --git a/java/core/src/core/connector/mv/ClientInfoMvStore.java b/java/core/src/core/connector/mv/ClientInfoMvStore.java new file mode 100644 index 0000000..5f7635f --- /dev/null +++ b/java/core/src/core/connector/mv/ClientInfoMvStore.java @@ -0,0 +1,33 @@ +package core.connector.mv; + +import core.constants.ConstantsClient; +import core.constants.ConstantsMvStore; +import core.util.Environment; + +public class ClientInfoMvStore +{ + private String accessId; + private String secretKey; + + public ClientInfoMvStore (Environment e) + { + accessId = e.checkGet(ConstantsMvStore.AccessKeyId); + secretKey = e.checkGet(ConstantsMvStore.SecretKey); + } + + public String getBucketEndpoint () + { + return ConstantsClient.SERVER_TOMCAT; + } + + public String getAccessId() + { + return accessId; + } + + public String getSecretKey() + { + return secretKey; + } + +} diff --git a/java/core/src/core/connector/mv/async/ConnectorMvStore.java b/java/core/src/core/connector/mv/async/ConnectorMvStore.java new file mode 100644 index 0000000..dffba30 --- /dev/null +++ b/java/core/src/core/connector/mv/async/ConnectorMvStore.java @@ -0,0 +1,346 @@ +package core.connector.mv.async; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import core.callback.Callback; +import core.callback.CallbackDefault; +import core.callback.CallbackWithVariables; +import core.connector.FileInfo; +import core.connector.async.AsyncStoreConnectorHelper; +import core.connector.mv.ClientInfoMvStore; +import core.connector.s3.async.S3Connector; +import core.crypt.HashSha256; +import core.crypt.HmacSha1; +import core.util.Base64; +import core.util.DateFormat; +import core.util.FastRandom; +import core.util.HttpDelegate; +import core.util.JSON_; +import core.util.LogNull; +import core.util.Strings; +import core.util.XML; + +public class ConnectorMvStore extends AsyncStoreConnectorHelper +{ + static LogNull log = new LogNull(S3Connector.class); + final int LOCK_INTERVAL = 10 * 1000; + + ClientInfoMvStore info; + HttpDelegate httpDelegate; + + HmacSha1 mac; + static FastRandom fastRandom = new FastRandom(); + + protected String createUrlPrefix () + { + return "https://" + info.getBucketEndpoint() + "/"; + } + + protected String createRandomPostfix () + { + return "random=" + fastRandom.nextLong(); + } + + // This method converts AWSSecretKey into crypto instance. + protected void setKey(String secretKey) throws Exception + { + mac = new HmacSha1(Base64.decode(secretKey)); + } + + // This method creates S3 signature for a given String. + protected String sign(String data) throws Exception + { + // Signed String must be BASE64 encoded. + byte[] signBytes = mac.mac(Strings.toBytes(data)); + String signature = Base64.encode(signBytes); + return signature; + } + + protected String format(String format, Date date) + { + DateFormat df = new DateFormat(format); + String dateString = df.format(date, 0) + " GMT"; + return dateString; + } + + protected String[][] makeHeaders (String keyId, String method, String contentMD5, String contentType, int contentLength, Date date, String resource) throws Exception + { + String fmt = "EEE, dd MMM yyyy HH:mm:ss"; + String dateString = format(fmt, date); + + // Generate signature + StringBuffer buf = new StringBuffer(); + buf.append(method).append("\n"); + buf.append(contentMD5).append("\n"); + buf.append(contentType).append("\n"); + buf.append("\n"); // empty real date header + buf.append(dateString).append("\n"); + buf.append(resource); + + log.debug("Signing:{" + buf.toString() + "}"); + String signature = sign(buf.toString()); + + String[][] headers; + if (method.equals("PUT")) + { + headers = new String[][] { + {"X-Mv-Date" , dateString }, + {"Content-Type", contentType }, + {"Content-Length", ""+contentLength }, + {"Authorization", "MV " + keyId + ":" + signature } + }; + } + else + { + headers = new String[][] { + {"X-Mv-Date" , dateString }, + {"Authorization", "MV " + keyId + ":" + signature } + }; + } + + return headers; + } + + public ConnectorMvStore(ClientInfoMvStore clientInfo, HttpDelegate httpDelegate) throws Exception + { + this.info = clientInfo; + this.httpDelegate = httpDelegate; + + setKey (info.getSecretKey()); + } + + long toVersionFromString (String s) throws Exception + { + HashSha256 hash = new HashSha256(); + byte[] result = hash.hash(Strings.toBytes(s)); + + long l = + ((long)result[0]) | + ((long)result[1] << 8) | + ((long)result[2] << 16) | + ((long)result[3] << 24); + + return l; + } + + public void listDirectoryFinished (List files, Callback callback, String path, Object... arguments) + { + log.debug("listDirectoryFinished"); + try + { + + if (arguments[0] instanceof Exception) + throw (Exception)arguments[0]; + + String result = (String)arguments[0]; + log.trace(result); + + DateFormat dateTimeFormat = new DateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z' Z"); + + Object doc = JSON_.parse(result); + Object nodes = JSON_.getArray(doc, "contents"); + for (int i=0; i(), path, callback); + } + + public void listIterative(List files, String path, Callback callback) + { + log.debug("listDirectory",path); + try + { + String url = + createUrlPrefix() + + "StoreList?Resource=" + path + "&max-keys=1000" + + (!files.isEmpty() ? ("&marker=" + files.get(files.size()-1).path) : "") + + "&" + createRandomPostfix(); + + log.debug(url); + + String[][] headers = makeHeaders ( + info.getAccessId(), "GET", "", "", 0, new Date(), path + ); + + httpDelegate.execute (HttpDelegate.GET, url, headers, false, false, null, + new CallbackWithVariables(files, callback, path) { + @Override + public void invoke(Object... arguments) + { + List files = V(0); + Callback callback = V(1); + String path = V(2); + listDirectoryFinished(files, callback, path, arguments); + } + } + ); + } + catch (Throwable e) + { + e.printStackTrace(); + + callback.invoke(e); + } + } + + @Override + public void createDirectory(String path, Callback callback) + { + log.debug("createDirectory",path); + callback.invoke(path); + } + + @Override + public void get(String path, Callback callback) + { + log.debug("get",path); + try + { + String url = + createUrlPrefix() + + "StoreGet?Resource=" + path + + "&" + createRandomPostfix(); + + log.debug(url); + + String[][] headers = makeHeaders ( + info.getAccessId(), "GET", "", "", 0, new Date(), path + ); + + httpDelegate.execute(HttpDelegate.GET, url, headers, false, true, null, grabVersion_(true).setReturn(callback)); + } + catch (Throwable e) + { + callback.invoke(e); + } + } + + public Callback grabVersion_(boolean includeResponseData) + { + return new CallbackDefault(includeResponseData) { + public void onSuccess(Object... arguments) throws Exception { + + boolean includeResponseData = (Boolean)V(0); + String[][] headers = (String[][])arguments[1]; + + for (String[] pair : headers) + { + if (pair[0].equals("Version")) + { + if (includeResponseData) + next(arguments[0], pair[1]); + else + next(pair[1]); + + return; + } + } + + throw new Exception("No ETag Response header"); + } + }; + } + + @Override + public void put(String path, byte[] contents, Callback callback) + { + log.debug("put",path); + try + { + String url = + createUrlPrefix() + + "StorePut?Resource=" + path + + "&" + createRandomPostfix(); + + log.debug(url); + + String[][] headers = makeHeaders ( + info.getAccessId(), "PUT", "", "application/octet-stream", contents.length, new Date(), path + ); + + httpDelegate.execute(HttpDelegate.PUT, url, headers, true, false, contents, grabVersion_(false).setReturn(callback)); + } + catch (Throwable e) + { + callback.invoke(e); + } + } + + @Override + public void delete(String path, Callback callback) + { + log.debug("delete",path); + + try + { + String url = + createUrlPrefix() + + "StoreDelete?Resource=" + path + + "&" + createRandomPostfix(); + + log.debug(url); + + String[][] headers = makeHeaders ( + info.getAccessId(), "DELETE", "", "", 0, new Date(), path + ); + + httpDelegate.execute(HttpDelegate.DELETE, url, headers, true, false, null, callback); + } + catch (Throwable e) + { + callback.invoke(e); + } + } +} diff --git a/java/core/src/core/connector/mv/sync/ConnectorMvStore.java b/java/core/src/core/connector/mv/sync/ConnectorMvStore.java new file mode 100644 index 0000000..ddb70df --- /dev/null +++ b/java/core/src/core/connector/mv/sync/ConnectorMvStore.java @@ -0,0 +1,137 @@ +package core.connector.mv.sync; + +import java.util.List; + +import core.callback.CallbackSync; +import core.connector.ConnectorException; +import core.connector.FileInfo; +import core.connector.mv.ClientInfoMvStore; +import core.connector.sync.StoreConnector; +import core.util.HttpDelegateFactory; + +public class ConnectorMvStore implements StoreConnector +{ + private ClientInfoMvStore clientInfo; + private core.connector.mv.async.ConnectorMvStore connector; + + public ConnectorMvStore (ClientInfoMvStore clientInfo) throws Exception + { + this.clientInfo = clientInfo; + this.connector = new core.connector.mv.async.ConnectorMvStore(clientInfo, HttpDelegateFactory.create()); + } + + @Override + public void open() throws ConnectorException + { + } + + @Override + public void close() throws ConnectorException + { + } + + @Override + public List listDirectory(String path) throws ConnectorException + { + try + { + CallbackSync sync = new CallbackSync(connector.list_(path)).invoke(); + return sync.>export(); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public void createDirectory(String path) throws ConnectorException + { + try + { + CallbackSync sync = new CallbackSync(connector.createDirectory_(path)).invoke(); + sync.checkException(); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public byte[] get(String path) throws ConnectorException + { + try + { + CallbackSync sync = new CallbackSync(connector.get_(path)).invoke(); + return sync.export(); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public byte[] get(String path, long size) throws ConnectorException + { + try + { + CallbackSync sync = new CallbackSync(connector.get_(path)).invoke(); + return sync.export(); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public void put(String path, byte[] contents) throws ConnectorException + { + try + { + CallbackSync sync = new CallbackSync(connector.put_(path, contents)).invoke(); + sync.checkException(); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public void move(String from, String to) throws ConnectorException + { + try + { + CallbackSync sync = new CallbackSync(connector.move_(from, to)).invoke(); + sync.checkException(); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public void delete(String path) throws ConnectorException + { + try + { + CallbackSync sync = new CallbackSync(connector.delete_(path)).invoke(); + sync.checkException(); + } + catch (Exception e) + { + throw new ConnectorException(e); + } + } + + @Override + public boolean ensureDirectories(String... folders) + { + CallbackSync sync = new CallbackSync(connector.ensureDirectories_(folders)).invoke(); + return sync.exportNoException(); + } +} diff --git a/java/core/src/core/constants/ConstantsMvStore.java b/java/core/src/core/constants/ConstantsMvStore.java new file mode 100644 index 0000000..2dc2674 --- /dev/null +++ b/java/core/src/core/constants/ConstantsMvStore.java @@ -0,0 +1,8 @@ +package core.constants; + +public class ConstantsMvStore +{ + public static final String AccessKeyId = "AccessKeyId"; + public static final String SecretKey = "SecretKey"; + +} diff --git a/java/core/src/core/constants/ConstantsStorage.java b/java/core/src/core/constants/ConstantsStorage.java index c831031..151f477 100644 --- a/java/core/src/core/constants/ConstantsStorage.java +++ b/java/core/src/core/constants/ConstantsStorage.java @@ -4,7 +4,8 @@ public class ConstantsStorage { public final static String HANDLER_DROPBOX = "DB", - HANDLER_S3 = "S3"; + HANDLER_S3 = "S3", + HANDLER_MV = "MV"; public final static String IN = "In", @@ -28,4 +29,5 @@ public class ConstantsStorage public static final int MAIL_CHECK_LOCK_TIME_SECONDS = 120; public static final int MAIL_CHECK_LOCK_TIME_ALLOWED_BEFORE_RELOCK_SECONDS = 60; + } diff --git a/java/core/src/core/util/HttpDelegateFactory.java b/java/core/src/core/util/HttpDelegateFactory.java new file mode 100644 index 0000000..f9216bb --- /dev/null +++ b/java/core/src/core/util/HttpDelegateFactory.java @@ -0,0 +1,9 @@ +package core.util; + +public class HttpDelegateFactory +{ + static public HttpDelegate create () + { + return new HttpDelegateJava(); + } +} diff --git a/java/core/src/core/util/SqlCatalog.java b/java/core/src/core/util/SqlCatalog.java index 1b74321..d6b32fa 100644 --- a/java/core/src/core/util/SqlCatalog.java +++ b/java/core/src/core/util/SqlCatalog.java @@ -10,6 +10,20 @@ import java.util.ArrayList; public class SqlCatalog { + static LogOut log = new LogOut(SqlCatalog.class); + public SqlCatalog () + { + try + { + Class.forName("com.mysql.jdbc.Driver"); + } + catch (Exception e) + { + log.exception(e); + throw new RuntimeException(e); + } + } + public String getSingle (String name) throws IOException { return Streams.readFullyString(getClass().getResourceAsStream(name), "UTF-8"); diff --git a/java/core/src/mail/client/CacheManager.java b/java/core/src/mail/client/CacheManager.java index 8a3932b..a84d371 100644 --- a/java/core/src/mail/client/CacheManager.java +++ b/java/core/src/mail/client/CacheManager.java @@ -36,6 +36,8 @@ import mail.client.model.Model; import mail.client.model.ModelFactory; import mail.client.model.Mail; import mail.client.model.ModelSerializer; +import mail.client.model.PublicKey; +import mail.client.model.PublicKeyRing; import mail.client.model.Settings; public class CacheManager extends Servent @@ -46,6 +48,9 @@ public class CacheManager extends Servent IndexedCache cacheMail; IndexedCache cacheConversation; IndexedCache cacheFolder; + IndexedCache cacheKeys; + + PublicKeyRing keyRing; Settings settings; boolean isCaching = false; @@ -84,6 +89,10 @@ public class CacheManager extends Servent this.settings.setId(Constants.SETTINGS_ID); this.masterCache.link(settings); + this.keyRing = new PublicKeyRing(this); + this.keyRing.setId(Constants.KEYRING_ID); + this.masterCache.link(keyRing); + ItemSerializer itemSerializer = new ModelSerializer(json); this.cacheMail = new IndexedCache( @@ -104,11 +113,17 @@ public class CacheManager extends Servent this.cacheFolder.setId(Constants.FOLDER_ID); masterCache.link(cacheFolder); + this.cacheKeys = new IndexedCache( + new ItemCacheFactory ("K", library, itemFactory, itemSerializer) + ); + this.cacheKeys.setId(Constants.KEY_ID); + masterCache.link(cacheKeys); + //------------------------------------------------------------- Callback countDown = new CountDown( - 4, + 5, getMaster().getEventPropagator().signal_(Events.Initialize_IndexedCacheLoadComplete) ); @@ -117,6 +132,7 @@ public class CacheManager extends Servent cacheMail.apply(new Split(countDown)); cacheConversation.apply(new Split(countDown)); cacheFolder.apply(new Split(countDown)); + cacheKeys.apply(new Split(countDown)); //------------------------------------------------------------- @@ -143,6 +159,7 @@ public class CacheManager extends Servent cacheMail.markCreate(); cacheConversation.markCreate(); cacheFolder.markCreate(); + cacheKeys.markCreate(); masterCache.markCreate(); settings.markCreate(); @@ -261,6 +278,16 @@ public class CacheManager extends Servent cacheFolder.put(f); } + public void putKey(PublicKey k) + { + cacheKeys.put(k); + } + + public PublicKey getKey (ID id) + { + return (PublicKey)cacheKeys.getAndAcquire(Type.PublicKey, id); + } + public boolean isFullyCached () { return (!library.hasDirtyChildren() && !masterCache.hasDirtyChildren()); diff --git a/java/core/src/mail/client/Client.java b/java/core/src/mail/client/Client.java index eddc36e..f62ee11 100644 --- a/java/core/src/mail/client/Client.java +++ b/java/core/src/mail/client/Client.java @@ -13,6 +13,8 @@ import core.connector.async.AsyncStoreConnectorBase64; import core.connector.async.AsyncStoreConnectorEncrypted; import core.connector.dropbox.ClientInfoDropbox; import core.connector.dropbox.async.ConnectorDropbox; +import core.connector.mv.ClientInfoMvStore; +import core.connector.mv.async.ConnectorMvStore; import core.connector.s3.ClientInfoS3; import core.connector.s3.async.S3Connector; import core.constants.ConstantsEnvironmentKeys; @@ -82,6 +84,13 @@ public class Client connector = new S3Connector(clientInfo, httpDelegate); } else + if (handler.equals(ConstantsStorage.HANDLER_MV)) + { + Environment mvEnvironment = mailBoxEnvironment.childEnvironment(handler); + ClientInfoMvStore clientInfo = new ClientInfoMvStore(mvEnvironment); + connector = new ConnectorMvStore(clientInfo, httpDelegate); + } + else { throw new Exception ("Unknown handler"); } diff --git a/java/core/src/mail/client/Constants.java b/java/core/src/mail/client/Constants.java index a6732ef..1c9af69 100644 --- a/java/core/src/mail/client/Constants.java +++ b/java/core/src/mail/client/Constants.java @@ -26,5 +26,7 @@ public class Constants SETTINGS_ID = ID.fromLong(1), MAIL_ID = ID.fromLong(2), CONVERSATION_ID = ID.fromLong(3), - FOLDER_ID = ID.fromLong(4); + FOLDER_ID = ID.fromLong(4), + KEYRING_ID = ID.fromLong(5), + KEY_ID = ID.fromLong(6); } diff --git a/java/core/src/mail/client/MasterCacheSerializer.java b/java/core/src/mail/client/MasterCacheSerializer.java index 38a9e9d..2287bff 100644 --- a/java/core/src/mail/client/MasterCacheSerializer.java +++ b/java/core/src/mail/client/MasterCacheSerializer.java @@ -28,7 +28,7 @@ public class MasterCacheSerializer implements ItemSerializer { if (item instanceof Settings) return settingsSerializer.serialize_(item); - + return indexedCacheSerializer.serialize_(item); } @Override diff --git a/java/core/src/mail/client/cache/JSON.java b/java/core/src/mail/client/cache/JSON.java index af8ece3..74fafb6 100644 --- a/java/core/src/mail/client/cache/JSON.java +++ b/java/core/src/mail/client/cache/JSON.java @@ -26,6 +26,8 @@ import mail.client.model.FolderSet; import mail.client.model.Header; import mail.client.model.Identity; import mail.client.model.Mail; +import mail.client.model.PublicKey; +import mail.client.model.PublicKeyRing; import mail.client.model.Recipients; import mail.client.model.Settings; import mail.client.model.TransportState; @@ -573,4 +575,45 @@ public class JSON extends Servent return o; } + + public void fromJSON(PublicKey item, Object parse) throws JSONException + { + item.setEmail(JSON_.getString(parse, "email")); + item.setPublicKey(JSON_.getString(parse, "publicKey")); + } + + public Object toJSON(PublicKey item) throws JSONException + { + Object o = JSON_.newObject(); + JSON_.put(o, "email", item.toString()); + JSON_.put(o, "publicKey", item.getPublicKey()); + + return o; + } + + public void fromJSON(PublicKeyRing item, Object a) throws JSONException + { + for (int i=0; i iNs : item.getPublicKeys()) + { + Object o = JSON_.newObject(); + JSON_.put(o, "id", toJSON(iNs.first)); + JSON_.put(o, "identity", toJSON(iNs.second)); + } + + return a; + } + } diff --git a/java/core/src/mail/client/cache/Type.java b/java/core/src/mail/client/cache/Type.java index 3d0d7ad..8e5c85b 100644 --- a/java/core/src/mail/client/cache/Type.java +++ b/java/core/src/mail/client/cache/Type.java @@ -13,5 +13,7 @@ public enum Type { FolderFilterSet, FolderMaster, Index, - Settings + Settings, + PublicKeyRing, + PublicKey, } diff --git a/java/core/src/mail/client/model/Identity.java b/java/core/src/mail/client/model/Identity.java index 8f46a21..74bf655 100644 --- a/java/core/src/mail/client/model/Identity.java +++ b/java/core/src/mail/client/model/Identity.java @@ -24,7 +24,6 @@ public class Identity implements Serializable, Exportable String name; String email; boolean isPrimary = false; - String publicKey; protected Identity () { @@ -169,19 +168,4 @@ public class Identity implements Serializable, Exportable this.email = identity.email; } } - - public void setPublicKey (String publicKey) - { - this.publicKey = publicKey; - } - - public boolean hasPublicKey() - { - return publicKey != null; - } - - public String getPublicKey() - { - return publicKey; - } } diff --git a/java/core/src/mail/client/model/Mail.java b/java/core/src/mail/client/model/Mail.java index 5edad38..be63ab4 100644 --- a/java/core/src/mail/client/model/Mail.java +++ b/java/core/src/mail/client/model/Mail.java @@ -124,7 +124,8 @@ public class Mail extends Model { this.attachments = attachments; } - + +/* public boolean isPresendEncryptable() { for (Identity i : getHeader().getRecipients().getAll()) @@ -191,4 +192,5 @@ public class Mail extends Model Base64.encode(aes.encrypt(Strings.toBytes(JSON_.asString(container)))) ); } +*/ } diff --git a/java/core/src/mail/client/model/ModelSerializer.java b/java/core/src/mail/client/model/ModelSerializer.java index 15060a5..92367eb 100644 --- a/java/core/src/mail/client/model/ModelSerializer.java +++ b/java/core/src/mail/client/model/ModelSerializer.java @@ -40,6 +40,8 @@ public class ModelSerializer implements ItemSerializer return Strings.toBytes(json.toJSON((Folder)item).toString()); if (item instanceof Settings) return Strings.toBytes(json.toJSON((Settings)item).toString()); + if (item instanceof PublicKey) + return Strings.toBytes(json.toJSON((PublicKey)item).toString()); return null; } @@ -58,6 +60,8 @@ public class ModelSerializer implements ItemSerializer json.fromJSON((Folder)item, JSON_.parse(Strings.toString(bytes))); if (item instanceof Settings) json.fromJSON((Settings)item, JSON_.parse(Strings.toString(bytes))); + if (item instanceof PublicKey) + json.fromJSON((PublicKey)item, JSON_.parse(Strings.toString(bytes))); } @Override diff --git a/java/core/src/mail/client/model/PublicKey.java b/java/core/src/mail/client/model/PublicKey.java new file mode 100644 index 0000000..3f05cf1 --- /dev/null +++ b/java/core/src/mail/client/model/PublicKey.java @@ -0,0 +1,35 @@ +package mail.client.model; + +import mail.client.CacheManager; + +public class PublicKey extends Model +{ + String email; + String publicKey; + + public PublicKey(CacheManager manager) + { + super(manager); + } + + public void setEmail (String email) + { + this.email = email; + } + + public String getEmail (String email) + { + return email; + } + + public void setPublicKey (String publicKey) + { + this.publicKey = publicKey; + } + + public String getPublicKey () + { + return publicKey; + } + +} diff --git a/java/core/src/mail/client/model/PublicKeyRing.java b/java/core/src/mail/client/model/PublicKeyRing.java new file mode 100644 index 0000000..f396eea --- /dev/null +++ b/java/core/src/mail/client/model/PublicKeyRing.java @@ -0,0 +1,34 @@ +package mail.client.model; + +import java.util.List; + +import core.util.Pair; +import mail.client.CacheManager; +import mail.client.cache.ID; + +public class PublicKeyRing extends Model +{ + List> ring; + + public PublicKeyRing(CacheManager manager) + { + super(manager); + } + + public void addPublicKeyFromCache(ID id, Identity identity) + { + ring.add(Pair.create(id, identity)); + } + + public void addPublicKey(ID id, Identity identity) + { + ring.add(Pair.create(id, identity)); + markDirty(); + } + + public List> getPublicKeys() + { + return ring; + } + +} diff --git a/java/core/src/mail/server/handler/MailetHandlerMvStore.java b/java/core/src/mail/server/handler/MailetHandlerMvStore.java new file mode 100644 index 0000000..4a5c076 --- /dev/null +++ b/java/core/src/mail/server/handler/MailetHandlerMvStore.java @@ -0,0 +1,32 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ +package mail.server.handler; + + +import core.connector.mv.ClientInfoMvStore; +import core.connector.mv.sync.ConnectorMvStore; +import core.connector.sync.EncryptedStoreConnector; +import core.connector.sync.StoreConnector; +import core.constants.ConstantsStorage; +import core.crypt.CryptorRSAAES; +import core.crypt.CryptorRSAFactoryEnvironment; +import core.util.Environment; +import core.util.LogOut; + + +public class MailetHandlerMvStore extends MailetHandlerDefault +{ + static LogOut log = new LogOut(MailetHandlerMvStore.class); + + public StoreConnector createConnector (Environment e) throws Exception + { + log.debug("createConnector"); + + return new EncryptedStoreConnector( + new CryptorRSAAES(CryptorRSAFactoryEnvironment.create(e)), + new ConnectorMvStore(new ClientInfoMvStore(e.childEnvironment(ConstantsStorage.HANDLER_MV))) + ); + } +} diff --git a/java/core/src/mail/server/handler/UserInformationFactory.java b/java/core/src/mail/server/handler/UserInformationFactory.java index 8bc9236..fa166a0 100644 --- a/java/core/src/mail/server/handler/UserInformationFactory.java +++ b/java/core/src/mail/server/handler/UserInformationFactory.java @@ -49,6 +49,9 @@ public class UserInformationFactory else if (handlerName.equals(ConstantsStorage.HANDLER_DROPBOX)) mailetHandler = new MailetHandlerDropbox(); + else + if (handlerName.equals(ConstantsStorage.HANDLER_MV)) + mailetHandler = new MailetHandlerMvStore(); else throw new Exception("Unknown handler"); diff --git a/java/core/src/store/server/ConstantsMvServer.java b/java/core/src/store/server/ConstantsMvServer.java new file mode 100644 index 0000000..fcc344f --- /dev/null +++ b/java/core/src/store/server/ConstantsMvServer.java @@ -0,0 +1,13 @@ +package store.server; + +public class ConstantsMvServer { + + public static final String + HEADER_DATE = "X-Mv-Date", + HEADER_CONTENT_TYPE = "Content-Type", + HEADER_CONTENT_LENGTH = "Content-Length", + HEADER_AUTHORIZATION = "Authorization", + + PARAMETER_RESOURCE = "Resource"; + +} diff --git a/java/core/src/store/server/StoreServer.java b/java/core/src/store/server/StoreServer.java new file mode 100644 index 0000000..cc519f0 --- /dev/null +++ b/java/core/src/store/server/StoreServer.java @@ -0,0 +1,64 @@ +package store.server; + +import java.util.List; + +import store.server.db.DbStore; +import core.connector.FileInfo; +import core.util.Pair; + +public class StoreServer +{ + DbStore store; + + public StoreServer () throws Exception + { + store = new DbStore(); + store.ensureTables(); + } + + public Pair newKeyPair (String userName) throws Exception + { + Pair v = StoreUtils.createKeyIdAndSecretKey(); + DbStore store = new DbStore(); + int userId = store.getUserId(userName); + store.addUserKeyPair(userId, v.first, v.second); + + return v; + } + + public void newUser (String userName) throws Exception + { + store.addUser(userName); + } + + public Pair newUserWithKeyPair (String userName) throws Exception + { + newUser (userName); + return newKeyPair(userName); + } + + public String putKeyValue (String keyId, String key, byte[] value) throws Exception + { + return store.putKeyValue(store.getUserIdAndSecretKey(keyId).first, key, value); + } + + public Pair getKeyValue (String keyId, String key) throws Exception + { + return store.getKeyValue(store.getUserIdAndSecretKey(keyId).first, key); + } + + public void removeKeyValue (String keyId, String key) throws Exception + { + store.removeKeyValue (store.getUserIdAndSecretKey(keyId).first, key); + } + + public void removeUser (String name) throws Exception + { + store.removeUser (name); + } + + public List listKeys (String keyId, String key) throws Exception + { + return store.listKeys(store.getUserIdAndSecretKey(keyId).first, key); + } +} diff --git a/java/core/src/store/server/StoreUtils.java b/java/core/src/store/server/StoreUtils.java new file mode 100644 index 0000000..e7fc4e5 --- /dev/null +++ b/java/core/src/store/server/StoreUtils.java @@ -0,0 +1,142 @@ +package store.server; + +import java.util.Map; + +import store.server.db.DbStore; + +import core.crypt.HmacSha1; +import core.util.Base64; +import core.util.LogOut; +import core.util.Pair; +import core.util.SecureRandom; +import core.util.Strings; + +public class StoreUtils +{ + static LogOut log = new LogOut(StoreUtils.class); + + // This method creates S3 signature for a given String. + static protected String sign(HmacSha1 mac, String data) throws Exception + { + // Signed String must be BASE64 encoded. + byte[] signBytes = mac.mac(Strings.toBytes(data)); + String signature = Base64.encode(signBytes); + return signature; + } + + static public Pair createKeyIdAndSecretKey () + { + byte[] keyId = new byte[32]; + byte[] secretKey = new byte[32]; + SecureRandom random = new SecureRandom(); + random.nextBytes(keyId); + random.nextBytes(secretKey); + + return Pair.create(Base64.encode(keyId), Base64.encode(secretKey)); + } + + static public void verifySignature (String data, String keyId, String signature) throws Exception + { + DbStore dbStore = new DbStore(); + Pair userIdAndSecretKey = dbStore.getUserIdAndSecretKey(keyId); + + log.debug ("verifySignature", data, userIdAndSecretKey.second); + + HmacSha1 userMac = new HmacSha1(Base64.decode(userIdAndSecretKey.second)); + String correctSignature = sign(userMac, data); + + if (!correctSignature.equals(signature)) + throw new Exception ("Signature mismatch"); + } + + /** + * returns the keyId of the user + * + * @param action + * @param headers + * @param resource + * @param content + * @return + * @throws Exception + */ + static public String verifyUser (String action, Map headers, String resource, byte[] content) throws Exception + { + String method = action; + String dateString = headers.get(ConstantsMvServer.HEADER_DATE); + String contentType = headers.get(ConstantsMvServer.HEADER_CONTENT_TYPE); + String contentLength = headers.get(ConstantsMvServer.HEADER_CONTENT_LENGTH); + + if (contentLength != null) + if (Integer.parseInt(contentLength) != content.length) + throw new Exception ("Length mismatch"); + + StringBuffer buf = new StringBuffer(); + buf.append(method).append("\n"); + buf.append("").append("\n"); + buf.append(contentType != null ? contentType : "").append("\n"); + buf.append("\n"); // empty real date header + buf.append(dateString).append("\n"); + buf.append(resource); + + String[] authorizationParts = headers.get(ConstantsMvServer.HEADER_AUTHORIZATION).split(":"); + String keyId = authorizationParts[0].split(" ")[1]; + String signature = authorizationParts[1]; + + verifySignature (buf.toString(), keyId, signature); + + return keyId; + } + +/* + public byte[] getFile (String userPath) throws Exception + { + FileInputStream f = new FileInputStream(ConstantsMvServer.PATH + userPath); + byte[] result = Streams.readFullyBytes(f); + f.close(); + + return result; + } + + public void putFile (String userPath, byte[] bytes) throws Exception + { + FileOutputStream f = new FileOutputStream (ConstantsMvServer.PATH + userPath); + f.write(bytes); + f.close(); + } + + private List listFiles(File dir, List files) + { + if (!dir.isDirectory()) + { + files.add(dir); + return files; + } + + for (File file : dir.listFiles()) + listFiles(file, files); + + return files; + } + + public List listKeys (String userPath) + { + List files = listFiles (new File (ConstantsMvServer.PATH + userPath), new ArrayList()); + + List fileInfos = new ArrayList(); + for (File file : files) + { + FileInfo info = new FileInfo( + file.getPath(), + file.getPath(), + file.length(), + new Date(file.lastModified()), + "" + file.lastModified() + ); + + fileInfos.add(info); + } + + return fileInfos; + } +*/ +} diff --git a/java/core/src/store/server/db/DbStore.java b/java/core/src/store/server/db/DbStore.java new file mode 100644 index 0000000..6d3fad5 --- /dev/null +++ b/java/core/src/store/server/db/DbStore.java @@ -0,0 +1,304 @@ +package store.server.db; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import core.connector.FileInfo; +import core.util.Base64; +import core.util.FastRandom; +import core.util.LogOut; +import core.util.Pair; +import core.util.Passwords; +import store.server.db.sql.Catalog; + +public class DbStore +{ + FastRandom random = new FastRandom(); + LogOut log = new LogOut(DbStore.class); + Catalog catalog = new Catalog(); + + public void ensureTables() throws Exception + { + Connection connection = openConnection(); + + try + { + for (String sql : catalog.getMulti(Catalog.ENSURE_TABLES)) + { + PreparedStatement statement = connection.prepareStatement (sql); + log(statement); + statement.executeUpdate(); + } + } + finally + { + closeConnection(connection); + } + } + + public String putKeyValue (int userId, String key, byte[] value) throws Exception + { + Connection connection = null; + + try + { + connection = openConnection(); + String sql = catalog.getSingle(Catalog.PUT_KEY_VALUE); + PreparedStatement statement = connection.prepareStatement(sql); + statement.setInt(1, userId); + statement.setString(2, key); + statement.setBytes(3, value); + + byte[] randomBytes = new byte[16]; + random.nextBytes(randomBytes); + String randomB64 = Base64.encode(randomBytes); + + statement.setString(4, randomB64); + statement.execute(); + + return randomB64; + } + finally + { + closeConnection (connection); + } + } + + public Pair getKeyValue (int userId, String key) throws Exception + { + Connection connection = null; + + try + { + connection = openConnection(); + String sql = catalog.getSingle(Catalog.GET_KEY_VALUE); + PreparedStatement statement = connection.prepareStatement(sql); + statement.setInt(1, userId); + statement.setString(2, key); + + ResultSet results = statement.executeQuery(); + if (results.next()) + { + return Pair.create(results.getString("version"), results.getBytes("v")); + } + + results.close(); + } + finally + { + closeConnection (connection); + } + + throw new Exception("Unknown key"); + } + + public List listKeys (int userId, String key) throws Exception + { + Connection connection = null; + + try + { + connection = openConnection(); + String sql = catalog.getSingle(Catalog.LIST_KEY_VALUES); + PreparedStatement statement = connection.prepareStatement(sql); + statement.setInt(1, userId); + statement.setString(2, key); + + List retval = new ArrayList(); + ResultSet results = statement.executeQuery(); + while (results.next()) + { + String path = results.getString("k"); + int size = results.getInt("size"); + Date date = results.getTimestamp("mark"); + String version = results.getString("version"); + + FileInfo fileInfo = new FileInfo(path, path.substring(key.length()), size, date, version); + retval.add(fileInfo); + } + + results.close(); + + return retval; + } + finally + { + closeConnection (connection); + } + } + + public Pair getUserIdAndSecretKey (String keyId) throws Exception + { + Connection connection = null; + + try + { + connection = openConnection(); + String sql = catalog.getSingle(Catalog.GET_USER_ID_AND_SECRET_KEY); + PreparedStatement statement = connection.prepareStatement(sql); + statement.setString(1, keyId); + + ResultSet results = statement.executeQuery(); + if (results.next()) + { + return Pair.create(results.getInt("user_id"), results.getString("secret_key")); + } + + results.close(); + } + finally + { + closeConnection (connection); + } + + throw new Exception("Unknown KeyId"); + } + + public void addUser (String userName) throws Exception + { + Connection connection = null; + + try + { + connection = openConnection(); + String sql = catalog.getSingle(Catalog.ADD_USER); + PreparedStatement statement = connection.prepareStatement(sql); + statement.setString(1, userName); + + statement.execute(); + } + finally + { + closeConnection (connection); + } + } + + public void removeUser (String userName) throws Exception + { + Connection connection = null; + + try + { + connection = openConnection(); + String[] sqls = catalog.getMulti(Catalog.REMOVE_USER); + + boolean first = true; + for (String sql : sqls) + { + PreparedStatement statement = connection.prepareStatement(sql); + if (first) + statement.setString(1, userName); + + statement.execute(); + first = false; + } + } + finally + { + closeConnection (connection); + } + } + + public void removeKeyValue(int userId, String key) throws Exception + { + Connection connection = null; + + try + { + connection = openConnection(); + String sql = catalog.getSingle(Catalog.REMOVE_KEY_VALUE); + PreparedStatement statement = connection.prepareStatement(sql); + statement.setInt(1, userId); + statement.setString(2, key); + + statement.execute(); + } + finally + { + closeConnection (connection); + } + } + + public int getUserId(String userName) throws Exception + { + Connection connection = null; + + try + { + connection = openConnection(); + String sql = catalog.getSingle(Catalog.GET_USER_ID); + PreparedStatement statement = connection.prepareStatement(sql); + statement.setString(1, userName); + + ResultSet results = statement.executeQuery(); + if (results.next()) + { + return results.getInt("user_id"); + } + + results.close(); + } + finally + { + closeConnection (connection); + } + + throw new Exception("Unknown user name"); + } + + + public void addUserKeyPair (int userId, String keyId, String secretKey) throws Exception + { + Connection connection = null; + + try + { + connection = openConnection(); + String sql = catalog.getSingle(Catalog.ADD_USER_KEY_PAIR); + PreparedStatement statement = connection.prepareStatement(sql); + statement.setInt(1, userId); + statement.setString(2, keyId); + statement.setString(3, secretKey); + + statement.execute(); + } + finally + { + closeConnection (connection); + } + + } + + public Connection openConnection () throws IOException, SQLException + { + log.debug("Connecting to", catalog.CONNECTION_STRING); + return DriverManager.getConnection(catalog.CONNECTION_STRING, catalog.USER, Passwords.getPasswordFor(catalog.USER)); + } + + public void closeConnection (Connection connection) + { + try + { + if (connection != null) + connection.close(); + } + catch (Exception e) + { + e.printStackTrace(); + } + + } + public void log (Statement sql) + { + log.debug (sql); + } + +} diff --git a/java/core/src/store/server/db/sql/Catalog.java b/java/core/src/store/server/db/sql/Catalog.java new file mode 100644 index 0000000..4b0ad43 --- /dev/null +++ b/java/core/src/store/server/db/sql/Catalog.java @@ -0,0 +1,28 @@ +/** + * Author: Timothy Prepscius + * License: GPLv3 Affero + keep my name in the code! + */ +package store.server.db.sql; + +import core.constants.ConstantsServer; +import core.util.SqlCatalog; + +public class Catalog extends SqlCatalog +{ + public static final String + ENSURE_TABLES = "ensure_tables.sql", + + ADD_USER = "add_user.sql", + ADD_USER_KEY_PAIR = "add_user_key_pair.sql", + GET_USER_ID = "get_user_id.sql", + REMOVE_USER = "remove_user.sql", + GET_USER_ID_AND_SECRET_KEY = "get_user_id_and_secret_key.sql", + + LIST_KEY_VALUES = "list_keys.sql", + REMOVE_KEY_VALUE = "remove_key_value.sql", + PUT_KEY_VALUE = "put_key_value.sql", + GET_KEY_VALUE = "get_key_value.sql"; + + public String CONNECTION_STRING = ConstantsServer.DBCONNECTION_PREFIX + "store"; + public String USER = "store"; +} diff --git a/java/core/src/store/server/db/sql/add_user.sql b/java/core/src/store/server/db/sql/add_user.sql new file mode 100644 index 0000000..f374ebc --- /dev/null +++ b/java/core/src/store/server/db/sql/add_user.sql @@ -0,0 +1 @@ +INSERT INTO users (name) VALUES (?) \ No newline at end of file diff --git a/java/core/src/store/server/db/sql/add_user_key_pair.sql b/java/core/src/store/server/db/sql/add_user_key_pair.sql new file mode 100644 index 0000000..d5367d8 --- /dev/null +++ b/java/core/src/store/server/db/sql/add_user_key_pair.sql @@ -0,0 +1 @@ +INSERT INTO key_pairs (user_id, key_id, secret_key) VALUES (?,?,?); \ No newline at end of file diff --git a/java/core/src/store/server/db/sql/ensure_tables.sql b/java/core/src/store/server/db/sql/ensure_tables.sql new file mode 100644 index 0000000..b52e6df --- /dev/null +++ b/java/core/src/store/server/db/sql/ensure_tables.sql @@ -0,0 +1,24 @@ +CREATE TABLE IF NOT EXISTS users ( + name VARCHAR(255) NOT NULL, + user_id INTEGER AUTO_INCREMENT NOT NULL, + PRIMARY KEY (user_id), + KEY (name) +); + +CREATE TABLE IF NOT EXISTS key_pairs ( + user_id INTEGER NOT NULL, + key_id VARCHAR(255) NOT NULL, + secret_key VARCHAR(255) NOT NULL, + mark TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (user_id, key_id), + KEY (key_id) +); + +CREATE TABLE IF NOT EXISTS key_values ( + user_id INTEGER NOT NULL, + k VARCHAR(2048), + v LONGBLOB, + mark TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + version VARCHAR(255) NOT NULL, + PRIMARY KEY (user_id, k(128)) +); \ No newline at end of file diff --git a/java/core/src/store/server/db/sql/get_key_value.sql b/java/core/src/store/server/db/sql/get_key_value.sql new file mode 100644 index 0000000..051e3ab --- /dev/null +++ b/java/core/src/store/server/db/sql/get_key_value.sql @@ -0,0 +1 @@ +SELECT version, v FROM key_values WHERE user_id = ? AND k = ? \ No newline at end of file diff --git a/java/core/src/store/server/db/sql/get_user_id.sql b/java/core/src/store/server/db/sql/get_user_id.sql new file mode 100644 index 0000000..0a58814 --- /dev/null +++ b/java/core/src/store/server/db/sql/get_user_id.sql @@ -0,0 +1 @@ +SELECT user_id FROM users WHERE name = ? \ No newline at end of file diff --git a/java/core/src/store/server/db/sql/get_user_id_and_secret_key.sql b/java/core/src/store/server/db/sql/get_user_id_and_secret_key.sql new file mode 100644 index 0000000..12768ea --- /dev/null +++ b/java/core/src/store/server/db/sql/get_user_id_and_secret_key.sql @@ -0,0 +1 @@ +SELECT user_id, secret_key FROM key_pairs WHERE key_id = ? \ No newline at end of file diff --git a/java/core/src/store/server/db/sql/list_keys.sql b/java/core/src/store/server/db/sql/list_keys.sql new file mode 100644 index 0000000..d2f4a31 --- /dev/null +++ b/java/core/src/store/server/db/sql/list_keys.sql @@ -0,0 +1,13 @@ +# this is really lame and I know it +# but I need to avoid the LIKE clause, until I have time to figure out +# where the function is for proper mysql escaping. + +SELECT + k, + mark, + version, + LENGTH(v) as size +FROM + key_values +WHERE + user_id = ? AND LOCATE(?,k)=1 diff --git a/java/core/src/store/server/db/sql/put_key_value.sql b/java/core/src/store/server/db/sql/put_key_value.sql new file mode 100644 index 0000000..b9cea1a --- /dev/null +++ b/java/core/src/store/server/db/sql/put_key_value.sql @@ -0,0 +1 @@ +REPLACE INTO key_values (user_id,k,v,version) VALUES (?,?,?,?) \ No newline at end of file diff --git a/java/core/src/store/server/db/sql/remove_key_value.sql b/java/core/src/store/server/db/sql/remove_key_value.sql new file mode 100644 index 0000000..41f0ed1 --- /dev/null +++ b/java/core/src/store/server/db/sql/remove_key_value.sql @@ -0,0 +1 @@ +DELETE FROM key_value WHERE user_id = ? AND k = ? \ No newline at end of file diff --git a/java/core/src/store/server/db/sql/remove_user.sql b/java/core/src/store/server/db/sql/remove_user.sql new file mode 100644 index 0000000..e69de29 diff --git a/java/webserver/src/core/constants/ConstantsMvStore.java b/java/webserver/src/core/constants/ConstantsMvStore.java new file mode 120000 index 0000000..7860884 --- /dev/null +++ b/java/webserver/src/core/constants/ConstantsMvStore.java @@ -0,0 +1 @@ +../../../../core/src/core/constants/ConstantsMvStore.java \ No newline at end of file diff --git a/java/webserver/src/mail/web/StoreEnable.java b/java/webserver/src/mail/web/StoreEnable.java new file mode 100644 index 0000000..da2b713 --- /dev/null +++ b/java/webserver/src/mail/web/StoreEnable.java @@ -0,0 +1,105 @@ +package mail.web; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Map; + +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import core.constants.ConstantsMvStore; +import core.server.captcha.Captcha; +import core.util.LogOut; +import core.util.Pair; +import core.util.Strings; + +import store.server.StoreServer; + +/** + * Servlet implementation class StoreEnable + */ +@WebServlet("/StoreEnable") +public class StoreEnable extends HttpServlet +{ + static LogOut log = new LogOut(StoreEnable.class); + private static final long serialVersionUID = 1L; + + StoreServer store; + Captcha captcha; + + /** + * @see HttpServlet#HttpServlet() + */ + public StoreEnable() throws Exception + { + super(); + + try + { + store = new StoreServer(); + captcha = new Captcha(); + captcha.ensureTables(); + } + catch (Exception e) + { + log.exception(e); + throw e; + } + } + + void doCors(HttpServletResponse response) + { + response.setHeader("Access-Control-Allow-Origin", ConstantsWeb.WEB_SERVER_URL); + response.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS"); + response.setHeader("Access-Control-Allow-Headers", "X-Requested-With"); + } + + /** + * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) + */ + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + { + doCors(response); + + try + { + String email = request.getParameter("email"); + String captchaToken = request.getParameter("captcha"); + + log.debug("email",email, "captchaToken",captchaToken); + + captcha.useToken(captchaToken, Captcha.CreateBucket); + Pair result = store.newUserWithKeyPair(email); + + ArrayList keyValues = new ArrayList(); + keyValues.add(ConstantsMvStore.AccessKeyId + "!" + result.first); + keyValues.add(ConstantsMvStore.SecretKey + "!" + result.second); + + response.getWriter().write(Strings.concat(keyValues, "&")); + } + catch (Exception e) + { + log.exception(e); + throw new ServletException(e); + } + } + + /** + * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response) + */ + protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + { + doGet(request, response); + } + + @Override + protected void doOptions (HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + { + doCors(response); + super.doOptions(request, response); + } + +} diff --git a/java/webserver/src/mail/web/StoreGet.java b/java/webserver/src/mail/web/StoreGet.java new file mode 100644 index 0000000..0b09cbd --- /dev/null +++ b/java/webserver/src/mail/web/StoreGet.java @@ -0,0 +1,88 @@ +package mail.web; + +import java.io.IOException; +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import store.server.ConstantsMvServer; +import store.server.StoreServer; + +import core.util.Base64; +import core.util.HttpDelegate; +import core.util.LogOut; +import core.util.Pair; +import core.util.Streams; + +/** + * Servlet implementation class StoreGet + */ +@WebServlet("/StoreGet") +public class StoreGet extends HttpServlet +{ + static LogOut log = new LogOut(StoreGet.class); + private static final long serialVersionUID = 1L; + + StoreServer store; + + /** + * @see HttpServlet#HttpServlet() + */ + public StoreGet() throws Exception + { + super(); + try + { + store = new StoreServer(); + } + catch (Exception e) + { + log.exception(e); + throw e; + } + } + + void doCors(HttpServletResponse response) + { + response.setHeader("Access-Control-Allow-Origin", ConstantsWeb.WEB_SERVER_URL); + response.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS"); + response.setHeader("Access-Control-Allow-Headers", "X-Requested-With"); + } + + protected void doRequest (String action, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + { + try + { + doCors(response); + + String resource = request.getParameter(ConstantsMvServer.PARAMETER_RESOURCE); + String keyId = StoreWebUtils.verifyUser(action, request, resource, null); + + Pair versionAndData = store.getKeyValue(keyId, resource); + response.setHeader("Version", versionAndData.first); + response.getOutputStream().write(versionAndData.second); + } + catch (Exception e) + { + log.exception(e); + throw new ServletException(e); + } + } + + /** + * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) + */ + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + { + doRequest(HttpDelegate.GET, request, response); + } + + @Override + protected void doOptions (HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + { + doCors(response); + super.doOptions(request, response); + } +} diff --git a/java/webserver/src/mail/web/StoreList.java b/java/webserver/src/mail/web/StoreList.java new file mode 100644 index 0000000..7b125f8 --- /dev/null +++ b/java/webserver/src/mail/web/StoreList.java @@ -0,0 +1,83 @@ +package mail.web; + +import java.io.IOException; +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import store.server.ConstantsMvServer; +import store.server.StoreServer; +import core.util.HttpDelegate; +import core.util.LogOut; + +/** + * Servlet implementation class StoreList + */ +@WebServlet("/StoreList") +public class StoreList extends HttpServlet { + private static final long serialVersionUID = 1L; + + static LogOut log = new LogOut(StoreList.class); + StoreServer store; + + /** + * @see HttpServlet#HttpServlet() + */ + public StoreList() throws Exception + { + super(); + + try + { + store = new StoreServer(); + } + catch (Exception e) + { + log.exception(e); + throw e; + } + } + + void doCors(HttpServletResponse response) + { + response.setHeader("Access-Control-Allow-Origin", ConstantsWeb.WEB_SERVER_URL); + response.setHeader("Access-Control-Allow-Methods", "GET, OPTIONS"); + response.setHeader("Access-Control-Allow-Headers", "X-Requested-With"); + } + + protected void doRequest (String action, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + { + try + { + doCors(response); + + String resource = request.getParameter(ConstantsMvServer.PARAMETER_RESOURCE); + String keyId = StoreWebUtils.verifyUser(action, request, resource, null); + + String json = StoreWebUtils.transformFileInfoListToJson(store.listKeys(keyId, resource)); + response.getOutputStream().write(json.getBytes("UTF-8")); + } + catch (Exception e) + { + log.exception(e); + throw new ServletException(e); + } + } + + /** + * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) + */ + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + { + doRequest(HttpDelegate.GET, request, response); + } + + @Override + protected void doOptions (HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + { + doCors(response); + super.doOptions(request, response); + } +} diff --git a/java/webserver/src/mail/web/StorePut.java b/java/webserver/src/mail/web/StorePut.java new file mode 100644 index 0000000..de1c4f6 --- /dev/null +++ b/java/webserver/src/mail/web/StorePut.java @@ -0,0 +1,85 @@ +package mail.web; + +import java.io.IOException; +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import store.server.ConstantsMvServer; +import store.server.StoreServer; +import core.util.HttpDelegate; +import core.util.LogOut; +import core.util.Streams; + +/** + * Servlet implementation class StorePut + */ +@WebServlet("/StorePut") +public class StorePut extends HttpServlet { + private static final long serialVersionUID = 1L; + + static LogOut log = new LogOut(StorePut.class); + StoreServer store; + + /** + * @see HttpServlet#HttpServlet() + */ + public StorePut() throws Exception + { + super(); + + try + { + store = new StoreServer(); + } + catch (Exception e) + { + log.exception(e); + throw e; + } + } + + void doCors(HttpServletResponse response) + { + response.setHeader("Access-Control-Allow-Origin", ConstantsWeb.WEB_SERVER_URL); + response.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS"); + response.setHeader("Access-Control-Allow-Headers", "X-Requested-With"); + } + + protected void doRequest (String action, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + { + try + { + doCors(response); + + byte[] content = Streams.readFullyBytes(request.getInputStream()); + String resource = request.getParameter(ConstantsMvServer.PARAMETER_RESOURCE); + String keyId = StoreWebUtils.verifyUser(action, request, resource, content); + + String version = store.putKeyValue(keyId, resource, content); + response.setHeader("Version", version); + } + catch (Exception e) + { + log.exception(e); + throw new ServletException(e); + } + } + + /** + * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response) + */ + protected void doPut(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + { + doRequest(HttpDelegate.PUT, request, response); + } + + @Override + protected void doOptions (HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException + { + doCors(response); + super.doOptions(request, response); + } +} diff --git a/java/webserver/src/mail/web/StoreWebUtils.java b/java/webserver/src/mail/web/StoreWebUtils.java new file mode 100644 index 0000000..fdc627f --- /dev/null +++ b/java/webserver/src/mail/web/StoreWebUtils.java @@ -0,0 +1,67 @@ +package mail.web; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONArray; +import org.json.JSONObject; + +import core.connector.FileInfo; +import core.util.DateFormat; +import core.util.LogOut; + +import store.server.ConstantsMvServer; +import store.server.StoreUtils; + +public class StoreWebUtils +{ + static LogOut log = new LogOut(StoreWebUtils.class); + + public static String verifyUser (String action, HttpServletRequest request, String resource, byte[] content) throws Exception + { + String possibleHeaders[] = { + ConstantsMvServer.HEADER_DATE, + ConstantsMvServer.HEADER_CONTENT_TYPE, + ConstantsMvServer.HEADER_CONTENT_LENGTH, + ConstantsMvServer.HEADER_AUTHORIZATION + }; + + Map headers = new HashMap(); + for (String possibleHeader : possibleHeaders) + { + String value = request.getHeader(possibleHeader); + if (value != null) + headers.put(possibleHeader, value); + } + + return StoreUtils.verifyUser(action, headers, resource, content); + } + + static public String transformFileInfoListToJson (List fileInfos) throws Exception + { + DateFormat dateTimeFormat = new DateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z GMT'"); + + JSONArray a = new JSONArray(); + for (FileInfo fileInfo : fileInfos) + { + JSONObject o = new JSONObject(); + o.put("path", fileInfo.path); + o.put("size", fileInfo.size); + o.put("version", fileInfo.version); + o.put("date", dateTimeFormat.format(fileInfo.date)); + + log.debug ("converted date", fileInfo.date, "to", o.get("date")); + + a.put(o); + } + + JSONObject r = new JSONObject(); + r.put("contents", a); + r.put("isTruncated", false); + + return r.toString(); + } +} diff --git a/java/webserver/src/store b/java/webserver/src/store new file mode 120000 index 0000000..32011c5 --- /dev/null +++ b/java/webserver/src/store @@ -0,0 +1 @@ +../../core/src/store \ No newline at end of file diff --git a/keys/nginx/convert-authority b/keys/nginx/convert-authority index b02efdb..2cd0239 100755 --- a/keys/nginx/convert-authority +++ b/keys/nginx/convert-authority @@ -8,4 +8,6 @@ openssl pkcs12 -in store.p12 -nocerts -out store.key openssl rsa -in store.key -out final.key rm final.crt -cat authority-response/* >> final.crt +cat authority-response/server.crt >> final.crt +cat authority-response/PositiveSSLCA2.crt >> final.crt +cat authority-response/AddTrustExternalCARoot.crt >> final.crt diff --git a/passwords/make b/passwords/make index 4ffa879..e350038 100755 --- a/passwords/make +++ b/passwords/make @@ -4,5 +4,6 @@ echo `./make-single-password` > mail echo `./make-single-password` > mail-pbe echo `./make-single-password` > mail_extra echo `./make-single-password` > push-certificate +echo `./make-single-password` > store diff --git a/passwords/store b/passwords/store new file mode 100644 index 0000000..127b2b7 --- /dev/null +++ b/passwords/store @@ -0,0 +1 @@ +PASSWORD FOR STORE DB USER diff --git a/web/WebContent/js/mDispatch.js b/web/WebContent/js/mDispatch.js index a0dcc24..fdcf413 100644 --- a/web/WebContent/js/mDispatch.js +++ b/web/WebContent/js/mDispatch.js @@ -154,4 +154,4 @@ mDispatch = { }, }; -$(document).ready(setTimeout(function() { mDispatch.startWorker(); }, 250)); +// $(document).ready(setTimeout(function() { mDispatch.startWorker(); }, 250)); diff --git a/web/common/Delegate.js b/web/common/Delegate.js index 6c82fa9..25d4b9a 100644 --- a/web/common/Delegate.js +++ b/web/common/Delegate.js @@ -103,7 +103,7 @@ var mDelegateCommon = log("finishing url ", url, " ", this.status); var getResponseHeadersHack = function(xhr) { - var headers = ['ETag','Content-Type', 'x-dropbox-metadata']; + var headers = ['ETag','Content-Type', 'x-dropbox-metadata', 'Version']; var result = []; for (var i=0; i
I want to use the storage provider:
- - +
+
+
+
- +
-

Mailiverse storage

+

Mailiverse S3 backed storage

We provide infinite mail storage.
- Mailiverse storage is simple, no hassle. We take care of everything.
+ Mailiverse S3 backed storage is simple, no hassle. We take care of everything.

Pick the region you are closest to most of the time.
- @@ -204,6 +206,20 @@ + + +
+

Mailiverse storage

+ We provide infinite mail storage. This storage is backed by our local db. +
+ + +
+ Mailiverse storage is simple, no hassle. We take care of everything.
+
+ + +
diff --git a/web/signup/mSignUp.js b/web/signup/mSignUp.js index 1e58687..e62081e 100644 --- a/web/signup/mSignUp.js +++ b/web/signup/mSignUp.js @@ -20,7 +20,7 @@ mSignUp = { onStorageChange: function() { - var possible = [ 'mailiverse', 'dropbox' ]; + var possible = [ 's3', 'mailiverse', 'dropbox' ]; var value = $('input[name=storage]:checked').val(); $('#storage_' + value).show(); @@ -35,6 +35,11 @@ mSignUp = { mSignUp.validate['storageAuthorized'] = true; } else + if (value == 's3') + { + mSignUp.validate['storageAuthorized'] = true; + } + else { mSignUp.validate['storageAuthorized'] = false; mSignUp.manualTestDropboxAlreadyAuthorized(); @@ -296,8 +301,8 @@ mSignUp = { $('#_mSignUpExecute').show(); var storageInfo = {}; - if (mSignUp.storage == "mailiverse") - storageInfo = { region: $('#storage_mailiverse_region').val() }; + if (mSignUp.storage == "s3") + storageInfo = { region: $('#storage_s3_region').val() }; var signUpDelegate = { progress: function(x) {