moparisthebest/mailiverse
1
0
Fork 0
mirror of https://github.com/moparisthebest/mailiverse synced 2024-11-21 08:25:00 -05:00
Code Issues Releases Wiki Activity

adds another cut of the pgp key ring, adds first cut of internal store so test-setups don't need to use S3

Browse Source
This commit is contained in:
Timothy Prepscius 2013-09-23 16:10:07 -04:00
parent f2e97bec7f
commit 08b39c1af7
70 changed files with 1999 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
DEV_BRAIN_DUMP.txt
View File

@ -92,5 +92,15 @@ You are screwed. Totally screwed. Make backups to a safe place of the mysql se
* How should I have done the code to avoid all that nasty call back stuff
I'm thinking that I should have done a master dispatcher.
I should have done all http requests sync not async.
And had like 20 threads to do request on and computation.
Instead of creating all of these callback chains, I should have just dispatched to one of the workers.
* The StoreEnable and CreateBucket should happen behind the scenes.
The results should be encrypted with a user supplied key
also, CreateBucket could be hacked.

4
build/compile-web
View File

@ -1,9 +1,7 @@
#!/bin/bash
if [ -z "$1" ]; then echo "Must supply prod or dev"; exit 0; fi
V=`date "+%Y%m%d_%H%M%S"`
M=$1
M=prod
rsync -avL --delete ../web/WebContent/ www/
rsync -avL --delete ../gwt/war/mailiverse_gwt/ www/mailiverse_gwt/

3
deploy/deploy-all
View File

@ -1,7 +1,8 @@
#./server-shutdown tunnel
./server-deploy tomcat
#./server-deploy james
./server-deploy postfix
./server-deploy postfix-user
./server-deploy mail-user
./server-deploy mail-key
./server-deploy tools

2
deploy/dev-deploy-all
View File

@ -1,6 +1,6 @@
./dev-server-deploy tomcat
#./dev-server-deploy james
./dev-server-deploy postfix
./dev-server-deploy postfix-user
./dev-server-deploy mail-user
./dev-server-deploy mail-key
./dev-server-deploy tools

6
deploy/postfix-user/config/main.cf
View File

@ -33,8 +33,10 @@ inet_interfaces = all
inet_protocols = all
javapipe_destination_recipient_limit = 1
virtual_mailbox_domains = hash:/etc/postfix/virtual_domains
virtual_mailbox_maps = mysql:/etc/postfix/virtual_mailbox_maps.cf
virtual_mailbox_domains = hash:/etc/postfix/config/virtual_domains
virtual_mailbox_maps = mysql:/etc/postfix/config/virtual_mailbox_maps.cf
virtual_transport = javapipe
smtp_sasl_auth_enable = yes
smtp_sasl_password_maps = mysql:/etc/postfix/config/sasl_password.cf

16
deploy/postfix-user/config/sasl_password.cf Normal file
View File

@ -0,0 +1,16 @@
#
# mysql config file for local(8) aliases(5) lookups
#
# The user name and password to log into the mysql server.
hosts = 127.0.0.1
user = postfix
password = postfix
# The database name on the servers.
dbname = postfix
# For Postfix 2.2 and later The SQL query template.
# See mysql_table(5) for details.
query = SELECT password FROM user WHERE name='%s'

1
deploy/postfix-user/resources/passwords/store Symbolic link
View File

@ -0,0 +1 @@
../../../../passwords/store

1
deploy/tomcat/resources/passwords/store Symbolic link
View File

@ -0,0 +1 @@
../../../../passwords/store

4
deploy/tomcat/restart
View File

@ -1,3 +1,7 @@
pushd ~/tomcat/webapps
rm -rf Mailiverse
popd
cd ~/tomcat/bin
./shutdown.sh

1
gwt/src/core/connector/mv/ClientInfoMvStore.java Symbolic link
View File

@ -0,0 +1 @@
../../../../../java/core/src/core/connector/mv/ClientInfoMvStore.java

1
gwt/src/core/connector/mv/async Symbolic link
View File

@ -0,0 +1 @@
../../../../../java/core/src/core/connector/mv/async

1
gwt/src/core/constants/ConstantsMvStore.java Symbolic link
View File

@ -0,0 +1 @@
../../../../java/core/src/core/constants/ConstantsMvStore.java

2
gwt/src/core/util/HttpDelegateFactory.java
View File

@ -5,7 +5,7 @@ import app.service.Main;
public class HttpDelegateFactory
{
static HttpDelegate create ()
static public HttpDelegate create ()
{
return new JSHttpDelegate(Main.delegate);
}

1
gwt/src/mail/client/model/PublicKey.java Symbolic link
View File

@ -0,0 +1 @@
../../../../../java/core/src/mail/client/model/PublicKey.java

1
gwt/src/mail/client/model/PublicKeyRing.java Symbolic link
View File

@ -0,0 +1 @@
../../../../../java/core/src/mail/client/model/PublicKeyRing.java

2
install/requirements/server_root_account
View File

@ -1 +1 @@
ubuntu
root

2
install/setup-mysql
View File

@ -6,6 +6,7 @@ MAIL_PASSWORD=`cat ../passwords/mail`
MAIL_EXTRA_PASSWORD=`cat ../passwords/mail_extra`
JAMES_PASSWORD=`cat ../passwords/james`
CAPTCHA_PASSWORD=`cat ../passwords/captcha`
STORE_PASSWORD=`cat ../passwords/store`
M_HOST=$1
@ -15,6 +16,7 @@ sed \
-e ''s/MAIL_EXTRA_PASSWORD/$MAIL_EXTRA_PASSWORD/'' \
-e ''s/JAMES_PASSWORD/$JAMES_PASSWORD/'' \
-e ''s/CAPTCHA_PASSWORD/$CAPTCHA_PASSWORD/'' \
-e ''s/STORE_PASSWORD/$STORE_PASSWORD/'' \
setup-mysql.sql.template > setup-mysql.sql \
`

3
install/setup-mysql.sql.template
View File

@ -1,17 +1,20 @@
CREATE DATABASE mail DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
CREATE DATABASE mail_extra DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
CREATE DATABASE captcha DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
CREATE DATABASE store DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
CREATE DATABASE postfix DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
CREATE DATABASE james CHARACTER SET utf8;
CREATE USER 'mail'@'localhost' IDENTIFIED BY 'MAIL_PASSWORD';
CREATE USER 'mail_extra'@'localhost' IDENTIFIED BY 'MAIL_EXTRA_PASSWORD';
CREATE USER 'captcha'@'localhost' IDENTIFIED BY 'CAPTCHA_PASSWORD';
CREATE USER 'store'@'localhost' IDENTIFIED BY 'STORE_PASSWORD';
CREATE USER 'james'@'localhost' IDENTIFIED BY 'JAMES_PASSWORD';
CREATE USER 'postfix'@'localhost' IDENTIFIED BY 'postfix';
GRANT ALL PRIVILEGES ON mail.* TO 'mail'@'localhost';
GRANT ALL PRIVILEGES ON mail_extra.* TO 'mail_extra'@'localhost';
GRANT ALL PRIVILEGES ON captcha.* TO 'captcha'@'localhost';
GRANT ALL PRIVILEGES ON store.* TO 'store'@'localhost';
GRANT ALL PRIVILEGES ON james.* TO 'james'@'localhost';
GRANT ALL PRIVILEGES ON postfix.* TO 'postfix'@'localhost';

7
install/setup-postfix.remote
View File

@ -1,6 +1,8 @@
set -x
apt-get update
echo "1" | apt-get install postfix postfix-mysql postfix-pcre --yes
apt-get install sasl2-bin --yes
cp sudoers.d-postfix-user /etc/sudoers.d/postfix-user
chmod 0440 /etc/sudoers.d/postfix-user
@ -11,7 +13,6 @@ rm master.cf
ln -fs /home/postfix-user/config/main.cf
ln -fs /home/postfix-user/config/master.cf
#ln -fs /home/postfix-user/config/virtual_domains.pcre
ln -fs /home/postfix-user/config/virtual_domains
ln -fs /home/postfix-user/config/virtual_mailbox_maps.cf
ln -fs /home/postfix-user/config

130
java/app/service/src/app/service/JSSignUp.java
View File

@ -23,6 +23,7 @@ import core.callback.Callback;
import core.callback.CallbackChain;
import core.callback.CallbackDefault;
import core.callback.CallbackWithVariables;
import core.constants.ConstantsMvStore;
import core.constants.ConstantsS3;
import core.constants.ConstantsEnvironmentKeys;
import core.constants.ConstantsClient;
@ -125,7 +126,7 @@ public class JSSignUp implements Exportable, SRPClientListener
}
static class SignUpInfo {
public static enum Storage { Mailiverse, Dropbox };
public static enum Storage { S3, Mailiverse, Dropbox };
String name, password, captchaToken;
Environment serverEnvironment, clientEnvironment, completeEnvironment;
@ -144,6 +145,8 @@ public class JSSignUp implements Exportable, SRPClientListener
String awsBucketName, awsBucketRegion;
String awsWriteAccessKey, awsWriteSecretKey;
String awsReadWriteAccessKey, awsReadWriteSecretKey;
String mvAccessKey, mvSecretKey;
String smtpPassword;
byte[] rsaPublicKey;
@ -180,10 +183,15 @@ public class JSSignUp implements Exportable, SRPClientListener
this.dropboxAuthSecret = authSecret;
}
void initializeStorageMailiverse (String storageRegion)
void initializeStorageS3 (String storageRegion)
{
this.storage = Storage.S3;
this.storageRegion = storageRegion;
}
void initializeStorageMailiverse ()
{
this.storage = Storage.Mailiverse;
this.storageRegion = storageRegion;
}
public void setDropboxInfo (String userToken, String userSecret)
@ -205,6 +213,13 @@ public class JSSignUp implements Exportable, SRPClientListener
this.awsReadWriteSecretKey = awsReadWriteSecretKey;
}
public void setMailiverseInfo (String mvAccessKey, String mvSecretKey)
{
this.mvAccessKey = mvAccessKey;
this.mvSecretKey = mvSecretKey;
}
public void calculateRSA (Callback callback) throws NoSuchAlgorithmException
{
new CryptorRSAFactory().generate(2048, new CallbackDefault() {
@ -347,13 +362,65 @@ public class JSSignUp implements Exportable, SRPClientListener
completeEnvironment.addChildEnvironment(ConstantsEnvironmentKeys.CLIENT_ENVIRONMENT, clientEnvironment);
}
public void calculateEnvironmentMailiverse ()
{
String handler = ConstantsStorage.HANDLER_MV;
String prefix = handler + "/";
serverEnvironment = new Environment();
serverEnvironment.put(ConstantsEnvironmentKeys.CONFIGURATION_VERSION, ConstantsVersion.CONFIGURATION);
serverEnvironment.put(ConstantsEnvironmentKeys.HANDLER, handler);
serverEnvironment.put(ConstantsEnvironmentKeys.SMTP_PASSWORD, smtpPassword);
serverEnvironment.put(prefix + ConstantsMvStore.AccessKeyId, mvAccessKey);
serverEnvironment.put(prefix + ConstantsMvStore.SecretKey, mvSecretKey);
serverEnvironment.put(
ConstantsEnvironmentKeys.PUBLIC_ENCRYPTION_KEY,
Base64.encode(rsaPublicKey)
);
serverEnvironment.put(
ConstantsEnvironmentKeys.PGP_PUBLIC_KEY,
Base64.encode(pgpPublicKey)
);
clientEnvironment = new Environment();
clientEnvironment.put(ConstantsEnvironmentKeys.CONFIGURATION_VERSION, ConstantsVersion.CONFIGURATION);
clientEnvironment.put(ConstantsEnvironmentKeys.HANDLER, handler);
clientEnvironment.put(ConstantsEnvironmentKeys.SMTP_PASSWORD, smtpPassword);
clientEnvironment.put(prefix + ConstantsMvStore.AccessKeyId, mvAccessKey);
clientEnvironment.put(prefix + ConstantsMvStore.SecretKey, mvSecretKey);
clientEnvironment.put(
ConstantsEnvironmentKeys.PUBLIC_ENCRYPTION_KEY,
Base64.encode(rsaPublicKey)
);
clientEnvironment.put(
ConstantsEnvironmentKeys.PRIVATE_DECRYPTION_KEY,
Base64.encode(rsaPrivateKey)
);
clientEnvironment.put(
ConstantsEnvironmentKeys.PGP_PUBLIC_KEY,
Base64.encode(pgpPublicKey)
);
clientEnvironment.put(
ConstantsEnvironmentKeys.PGP_PRIVATE_KEY,
Base64.encode(pgpPrivateKey)
);
completeEnvironment = new Environment();
completeEnvironment.put(ConstantsEnvironmentKeys.CONFIGURATION_VERSION, ConstantsVersion.CONFIGURATION);
completeEnvironment.addChildEnvironment(ConstantsEnvironmentKeys.SERVER_ENVIRONMENT, serverEnvironment);
completeEnvironment.addChildEnvironment(ConstantsEnvironmentKeys.CLIENT_ENVIRONMENT, clientEnvironment);
}
public void calculateEnvironment ()
{
if (storage == Storage.Dropbox)
calculateEnvironmentDropbox();
else
if (storage == Storage.Mailiverse)
if (storage == Storage.S3)
calculateEnvironmentAWS();
else
if (storage == Storage.Mailiverse)
calculateEnvironmentMailiverse();
}
};
@ -376,8 +443,11 @@ public class JSSignUp implements Exportable, SRPClientListener
if (storage.equals("dropbox"))
info.intializeStorageDropbox(ConstantsClient.DROPBOX_APPKEY, ConstantsClient.DROPBOX_APPSECRET, dropboxUserKey, dropboxUserSecret);
else
if (storage.equals("s3"))
info.initializeStorageS3(JSON_.getString(JSON_.parse(storageInfo), "region"));
else
if (storage.equals("mailiverse"))
info.initializeStorageMailiverse(JSON_.getString(JSON_.parse(storageInfo), "region"));
info.initializeStorageMailiverse();
CallbackChain signUpChain = new CallbackChain();
@ -489,6 +559,9 @@ public class JSSignUp implements Exportable, SRPClientListener
protected void signUp_step_requestAccess (SignUpInfo info, Callback callback)
{
if (info.storage == SignUpInfo.Storage.S3)
signUp_step_requestS3Bucket(info, callback);
else
if (info.storage == SignUpInfo.Storage.Mailiverse)
signUp_step_requestMailiverseBucket(info, callback);
else
@ -496,9 +569,9 @@ public class JSSignUp implements Exportable, SRPClientListener
signUp_step_requestDropboxAccessToken(info, callback);
}
protected void signUp_step_requestMailiverseBucket (SignUpInfo info, Callback callback)
protected void signUp_step_requestS3Bucket (SignUpInfo info, Callback callback)
{
log.debug("signUp_step_requestMailiverseBucket");
log.debug("signUp_step_requestS3Bucket");
String url =
ConstantsClient.WEB_SERVER_TOMCAT + "CreateBucket" +
@ -509,7 +582,7 @@ public class JSSignUp implements Exportable, SRPClientListener
new CallbackDefault(info) {
public void onSuccess(Object... arguments)
{
log.debug("signUp_step_requestMailiverseBucket callback");
log.debug("signUp_step_requestS3Bucket callback");
SignUpInfo info = V(0);
String response = (String)arguments[0];
String awsBucketName=null, awsBucketRegion=null,
@ -549,7 +622,46 @@ public class JSSignUp implements Exportable, SRPClientListener
);
}
protected void signUp_step_requestDropboxAccessToken (SignUpInfo info, Callback callback)
protected void signUp_step_requestMailiverseBucket (SignUpInfo info, Callback callback)
{
log.debug("signUp_step_requestMailiverseBucket");
String url =
ConstantsClient.WEB_SERVER_TOMCAT + "StoreEnable" +
"?email=" + info.name + "&captcha=" + info.captchaToken;
JSHttpDelegate http = new JSHttpDelegate(main.delegate);
http.execute(HttpDelegate.GET, url, null, false, false, null,
new CallbackDefault(info) {
public void onSuccess(Object... arguments)
{
log.debug("signUp_step_requestMailiverseBucket callback");
SignUpInfo info = V(0);
String response = (String)arguments[0];
String mvAccessKey=null, mvSecretKey=null;
String[] parts = response.split("&");
for (String part : parts)
{
String[] keyValue = part.split("!");
String key = keyValue[0];
String value = keyValue[1];
if (key.equalsIgnoreCase(ConstantsMvStore.AccessKeyId))
mvAccessKey = value;
else
if (key.equalsIgnoreCase(ConstantsMvStore.SecretKey))
mvSecretKey = value;
}
info.setMailiverseInfo(mvAccessKey, mvSecretKey);
next();
}
}.setReturn(callback)
);
}
protected void signUp_step_requestDropboxAccessToken (SignUpInfo info, Callback callback)
{
log.debug("signUp_step_requestAccessToken");

12
java/core/src/core/callback/CallbackSync.java
View File

@ -79,4 +79,16 @@ public class CallbackSync
return null;
}
public void checkException () throws Exception
{
if (results != null && results.length > 0)
{
if (results[0] instanceof Exception)
{
Exception e = (Exception)results[0];
throw e;
}
}
}
}

33
java/core/src/core/connector/mv/ClientInfoMvStore.java Normal file
View File

@ -0,0 +1,33 @@
package core.connector.mv;
import core.constants.ConstantsClient;
import core.constants.ConstantsMvStore;
import core.util.Environment;
public class ClientInfoMvStore
{
private String accessId;
private String secretKey;
public ClientInfoMvStore (Environment e)
{
accessId = e.checkGet(ConstantsMvStore.AccessKeyId);
secretKey = e.checkGet(ConstantsMvStore.SecretKey);
}
public String getBucketEndpoint ()
{
return ConstantsClient.SERVER_TOMCAT;
}
public String getAccessId()
{
return accessId;
}
public String getSecretKey()
{
return secretKey;
}
}

346
java/core/src/core/connector/mv/async/ConnectorMvStore.java Normal file
View File

@ -0,0 +1,346 @@
package core.connector.mv.async;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import core.callback.Callback;
import core.callback.CallbackDefault;
import core.callback.CallbackWithVariables;
import core.connector.FileInfo;
import core.connector.async.AsyncStoreConnectorHelper;
import core.connector.mv.ClientInfoMvStore;
import core.connector.s3.async.S3Connector;
import core.crypt.HashSha256;
import core.crypt.HmacSha1;
import core.util.Base64;
import core.util.DateFormat;
import core.util.FastRandom;
import core.util.HttpDelegate;
import core.util.JSON_;
import core.util.LogNull;
import core.util.Strings;
import core.util.XML;
public class ConnectorMvStore extends AsyncStoreConnectorHelper
{
static LogNull log = new LogNull(S3Connector.class);
final int LOCK_INTERVAL = 10 * 1000;
ClientInfoMvStore info;
HttpDelegate httpDelegate;
HmacSha1 mac;
static FastRandom fastRandom = new FastRandom();
protected String createUrlPrefix ()
{
return "https://" + info.getBucketEndpoint() + "/";
}
protected String createRandomPostfix ()
{
return "random=" + fastRandom.nextLong();
}
// This method converts AWSSecretKey into crypto instance.
protected void setKey(String secretKey) throws Exception
{
mac = new HmacSha1(Base64.decode(secretKey));
}
// This method creates S3 signature for a given String.
protected String sign(String data) throws Exception
{
// Signed String must be BASE64 encoded.
byte[] signBytes = mac.mac(Strings.toBytes(data));
String signature = Base64.encode(signBytes);
return signature;
}
protected String format(String format, Date date)
{
DateFormat df = new DateFormat(format);
String dateString = df.format(date, 0) + " GMT";
return dateString;
}
protected String[][] makeHeaders (String keyId, String method, String contentMD5, String contentType, int contentLength, Date date, String resource) throws Exception
{
String fmt = "EEE, dd MMM yyyy HH:mm:ss";
String dateString = format(fmt, date);
// Generate signature
StringBuffer buf = new StringBuffer();
buf.append(method).append("\n");
buf.append(contentMD5).append("\n");
buf.append(contentType).append("\n");
buf.append("\n"); // empty real date header
buf.append(dateString).append("\n");
buf.append(resource);
log.debug("Signing:{" + buf.toString() + "}");
String signature = sign(buf.toString());
String[][] headers;
if (method.equals("PUT"))
{
headers = new String[][] {
{"X-Mv-Date" , dateString },
{"Content-Type", contentType },
{"Content-Length", ""+contentLength },
{"Authorization", "MV " + keyId + ":" + signature }
};
}
else
{
headers = new String[][] {
{"X-Mv-Date" , dateString },
{"Authorization", "MV " + keyId + ":" + signature }
};
}
return headers;
}
public ConnectorMvStore(ClientInfoMvStore clientInfo, HttpDelegate httpDelegate) throws Exception
{
this.info = clientInfo;
this.httpDelegate = httpDelegate;
setKey (info.getSecretKey());
}
long toVersionFromString (String s) throws Exception
{
HashSha256 hash = new HashSha256();
byte[] result = hash.hash(Strings.toBytes(s));
long l =
((long)result[0]) |
((long)result[1] << 8) |
((long)result[2] << 16) |
((long)result[3] << 24);
return l;
}
public void listDirectoryFinished (List<FileInfo> files, Callback callback, String path, Object... arguments)
{
log.debug("listDirectoryFinished");
try
{
if (arguments[0] instanceof Exception)
throw (Exception)arguments[0];
String result = (String)arguments[0];
log.trace(result);
DateFormat dateTimeFormat = new DateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z' Z");
Object doc = JSON_.parse(result);
Object nodes = JSON_.getArray(doc, "contents");
for (int i=0; i<JSON_.size(nodes); ++i)
{
Object currentNode = JSON_.get(nodes, i);
String keyNode = JSON_.getString(currentNode, "path");
String etagNode = JSON_.getString(currentNode, "version");
long sizeNode = JSON_.getInt(currentNode, "size");
String lastModifiedNode = JSON_.getString(currentNode, "date");
log.trace(keyNode, etagNode, sizeNode, lastModifiedNode);
String fullPath = keyNode;
String relativePath = fullPath.substring(path.length());
FileInfo fi = new FileInfo(
fullPath,
relativePath,
sizeNode,
dateTimeFormat.parse(lastModifiedNode),
etagNode
);
files.add(fi);
}
if (JSON_.getBoolean(doc, "isTruncated"))
{
log.debug("results were truncated, requesting more...");
listIterative(files, path, callback);
}
else
{
log.debug("results were complete, invoking callback");
Collections.sort(files, new FileInfo.SortByDateAscending());
for (FileInfo i : files)
log.trace ("path: ", i.path, " date:", i.date);
callback.invoke(files);
}
}
catch (Throwable e)
{
e.printStackTrace();
callback.invoke(e);
}
}
@Override
public void list(String path, Callback callback)
{
listIterative(new ArrayList<FileInfo>(), path, callback);
}
public void listIterative(List<FileInfo> files, String path, Callback callback)
{
log.debug("listDirectory",path);
try
{
String url =
createUrlPrefix() +
"StoreList?Resource=" + path + "&max-keys=1000" +
(!files.isEmpty() ? ("&marker=" + files.get(files.size()-1).path) : "") +
"&" + createRandomPostfix();
log.debug(url);
String[][] headers = makeHeaders (
info.getAccessId(), "GET", "", "", 0, new Date(), path
);
httpDelegate.execute (HttpDelegate.GET, url, headers, false, false, null,
new CallbackWithVariables(files, callback, path) {
@Override
public void invoke(Object... arguments)
{
List<FileInfo> files = V(0);
Callback callback = V(1);
String path = V(2);
listDirectoryFinished(files, callback, path, arguments);
}
}
);
}
catch (Throwable e)
{
e.printStackTrace();
callback.invoke(e);
}
}
@Override
public void createDirectory(String path, Callback callback)
{
log.debug("createDirectory",path);
callback.invoke(path);
}
@Override
public void get(String path, Callback callback)
{
log.debug("get",path);
try
{
String url =
createUrlPrefix() +
"StoreGet?Resource=" + path +
"&" + createRandomPostfix();
log.debug(url);
String[][] headers = makeHeaders (
info.getAccessId(), "GET", "", "", 0, new Date(), path
);
httpDelegate.execute(HttpDelegate.GET, url, headers, false, true, null, grabVersion_(true).setReturn(callback));
}
catch (Throwable e)
{
callback.invoke(e);
}
}
public Callback grabVersion_(boolean includeResponseData)
{
return new CallbackDefault(includeResponseData) {
public void onSuccess(Object... arguments) throws Exception {
boolean includeResponseData = (Boolean)V(0);
String[][] headers = (String[][])arguments[1];
for (String[] pair : headers)
{
if (pair[0].equals("Version"))
{
if (includeResponseData)
next(arguments[0], pair[1]);
else
next(pair[1]);
return;
}
}
throw new Exception("No ETag Response header");
}
};
}
@Override
public void put(String path, byte[] contents, Callback callback)
{
log.debug("put",path);
try
{
String url =
createUrlPrefix() +
"StorePut?Resource=" + path +
"&" + createRandomPostfix();
log.debug(url);
String[][] headers = makeHeaders (
info.getAccessId(), "PUT", "", "application/octet-stream", contents.length, new Date(), path
);
httpDelegate.execute(HttpDelegate.PUT, url, headers, true, false, contents, grabVersion_(false).setReturn(callback));
}
catch (Throwable e)
{
callback.invoke(e);
}
}
@Override
public void delete(String path, Callback callback)
{
log.debug("delete",path);
try
{
String url =
createUrlPrefix() +
"StoreDelete?Resource=" + path +
"&" + createRandomPostfix();
log.debug(url);
String[][] headers = makeHeaders (
info.getAccessId(), "DELETE", "", "", 0, new Date(), path
);
httpDelegate.execute(HttpDelegate.DELETE, url, headers, true, false, null, callback);
}
catch (Throwable e)
{
callback.invoke(e);
}
}
}

137
java/core/src/core/connector/mv/sync/ConnectorMvStore.java Normal file
View File

@ -0,0 +1,137 @@
package core.connector.mv.sync;
import java.util.List;
import core.callback.CallbackSync;
import core.connector.ConnectorException;
import core.connector.FileInfo;
import core.connector.mv.ClientInfoMvStore;
import core.connector.sync.StoreConnector;
import core.util.HttpDelegateFactory;
public class ConnectorMvStore implements StoreConnector
{
private ClientInfoMvStore clientInfo;
private core.connector.mv.async.ConnectorMvStore connector;
public ConnectorMvStore (ClientInfoMvStore clientInfo) throws Exception
{
this.clientInfo = clientInfo;
this.connector = new core.connector.mv.async.ConnectorMvStore(clientInfo, HttpDelegateFactory.create());
}
@Override
public void open() throws ConnectorException
{
}
@Override
public void close() throws ConnectorException
{
}
@Override
public List<FileInfo> listDirectory(String path) throws ConnectorException
{
try
{
CallbackSync sync = new CallbackSync(connector.list_(path)).invoke();
return sync.<List<FileInfo>>export();
}
catch (Exception e)
{
throw new ConnectorException(e);
}
}
@Override
public void createDirectory(String path) throws ConnectorException
{
try
{
CallbackSync sync = new CallbackSync(connector.createDirectory_(path)).invoke();
sync.checkException();
}
catch (Exception e)
{
throw new ConnectorException(e);
}
}
@Override
public byte[] get(String path) throws ConnectorException
{
try
{
CallbackSync sync = new CallbackSync(connector.get_(path)).invoke();
return sync.<byte[]>export();
}
catch (Exception e)
{
throw new ConnectorException(e);
}
}
@Override
public byte[] get(String path, long size) throws ConnectorException
{
try
{
CallbackSync sync = new CallbackSync(connector.get_(path)).invoke();
return sync.<byte[]>export();
}
catch (Exception e)
{
throw new ConnectorException(e);
}
}
@Override
public void put(String path, byte[] contents) throws ConnectorException
{
try
{
CallbackSync sync = new CallbackSync(connector.put_(path, contents)).invoke();
sync.checkException();
}
catch (Exception e)
{
throw new ConnectorException(e);
}
}
@Override
public void move(String from, String to) throws ConnectorException
{
try
{
CallbackSync sync = new CallbackSync(connector.move_(from, to)).invoke();
sync.checkException();
}
catch (Exception e)
{
throw new ConnectorException(e);
}
}
@Override
public void delete(String path) throws ConnectorException
{
try
{
CallbackSync sync = new CallbackSync(connector.delete_(path)).invoke();
sync.checkException();
}
catch (Exception e)
{
throw new ConnectorException(e);
}
}
@Override
public boolean ensureDirectories(String... folders)
{
CallbackSync sync = new CallbackSync(connector.ensureDirectories_(folders)).invoke();
return sync.<Boolean>exportNoException();
}
}

8
java/core/src/core/constants/ConstantsMvStore.java Normal file
View File

@ -0,0 +1,8 @@
package core.constants;
public class ConstantsMvStore
{
public static final String AccessKeyId = "AccessKeyId";
public static final String SecretKey = "SecretKey";
}

4
java/core/src/core/constants/ConstantsStorage.java
View File

@ -4,7 +4,8 @@ public class ConstantsStorage
{
public final static String
HANDLER_DROPBOX = "DB",
HANDLER_S3 = "S3";
HANDLER_S3 = "S3",
HANDLER_MV = "MV";
public final static String
IN = "In",
@ -28,4 +29,5 @@ public class ConstantsStorage
public static final int MAIL_CHECK_LOCK_TIME_SECONDS = 120;
public static final int MAIL_CHECK_LOCK_TIME_ALLOWED_BEFORE_RELOCK_SECONDS = 60;
}

9
java/core/src/core/util/HttpDelegateFactory.java Normal file
View File

@ -0,0 +1,9 @@
package core.util;
public class HttpDelegateFactory
{
static public HttpDelegate create ()
{
return new HttpDelegateJava();
}
}

14
java/core/src/core/util/SqlCatalog.java
View File

@ -10,6 +10,20 @@ import java.util.ArrayList;
public class SqlCatalog
{
static LogOut log = new LogOut(SqlCatalog.class);
public SqlCatalog ()
{
try
{
Class.forName("com.mysql.jdbc.Driver");
}
catch (Exception e)
{
log.exception(e);
throw new RuntimeException(e);
}
}
public String getSingle (String name) throws IOException
{
return Streams.readFullyString(getClass().getResourceAsStream(name), "UTF-8");

29
java/core/src/mail/client/CacheManager.java
View File

@ -36,6 +36,8 @@ import mail.client.model.Model;
import mail.client.model.ModelFactory;
import mail.client.model.Mail;
import mail.client.model.ModelSerializer;
import mail.client.model.PublicKey;
import mail.client.model.PublicKeyRing;
import mail.client.model.Settings;
public class CacheManager extends Servent<Master>
@ -46,6 +48,9 @@ public class CacheManager extends Servent<Master>
IndexedCache cacheMail;
IndexedCache cacheConversation;
IndexedCache cacheFolder;
IndexedCache cacheKeys;
PublicKeyRing keyRing;
Settings settings;
boolean isCaching = false;
@ -84,6 +89,10 @@ public class CacheManager extends Servent<Master>
this.settings.setId(Constants.SETTINGS_ID);
this.masterCache.link(settings);
this.keyRing = new PublicKeyRing(this);
this.keyRing.setId(Constants.KEYRING_ID);
this.masterCache.link(keyRing);
ItemSerializer itemSerializer = new ModelSerializer(json);
this.cacheMail = new IndexedCache(
@ -104,11 +113,17 @@ public class CacheManager extends Servent<Master>
this.cacheFolder.setId(Constants.FOLDER_ID);
masterCache.link(cacheFolder);
this.cacheKeys = new IndexedCache(
new ItemCacheFactory ("K", library, itemFactory, itemSerializer)
);
this.cacheKeys.setId(Constants.KEY_ID);
masterCache.link(cacheKeys);
//-------------------------------------------------------------
Callback countDown =
new CountDown(
4,
5,
getMaster().getEventPropagator().signal_(Events.Initialize_IndexedCacheLoadComplete)
);
@ -117,6 +132,7 @@ public class CacheManager extends Servent<Master>
cacheMail.apply(new Split(countDown));
cacheConversation.apply(new Split(countDown));
cacheFolder.apply(new Split(countDown));
cacheKeys.apply(new Split(countDown));
//-------------------------------------------------------------
@ -143,6 +159,7 @@ public class CacheManager extends Servent<Master>
cacheMail.markCreate();
cacheConversation.markCreate();
cacheFolder.markCreate();
cacheKeys.markCreate();
masterCache.markCreate();
settings.markCreate();
@ -261,6 +278,16 @@ public class CacheManager extends Servent<Master>
cacheFolder.put(f);
}
public void putKey(PublicKey k)
{
cacheKeys.put(k);
}
public PublicKey getKey (ID id)
{
return (PublicKey)cacheKeys.getAndAcquire(Type.PublicKey, id);
}
public boolean isFullyCached ()
{
return (!library.hasDirtyChildren() && !masterCache.hasDirtyChildren());

9
java/core/src/mail/client/Client.java
View File

@ -13,6 +13,8 @@ import core.connector.async.AsyncStoreConnectorBase64;
import core.connector.async.AsyncStoreConnectorEncrypted;
import core.connector.dropbox.ClientInfoDropbox;
import core.connector.dropbox.async.ConnectorDropbox;
import core.connector.mv.ClientInfoMvStore;
import core.connector.mv.async.ConnectorMvStore;
import core.connector.s3.ClientInfoS3;
import core.connector.s3.async.S3Connector;
import core.constants.ConstantsEnvironmentKeys;
@ -82,6 +84,13 @@ public class Client
connector = new S3Connector(clientInfo, httpDelegate);
}
else
if (handler.equals(ConstantsStorage.HANDLER_MV))
{
Environment mvEnvironment = mailBoxEnvironment.childEnvironment(handler);
ClientInfoMvStore clientInfo = new ClientInfoMvStore(mvEnvironment);
connector = new ConnectorMvStore(clientInfo, httpDelegate);
}
else
{
throw new Exception ("Unknown handler");
}

4
java/core/src/mail/client/Constants.java
View File

@ -26,5 +26,7 @@ public class Constants
SETTINGS_ID = ID.fromLong(1),
MAIL_ID = ID.fromLong(2),
CONVERSATION_ID = ID.fromLong(3),
FOLDER_ID = ID.fromLong(4);
FOLDER_ID = ID.fromLong(4),
KEYRING_ID = ID.fromLong(5),
KEY_ID = ID.fromLong(6);
}

2
java/core/src/mail/client/MasterCacheSerializer.java
View File

@ -28,7 +28,7 @@ public class MasterCacheSerializer implements ItemSerializer
{
if (item instanceof Settings)
return settingsSerializer.serialize_(item);
return indexedCacheSerializer.serialize_(item);
}
@Override

43
java/core/src/mail/client/cache/JSON.java vendored
View File

@ -26,6 +26,8 @@ import mail.client.model.FolderSet;
import mail.client.model.Header;
import mail.client.model.Identity;
import mail.client.model.Mail;
import mail.client.model.PublicKey;
import mail.client.model.PublicKeyRing;
import mail.client.model.Recipients;
import mail.client.model.Settings;
import mail.client.model.TransportState;
@ -573,4 +575,45 @@ public class JSON extends Servent<Master>
return o;
}
public void fromJSON(PublicKey item, Object parse) throws JSONException
{
item.setEmail(JSON_.getString(parse, "email"));
item.setPublicKey(JSON_.getString(parse, "publicKey"));
}
public Object toJSON(PublicKey item) throws JSONException
{
Object o = JSON_.newObject();
JSON_.put(o, "email", item.toString());
JSON_.put(o, "publicKey", item.getPublicKey());
return o;
}
public void fromJSON(PublicKeyRing item, Object a) throws JSONException
{
for (int i=0; i<JSON_.size(a); ++i) // reverse it
{
Object iNs = JSON_.getObject(a, i);
item.addPublicKeyFromCache(
toID(JSON_.getString(iNs,"id")),
toIdentity(JSON_.getString(iNs, "identity"))
);
}
}
public Object toJSON(PublicKeyRing item) throws JSONException
{
Object a = JSON_.newArray();
for (Pair<ID,Identity> iNs : item.getPublicKeys())
{
Object o = JSON_.newObject();
JSON_.put(o, "id", toJSON(iNs.first));
JSON_.put(o, "identity", toJSON(iNs.second));
}
return a;
}
}

4
java/core/src/mail/client/cache/Type.java vendored
View File

@ -13,5 +13,7 @@ public enum Type {
FolderFilterSet,
FolderMaster,
Index,
Settings
Settings,
PublicKeyRing,
PublicKey,
}

16
java/core/src/mail/client/model/Identity.java
View File

@ -24,7 +24,6 @@ public class Identity implements Serializable, Exportable
String name;
String email;
boolean isPrimary = false;
String publicKey;
protected Identity ()
{
@ -169,19 +168,4 @@ public class Identity implements Serializable, Exportable
this.email = identity.email;
}
}
public void setPublicKey (String publicKey)
{
this.publicKey = publicKey;
}
public boolean hasPublicKey()
{
return publicKey != null;
}
public String getPublicKey()
{
return publicKey;
}
}

4
java/core/src/mail/client/model/Mail.java
View File

@ -124,7 +124,8 @@ public class Mail extends Model
{
this.attachments = attachments;
}
/*
public boolean isPresendEncryptable()
{
for (Identity i : getHeader().getRecipients().getAll())
@ -191,4 +192,5 @@ public class Mail extends Model
Base64.encode(aes.encrypt(Strings.toBytes(JSON_.asString(container))))
);
}
*/
}

4
java/core/src/mail/client/model/ModelSerializer.java
View File

@ -40,6 +40,8 @@ public class ModelSerializer implements ItemSerializer
return Strings.toBytes(json.toJSON((Folder)item).toString());
if (item instanceof Settings)
return Strings.toBytes(json.toJSON((Settings)item).toString());
if (item instanceof PublicKey)
return Strings.toBytes(json.toJSON((PublicKey)item).toString());
return null;
}
@ -58,6 +60,8 @@ public class ModelSerializer implements ItemSerializer
json.fromJSON((Folder)item, JSON_.parse(Strings.toString(bytes)));
if (item instanceof Settings)
json.fromJSON((Settings)item, JSON_.parse(Strings.toString(bytes)));
if (item instanceof PublicKey)
json.fromJSON((PublicKey)item, JSON_.parse(Strings.toString(bytes)));
}
@Override

35
java/core/src/mail/client/model/PublicKey.java Normal file
View File

@ -0,0 +1,35 @@
package mail.client.model;
import mail.client.CacheManager;
public class PublicKey extends Model
{
String email;
String publicKey;
public PublicKey(CacheManager manager)
{
super(manager);
}
public void setEmail (String email)
{
this.email = email;
}
public String getEmail (String email)
{
return email;
}
public void setPublicKey (String publicKey)
{
this.publicKey = publicKey;
}
public String getPublicKey ()
{
return publicKey;
}
}

34
java/core/src/mail/client/model/PublicKeyRing.java Normal file
View File

@ -0,0 +1,34 @@
package mail.client.model;
import java.util.List;
import core.util.Pair;
import mail.client.CacheManager;
import mail.client.cache.ID;
public class PublicKeyRing extends Model
{
List<Pair<ID, Identity>> ring;
public PublicKeyRing(CacheManager manager)
{
super(manager);
}
public void addPublicKeyFromCache(ID id, Identity identity)
{
ring.add(Pair.create(id, identity));
}
public void addPublicKey(ID id, Identity identity)
{
ring.add(Pair.create(id, identity));
markDirty();
}
public List<Pair<ID,Identity>> getPublicKeys()
{
return ring;
}
}

32
java/core/src/mail/server/handler/MailetHandlerMvStore.java Normal file
View File

@ -0,0 +1,32 @@
/**
* Author: Timothy Prepscius
* License: GPLv3 Affero + keep my name in the code!
*/
package mail.server.handler;
import core.connector.mv.ClientInfoMvStore;
import core.connector.mv.sync.ConnectorMvStore;
import core.connector.sync.EncryptedStoreConnector;
import core.connector.sync.StoreConnector;
import core.constants.ConstantsStorage;
import core.crypt.CryptorRSAAES;
import core.crypt.CryptorRSAFactoryEnvironment;
import core.util.Environment;
import core.util.LogOut;
public class MailetHandlerMvStore extends MailetHandlerDefault
{
static LogOut log = new LogOut(MailetHandlerMvStore.class);
public StoreConnector createConnector (Environment e) throws Exception
{
log.debug("createConnector");
return new EncryptedStoreConnector(
new CryptorRSAAES(CryptorRSAFactoryEnvironment.create(e)),
new ConnectorMvStore(new ClientInfoMvStore(e.childEnvironment(ConstantsStorage.HANDLER_MV)))
);
}
}

3
java/core/src/mail/server/handler/UserInformationFactory.java
View File

@ -49,6 +49,9 @@ public class UserInformationFactory
else
if (handlerName.equals(ConstantsStorage.HANDLER_DROPBOX))
mailetHandler = new MailetHandlerDropbox();
else
if (handlerName.equals(ConstantsStorage.HANDLER_MV))
mailetHandler = new MailetHandlerMvStore();
else
throw new Exception("Unknown handler");

13
java/core/src/store/server/ConstantsMvServer.java Normal file
View File

@ -0,0 +1,13 @@
package store.server;
public class ConstantsMvServer {
public static final String
HEADER_DATE = "X-Mv-Date",
HEADER_CONTENT_TYPE = "Content-Type",
HEADER_CONTENT_LENGTH = "Content-Length",
HEADER_AUTHORIZATION = "Authorization",
PARAMETER_RESOURCE = "Resource";
}

64
java/core/src/store/server/StoreServer.java Normal file
View File

@ -0,0 +1,64 @@
package store.server;
import java.util.List;
import store.server.db.DbStore;
import core.connector.FileInfo;
import core.util.Pair;
public class StoreServer
{
DbStore store;
public StoreServer () throws Exception
{
store = new DbStore();
store.ensureTables();
}
public Pair<String,String> newKeyPair (String userName) throws Exception
{
Pair<String, String> v = StoreUtils.createKeyIdAndSecretKey();
DbStore store = new DbStore();
int userId = store.getUserId(userName);
store.addUserKeyPair(userId, v.first, v.second);
return v;
}
public void newUser (String userName) throws Exception
{
store.addUser(userName);
}
public Pair<String,String> newUserWithKeyPair (String userName) throws Exception
{
newUser (userName);
return newKeyPair(userName);
}
public String putKeyValue (String keyId, String key, byte[] value) throws Exception
{
return store.putKeyValue(store.getUserIdAndSecretKey(keyId).first, key, value);
}
public Pair<String, byte[]> getKeyValue (String keyId, String key) throws Exception
{
return store.getKeyValue(store.getUserIdAndSecretKey(keyId).first, key);
}
public void removeKeyValue (String keyId, String key) throws Exception
{
store.removeKeyValue (store.getUserIdAndSecretKey(keyId).first, key);
}
public void removeUser (String name) throws Exception
{
store.removeUser (name);
}
public List<FileInfo> listKeys (String keyId, String key) throws Exception
{
return store.listKeys(store.getUserIdAndSecretKey(keyId).first, key);
}
}

142
java/core/src/store/server/StoreUtils.java Normal file
View File

@ -0,0 +1,142 @@
package store.server;
import java.util.Map;
import store.server.db.DbStore;
import core.crypt.HmacSha1;
import core.util.Base64;
import core.util.LogOut;
import core.util.Pair;
import core.util.SecureRandom;
import core.util.Strings;
public class StoreUtils
{
static LogOut log = new LogOut(StoreUtils.class);
// This method creates S3 signature for a given String.
static protected String sign(HmacSha1 mac, String data) throws Exception
{
// Signed String must be BASE64 encoded.
byte[] signBytes = mac.mac(Strings.toBytes(data));
String signature = Base64.encode(signBytes);
return signature;
}
static public Pair<String, String> createKeyIdAndSecretKey ()
{
byte[] keyId = new byte[32];
byte[] secretKey = new byte[32];
SecureRandom random = new SecureRandom();
random.nextBytes(keyId);
random.nextBytes(secretKey);
return Pair.create(Base64.encode(keyId), Base64.encode(secretKey));
}
static public void verifySignature (String data, String keyId, String signature) throws Exception
{
DbStore dbStore = new DbStore();
Pair<Integer, String> userIdAndSecretKey = dbStore.getUserIdAndSecretKey(keyId);
log.debug ("verifySignature", data, userIdAndSecretKey.second);
HmacSha1 userMac = new HmacSha1(Base64.decode(userIdAndSecretKey.second));
String correctSignature = sign(userMac, data);
if (!correctSignature.equals(signature))
throw new Exception ("Signature mismatch");
}
/**
* returns the keyId of the user
*
* @param action
* @param headers
* @param resource
* @param content
* @return
* @throws Exception
*/
static public String verifyUser (String action, Map<String,String> headers, String resource, byte[] content) throws Exception
{
String method = action;
String dateString = headers.get(ConstantsMvServer.HEADER_DATE);
String contentType = headers.get(ConstantsMvServer.HEADER_CONTENT_TYPE);
String contentLength = headers.get(ConstantsMvServer.HEADER_CONTENT_LENGTH);
if (contentLength != null)
if (Integer.parseInt(contentLength) != content.length)
throw new Exception ("Length mismatch");
StringBuffer buf = new StringBuffer();
buf.append(method).append("\n");
buf.append("").append("\n");
buf.append(contentType != null ? contentType : "").append("\n");
buf.append("\n"); // empty real date header
buf.append(dateString).append("\n");
buf.append(resource);
String[] authorizationParts = headers.get(ConstantsMvServer.HEADER_AUTHORIZATION).split(":");
String keyId = authorizationParts[0].split(" ")[1];
String signature = authorizationParts[1];
verifySignature (buf.toString(), keyId, signature);
return keyId;
}
/*
public byte[] getFile (String userPath) throws Exception
{
FileInputStream f = new FileInputStream(ConstantsMvServer.PATH + userPath);
byte[] result = Streams.readFullyBytes(f);
f.close();
return result;
}
public void putFile (String userPath, byte[] bytes) throws Exception
{
FileOutputStream f = new FileOutputStream (ConstantsMvServer.PATH + userPath);
f.write(bytes);
f.close();
}
private List<File> listFiles(File dir, List<File> files)
{
if (!dir.isDirectory())
{
files.add(dir);
return files;
}
for (File file : dir.listFiles())
listFiles(file, files);
return files;
}
public List<FileInfo> listKeys (String userPath)
{
List<File> files = listFiles (new File (ConstantsMvServer.PATH + userPath), new ArrayList<File>());
List<FileInfo> fileInfos = new ArrayList<FileInfo>();
for (File file : files)
{
FileInfo info = new FileInfo(
file.getPath(),
file.getPath(),
file.length(),
new Date(file.lastModified()),
"" + file.lastModified()
);
fileInfos.add(info);
}
return fileInfos;
}
*/
}

304
java/core/src/store/server/db/DbStore.java Normal file
View File

@ -0,0 +1,304 @@
package store.server.db;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import core.connector.FileInfo;
import core.util.Base64;
import core.util.FastRandom;
import core.util.LogOut;
import core.util.Pair;
import core.util.Passwords;
import store.server.db.sql.Catalog;
public class DbStore
{
FastRandom random = new FastRandom();
LogOut log = new LogOut(DbStore.class);
Catalog catalog = new Catalog();
public void ensureTables() throws Exception
{
Connection connection = openConnection();
try
{
for (String sql : catalog.getMulti(Catalog.ENSURE_TABLES))
{
PreparedStatement statement = connection.prepareStatement (sql);
log(statement);
statement.executeUpdate();
}
}
finally
{
closeConnection(connection);
}
}
public String putKeyValue (int userId, String key, byte[] value) throws Exception
{
Connection connection = null;
try
{
connection = openConnection();
String sql = catalog.getSingle(Catalog.PUT_KEY_VALUE);
PreparedStatement statement = connection.prepareStatement(sql);
statement.setInt(1, userId);
statement.setString(2, key);
statement.setBytes(3, value);
byte[] randomBytes = new byte[16];
random.nextBytes(randomBytes);
String randomB64 = Base64.encode(randomBytes);
statement.setString(4, randomB64);
statement.execute();
return randomB64;
}
finally
{
closeConnection (connection);
}
}
public Pair<String, byte[]> getKeyValue (int userId, String key) throws Exception
{
Connection connection = null;
try
{
connection = openConnection();
String sql = catalog.getSingle(Catalog.GET_KEY_VALUE);
PreparedStatement statement = connection.prepareStatement(sql);
statement.setInt(1, userId);
statement.setString(2, key);
ResultSet results = statement.executeQuery();
if (results.next())
{
return Pair.create(results.getString("version"), results.getBytes("v"));
}
results.close();
}
finally
{
closeConnection (connection);
}
throw new Exception("Unknown key");
}
public List<FileInfo> listKeys (int userId, String key) throws Exception
{
Connection connection = null;
try
{
connection = openConnection();
String sql = catalog.getSingle(Catalog.LIST_KEY_VALUES);
PreparedStatement statement = connection.prepareStatement(sql);
statement.setInt(1, userId);
statement.setString(2, key);
List<FileInfo> retval = new ArrayList<FileInfo>();
ResultSet results = statement.executeQuery();
while (results.next())
{
String path = results.getString("k");
int size = results.getInt("size");
Date date = results.getTimestamp("mark");
String version = results.getString("version");
FileInfo fileInfo = new FileInfo(path, path.substring(key.length()), size, date, version);
retval.add(fileInfo);
}
results.close();
return retval;
}
finally
{
closeConnection (connection);
}
}
public Pair<Integer, String> getUserIdAndSecretKey (String keyId) throws Exception
{
Connection connection = null;
try
{
connection = openConnection();
String sql = catalog.getSingle(Catalog.GET_USER_ID_AND_SECRET_KEY);
PreparedStatement statement = connection.prepareStatement(sql);
statement.setString(1, keyId);
ResultSet results = statement.executeQuery();
if (results.next())
{
return Pair.create(results.getInt("user_id"), results.getString("secret_key"));
}
results.close();
}
finally
{
closeConnection (connection);
}
throw new Exception("Unknown KeyId");
}
public void addUser (String userName) throws Exception
{
Connection connection = null;
try
{
connection = openConnection();
String sql = catalog.getSingle(Catalog.ADD_USER);
PreparedStatement statement = connection.prepareStatement(sql);
statement.setString(1, userName);
statement.execute();
}
finally
{
closeConnection (connection);
}
}
public void removeUser (String userName) throws Exception
{
Connection connection = null;
try
{
connection = openConnection();
String[] sqls = catalog.getMulti(Catalog.REMOVE_USER);
boolean first = true;
for (String sql : sqls)
{
PreparedStatement statement = connection.prepareStatement(sql);
if (first)
statement.setString(1, userName);
statement.execute();
first = false;
}
}
finally
{
closeConnection (connection);
}
}
public void removeKeyValue(int userId, String key) throws Exception
{
Connection connection = null;
try
{
connection = openConnection();
String sql = catalog.getSingle(Catalog.REMOVE_KEY_VALUE);
PreparedStatement statement = connection.prepareStatement(sql);
statement.setInt(1, userId);
statement.setString(2, key);
statement.execute();
}
finally
{
closeConnection (connection);
}
}
public int getUserId(String userName) throws Exception
{
Connection connection = null;
try
{
connection = openConnection();
String sql = catalog.getSingle(Catalog.GET_USER_ID);
PreparedStatement statement = connection.prepareStatement(sql);
statement.setString(1, userName);
ResultSet results = statement.executeQuery();
if (results.next())
{
return results.getInt("user_id");
}
results.close();
}
finally
{
closeConnection (connection);
}
throw new Exception("Unknown user name");
}
public void addUserKeyPair (int userId, String keyId, String secretKey) throws Exception
{
Connection connection = null;
try
{
connection = openConnection();
String sql = catalog.getSingle(Catalog.ADD_USER_KEY_PAIR);
PreparedStatement statement = connection.prepareStatement(sql);
statement.setInt(1, userId);
statement.setString(2, keyId);
statement.setString(3, secretKey);
statement.execute();
}
finally
{
closeConnection (connection);
}
}
public Connection openConnection () throws IOException, SQLException
{
log.debug("Connecting to", catalog.CONNECTION_STRING);
return DriverManager.getConnection(catalog.CONNECTION_STRING, catalog.USER, Passwords.getPasswordFor(catalog.USER));
}
public void closeConnection (Connection connection)
{
try
{
if (connection != null)
connection.close();
}
catch (Exception e)
{
e.printStackTrace();
}
}
public void log (Statement sql)
{
log.debug (sql);
}
}

28
java/core/src/store/server/db/sql/Catalog.java Normal file
View File

@ -0,0 +1,28 @@
/**
* Author: Timothy Prepscius
* License: GPLv3 Affero + keep my name in the code!
*/
package store.server.db.sql;
import core.constants.ConstantsServer;
import core.util.SqlCatalog;
public class Catalog extends SqlCatalog
{
public static final String
ENSURE_TABLES = "ensure_tables.sql",
ADD_USER = "add_user.sql",
ADD_USER_KEY_PAIR = "add_user_key_pair.sql",
GET_USER_ID = "get_user_id.sql",
REMOVE_USER = "remove_user.sql",
GET_USER_ID_AND_SECRET_KEY = "get_user_id_and_secret_key.sql",
LIST_KEY_VALUES = "list_keys.sql",
REMOVE_KEY_VALUE = "remove_key_value.sql",
PUT_KEY_VALUE = "put_key_value.sql",
GET_KEY_VALUE = "get_key_value.sql";
public String CONNECTION_STRING = ConstantsServer.DBCONNECTION_PREFIX + "store";
public String USER = "store";
}

1
java/core/src/store/server/db/sql/add_user.sql Normal file
View File

@ -0,0 +1 @@
INSERT INTO users (name) VALUES (?)

1
java/core/src/store/server/db/sql/add_user_key_pair.sql Normal file
View File

@ -0,0 +1 @@
INSERT INTO key_pairs (user_id, key_id, secret_key) VALUES (?,?,?);

24
java/core/src/store/server/db/sql/ensure_tables.sql Normal file
View File

@ -0,0 +1,24 @@
CREATE TABLE IF NOT EXISTS users (
name VARCHAR(255) NOT NULL,
user_id INTEGER AUTO_INCREMENT NOT NULL,
PRIMARY KEY (user_id),
KEY (name)
);
CREATE TABLE IF NOT EXISTS key_pairs (
user_id INTEGER NOT NULL,
key_id VARCHAR(255) NOT NULL,
secret_key VARCHAR(255) NOT NULL,
mark TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (user_id, key_id),
KEY (key_id)
);
CREATE TABLE IF NOT EXISTS key_values (
user_id INTEGER NOT NULL,
k VARCHAR(2048),
v LONGBLOB,
mark TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
version VARCHAR(255) NOT NULL,
PRIMARY KEY (user_id, k(128))
);

1
java/core/src/store/server/db/sql/get_key_value.sql Normal file
View File

@ -0,0 +1 @@
SELECT version, v FROM key_values WHERE user_id = ? AND k = ?

1
java/core/src/store/server/db/sql/get_user_id.sql Normal file
View File

@ -0,0 +1 @@
SELECT user_id FROM users WHERE name = ?

1
java/core/src/store/server/db/sql/get_user_id_and_secret_key.sql Normal file
View File

@ -0,0 +1 @@
SELECT user_id, secret_key FROM key_pairs WHERE key_id = ?

13
java/core/src/store/server/db/sql/list_keys.sql Normal file
View File

@ -0,0 +1,13 @@
# this is really lame and I know it
# but I need to avoid the LIKE clause, until I have time to figure out
# where the function is for proper mysql escaping.
SELECT
k,
mark,
version,
LENGTH(v) as size
FROM
key_values
WHERE
user_id = ? AND LOCATE(?,k)=1

1
java/core/src/store/server/db/sql/put_key_value.sql Normal file
View File

@ -0,0 +1 @@
REPLACE INTO key_values (user_id,k,v,version) VALUES (?,?,?,?)

1
java/core/src/store/server/db/sql/remove_key_value.sql Normal file
View File

@ -0,0 +1 @@
DELETE FROM key_value WHERE user_id = ? AND k = ?

0
java/core/src/store/server/db/sql/remove_user.sql Normal file
View File

1
java/webserver/src/core/constants/ConstantsMvStore.java Symbolic link
View File

@ -0,0 +1 @@
../../../../core/src/core/constants/ConstantsMvStore.java

105
java/webserver/src/mail/web/StoreEnable.java Normal file
View File

@ -0,0 +1,105 @@
package mail.web;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import core.constants.ConstantsMvStore;
import core.server.captcha.Captcha;
import core.util.LogOut;
import core.util.Pair;
import core.util.Strings;
import store.server.StoreServer;
/**
* Servlet implementation class StoreEnable
*/
@WebServlet("/StoreEnable")
public class StoreEnable extends HttpServlet
{
static LogOut log = new LogOut(StoreEnable.class);
private static final long serialVersionUID = 1L;
StoreServer store;
Captcha captcha;
/**
* @see HttpServlet#HttpServlet()
*/
public StoreEnable() throws Exception
{
super();
try
{
store = new StoreServer();
captcha = new Captcha();
captcha.ensureTables();
}
catch (Exception e)
{
log.exception(e);
throw e;
}
}
void doCors(HttpServletResponse response)
{
response.setHeader("Access-Control-Allow-Origin", ConstantsWeb.WEB_SERVER_URL);
response.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
response.setHeader("Access-Control-Allow-Headers", "X-Requested-With");
}
/**
* @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
*/
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
doCors(response);
try
{
String email = request.getParameter("email");
String captchaToken = request.getParameter("captcha");
log.debug("email",email, "captchaToken",captchaToken);
captcha.useToken(captchaToken, Captcha.CreateBucket);
Pair<String,String> result = store.newUserWithKeyPair(email);
ArrayList<String> keyValues = new ArrayList<String>();
keyValues.add(ConstantsMvStore.AccessKeyId + "!" + result.first);
keyValues.add(ConstantsMvStore.SecretKey + "!" + result.second);
response.getWriter().write(Strings.concat(keyValues, "&"));
}
catch (Exception e)
{
log.exception(e);
throw new ServletException(e);
}
}
/**
* @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
*/
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
doGet(request, response);
}
@Override
protected void doOptions (HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
doCors(response);
super.doOptions(request, response);
}
}

88
java/webserver/src/mail/web/StoreGet.java Normal file
View File

@ -0,0 +1,88 @@
package mail.web;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import store.server.ConstantsMvServer;
import store.server.StoreServer;
import core.util.Base64;
import core.util.HttpDelegate;
import core.util.LogOut;
import core.util.Pair;
import core.util.Streams;
/**
* Servlet implementation class StoreGet
*/
@WebServlet("/StoreGet")
public class StoreGet extends HttpServlet
{
static LogOut log = new LogOut(StoreGet.class);
private static final long serialVersionUID = 1L;
StoreServer store;
/**
* @see HttpServlet#HttpServlet()
*/
public StoreGet() throws Exception
{
super();
try
{
store = new StoreServer();
}
catch (Exception e)
{
log.exception(e);
throw e;
}
}
void doCors(HttpServletResponse response)
{
response.setHeader("Access-Control-Allow-Origin", ConstantsWeb.WEB_SERVER_URL);
response.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
response.setHeader("Access-Control-Allow-Headers", "X-Requested-With");
}
protected void doRequest (String action, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
try
{
doCors(response);
String resource = request.getParameter(ConstantsMvServer.PARAMETER_RESOURCE);
String keyId = StoreWebUtils.verifyUser(action, request, resource, null);
Pair<String, byte[]> versionAndData = store.getKeyValue(keyId, resource);
response.setHeader("Version", versionAndData.first);
response.getOutputStream().write(versionAndData.second);
}
catch (Exception e)
{
log.exception(e);
throw new ServletException(e);
}
}
/**
* @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
*/
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
doRequest(HttpDelegate.GET, request, response);
}
@Override
protected void doOptions (HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
doCors(response);
super.doOptions(request, response);
}
}

83
java/webserver/src/mail/web/StoreList.java Normal file
View File

@ -0,0 +1,83 @@
package mail.web;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import store.server.ConstantsMvServer;
import store.server.StoreServer;
import core.util.HttpDelegate;
import core.util.LogOut;
/**
* Servlet implementation class StoreList
*/
@WebServlet("/StoreList")
public class StoreList extends HttpServlet {
private static final long serialVersionUID = 1L;
static LogOut log = new LogOut(StoreList.class);
StoreServer store;
/**
* @see HttpServlet#HttpServlet()
*/
public StoreList() throws Exception
{
super();
try
{
store = new StoreServer();
}
catch (Exception e)
{
log.exception(e);
throw e;
}
}
void doCors(HttpServletResponse response)
{
response.setHeader("Access-Control-Allow-Origin", ConstantsWeb.WEB_SERVER_URL);
response.setHeader("Access-Control-Allow-Methods", "GET, OPTIONS");
response.setHeader("Access-Control-Allow-Headers", "X-Requested-With");
}
protected void doRequest (String action, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
try
{
doCors(response);
String resource = request.getParameter(ConstantsMvServer.PARAMETER_RESOURCE);
String keyId = StoreWebUtils.verifyUser(action, request, resource, null);
String json = StoreWebUtils.transformFileInfoListToJson(store.listKeys(keyId, resource));
response.getOutputStream().write(json.getBytes("UTF-8"));
}
catch (Exception e)
{
log.exception(e);
throw new ServletException(e);
}
}
/**
* @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
*/
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
doRequest(HttpDelegate.GET, request, response);
}
@Override
protected void doOptions (HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
doCors(response);
super.doOptions(request, response);
}
}

85
java/webserver/src/mail/web/StorePut.java Normal file
View File

@ -0,0 +1,85 @@
package mail.web;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import store.server.ConstantsMvServer;
import store.server.StoreServer;
import core.util.HttpDelegate;
import core.util.LogOut;
import core.util.Streams;
/**
* Servlet implementation class StorePut
*/
@WebServlet("/StorePut")
public class StorePut extends HttpServlet {
private static final long serialVersionUID = 1L;
static LogOut log = new LogOut(StorePut.class);
StoreServer store;
/**
* @see HttpServlet#HttpServlet()
*/
public StorePut() throws Exception
{
super();
try
{
store = new StoreServer();
}
catch (Exception e)
{
log.exception(e);
throw e;
}
}
void doCors(HttpServletResponse response)
{
response.setHeader("Access-Control-Allow-Origin", ConstantsWeb.WEB_SERVER_URL);
response.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
response.setHeader("Access-Control-Allow-Headers", "X-Requested-With");
}
protected void doRequest (String action, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
try
{
doCors(response);
byte[] content = Streams.readFullyBytes(request.getInputStream());
String resource = request.getParameter(ConstantsMvServer.PARAMETER_RESOURCE);
String keyId = StoreWebUtils.verifyUser(action, request, resource, content);
String version = store.putKeyValue(keyId, resource, content);
response.setHeader("Version", version);
}
catch (Exception e)
{
log.exception(e);
throw new ServletException(e);
}
}
/**
* @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
*/
protected void doPut(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
doRequest(HttpDelegate.PUT, request, response);
}
@Override
protected void doOptions (HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
doCors(response);
super.doOptions(request, response);
}
}

67
java/webserver/src/mail/web/StoreWebUtils.java Normal file
View File

@ -0,0 +1,67 @@
package mail.web;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.servlet.http.HttpServletRequest;
import org.json.JSONArray;
import org.json.JSONObject;
import core.connector.FileInfo;
import core.util.DateFormat;
import core.util.LogOut;
import store.server.ConstantsMvServer;
import store.server.StoreUtils;
public class StoreWebUtils
{
static LogOut log = new LogOut(StoreWebUtils.class);
public static String verifyUser (String action, HttpServletRequest request, String resource, byte[] content) throws Exception
{
String possibleHeaders[] = {
ConstantsMvServer.HEADER_DATE,
ConstantsMvServer.HEADER_CONTENT_TYPE,
ConstantsMvServer.HEADER_CONTENT_LENGTH,
ConstantsMvServer.HEADER_AUTHORIZATION
};
Map<String, String> headers = new HashMap<String,String>();
for (String possibleHeader : possibleHeaders)
{
String value = request.getHeader(possibleHeader);
if (value != null)
headers.put(possibleHeader, value);
}
return StoreUtils.verifyUser(action, headers, resource, content);
}
static public String transformFileInfoListToJson (List<FileInfo> fileInfos) throws Exception
{
DateFormat dateTimeFormat = new DateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z GMT'");
JSONArray a = new JSONArray();
for (FileInfo fileInfo : fileInfos)
{
JSONObject o = new JSONObject();
o.put("path", fileInfo.path);
o.put("size", fileInfo.size);
o.put("version", fileInfo.version);
o.put("date", dateTimeFormat.format(fileInfo.date));
log.debug ("converted date", fileInfo.date, "to", o.get("date"));
a.put(o);
}
JSONObject r = new JSONObject();
r.put("contents", a);
r.put("isTruncated", false);
return r.toString();
}
}

1
java/webserver/src/store Symbolic link
View File

@ -0,0 +1 @@
../../core/src/store

4
keys/nginx/convert-authority
View File

@ -8,4 +8,6 @@ openssl pkcs12 -in store.p12 -nocerts -out store.key
openssl rsa -in store.key -out final.key
rm final.crt
cat authority-response/* >> final.crt
cat authority-response/server.crt >> final.crt
cat authority-response/PositiveSSLCA2.crt >> final.crt
cat authority-response/AddTrustExternalCARoot.crt >> final.crt

1
passwords/make
View File

@ -4,5 +4,6 @@ echo `./make-single-password` > mail
echo `./make-single-password` > mail-pbe
echo `./make-single-password` > mail_extra
echo `./make-single-password` > push-certificate
echo `./make-single-password` > store

1
passwords/store Normal file
View File

@ -0,0 +1 @@
PASSWORD FOR STORE DB USER

2
web/WebContent/js/mDispatch.js
View File

@ -154,4 +154,4 @@ mDispatch = {
},
};
$(document).ready(setTimeout(function() { mDispatch.startWorker(); }, 250));
// $(document).ready(setTimeout(function() { mDispatch.startWorker(); }, 250));

2
web/common/Delegate.js
View File

@ -103,7 +103,7 @@ var mDelegateCommon =
log("finishing url ", url, " ", this.status);
var getResponseHeadersHack = function(xhr) {
var headers = ['ETag','Content-Type', 'x-dropbox-metadata'];
var headers = ['ETag','Content-Type', 'x-dropbox-metadata', 'Version'];
var result = [];
for (var i=0; i<headers.length; ++i)
{

28
web/signup/SignUp.html
View File

@ -172,25 +172,27 @@
<td class="right-side">
<div class="inner-td">
I want to use the storage provider:<br/>
<label class="checkbox inline"><input name="storage" type="radio" onchange="mSignUp.onStorageChange();" value="mailiverse" checked>Mailiverse</label>
<label class="checkbox inline"><input name="storage" type="radio" onchange="mSignUp.onStorageChange();" value="dropbox">Dropbox</label>
<br/>
<label class="checkbox inline"><input name="storage" type="radio" onchange="mSignUp.onStorageChange();" value="mailiverse" checked>Mailiverse</label><br/>
<label class="checkbox inline"><input name="storage" type="radio" onchange="mSignUp.onStorageChange();" value="s3">Mailiverse (S3 Backed)</label><br>
<label class="checkbox inline"><input name="storage" type="radio" onchange="mSignUp.onStorageChange();" value="dropbox">Dropbox</label><br/>
</div>
</td>
</tr>
<tr id="storage_mailiverse" class="storage-option">
<tr id="storage_s3" class="storage-option" style="display:none">
<td class="left-side">
<div class="inner-td">
<h4>Mailiverse storage</h4>
<h4>Mailiverse S3 backed storage</h4>
We provide infinite mail storage.
</div>
</td>
<td class="right-side">
<div class="inner-td">
Mailiverse storage is simple, no hassle. We take care of everything.<br/>
Mailiverse S3 backed storage is simple, no hassle. We take care of everything.<br/>
<br/>
Pick the region you are closest to most of the time.<br/>
<select id="storage_mailiverse_region">
<select id="storage_s3_region">
<option value="US_Standard" selected="selected">US Standard</option>
<option value="US_West">US-West (Northern California)</option>
<option value="US_West_2">US-West-2 (Oregon)</option>
@ -204,6 +206,20 @@
</td>
</tr>
<tr id="storage_mailiverse" class="storage-option">
<td class="left-side">
<div class="inner-td">
<h4>Mailiverse storage</h4>
We provide infinite mail storage. This storage is backed by our local db.
</div>
</td>
<td class="right-side">
<div class="inner-td">
Mailiverse storage is simple, no hassle. We take care of everything.<br/>
</div>
</td>
</tr>
<tr id="storage_dropbox" class="storage-option" style="display:none">
<td class="left-side">
<div class="inner-td">

11
web/signup/mSignUp.js
View File

@ -20,7 +20,7 @@ mSignUp = {
onStorageChange: function()
{
var possible = [ 'mailiverse', 'dropbox' ];
var possible = [ 's3', 'mailiverse', 'dropbox' ];
var value = $('input[name=storage]:checked').val();
$('#storage_' + value).show();
@ -35,6 +35,11 @@ mSignUp = {
mSignUp.validate['storageAuthorized'] = true;
}
else
if (value == 's3')
{
mSignUp.validate['storageAuthorized'] = true;
}
else
{
mSignUp.validate['storageAuthorized'] = false;
mSignUp.manualTestDropboxAlreadyAuthorized();
@ -296,8 +301,8 @@ mSignUp = {
$('#_mSignUpExecute').show();
var storageInfo = {};
if (mSignUp.storage == "mailiverse")
storageInfo = { region: $('#storage_mailiverse_region').val() };
if (mSignUp.storage == "s3")
storageInfo = { region: $('#storage_s3_region').val() };
var signUpDelegate = {
progress: function(x) {