Merge pull request #36 from whiteout-io/dev/database-migration

introduce update-handler

src/js/app-config.js

@@ -43,7 +43,8 @@ define(function(require) {
         checkOutboxInterval: 5000,
         iconPath: '/img/icon.png',
         verificationUrl: '/verify/',
-        verificationUuidLength: 36
+        verificationUuidLength: 36,
+        dbVersion: 1
     };
 
     /**

src/js/app-controller.js

@@ -17,6 +17,7 @@ define(function(require) {
         OutboxBO = require('js/bo/outbox'),
         PGP = require('js/crypto/pgp'),
         PgpBuilder = require('pgpbuilder'),
+        UpdateHandler = require('js/util/update/update-handler'),
         config = require('js/app-config').config;
 
     var self = {};
@@ -334,23 +335,24 @@ define(function(require) {
     };
 
     self.buildModules = function() {
-        var lawnchairDao, restDao, pubkeyDao, invitationDao,
-            emailDao, keychain, pgp, userStorage, pgpbuilder;
+        var lawnchairDao, restDao, pubkeyDao, emailDao, keychain, pgp, userStorage, pgpbuilder;
 
+        // start the mailreader's worker thread 
         mailreader.startWorker(config.workerPath + '/../lib/mailreader-parser-worker.js');
 
         // init objects and inject dependencies
         restDao = new RestDAO();
         pubkeyDao = new PublicKeyDAO(restDao);
         lawnchairDao = new LawnchairDAO();
-        userStorage = new DeviceStorageDAO(lawnchairDao);
 
-        self._invitationDao = invitationDao = new InvitationDAO(restDao);
+        self._userStorage = userStorage = new DeviceStorageDAO(lawnchairDao);
+        self._invitationDao = new InvitationDAO(restDao);
         self._keychain = keychain = new KeychainDAO(lawnchairDao, pubkeyDao);
         self._crypto = pgp = new PGP();
         self._pgpbuilder = pgpbuilder = new PgpBuilder();
         self._emailDao = emailDao = new EmailDAO(keychain, pgp, userStorage, pgpbuilder, mailreader);
         self._outboxBo = new OutboxBO(emailDao, keychain, userStorage);
+        self._updateHandler = new UpdateHandler(self._appConfigStore, userStorage);
     };
 
     /**
@@ -359,30 +361,52 @@ define(function(require) {
     self.init = function(options, callback) {
         self.buildModules();
 
-        // init email dao
-        var account = {
-            emailAddress: options.emailAddress,
-            asymKeySize: config.asymKeySize
-        };
-
-        self._emailDao.init({
-            account: account
-        }, function(err, keypair) {
+        // init user's local database
+        self._userStorage.init(options.emailAddress, function(err) {
             if (err) {
                 callback(err);
                 return;
             }
 
-            // connect tcp clients on first startup
-            self.onConnect(function(err) {
+            // Migrate the databases if necessary
+            self._updateHandler.update(onUpdate);
+        });
+
+        function onUpdate(err) {
+            if (err) {
+                callback({
+                    errMsg: 'Update failed, please reinstall the app.',
+                    err: err
+                });
+                return;
+            }
+
+            // account information for the email dao
+            var account = {
+                emailAddress: options.emailAddress,
+                asymKeySize: config.asymKeySize
+            };
+
+            // init email dao
+            self._emailDao.init({
+                account: account
+            }, function(err, keypair) {
                 if (err) {
                     callback(err);
                     return;
                 }
 
-                callback(null, keypair);
+                // connect tcp clients on first startup
+                self.onConnect(function(err) {
+                    if (err) {
+                        callback(err);
+                        return;
+                    }
+
+                    callback(null, keypair);
+                });
             });
-        });
+        }
     };
 
     return self;

src/js/dao/email-dao.js

@@ -39,18 +39,15 @@ define(function(require) {
         initKeychain();
 
         function initKeychain() {
-            // init user's local database
-            self._devicestorage.init(emailAddress, function() {
-                // call getUserKeyPair to read/sync keypair with devicestorage/cloud
-                self._keychain.getUserKeyPair(emailAddress, function(err, storedKeypair) {
-                    if (err) {
-                        callback(err);
-                        return;
-                    }
+            // call getUserKeyPair to read/sync keypair with devicestorage/cloud
+            self._keychain.getUserKeyPair(emailAddress, function(err, storedKeypair) {
+                if (err) {
+                    callback(err);
+                    return;
+                }
 
-                    keypair = storedKeypair;
-                    initFolders();
-                });
+                keypair = storedKeypair;
+                initFolders();
             });
         }
 

src/js/util/update/update-handler.js

@@ -0,0 +1,90 @@
+define(function(require) {
+    'use strict';
+
+    var cfg = require('js/app-config').config,
+        updateV1 = require('js/util/update/update-v1');
+
+    /**
+     * Handles database migration
+     */
+    var UpdateHandler = function(appConfigStorage, userStorage) {
+        this._appConfigStorage = appConfigStorage;
+        this._userStorage = userStorage;
+        this._updateScripts = [updateV1];
+    };
+
+    /**
+     * Executes all the necessary updates
+     * @param  {Function} callback(error) Invoked when all the database updates were executed, or if an error occurred
+     */
+    UpdateHandler.prototype.update = function(callback) {
+        var self = this,
+            currentVersion = 0,
+            targetVersion = cfg.dbVersion,
+            versionDbType = 'dbVersion';
+
+        self._appConfigStorage.listItems(versionDbType, 0, null, function(err, items) {
+            if (err) {
+                callback(err);
+                return;
+            }
+
+            // parse the database version number
+            if (items && items.length > 0) {
+                currentVersion = parseInt(items[0], 10);
+            }
+
+            self._applyUpdate({
+                currentVersion: currentVersion,
+                targetVersion: targetVersion
+            }, callback);
+        });
+    };
+
+    /**
+     * Schedules necessary updates and executes thom in order
+     */
+    UpdateHandler.prototype._applyUpdate = function(options, callback) {
+        var self = this,
+            storage,
+            queue = [];
+
+        if (options.currentVersion >= options.targetVersion) {
+            // the current database version is up to date
+            callback();
+            return;
+        }
+
+        storage = {
+            appConfigStorage: self._appConfigStorage,
+            userStorage: self._userStorage
+        };
+
+        // add all the necessary database updates to the queue
+        for (var i = options.currentVersion; i < options.targetVersion; i++) {
+            queue.push(self._updateScripts[i]);
+        }
+
+        // takes the next update from the queue and executes it
+        function executeNextUpdate(err) {
+            if (err) {
+                callback(err);
+                return;
+            }
+
+            if (queue.length < 1) {
+                // we're done
+                callback();
+                return;
+            }
+
+            // process next update
+            var script = queue.shift();
+            script(storage, executeNextUpdate);
+        }
+
+        executeNextUpdate();
+    };
+
+    return UpdateHandler;
+});

src/js/util/update/update-v1.js

@@ -0,0 +1,29 @@
+define(function() {
+    'use strict';
+
+    /**
+     * Update handler for transition database version 0 -> 1
+     *
+     * In database version 1, the stored email objects have to be purged, otherwise
+     * every non-prefixed mail in the IMAP folders would be nuked due to the implementation
+     * of the delta sync.
+     */
+    function updateV1(options, callback) {
+        var emailDbType = 'email_',
+            versionDbType = 'dbVersion',
+            postUpdateDbVersion = 1;
+
+        // remove the emails
+        options.userStorage.removeList(emailDbType, function(err) {
+            if (err) {
+                callback(err);
+                return;
+            }
+
+            // update the database version to postUpdateDbVersion
+            options.appConfigStorage.storeList([postUpdateDbVersion], versionDbType, callback);
+        });
+    }
+
+    return updateV1;
+});

test/new-unit/app-controller-test.js

@@ -5,19 +5,19 @@ define(function(require) {
         EmailDAO = require('js/dao/email-dao'),
         OutboxBO = require('js/bo/outbox'),
         DeviceStorageDAO = require('js/dao/devicestorage-dao'),
+        UpdateHandler = require('js/util/update/update-handler'),
         expect = chai.expect;
 
     describe('App Controller unit tests', function() {
-        var emailDaoStub, outboxStub, appConfigStoreStub, isOnlineStub,
+        var emailDaoStub, outboxStub, updateHandlerStub, appConfigStoreStub, devicestorageStub, isOnlineStub,
             identityStub;
 
         beforeEach(function() {
-            emailDaoStub = sinon.createStubInstance(EmailDAO);
-            controller._emailDao = emailDaoStub;
-            outboxStub = sinon.createStubInstance(OutboxBO);
-            controller._outboxBo = outboxStub;
-            appConfigStoreStub = sinon.createStubInstance(DeviceStorageDAO);
-            controller._appConfigStore = appConfigStoreStub;
+            controller._emailDao = emailDaoStub = sinon.createStubInstance(EmailDAO);
+            controller._outboxBo = outboxStub = sinon.createStubInstance(OutboxBO);
+            controller._appConfigStore = appConfigStoreStub = sinon.createStubInstance(DeviceStorageDAO);
+            controller._userStorage = devicestorageStub = sinon.createStubInstance(DeviceStorageDAO);
+            controller._updateHandler = updateHandlerStub = sinon.createStubInstance(UpdateHandler);
 
             isOnlineStub = sinon.stub(controller, 'isOnline');
 
@@ -226,9 +226,11 @@ define(function(require) {
         });
 
         describe('init', function() {
-            var buildModulesStub, onConnectStub;
+            var buildModulesStub, onConnectStub, emailAddress;
 
             beforeEach(function() {
+                emailAddress = 'alice@bob.com';
+
                 // buildModules
                 buildModulesStub = sinon.stub(controller, 'buildModules');
                 buildModulesStub.returns();
@@ -241,42 +243,88 @@ define(function(require) {
                 onConnectStub.restore();
             });
 
-            it('should fail due to error in emailDao.init', function(done) {
-                emailDaoStub.init.yields({});
+            it('should fail due to error in storage initialization', function(done) {
+                devicestorageStub.init.withArgs(undefined).yields({});
 
                 controller.init({}, function(err, keypair) {
                     expect(err).to.exist;
                     expect(keypair).to.not.exist;
+                    expect(devicestorageStub.init.calledOnce).to.be.true;
+                    expect(updateHandlerStub.update.calledOnce).to.be.false;
+                    done();
+                });
+            });
+
+            it('should fail due to error in update handler', function(done) {
+                devicestorageStub.init.yields();
+                updateHandlerStub.update.yields({});
+
+                controller.init({
+                    emailAddress: emailAddress
+                }, function(err, keypair) {
+                    expect(err).to.exist;
+                    expect(keypair).to.not.exist;
+                    expect(updateHandlerStub.update.calledOnce).to.be.true;
+                    expect(devicestorageStub.init.calledOnce).to.be.true;
+                    done();
+                });
+            });
+
+            it('should fail due to error in emailDao.init', function(done) {
+                devicestorageStub.init.yields();
+                updateHandlerStub.update.yields();
+                emailDaoStub.init.yields({});
+
+                controller.init({
+                    emailAddress: emailAddress
+                }, function(err, keypair) {
+                    expect(err).to.exist;
+                    expect(keypair).to.not.exist;
+                    expect(updateHandlerStub.update.calledOnce).to.be.true;
+                    expect(emailDaoStub.init.calledOnce).to.be.true;
+                    expect(devicestorageStub.init.calledOnce).to.be.true;
                     done();
                 });
             });
 
             it('should fail due to error in onConnect', function(done) {
+                devicestorageStub.init.yields();
+                updateHandlerStub.update.yields();
                 emailDaoStub.init.yields();
 
                 onConnectStub.yields({});
 
-                controller.init({}, function(err) {
+                controller.init({
+                    emailAddress: emailAddress
+                }, function(err) {
                     expect(err).to.exist;
+                    expect(updateHandlerStub.update.calledOnce).to.be.true;
+                    expect(emailDaoStub.init.calledOnce).to.be.true;
+                    expect(devicestorageStub.init.calledOnce).to.be.true;
                     expect(onConnectStub.calledOnce).to.be.true;
                     done();
                 });
             });
 
             it('should work and return a keypair', function(done) {
+                devicestorageStub.init.withArgs(emailAddress).yields();
                 emailDaoStub.init.yields(null, {});
+                updateHandlerStub.update.yields();
 
                 onConnectStub.yields();
 
-                controller.init({}, function(err, keypair) {
+                controller.init({
+                    emailAddress: emailAddress
+                }, function(err, keypair) {
                     expect(err).to.not.exist;
                     expect(keypair).to.exist;
+                    expect(updateHandlerStub.update.calledOnce).to.be.true;
+                    expect(emailDaoStub.init.calledOnce).to.be.true;
+                    expect(devicestorageStub.init.calledOnce).to.be.true;
                     expect(onConnectStub.calledOnce).to.be.true;
                     done();
                 });
             });
         });
-
     });
-
 });

test/new-unit/email-dao-test.js

@@ -177,7 +177,6 @@ define(function(require) {
                 folders = [{}, {}];
 
                 // initKeychain
-                devicestorageStub.init.withArgs(emailAddress).yields();
                 keychainStub.getUserKeyPair.yields(null, mockKeyPair);
 
                 // initFolders
@@ -194,7 +193,6 @@ define(function(require) {
 
                     expect(dao._account).to.equal(account);
                     expect(dao._account.folders).to.equal(folders);
-                    expect(devicestorageStub.init.calledOnce).to.be.true;
                     expect(keychainStub.getUserKeyPair.calledOnce).to.be.true;
 
                     expect(listFolderStub.calledOnce).to.be.true;
@@ -207,7 +205,6 @@ define(function(require) {
                 var listFolderStub;
 
                 // initKeychain
-                devicestorageStub.init.withArgs(emailAddress).yields();
                 keychainStub.getUserKeyPair.yields(null, mockKeyPair);
 
                 // initFolders
@@ -226,7 +223,6 @@ define(function(require) {
 
                     expect(dao._account).to.equal(account);
                     expect(dao._account.folders).to.equal(undefined);
-                    expect(devicestorageStub.init.calledOnce).to.be.true;
                     expect(keychainStub.getUserKeyPair.calledOnce).to.be.true;
                     expect(listFolderStub.calledOnce).to.be.true;
 
@@ -238,7 +234,6 @@ define(function(require) {
                 var listFolderStub;
 
                 // initKeychain
-                devicestorageStub.init.withArgs(emailAddress).yields();
                 keychainStub.getUserKeyPair.yields(null, mockKeyPair);
 
                 // initFolders
@@ -252,7 +247,6 @@ define(function(require) {
                     expect(keyPair).to.not.exist;
 
                     expect(dao._account).to.equal(account);
-                    expect(devicestorageStub.init.calledOnce).to.be.true;
                     expect(keychainStub.getUserKeyPair.calledOnce).to.be.true;
                     expect(listFolderStub.calledOnce).to.be.true;
 
@@ -261,7 +255,6 @@ define(function(require) {
             });
 
             it('should fail due to error in getUserKeyPair', function(done) {
-                devicestorageStub.init.yields();
                 keychainStub.getUserKeyPair.yields({});
 
                 dao.init({
@@ -270,8 +263,6 @@ define(function(require) {
                     expect(err).to.exist;
                     expect(keyPair).to.not.exist;
 
-                    expect(devicestorageStub.init.calledOnce).to.be.true;
-
                     done();
                 });
             });

test/new-unit/main.js

@@ -49,7 +49,8 @@ function startTests() {
 			'test/new-unit/mail-list-ctrl-test',
 			'test/new-unit/write-ctrl-test',
 			'test/new-unit/outbox-bo-test',
-			'test/new-unit/invitation-dao-test'
+			'test/new-unit/invitation-dao-test',
+			'test/new-unit/update-handler-test'
 		], function() {
 			//Tests loaded, run tests
 			mocha.run();

test/new-unit/update-handler-test.js

@@ -0,0 +1,165 @@
+define(function(require) {
+    'use strict';
+
+    var DeviceStorageDAO = require('js/dao/devicestorage-dao'),
+        cfg = require('js/app-config').config,
+        UpdateHandler = require('js/util/update/update-handler'),
+        expect = chai.expect;
+
+    chai.Assertion.includeStack = true;
+
+    describe('UpdateHandler', function() {
+        var updateHandler, appConfigStorageStub, userStorageStub, origDbVersion;
+
+        beforeEach(function() {
+            origDbVersion = cfg.dbVersion;
+            appConfigStorageStub = sinon.createStubInstance(DeviceStorageDAO);
+            userStorageStub = sinon.createStubInstance(DeviceStorageDAO);
+            updateHandler = new UpdateHandler(appConfigStorageStub, userStorageStub);
+        });
+
+        afterEach(function() {
+            cfg.dbVersion = origDbVersion;
+        });
+
+        describe('#constructor', function() {
+            it('should create instance', function() {
+                expect(updateHandler).to.exist;
+                expect(updateHandler._appConfigStorage).to.equal(appConfigStorageStub);
+                expect(updateHandler._userStorage).to.equal(userStorageStub);
+
+                // the update handler must contain as many db update sripts as there are database versions
+                expect(updateHandler._updateScripts.length).to.equal(cfg.dbVersion);
+            });
+        });
+
+        describe('#update', function() {
+            var versionDbType = 'dbVersion';
+
+            it('should not update when up to date', function(done) {
+                cfg.dbVersion = 10; // app requires database version 10
+                appConfigStorageStub.listItems.withArgs(versionDbType).yieldsAsync(null, ['10']); // database version is 10
+
+                updateHandler.update(function(error) {
+                    expect(error).to.not.exist;
+                    expect(appConfigStorageStub.listItems.calledOnce).to.be.true;
+
+                    done();
+                });
+            });
+
+            describe('dummy updates for v2 to v4', function() {
+                var updateCounter;
+
+                beforeEach(function() {
+                    updateCounter = 0;
+                    appConfigStorageStub.listItems.withArgs(versionDbType).yieldsAsync(null, ['2']); // database version is 0
+                });
+
+                afterEach(function() {
+                    expect(appConfigStorageStub.listItems.calledOnce).to.be.true;
+                });
+
+
+                it('should work', function(done) {
+                    cfg.dbVersion = 4; // app requires database version 4
+
+                    // a simple dummy update to executed that only increments the update counter
+                    function dummyUpdate(options, callback) {
+                        updateCounter++;
+                        callback();
+                    }
+
+                    // inject the dummy updates instead of live ones
+                    updateHandler._updateScripts = [dummyUpdate, dummyUpdate, dummyUpdate, dummyUpdate];
+
+                    // execute test
+                    updateHandler.update(function(error) {
+                        expect(error).to.not.exist;
+                        expect(updateCounter).to.equal(2);
+
+                        done();
+                    });
+                });
+
+                it('should fail while updating to v3', function(done) {
+                    cfg.dbVersion = 4; // app requires database version 4
+
+                    function dummyUpdate(options, callback) {
+                        updateCounter++;
+                        callback();
+                    }
+
+                    function failingUpdate(options, callback) {
+                        callback({});
+                    }
+
+                    // inject the dummy updates instead of live ones
+                    updateHandler._updateScripts = [dummyUpdate, dummyUpdate, failingUpdate, dummyUpdate];
+
+                    // execute test
+                    updateHandler.update(function(error) {
+                        expect(error).to.exist;
+                        expect(updateCounter).to.equal(0);
+
+                        done();
+                    });
+                });
+
+            });
+
+            describe('v0 -> v1', function() {
+                var emailDbType = 'email_';
+
+                beforeEach(function() {
+                    cfg.dbVersion = 1; // app requires database version 1
+                    appConfigStorageStub.listItems.withArgs(versionDbType).yieldsAsync(); // database version is 0
+                });
+
+                afterEach(function() {
+                    // database version is only queried for version checking prior to the update script
+                    // so no need to check this in case-specific tests
+                    expect(appConfigStorageStub.listItems.calledOnce).to.be.true;
+                });
+
+                it('should work', function(done) {
+                    userStorageStub.removeList.withArgs(emailDbType).yieldsAsync();
+                    appConfigStorageStub.storeList.withArgs([1], versionDbType).yieldsAsync();
+
+                    updateHandler.update(function(error) {
+                        expect(error).to.not.exist;
+                        expect(userStorageStub.removeList.calledOnce).to.be.true;
+                        expect(appConfigStorageStub.storeList.calledOnce).to.be.true;
+
+                        done();
+                    });
+                });
+
+                it('should fail when persisting database version fails', function(done) {
+                    userStorageStub.removeList.yieldsAsync();
+                    appConfigStorageStub.storeList.yieldsAsync({});
+
+                    updateHandler.update(function(error) {
+                        expect(error).to.exist;
+                        expect(userStorageStub.removeList.calledOnce).to.be.true;
+                        expect(appConfigStorageStub.storeList.calledOnce).to.be.true;
+
+                        done();
+                    });
+                });
+
+                it('should fail when wiping emails from database fails', function(done) {
+                    userStorageStub.removeList.yieldsAsync({});
+
+                    updateHandler.update(function(error) {
+                        expect(error).to.exist;
+                        expect(userStorageStub.removeList.calledOnce).to.be.true;
+                        expect(appConfigStorageStub.storeList.called).to.be.false;
+
+                        done();
+                    });
+                });
+            });
+        });
+    });
+});