From 77e5e2a97e9a497f8d4fa150819dbc4b43fbaa02 Mon Sep 17 00:00:00 2001 From: Felix Hammerl Date: Fri, 23 May 2014 14:23:50 +0200 Subject: [PATCH 1/5] [WO-373] refactor sync remove email-sync.js remove delta sync minor refactorings --- package.json | 2 +- src/js/app-config.js | 9 +- src/js/app-controller.js | 18 +- src/js/bo/outbox.js | 4 +- src/js/controller/mail-list.js | 350 ++-- src/js/controller/navigation.js | 71 +- src/js/controller/read.js | 2 +- src/js/controller/write.js | 59 +- src/js/dao/email-dao.js | 948 +++++++-- src/js/dao/email-sync.js | 848 -------- src/js/util/update/update-handler.js | 5 +- src/js/util/update/update-v3.js | 28 + src/tpl/mail-list.html | 2 +- test/new-unit/app-controller-test.js | 13 +- test/new-unit/email-dao-test.js | 2770 ++++++++++++++++---------- test/new-unit/email-sync-test.js | 1652 --------------- test/new-unit/mail-list-ctrl-test.js | 99 +- test/new-unit/main.js | 1 - test/new-unit/update-handler-test.js | 53 + test/new-unit/write-ctrl-test.js | 4 +- 20 files changed, 2796 insertions(+), 4142 deletions(-) delete mode 100644 src/js/dao/email-sync.js create mode 100644 src/js/util/update/update-v3.js delete mode 100644 test/new-unit/email-sync-test.js diff --git a/package.json b/package.json index e9b91c7..b898d04 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ }, "dependencies": { "crypto-lib": "https://github.com/whiteout-io/crypto-lib/tarball/v0.1.1", - "imap-client": "https://github.com/whiteout-io/imap-client/tarball/v0.3.2", + "imap-client": "https://github.com/whiteout-io/imap-client/tarball/dev/WO-373", "mailreader": "https://github.com/whiteout-io/mailreader/tarball/v0.3.2", "pgpmailer": "https://github.com/whiteout-io/pgpmailer/tarball/v0.3.2", "pgpbuilder": "https://github.com/whiteout-io/pgpbuilder/tarball/v0.3.2", diff --git a/src/js/app-config.js b/src/js/app-config.js index cc0f818..38be9a9 100644 --- a/src/js/app-config.js +++ b/src/js/app-config.js @@ -46,15 +46,16 @@ define(function(require) { iconPath: '/img/icon.png', verificationUrl: '/verify/', verificationUuidLength: 36, - dbVersion: 2, - appVersion: appVersion + dbVersion: 3, + appVersion: appVersion, + outboxMailboxPath: 'OUTBOX', + outboxMailboxType: 'Outbox' }; /** * Strings are maintained here */ app.string = { - subjectPrefix: '[whiteout] ', fallbackSubject: '(no subject)', invitationSubject: 'Invitation to a private conversation', invitationMessage: 'Hi,\n\nI use Whiteout Mail to send and receive encrypted email. I would like to exchange encrypted messages with you as well.\n\nPlease install the Whiteout Mail application. This application makes it easy to read and write messages securely with PGP encryption applied.\n\nGo to the Whiteout Networks homepage to learn more and to download the application: https://whiteout.io\n\n', @@ -63,7 +64,7 @@ define(function(require) { cryptSuffix: '-----END PGP MESSAGE-----', signature: '\n\n\n--\nSent from Whiteout Mail - Email encryption for the rest of us\nhttps://whiteout.io\n\n', webSite: 'http://whiteout.io', - verificationSubject: 'New public key uploaded', + verificationSubject: '[whiteout] New public key uploaded', sendBtnClear: 'Send', sendBtnSecure: 'Send securely' }; diff --git a/src/js/app-controller.js b/src/js/app-controller.js index f00c318..0ad3a41 100644 --- a/src/js/app-controller.js +++ b/src/js/app-controller.js @@ -15,7 +15,6 @@ define(function(require) { RestDAO = require('js/dao/rest-dao'), EmailDAO = require('js/dao/email-dao'), config = require('js/app-config').config, - EmailSync = require('js/dao/email-sync'), KeychainDAO = require('js/dao/keychain-dao'), PublicKeyDAO = require('js/dao/publickey-dao'), LawnchairDAO = require('js/dao/lawnchair-dao'), @@ -43,18 +42,18 @@ define(function(require) { function onDeviceReady() { console.log('Starting app.'); - self.buildModules(); + self.buildModules(options); // Handle offline and online gracefully window.addEventListener('online', self.onConnect.bind(self, options.onError)); - window.addEventListener('offline', self.onDisconnect.bind(self, options.onError)); + window.addEventListener('offline', self.onDisconnect.bind(self)); self._appConfigStore.init('app-config', callback); } }; - self.buildModules = function() { - var lawnchairDao, restDao, pubkeyDao, emailDao, emailSync, keychain, pgp, userStorage, pgpbuilder, oauth, appConfigStore; + self.buildModules = function(options) { + var lawnchairDao, restDao, pubkeyDao, emailDao, keychain, pgp, userStorage, pgpbuilder, oauth, appConfigStore; // start the mailreader's worker thread mailreader.startWorker(config.workerPath + '/../lib/mailreader-parser-worker.js'); @@ -72,18 +71,19 @@ define(function(require) { self._keychain = keychain = new KeychainDAO(lawnchairDao, pubkeyDao); self._crypto = pgp = new PGP(); self._pgpbuilder = pgpbuilder = new PgpBuilder(); - self._emailSync = emailSync = new EmailSync(keychain, userStorage, mailreader); - self._emailDao = emailDao = new EmailDAO(keychain, pgp, userStorage, pgpbuilder, mailreader, emailSync); + self._emailDao = emailDao = new EmailDAO(keychain, pgp, userStorage, pgpbuilder, mailreader); self._outboxBo = new OutboxBO(emailDao, keychain, userStorage); self._updateHandler = new UpdateHandler(appConfigStore, userStorage); + + emailDao.onError = options.onError; }; self.isOnline = function() { return navigator.onLine; }; - self.onDisconnect = function(callback) { - self._emailDao.onDisconnect(null, callback); + self.onDisconnect = function() { + self._emailDao.onDisconnect(); }; self.onConnect = function(callback) { diff --git a/src/js/bo/outbox.js b/src/js/bo/outbox.js index 7d04a52..a9dd582 100644 --- a/src/js/bo/outbox.js +++ b/src/js/bo/outbox.js @@ -60,7 +60,7 @@ define(function(require) { allReaders = mail.from.concat(mail.to.concat(mail.cc.concat(mail.bcc))); // all the users that should be able to read the mail mail.publicKeysArmored = []; // gather the public keys - mail.id = util.UUID(); // the mail needs a random uuid for storage in the database + mail.uid = mail.id = util.UUID(); // the mail needs a random id & uid for storage in the database // do not encrypt mails with a bcc recipient, due to a possible privacy leak if (mail.bcc.length > 0) { @@ -216,7 +216,7 @@ define(function(require) { // removes the mail object from disk after successfully sending it function removeFromStorage(mail, done) { - self._devicestorage.removeList(outboxDb + '_' + mail.id, function(err) { + self._devicestorage.removeList(outboxDb + '_' + mail.uid, function(err) { if (err) { self._outboxBusy = false; callback(err); diff --git a/src/js/controller/mail-list.js b/src/js/controller/mail-list.js index 08ce610..a6c19f0 100644 --- a/src/js/controller/mail-list.js +++ b/src/js/controller/mail-list.js @@ -6,7 +6,7 @@ define(function(require) { appController = require('js/app-controller'), IScroll = require('iscroll'), notification = require('js/util/notification'), - emailDao, outboxBo, emailSync; + emailDao, outboxBo; var MailListCtrl = function($scope, $timeout) { // @@ -15,7 +15,6 @@ define(function(require) { emailDao = appController._emailDao; outboxBo = appController._outboxBo; - emailSync = appController._emailSync; // // scope functions @@ -23,7 +22,7 @@ define(function(require) { $scope.getBody = function(email) { emailDao.getBody({ - folder: currentFolder().path, + folder: currentFolder(), message: email }, function(err) { if (err && err.code !== 42) { @@ -35,7 +34,7 @@ define(function(require) { $scope.$digest(); // automatically decrypt if it's the selected email - if (email === $scope.state.mailList.selected) { + if (email === currentMessage()) { emailDao.decryptBody({ message: email }, $scope.onError); @@ -66,49 +65,24 @@ define(function(require) { return; } - email.unread = false; - $scope.synchronize(); + $scope.toggleUnread(email); }; /** * Mark an email as unread or read, respectively */ - $scope.toggleUnread = function(email) { - email.unread = !email.unread; - $scope.synchronize(); - }; - - /** - * Synchronize the selected imap folder to local storage - */ - $scope.synchronize = function(options) { - updateStatus('Syncing ...'); - - options = options || {}; - options.folder = options.folder || currentFolder().path; - - // let email dao handle sync transparently - if (currentFolder().type === 'Outbox') { - emailDao.syncOutbox({ - folder: currentFolder().path - }, done); - } else { - emailDao.sync({ - folder: options.folder || currentFolder().path - }, done); - } - - - function done(err) { - if (err && err.code === 409) { - // sync still busy - return; - } + $scope.toggleUnread = function(message) { + updateStatus('Updating unread flag...'); + message.unread = !message.unread; + emailDao.setFlags({ + folder: currentFolder(), + message: message + }, function(err) { if (err && err.code === 42) { - // offline - updateStatus('Offline mode'); - $scope.$apply(); + // offline, restore + message.unread = !message.unread; + updateStatus('Unable to mark unread flag in offline mode!'); return; } @@ -118,59 +92,46 @@ define(function(require) { return; } - // display last update - updateStatus('Last update: ', new Date()); - - // do not change the selection if we just updated another folder in the background - if (currentFolder().path === options.folder) { - selectFirstMessage(); - } - + updateStatus('Flag updated!'); $scope.$apply(); - - // fetch visible bodies at the end of a successful sync - $scope.loadVisibleBodies(); - } + }); }; /** - * Delete an email by moving it to the trash folder or purging it. + * Delete a message */ - $scope.remove = function(email) { - if (!email) { + $scope.remove = function(message) { + if (!message) { return; } - if (currentFolder().type === 'Outbox') { - $scope.onError({ - errMsg: 'Deleting messages from the outbox is not yet supported.' + updateStatus('Deleting message...'); + remove(); + + function remove() { + emailDao.deleteMessage({ + folder: currentFolder(), + message: message + }, function(err) { + if (err) { + // show errors where appropriate + if (err.code === 42) { + $scope.select(message); + updateStatus('Unable to delete message in offline mode!'); + return; + } + updateStatus('Error during delete!'); + $scope.onError(err); + } + updateStatus('Message deleted!'); + $scope.$apply(); }); - return; - } - - removeAndShowNext(); - $scope.synchronize(); - - function removeAndShowNext() { - var index = currentFolder().messages.indexOf(email); - // show the next mail - if (currentFolder().messages.length > 1) { - // if we're about to delete the last entry of the array, show the previous (i.e. the one below in the list), - // otherwise show the next one (i.e. the one above in the list) - $scope.select(_.last(currentFolder().messages) === email ? currentFolder().messages[index - 1] : currentFolder().messages[index + 1]); - } else { - // if we have only one email in the array, show nothing - $scope.select(); - $scope.state.mailList.selected = undefined; - } - currentFolder().messages.splice(index, 1); } }; // share local scope functions with root state $scope.state.mailList = { - remove: $scope.remove, - synchronize: $scope.synchronize + remove: $scope.remove }; // @@ -185,81 +146,84 @@ define(function(require) { return; } - // development... display dummy mail objects + // in development, display dummy mail objects if (!window.chrome || !chrome.identity) { updateStatus('Last update: ', new Date()); currentFolder().messages = createDummyMails(); - selectFirstMessage(); return; } - // production... in chrome packaged app - - // unselect selection from old folder - $scope.select(); // display and select first - selectFirstMessage(); - - $scope.synchronize(); + openCurrentFolder(); }); + $scope.$watch('state.nav.currentFolder.messages', selectFirstMessage); + $scope.$watch('state.nav.currentFolder.messages.length', selectFirstMessage); + + function selectFirstMessage() { + if (!currentMessages()) { + return; + } + + // Shows the next message based on the uid of the currently selected element + if (currentMessages().indexOf(currentMessage()) === -1) { + // wait until after first $digest() so $scope.filteredMessages is set + $timeout(function() { + $scope.select($scope.filteredMessages ? $scope.filteredMessages[0] : undefined); + }); + } + } + /** * Sync current folder when client comes back online */ $scope.$watch('account.online', function(isOnline) { if (isOnline) { - $scope.synchronize(); + updateStatus('Online'); + openCurrentFolder(); } else { updateStatus('Offline mode'); } }, true); // - // helper functions + // Helper Functions // + function openCurrentFolder() { + emailDao.openFolder({ + folder: currentFolder() + }, function(error) { + if (error && error.code === 42) { + return; + } + + $scope.onError(error); + }); + } + function updateStatus(lbl, time) { $scope.lastUpdateLbl = lbl; $scope.lastUpdate = (time) ? time : ''; } - function selectFirstMessage() { - // wait until after first $digest() so $scope.filteredMessages is set - $timeout(function() { - var emails = $scope.filteredMessages; - - if (!emails || emails.length < 1) { - $scope.select(); - return; - } - - if (!$scope.state.mailList.selected) { - // select first message - $scope.select(emails[0]); - } - }); - } - function currentFolder() { return $scope.state.nav.currentFolder; } - if (!emailDao || !emailSync) { - return; // development mode + function currentMessages() { + return currentFolder() && currentFolder().messages; } - emailDao.onNeedsSync = function(error, folder) { - if (error) { - $scope.onError(error); - return; - } + function currentMessage() { + return $scope.state.mailList.selected; + } - $scope.synchronize({ - folder: folder - }); - }; + // + // Notification API + // - emailSync.onIncomingMessage = function(msgs) { + (emailDao || {}).onIncomingMessage = function(msgs) { var popupId, popupTitle, popupMessage, unreadMsgs; unreadMsgs = msgs.filter(function(msg) { @@ -299,6 +263,81 @@ define(function(require) { }); }; + // + // Directives + // + + var ngModule = angular.module('mail-list', []); + + ngModule.directive('ngIscroll', function($timeout) { + return { + link: function(scope, elm, attrs) { + var model = attrs.ngIscroll, + listEl = elm[0], + myScroll; + + /* + * iterates over the mails in the mail list and loads their bodies if they are visible in the viewport + */ + scope.loadVisibleBodies = function() { + var listBorder = listEl.getBoundingClientRect(), + top = listBorder.top, + bottom = listBorder.bottom, + listItems = listEl.children[0].children, + inViewport = false, + listItem, message, + isPartiallyVisibleTop, isPartiallyVisibleBottom, isVisible; + + for (var i = 0, len = listItems.length; i < len; i++) { + // the n-th list item (the dom representation of an email) corresponds to + // the n-th message model in the filteredMessages array + listItem = listItems.item(i).getBoundingClientRect(); + + if (!scope.filteredMessages || scope.filteredMessages.length <= i) { + // stop if i get larger than the size of filtered messages + break; + } + message = scope.filteredMessages[i]; + + + isPartiallyVisibleTop = listItem.top < top && listItem.bottom > top; // a portion of the list item is visible on the top + isPartiallyVisibleBottom = listItem.top < bottom && listItem.bottom > bottom; // a portion of the list item is visible on the bottom + isVisible = listItem.top >= top && listItem.bottom <= bottom; // the list item is visible as a whole + + if (isPartiallyVisibleTop || isVisible || isPartiallyVisibleBottom) { + // we are now iterating over visible elements + inViewport = true; + // load mail body of visible + scope.getBody(message); + } else if (inViewport) { + // we are leaving the viewport, so stop iterating over the items + break; + } + } + }; + + // activate iscroll + myScroll = new IScroll(listEl, { + mouseWheel: true, + scrollbars: true, + fadeScrollbars: true + }); + myScroll.on('scrollEnd', scope.loadVisibleBodies); + + // refresh iScroll when model length changes + scope.$watchCollection(model, function() { + $timeout(function() { + myScroll.refresh(); + }); + // load the visible message bodies, when the list is re-initialized and when scrolling stopped + scope.loadVisibleBodies(); + }); + } + }; + }); + + // Helper for development mode + function createDummyMails() { var uid = 0; @@ -428,78 +467,5 @@ define(function(require) { return dummys; } - // - // Directives - // - - var ngModule = angular.module('mail-list', []); - - ngModule.directive('ngIscroll', function($timeout) { - return { - link: function(scope, elm, attrs) { - var model = attrs.ngIscroll, - listEl = elm[0], - myScroll; - - /* - * iterates over the mails in the mail list and loads their bodies if they are visible in the viewport - */ - scope.loadVisibleBodies = function() { - var listBorder = listEl.getBoundingClientRect(), - top = listBorder.top, - bottom = listBorder.bottom, - listItems = listEl.children[0].children, - inViewport = false, - listItem, message, - isPartiallyVisibleTop, isPartiallyVisibleBottom, isVisible; - - for (var i = 0, len = listItems.length; i < len; i++) { - // the n-th list item (the dom representation of an email) corresponds to - // the n-th message model in the filteredMessages array - listItem = listItems.item(i).getBoundingClientRect(); - - if (!scope.filteredMessages || scope.filteredMessages.length <= i) { - // stop if i get larger than the size of filtered messages - break; - } - message = scope.filteredMessages[i]; - - - isPartiallyVisibleTop = listItem.top < top && listItem.bottom > top; // a portion of the list item is visible on the top - isPartiallyVisibleBottom = listItem.top < bottom && listItem.bottom > bottom; // a portion of the list item is visible on the bottom - isVisible = listItem.top >= top && listItem.bottom <= bottom; // the list item is visible as a whole - - if (isPartiallyVisibleTop || isVisible || isPartiallyVisibleBottom) { - // we are now iterating over visible elements - inViewport = true; - // load mail body of visible - scope.getBody(message); - } else if (inViewport) { - // we are leaving the viewport, so stop iterating over the items - break; - } - } - }; - - // activate iscroll - myScroll = new IScroll(listEl, { - mouseWheel: true, - scrollbars: true, - fadeScrollbars: true - }); - myScroll.on('scrollEnd', scope.loadVisibleBodies); - - // refresh iScroll when model length changes - scope.$watchCollection(model, function() { - $timeout(function() { - myScroll.refresh(); - }); - // load the visible message bodies, when the list is re-initialized and when scrolling stopped - scope.loadVisibleBodies(); - }); - } - }; - }); - return MailListCtrl; }); \ No newline at end of file diff --git a/src/js/controller/navigation.js b/src/js/controller/navigation.js index 6fc2c35..e9b1e3d 100644 --- a/src/js/controller/navigation.js +++ b/src/js/controller/navigation.js @@ -2,8 +2,8 @@ define(function(require) { 'use strict'; var angular = require('angular'), - str = require('js/app-config').string, appController = require('js/app-controller'), + config = require('js/app-config').config, notification = require('js/util/notification'), _ = require('underscore'), emailDao, outboxBo; @@ -38,19 +38,16 @@ define(function(require) { return; } - // update the outbox mail count. this should normally happen during the delta sync - // problem is that the outbox continuously retries in the background, whereas the delta sync only runs - // when the outbox is currently viewed... + // update the outbox mail count var outbox = _.findWhere($scope.account.folders, { - type: 'Outbox' + type: config.outboxMailboxType }); - - if (outbox === $scope.state.nav.currentFolder) { - $scope.state.mailList.synchronize(); - } else { - outbox.count = count; - $scope.$apply(); - } + outbox.count = count; + $scope.$apply(); + + emailDao.refreshFolder({ + folder: outbox + }, $scope.onError); }; // @@ -58,7 +55,7 @@ define(function(require) { // // init folders - initFolders(); + initializeFolders(); // select inbox as the current folder on init if ($scope.account.folders && $scope.account.folders.length > 0) { @@ -82,20 +79,34 @@ define(function(require) { // helper functions // - function initFolders() { - if (window.chrome && chrome.identity) { - // get pointer to account/folder/message tree on root scope - $scope.$root.account = emailDao._account; - - // set notificatio handler for sent messages - outboxBo.onSent = sentNotification; - // start checking outbox periodically - outboxBo.startChecking($scope.onOutboxUpdate); - + function initializeFolders() { + // create dummy folder in dev environment only + if (!window.chrome || !chrome.identity) { + createDummyFolders(); return; } - // attach dummy folders for development + // get pointer to account/folder/message tree on root scope + $scope.$root.account = emailDao._account; + + // set notificatio handler for sent messages + outboxBo.onSent = sentNotification; + // start checking outbox periodically + outboxBo.startChecking($scope.onOutboxUpdate); + + } + + function sentNotification(email) { + notification.create({ + id: 'o' + email.id, + title: 'Message sent', + message: email.subject + }, function() {}); + } + + + // attach dummy folders for development + function createDummyFolders() { $scope.$root.account = {}; $scope.account.folders = [{ type: 'Inbox', @@ -106,9 +117,9 @@ define(function(require) { count: 0, path: 'SENT' }, { - type: 'Outbox', + type: config.outboxMailboxType, count: 0, - path: 'OUTBOX' + path: config.outboxMailboxPath }, { type: 'Drafts', count: 0, @@ -119,14 +130,6 @@ define(function(require) { path: 'TRASH' }]; } - - function sentNotification(email) { - notification.create({ - id: 'o' + email.id, - title: 'Message sent', - message: email.subject.replace(str.subjectPrefix, '') - }, function() {}); - } }; // diff --git a/src/js/controller/read.js b/src/js/controller/read.js index 750427e..8b3245b 100644 --- a/src/js/controller/read.js +++ b/src/js/controller/read.js @@ -117,7 +117,7 @@ define(function(require) { var email = $scope.state.mailList.selected; emailDao.getAttachment({ - folder: folder.path, + folder: folder, uid: email.uid, attachment: attachment }, function(err) { diff --git a/src/js/controller/write.js b/src/js/controller/write.js index 1a906bf..ed96ef8 100644 --- a/src/js/controller/write.js +++ b/src/js/controller/write.js @@ -326,53 +326,46 @@ define(function(require) { return; } - // helper flag to remember if we need to sync back to imap - // in case the replyTo.answered changed - var needsSync = false; - - // mark replyTo as answered, if necessary - if ($scope.replyTo && !$scope.replyTo.answered) { - $scope.replyTo.answered = true; - // update the ui - $scope.$apply(); - needsSync = true; - } - - // if we need to synchronize replyTo.answered, let's do that. - // otherwise, we're done - if (!needsSync) { + // if we need to synchronize replyTo.answered = true to imap, + // let's do that. otherwise, we're done + if (!$scope.replyTo || $scope.replyTo.answered) { return; } - emailDao.sync({ - folder: $scope.state.nav.currentFolder.path + $scope.replyTo.answered = true; + emailDao.setFlags({ + folder: currentFolder(), + message: $scope.replyTo }, function(err) { - if (err && err.code === 42) { - // offline - $scope.onError(); + if (err && err.code !== 42) { + $scope.onError(err); return; } - $scope.onError(err); + // offline or no error, let's apply the ui changes + $scope.$apply(); }); }); }; + + // + // Helpers + // + + function currentFolder() { + return $scope.state.nav.currentFolder; + } + + /* + * Visitor to filter out objects without an address property, i.e. empty addresses + */ + function filterEmptyAddresses(addr) { + return !!addr.address; + } }; - // - // Helpers - // - - /* - * Visitor to filter out objects without an address property, i.e. empty addresses - */ - function filterEmptyAddresses(addr) { - return !!addr.address; - } - - // // Directives // diff --git a/src/js/dao/email-dao.js b/src/js/dao/email-dao.js index 70861bc..abdb954 100644 --- a/src/js/dao/email-dao.js +++ b/src/js/dao/email-dao.js @@ -3,27 +3,31 @@ define(function(require) { var util = require('cryptoLib/util'), _ = require('underscore'), + config = require('js/app-config').config, str = require('js/app-config').string; - var EmailDAO = function(keychain, crypto, devicestorage, pgpbuilder, mailreader, emailSync) { + var EmailDAO = function(keychain, crypto, devicestorage, pgpbuilder, mailreader) { this._keychain = keychain; this._crypto = crypto; this._devicestorage = devicestorage; this._pgpbuilder = pgpbuilder; this._mailreader = mailreader; - this._emailSync = emailSync; }; + // - // External API // + // Public API + // + // + EmailDAO.prototype.init = function(options, callback) { var self = this, keypair; self._account = options.account; - self._account.busy = false; + self._account.busy = false; // triggers the spinner self._account.online = false; self._account.loggingIn = false; @@ -48,113 +52,24 @@ define(function(require) { } keypair = storedKeypair; - initEmailSync(); - }); - } - - function initEmailSync() { - self._emailSync.init({ - account: self._account - }, function(err) { - if (err) { - callback(err); - return; - } - initFolders(); }); } function initFolders() { // try init folders from memory, since imap client not initiated yet - self._imapListFolders(function(err, folders) { + self._initFolders(function(err) { // dont handle offline case this time if (err && err.code !== 42) { callback(err); return; } - self._account.folders = folders; - callback(null, keypair); }); } }; - EmailDAO.prototype.onConnect = function(options, callback) { - var self = this; - - self._account.loggingIn = true; - - self._imapClient = options.imapClient; - self._pgpMailer = options.pgpMailer; - - // notify emailSync - self._emailSync.onConnect({ - imapClient: self._imapClient - }, function(err) { - if (err) { - self._account.loggingIn = false; - callback(err); - return; - } - - // connect to newly created imap client - self._imapLogin(onLogin); - }); - - function onLogin(err) { - if (err) { - self._account.loggingIn = false; - callback(err); - return; - } - - // set status to online - self._account.loggingIn = false; - self._account.online = true; - - // init folders - self._imapListFolders(function(err, folders) { - if (err) { - callback(err); - return; - } - - // only overwrite folders if they are not yet set - if (!self._account.folders) { - self._account.folders = folders; - } - - var inbox = _.findWhere(self._account.folders, { - type: 'Inbox' - }); - - if (inbox) { - self._imapClient.listenForChanges({ - path: inbox.path - }, function(error, path) { - if (typeof self.onNeedsSync === 'function') { - self.onNeedsSync(error, path); - } - }); - } - - callback(); - }); - } - }; - - EmailDAO.prototype.onDisconnect = function(options, callback) { - // set status to online - this._account.online = false; - this._imapClient = undefined; - this._pgpMailer = undefined; - - // notify emailSync - this._emailSync.onDisconnect(null, callback); - }; - EmailDAO.prototype.unlock = function(options, callback) { var self = this; @@ -255,12 +170,335 @@ define(function(require) { } }; - EmailDAO.prototype.syncOutbox = function(options, callback) { - this._emailSync.syncOutbox(options, callback); + EmailDAO.prototype.openFolder = function(options, callback) { + var self = this; + + if (!self._account.online) { + callback({ + errMsg: 'Client is currently offline!', + code: 42 + }); + return; + } + + if (options.folder.path === config.outboxMailboxPath) { + return; + } + + this._imapClient.selectMailbox({ + path: options.folder.path + }, callback); }; - EmailDAO.prototype.sync = function(options, callback) { - this._emailSync.sync(options, callback); + EmailDAO.prototype.refreshFolder = function(options, callback) { + var self = this, + folder = options.folder; + + folder.messages = folder.messages || []; + self._localListMessages({ + folder: folder + }, function(err, storedMessages) { + if (err) { + done(err); + return; + } + + var storedUids = _.pluck(storedMessages, 'uid'), + memoryUids = _.pluck(folder.messages, 'uid'), + newUids = _.difference(storedUids, memoryUids), + removedUids = _.difference(memoryUids, storedUids); + + // which messages are new on the disk that are not yet in memory? + _.filter(storedMessages, function(msg) { + return _.contains(newUids, msg.uid); + }).forEach(function(newMessage) { + // remove the body parts to not load unnecessary data to memory + // however, don't do that for the outbox. load the full message there. + if (folder.path !== config.outboxMailboxPath) { + delete newMessage.bodyParts; + } + + folder.messages.push(newMessage); + }); + + // which messages are no longer on disk, i.e. have been removed/sent/... + _.filter(folder.messages, function(msg) { + return _.contains(removedUids, msg.uid); + }).forEach(function(removedMessage) { + // remove the message + var index = folder.messages.indexOf(removedMessage); + folder.messages.splice(index, 1); + }); + + done(); + }); + + function done(err) { + self._account.busy = false; // stop the spinner + updateUnreadCount(folder); // update the unread count + callback(err); + } + }; + + EmailDAO.prototype.fetchMessages = function(options, callback) { + var self = this, + folder = options.folder; + + self._account.busy = true; + + if (!self._account.online) { + done({ + errMsg: 'Client is currently offline!', + code: 42 + }); + return; + } + + // list the messages starting from the lowest new uid to the highest new uid + self._imapListMessages(options, function(err, messages) { + if (err) { + done(err); + return; + } + + // if there are verification messages in the synced messages, handle it + var verificationMessages = _.filter(messages, function(message) { + return message.subject === str.verificationSubject; + }); + + // if there are verification messages, continue after we've tried to verify + if (verificationMessages.length > 0) { + var after = _.after(verificationMessages.length, storeHeaders); + + verificationMessages.forEach(function(verificationMessage) { + handleVerification(verificationMessage, function(err, isValid) { + // if it was NOT a valid verification mail, do nothing + // if an error occurred and the mail was a valid verification mail, + // keep the mail in the list so the user can see it and verify manually + if (!isValid || err) { + after(); + return; + } + + // if verification worked, we remove the mail from the list. + messages.splice(messages.indexOf(verificationMessage), 1); + after(); + }); + }); + return; + } + + // no verification messages, just proceed as usual + storeHeaders(); + + function storeHeaders() { + if (_.isEmpty(messages)) { + // nothing to do, we're done here + done(); + return; + } + + // persist the encrypted message to the local storage + self._localStoreMessages({ + folder: folder, + emails: messages + }, function(err) { + if (err) { + done(err); + return; + } + + // this enables us to already show the attachment clip in the message list ui + messages.forEach(function(message) { + message.attachments = message.bodyParts.filter(function(bodyPart) { + return bodyPart.type === 'attachment'; + }); + }); + + [].unshift.apply(folder.messages, messages); // add the new messages to the folder + updateUnreadCount(folder); // update the unread count + self.onIncomingMessage(messages); // notify about new messages + done(); + }); + } + }); + + function done(err) { + self._account.busy = false; // stop the spinner + callback(err); + } + + function handleVerification(message, localCallback) { + self._getBodyParts({ + folder: folder, + uid: message.uid, + bodyParts: message.bodyParts + }, function(error, parsedBodyParts) { + // we could not stream the text to determine if the verification was valid or not + // so handle it as if it were valid + if (error) { + localCallback(error, true); + return; + } + + var body = _.pluck(filterBodyParts(parsedBodyParts, 'text'), 'content').join('\n'), + verificationUrlPrefix = config.cloudUrl + config.verificationUrl, + uuid = body.split(verificationUrlPrefix).pop().substr(0, config.verificationUuidLength), + uuidRegex = /[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/; + + // there's no valid uuid in the message, so forget about it + if (!uuidRegex.test(uuid)) { + localCallback(null, false); + return; + } + + // there's a valid uuid in the message, so try to verify it + self._keychain.verifyPublicKey(uuid, function(err) { + if (err) { + localCallback({ + errMsg: 'Verifying your public key failed: ' + err.errMsg + }, true); + return; + } + + // public key has been verified, delete the message + self._imapDeleteMessage({ + folder: folder, + uid: message.uid + }, function() { + // if we could successfully not delete the message or not doesn't matter. + // just don't show it in whiteout and keep quiet about it + localCallback(null, true); + }); + }); + }); + } + }; + + EmailDAO.prototype.deleteMessage = function(options, callback) { + var self = this, + folder = options.folder, + message = options.message; + + self._account.busy = true; + + folder.messages.splice(folder.messages.indexOf(message), 1); + + if (options.localOnly || options.folder.path === config.outboxMailboxPath) { + deleteLocal(); + return; + } + + deleteImap(); + + function deleteImap() { + if (!self._account.online) { + done({ + errMsg: 'Client is currently offline!', + code: 42 + }); + return; + } + + self._imapDeleteMessage({ + folder: folder, + uid: message.uid + }, function(err) { + if (err) { + done(err); + return; + } + + deleteLocal(); + }); + } + + function deleteLocal() { + self._localDeleteMessage({ + folder: folder, + uid: message.uid + }, done); + } + + function done(err) { + self._account.busy = false; // stop the spinner + if (err) { + folder.messages.unshift(message); // re-add the message to the folder in case of an error + } + updateUnreadCount(folder); // update the unread count, if necessary + callback(err); + } + }; + + EmailDAO.prototype.setFlags = function(options, callback) { + var self = this, + folder = options.folder, + message = options.message; + + self._account.busy = true; + + if (folder.messages.indexOf(message) < 0) { + self._account.busy = false; // stop the spinner + return; + } + + if (options.localOnly || options.folder.path === config.outboxMailboxPath) { + markStorage(); + return; + } + + if (!self._account.online) { + done({ + errMsg: 'Client is currently offline!', + code: 42 + }); + return; + } + + markImap(); + + function markImap() { + self._imapMark({ + folder: folder, + uid: options.message.uid, + unread: options.message.unread, + answered: options.message.answered + }, function(err) { + if (err) { + done(err); + return; + } + + markStorage(); + }); + } + + function markStorage() { + self._localListMessages({ + folder: folder, + uid: options.message.uid, + }, function(err, storedMessages) { + if (err) { + done(err); + return; + } + + var storedMessage = storedMessages[0]; + storedMessage.unread = options.message.unread; + storedMessage.answered = options.message.answered; + + self._localStoreMessages({ + folder: folder, + emails: [storedMessage] + }, done); + }); + } + + function done(err) { + self._account.busy = false; // stop the spinner // + updateUnreadCount(folder); // update the unread count + callback(err); + } }; /** @@ -295,7 +533,7 @@ define(function(require) { function retrieveContent() { // load the local message from memory - self._emailSync._localListMessages({ + self._localListMessages({ folder: folder, uid: message.uid }, function(err, localMessages) { @@ -331,7 +569,7 @@ define(function(require) { } // get the raw content from the imap server - self._emailSync._getBodyParts({ + self._getBodyParts({ folder: folder, uid: localMessage.uid, bodyParts: contentParts @@ -346,7 +584,7 @@ define(function(require) { localMessage.bodyParts = parsedBodyParts.concat(attachmentParts); // persist it to disk - self._emailSync._localStoreMessages({ + self._localStoreMessages({ folder: folder, emails: [localMessage] }, function(error) { @@ -365,7 +603,7 @@ define(function(require) { function extractContent() { if (message.encrypted) { // show the encrypted message - message.body = self._emailSync.filterBodyParts(message.bodyParts, 'encrypted')[0].content; + message.body = filterBodyParts(message.bodyParts, 'encrypted')[0].content; done(); return; } @@ -374,7 +612,7 @@ define(function(require) { var root = message.bodyParts; if (message.signed) { - var signedPart = self._emailSync.filterBodyParts(message.bodyParts, 'signed')[0]; + var signedPart = filterBodyParts(message.bodyParts, 'signed')[0]; message.message = signedPart.message; message.signature = signedPart.signature; // TODO check integrity @@ -384,7 +622,7 @@ define(function(require) { // if the message is plain text and contains pgp/inline, we are only interested in the encrypted // content, the rest (corporate mail footer, attachments, etc.) is discarded. - var body = _.pluck(self._emailSync.filterBodyParts(root, 'text'), 'content').join('\n'); + var body = _.pluck(filterBodyParts(root, 'text'), 'content').join('\n'); /* * here's how the regex works: @@ -412,9 +650,9 @@ define(function(require) { return; } - message.attachments = self._emailSync.filterBodyParts(root, 'attachment'); + message.attachments = filterBodyParts(root, 'attachment'); message.body = body; - message.html = _.pluck(self._emailSync.filterBodyParts(root, 'html'), 'content').join('\n'); + message.html = _.pluck(filterBodyParts(root, 'html'), 'content').join('\n'); done(); } @@ -426,7 +664,7 @@ define(function(require) { }; EmailDAO.prototype.getAttachment = function(options, callback) { - this._emailSync._getBodyParts({ + this._getBodyParts({ folder: options.folder, uid: options.uid, bodyParts: [options.attachment] @@ -446,7 +684,7 @@ define(function(require) { message = options.message; // the message is decrypting has no body, is not encrypted or has already been decrypted - if (message.decryptingBody || !message.body || !message.encrypted || message.decrypted) { + if (!message.bodyParts || message.decryptingBody || !message.body || !message.encrypted || message.decrypted) { return; } @@ -466,7 +704,7 @@ define(function(require) { } // get the receiver's public key to check the message signature - var encryptedNode = self._emailSync.filterBodyParts(message.bodyParts, 'encrypted')[0]; + var encryptedNode = filterBodyParts(message.bodyParts, 'encrypted')[0]; self._crypto.decrypt(encryptedNode.content, senderPublicKey.publicKey, function(err, decrypted) { if (err || !decrypted) { showError(err.errMsg || err.message || 'An error occurred during the decryption.'); @@ -497,9 +735,9 @@ define(function(require) { // we have successfully interpreted the descrypted message, // so let's update the views on the message parts - message.body = _.pluck(self._emailSync.filterBodyParts(parsedBodyParts, 'text'), 'content').join('\n'); - message.html = _.pluck(self._emailSync.filterBodyParts(parsedBodyParts, 'html'), 'content').join('\n'); - message.attachments = _.reject(self._emailSync.filterBodyParts(parsedBodyParts, 'attachment'), function(attmt) { + message.body = _.pluck(filterBodyParts(parsedBodyParts, 'text'), 'content').join('\n'); + message.html = _.pluck(filterBodyParts(parsedBodyParts, 'html'), 'content').join('\n'); + message.attachments = _.reject(filterBodyParts(parsedBodyParts, 'attachment'), function(attmt) { // remove the pgp-signature from the attachments return attmt.mimeType === "application/pgp-signature"; }); @@ -527,7 +765,7 @@ define(function(require) { EmailDAO.prototype.sendEncrypted = function(options, callback) { var self = this; - if (!this._account.online) { + if (!self._account.online) { callback({ errMsg: 'Client is currently offline!', code: 42 @@ -563,32 +801,295 @@ define(function(require) { this._pgpbuilder.encrypt(options, callback); }; + + // + // + // Event Handlers + // + // + + + EmailDAO.prototype.onConnect = function(options, callback) { + var self = this; + + self._account.loggingIn = true; + + self._imapClient = options.imapClient; + self._pgpMailer = options.pgpMailer; + + this._imapClient.login(function(err) { + self._account.loggingIn = false; + + if (err) { + callback(err); + return; + } + + // set status to online + self._account.online = true; + + // init folders + self._initFolders(function(err) { + if (err) { + callback(err); + return; + } + + // attach sync update handler + self._imapClient.onSyncUpdate = self._onSyncUpdate.bind(self); + + // fill the imap mailboxCache + var mailboxCache = {}; + self._account.folders.forEach(function(folder) { + if (folder.messages.length === 0) { + return; + } + + var uids, highestModseq, lastUid; + + uids = _.pluck(folder.messages, 'uid').sort(function(a, b) { + return a - b; + }); + lastUid = uids[uids.length - 1]; + + highestModseq = _.pluck(folder.messages, 'modseq').sort(function(a, b) { + return a - b; + }).pop(); + + mailboxCache[folder.path] = { + exists: lastUid, + uidNext: lastUid + 1, + uidlist: uids, + highestModseq: highestModseq + }; + }); + self._imapClient.mailboxCache = mailboxCache; + + var inbox = _.findWhere(self._account.folders, { + type: 'Inbox' + }); + + if (!inbox) { + callback(); + return; + } + + self._imapClient.listenForChanges({ + path: inbox.path + }, callback); + }); + }); + }; + + EmailDAO.prototype.onDisconnect = function() { + this._account.online = false; + this._imapClient = undefined; + this._pgpMailer = undefined; + }; + + EmailDAO.prototype._onSyncUpdate = function(options) { + var self = this; + + var folder = _.findWhere(self._account.folders, { + path: options.path + }); + + if (!folder) { + // ignore updates for an unknown folder + return; + } + + if (options.type === 'new') { + // new messages available on imap, fetch from imap and store to disk and memory + self.fetchMessages({ + folder: folder, + firstUid: Math.min.apply(null, options.list), + lastUid: Math.max.apply(null, options.list) + }, self.onError.bind(self)); + } else if (options.type === 'deleted') { + // messages have been deleted, remove from local storage and memory + options.list.forEach(function(uid) { + var message = _.findWhere(folder.messages, { + uid: uid + }); + + if (!message) { + return; + } + + self.deleteMessage({ + folder: folder, + message: message, + localOnly: true + }, self.onError.bind(self)); + }); + } else if (options.type === 'messages') { + // NB! several possible reasons why this could be called. + // if a message in the array has uid value and flag array, it had a possible flag update + options.list.forEach(function(changedMsg) { + if (!changedMsg.uid || !changedMsg.flags) { + return; + } + + var message = _.findWhere(folder.messages, { + uid: changedMsg.uid + }); + + if (!message) { + return; + } + + message.answered = changedMsg.flags.indexOf('\\Answered') > -1; + message.unread = changedMsg.flags.indexOf('\\Seen') === -1; + + if (!message) { + return; + } + + self.setFlags({ + folder: folder, + message: message + }, self.onError.bind(self)); + }); + } + }; + + + // // // Internal API // + // - // IMAP API /** - * Login the imap client + * List the folders in the user's IMAP mailbox. */ - EmailDAO.prototype._imapLogin = function(callback) { - if (!this._imapClient) { - callback({ - errMsg: 'Client is currently offline!', - code: 42 + EmailDAO.prototype._initFolders = function(callback) { + var self = this, + folderDbType = 'folders', + folders; + + self._account.busy = true; + + if (!self._account.online) { + // fetch list from local cache + self._devicestorage.listItems(folderDbType, 0, null, function(err, stored) { + if (err) { + done(err); + return; + } + + self._account.folders = stored[0] || []; + readCache(); + }); + return; + } else { + // fetch list from imap server + self._imapClient.listWellKnownFolders(function(err, wellKnownFolders) { + var foldersChanged = false; + + if (err) { + done(err); + return; + } + + // this array is dropped directly into the ui to create the folder list + folders = [ + wellKnownFolders.inbox, + wellKnownFolders.sent, { + type: 'Outbox', + path: config.outboxMailboxPath + }, + wellKnownFolders.drafts, + wellKnownFolders.trash + ]; + + // are there any new folders? + folders.forEach(function(folder) { + if (!_.findWhere(self._account.folders, { + path: folder.path + })) { + // add the missing folder + self._account.folders.push(folder); + foldersChanged = true; + } + }); + + // have any folders been deleted? + self._account.folders.forEach(function(folder) { + if (!_.findWhere(folders, { + path: folder.path + })) { + // remove the obsolete folder + self._account.folders.splice(self._account.folder.indexOf(folder), 1); + foldersChanged = true; + } + }); + + if (!foldersChanged) { + readCache(); + return; + } + + // persist encrypted list in device storage + self._devicestorage.storeList([folders], folderDbType, function(err) { + if (err) { + done(err); + return; + } + + readCache(); + }); }); return; } - // login IMAP client if existent - this._imapClient.login(callback); + function readCache() { + if (!self._account.folders || self._account.folders.length === 0) { + done(); + return; + } + + var after = _.after(self._account.folders.length, done); + + self._account.folders.forEach(function(folder) { + if (folder.messages) { + // the folder is already initialized + after(); + return; + } + + self.refreshFolder({ + folder: folder + }, function(err) { + if (err) { + done(err); + return; + } + + after(); + }); + }); + } + + function done(err) { + self._account.busy = false; // stop the spinner + callback(err); + } }; + + // + // + // IMAP API + // + // + /** - * Cleanup by logging the user off. + * Mark imap messages as un-/read or un-/answered */ - EmailDAO.prototype._imapLogout = function(callback) { + EmailDAO.prototype._imapMark = function(options, callback) { if (!this._account.online) { callback({ errMsg: 'Client is currently offline!', @@ -597,73 +1098,160 @@ define(function(require) { return; } - this._imapClient.logout(callback); + options.path = options.folder.path; + this._imapClient.updateFlags(options, callback); + }; + + EmailDAO.prototype._imapDeleteMessage = function(options, callback) { + if (!this._account.online) { + callback({ + errMsg: 'Client is currently offline!', + code: 42 + }); + return; + } + + var trash = _.findWhere(this._account.folders, { + type: 'Trash' + }); + + // there's no known trash folder to move the mail to or we're in the trash folder, so we can purge the message + if (!trash || options.folder === trash) { + this._imapClient.deleteMessage({ + path: options.folder.path, + uid: options.uid + }, callback); + + return; + } + + this._imapClient.moveMessage({ + path: options.folder.path, + destination: trash.path, + uid: options.uid + }, callback); }; /** - * List the folders in the user's IMAP mailbox. + * Get an email messsage without the body + * @param {String} options.folder The folder + * @param {Number} options.firstUid The lower bound of the uid (inclusive) + * @param {Number} options.lastUid The upper bound of the uid range (inclusive) + * @param {Function} callback (error, messages) The callback when the imap client is done fetching message metadata */ - EmailDAO.prototype._imapListFolders = function(callback) { - var self = this, - dbType = 'folders'; + EmailDAO.prototype._imapListMessages = function(options, callback) { + var self = this; - // check local cache - self._devicestorage.listItems(dbType, 0, null, function(err, stored) { + if (!this._account.online) { + callback({ + errMsg: 'Client is currently offline!', + code: 42 + }); + return; + } + + options.path = options.folder.path; + self._imapClient.listMessages(options, callback); + }; + + /** + * Stream an email messsage's body + * @param {String} options.folder The folder + * @param {String} options.uid the message's uid + * @param {Object} options.bodyParts The message, as retrieved by _imapListMessages + * @param {Function} callback (error, message) The callback when the imap client is done streaming message text content + */ + EmailDAO.prototype._getBodyParts = function(options, callback) { + var self = this; + + if (!self._account.online) { + callback({ + errMsg: 'Client is currently offline!', + code: 42 + }); + return; + } + + options.path = options.folder.path; + self._imapClient.getBodyParts(options, function(err) { if (err) { callback(err); return; } - - if (!stored || stored.length < 1) { - // no folders cached... fetch from server - fetchFromServer(); - return; - } - - callback(null, stored[0]); + // interpret the raw content of the email + self._mailreader.parse(options, callback); }); - - function fetchFromServer() { - var folders; - - if (!self._account.online) { - callback({ - errMsg: 'Client is currently offline!', - code: 42 - }); - return; - } - - // fetch list from imap server - self._imapClient.listWellKnownFolders(function(err, wellKnownFolders) { - if (err) { - callback(err); - return; - } - - folders = [ - wellKnownFolders.inbox, - wellKnownFolders.sent, { - type: 'Outbox', - path: 'OUTBOX' - }, - wellKnownFolders.drafts, - wellKnownFolders.trash - ]; - - // cache locally - // persist encrypted list in device storage - self._devicestorage.storeList([folders], dbType, function(err) { - if (err) { - callback(err); - return; - } - - callback(null, folders); - }); - }); - } }; + + // + // + // Local Storage API + // + // + + + EmailDAO.prototype._localListMessages = function(options, callback) { + var dbType = 'email_' + options.folder.path + (options.uid ? '_' + options.uid : ''); + this._devicestorage.listItems(dbType, 0, null, callback); + }; + + EmailDAO.prototype._localStoreMessages = function(options, callback) { + var dbType = 'email_' + options.folder.path; + this._devicestorage.storeList(options.emails, dbType, callback); + }; + + EmailDAO.prototype._localDeleteMessage = function(options, callback) { + var path = options.folder.path, + uid = options.uid, + id = options.id; + + if (!path || !(uid || id)) { + callback({ + errMsg: 'Invalid options!' + }); + return; + } + + var dbType = 'email_' + path + '_' + (uid || id); + this._devicestorage.removeList(dbType, callback); + }; + + + // + // + // Helper Functions + // + // + + function updateUnreadCount(folder) { + var allMsgs = folder.messages.length, + unreadMsgs = _.filter(folder.messages, function(msg) { + return msg.unread; + }).length; + + // for the outbox, the unread count is determined by ALL the messages + // whereas for normal folders, only the unread messages matter + folder.count = folder.path === config.outboxMailboxPath ? allMsgs : unreadMsgs; + } + + /** + * Helper function that recursively traverses the body parts tree. Looks for bodyParts that match the provided type and aggregates them + * @param {[type]} bodyParts The bodyParts array + * @param {[type]} type The type to look up + * @param {undefined} result Leave undefined, only used for recursion + */ + function filterBodyParts(bodyParts, type, result) { + result = result || []; + bodyParts.forEach(function(part) { + if (part.type === type) { + result.push(part); + } else if (Array.isArray(part.content)) { + filterBodyParts(part.content, type, result); + } + }); + return result; + } + return EmailDAO; }); \ No newline at end of file diff --git a/src/js/dao/email-sync.js b/src/js/dao/email-sync.js deleted file mode 100644 index 0f8c7db..0000000 --- a/src/js/dao/email-sync.js +++ /dev/null @@ -1,848 +0,0 @@ -define(function(require) { - 'use strict'; - - var _ = require('underscore'), - config = require('js/app-config').config, - str = require('js/app-config').string; - - var EmailSync = function(keychain, devicestorage, mailreader) { - this._keychain = keychain; - this._devicestorage = devicestorage; - this._mailreader = mailreader; - }; - - EmailSync.prototype.init = function(options, callback) { - this._account = options.account; - - callback(); - }; - - EmailSync.prototype.onConnect = function(options, callback) { - this._imapClient = options.imapClient; - - callback(); - }; - - EmailSync.prototype.onDisconnect = function(options, callback) { - this._imapClient = undefined; - - callback(); - }; - - /** - * Syncs outbox content from disk to memory, not vice-versa - */ - EmailSync.prototype.syncOutbox = function(options, callback) { - var self = this; - - // check busy status - if (self._account.busy) { - callback({ - errMsg: 'Sync aborted: Previous sync still in progress', - code: 409 - }); - return; - } - - // make sure two syncs for the same folder don't interfere - self._account.busy = true; - - var folder = _.findWhere(self._account.folders, { - path: options.folder - }); - - folder.messages = folder.messages || []; - - self._localListMessages({ - folder: folder.path - }, function(err, storedMessages) { - if (err) { - self._account.busy = false; - callback(err); - return; - } - - // calculate the diffs between memory and disk - var storedIds = _.pluck(storedMessages, 'id'), - inMemoryIds = _.pluck(folder.messages, 'id'), - newIds = _.difference(storedIds, inMemoryIds), - removedIds = _.difference(inMemoryIds, storedIds); - - // which messages are new on the disk that are not yet in memory? - var newMessages = _.filter(storedMessages, function(msg) { - return _.contains(newIds, msg.id); - }); - - // which messages are no longer on disk, i.e. have been sent - var removedMessages = _.filter(folder.messages, function(msg) { - return _.contains(removedIds, msg.id); - }); - - // add the new messages to memory - newMessages.forEach(function(newMessage) { - folder.messages.push(newMessage); - }); - - // remove the sent messages from memory - removedMessages.forEach(function(removedMessage) { - var index = folder.messages.indexOf(removedMessage); - folder.messages.splice(index, 1); - }); - - // update the folder count and we're done. - folder.count = folder.messages.length; - self._account.busy = false; - - callback(); - }); - }; - - EmailSync.prototype.sync = function(options, callback) { - /* - * Here's how delta sync works: - * - * First, we sync the messages between memory and local storage, based on their uid - * delta1: storage > memory => we deleted messages, remove from remote and memory - * delta2: memory > storage => we added messages, push to remote <<< not supported yet - * - * Second, we check the delta for the flags - * deltaF2: memory > storage => we changed flags, sync them to the remote and memory - * - * Third, we go on to sync between imap and memory, again based on uid - * delta3: memory > imap => we deleted messages directly from the remote, remove from memory and storage - * delta4: imap > memory => we have new messages available, fetch to memory and storage - * - * Fourth, we pull changes in the flags downstream - * deltaF4: imap > memory => we changed flags directly on the remote, sync them to the storage and memory - */ - - var self = this; - - // validate options - if (!options.folder) { - callback({ - errMsg: 'Invalid options!' - }); - return; - } - - // check busy status - if (self._account.busy) { - callback({ - errMsg: 'Sync aborted: Previous sync still in progress', - code: 409 - }); - return; - } - - // make sure two syncs for the same folder don't interfere - self._account.busy = true; - - var folder = _.findWhere(self._account.folders, { - path: options.folder - }); - - /* - * if the folder is not initialized with the messages from the memory, we need to fill it first, otherwise the delta sync obviously breaks. - * initial filling from local storage is an exception from the normal sync. after reading from local storage, do imap sync - */ - var isFolderInitialized = !! folder.messages; - if (!isFolderInitialized) { - initFolderMessages(); - return; - } - - doLocalDelta(); - - /* - * pre-fill the memory with the messages stored on the hard disk - */ - function initFolderMessages() { - folder.messages = []; - self._localListMessages({ - folder: folder.path - }, function(err, storedMessages) { - if (err) { - self._account.busy = false; - callback(err); - return; - } - - storedMessages.forEach(function(storedMessage) { - // remove the body parts to not load unnecessary data to memory - delete storedMessage.bodyParts; - - folder.messages.push(storedMessage); - }); - - callback(); - doImapDelta(); - }); - } - - /* - * compares the messages in memory to the messages on the disk - */ - function doLocalDelta() { - self._localListMessages({ - folder: folder.path - }, function(err, storedMessages) { - if (err) { - self._account.busy = false; - callback(err); - return; - } - - doDelta1(); - - /* - * delta1: - * storage contains messages that are not present in memory => we deleted messages from the memory, so remove the messages from the remote and the disk - */ - function doDelta1() { - var inMemoryUids = _.pluck(folder.messages, 'uid'), - storedMessageUids = _.pluck(storedMessages, 'uid'), - delta1 = _.difference(storedMessageUids, inMemoryUids); // delta1 contains only uids - - // if we're we are done here - if (_.isEmpty(delta1)) { - doDeltaF2(); - return; - } - - var after = _.after(delta1.length, function() { - doDeltaF2(); - }); - - // delta1 contains uids of messages on the disk - delta1.forEach(function(inMemoryUid) { - var deleteMe = { - folder: folder.path, - uid: inMemoryUid - }; - - self._imapDeleteMessage(deleteMe, function(err) { - if (err) { - self._account.busy = false; - callback(err); - return; - } - - self._localDeleteMessage(deleteMe, function(err) { - if (err) { - self._account.busy = false; - callback(err); - return; - } - - after(); - }); - }); - }); - } - - /* - * deltaF2: - * memory contains messages that have flags other than those in storage => we changed flags, sync them to the remote and memory - */ - function doDeltaF2() { - var deltaF2 = checkFlags(folder.messages, storedMessages); // deltaF2 contains the message objects, we need those to sync the flags - - if (_.isEmpty(deltaF2)) { - callback(); - doImapDelta(); - return; - } - - var after = _.after(deltaF2.length, function() { - callback(); - doImapDelta(); - }); - - // deltaF2 contains references to the in-memory messages - deltaF2.forEach(function(inMemoryMessage) { - self._imapMark({ - folder: folder.path, - uid: inMemoryMessage.uid, - unread: inMemoryMessage.unread, - answered: inMemoryMessage.answered - }, function(err) { - if (err) { - self._account.busy = false; - callback(err); - return; - } - - var storedMessage = _.findWhere(storedMessages, { - uid: inMemoryMessage.uid - }); - - storedMessage.unread = inMemoryMessage.unread; - storedMessage.answered = inMemoryMessage.answered; - - self._localStoreMessages({ - folder: folder.path, - emails: [storedMessage] - }, function(err) { - if (err) { - self._account.busy = false; - callback(err); - return; - } - - after(); - }); - }); - }); - } - }); - } - - /* - * compare the messages on the imap server to the in memory messages - */ - function doImapDelta() { - self._imapSearch({ - folder: folder.path - }, function(err, inImapUids) { - if (err) { - self._account.busy = false; - callback(err); - return; - } - - doDelta3(); - - /* - * delta3: - * memory contains messages that are not present on the imap => we deleted messages directly from the remote, remove from memory and storage - */ - function doDelta3() { - var inMemoryUids = _.pluck(folder.messages, 'uid'), - delta3 = _.difference(inMemoryUids, inImapUids); - - if (_.isEmpty(delta3)) { - doDelta4(); - return; - } - - var after = _.after(delta3.length, function() { - doDelta4(); - }); - - // delta3 contains uids of the in-memory messages that have been deleted from the remote - delta3.forEach(function(inMemoryUid) { - // remove from local storage - self._localDeleteMessage({ - folder: folder.path, - uid: inMemoryUid - }, function(err) { - if (err) { - self._account.busy = false; - callback(err); - return; - } - - // remove from memory - var inMemoryMessage = _.findWhere(folder.messages, function(msg) { - return msg.uid === inMemoryUid; - }); - folder.messages.splice(folder.messages.indexOf(inMemoryMessage), 1); - - after(); - }); - }); - } - - /* - * delta4: - * imap contains messages that are not present in memory => we have new messages available, fetch downstream to memory and storage - */ - function doDelta4() { - var inMemoryUids = _.pluck(folder.messages, 'uid'), - delta4 = _.difference(inImapUids, inMemoryUids); - - // eliminate uids smaller than the biggest local uid, i.e. just fetch everything - // that came in AFTER the most recent email we have in memory. Keep in mind that - // uids are strictly ascending, so there can't be a NEW mail in the mailbox with a - // uid smaller than anything we've encountered before. - if (!_.isEmpty(inMemoryUids)) { - var maxInMemoryUid = Math.max.apply(null, inMemoryUids); // apply works with separate arguments rather than an array - - // eliminate everything prior to maxInMemoryUid, i.e. everything that was already synced - delta4 = _.filter(delta4, function(uid) { - return uid > maxInMemoryUid; - }); - } - - // no delta, we're done here - if (_.isEmpty(delta4)) { - doDeltaF4(); - return; - } - - // list the messages starting from the lowest new uid to the highest new uid - self._imapListMessages({ - folder: folder.path, - firstUid: Math.min.apply(null, delta4), - lastUid: Math.max.apply(null, delta4) - }, function(err, messages) { - if (err) { - self._account.busy = false; - callback(err); - return; - } - - // if there are verification messages in the synced messages, handle it - var verificationMessages = _.filter(messages, function(message) { - return message.subject === (str.subjectPrefix + str.verificationSubject); - }); - - // if there are verification messages, continue after we've tried to verify - if (verificationMessages.length > 0) { - var after = _.after(verificationMessages.length, storeHeaders); - - verificationMessages.forEach(function(verificationMessage) { - handleVerification(verificationMessage, function(err, isValid) { - // if it was NOT a valid verification mail, do nothing - if (!isValid) { - after(); - return; - } - - // if an error occurred and the mail was a valid verification mail, display the error, but - // keep the mail in the list so the user can see it and verify manually - if (err) { - callback(err); - after(); - return; - } - - // if verification worked, we remove the mail from the list. - messages.splice(messages.indexOf(verificationMessage), 1); - after(); - }); - }); - return; - } - - // no verification messages, just proceed as usual - storeHeaders(); - - function storeHeaders() { - // no delta, we're done here - if (_.isEmpty(messages)) { - doDeltaF4(); - return; - } - - // persist the encrypted message to the local storage - self._localStoreMessages({ - folder: folder.path, - emails: messages - }, function(err) { - if (err) { - self._account.busy = false; - callback(err); - return; - } - - // this enables us to already show the attachment clip in the message list ui - messages.forEach(function(message) { - message.attachments = message.bodyParts.filter(function(bodyPart) { - return bodyPart.type === 'attachment'; - }); - }); - - // if persisting worked, add them to the messages array - folder.messages = folder.messages.concat(messages); - self.onIncomingMessage(messages); - doDeltaF4(); - }); - } - }); - } - }); - - /** - * deltaF4: imap > memory => we changed flags directly on the remote, sync them to the storage and memory - */ - function doDeltaF4() { - var answeredUids, unreadUids, - deltaF4 = []; - - getUnreadUids(); - - // find all the relevant unread mails - function getUnreadUids() { - self._imapSearch({ - folder: folder.path, - unread: true - }, function(err, uids) { - if (err) { - self._account.busy = false; - callback(err); - return; - } - - // we're done here, let's get all the answered mails - unreadUids = uids; - getAnsweredUids(); - }); - } - - // find all the relevant answered mails - function getAnsweredUids() { - // find all the relevant answered mails - self._imapSearch({ - folder: folder.path, - answered: true - }, function(err, uids) { - if (err) { - self._account.busy = false; - callback(err); - return; - } - - // we're done here, let's update what we have in memory and persist that! - answeredUids = uids; - updateFlags(); - }); - - } - - function updateFlags() { - folder.messages.forEach(function(msg) { - // if the message's uid is among the uids that should be unread, - // AND the message is not unread, we clearly have to change that - var shouldBeUnread = _.contains(unreadUids, msg.uid); - if (msg.unread === shouldBeUnread) { - // everything is in order, we're good here - return; - } - - msg.unread = shouldBeUnread; - deltaF4.push(msg); - }); - - folder.messages.forEach(function(msg) { - // if the message's uid is among the uids that should be answered, - // AND the message is not answered, we clearly have to change that - var shouldBeAnswered = _.contains(answeredUids, msg.uid); - if (msg.answered === shouldBeAnswered) { - // everything is in order, we're good here - return; - } - - msg.answered = shouldBeAnswered; - deltaF4.push(msg); - }); - - // maybe a mail had BOTH flags wrong, so let's create - // a duplicate-free version of deltaF4 - deltaF4 = _.uniq(deltaF4); - - // everything up to date? fine, we're done! - if (_.isEmpty(deltaF4)) { - finishSync(); - return; - } - - var after = _.after(deltaF4.length, function() { - // we're doing updating everything - finishSync(); - }); - - // alright, so let's sync the corrected messages - deltaF4.forEach(function(inMemoryMessage) { - // do a short round trip to the database to avoid re-encrypting, - // instead use the encrypted object in the storage - self._localListMessages({ - folder: folder.path, - uid: inMemoryMessage.uid - }, function(err, storedMessages) { - if (err) { - self._account.busy = false; - callback(err); - return; - } - - var storedMessage = storedMessages[0]; - storedMessage.unread = inMemoryMessage.unread; - storedMessage.answered = inMemoryMessage.answered; - - // persist the modified object - self._localStoreMessages({ - folder: folder.path, - emails: [storedMessage] - }, function(err) { - if (err) { - self._account.busy = false; - callback(err); - return; - } - - // and we're done. - after(); - }); - }); - - }); - } - } - } - - function finishSync() { - // whereas normal folders show the unread messages count only, - // the outbox shows the total count - // after all the tags are up to date, let's adjust the unread mail count - folder.count = _.filter(folder.messages, function(msg) { - return msg.unread === true; - }).length; - - // allow the next sync to take place - self._account.busy = false; - callback(); - } - - /* - * checks if there are some flags that have changed in a and b - */ - function checkFlags(a, b) { - var i, aI, bI, - delta = []; - - // find the delta - for (i = a.length - 1; i >= 0; i--) { - aI = a[i]; - bI = _.findWhere(b, { - uid: aI.uid - }); - if (bI && (aI.unread !== bI.unread || aI.answered !== bI.answered)) { - delta.push(aI); - } - } - - return delta; - } - - function handleVerification(message, localCallback) { - self._getBodyParts({ - folder: options.folder, - uid: message.uid, - bodyParts: message.bodyParts - }, function(error, parsedBodyParts) { - // we could not stream the text to determine if the verification was valid or not - // so handle it as if it were valid - if (error) { - localCallback(error, true); - return; - } - - var body = _.pluck(self.filterBodyParts(parsedBodyParts, 'text'), 'content').join('\n'), - verificationUrlPrefix = config.cloudUrl + config.verificationUrl, - uuid = body.split(verificationUrlPrefix).pop().substr(0, config.verificationUuidLength), - isValidUuid = new RegExp('[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}').test(uuid); - - // there's no valid uuid in the message, so forget about it - if (!isValidUuid) { - localCallback(null, false); - return; - } - - // there's a valid uuid in the message, so try to verify it - self._keychain.verifyPublicKey(uuid, function(err) { - if (err) { - localCallback({ - errMsg: 'Verifying your public key failed: ' + err.errMsg - }, true); - return; - } - - // public key has been verified, delete the message - self._imapDeleteMessage({ - folder: options.folder, - uid: message.uid - }, function() { - // if we could successfully not delete the message or not doesn't matter. - // just don't show it in whiteout and keep quiet about it - localCallback(null, true); - }); - }); - }); - } - }; - - // - // Internal APIs - // - - // Local Storage API - - EmailSync.prototype._localListMessages = function(options, callback) { - var dbType = 'email_' + options.folder; - if (typeof options.uid !== 'undefined') { - dbType = dbType + '_' + options.uid; - } - this._devicestorage.listItems(dbType, 0, null, callback); - }; - - EmailSync.prototype._localStoreMessages = function(options, callback) { - var dbType = 'email_' + options.folder; - this._devicestorage.storeList(options.emails, dbType, callback); - }; - - EmailSync.prototype._localDeleteMessage = function(options, callback) { - if (!options.folder || !options.uid) { - callback({ - errMsg: 'Invalid options!' - }); - return; - } - var dbType = 'email_' + options.folder + '_' + options.uid; - this._devicestorage.removeList(dbType, callback); - }; - - // IMAP API - - /** - * Mark imap messages as un-/read or un-/answered - */ - EmailSync.prototype._imapMark = function(options, callback) { - if (!this._account.online) { - callback({ - errMsg: 'Client is currently offline!', - code: 42 - }); - return; - } - - options.path = options.folder; - this._imapClient.updateFlags(options, callback); - }; - - /** - * Returns the relevant messages corresponding to the search terms in the options - * @param {String} options.folder The folder's path - * @param {Boolean} options.answered (optional) Mails with or without the \Answered flag set. - * @param {Boolean} options.unread (optional) Mails with or without the \Seen flag set. - * @param {Function} callback(error, uids) invoked with the uids of messages matching the search terms, or an error object if an error occurred - */ - EmailSync.prototype._imapSearch = function(options, callback) { - if (!this._account.online) { - callback({ - errMsg: 'Client is currently offline!', - code: 42 - }); - return; - } - - options.path = options.folder; - this._imapClient.search(options, callback); - }; - - EmailSync.prototype._imapDeleteMessage = function(options, callback) { - if (!this._account.online) { - callback({ - errMsg: 'Client is currently offline!', - code: 42 - }); - return; - } - - var trash = _.findWhere(this._account.folders, { - type: 'Trash' - }); - - // there's no known trash folder to move the mail to or we're in the trash folder, - // so we can purge the message - if (!trash || options.folder === trash.path) { - this._imapClient.deleteMessage({ - path: options.folder, - uid: options.uid - }, callback); - - return; - } - - this._imapClient.moveMessage({ - path: options.folder, - destination: trash.path, - uid: options.uid - }, callback); - }; - - /** - * Get an email messsage without the body - * @param {String} options.folder The folder - * @param {Number} options.firstUid The lower bound of the uid (inclusive) - * @param {Number} options.lastUid The upper bound of the uid range (inclusive) - * @param {Function} callback (error, messages) The callback when the imap client is done fetching message metadata - */ - EmailSync.prototype._imapListMessages = function(options, callback) { - var self = this; - - if (!this._account.online) { - callback({ - errMsg: 'Client is currently offline!', - code: 42 - }); - return; - } - - options.path = options.folder; - self._imapClient.listMessages(options, callback); - }; - - /** - * Stream an email messsage's body - * @param {String} options.folder The folder - * @param {String} options.uid the message's uid - * @param {Object} options.bodyParts The message, as retrieved by _imapListMessages - * @param {Function} callback (error, message) The callback when the imap client is done streaming message text content - */ - EmailSync.prototype._getBodyParts = function(options, callback) { - var self = this; - - if (!this._account.online) { - callback({ - errMsg: 'Client is currently offline!', - code: 42 - }); - return; - } - - options.path = options.folder; - self._imapClient.getBodyParts(options, function(err) { - if (err) { - callback(err); - return; - } - // interpret the raw content of the email - self._mailreader.parse(options, callback); - }); - }; - - /** - * Helper function that recursively traverses the body parts tree. Looks for bodyParts that match the provided type and aggregates them - * @param {[type]} bodyParts The bodyParts array - * @param {[type]} type The type to look up - * @param {undefined} result Leave undefined, only used for recursion - */ - EmailSync.prototype.filterBodyParts = function(bodyParts, type, result) { - var self = this; - - result = result || []; - bodyParts.forEach(function(part) { - if (part.type === type) { - result.push(part); - } else if (Array.isArray(part.content)) { - self.filterBodyParts(part.content, type, result); - } - }); - return result; - }; - - - return EmailSync; -}); \ No newline at end of file diff --git a/src/js/util/update/update-handler.js b/src/js/util/update/update-handler.js index 8836c93..8044ef4 100644 --- a/src/js/util/update/update-handler.js +++ b/src/js/util/update/update-handler.js @@ -3,7 +3,8 @@ define(function(require) { var cfg = require('js/app-config').config, updateV1 = require('js/util/update/update-v1'), - updateV2 = require('js/util/update/update-v2'); + updateV2 = require('js/util/update/update-v2'), + updateV3 = require('js/util/update/update-v3'); /** * Handles database migration @@ -11,7 +12,7 @@ define(function(require) { var UpdateHandler = function(appConfigStorage, userStorage) { this._appConfigStorage = appConfigStorage; this._userStorage = userStorage; - this._updateScripts = [updateV1, updateV2]; + this._updateScripts = [updateV1, updateV2, updateV3]; }; /** diff --git a/src/js/util/update/update-v3.js b/src/js/util/update/update-v3.js new file mode 100644 index 0000000..87180fc --- /dev/null +++ b/src/js/util/update/update-v3.js @@ -0,0 +1,28 @@ +define(function() { + 'use strict'; + + /** + * Update handler for transition database version 2 -> 3 + * + * In database version 3, we introduced new flags to the messages, also + * the outbox uses artificial uids + */ + function updateV2(options, callback) { + var emailDbType = 'email_', + versionDbType = 'dbVersion', + postUpdateDbVersion = 3; + + // remove the emails + options.userStorage.removeList(emailDbType, function(err) { + if (err) { + callback(err); + return; + } + + // update the database version to postUpdateDbVersion + options.appConfigStorage.storeList([postUpdateDbVersion], versionDbType, callback); + }); + } + + return updateV2; +}); \ No newline at end of file diff --git a/src/tpl/mail-list.html b/src/tpl/mail-list.html index 45b69a5..e32eccd 100644 --- a/src/tpl/mail-list.html +++ b/src/tpl/mail-list.html @@ -23,7 +23,7 @@ -