add sliding window delta sync

src/js/dao/email-dao.js

@@ -77,7 +77,7 @@ define(function(require) {
 
                     // every folder is initially created with an unread count of 0.
                     // the unread count will be updated after every sync
-                    folders.forEach(function(folder){
+                    folders.forEach(function(folder) {
                         folder.count = 0;
                     });
 
@@ -352,31 +352,29 @@ define(function(require) {
         }
 
         function doImapDelta() {
-            self._imapListMessages({
+            self._imapSearch({
                 folder: folder.path
-            }, function(err, headers) {
+            }, function(err, uids) {
                 if (err) {
                     self._account.busy = false;
                     callback(err);
                     return;
                 }
 
-                // ignore non-whitelisted mails
+                // uidWrappers is just to wrap the bare uids in an object { uid: 123 } so 
-                var nonWhitelisted = _.filter(headers, function(header) {
+                // the checkDelta function can treat it like something that resembles a stripped down email object...
-                    return header.subject.indexOf(str.subjectPrefix) === -1;
+                var uidWrappers = _.map(uids, function(uid) {
-                });
+                    return {
-                nonWhitelisted.forEach(function(i) {
+                        uid: uid
-                    headers.splice(headers.indexOf(i), 1);
+                    };
                 });
 
                 /*
                  * delta3: memory > imap   => we deleted messages directly from the remote, remove from memory and storage
                  * delta4:   imap > memory => we have new messages available, fetch to memory and storage
-                 * deltaF4:  imap > memory => we changed flags directly on the remote, sync them to the storage and memory
                  */
-                delta3 = checkDelta(folder.messages, headers);
+                delta3 = checkDelta(folder.messages, uidWrappers);
-                delta4 = checkDelta(headers, folder.messages);
+                delta4 = checkDelta(uidWrappers, folder.messages);
-                deltaF4 = checkFlags(headers, folder.messages);
 
                 doDelta3();
 
@@ -417,22 +415,36 @@ define(function(require) {
                 // we have new messages available, fetch to memory and storage
                 // (downstream sync)
                 function doDelta4() {
-                    // no delta, we're done here
+                    // eliminate uids smaller than the biggest local uid, i.e. just fetch everything
-                    if (_.isEmpty(delta4)) {
+                    // that came in AFTER the most recent email we have in memory. Keep in mind that
-                        doDeltaF4();
+                    // uids are strictly ascending, so there can't be a NEW mail in the mailbox with a 
-                        return;
+                    // uid smaller than anything we've encountered before.
+                    if (!_.isEmpty(folder.messages)) {
+                        var localUids = _.pluck(folder.messages, 'uid'),
+                            maxLocalUid = Math.max.apply(null, localUids);
+
+                        // eliminate everything prior to maxLocalUid
+                        delta4 = _.filter(delta4, function(uidWrapper) {
+                            return uidWrapper.uid > maxLocalUid;
+                        });
                     }
 
-                    var after = _.after(delta4.length, function() {
+                    syncNextItem();
-                        doDeltaF4();
+
-                    });
+                    function syncNextItem() {
+                        // no delta, we're done here
+                        if (_.isEmpty(delta4)) {
+                            doDeltaF4();
+                            return;
+                        }
+
+                        // delta4 contains the headers that are newly available on the remote
+                        var nextUidWrapper = delta4.shift();
 
-                    // delta4 contains the headers that are newly available on the remote
-                    delta4.forEach(function(imapHeader) {
                         // get the whole message
                         self._imapGetMessage({
                             folder: folder.path,
-                            uid: imapHeader.uid
+                            uid: nextUidWrapper.uid
                         }, function(err, message) {
                             if (err) {
                                 self._account.busy = false;
@@ -440,22 +452,22 @@ define(function(require) {
                                 return;
                             }
 
-                            // create a bastard child of smtp and imap.
+                            // imap filtering may not be sufficient, since google filters out
-                            // before thinking this is stupid, talk to the guys who wrote this.
+                            // non-alphabetical characters
-                            imapHeader.id = message.id;
+                            if (message.subject.indexOf(str.subjectPrefix) === -1) {
-                            imapHeader.body = message.body;
+                                syncNextItem();
-                            imapHeader.html = message.html;
+                                return;
-                            imapHeader.attachments = message.attachments;
+                            }
 
-                            if (isVerificationMail(imapHeader)) {
+                            if (isVerificationMail(message)) {
-                                verify(imapHeader, function(err) {
+                                verify(message, function(err) {
                                     if (err) {
                                         self._account.busy = false;
                                         callback(err);
                                         return;
                                     }
 
-                                    after();
+                                    syncNextItem();
                                 });
                                 return;
                             }
@@ -463,7 +475,7 @@ define(function(require) {
                             // add the encrypted message to the local storage
                             self._localStoreMessages({
                                 folder: folder.path,
-                                emails: [imapHeader]
+                                emails: [message]
                             }, function(err) {
                                 if (err) {
                                     self._account.busy = false;
@@ -472,7 +484,7 @@ define(function(require) {
                                 }
 
                                 // decrypt and add to folder in memory
-                                handleMessage(imapHeader, function(err, cleartextMessage) {
+                                handleMessage(message, function(err, cleartextMessage) {
                                     if (err) {
                                         self._account.busy = false;
                                         callback(err);
@@ -480,40 +492,109 @@ define(function(require) {
                                     }
 
                                     folder.messages.push(cleartextMessage);
-                                    after();
+                                    syncNextItem();
                                 });
                             });
                         });
+                    }
+                }
+            });
+
+            function doDeltaF4() {
+                var answeredUids, unreadUids;
+
+                getUnreadUids();
+
+                // find all the relevant unread mails
+                function getUnreadUids() {
+                    self._imapSearch({
+                        folder: folder.path,
+                        unread: true
+                    }, function(err, uids) {
+                        if (err) {
+                            self._account.busy = false;
+                            callback(err);
+                            return;
+                        }
+
+                        // we're done here, let's get all the answered mails
+                        unreadUids = uids;
+                        getAnsweredUids();
                     });
                 }
 
-                // we have a mismatch concerning flags between imap and memory.
+                // find all the relevant answered mails
-                // pull changes from imap.
+                function getAnsweredUids() {
-                function doDeltaF4() {
+                    // find all the relevant answered mails
-                    function finishSync() {
+                    self._imapSearch({
-                        self._account.busy = false;
+                        folder: folder.path,
-                        folder.count = _.filter(folder.messages, function(msg) {
+                        answered: true
-                            return msg.unread === true;
+                    }, function(err, uids) {
-                        }).length;
+                        if (err) {
-                        callback();
+                            self._account.busy = false;
-                    }
+                            callback(err);
+                            return;
+                        }
 
+                        // we're done here, let's update what we have in memory and persist that!
+                        answeredUids = uids;
+                        updateFlags();
+                    });
+
+                }
+
+                function updateFlags() {
+                    // deltaF4:  imap > memory => we changed flags directly on the remote, sync them to the storage and memory
+                    deltaF4 = [];
+
+                    folder.messages.forEach(function(msg) {
+                        // if the message's uid is among the uids that should be unread,
+                        // AND the message is not unread, we clearly have to change that
+                        var shouldBeUnread = _.contains(unreadUids, msg.uid);
+                        if (msg.unread === shouldBeUnread) {
+                            // everything is in order, we're good here
+                            return;
+                        }
+
+                        msg.unread = shouldBeUnread;
+                        deltaF4.push(msg);
+                    });
+
+                    folder.messages.forEach(function(msg) {
+                        // if the message's uid is among the uids that should be answered,
+                        // AND the message is not answered, we clearly have to change that
+                        var shouldBeAnswered = _.contains(answeredUids, msg.uid);
+                        if (msg.answered === shouldBeAnswered) {
+                            // everything is in order, we're good here
+                            return;
+                        }
+
+                        msg.answered = shouldBeAnswered;
+                        deltaF4.push(msg);
+                    });
+
+                    // maybe a mail had BOTH flags wrong, so let's create
+                    // a duplicate-free version of deltaF4
+                    deltaF4 = _.uniq(deltaF4);
+
+                    // everything up to date? fine, we're done!
                     if (_.isEmpty(deltaF4)) {
                         finishSync();
                         return;
                     }
 
                     var after = _.after(deltaF4.length, function() {
+                        // we're doing updating everything
                         finishSync();
                     });
 
-                    // deltaF4 contains the imap headers that have changed flags
+                    // alright, so let's sync the corr
-                    deltaF4.forEach(function(imapHeader) {
+                    deltaF4.forEach(function(inMemoryMessage) {
                         // do a short round trip to the database to avoid re-encrypting,
                         // instead use the encrypted object in the storage
                         self._localListMessages({
                             folder: folder.path,
-                            uid: imapHeader.uid
+                            uid: inMemoryMessage.uid
                         }, function(err, storedMessages) {
                             if (err) {
                                 self._account.busy = false;
@@ -522,9 +603,10 @@ define(function(require) {
                             }
 
                             var storedMessage = storedMessages[0];
-                            storedMessage.unread = imapHeader.unread;
+                            storedMessage.unread = inMemoryMessage.unread;
-                            storedMessage.answered = imapHeader.answered;
+                            storedMessage.answered = inMemoryMessage.answered;
 
+                            // persist the modified object
                             self._localStoreMessages({
                                 folder: folder.path,
                                 emails: [storedMessage]
@@ -535,20 +617,24 @@ define(function(require) {
                                     return;
                                 }
 
-                                // after the metadata of the encrypted object has changed, proceed with the live object
+                                // and we're done.
-                                var inMemoryMessage = _.findWhere(folder.messages, {
-                                    uid: imapHeader.uid
-                                });
-                                inMemoryMessage.unread = imapHeader.unread;
-                                inMemoryMessage.answered = imapHeader.answered;
-
                                 after();
                             });
                         });
+
                     });
                 }
 
-            });
+                function finishSync() {
+                    // after all the tags are up to date, let's adjust the unread mail count
+                    folder.count = _.filter(folder.messages, function(msg) {
+                        return msg.unread === true;
+                    }).length;
+                    // allow the next sync to take place
+                    self._account.busy = false;
+                    callback();
+                }
+            }
         }
 
         /*
@@ -878,15 +964,26 @@ define(function(require) {
     };
 
     /**
-     * List messages from an imap folder. This will not yet fetch the email body.
+     * Returns the relevant messages corresponding to the search terms in the options
-     * @param {String} options.folderName The name of the imap folder.
+     * @param {String} options.folder The folder's path
+     * @param {Boolean} options.answered (optional) Mails with or without the \Answered flag set.
+     * @param {Boolean} options.unread (optional) Mails with or without the \Seen flag set.
+     * @param {Function} callback(error, uids) invoked with the uids of messages matching the search terms, or an error object if an error occurred
      */
-    EmailDAO.prototype._imapListMessages = function(options, callback) {
+    EmailDAO.prototype._imapSearch = function(options, callback) {
-        this._imapClient.listMessages({
+        var o = {
             path: options.folder,
-            offset: 0,
+            subject: str.subjectPrefix
-            length: 100
+        };
-        }, callback);
+
+        if (typeof options.answered !== 'undefined') {
+            o.answered = options.answered;
+        }
+        if (typeof options.unread !== 'undefined') {
+            o.unread = options.unread;
+        }
+
+        this._imapClient.search(o, callback);
     };
 
     EmailDAO.prototype._imapDeleteMessage = function(options, callback) {
@@ -901,10 +998,39 @@ define(function(require) {
      * @param {String} options.messageId The
      */
     EmailDAO.prototype._imapGetMessage = function(options, callback) {
-        this._imapClient.getMessagePreview({
+        var self = this;
+
+        self._imapClient.listMessagesByUid({
             path: options.folder,
-            uid: options.uid
+            firstUid: options.uid,
-        }, callback);
+            lastUid: options.uid
+        }, function(err, imapHeaders) {
+            if (err) {
+                callback(err);
+                return;
+            }
+
+            var imapHeader = imapHeaders[0];
+            self._imapClient.getMessagePreview({
+                path: options.folder,
+                uid: options.uid
+            }, function(err, message) {
+                if (err) {
+                    callback(err);
+                    return;
+                }
+
+                // create a bastard child of smtp and imap. before thinking this is stupid, talk to the guys who wrote this.
+                // p.s. it's a parsing issue.
+
+                imapHeader.id = message.id;
+                imapHeader.body = message.body;
+                imapHeader.html = message.html;
+                imapHeader.attachments = message.attachments;
+
+                callback(null, imapHeader);
+            });
+        });
     };
 
     /**