Performance improvements and fixes for mail-list

* Use same list style in desktop as mobile
* Bugfix: don't download all body when list not displayed
* Use ng-infinite-scroll to load DOM nodes on demand
* Custom search filter for FTS and reomve angular filters
* Rubber band scrolling on iOS
* Add CSP support in cordova via html meta tag

Gruntfile.js

@@ -142,7 +142,9 @@ module.exports = function(grunt) {
                     'pgpmailer/node_modules/smtpclient/src/*.js',
                     'pgpmailer/node_modules/smtpclient/node_modules/stringencoding/dist/stringencoding.js',
                     'axe/axe.js',
-                    'dompurify/purify.js'
+                    'dompurify/purify.js',
+                    'jquery/dist/jquery.min.js',
+                    'ng-infinite-scroll/build/ng-infinite-scroll.min.js'
                 ],
                 dest: 'src/lib/'
             },

package.json

@@ -17,7 +17,9 @@
         "pgpbuilder": "https://github.com/whiteout-io/pgpbuilder/tarball/v0.3.5",
         "requirejs": "2.1.14",
         "axe": "https://github.com/whiteout-io/axe/tarball/v0.0.2",
-        "dompurify": "~0.4.2"
+        "dompurify": "~0.4.2",
+        "jquery": "~2.1.1",
+        "ng-infinite-scroll": "https://github.com/sroze/ngInfiniteScroll/tarball/1.1.1"
     },
     "devDependencies": {
         "angularjs": "https://github.com/angular/angular.js/tarball/v1.2.8",

src/index.html

@@ -4,6 +4,9 @@
     <meta charset="utf-8">
     <title>Mail</title>
 
+    <!-- Theses CSP rules are used as a fallback in runtimes such as Cordova -->
+    <meta http-equiv="Content-Security-Policy" content="default-src 'self' chrome-extension:; object-src 'none'; connect-src 'self' chrome-extension: https://*.whiteout.io; style-src 'self' chrome-extension: 'unsafe-inline'; img-src 'self' chrome-extension: data:">
+
     <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0">
     <link rel="stylesheet" media="all" href="css/all.min.css" type="text/css">
 

src/js/app.js

@@ -24,7 +24,8 @@ requirejs([
     'js/util/error',
     'fastclick',
     'angularRoute',
-    'angularAnimate'
+    'angularAnimate',
+    'ngInfiniteScroll'
 ], function(
     angular,
     DialogCtrl,
@@ -63,7 +64,8 @@ requirejs([
         'read',
         'contacts',
         'login-new-device',
-        'popover'
+        'popover',
+        'infinite-scroll'
     ]);
 
     // set router paths

src/js/controller/mail-list.js

@@ -5,7 +5,10 @@ define(function(require) {
         _ = require('underscore'),
         appController = require('js/app-controller'),
         notification = require('js/util/notification'),
-        emailDao, outboxBo, keychainDao;
+        emailDao, outboxBo, keychainDao, searchTimeout;
+
+    var INIT_DISPLAY_LEN = 20,
+        SCROLL_DISPLAY_LEN = 10;
 
     var MailListCtrl = function($scope, $timeout) {
         //
@@ -154,6 +157,9 @@ define(function(require) {
                 return;
             }
 
+            // reset searchFilter
+            $scope.searchText = undefined;
+
             // in development, display dummy mail objects
             if (!window.chrome || !chrome.identity) {
                 updateStatus('Last update: ', new Date());
@@ -165,26 +171,145 @@ define(function(require) {
             openCurrentFolder();
         });
 
-        $scope.$watchCollection('state.nav.currentFolder.messages', selectFirstMessage);
+        $scope.watchMessages = $scope.$watchCollection('state.nav.currentFolder.messages', function(messages) {
-
-        function selectFirstMessage(messages) {
             if (!messages) {
                 return;
             }
 
+            // sort message by uid
+            currentFolder().messages.sort(byUidDescending);
+            // set display buffer to first messages
+            $scope.displayMessages = currentFolder().messages.slice(0, INIT_DISPLAY_LEN);
+
             // Shows the next message based on the uid of the currently selected element
-            if (messages.indexOf(currentMessage()) === -1) {
+            if (currentFolder().messages.indexOf(currentMessage()) === -1) {
-                // wait until after first $digest() so $scope.filteredMessages is set
                 $timeout(function() {
-                    $scope.select($scope.filteredMessages ? $scope.filteredMessages[0] : undefined);
+                    $scope.select($scope.displayMessages[0]);
                 });
             }
+        });
+
+        /**
+         * display more items (for infinite scrolling)
+         */
+        $scope.displayMore = function() {
+            var len = currentFolder().messages.length,
+                dLen = $scope.displayMessages.length;
+
+            if (dLen === len || $scope.searchText) {
+                // all messages are already displayed or we're in search mode
+                return;
             }
 
+            // copy next interval of messages to the end of the display messages array
+            var next = currentFolder().messages.slice(dLen, dLen + SCROLL_DISPLAY_LEN);
+            Array.prototype.push.apply($scope.displayMessages, next);
+        };
+
+        /**
+         * This method is called when the user changes the searchText
+         */
+        $scope.displaySearchResults = function(searchText) {
+            if (searchTimeout) {
+                // remove timeout to wait for user typing query
+                clearTimeout(searchTimeout);
+            }
+
+            if (!searchText) {
+                // set display buffer to first messages
+                $scope.displayMessages = currentFolder().messages.slice(0, INIT_DISPLAY_LEN);
+                $scope.searching = false;
+                updateStatus('Online');
+                return;
+            }
+
+            // display searching spinner
+            $scope.searching = true;
+            updateStatus('Searching ...');
+            searchTimeout = setTimeout(function() {
+                $scope.$apply(function() {
+                    // filter relevant messages
+                    $scope.displayMessages = $scope.search(currentFolder().messages, searchText);
+                    $scope.searching = false;
+                    updateStatus('Matches in this folder');
+                });
+            }, 500);
+        };
+
+        /**
+         * Do full text search on messages. Parse meta data first
+         */
+        $scope.search = function(messages, searchText) {
+            // don't filter on empty searchText
+            if (!searchText) {
+                return messages;
+            }
+
+            // escape search string
+            searchText = searchText.replace(/([.*+?^${}()|\[\]\/\\])/g, "\\$1");
+            // compare all strings (case insensitive)
+            var regex = new RegExp(searchText, 'i');
+
+            function contains(input) {
+                if (!input) {
+                    return false;
+                }
+                return regex.test(input);
+            }
+
+            function checkAddresses(header) {
+                if (!header || !header.length) {
+                    return false;
+                }
+
+                for (var i = 0; i < header.length; i++) {
+                    if (contains(header[i].name) || contains(header[i].address)) {
+                        return true;
+                    }
+                }
+
+                return false;
+            }
+
+            /**
+             * Filter meta data first and then only look at plaintext and decrypted message bodies
+             */
+            function matchMetaDataFirst(m) {
+                // compare subject
+                if (contains(m.subject)) {
+                    return true;
+                }
+                // compares address headers
+                if (checkAddresses(m.from) || checkAddresses(m.to) || checkAddresses(m.cc) || checkAddresses(m.bcc)) {
+                    return true;
+                }
+                // compare plaintext body
+                if (m.body && !m.encrypted && contains(m.body)) {
+                    return true;
+                }
+                // compare decrypted body
+                if (m.body && m.encrypted && m.decrypted && contains(m.body)) {
+                    return true;
+                }
+                // compare plaintex html body
+                if (m.html && !m.encrypted && contains(m.html)) {
+                    return true;
+                }
+                // compare decrypted html body
+                if (m.html && m.encrypted && m.decrypted && contains(m.html)) {
+                    return true;
+                }
+                return false;
+            }
+
+            // user native js Array.filter
+            return messages.filter(matchMetaDataFirst);
+        };
+
         /**
          * Sync current folder when client comes back online
          */
-        $scope.$watch('account.online', function(isOnline) {
+        $scope.watchOnline = $scope.$watch('account.online', function(isOnline) {
             if (isOnline) {
                 updateStatus('Online');
                 openCurrentFolder();
@@ -315,23 +440,29 @@ define(function(require) {
                         listItems = listEl.children[0].children,
                         inViewport = false,
                         listItem, message,
-                        isPartiallyVisibleTop, isPartiallyVisibleBottom, isVisible;
+                        isPartiallyVisibleTop, isPartiallyVisibleBottom, isVisible,
+                        displayMessages = scope[model];
+
+                    if (!top && !bottom) {
+                        // list not visible
+                        return;
+                    }
 
                     for (var i = 0, len = listItems.length; i < len; i++) {
                         // the n-th list item (the dom representation of an email) corresponds to
                         // the n-th message model in the filteredMessages array
                         listItem = listItems.item(i).getBoundingClientRect();
 
-                        if (!scope.filteredMessages || scope.filteredMessages.length <= i) {
+                        if (!displayMessages || displayMessages.length <= i) {
                             // stop if i get larger than the size of filtered messages
                             break;
                         }
-                        message = scope.filteredMessages[i];
+                        message = displayMessages[i];
 
 
                         isPartiallyVisibleTop = listItem.top < top && listItem.bottom > top; // a portion of the list item is visible on the top
                         isPartiallyVisibleBottom = listItem.top < bottom && listItem.bottom > bottom; // a portion of the list item is visible on the bottom
-                        isVisible = listItem.top >= top && listItem.bottom <= bottom; // the list item is visible as a whole
+                        isVisible = (listItem.top || listItem.bottom) && listItem.top >= top && listItem.bottom <= bottom; // the list item is visible as a whole
 
                         if (isPartiallyVisibleTop || isVisible || isPartiallyVisibleBottom) {
                             // we are now iterating over visible elements
@@ -364,13 +495,23 @@ define(function(require) {
         };
     });
 
+    function byUidDescending(a, b) {
+        if (a.uid < b.uid) {
+            return 1;
+        } else if (b.uid < a.uid) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
     // Helper for development mode
 
     function createDummyMails() {
-        var uid = 0;
+        var uid = 1000000;
 
         var Email = function(unread, attachments, answered) {
-            this.uid = uid++;
+            this.uid = uid--;
             this.from = [{
                 name: 'Whiteout Support',
                 address: 'support@whiteout.io'
@@ -489,7 +630,11 @@ define(function(require) {
             this.decrypted = true;
         };
 
-        var dummys = [new Email(true, true), new Email(true, false, true), new Email(false, true, true), new Email(false, true), new Email(false), new Email(false), new Email(false), new Email(false), new Email(false), new Email(false), new Email(false), new Email(false), new Email(false), new Email(false), new Email(false), new Email(false), new Email(false), new Email(false)];
+        var dummys = [new Email(true, true), new Email(true, false, true), new Email(false, true, true), new Email(false, true)];
+
+        for (var i = 0; i < 100; i++) {
+            dummys.push(new Email(false));
+        }
 
         return dummys;
     }

src/require-config.js

@@ -7,6 +7,7 @@
 		paths: {
 			js: '../js',
 			test: '../../test',
+			jquery: 'jquery.min',
 			underscore: 'underscore/underscore-min',
 			lawnchair: 'lawnchair/lawnchair-git',
 			lawnchairSQL: 'lawnchair/lawnchair-adapter-webkit-sqlite-git',
@@ -14,6 +15,7 @@
 			angular: 'angular/angular.min',
 			angularRoute: 'angular/angular-route.min',
 			angularAnimate: 'angular/angular-animate.min',
+			ngInfiniteScroll: 'ng-infinite-scroll.min',
 			uuid: 'uuid/uuid',
 			forge: 'forge/forge.min',
 			punycode: 'punycode.min',
@@ -24,8 +26,12 @@
 			forge: {
 				exports: 'forge'
 			},
+			jquery: {
+				exports: '$'
+			},
 			angular: {
-				exports: 'angular'
+				exports: 'angular',
+				deps: ['jquery']
 			},
 			angularRoute: {
 				exports: 'angular',
@@ -35,6 +41,10 @@
 				exports: 'angular',
 				deps: ['angular']
 			},
+			ngInfiniteScroll: {
+				exports: 'angular',
+				deps: ['jquery', 'angular']
+			},
 			lawnchair: {
 				exports: 'Lawnchair'
 			},

src/sass/components/_mail-list.scss

@@ -159,16 +159,6 @@
                 }
             }
         }
+    }
 
-        @include respond-to(desktop) {
-            margin-top: 0;
-            margin-bottom: 7px;
-            &:before {
-                display: none;
-            }
-            &:last-child {
-                margin-bottom: 0;
-            }
-        }
-    }
 }

src/sass/views/_mail-list.scss

@@ -85,6 +85,9 @@
         flex-grow: 1;
         padding: 0 ($padding-horizontal - $scrollbar-width) 0 $padding-horizontal;
         overflow-y: scroll;
+        // allow scrolling on iOS
+        -webkit-overflow-scrolling: touch;
+        // put layer on GPU
         transform: translatez(0);
     }
 
@@ -127,15 +130,4 @@
         }
     }
 
-    @include respond-to(desktop) {
-        background: $color-grey-lighterer;
-        background-image: linear-gradient(to right ,$color-grey-lighterer 98%, darken($color-grey-lighterer, 1%) 100%);
-
-        footer {
-            background: darken($color-grey-lighterer, 1%);
-            &:before {
-                display: none;
-            }
-        }
-    }
 }

src/tpl/mail-list.html

@@ -9,21 +9,24 @@
   </div>
 
   <div class="search" data-icon="&#xe017;">
-    <input class="input-text" type="text" ng-model="searchText" placeholder="Filter..." focus-me="state.mailList.searching">
+    <input class="input-text" type="text" ng-model="searchText"
+      ng-change="displaySearchResults(searchText)"
+      placeholder="Search" focus-me="state.mailList.searching">
   </div>
 
-  <div class="list-wrapper" list-scroll="filteredMessages">
+  <div class="list-wrapper" list-scroll="displayMessages">
-    <ul class="mail-list">
+    <ul class="mail-list" infinite-scroll="displayMore()"
+      infinite-scroll-distance="1" infinite-scroll-parent="true">
       <li ng-class="{'mail-list-active': email === state.mailList.selected}"
         wo-touch="select(email)"
-        ng-repeat="email in (filteredMessages = (state.nav.currentFolder.messages | filter:searchText | orderBy:'uid':true | limitTo:100))">
+        ng-repeat="email in displayMessages">
         <h3>{{email.from[0].name || email.from[0].address}}</h3>
         <div class="encrypted" data-icon="{{email.encrypted && email.decrypted ? '&#xe012;' : email.encrypted ? '&#xe009;' : ''}}"></div>
         <div class="head">
           <p class="subject">{{email.subject || 'No subject'}}</p>
           <time>{{email.sentDate | date:'mediumDate'}}</time>
         </div>
-        <p class="body">{{email.body}}</p>
+        <p class="body">{{email.body ? email.body.substr(0, 200) : ''}}</p>
         <ul class="flags">
           <li ng-show="email.unread"></li>
           <li data-icon="&#xe002;" ng-show="!email.unread && email.answered"></li>
@@ -33,7 +36,7 @@
     </ul><!--/.mail-list-->
   </div>
 
-  <footer ng-class="{syncing: account.loggingIn || account.busy}">
+  <footer ng-class="{syncing: account.loggingIn || account.busy || searching}">
     <span class="spinner"></span>
     <span class="text" ng-switch="account.online">
       <span ng-switch-when="false">

test/unit/mail-list-ctrl-test.js

@@ -104,6 +104,148 @@ define(function(require) {
             appController._emailDao = origEmailDao;
         });
 
+        describe('displayMore', function() {
+            beforeEach(function() {
+                scope.state.nav = {
+                    currentFolder: {
+                        messages: ['a', 'b']
+                    }
+                };
+            });
+            it('should not do anything when display length equals messages length', function() {
+                scope.displayMessages = ['a', 'b'];
+
+                scope.displayMore();
+                expect(scope.displayMessages.length).to.equal(scope.state.nav.currentFolder.messages.length);
+            });
+            it('should append next message interval', function() {
+                scope.displayMessages = ['a'];
+
+                scope.displayMore();
+                expect(scope.displayMessages.length).to.equal(scope.state.nav.currentFolder.messages.length);
+            });
+        });
+
+        describe('displaySearchResults', function() {
+            var clock;
+
+            beforeEach(function() {
+                scope.state.nav = {
+                    currentFolder: {
+                        messages: ['a', 'b']
+                    }
+                };
+                scope.watchMessages();
+                scope.watchOnline();
+                clock = sinon.useFakeTimers();
+            });
+            afterEach(function() {
+                clock.restore();
+            });
+
+            it('should show initial message on empty', function() {
+                scope.displaySearchResults();
+                expect(scope.searching).to.be.false;
+                expect(scope.lastUpdateLbl).to.equal('Online');
+                expect(scope.displayMessages.length).to.equal(2);
+            });
+            it('should show initial message on empty', function() {
+                var searchStub = sinon.stub(scope, 'search');
+                searchStub.returns(['a']);
+
+
+                scope.displaySearchResults('query');
+                expect(scope.searching).to.be.true;
+                expect(scope.lastUpdateLbl).to.equal('Searching ...');
+                clock.tick(500);
+
+                expect(scope.displayMessages).to.deep.equal(['a']);
+                expect(scope.searching).to.be.false;
+                expect(scope.lastUpdateLbl).to.equal('Matches in this folder');
+
+            });
+        });
+
+        describe('search', function() {
+            var message1 = {
+                    to: [{
+                        name: 'name1',
+                        address: 'address1'
+                    }],
+                    subject: 'subject1',
+                    body: 'body1',
+                    html: 'html1'
+                },
+                message2 = {
+                    to: [{
+                        name: 'name2',
+                        address: 'address2'
+                    }],
+                    subject: 'subject2',
+                    body: 'body2',
+                    html: 'html2'
+                },
+                message3 = {
+                    to: [{
+                        name: 'name3',
+                        address: 'address3'
+                    }],
+                    subject: 'subject3',
+                    body: 'body1',
+                    html: 'html1',
+                    encrypted: true
+                },
+                message4 = {
+                    to: [{
+                        name: 'name4',
+                        address: 'address4'
+                    }],
+                    subject: 'subject4',
+                    body: 'body1',
+                    html: 'html1',
+                    encrypted: true,
+                    decrypted: true
+                },
+                testMessages = [message1, message2, message3, message4];
+
+            it('return same messages array on empty query string', function() {
+                var result = scope.search(testMessages, '');
+                expect(result).to.equal(testMessages);
+            });
+
+            it('return message1 on matching subject', function() {
+                var result = scope.search(testMessages, 'subject1');
+                expect(result.length).to.equal(1);
+                expect(result[0]).to.equal(message1);
+            });
+
+            it('return message1 on matching name', function() {
+                var result = scope.search(testMessages, 'name1');
+                expect(result.length).to.equal(1);
+                expect(result[0]).to.equal(message1);
+            });
+
+            it('return message1 on matching address', function() {
+                var result = scope.search(testMessages, 'address1');
+                expect(result.length).to.equal(1);
+                expect(result[0]).to.equal(message1);
+            });
+
+            it('return plaintext and decrypted messages on matching body', function() {
+                var result = scope.search(testMessages, 'body1');
+                expect(result.length).to.equal(2);
+                expect(result[0]).to.equal(message1);
+                expect(result[1]).to.equal(message4);
+            });
+
+            it('return plaintext and decrypted messages on matching html', function() {
+                var result = scope.search(testMessages, 'html1');
+                expect(result.length).to.equal(2);
+                expect(result[0]).to.equal(message1);
+                expect(result[1]).to.equal(message4);
+            });
+        });
+
         describe('scope variables', function() {
             it('should be set correctly', function() {
                 expect(scope.select).to.exist;