consolidate: working implementation, lacking ui

OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpImportExport.java

@@ -36,6 +36,7 @@ import org.sufficientlysecure.keychain.service.KeychainIntentService;
 import org.sufficientlysecure.keychain.service.OperationResultParcel.OperationLog;
 import org.sufficientlysecure.keychain.service.OperationResults.ImportKeyResult;
 import org.sufficientlysecure.keychain.service.OperationResults.SaveKeyringResult;
+import org.sufficientlysecure.keychain.util.IterableIterator;
 import org.sufficientlysecure.keychain.util.Log;
 import org.sufficientlysecure.keychain.util.ProgressScaler;
 
@@ -43,6 +44,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 
 public class PgpImportExport {
@@ -60,10 +62,14 @@ public class PgpImportExport {
     private ProviderHelper mProviderHelper;
 
     public PgpImportExport(Context context, Progressable progressable) {
+        this(context, new ProviderHelper(context), progressable);
+    }
+
+    public PgpImportExport(Context context, ProviderHelper providerHelper, Progressable progressable) {
         super();
         this.mContext = context;
         this.mProgressable = progressable;
-        this.mProviderHelper = new ProviderHelper(context);
+        this.mProviderHelper = providerHelper;
     }
 
     public PgpImportExport(Context context,
@@ -124,11 +130,14 @@ public class PgpImportExport {
 
     /** Imports keys from given data. If keyIds is given only those are imported */
     public ImportKeyResult importKeyRings(List<ParcelableKeyRing> entries) {
+        return importKeyRings(entries.iterator(), entries.size());
+    }
 
+    public ImportKeyResult importKeyRings(Iterator<ParcelableKeyRing> entries, int num) {
         updateProgress(R.string.progress_importing, 0, 100);
 
         // If there aren't even any keys, do nothing here.
-        if (entries == null || entries.size() == 0) {
+        if (entries == null || !entries.hasNext()) {
             return new ImportKeyResult(
                     ImportKeyResult.RESULT_FAIL_NOTHING, mProviderHelper.getLog(), 0, 0, 0);
         }
@@ -136,8 +145,8 @@ public class PgpImportExport {
         int newKeys = 0, oldKeys = 0, badKeys = 0;
 
         int position = 0;
-        int progSteps = 100 / entries.size();
+        double progSteps = 100.0 / num;
-        for (ParcelableKeyRing entry : entries) {
+        for (ParcelableKeyRing entry : new IterableIterator<ParcelableKeyRing>(entries)) {
             try {
                 UncachedKeyRing key = UncachedKeyRing.decodeFromData(entry.getBytes());
 
@@ -157,10 +166,10 @@ public class PgpImportExport {
                 SaveKeyringResult result;
                 if (key.isSecret()) {
                     result = mProviderHelper.saveSecretKeyRing(key,
-                            new ProgressScaler(mProgressable, position, (position+1)*progSteps, 100));
+                            new ProgressScaler(mProgressable, (int)(position*progSteps), (int)((position+1)*progSteps), 100));
                 } else {
                     result = mProviderHelper.savePublicKeyRing(key,
-                            new ProgressScaler(mProgressable, position, (position+1)*progSteps, 100));
+                            new ProgressScaler(mProgressable, (int)(position*progSteps), (int)((position+1)*progSteps), 100));
                 }
                 if (!result.success()) {
                     badKeys += 1;

OpenKeychain/src/main/java/org/sufficientlysecure/keychain/provider/KeychainDatabase.java

@@ -349,7 +349,7 @@ public class KeychainDatabase extends SQLiteOpenHelper {
         copy(in, out);
     }
 
-    // for test cases ONLY!!
+    // DANGEROUS
     public void clearDatabase() {
         getWritableDatabase().execSQL("delete from " + Tables.KEY_RINGS_PUBLIC);
     }

OpenKeychain/src/main/java/org/sufficientlysecure/keychain/provider/ProviderHelper.java

@@ -28,12 +28,14 @@ import android.os.RemoteException;
 import android.support.v4.util.LongSparseArray;
 
 import org.sufficientlysecure.keychain.Constants;
+import org.sufficientlysecure.keychain.keyimport.ParcelableKeyRing;
 import org.sufficientlysecure.keychain.pgp.CanonicalizedPublicKey;
 import org.sufficientlysecure.keychain.pgp.CanonicalizedPublicKeyRing;
 import org.sufficientlysecure.keychain.pgp.CanonicalizedSecretKeyRing;
 import org.sufficientlysecure.keychain.pgp.KeyRing;
 import org.sufficientlysecure.keychain.pgp.NullProgressable;
 import org.sufficientlysecure.keychain.pgp.PgpHelper;
+import org.sufficientlysecure.keychain.pgp.PgpImportExport;
 import org.sufficientlysecure.keychain.pgp.PgpKeyHelper;
 import org.sufficientlysecure.keychain.pgp.Progressable;
 import org.sufficientlysecure.keychain.pgp.UncachedKeyRing;
@@ -51,9 +53,12 @@ import org.sufficientlysecure.keychain.remote.AppSettings;
 import org.sufficientlysecure.keychain.service.OperationResultParcel.LogLevel;
 import org.sufficientlysecure.keychain.service.OperationResultParcel.LogType;
 import org.sufficientlysecure.keychain.service.OperationResultParcel.OperationLog;
+import org.sufficientlysecure.keychain.service.OperationResults.ConsolidateResult;
 import org.sufficientlysecure.keychain.service.OperationResults.SaveKeyringResult;
+import org.sufficientlysecure.keychain.util.FileImportCache;
 import org.sufficientlysecure.keychain.util.IterableIterator;
 import org.sufficientlysecure.keychain.util.Log;
+import org.sufficientlysecure.keychain.util.ProgressScaler;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -63,6 +68,7 @@ import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 
@@ -97,14 +103,6 @@ public class ProviderHelper {
         mIndent = indent;
     }
 
-    public void resetLog() {
-        if(mLog != null) {
-            // Start a new log (leaving the old one intact)
-            mLog = new OperationLog();
-            mIndent = 0;
-        }
-    }
-
     public OperationLog getLog() {
         return mLog;
     }
@@ -825,6 +823,173 @@ public class ProviderHelper {
 
     }
 
+    public ConsolidateResult consolidateDatabase(Progressable progress) {
+
+        // 1a. fetch all secret keyrings into a cache file
+        int numSecrets, numPublics;
+
+        log(LogLevel.START, LogType.MSG_CON, mIndent);
+        mIndent += 1;
+
+        try {
+
+            log(LogLevel.DEBUG, LogType.MSG_CON_SAVE_SECRET, mIndent);
+            mIndent += 1;
+
+            final Cursor cursor = mContentResolver.query(KeyRings.buildUnifiedKeyRingsUri(), new String[] {
+                    KeyRings.PRIVKEY_DATA, KeyRings.FINGERPRINT, KeyRings.HAS_ANY_SECRET
+            }, KeyRings.HAS_ANY_SECRET + " = 1", null, null);
+
+            if (cursor == null || !cursor.moveToFirst()) {
+                return new ConsolidateResult(ConsolidateResult.RESULT_ERROR, mLog);
+            }
+
+            numSecrets = cursor.getCount();
+
+            FileImportCache<ParcelableKeyRing> cache =
+                    new FileImportCache<ParcelableKeyRing>(mContext, "consolidate_secret.pcl");
+            cache.writeCache(new Iterator<ParcelableKeyRing>() {
+                ParcelableKeyRing ring;
+
+                @Override
+                public boolean hasNext() {
+                    if (ring != null) {
+                        return true;
+                    }
+                    if (cursor.isAfterLast()) {
+                        return false;
+                    }
+                    ring = new ParcelableKeyRing(cursor.getBlob(0),
+                            PgpKeyHelper.convertFingerprintToHex(cursor.getBlob(1)));
+                    cursor.moveToNext();
+                    return true;
+                }
+
+                @Override
+                public ParcelableKeyRing next() {
+                    try {
+                        return ring;
+                    } finally {
+                        ring = null;
+                    }
+                }
+
+                @Override
+                public void remove() {
+                    throw new UnsupportedOperationException();
+                }
+
+            });
+
+        } catch (IOException e) {
+            Log.e(Constants.TAG, "error saving secret");
+            return new ConsolidateResult(ConsolidateResult.RESULT_ERROR, mLog);
+        } finally {
+            mIndent -= 1;
+        }
+
+        // 1b. fetch all public keyrings into a cache file
+        try {
+
+            log(LogLevel.DEBUG, LogType.MSG_CON_SAVE_PUBLIC, mIndent);
+            mIndent += 1;
+
+            final Cursor cursor = mContentResolver.query(KeyRings.buildUnifiedKeyRingsUri(), new String[] {
+                    KeyRings.PUBKEY_DATA, KeyRings.FINGERPRINT
+            }, null, null, null);
+
+            if (cursor == null || !cursor.moveToFirst()) {
+                return new ConsolidateResult(ConsolidateResult.RESULT_ERROR, mLog);
+            }
+
+            numPublics = cursor.getCount();
+
+            FileImportCache<ParcelableKeyRing> cache =
+                    new FileImportCache<ParcelableKeyRing>(mContext, "consolidate_public.pcl");
+            cache.writeCache(new Iterator<ParcelableKeyRing>() {
+                ParcelableKeyRing ring;
+
+                @Override
+                public boolean hasNext() {
+                    if (ring != null) {
+                        return true;
+                    }
+                    if (cursor.isAfterLast()) {
+                        return false;
+                    }
+                    ring = new ParcelableKeyRing(cursor.getBlob(0),
+                            PgpKeyHelper.convertFingerprintToHex(cursor.getBlob(1)));
+                    cursor.moveToNext();
+                    return true;
+                }
+
+                @Override
+                public ParcelableKeyRing next() {
+                    try {
+                        return ring;
+                    } finally {
+                        ring = null;
+                    }
+                }
+
+                @Override
+                public void remove() {
+                    throw new UnsupportedOperationException();
+                }
+
+            });
+
+        } catch (IOException e) {
+            Log.e(Constants.TAG, "error saving public");
+            return new ConsolidateResult(ConsolidateResult.RESULT_ERROR, mLog);
+        } finally {
+            mIndent -= 1;
+        }
+
+        // 2. wipe database (IT'S DANGEROUS)
+        log(LogLevel.DEBUG, LogType.MSG_CON_DB_CLEAR, mIndent);
+        new KeychainDatabase(mContext).clearDatabase();
+
+        // 3. Re-Import secret keyrings from cache
+        try {
+            log(LogLevel.DEBUG, LogType.MSG_CON_REIMPORT_SECRET, mIndent, numSecrets);
+            mIndent += 1;
+
+            FileImportCache<ParcelableKeyRing> cache =
+                    new FileImportCache<ParcelableKeyRing>(mContext, "consolidate_secret.pcl");
+            new PgpImportExport(mContext, this, new ProgressScaler(progress, 10, 25, 100))
+                    .importKeyRings(cache.readCache(), numSecrets);
+        } catch (IOException e) {
+            Log.e(Constants.TAG, "error importing secret");
+            return new ConsolidateResult(ConsolidateResult.RESULT_ERROR, mLog);
+        } finally {
+            mIndent -= 1;
+        }
+
+        // 3. Re-Import public keyrings from cache
+        try {
+            log(LogLevel.DEBUG, LogType.MSG_CON_REIMPORT_PUBLIC, mIndent, numPublics);
+            mIndent += 1;
+
+            FileImportCache<ParcelableKeyRing> cache =
+                    new FileImportCache<ParcelableKeyRing>(mContext, "consolidate_public.pcl");
+            new PgpImportExport(mContext, this, new ProgressScaler(progress, 25, 99, 100))
+                    .importKeyRings(cache.readCache(), numPublics);
+        } catch (IOException e) {
+            Log.e(Constants.TAG, "error importing public");
+            return new ConsolidateResult(ConsolidateResult.RESULT_ERROR, mLog);
+        } finally {
+            mIndent -= 1;
+        }
+
+        progress.setProgress(100, 100);
+        log(LogLevel.OK, LogType.MSG_CON_SUCCESS, mIndent);
+        mIndent -= 1;
+
+        return new ConsolidateResult(ConsolidateResult.RESULT_OK, mLog);
+
+    }
+
     /**
      * Build ContentProviderOperation to add PGPPublicKey to database corresponding to a keyRing
      */

OpenKeychain/src/main/java/org/sufficientlysecure/keychain/service/KeychainIntentService.java

@@ -52,6 +52,7 @@ import org.sufficientlysecure.keychain.pgp.exception.PgpGeneralMsgIdException;
 import org.sufficientlysecure.keychain.provider.KeychainContract.KeyRings;
 import org.sufficientlysecure.keychain.provider.KeychainDatabase;
 import org.sufficientlysecure.keychain.provider.ProviderHelper;
+import org.sufficientlysecure.keychain.service.OperationResults.ConsolidateResult;
 import org.sufficientlysecure.keychain.service.OperationResults.EditKeyResult;
 import org.sufficientlysecure.keychain.service.OperationResults.ImportKeyResult;
 import org.sufficientlysecure.keychain.service.OperationResults.SaveKeyringResult;
@@ -103,6 +104,8 @@ public class KeychainIntentService extends IntentService
 
     public static final String ACTION_CERTIFY_KEYRING = Constants.INTENT_PREFIX + "SIGN_KEYRING";
 
+    public static final String ACTION_CONSOLIDATE = Constants.INTENT_PREFIX + "CONSOLIDATE";
+
     /* keys for data bundle */
 
     // encrypt, decrypt, import export
@@ -142,6 +145,7 @@ public class KeychainIntentService extends IntentService
 
     // import key
     public static final String IMPORT_KEY_LIST = "import_key_list";
+    public static final String IMPORT_KEY_FILE = "import_key_file";
 
     // export key
     public static final String EXPORT_OUTPUT_STREAM = "export_output_stream";
@@ -179,6 +183,8 @@ public class KeychainIntentService extends IntentService
 
     public static final String RESULT_IMPORT = "result";
 
+    public static final String RESULT_CONSOLIDATE = "consolidate_result";
+
     Messenger mMessenger;
 
     private boolean mIsCanceled;
@@ -662,7 +668,16 @@ public class KeychainIntentService extends IntentService
             } catch (Exception e) {
                 sendErrorToHandler(e);
             }
+
+        } else if (ACTION_CONSOLIDATE.equals(action)) {
+            ConsolidateResult result = new ProviderHelper(this).consolidateDatabase(this);
+
+            Bundle resultData = new Bundle();
+            resultData.putParcelable(RESULT_CONSOLIDATE, result);
+
+            sendMessageToHandler(KeychainIntentServiceHandler.MESSAGE_OKAY, resultData);
         }
+
     }
 
     private void sendErrorToHandler(Exception e) {

OpenKeychain/src/main/java/org/sufficientlysecure/keychain/service/OperationResultParcel.java

@@ -388,6 +388,15 @@ public class OperationResultParcel implements Parcelable {
         MSG_MF_UID_ERROR_EMPTY (R.string.msg_mf_uid_error_empty),
         MSG_MF_UNLOCK_ERROR (R.string.msg_mf_unlock_error),
         MSG_MF_UNLOCK (R.string.msg_mf_unlock),
+
+        // consolidate
+        MSG_CON (R.string.msg_con),
+        MSG_CON_SAVE_SECRET (R.string.msg_con_save_secret),
+        MSG_CON_SAVE_PUBLIC (R.string.msg_con_save_public),
+        MSG_CON_DB_CLEAR (R.string.msg_con_db_clear),
+        MSG_CON_REIMPORT_SECRET (R.plurals.msg_con_reimport_secret),
+        MSG_CON_REIMPORT_PUBLIC (R.plurals.msg_con_reimport_public),
+        MSG_CON_SUCCESS (R.string.msg_con_success),
         ;
 
         private final int mMsgId;

OpenKeychain/src/main/java/org/sufficientlysecure/keychain/service/OperationResults.java

@@ -272,4 +272,12 @@ public abstract class OperationResults {
         };
     }
 
+    public static class ConsolidateResult extends OperationResultParcel {
+
+        public ConsolidateResult(int result, OperationLog log) {
+            super(result, log);
+        }
+
+    }
+
 }

OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/KeyListActivity.java

@@ -17,8 +17,11 @@
 
 package org.sufficientlysecure.keychain.ui;
 
+import android.app.ProgressDialog;
 import android.content.Intent;
 import android.os.Bundle;
+import android.os.Message;
+import android.os.Messenger;
 import android.view.Menu;
 import android.view.MenuItem;
 
@@ -28,6 +31,9 @@ import org.sufficientlysecure.keychain.helper.ExportHelper;
 import org.sufficientlysecure.keychain.helper.Preferences;
 import org.sufficientlysecure.keychain.provider.KeychainContract;
 import org.sufficientlysecure.keychain.provider.KeychainDatabase;
+import org.sufficientlysecure.keychain.service.KeychainIntentService;
+import org.sufficientlysecure.keychain.service.KeychainIntentServiceHandler;
+import org.sufficientlysecure.keychain.service.OperationResults.ConsolidateResult;
 import org.sufficientlysecure.keychain.util.Log;
 import org.sufficientlysecure.keychain.util.Notify;
 
@@ -63,6 +69,7 @@ public class KeyListActivity extends DrawerActivity {
         getMenuInflater().inflate(R.menu.key_list, menu);
 
         if (Constants.DEBUG) {
+            menu.findItem(R.id.menu_key_list_debug_cons).setVisible(true);
             menu.findItem(R.id.menu_key_list_debug_read).setVisible(true);
             menu.findItem(R.id.menu_key_list_debug_write).setVisible(true);
             menu.findItem(R.id.menu_key_list_debug_first_time).setVisible(true);
@@ -92,6 +99,10 @@ public class KeyListActivity extends DrawerActivity {
                 mExportHelper.showExportKeysDialog(null, Constants.Path.APP_DIR_FILE, true);
                 return true;
 
+            case R.id.menu_key_list_debug_cons:
+                consolidate();
+                return true;
+
             case R.id.menu_key_list_debug_read:
                 try {
                     KeychainDatabase.debugBackup(this, true);
@@ -136,4 +147,53 @@ public class KeyListActivity extends DrawerActivity {
         startActivity(intent);
     }
 
+    private void consolidate() {
+        // Message is received after importing is done in KeychainIntentService
+        KeychainIntentServiceHandler saveHandler = new KeychainIntentServiceHandler(
+                this,
+                getString(R.string.progress_importing),
+                ProgressDialog.STYLE_HORIZONTAL) {
+            public void handleMessage(Message message) {
+                // handle messages by standard KeychainIntentServiceHandler first
+                super.handleMessage(message);
+
+                if (message.arg1 == KeychainIntentServiceHandler.MESSAGE_OKAY) {
+                    // get returned data bundle
+                    Bundle returnData = message.getData();
+                    if (returnData == null) {
+                        return;
+                    }
+                    final ConsolidateResult result =
+                            returnData.getParcelable(KeychainIntentService.RESULT_CONSOLIDATE);
+                    if (result == null) {
+                        return;
+                    }
+
+                    result.createNotify(KeyListActivity.this).show();
+                }
+            }
+        };
+
+        // Send all information needed to service to import key in other thread
+        Intent intent = new Intent(this, KeychainIntentService.class);
+
+        intent.setAction(KeychainIntentService.ACTION_CONSOLIDATE);
+
+        // fill values for this action
+        Bundle data = new Bundle();
+
+        intent.putExtra(KeychainIntentService.EXTRA_DATA, data);
+
+        // Create a new Messenger for the communication back
+        Messenger messenger = new Messenger(saveHandler);
+        intent.putExtra(KeychainIntentService.EXTRA_MESSENGER, messenger);
+
+        // show progress dialog
+        saveHandler.showProgressDialog(this);
+
+        // start service with intent
+        startService(intent);
+
+    }
+
 }

OpenKeychain/src/main/res/menu/key_list.xml

@@ -31,6 +31,12 @@
         app:showAsAction="never"
         android:title="@string/menu_import_existing_key" />
 
+    <item
+        android:id="@+id/menu_key_list_debug_cons"
+        app:showAsAction="never"
+        android:title="Debug / Consolidate"
+        android:visible="false" />
+
     <item
         android:id="@+id/menu_key_list_debug_read"
         app:showAsAction="never"

OpenKeychain/src/main/res/values/strings.xml

@@ -668,6 +668,21 @@
     <string name="msg_mf_unlock_error">Error unlocking keyring!</string>
     <string name="msg_mf_unlock">Unlocking keyring</string>
 
+    <!-- Consolidate -->
+    <string name="msg_con">Consolidating database</string>
+    <string name="msg_con_save_secret">Saving secret keyrings</string>
+    <string name="msg_con_save_public">Saving public keyrings</string>
+    <string name="msg_con_db_clear">Clearing database</string>
+    <plurals name="msg_con_reimport_secret">
+        <item quantity="one">Reimporting one secret key</item>
+        <item quantity="other">Reimporting %d secret keys</item>
+    </plurals>
+    <plurals name="msg_con_reimport_public">
+        <item quantity="one">Reimporting one public key</item>
+        <item quantity="other">Reimporting %d public keys</item>
+    </plurals>
+    <string name="msg_con_success">Successfully consolidated database</string>
+
     <!-- PassphraseCache -->
     <string name="passp_cache_notif_click_to_clear">Click to clear cached passphrases</string>
     <string name="passp_cache_notif_n_keys">OpenKeychain has cached %d passphrases</string>