From 19775c399b106149e8906a7482a4b5a4af2ac8fa Mon Sep 17 00:00:00 2001 From: Adithya Abraham Philip Date: Thu, 19 Mar 2015 20:31:34 +0530 Subject: [PATCH] introduced multi-threading refactored oldKeys to updatedKeys added update all keys, ThreadPoolExecutor used modified CachedThreadPoolExecutor --- .../operations/ImportExportOperation.java | 20 +- .../service/KeychainIntentService.java | 235 ++++++++++++++++-- .../keychain/ui/KeyListFragment.java | 88 +++++++ .../keychain/util/ParcelableFileCache.java | 12 + OpenKeychain/src/main/res/menu/key_list.xml | 5 + OpenKeychain/src/main/res/values/strings.xml | 2 + 6 files changed, 327 insertions(+), 35 deletions(-) diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/ImportExportOperation.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/ImportExportOperation.java index 20dba95e9..f2516f1bd 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/ImportExportOperation.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/ImportExportOperation.java @@ -168,7 +168,7 @@ public class ImportExportOperation extends BaseOperation { return new ImportKeyResult(ImportKeyResult.RESULT_FAIL_NOTHING, log); } - int newKeys = 0, oldKeys = 0, badKeys = 0, secret = 0; + int newKeys = 0, updatedKeys = 0, badKeys = 0, secret = 0; ArrayList importedMasterKeyIds = new ArrayList<>(); boolean cancelled = false; @@ -302,7 +302,7 @@ public class ImportExportOperation extends BaseOperation { if (!result.success()) { badKeys += 1; } else if (result.updated()) { - oldKeys += 1; + updatedKeys += 1; importedMasterKeyIds.add(key.getMasterKeyId()); } else { newKeys += 1; @@ -333,7 +333,9 @@ public class ImportExportOperation extends BaseOperation { } // Special: make sure new data is synced into contacts - ContactSyncAdapterService.requestSync(); + // disabling sync right now since it reduces speed while multi-threading + // so, we expect calling functions to take care of it. KeychainIntentService handles this + //ContactSyncAdapterService.requestSync(); // convert to long array long[] importedMasterKeyIdsArray = new long[importedMasterKeyIds.size()]; @@ -348,18 +350,18 @@ public class ImportExportOperation extends BaseOperation { } // special return case: no new keys at all - if (badKeys == 0 && newKeys == 0 && oldKeys == 0) { + if (badKeys == 0 && newKeys == 0 && updatedKeys == 0) { resultType = ImportKeyResult.RESULT_FAIL_NOTHING; } else { if (newKeys > 0) { resultType |= ImportKeyResult.RESULT_OK_NEWKEYS; } - if (oldKeys > 0) { + if (updatedKeys > 0) { resultType |= ImportKeyResult.RESULT_OK_UPDATED; } if (badKeys > 0) { resultType |= ImportKeyResult.RESULT_WITH_ERRORS; - if (newKeys == 0 && oldKeys == 0) { + if (newKeys == 0 && updatedKeys == 0) { resultType |= ImportKeyResult.RESULT_ERROR; } } @@ -369,15 +371,15 @@ public class ImportExportOperation extends BaseOperation { } // Final log entry, it's easier to do this individually - if ( (newKeys > 0 || oldKeys > 0) && badKeys > 0) { + if ( (newKeys > 0 || updatedKeys > 0) && badKeys > 0) { log.add(LogType.MSG_IMPORT_PARTIAL, 1); - } else if (newKeys > 0 || oldKeys > 0) { + } else if (newKeys > 0 || updatedKeys > 0) { log.add(LogType.MSG_IMPORT_SUCCESS, 1); } else { log.add(LogType.MSG_IMPORT_ERROR, 1); } - return new ImportKeyResult(resultType, log, newKeys, oldKeys, badKeys, secret, + return new ImportKeyResult(resultType, log, newKeys, updatedKeys, badKeys, secret, importedMasterKeyIdsArray); } diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/service/KeychainIntentService.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/service/KeychainIntentService.java index d5f13f7ce..0cac8fb32 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/service/KeychainIntentService.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/service/KeychainIntentService.java @@ -21,11 +21,11 @@ package org.sufficientlysecure.keychain.service; import android.app.IntentService; import android.content.Intent; import android.net.Uri; + import android.os.Bundle; import android.os.Message; import android.os.Messenger; import android.os.RemoteException; - import com.textuality.keybase.lib.Proof; import com.textuality.keybase.lib.prover.Prover; @@ -74,7 +74,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import de.measite.minidns.Client; @@ -136,7 +138,7 @@ public class KeychainIntentService extends IntentService implements Progressable private static final IOType[] values = values(); public static IOType fromInt(int n) { - if(n < 0 || n >= values.length) { + if (n < 0 || n >= values.length) { return UNKNOWN; } else { return values[n]; @@ -201,7 +203,127 @@ public class KeychainIntentService extends IntentService implements Progressable Messenger mMessenger; // this attribute can possibly merged with the one above? not sure... - private AtomicBoolean mActionCanceled = new AtomicBoolean(false); + private static AtomicBoolean sActionCanceled = new AtomicBoolean(false); + + /** + * accumulates the results from a multi-threaded key import from a keyserver and + * consolidates them into a single ImportKeyResult, besides keeping count of keys imported and + * total keys to be imported. Also provides the Progressable used by these threads, which + * currently ignores updates + */ + private class KeyImportAccumulator { + private OperationLog mImportLog = new OperationLog(); + private int mTotalKeys; + private int mImportedKeys = 0; + private Progressable mImportProgressable; + ArrayList mImportedMasterKeyIds = new ArrayList(); + private int mBadKeys = 0; + private int mNewKeys = 0; + private int mUpdatedKeys = 0; + private int mSecret = 0; + private int mResultType = 0; + + public KeyImportAccumulator(int totalKeys) { + mTotalKeys = totalKeys; + //ignore updates from ImportExportOperation for now + mImportProgressable = new Progressable() { + @Override + public void setProgress(String message, int current, int total) { + + } + + @Override + public void setProgress(int resourceId, int current, int total) { + + } + + @Override + public void setProgress(int current, int total) { + + } + + @Override + public void setPreventCancel() { + + } + }; + } + + public Progressable getImportProgressable() { + return mImportProgressable; + } + + public int getTotalKeys() { + return mTotalKeys; + } + + public int getImportedKeys() { + return mImportedKeys; + } + + public synchronized void accumulateKeyImport(ImportKeyResult result) { + mImportedKeys++; + mImportLog.addAll(result.getLog().toList());//accumulates log + mBadKeys += result.mBadKeys; + mNewKeys += result.mNewKeys; + mUpdatedKeys += result.mUpdatedKeys; + mSecret += result.mSecret; + + long[] masterKeyIds = result.getImportedMasterKeyIds(); + for (int i = 0; i < masterKeyIds.length; i++) { + mImportedMasterKeyIds.add(masterKeyIds[i]); + } + + // if any key import has been cancelled, set result type to cancelled + // resultType is added to in getConsolidatedKayImport to account for remaining factors + mResultType |= result.getResult() & ImportKeyResult.RESULT_CANCELLED; + + } + + /** + * returns accumulated result of all imports so far + * + * @return + */ + public ImportKeyResult getConsolidatedImportKeyResult() { + + // adding required information to mResultType + // special case,no keys requested for import + if (mBadKeys == 0 && mNewKeys == 0 && mUpdatedKeys == 0) { + mResultType = ImportKeyResult.RESULT_FAIL_NOTHING; + } else { + if (mNewKeys > 0) { + mResultType |= ImportKeyResult.RESULT_OK_NEWKEYS; + } + if (mUpdatedKeys > 0) { + mResultType |= ImportKeyResult.RESULT_OK_UPDATED; + } + if (mBadKeys > 0) { + mResultType |= ImportKeyResult.RESULT_WITH_ERRORS; + if (mNewKeys == 0 && mUpdatedKeys == 0) { + mResultType |= ImportKeyResult.RESULT_ERROR; + } + } + if (mImportLog.containsWarnings()) { + mResultType |= ImportKeyResult.RESULT_WARNINGS; + } + } + + long masterKeyIds[] = new long[mImportedMasterKeyIds.size()]; + for (int i = 0; i < masterKeyIds.length; i++) { + masterKeyIds[i] = mImportedMasterKeyIds.get(i); + } + + return new ImportKeyResult(mResultType, mImportLog, mNewKeys, mUpdatedKeys, mBadKeys, + mSecret, masterKeyIds); + } + + public boolean isImportFinished() { + return mTotalKeys == mImportedKeys; + } + } + + private KeyImportAccumulator mKeyImportAccumulator; public KeychainIntentService() { super("KeychainIntentService"); @@ -216,7 +338,7 @@ public class KeychainIntentService extends IntentService implements Progressable protected void onHandleIntent(Intent intent) { // We have not been cancelled! (yet) - mActionCanceled.set(false); + sActionCanceled.set(false); Bundle extras = intent.getExtras(); if (extras == null) { @@ -242,7 +364,7 @@ public class KeychainIntentService extends IntentService implements Progressable Log.logDebugBundle(data, "EXTRA_DATA"); - ProviderHelper providerHelper = new ProviderHelper(this); + final ProviderHelper providerHelper = new ProviderHelper(this); String action = intent.getAction(); @@ -255,7 +377,7 @@ public class KeychainIntentService extends IntentService implements Progressable String keyServerUri = data.getString(UPLOAD_KEY_SERVER); // Operation - CertifyOperation op = new CertifyOperation(this, providerHelper, this, mActionCanceled); + CertifyOperation op = new CertifyOperation(this, providerHelper, this, sActionCanceled); CertifyResult result = op.certify(parcel, keyServerUri); // Result @@ -473,7 +595,7 @@ public class KeychainIntentService extends IntentService implements Progressable Passphrase passphrase = data.getParcelable(EDIT_KEYRING_PASSPHRASE); // Operation - EditKeyOperation op = new EditKeyOperation(this, providerHelper, this, mActionCanceled); + EditKeyOperation op = new EditKeyOperation(this, providerHelper, this, sActionCanceled); EditKeyResult result = op.execute(saveParcel, passphrase); // Result @@ -487,7 +609,7 @@ public class KeychainIntentService extends IntentService implements Progressable long keyRingId = data.getInt(EXPORT_KEY_RING_MASTER_KEY_ID); // Operation - PromoteKeyOperation op = new PromoteKeyOperation(this, providerHelper, this, mActionCanceled); + PromoteKeyOperation op = new PromoteKeyOperation(this, providerHelper, this, sActionCanceled); PromoteKeyResult result = op.execute(keyRingId); // Result @@ -520,26 +642,68 @@ public class KeychainIntentService extends IntentService implements Progressable break; } case ACTION_IMPORT_KEYRING: { - // Input - String keyServer = data.getString(IMPORT_KEY_SERVER); + final String keyServer = data.getString(IMPORT_KEY_SERVER); ArrayList list = data.getParcelableArrayList(IMPORT_KEY_LIST); ParcelableFileCache cache = new ParcelableFileCache<>(this, "key_import.pcl"); + int totKeys = 0; + Iterator keyListIterator = null; + //either list or cache must be null, no guarantees otherwise + if (list == null) {//export from cache, copied from ImportExportOperation.importKeyRings - // Operation - ImportExportOperation importExportOperation = new ImportExportOperation( - this, providerHelper, this, mActionCanceled); - // Either list or cache must be null, no guarantees otherwise. - ImportKeyResult result = list != null - ? importExportOperation.importKeyRings(list, keyServer) - : importExportOperation.importKeyRings(cache, keyServer); + try { + ParcelableFileCache.IteratorWithSize it = cache.readCache(); + keyListIterator = it; + totKeys = it.getSize(); + } catch (IOException e) { - // Result - sendMessageToHandler(MessageStatus.OKAY, result); + // Special treatment here, we need a lot + OperationLog log = new OperationLog(); + log.add(OperationResult.LogType.MSG_IMPORT, 0, 0); + log.add(OperationResult.LogType.MSG_IMPORT_ERROR_IO, 0, 0); + + keyImportFailed(new ImportKeyResult(ImportKeyResult.RESULT_ERROR, log)); + } + } else { + keyListIterator = list.iterator(); + totKeys = list.size(); + } + + + if (keyListIterator != null) { + mKeyImportAccumulator = new KeyImportAccumulator(totKeys); + setProgress(0, totKeys); + + final int maxThreads = 200; + ExecutorService importExecutor = new ThreadPoolExecutor(0, maxThreads, + 30L, TimeUnit.SECONDS, + new SynchronousQueue()); + + while (keyListIterator.hasNext()) { + final ParcelableKeyRing pkRing = keyListIterator.next(); + Runnable importOperationRunnable = new Runnable() { + @Override + public void run() { + // Operation + ImportExportOperation importExportOperation = new ImportExportOperation( + KeychainIntentService.this, + new ProviderHelper(KeychainIntentService.this), + mKeyImportAccumulator.getImportProgressable(), + sActionCanceled); + + ArrayList list = new ArrayList<>(); + list.add(pkRing); + ImportKeyResult result = importExportOperation.importKeyRings(list, + keyServer); + singleKeyRingImportCompleted(result); + } + }; + importExecutor.execute(importOperationRunnable); + } + } break; - } case ACTION_SIGN_ENCRYPT: { @@ -548,7 +712,7 @@ public class KeychainIntentService extends IntentService implements Progressable // Operation SignEncryptOperation op = new SignEncryptOperation( - this, new ProviderHelper(this), this, mActionCanceled); + this, new ProviderHelper(this), this, sActionCanceled); SignEncryptResult result = op.execute(inputParcel); // Result @@ -584,6 +748,23 @@ public class KeychainIntentService extends IntentService implements Progressable } } + private synchronized void singleKeyRingImportCompleted(ImportKeyResult result) { + mKeyImportAccumulator.accumulateKeyImport(result); + + setProgress(mKeyImportAccumulator.getImportedKeys(), mKeyImportAccumulator.getTotalKeys()); + + if (mKeyImportAccumulator.isImportFinished()) { + ContactSyncAdapterService.requestSync(); + + sendMessageToHandler(MessageStatus.OKAY, + mKeyImportAccumulator.getConsolidatedImportKeyResult()); + } + } + + private void keyImportFailed(ImportKeyResult result) { + sendMessageToHandler(MessageStatus.OKAY, result); + } + private void sendProofError(List log, String label) { String msg = null; label = (label == null) ? "" : label + ": "; @@ -655,7 +836,7 @@ public class KeychainIntentService extends IntentService implements Progressable /** * Set progress of ProgressDialog by sending message to handler on UI thread */ - public void setProgress(String message, int progress, int max) { + public synchronized void setProgress(String message, int progress, int max) { Log.d(Constants.TAG, "Send message by setProgress with progress=" + progress + ", max=" + max); @@ -669,16 +850,16 @@ public class KeychainIntentService extends IntentService implements Progressable sendMessageToHandler(MessageStatus.UPDATE_PROGRESS, null, data); } - public void setProgress(int resourceId, int progress, int max) { + public synchronized void setProgress(int resourceId, int progress, int max) { setProgress(getString(resourceId), progress, max); } - public void setProgress(int progress, int max) { + public synchronized void setProgress(int progress, int max) { setProgress(null, progress, max); } @Override - public void setPreventCancel() { + public synchronized void setPreventCancel() { sendMessageToHandler(MessageStatus.PREVENT_CANCEL); } @@ -743,8 +924,10 @@ public class KeychainIntentService extends IntentService implements Progressable @Override public int onStartCommand(Intent intent, int flags, int startId) { + // onStartCommand will be run on the thread which starts the service + // cancel operation is introduced here as it must not be queued up if (ACTION_CANCEL.equals(intent.getAction())) { - mActionCanceled.set(true); + sActionCanceled.set(true); return START_NOT_STICKY; } return super.onStartCommand(intent, flags, startId); diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/KeyListFragment.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/KeyListFragment.java index 5f1189deb..513da7785 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/KeyListFragment.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/KeyListFragment.java @@ -20,6 +20,7 @@ package org.sufficientlysecure.keychain.ui; import android.animation.ObjectAnimator; import android.annotation.TargetApi; +import android.app.Activity; import android.app.ProgressDialog; import android.content.Context; import android.content.Intent; @@ -58,13 +59,17 @@ import com.getbase.floatingactionbutton.FloatingActionsMenu; import org.sufficientlysecure.keychain.Constants; import org.sufficientlysecure.keychain.R; +import org.sufficientlysecure.keychain.keyimport.ImportKeysListEntry; +import org.sufficientlysecure.keychain.keyimport.ParcelableKeyRing; import org.sufficientlysecure.keychain.operations.results.ConsolidateResult; import org.sufficientlysecure.keychain.operations.results.DeleteResult; +import org.sufficientlysecure.keychain.operations.results.ImportKeyResult; import org.sufficientlysecure.keychain.operations.results.OperationResult; import org.sufficientlysecure.keychain.pgp.KeyRing; import org.sufficientlysecure.keychain.provider.KeychainContract; import org.sufficientlysecure.keychain.provider.KeychainContract.KeyRings; import org.sufficientlysecure.keychain.provider.KeychainDatabase; +import org.sufficientlysecure.keychain.provider.ProviderHelper; import org.sufficientlysecure.keychain.service.KeychainIntentService; import org.sufficientlysecure.keychain.service.KeychainIntentServiceHandler; import org.sufficientlysecure.keychain.ui.dialog.DeleteKeyDialogFragment; @@ -78,6 +83,7 @@ import org.sufficientlysecure.keychain.util.Log; import org.sufficientlysecure.keychain.util.Preferences; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import se.emilsjolander.stickylistheaders.StickyListHeadersAdapter; @@ -477,6 +483,10 @@ public class KeyListFragment extends LoaderFragment mExportHelper.showExportKeysDialog(null, Constants.Path.APP_DIR_FILE, true); return true; + case R.id.menu_key_list_update_all_keys: + updateAllKeys(); + return true; + case R.id.menu_key_list_debug_cons: consolidate(); return true; @@ -561,6 +571,84 @@ public class KeyListFragment extends LoaderFragment startActivityForResult(intent, 0); } + private void updateAllKeys() { + Context context = this.getActivity(); + + ProviderHelper providerHelper = new ProviderHelper(context); + + Cursor cursor = providerHelper.getContentResolver().query( + KeyRings.buildUnifiedKeyRingsUri(), new String[]{ + KeyRings.FINGERPRINT + }, null, null, null + ); + + ArrayList keyList = new ArrayList<>(); + + while (cursor.moveToNext()) { + byte[] blob = cursor.getBlob(0);//fingerprint column is 0 + String fingerprint = KeyFormattingUtils.convertFingerprintToHex(blob); + ParcelableKeyRing keyEntry = new ParcelableKeyRing(fingerprint, null, null); + keyList.add(keyEntry); + } + + KeychainIntentServiceHandler serviceHandler = new KeychainIntentServiceHandler( + getActivity(), + getString(R.string.progress_importing), + ProgressDialog.STYLE_HORIZONTAL, + true) { + public void handleMessage(Message message) { + // handle messages by standard KeychainIntentServiceHandler first + super.handleMessage(message); + + if (message.arg1 == MessageStatus.OKAY.ordinal()) { + // get returned data bundle + Bundle returnData = message.getData(); + if (returnData == null) { + return; + } + final ImportKeyResult result = + returnData.getParcelable(OperationResult.EXTRA_RESULT); + if (result == null) { + Log.e(Constants.TAG, "result == null"); + return; + } + + result.createNotify(KeyListFragment.this.getActivity()).show(); + } + } + }; + + // Send all information needed to service to query keys in other thread + Intent intent = new Intent(getActivity(), KeychainIntentService.class); + + intent.setAction(KeychainIntentService.ACTION_IMPORT_KEYRING); + + // fill values for this action + Bundle data = new Bundle(); + + // search config + { + Preferences prefs = Preferences.getPreferences(getActivity()); + Preferences.CloudSearchPrefs cloudPrefs = + new Preferences.CloudSearchPrefs(true, true, prefs.getPreferredKeyserver()); + data.putString(KeychainIntentService.IMPORT_KEY_SERVER, cloudPrefs.keyserver); + } + + data.putParcelableArrayList(KeychainIntentService.IMPORT_KEY_LIST, keyList); + + intent.putExtra(KeychainIntentService.EXTRA_DATA, data); + + // Create a new Messenger for the communication back + Messenger messenger = new Messenger(serviceHandler); + intent.putExtra(KeychainIntentService.EXTRA_MESSENGER, messenger); + + // show progress dialog + serviceHandler.showProgressDialog(getActivity()); + + // start service with intent + getActivity().startService(intent); + } + private void consolidate() { // Message is received after importing is done in KeychainIntentService KeychainIntentServiceHandler saveHandler = new KeychainIntentServiceHandler( diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/util/ParcelableFileCache.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/util/ParcelableFileCache.java index 6f9cb277e..5a314ad0b 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/util/ParcelableFileCache.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/util/ParcelableFileCache.java @@ -83,10 +83,22 @@ public class ParcelableFileCache { } + /** + * Reads from cache file and deletes it afterward. Convenience function for readCache(boolean). + * @return an IteratorWithSize object containing entries read from the cache file + * @throws IOException + */ public IteratorWithSize readCache() throws IOException { return readCache(true); } + /** + * Reads entries from a cache file and returns an IteratorWithSize object containing the entries + * @param deleteAfterRead if true, the cache file will be deleted after being read + * @return an IteratorWithSize object containing entries read from the cache file + * @throws IOException if cache directory/parcel import file does not exist, or a read error + * occurs + */ public IteratorWithSize readCache(final boolean deleteAfterRead) throws IOException { File cacheDir = mContext.getCacheDir(); diff --git a/OpenKeychain/src/main/res/menu/key_list.xml b/OpenKeychain/src/main/res/menu/key_list.xml index 15e098138..c4797d5f7 100644 --- a/OpenKeychain/src/main/res/menu/key_list.xml +++ b/OpenKeychain/src/main/res/menu/key_list.xml @@ -19,6 +19,11 @@ android:title="@string/menu_manage_keys" app:showAsAction="never" /> + + "Add keys" "Search cloud" "Export all keys" + "Update all keys" "Show advanced info" "Confirm via fingerprint comparison" "Export Log" @@ -316,6 +317,7 @@ "cancelling…" "saving…" "importing…" + "Updating keys…" "exporting…" "uploading…" "building key…"