From 22ce159fe6bb8931cd273e42c055be44d57d75b1 Mon Sep 17 00:00:00 2001 From: cketti Date: Wed, 19 May 2010 13:31:48 +0000 Subject: [PATCH] Merge imap-parser branch. Fixes issue 1547. --- src/com/fsck/k9/MessagingController.java | 14 +- src/com/fsck/k9/mail/FetchProfile.java | 2 +- src/com/fsck/k9/mail/Folder.java | 6 + .../k9/mail/store/ImapResponseParser.java | 224 ++++++---- src/com/fsck/k9/mail/store/ImapStore.java | 393 ++++++++++++------ 5 files changed, 432 insertions(+), 207 deletions(-) diff --git a/src/com/fsck/k9/MessagingController.java b/src/com/fsck/k9/MessagingController.java index 4cd405bce..ce565ecfe 100644 --- a/src/com/fsck/k9/MessagingController.java +++ b/src/com/fsck/k9/MessagingController.java @@ -1727,11 +1727,7 @@ public class MessagingController implements Runnable */ for (Part part : viewables) { - fp.clear(); - fp.add(part); - // TODO what happens if the network connection dies? We've got partial - // messages with incorrect status stored. - remoteFolder.fetch(new Message[] { message }, fp, null); + remoteFolder.fetchPart(message, part, null); } // Store the updated message locally localFolder.appendMessages(new Message[] { message }); @@ -3175,9 +3171,11 @@ public class MessagingController implements Runnable remoteFolder = remoteStore.getFolder(message.getFolder().getName()); remoteFolder.open(OpenMode.READ_WRITE); - FetchProfile fp = new FetchProfile(); - fp.add(part); - remoteFolder.fetch(new Message[] { message }, fp, null); + //FIXME: This is an ugly hack that won't be needed once the Message objects have been united. + Message remoteMessage = remoteFolder.getMessage(message.getUid()); + remoteMessage.setBody(message.getBody()); + remoteFolder.fetchPart(remoteMessage, part, null); + localFolder.updateMessage((LocalMessage)message); for (MessagingListener l : getListeners()) { diff --git a/src/com/fsck/k9/mail/FetchProfile.java b/src/com/fsck/k9/mail/FetchProfile.java index 78e974c24..7dcf51401 100644 --- a/src/com/fsck/k9/mail/FetchProfile.java +++ b/src/com/fsck/k9/mail/FetchProfile.java @@ -15,7 +15,7 @@ import java.util.ArrayList; * any information it needs to download the content. * */ -public class FetchProfile extends ArrayList +public class FetchProfile extends ArrayList { /** * Default items available for pre-fetching. It should be expected that any diff --git a/src/com/fsck/k9/mail/Folder.java b/src/com/fsck/k9/mail/Folder.java index 330a70f12..83d185cf9 100644 --- a/src/com/fsck/k9/mail/Folder.java +++ b/src/com/fsck/k9/mail/Folder.java @@ -132,6 +132,12 @@ public abstract class Folder public abstract void fetch(Message[] messages, FetchProfile fp, MessageRetrievalListener listener) throws MessagingException; + public void fetchPart(Message message, Part part, + MessageRetrievalListener listener) throws MessagingException + { + throw new RuntimeException("fetchPart() not implemented."); + } + public abstract void delete(boolean recurse) throws MessagingException; public abstract String getName(); diff --git a/src/com/fsck/k9/mail/store/ImapResponseParser.java b/src/com/fsck/k9/mail/store/ImapResponseParser.java index f4afac0bf..91f0a68c5 100644 --- a/src/com/fsck/k9/mail/store/ImapResponseParser.java +++ b/src/com/fsck/k9/mail/store/ImapResponseParser.java @@ -1,7 +1,3 @@ -/** - * - */ - package com.fsck.k9.mail.store; import android.util.Log; @@ -9,7 +5,6 @@ import com.fsck.k9.K9; import com.fsck.k9.FixedLengthInputStream; import com.fsck.k9.PeekableInputStream; import com.fsck.k9.mail.MessagingException; - import java.io.IOException; import java.io.InputStream; import java.text.ParseException; @@ -20,19 +15,24 @@ import java.util.Locale; public class ImapResponseParser { - SimpleDateFormat mDateTimeFormat = new SimpleDateFormat("dd-MMM-yyyy HH:mm:ss Z", Locale.US); + private static final SimpleDateFormat mDateTimeFormat = new SimpleDateFormat("dd-MMM-yyyy HH:mm:ss Z", Locale.US); + private static final SimpleDateFormat badDateTimeFormat = new SimpleDateFormat("dd MMM yyyy HH:mm:ss Z", Locale.US); + private static final SimpleDateFormat badDateTimeFormat2 = new SimpleDateFormat("E, dd MMM yyyy HH:mm:ss Z", Locale.US); - SimpleDateFormat badDateTimeFormat = new SimpleDateFormat("dd MMM yyyy HH:mm:ss Z", Locale.US); - SimpleDateFormat badDateTimeFormat2 = new SimpleDateFormat("E, dd MMM yyyy HH:mm:ss Z", Locale.US); - - PeekableInputStream mIn; - InputStream mActiveLiteral; + private PeekableInputStream mIn; + private ImapResponse mResponse; + private Exception mException; public ImapResponseParser(PeekableInputStream in) { this.mIn = in; } + public ImapResponse readResponse() throws IOException + { + return readResponse(null); + } + /** * Reads the next response available on the stream and returns an * ImapResponse object that represents it. @@ -40,49 +40,60 @@ public class ImapResponseParser * @return * @throws IOException */ - public ImapResponse readResponse() throws IOException + public ImapResponse readResponse(IImapResponseCallback callback) throws IOException { - ImapResponse response = new ImapResponse(); - if (mActiveLiteral != null) + try { - while (mActiveLiteral.read() != -1) - ; - mActiveLiteral = null; + ImapResponse response = new ImapResponse(); + mResponse = response; + mResponse.mCallback = callback; + + int ch = mIn.peek(); + if (ch == '*') + { + parseUntaggedResponse(); + readTokens(response); + } + else if (ch == '+') + { + response.mCommandContinuationRequested = + parseCommandContinuationRequest(); + readTokens(response); + } + else + { + response.mTag = parseTaggedResponse(); + readTokens(response); + } + if (K9.DEBUG) + { + Log.v(K9.LOG_TAG, "<<< " + response.toString()); + } + + if (mException != null) + { + throw new RuntimeException("readResponse(): Exception in callback method", mException); + } + + return response; } - int ch = mIn.peek(); - if (ch == '*') + finally { - parseUntaggedResponse(); - readTokens(response); + mResponse.mCallback = null; + mResponse = null; + mException = null; } - else if (ch == '+') - { - response.mCommandContinuationRequested = - parseCommandContinuationRequest(); - readTokens(response); - } - else - { - response.mTag = parseTaggedResponse(); - readTokens(response); - } - if (K9.DEBUG) - { - Log.v(K9.LOG_TAG, "<<< " + response.toString()); - } - return response; } private void readTokens(ImapResponse response) throws IOException { response.clear(); Object token; - while ((token = readToken()) != null) + while ((token = readToken(response)) != null) { - response.add(token); - if (mActiveLiteral != null) + if (!(token instanceof ImapList)) { - break; + response.add(token); } } response.mCompleted = token == null; @@ -99,36 +110,30 @@ public class ImapResponseParser * tokens. * @throws IOException */ - public Object readToken() throws IOException + private Object readToken(ImapResponse response) throws IOException { while (true) { - Object token = parseToken(); - if (token == null || !token.equals(")") || !token.equals("]")) + Object token = parseToken(response); + if (token == null || !(token.equals(")") || token.equals("]"))) { return token; } } } - private Object parseToken() throws IOException + private Object parseToken(ImapList parent) throws IOException { - if (mActiveLiteral != null) - { - while (mActiveLiteral.read() != -1) - ; - mActiveLiteral = null; - } while (true) { int ch = mIn.peek(); if (ch == '(') { - return parseList(); + return parseList(parent); } else if (ch == '[') { - return parseSequence(); + return parseSequence(parent); } else if (ch == ')') { @@ -146,8 +151,7 @@ public class ImapResponseParser } else if (ch == '{') { - mActiveLiteral = parseLiteral(); - return mActiveLiteral; + return parseLiteral(); } else if (ch == ' ') { @@ -196,27 +200,27 @@ public class ImapResponseParser return tag; } - private ImapList parseList() throws IOException + private ImapList parseList(ImapList parent) throws IOException { expect('('); ImapList list = new ImapList(); + parent.add(list); Object token; while (true) { - token = parseToken(); + token = parseToken(list); if (token == null) { break; } - else if (token instanceof InputStream) - { - list.add(token); - break; - } else if (token.equals(")")) { break; } + else if (token instanceof ImapList) + { + // Do nothing + } else { list.add(token); @@ -225,27 +229,27 @@ public class ImapResponseParser return list; } - private ImapList parseSequence() throws IOException + private ImapList parseSequence(ImapList parent) throws IOException { expect('['); ImapList list = new ImapList(); + parent.add(list); Object token; while (true) { - token = parseToken(); + token = parseToken(list); if (token == null) { break; } - else if (token instanceof InputStream) - { - list.add(token); - break; - } else if (token.equals("]")) { break; } + else if (token instanceof ImapList) + { + // Do nothing + } else { list.add(token); @@ -297,14 +301,66 @@ public class ImapResponseParser * @param mListener * @throws IOException */ - private InputStream parseLiteral() throws IOException + private Object parseLiteral() throws IOException { expect('{'); int size = Integer.parseInt(readStringUntil('}')); expect('\r'); expect('\n'); - FixedLengthInputStream fixed = new FixedLengthInputStream(mIn, size); - return fixed; + + if (size == 0) + { + return ""; + } + + if (mResponse.mCallback != null) + { + FixedLengthInputStream fixed = new FixedLengthInputStream(mIn, size); + + Object result = null; + try + { + result = mResponse.mCallback.foundLiteral(mResponse, fixed); + } + catch (IOException e) + { + // Pass IOExceptions through + throw e; + } + catch (Exception e) + { + // Catch everything else and save it for later. + mException = e; + //Log.e(K9.LOG_TAG, "parseLiteral(): Exception in callback method", e); + } + + // Check if only some of the literal data was read + int available = fixed.available(); + if ((available > 0) && (available != size)) + { + // If so, skip the rest + fixed.skip(fixed.available()); + } + + if (result != null) + { + return result; + } + } + + byte[] data = new byte[size]; + int read = 0; + while (read != size) + { + int count = mIn.read(data, read, size - read); + if (count == -1) + { + throw new IOException("parseLiteral(): end of stream reached"); + } + read += count; + } + + return new String(data, "US-ASCII"); } /** @@ -536,6 +592,7 @@ public class ImapResponseParser public class ImapResponse extends ImapList { private boolean mCompleted; + private IImapResponseCallback mCallback; boolean mCommandContinuationRequested; String mTag; @@ -595,4 +652,27 @@ public class ImapResponseParser return o1 == o2; } } + + public interface IImapResponseCallback + { + /** + * Callback method that is called by the parser when a literal string + * is found in an IMAP response. + * + * @param response ImapResponse object with the fields that have been + * parsed up until now (excluding the literal string). + * @param literal FixedLengthInputStream that can be used to access + * the literal string. + * + * @return an Object that will be put in the ImapResponse object at the + * place of the literal string. + * + * @throws IOException passed-through if thrown by FixedLengthInputStream + * @throws Exception if something goes wrong. Parsing will be resumed + * and the exception will be thrown after the + * complete IMAP response has been parsed. + */ + public Object foundLiteral(ImapResponse response, FixedLengthInputStream literal) + throws IOException, Exception; + } } diff --git a/src/com/fsck/k9/mail/store/ImapStore.java b/src/com/fsck/k9/mail/store/ImapStore.java index 5d68001d7..fef5d88ac 100644 --- a/src/com/fsck/k9/mail/store/ImapStore.java +++ b/src/com/fsck/k9/mail/store/ImapStore.java @@ -7,6 +7,7 @@ import android.net.NetworkInfo; import android.os.PowerManager; import android.util.Log; import com.fsck.k9.Account; +import com.fsck.k9.FixedLengthInputStream; import com.fsck.k9.K9; import com.fsck.k9.PeekableInputStream; import com.fsck.k9.Utility; @@ -50,8 +51,6 @@ import java.util.concurrent.atomic.AtomicInteger; * 
  * TODO Need to start keeping track of UIDVALIDITY
  * TODO Need a default response handler for things like folder updates
- * TODO In fetch(), if we need a ImapMessage and were given
- * something else we can try to do a pre-fetch first.
  *
*/ public class ImapStore extends Store @@ -1133,26 +1132,6 @@ public class ImapStore extends Store { fetchFields.add("BODY.PEEK[]"); } - for (Object o : fp) - { - if (o != null && o instanceof Part) - { - Part part = (Part) o; - String[] parts = part.getHeader(MimeHeader.HEADER_ANDROID_ATTACHMENT_STORE_DATA); - if (parts != null) - { - String partId = parts[0]; - if ("TEXT".equalsIgnoreCase(partId)) - { - fetchFields.add(String.format("BODY.PEEK[TEXT]<0.%d>", FETCH_BODY_SANE_SUGGESTED_SIZE)); - } - else - { - fetchFields.add("BODY.PEEK[" + partId + "]"); - } - } - } - } try { @@ -1162,11 +1141,16 @@ public class ImapStore extends Store ), false); ImapResponse response; int messageNumber = 0; + + ImapResponseParser.IImapResponseCallback callback = null; + if (fp.contains(FetchProfile.Item.BODY) || fp.contains(FetchProfile.Item.BODY_SANE) || fp.contains(FetchProfile.Item.ENVELOPE)) + { + callback = new FetchBodyCallback(messageMap); + } + do { - response = mConnection.readResponse(); - if (K9.DEBUG) - Log.v(K9.LOG_TAG, "response for fetch: " + response + " for " + getLogId()); + response = mConnection.readResponse(callback); if (response.mTag == null && ImapResponseParser.equalsIgnoreCase(response.get(1), "FETCH")) { @@ -1205,117 +1189,25 @@ public class ImapStore extends Store ImapMessage imapMessage = (ImapMessage) message; - if (fetchList.containsKey("FLAGS")) - { - ImapList flags = fetchList.getKeyedList("FLAGS"); - if (flags != null) - { - for (int i = 0, count = flags.size(); i < count; i++) - { - String flag = flags.getString(i); - if (flag.equalsIgnoreCase("\\Deleted")) - { - imapMessage.setFlagInternal(Flag.DELETED, true); - } - else if (flag.equalsIgnoreCase("\\Answered")) - { - imapMessage.setFlagInternal(Flag.ANSWERED, true); - } - else if (flag.equalsIgnoreCase("\\Seen")) - { - imapMessage.setFlagInternal(Flag.SEEN, true); - } - else if (flag.equalsIgnoreCase("\\Flagged")) - { - imapMessage.setFlagInternal(Flag.FLAGGED, true); - } - } - } - } + Object literal = handleFetchResponse(imapMessage, fetchList); - if (fetchList.containsKey("INTERNALDATE")) + if (literal != null) { - Date internalDate = fetchList.getKeyedDate("INTERNALDATE"); - message.setInternalDate(internalDate); - } - if (fetchList.containsKey("RFC822.SIZE")) - { - int size = fetchList.getKeyedNumber("RFC822.SIZE"); - imapMessage.setSize(size); - } - if (fetchList.containsKey("BODYSTRUCTURE")) - { - ImapList bs = fetchList.getKeyedList("BODYSTRUCTURE"); - if (bs != null) - { - try - { - parseBodyStructure(bs, message, "TEXT"); - } - catch (MessagingException e) - { - if (K9.DEBUG) - Log.d(K9.LOG_TAG, "Error handling message for " + getLogId(), e); - message.setBody(null); - } - } - } - - if (fetchList.containsKey("BODY")) - { - Part part = null; - for (Object o : fp) - { - if (o instanceof Part) - { - part = (Part) o; - break; - } - } - - int index = fetchList.getKeyIndex("BODY") + 2; - Object literal = fetchList.getObject(index); - - // Check if there's an origin octet if (literal instanceof String) - { - String originOctet = (String)literal; - if (originOctet.startsWith("<")) - { - literal = fetchList.getObject(index + 1); - } - } - - InputStream bodyStream; - if (literal instanceof InputStream) - { - bodyStream = (InputStream)literal; - } - else if (literal instanceof String) { String bodyString = (String)literal; - - if (K9.DEBUG) - Log.v(K9.LOG_TAG, "Part is a String: '" + bodyString + "' for " + getLogId()); - - bodyStream = new ByteArrayInputStream(bodyString.getBytes()); + InputStream bodyStream = new ByteArrayInputStream(bodyString.getBytes()); + imapMessage.parse(bodyStream); + } + else if (literal instanceof Integer) + { + // All the work was done in FetchBodyCallback.foundLiteral() } else { // This shouldn't happen throw new MessagingException("Got FETCH response with bogus parameters"); } - - if (part != null) - { - String contentTransferEncoding = part.getHeader( - MimeHeader.HEADER_CONTENT_TRANSFER_ENCODING)[0]; - part.setBody(MimeUtility.decodeBody(bodyStream, contentTransferEncoding)); - } - else - { - imapMessage.parse(bodyStream); - } } if (listener != null) @@ -1339,6 +1231,193 @@ public class ImapStore extends Store } } + + @Override + public void fetchPart(Message message, Part part, MessageRetrievalListener listener) + throws MessagingException + { + checkOpen(); + + String[] parts = part.getHeader(MimeHeader.HEADER_ANDROID_ATTACHMENT_STORE_DATA); + if (parts == null) + { + return; + } + + String fetch; + String partId = parts[0]; + if ("TEXT".equalsIgnoreCase(partId)) + { + fetch = String.format("BODY.PEEK[TEXT]<0.%d>", FETCH_BODY_SANE_SUGGESTED_SIZE); + } + else + { + fetch = String.format("BODY.PEEK[%s]", partId); + } + + try + { + mConnection.sendCommand( + String.format("UID FETCH %s (UID %s)", message.getUid(), fetch), + false); + + ImapResponse response; + int messageNumber = 0; + + ImapResponseParser.IImapResponseCallback callback = new FetchPartCallback(part); + + do + { + response = mConnection.readResponse(callback); + + if ((response.mTag == null) && + (ImapResponseParser.equalsIgnoreCase(response.get(1), "FETCH"))) + { + ImapList fetchList = (ImapList)response.getKeyedValue("FETCH"); + String uid = fetchList.getKeyedString("UID"); + + if (!message.getUid().equals(uid)) + { + if (K9.DEBUG) + Log.d(K9.LOG_TAG, "Did not ask for UID " + uid + " for " + getLogId()); + + handleUntaggedResponse(response); + continue; + } + if (listener != null) + { + listener.messageStarted(uid, messageNumber++, 1); + } + + ImapMessage imapMessage = (ImapMessage) message; + + Object literal = handleFetchResponse(imapMessage, fetchList); + + if (literal != null) + { + if (literal instanceof Body) + { + // Most of the work was done in FetchAttchmentCallback.foundLiteral() + part.setBody((Body)literal); + } + else if (literal instanceof String) + { + String bodyString = (String)literal; + InputStream bodyStream = new ByteArrayInputStream(bodyString.getBytes()); + + String contentTransferEncoding = part.getHeader( + MimeHeader.HEADER_CONTENT_TRANSFER_ENCODING)[0]; + part.setBody(MimeUtility.decodeBody(bodyStream, contentTransferEncoding)); + } + else + { + // This shouldn't happen + throw new MessagingException("Got FETCH response with bogus parameters"); + } + } + + if (listener != null) + { + listener.messageFinished(message, messageNumber, 1); + } + } + else + { + handleUntaggedResponse(response); + } + + while (response.more()); + + } + while (response.mTag == null); + } + catch (IOException ioe) + { + throw ioExceptionHandler(mConnection, ioe); + } + } + + // Returns value of body field + private Object handleFetchResponse(ImapMessage message, ImapList fetchList) throws MessagingException + { + Object result = null; + if (fetchList.containsKey("FLAGS")) + { + ImapList flags = fetchList.getKeyedList("FLAGS"); + if (flags != null) + { + for (int i = 0, count = flags.size(); i < count; i++) + { + String flag = flags.getString(i); + if (flag.equalsIgnoreCase("\\Deleted")) + { + message.setFlagInternal(Flag.DELETED, true); + } + else if (flag.equalsIgnoreCase("\\Answered")) + { + message.setFlagInternal(Flag.ANSWERED, true); + } + else if (flag.equalsIgnoreCase("\\Seen")) + { + message.setFlagInternal(Flag.SEEN, true); + } + else if (flag.equalsIgnoreCase("\\Flagged")) + { + message.setFlagInternal(Flag.FLAGGED, true); + } + } + } + } + + if (fetchList.containsKey("INTERNALDATE")) + { + Date internalDate = fetchList.getKeyedDate("INTERNALDATE"); + message.setInternalDate(internalDate); + } + + if (fetchList.containsKey("RFC822.SIZE")) + { + int size = fetchList.getKeyedNumber("RFC822.SIZE"); + message.setSize(size); + } + + if (fetchList.containsKey("BODYSTRUCTURE")) + { + ImapList bs = fetchList.getKeyedList("BODYSTRUCTURE"); + if (bs != null) + { + try + { + parseBodyStructure(bs, message, "TEXT"); + } + catch (MessagingException e) + { + if (K9.DEBUG) + Log.d(K9.LOG_TAG, "Error handling message for " + getLogId(), e); + message.setBody(null); + } + } + } + + if (fetchList.containsKey("BODY")) + { + int index = fetchList.getKeyIndex("BODY") + 2; + result = fetchList.getObject(index); + + // Check if there's an origin octet + if (result instanceof String) + { + String originOctet = (String)result; + if (originOctet.startsWith("<")) + { + result = fetchList.getObject(index + 1); + } + } + } + + return result; + } + @Override public Flag[] getPermanentFlags() throws MessagingException { @@ -2384,10 +2463,15 @@ public class ImapStore extends Store } private ImapResponse readResponse() throws IOException, MessagingException + { + return readResponse(null); + } + + private ImapResponse readResponse(ImapResponseParser.IImapResponseCallback callback) throws IOException, MessagingException { try { - ImapResponse response = mParser.readResponse(); + ImapResponse response = mParser.readResponse(callback); if (K9.DEBUG) Log.v(K9.LOG_TAG, getLogId() + "<<<" + response); @@ -3369,4 +3453,61 @@ public class ImapStore extends Store { List search() throws IOException, MessagingException; } + + private class FetchBodyCallback implements ImapResponseParser.IImapResponseCallback + { + private HashMap mMessageMap; + + FetchBodyCallback(HashMap mesageMap) + { + mMessageMap = mesageMap; + } + + @Override + public Object foundLiteral(ImapResponse response, + FixedLengthInputStream literal) throws IOException, Exception + { + if (response.mTag == null && + ImapResponseParser.equalsIgnoreCase(response.get(1), "FETCH")) + { + ImapList fetchList = (ImapList)response.getKeyedValue("FETCH"); + String uid = fetchList.getKeyedString("UID"); + + ImapMessage message = (ImapMessage) mMessageMap.get(uid); + message.parse(literal); + + // Return placeholder object + return new Integer(1); + } + return null; + } + } + + private class FetchPartCallback implements ImapResponseParser.IImapResponseCallback + { + private Part mPart; + + FetchPartCallback(Part part) + { + mPart = part; + } + + @Override + public Object foundLiteral(ImapResponse response, + FixedLengthInputStream literal) throws IOException, Exception + { + if (response.mTag == null && + ImapResponseParser.equalsIgnoreCase(response.get(1), "FETCH")) + { + //TODO: check for correct UID + + String contentTransferEncoding = mPart.getHeader( + MimeHeader.HEADER_CONTENT_TRANSFER_ENCODING)[0]; + + return MimeUtility.decodeBody(literal, contentTransferEncoding); + } + return null; + } + } + }