Minimal version that reconstructs original message from the database

This change breaks all kinds of things, e.g.
- deleting messages
- updating messages
- downloading attachments
- deleting attachments
- searching in message bodies

k9mail-library/src/main/java/com/fsck/k9/mail/Multipart.java

@@ -28,7 +28,9 @@ public abstract class Multipart implements CompositeBody {
         return Collections.unmodifiableList(mParts);
     }
 
-    public abstract String getContentType();
+    public abstract String getMimeType();
+
+    public abstract String getBoundary();
 
     public int getCount() {
         return mParts.size();
@@ -64,4 +66,7 @@ public abstract class Multipart implements CompositeBody {
             ((TextBody)body).setCharset(charset);
         }
     }
+
+    public abstract byte[] getPreamble();
+    public abstract byte[] getEpilogue();
 }

k9mail-library/src/main/java/com/fsck/k9/mail/Part.java

@@ -31,6 +31,8 @@ public interface Part {
 
     void writeTo(OutputStream out) throws IOException, MessagingException;
 
+    void writeHeaderTo(OutputStream out) throws IOException, MessagingException;
+
     /**
      * Called just prior to transmission, once the type of transport is known to
      * be 7bit.

k9mail-library/src/main/java/com/fsck/k9/mail/internet/MimeBodyPart.java

@@ -5,7 +5,6 @@ import com.fsck.k9.mail.Body;
 import com.fsck.k9.mail.BodyPart;
 import com.fsck.k9.mail.CompositeBody;
 import com.fsck.k9.mail.MessagingException;
-import com.fsck.k9.mail.Multipart;
 
 import java.io.BufferedWriter;
 import java.io.IOException;
@@ -135,6 +134,11 @@ public class MimeBodyPart extends BodyPart {
         }
     }
 
+    @Override
+    public void writeHeaderTo(OutputStream out) throws IOException, MessagingException {
+        mHeader.writeTo(out);
+    }
+
     @Override
     public void setUsing7bitTransport() throws MessagingException {
         String type = getFirstHeader(MimeHeader.HEADER_CONTENT_TYPE);

k9mail-library/src/main/java/com/fsck/k9/mail/internet/MimeMessage.java

@@ -443,6 +443,11 @@ public class MimeMessage extends Message {
         }
     }
 
+    @Override
+    public void writeHeaderTo(OutputStream out) throws IOException, MessagingException {
+        mHeader.writeTo(out);
+    }
+
     @Override
     public InputStream getInputStream() throws MessagingException {
         return null;
@@ -518,7 +523,10 @@ public class MimeMessage extends Message {
 
             Part e = (Part)stack.peek();
             try {
-                MimeMultipart multiPart = new MimeMultipart(e.getContentType());
+                String contentType = e.getContentType();
+                String mimeType = MimeUtility.getHeaderParameter(contentType, null);
+                String boundary = MimeUtility.getHeaderParameter(contentType, "boundary");
+                MimeMultipart multiPart = new MimeMultipart(mimeType, boundary);
                 e.setBody(multiPart);
                 stack.addFirst(multiPart);
             } catch (MessagingException me) {

k9mail-library/src/main/java/com/fsck/k9/mail/internet/MimeMessageHelper.java

@@ -23,9 +23,10 @@ public class MimeMessageHelper {
         if (body instanceof Multipart) {
             Multipart multipart = ((Multipart) body);
             multipart.setParent(part);
-            String type = multipart.getContentType();
+            String mimeType = multipart.getMimeType();
-            part.setHeader(MimeHeader.HEADER_CONTENT_TYPE, type);
+            String contentType = String.format("%s; boundary=\"%s\"", mimeType, multipart.getBoundary());
-            if ("multipart/signed".equalsIgnoreCase(type)) {
+            part.setHeader(MimeHeader.HEADER_CONTENT_TYPE, contentType);
+            if ("multipart/signed".equalsIgnoreCase(mimeType)) {
                 setEncoding(part, MimeUtil.ENC_7BIT);
             } else {
                 setEncoding(part, MimeUtil.ENC_8BIT);

k9mail-library/src/main/java/com/fsck/k9/mail/internet/MimeMultipart.java

@@ -10,30 +10,26 @@ import java.util.Locale;
 import java.util.Random;
 
 public class MimeMultipart extends Multipart {
-    private byte[] mPreamble;
+    private String mimeType;
-    private byte[] mEpilogue;
+    private byte[] preamble;
-
+    private byte[] epilogue;
-    private String mContentType;
+    private final String boundary;
-
-    private final String mBoundary;
 
     public MimeMultipart() throws MessagingException {
-        mBoundary = generateBoundary();
+        boundary = generateBoundary();
         setSubType("mixed");
     }
 
-    public MimeMultipart(String contentType) throws MessagingException {
+    public MimeMultipart(String mimeType, String boundary) throws MessagingException {
-        this.mContentType = contentType;
+        if (mimeType == null) {
-        try {
+            throw new IllegalArgumentException("mimeType can't be null");
-            mBoundary = MimeUtility.getHeaderParameter(contentType, "boundary");
-            if (mBoundary == null) {
-                throw new MessagingException("MultiPart does not contain boundary: " + contentType);
-            }
-        } catch (Exception e) {
-            throw new MessagingException(
-                "Invalid MultiPart Content-Type; must contain subtype and boundary. ("
-                + contentType + ")", e);
         }
+        if (boundary == null) {
+            throw new IllegalArgumentException("boundary can't be null");
+        }
+
+        this.mimeType = mimeType;
+        this.boundary = boundary;
     }
 
     public String generateBoundary() {
@@ -46,40 +42,53 @@ public class MimeMultipart extends Multipart {
         return sb.toString().toUpperCase(Locale.US);
     }
 
+    @Override
+    public String getBoundary() {
+        return boundary;
+    }
+
+    public byte[] getPreamble() {
+        return preamble;
+    }
+
     public void setPreamble(byte[] preamble) {
-        this.mPreamble = preamble;
+        this.preamble = preamble;
+    }
+
+    public byte[] getEpilogue() {
+        return epilogue;
     }
 
     public void setEpilogue(byte[] epilogue) {
-        mEpilogue = epilogue;
+        this.epilogue = epilogue;
     }
 
     @Override
-    public String getContentType() {
+    public String getMimeType() {
-        return mContentType;
+        return mimeType;
     }
 
     public void setSubType(String subType) {
-        mContentType = String.format("multipart/%s; boundary=\"%s\"", subType, mBoundary);
+        mimeType = "multipart/" + subType;
     }
 
     @Override
     public void writeTo(OutputStream out) throws IOException, MessagingException {
         BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out), 1024);
 
-        if (mPreamble != null) {
+        if (preamble != null) {
-            out.write(mPreamble);
+            out.write(preamble);
             writer.write("\r\n");
         }
 
         if (getBodyParts().isEmpty()) {
             writer.write("--");
-            writer.write(mBoundary);
+            writer.write(boundary);
             writer.write("\r\n");
         } else {
             for (BodyPart bodyPart : getBodyParts()) {
                 writer.write("--");
-                writer.write(mBoundary);
+                writer.write(boundary);
                 writer.write("\r\n");
                 writer.flush();
                 bodyPart.writeTo(out);
@@ -88,11 +97,11 @@ public class MimeMultipart extends Multipart {
         }
 
         writer.write("--");
-        writer.write(mBoundary);
+        writer.write(boundary);
         writer.write("--\r\n");
         writer.flush();
-        if (mEpilogue != null) {
+        if (epilogue != null) {
-            out.write(mEpilogue);
+            out.write(epilogue);
         }
     }
 

k9mail/src/main/java/com/fsck/k9/activity/MessageCompose.java

@@ -1338,7 +1338,6 @@ public class MessageCompose extends K9Activity implements OnClickListener,
     private MimeMessage createMessage(boolean isDraft) throws MessagingException {
         MimeMessage message = new MimeMessage();
         message.addSentDate(new Date(), K9.hideTimeZone());
-        message.generateMessageId();
         Address from = new Address(mIdentity.getEmail(), mIdentity.getName());
         message.setFrom(from);
         message.setRecipients(RecipientType.TO, getAddresses(mToView));
@@ -1426,6 +1425,8 @@ public class MessageCompose extends K9Activity implements OnClickListener,
             message.addHeader(K9.IDENTITY_HEADER, buildIdentityHeader(body, bodyPlain));
         }
 
+        message.generateMessageId();
+
         return message;
     }
 

k9mail/src/main/java/com/fsck/k9/mailstore/BinaryMemoryBody.java

@@ -0,0 +1,42 @@
+package com.fsck.k9.mailstore;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import com.fsck.k9.mail.Body;
+import com.fsck.k9.mail.MessagingException;
+import com.fsck.k9.mail.internet.RawDataBody;
+
+
+public class BinaryMemoryBody implements Body, RawDataBody {
+    private final byte[] data;
+    private final String encoding;
+
+    public BinaryMemoryBody(byte[] data, String encoding) {
+        this.data = data;
+        this.encoding = encoding;
+    }
+
+    @Override
+    public String getEncoding() {
+        return encoding;
+    }
+
+    @Override
+    public InputStream getInputStream() throws MessagingException {
+        return new ByteArrayInputStream(data);
+    }
+
+    @Override
+    public void setEncoding(String encoding) throws UnavailableStorageException, MessagingException {
+        throw new RuntimeException("nope"); //FIXME
+    }
+
+    @Override
+    public void writeTo(OutputStream out) throws IOException, MessagingException {
+        out.write(data);
+    }
+}

k9mail/src/main/java/com/fsck/k9/mailstore/LocalFolder.java

@@ -1,6 +1,8 @@
 package com.fsck.k9.mailstore;
 
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -15,6 +17,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.Stack;
 import java.util.UUID;
 import java.util.regex.Pattern;
 
@@ -27,7 +30,6 @@ import android.net.Uri;
 import android.util.Log;
 
 import com.fsck.k9.Account;
-import com.fsck.k9.Account.MessageFormat;
 import com.fsck.k9.K9;
 import com.fsck.k9.activity.Search;
 import com.fsck.k9.helper.HtmlConverter;
@@ -42,6 +44,7 @@ import com.fsck.k9.mail.Message;
 import com.fsck.k9.mail.Message.RecipientType;
 import com.fsck.k9.mail.MessageRetrievalListener;
 import com.fsck.k9.mail.MessagingException;
+import com.fsck.k9.mail.Multipart;
 import com.fsck.k9.mail.Part;
 import com.fsck.k9.mail.internet.MimeBodyPart;
 import com.fsck.k9.mail.internet.MimeHeader;
@@ -49,11 +52,16 @@ import com.fsck.k9.mail.internet.MimeMessage;
 import com.fsck.k9.mail.internet.MimeMessageHelper;
 import com.fsck.k9.mail.internet.MimeMultipart;
 import com.fsck.k9.mail.internet.MimeUtility;
-import com.fsck.k9.mail.internet.TextBody;
 import com.fsck.k9.mailstore.LockableDatabase.DbCallback;
 import com.fsck.k9.mailstore.LockableDatabase.WrappedException;
 import com.fsck.k9.provider.AttachmentProvider;
 import org.apache.commons.io.IOUtils;
+import org.apache.james.mime4j.MimeException;
+import org.apache.james.mime4j.parser.ContentHandler;
+import org.apache.james.mime4j.parser.MimeStreamParser;
+import org.apache.james.mime4j.stream.BodyDescriptor;
+import org.apache.james.mime4j.stream.Field;
+import org.apache.james.mime4j.stream.MimeConfig;
 import org.apache.james.mime4j.util.MimeUtil;
 
 
@@ -616,182 +624,9 @@ public class LocalFolder extends Folder<LocalMessage> implements Serializable {
                         open(OPEN_MODE_RW);
                         if (fp.contains(FetchProfile.Item.BODY)) {
                             for (Message message : messages) {
-                                LocalMessage localMessage = (LocalMessage)message;
+                                LocalMessage localMessage = (LocalMessage) message;
-                                Cursor cursor = null;
-                                MimeMultipart mp = new MimeMultipart();
-                                mp.setSubType("mixed");
-                                try {
-                                    cursor = db.rawQuery("SELECT html_content, text_content, mime_type FROM messages "
-                                                         + "WHERE id = ?",
-                                                         new String[] { Long.toString(localMessage.getId()) });
-                                    cursor.moveToNext();
-                                    String htmlContent = cursor.getString(0);
-                                    String textContent = cursor.getString(1);
-                                    String mimeType = cursor.getString(2);
-                                    if (mimeType != null && mimeType.toLowerCase(Locale.US).startsWith("multipart/")) {
-                                        // If this is a multipart message, preserve both text
-                                        // and html parts, as well as the subtype.
-                                        mp.setSubType(mimeType.toLowerCase(Locale.US).replaceFirst("^multipart/", ""));
-                                        if (textContent != null) {
-                                            LocalTextBody body = new LocalTextBody(textContent, htmlContent);
-                                            MimeBodyPart bp = new MimeBodyPart(body, "text/plain");
-                                            mp.addBodyPart(bp);
-                                        }
 
-                                        if (getAccount().getMessageFormat() != MessageFormat.TEXT) {
+                                loadMessageParts(db, localMessage);
-                                            if (htmlContent != null) {
-                                                TextBody body = new TextBody(htmlContent);
-                                                MimeBodyPart bp = new MimeBodyPart(body, "text/html");
-                                                mp.addBodyPart(bp);
-                                            }
-
-                                            // If we have both text and html content and our MIME type
-                                            // isn't multipart/alternative, then corral them into a new
-                                            // multipart/alternative part and put that into the parent.
-                                            // If it turns out that this is the only part in the parent
-                                            // MimeMultipart, it'll get fixed below before we attach to
-                                            // the message.
-                                            if (textContent != null && htmlContent != null && !mimeType.equalsIgnoreCase("multipart/alternative")) {
-                                                MimeMultipart alternativeParts = mp;
-                                                alternativeParts.setSubType("alternative");
-                                                mp = new MimeMultipart();
-                                                mp.addBodyPart(new MimeBodyPart(alternativeParts));
-                                            }
-                                        }
-                                    } else if (mimeType != null && mimeType.equalsIgnoreCase("text/plain")) {
-                                        // If it's text, add only the plain part. The MIME
-                                        // container will drop away below.
-                                        if (textContent != null) {
-                                            LocalTextBody body = new LocalTextBody(textContent, htmlContent);
-                                            MimeBodyPart bp = new MimeBodyPart(body, "text/plain");
-                                            mp.addBodyPart(bp);
-                                        }
-                                    } else if (mimeType != null && mimeType.equalsIgnoreCase("text/html")) {
-                                        // If it's html, add only the html part. The MIME
-                                        // container will drop away below.
-                                        if (htmlContent != null) {
-                                            TextBody body = new TextBody(htmlContent);
-                                            MimeBodyPart bp = new MimeBodyPart(body, "text/html");
-                                            mp.addBodyPart(bp);
-                                        }
-                                    } else {
-                                        // MIME type not set. Grab whatever part we can get,
-                                        // with Text taking precedence. This preserves pre-HTML
-                                        // composition behaviour.
-                                        if (textContent != null) {
-                                            LocalTextBody body = new LocalTextBody(textContent, htmlContent);
-                                            MimeBodyPart bp = new MimeBodyPart(body, "text/plain");
-                                            mp.addBodyPart(bp);
-                                        } else if (htmlContent != null) {
-                                            TextBody body = new TextBody(htmlContent);
-                                            MimeBodyPart bp = new MimeBodyPart(body, "text/html");
-                                            mp.addBodyPart(bp);
-                                        }
-                                    }
-
-                                } catch (Exception e) {
-                                    Log.e(K9.LOG_TAG, "Exception fetching message:", e);
-                                } finally {
-                                    Utility.closeQuietly(cursor);
-                                }
-
-                                try {
-                                    cursor = db.query(
-                                                 "attachments",
-                                                 new String[] {
-                                                     "id",
-                                                     "size",
-                                                     "name",
-                                                     "mime_type",
-                                                     "store_data",
-                                                     "content_uri",
-                                                     "content_id",
-                                                     "content_disposition"
-                                                 },
-                                                 "message_id = ?",
-                                                 new String[] { Long.toString(localMessage.getId()) },
-                                                 null,
-                                                 null,
-                                                 null);
-
-                                    while (cursor.moveToNext()) {
-                                        long id = cursor.getLong(0);
-                                        int size = cursor.getInt(1);
-                                        String name = cursor.getString(2);
-                                        String type = cursor.getString(3);
-                                        String storeData = cursor.getString(4);
-                                        String contentUri = cursor.getString(5);
-                                        String contentId = cursor.getString(6);
-                                        String contentDisposition = cursor.getString(7);
-                                        String encoding = MimeUtility.getEncodingforType(type);
-                                        Body body = null;
-
-                                        if (contentDisposition == null) {
-                                            contentDisposition = "attachment";
-                                        }
-
-                                        if (contentUri != null) {
-                                            if (MimeUtil.isMessage(type)) {
-                                                body = new LocalAttachmentMessageBody(
-                                                        Uri.parse(contentUri),
-                                                        LocalFolder.this.localStore.context);
-                                            } else {
-                                                body = new LocalAttachmentBody(
-                                                        Uri.parse(contentUri),
-                                                        LocalFolder.this.localStore.context);
-                                            }
-                                        }
-
-                                        MimeBodyPart bp = new LocalAttachmentBodyPart(body, id);
-                                        bp.setEncoding(encoding);
-                                        if (name != null) {
-                                            bp.setHeader(MimeHeader.HEADER_CONTENT_TYPE,
-                                                         String.format("%s;\r\n name=\"%s\"",
-                                                                       type,
-                                                                       name));
-                                            bp.setHeader(MimeHeader.HEADER_CONTENT_DISPOSITION,
-                                                         String.format(Locale.US, "%s;\r\n filename=\"%s\";\r\n size=%d",
-                                                                       contentDisposition,
-                                                                       name, // TODO: Should use encoded word defined in RFC 2231.
-                                                                       size));
-                                        } else {
-                                            bp.setHeader(MimeHeader.HEADER_CONTENT_TYPE, type);
-                                            bp.setHeader(MimeHeader.HEADER_CONTENT_DISPOSITION,
-                                                    String.format(Locale.US, "%s;\r\n size=%d",
-                                                                  contentDisposition,
-                                                                  size));
-                                        }
-
-                                        bp.setHeader(MimeHeader.HEADER_CONTENT_ID, contentId);
-                                        /*
-                                         * HEADER_ANDROID_ATTACHMENT_STORE_DATA is a custom header we add to that
-                                         * we can later pull the attachment from the remote store if necessary.
-                                         */
-                                        bp.setHeader(MimeHeader.HEADER_ANDROID_ATTACHMENT_STORE_DATA, storeData);
-
-                                        mp.addBodyPart(bp);
-                                    }
-                                } finally {
-                                    Utility.closeQuietly(cursor);
-                                }
-
-                                if (mp.getCount() == 0) {
-                                    // If we have no body, remove the container and create a
-                                    // dummy plain text body. This check helps prevents us from
-                                    // triggering T_MIME_NO_TEXT and T_TVD_MIME_NO_HEADERS
-                                    // SpamAssassin rules.
-                                    localMessage.setHeader(MimeHeader.HEADER_CONTENT_TYPE, "text/plain");
-                                    MimeMessageHelper.setBody(localMessage, new TextBody(""));
-                                } else if (mp.getCount() == 1 &&
-                                        !(mp.getBodyPart(0) instanceof LocalAttachmentBodyPart)) {
-                                    // If we have only one part, drop the MimeMultipart container.
-                                    BodyPart part = mp.getBodyPart(0);
-                                    localMessage.setHeader(MimeHeader.HEADER_CONTENT_TYPE, part.getContentType());
-                                    MimeMessageHelper.setBody(localMessage, part.getBody());
-                                } else {
-                                    // Otherwise, attach the MimeMultipart to the message.
-                                    MimeMessageHelper.setBody(localMessage, mp);
-                                }
                             }
                         }
                     } catch (MessagingException e) {
@@ -801,7 +636,156 @@ public class LocalFolder extends Folder<LocalMessage> implements Serializable {
                 }
             });
         } catch (WrappedException e) {
-            throw(MessagingException) e.getCause();
+            throw (MessagingException) e.getCause();
+        }
+    }
+
+    private void loadMessageParts(SQLiteDatabase db, LocalMessage message) throws MessagingException {
+        Map<Long, Part> partById = new HashMap<Long, Part>();
+
+        String[] columns = {
+                "id",                   // 0
+                "type",                 // 1
+                "parent",               // 2
+                "mime_type",            // 3
+                "decoded_body_size",    // 4
+                "display_name",         // 5
+                "header",               // 6
+                "encoding",             // 7
+                "charset",              // 8
+                "data_location",        // 9
+                "data",                 // 10
+                "preamble",             // 11
+                "epilogue",             // 12
+                "boundary",             // 13
+                "content_id",           // 14
+                "server_extra",         // 15
+        };
+        Cursor cursor = db.query("message_parts", columns, "root = ?",
+                new String[] { String.valueOf(message.getMessagePartId()) }, null, null, "seq");
+        try {
+            while (cursor.moveToNext()) {
+                loadMessagePart(message, partById, cursor);
+            }
+        } finally {
+            cursor.close();
+        }
+    }
+
+    private void loadMessagePart(LocalMessage message, Map<Long, Part> partById, Cursor cursor)
+            throws MessagingException {
+
+        long id = cursor.getLong(0);
+        long parentId = cursor.getLong(2);
+        String mimeType = cursor.getString(3);
+        byte[] header = cursor.getBlob(6);
+
+        final Part part;
+        if (id == message.getMessagePartId()) {
+            part = message;
+        } else {
+            Part parentPart = partById.get(parentId);
+            if (parentPart == null) {
+                throw new IllegalStateException("Parent part not found");
+            }
+
+            String parentMimeType = parentPart.getMimeType();
+            if (parentMimeType.startsWith("multipart/")) {
+                BodyPart bodyPart = new MimeBodyPart();
+                ((Multipart) parentPart.getBody()).addBodyPart(bodyPart);
+                part = bodyPart;
+            } else if (parentMimeType.startsWith("message/")) {
+                Message innerMessage = new MimeMessage();
+                parentPart.setBody(innerMessage);
+                part = innerMessage;
+            } else {
+                throw new IllegalStateException("Parent is neither a multipart nor a message");
+            }
+
+            parseHeaderBytes(part, header);
+        }
+        partById.put(id, part);
+
+        boolean isMultipart = mimeType.startsWith("multipart/");
+        if (isMultipart) {
+            byte[] preamble = cursor.getBlob(11);
+            byte[] epilogue = cursor.getBlob(12);
+            String boundary = cursor.getString(13);
+
+            MimeMultipart multipart = new MimeMultipart(mimeType, boundary);
+            part.setBody(multipart);
+            multipart.setPreamble(preamble);
+            multipart.setEpilogue(epilogue);
+        } else {
+            String encoding = cursor.getString(7);
+            byte[] data = cursor.getBlob(10);
+
+            Body body = new BinaryMemoryBody(data, encoding);
+            part.setBody(body);
+        }
+    }
+
+    private void parseHeaderBytes(final Part part, byte[] header) throws MessagingException {
+        MimeConfig parserConfig  = new MimeConfig();
+        parserConfig.setMaxHeaderLen(-1);
+        parserConfig.setMaxLineLen(-1);
+        parserConfig.setMaxHeaderCount(-1);
+        MimeStreamParser parser = new MimeStreamParser(parserConfig);
+        parser.setContentHandler(new ContentHandler() {
+            @Override
+            public void field(Field rawField) throws MimeException {
+                String name = rawField.getName();
+                String raw = rawField.getRaw().toString();
+                try {
+                    part.addRawHeader(name, raw);
+                } catch (MessagingException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+
+            @Override
+            public void startMessage() throws MimeException { /* do nothing */ }
+
+            @Override
+            public void endMessage() throws MimeException { /* do nothing */ }
+
+            @Override
+            public void startBodyPart() throws MimeException { /* do nothing */ }
+
+            @Override
+            public void endBodyPart() throws MimeException { /* do nothing */ }
+
+            @Override
+            public void startHeader() throws MimeException { /* do nothing */ }
+
+            @Override
+            public void endHeader() throws MimeException { /* do nothing */ }
+
+            @Override
+            public void preamble(InputStream is) throws MimeException, IOException { /* do nothing */ }
+
+            @Override
+            public void epilogue(InputStream is) throws MimeException, IOException { /* do nothing */ }
+
+            @Override
+            public void startMultipart(BodyDescriptor bd) throws MimeException { /* do nothing */ }
+
+            @Override
+            public void endMultipart() throws MimeException { /* do nothing */ }
+
+            @Override
+            public void body(BodyDescriptor bd, InputStream is) throws MimeException, IOException { /* do nothing */ }
+
+            @Override
+            public void raw(InputStream is) throws MimeException, IOException { /* do nothing */ }
+        });
+
+        try {
+            parser.parse(new ByteArrayInputStream(header));
+        } catch (MimeException me) {
+            throw new MessagingException("Error parsing headers", me);
+        } catch (IOException e) {
+            throw new MessagingException("I/O error parsing headers", e);
         }
     }
 
@@ -813,49 +797,16 @@ public class LocalFolder extends Folder<LocalMessage> implements Serializable {
             "LocalStore.getMessages(int, int, MessageRetrievalListener) not yet implemented");
     }
 
-    /**
+    void populateHeaders(final LocalMessage message) throws MessagingException {
-     * Populate the header fields of the given list of messages by reading
-     * the saved header data from the database.
-     *
-     * @param messages
-     *            The messages whose headers should be loaded.
-     * @throws UnavailableStorageException
-     */
-    void populateHeaders(final List<LocalMessage> messages) throws MessagingException {
         this.localStore.database.execute(false, new DbCallback<Void>() {
             @Override
             public Void doDbWork(final SQLiteDatabase db) throws WrappedException, MessagingException {
-                Cursor cursor = null;
+                Cursor cursor = db.query("message_parts", new String[] { "header" }, "id = ?",
-                if (messages.isEmpty()) {
+                        new String[] { Long.toString(message.getMessagePartId()) }, null, null, null);
-                    return null;
-                }
                 try {
-                    Map<Long, LocalMessage> popMessages = new HashMap<Long, LocalMessage>();
+                    if (cursor.moveToFirst()) {
-                    List<String> ids = new ArrayList<String>();
+                        byte[] header = cursor.getBlob(0);
-                    StringBuilder questions = new StringBuilder();
+                        parseHeaderBytes(message, header);
-
-                    for (int i = 0; i < messages.size(); i++) {
-                        if (i != 0) {
-                            questions.append(", ");
-                        }
-                        questions.append("?");
-                        LocalMessage message = messages.get(i);
-                        Long id = message.getId();
-                        ids.add(Long.toString(id));
-                        popMessages.put(id, message);
-                    }
-
-                    cursor = db.rawQuery(
-                            "SELECT message_id, name, value FROM headers " +
-                            "WHERE message_id in ( " + questions + ") ORDER BY id ASC",
-                            ids.toArray(LocalStore.EMPTY_STRING_ARRAY));
-
-                    while (cursor.moveToNext()) {
-                        Long id = cursor.getLong(0);
-                        String name = cursor.getString(1);
-                        String value = cursor.getString(2);
-                        //Log.i(K9.LOG_TAG, "Retrieved header name= " + name + ", value = " + value + " for message " + id);
-                        popMessages.get(id).addHeader(name, value);
                     }
                 } finally {
                     Utility.closeQuietly(cursor);
@@ -1225,7 +1176,8 @@ public class LocalFolder extends Folder<LocalMessage> implements Serializable {
      * @param copy
      * @return uidMap of srcUids -> destUids
      */
-    private Map<String, String> appendMessages(final List<? extends Message> messages, final boolean copy) throws MessagingException {
+    private Map<String, String> appendMessages(final List<? extends Message> messages, final boolean copy)
+            throws MessagingException {
         open(OPEN_MODE_RW);
         try {
             final Map<String, String> uidMap = new HashMap<String, String>();
@@ -1234,136 +1186,7 @@ public class LocalFolder extends Folder<LocalMessage> implements Serializable {
                 public Void doDbWork(final SQLiteDatabase db) throws WrappedException, UnavailableStorageException {
                     try {
                         for (Message message : messages) {
-                            long oldMessageId = -1;
+                            saveMessage(db, message, copy, uidMap);
-                            String uid = message.getUid();
-                            if (uid == null || copy) {
-                                /*
-                                 * Create a new message in the database
-                                 */
-                                String randomLocalUid = K9.LOCAL_UID_PREFIX +
-                                        UUID.randomUUID().toString();
-
-                                if (copy) {
-                                    // Save mapping: source UID -> target UID
-                                    uidMap.put(uid, randomLocalUid);
-                                } else {
-                                    // Modify the Message instance to reference the new UID
-                                    message.setUid(randomLocalUid);
-                                }
-
-                                // The message will be saved with the newly generated UID
-                                uid = randomLocalUid;
-                            } else {
-                                /*
-                                 * Replace an existing message in the database
-                                 */
-                                LocalMessage oldMessage = getMessage(uid);
-
-                                if (oldMessage != null) {
-                                    oldMessageId = oldMessage.getId();
-                                }
-
-                                deleteAttachments(message.getUid());
-                            }
-
-                            long rootId = -1;
-                            long parentId = -1;
-
-                            if (oldMessageId == -1) {
-                                // This is a new message. Do the message threading.
-                                ThreadInfo threadInfo = doMessageThreading(db, message);
-                                oldMessageId = threadInfo.msgId;
-                                rootId = threadInfo.rootId;
-                                parentId = threadInfo.parentId;
-                            }
-
-                            boolean isDraft = (message.getHeader(K9.IDENTITY_HEADER) != null);
-
-                            List<Part> attachments;
-                            String text;
-                            String html;
-                            if (isDraft) {
-                                // Don't modify the text/plain or text/html part of our own
-                                // draft messages because this will cause the values stored in
-                                // the identity header to be wrong.
-                                ViewableContainer container =
-                                        LocalMessageExtractor.extractPartsFromDraft(message);
-
-                                text = container.text;
-                                html = container.html;
-                                attachments = container.attachments;
-                            } else {
-                                ViewableContainer container =
-                                        LocalMessageExtractor.extractTextAndAttachments(LocalFolder.this.localStore.context, message);
-
-                                attachments = container.attachments;
-                                text = container.text;
-                                html = HtmlConverter.convertEmoji2Img(container.html);
-                            }
-
-                            String preview = Message.calculateContentPreview(text);
-
-                            try {
-                                ContentValues cv = new ContentValues();
-                                cv.put("uid", uid);
-                                cv.put("subject", message.getSubject());
-                                cv.put("sender_list", Address.pack(message.getFrom()));
-                                cv.put("date", message.getSentDate() == null
-                                       ? System.currentTimeMillis() : message.getSentDate().getTime());
-                                cv.put("flags", LocalFolder.this.localStore.serializeFlags(message.getFlags()));
-                                cv.put("deleted", message.isSet(Flag.DELETED) ? 1 : 0);
-                                cv.put("read", message.isSet(Flag.SEEN) ? 1 : 0);
-                                cv.put("flagged", message.isSet(Flag.FLAGGED) ? 1 : 0);
-                                cv.put("answered", message.isSet(Flag.ANSWERED) ? 1 : 0);
-                                cv.put("forwarded", message.isSet(Flag.FORWARDED) ? 1 : 0);
-                                cv.put("folder_id", mFolderId);
-                                cv.put("to_list", Address.pack(message.getRecipients(RecipientType.TO)));
-                                cv.put("cc_list", Address.pack(message.getRecipients(RecipientType.CC)));
-                                cv.put("bcc_list", Address.pack(message.getRecipients(RecipientType.BCC)));
-                                cv.put("html_content", html.length() > 0 ? html : null);
-                                cv.put("text_content", text.length() > 0 ? text : null);
-                                cv.put("preview", preview.length() > 0 ? preview : null);
-                                cv.put("reply_to_list", Address.pack(message.getReplyTo()));
-                                cv.put("attachment_count", attachments.size());
-                                cv.put("internal_date",  message.getInternalDate() == null
-                                       ? System.currentTimeMillis() : message.getInternalDate().getTime());
-                                cv.put("mime_type", message.getMimeType());
-                                cv.put("empty", 0);
-
-                                String messageId = message.getMessageId();
-                                if (messageId != null) {
-                                    cv.put("message_id", messageId);
-                                }
-
-                                long msgId;
-
-                                if (oldMessageId == -1) {
-                                    msgId = db.insert("messages", "uid", cv);
-
-                                    // Create entry in 'threads' table
-                                    cv.clear();
-                                    cv.put("message_id", msgId);
-
-                                    if (rootId != -1) {
-                                        cv.put("root", rootId);
-                                    }
-                                    if (parentId != -1) {
-                                        cv.put("parent", parentId);
-                                    }
-
-                                    db.insert("threads", null, cv);
-                                } else {
-                                    db.update("messages", cv, "id = ?", new String[] { Long.toString(oldMessageId) });
-                                    msgId = oldMessageId;
-                                }
-
-                                for (Part attachment : attachments) {
-                                    saveAttachment(msgId, attachment, copy);
-                                }
-                                saveHeaders(msgId, (MimeMessage)message);
-                            } catch (Exception e) {
-                                throw new MessagingException("Error appending message", e);
-                            }
                         }
                     } catch (MessagingException e) {
                         throw new WrappedException(e);
@@ -1376,7 +1199,212 @@ public class LocalFolder extends Folder<LocalMessage> implements Serializable {
 
             return uidMap;
         } catch (WrappedException e) {
-            throw(MessagingException) e.getCause();
+            throw (MessagingException) e.getCause();
+        }
+    }
+
+    protected void saveMessage(SQLiteDatabase db, Message message, boolean copy, Map<String, String> uidMap)
+            throws MessagingException {
+        if (!(message instanceof MimeMessage)) {
+            throw new Error("LocalStore can only store Messages that extend MimeMessage");
+        }
+
+        long oldMessageId = -1;
+        String uid = message.getUid();
+        boolean shouldCreateNewMessage = uid == null || copy;
+        if (shouldCreateNewMessage) {
+            String randomLocalUid = K9.LOCAL_UID_PREFIX + UUID.randomUUID().toString();
+
+            if (copy) {
+                // Save mapping: source UID -> target UID
+                uidMap.put(uid, randomLocalUid);
+            } else {
+                // Modify the Message instance to reference the new UID
+                message.setUid(randomLocalUid);
+            }
+
+            // The message will be saved with the newly generated UID
+            uid = randomLocalUid;
+        } else {
+            LocalMessage oldMessage = getMessage(uid);
+
+            if (oldMessage != null) {
+                oldMessageId = oldMessage.getId();
+            }
+
+            //FIXME
+            deleteAttachments(message.getUid());
+        }
+
+        long rootId = -1;
+        long parentId = -1;
+
+        if (oldMessageId == -1) {
+            // This is a new message. Do the message threading.
+            ThreadInfo threadInfo = doMessageThreading(db, message);
+            oldMessageId = threadInfo.msgId;
+            rootId = threadInfo.rootId;
+            parentId = threadInfo.parentId;
+        }
+
+        //TODO: construct message preview
+        //TODO: get attachment count
+
+        try {
+            long rootMessagePartId = saveMessageParts(db, message);
+
+            ContentValues cv = new ContentValues();
+            cv.put("message_part_id", rootMessagePartId);
+            cv.put("uid", uid);
+            cv.put("subject", message.getSubject());
+            cv.put("sender_list", Address.pack(message.getFrom()));
+            cv.put("date", message.getSentDate() == null
+                    ? System.currentTimeMillis() : message.getSentDate().getTime());
+            cv.put("flags", this.localStore.serializeFlags(message.getFlags()));
+            cv.put("deleted", message.isSet(Flag.DELETED) ? 1 : 0);
+            cv.put("read", message.isSet(Flag.SEEN) ? 1 : 0);
+            cv.put("flagged", message.isSet(Flag.FLAGGED) ? 1 : 0);
+            cv.put("answered", message.isSet(Flag.ANSWERED) ? 1 : 0);
+            cv.put("forwarded", message.isSet(Flag.FORWARDED) ? 1 : 0);
+            cv.put("folder_id", mFolderId);
+            cv.put("to_list", Address.pack(message.getRecipients(RecipientType.TO)));
+            cv.put("cc_list", Address.pack(message.getRecipients(RecipientType.CC)));
+            cv.put("bcc_list", Address.pack(message.getRecipients(RecipientType.BCC)));
+            cv.put("preview", "");  //FIXME
+            cv.put("reply_to_list", Address.pack(message.getReplyTo()));
+            cv.put("attachment_count", 0);  //FIXME
+            cv.put("internal_date", message.getInternalDate() == null
+                    ? System.currentTimeMillis() : message.getInternalDate().getTime());
+            cv.put("mime_type", message.getMimeType());
+            cv.put("empty", 0);
+
+            String messageId = message.getMessageId();
+            if (messageId != null) {
+                cv.put("message_id", messageId);
+            }
+
+            if (oldMessageId == -1) {
+                long msgId = db.insert("messages", "uid", cv);
+
+                // Create entry in 'threads' table
+                cv.clear();
+                cv.put("message_id", msgId);
+
+                if (rootId != -1) {
+                    cv.put("root", rootId);
+                }
+                if (parentId != -1) {
+                    cv.put("parent", parentId);
+                }
+
+                db.insert("threads", null, cv);
+            } else {
+                db.update("messages", cv, "id = ?", new String[] { Long.toString(oldMessageId) });
+            }
+        } catch (Exception e) {
+            throw new MessagingException("Error appending message", e);
+        }
+    }
+
+    private long saveMessageParts(SQLiteDatabase db, Message message) throws IOException, MessagingException {
+        long rootMessagePartId = saveMessagePart(db, new PartContainer(-1, message), -1, 0);
+
+        Stack<PartContainer> partsToSave = new Stack<PartContainer>();
+        addChildrenToStack(partsToSave, message, rootMessagePartId);
+
+        int order = 1;
+        while (!partsToSave.isEmpty()) {
+            PartContainer partContainer = partsToSave.pop();
+            long messagePartId = saveMessagePart(db, partContainer, rootMessagePartId, order);
+            order++;
+
+            addChildrenToStack(partsToSave, partContainer.part, messagePartId);
+        }
+
+        return rootMessagePartId;
+    }
+
+    private long saveMessagePart(SQLiteDatabase db, PartContainer partContainer, long rootMessagePartId, int order)
+            throws IOException, MessagingException {
+
+        Part part = partContainer.part;
+
+        byte[] headerBytes = getHeaderBytes(part);
+
+        ContentValues cv = new ContentValues();
+        if (rootMessagePartId != -1) {
+            cv.put("root", rootMessagePartId);
+        }
+        cv.put("parent", partContainer.parent);
+        cv.put("seq", order);
+        cv.put("mime_type", part.getMimeType());
+        cv.put("header", headerBytes);
+        cv.put("type", MessagePartType.UNKNOWN);
+
+        Body body = part.getBody();
+        if (body instanceof Multipart) {
+            cv.put("data_location", DataLocation.IN_DATABASE);
+
+            Multipart multipart = (Multipart) body;
+            cv.put("preamble", multipart.getPreamble());
+            cv.put("epilogue", multipart.getEpilogue());
+            cv.put("boundary", multipart.getBoundary());
+        } else if (body == null) {
+            //TODO: deal with missing parts
+            cv.put("data_location", DataLocation.MISSING);
+        } else {
+            cv.put("data_location", DataLocation.IN_DATABASE);
+
+            byte[] bodyData = getBodyBytes(body);
+            String encoding = getTransferEncoding(part);
+
+            cv.put("encoding", encoding);
+            cv.put("data", bodyData);
+            cv.put("content_id", part.getContentId());
+        }
+
+        return db.insertOrThrow("message_parts", null, cv);
+    }
+
+    private byte[] getHeaderBytes(Part part) throws IOException, MessagingException {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        part.writeHeaderTo(output);
+        return output.toByteArray();
+    }
+
+    private byte[] getBodyBytes(Body body) throws IOException, MessagingException {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        body.writeTo(output);
+        return output.toByteArray();
+    }
+
+    private String getTransferEncoding(Part part) throws MessagingException {
+        String[] contentTransferEncoding = part.getHeader(MimeHeader.HEADER_CONTENT_TRANSFER_ENCODING);
+        if (contentTransferEncoding != null && contentTransferEncoding.length > 0) {
+            return contentTransferEncoding[0].toLowerCase(Locale.US);
+        }
+
+        return MimeUtil.ENC_7BIT;
+    }
+
+    private void addChildrenToStack(Stack<PartContainer> stack, Part part, long parentMessageId) {
+        Body body = part.getBody();
+        if (body instanceof Multipart) {
+            Multipart multipart = (Multipart) body;
+            for (int i = multipart.getCount() - 1; i >= 0; i--) {
+                BodyPart childPart = multipart.getBodyPart(i);
+                stack.push(new PartContainer(parentMessageId, childPart));
+            }
+        }
+    }
+
+    private static class PartContainer {
+        public final long parent;
+        public final Part part;
+
+        PartContainer(long parent, Part part) {
+            this.parent = parent;
+            this.part = part;
         }
     }
 
@@ -2191,4 +2219,21 @@ public class LocalFolder extends Folder<LocalMessage> implements Serializable {
     private Account getAccount() {
         return localStore.getAccount();
     }
+
+    // Note: The contents of the 'message_parts' table depend on these values.
+    private static class MessagePartType {
+        static final int UNKNOWN = 0;
+        static final int ALTERNATIVE_PLAIN = 1;
+        static final int ALTERNATIVE_HTML = 2;
+        static final int TEXT = 3;
+        static final int RELATED = 4;
+        static final int ATTACHMENT = 5;
+    }
+
+    // Note: The contents of the 'message_parts' table depend on these values.
+    private static class DataLocation {
+        static final int MISSING = 0;
+        static final int IN_DATABASE = 1;
+        static final int ON_DISK = 2;
+    }
 }

k9mail/src/main/java/com/fsck/k9/mailstore/LocalMessage.java

@@ -2,9 +2,7 @@ package com.fsck.k9.mailstore;
 
 import java.io.IOException;
 import java.io.OutputStream;
-import java.util.ArrayList;
 import java.util.Date;
-import java.util.List;
 import java.util.Set;
 
 import android.content.ContentValues;
@@ -40,6 +38,8 @@ public class LocalMessage extends MimeMessage {
 
     private long mThreadId;
     private long mRootId;
+    private long messagePartId;
+    private String mimeType;
 
     private LocalMessage(LocalStore localStore) {
         this.localStore = localStore;
@@ -110,6 +110,18 @@ public class LocalMessage extends MimeMessage {
         setFlagInternal(Flag.FLAGGED, flagged);
         setFlagInternal(Flag.ANSWERED, answered);
         setFlagInternal(Flag.FORWARDED, forwarded);
+
+        messagePartId = cursor.getLong(22);
+        mimeType = cursor.getString(23);
+    }
+
+    long getMessagePartId() {
+        return messagePartId;
+    }
+
+    @Override
+    public String getMimeType() {
+        return mimeType;
     }
 
     /**
@@ -477,23 +489,8 @@ public class LocalMessage extends MimeMessage {
     }
 
     private void loadHeaders() throws MessagingException {
-        List<LocalMessage> messages = new ArrayList<LocalMessage>();
+        mHeadersLoaded = true;
-        messages.add(this);
+        getFolder().populateHeaders(this);
-        mHeadersLoaded = true; // set true before calling populate headers to stop recursion
-        getFolder().populateHeaders(messages);
-
-    }
-
-    @Override
-    public void addHeader(String name, String value) throws MessagingException {
-        if (!mHeadersLoaded)
-            loadHeaders();
-        super.addHeader(name, value);
-    }
-
-    @Override
-    public void addRawHeader(String name, String raw) {
-        throw new RuntimeException("Not supported");
     }
 
     @Override

k9mail/src/main/java/com/fsck/k9/mailstore/LocalStore.java

@@ -76,7 +76,7 @@ public class LocalStore extends Store implements Serializable {
         "subject, sender_list, date, uid, flags, messages.id, to_list, cc_list, " +
         "bcc_list, reply_to_list, attachment_count, internal_date, messages.message_id, " +
         "folder_id, preview, threads.id, threads.root, deleted, read, flagged, answered, " +
-        "forwarded ";
+        "forwarded, message_part_id, mime_type ";
 
     static final String GET_FOLDER_COLS =
         "folders.id, name, visible_limit, last_updated, status, push_state, last_pushed, " +
@@ -119,7 +119,7 @@ public class LocalStore extends Store implements Serializable {
      */
     private static final int THREAD_FLAG_UPDATE_BATCH_SIZE = 500;
 
-    public static final int DB_VERSION = 50;
+    public static final int DB_VERSION = 51;
 
 
     public static String getColumnNameForFlag(Flag flag) {

k9mail/src/main/java/com/fsck/k9/mailstore/StoreSchemaDefinition.java

@@ -83,8 +83,6 @@ class StoreSchemaDefinition implements LockableDatabase.SchemaDefinition {
                         "cc_list TEXT, " +
                         "bcc_list TEXT, " +
                         "reply_to_list TEXT, " +
-                        "html_content TEXT, " +
-                        "text_content TEXT, " +
                         "attachment_count INTEGER, " +
                         "internal_date INTEGER, " +
                         "message_id TEXT, " +
@@ -95,12 +93,36 @@ class StoreSchemaDefinition implements LockableDatabase.SchemaDefinition {
                         "read INTEGER default 0, " +
                         "flagged INTEGER default 0, " +
                         "answered INTEGER default 0, " +
-                        "forwarded INTEGER default 0" +
+                        "forwarded INTEGER default 0, " +
+                        "message_part_id INTEGER" +
                         ")");
 
-                db.execSQL("DROP TABLE IF EXISTS headers");
+                db.execSQL("CREATE TABLE message_parts (" +
-                db.execSQL("CREATE TABLE headers (id INTEGER PRIMARY KEY, message_id INTEGER, name TEXT, value TEXT)");
+                        "id INTEGER PRIMARY KEY, " +
-                db.execSQL("CREATE INDEX IF NOT EXISTS header_folder ON headers (message_id)");
+                        "type INTEGER NOT NULL, " +
+                        "root INTEGER, " +
+                        "parent INTEGER NOT NULL, " +
+                        "seq INTEGER NOT NULL, " +
+                        "mime_type TEXT, " +
+                        "decoded_body_size INTEGER, " +
+                        "display_name TEXT, " +
+                        "header TEXT, " +
+                        "encoding TEXT, " +
+                        "charset TEXT, " +
+                        "data_location INTEGER NOT NULL, " +
+                        "data TEXT, " +
+                        "preamble TEXT, " +
+                        "epilogue TEXT, " +
+                        "boundary TEXT, " +
+                        "content_id TEXT, " +
+                        "server_extra TEXT" +
+                        ")");
+
+                db.execSQL("CREATE TRIGGER set_message_part_root " +
+                        "AFTER INSERT ON message_parts " +
+                        "BEGIN " +
+                        "UPDATE message_parts SET root=id WHERE root IS NULL AND ROWID = NEW.ROWID; " +
+                        "END");
 
                 db.execSQL("CREATE INDEX IF NOT EXISTS msg_uid ON messages (uid, folder_id)");
                 db.execSQL("DROP INDEX IF EXISTS msg_folder_id");
@@ -541,6 +563,9 @@ class StoreSchemaDefinition implements LockableDatabase.SchemaDefinition {
                     db.update("folders", cv, "name = ?",
                             new String[] { this.localStore.getAccount().getInboxFolderName() });
                 }
+                if (db.getVersion() < 51) {
+                    throw new IllegalStateException("Database upgrade not supported yet!");
+                }
             }
 
             db.setVersion(LocalStore.DB_VERSION);