Generally replace \n with \r\n when part of a message

This builds upon the efforts started 2 commits back where \r\n is used for
all message text and \n is only used when the text is inside an
EolConvertingEditText widget.

src/com/fsck/k9/activity/MessageCompose.java

@@ -1394,10 +1394,10 @@ public class MessageCompose extends K9Activity implements OnClickListener,
             if (includeQuotedText) {
                 String quotedText = mQuotedText.getCharacters();
                 if (replyAfterQuote) {
-                    composedMessageOffset = quotedText.length() + "\n".length();
+                    composedMessageOffset = quotedText.length() + "\r\n".length();
-                    text = quotedText + "\n" + text;
+                    text = quotedText + "\r\n" + text;
                 } else {
-                    text += "\n\n" + quotedText.toString();
+                    text += "\r\n\r\n" + quotedText.toString();
                 }
             }
 
@@ -1541,7 +1541,7 @@ public class MessageCompose extends K9Activity implements OnClickListener,
              * header value (all parameters at once) will be encoded by
              * MimeHeader.writeTo().
              */
-            bp.addHeader(MimeHeader.HEADER_CONTENT_TYPE, String.format("%s;\n name=\"%s\"",
+            bp.addHeader(MimeHeader.HEADER_CONTENT_TYPE, String.format("%s;\r\n name=\"%s\"",
                          contentType,
                          EncoderUtil.encodeIfNecessary(attachment.name,
                                  EncoderUtil.Usage.WORD_ENTITY, 7)));
@@ -1564,7 +1564,7 @@ public class MessageCompose extends K9Activity implements OnClickListener,
              *  title*3="isn't it!"
              */
             bp.addHeader(MimeHeader.HEADER_CONTENT_DISPOSITION, String.format(
-                             "attachment;\n filename=\"%s\";\n size=%d",
+                             "attachment;\r\n filename=\"%s\";\r\n size=%d",
                              attachment.name, attachment.size));
 
             mp.addBodyPart(bp);
@@ -1770,7 +1770,7 @@ public class MessageCompose extends K9Activity implements OnClickListener,
             String signature = mSignatureView.getCharacters();
 
             if (signature != null && !signature.contentEquals("")) {
-                text += "\n" + signature;
+                text += "\r\n" + signature;
             }
         }
 
@@ -1786,7 +1786,7 @@ public class MessageCompose extends K9Activity implements OnClickListener,
         if (mIdentity.getSignatureUse()) {
             signature = mSignatureView.getCharacters();
             if(!StringUtils.isNullOrEmpty(signature)) {
-                signature = HtmlConverter.textToHtmlFragment("\n" + signature);
+                signature = HtmlConverter.textToHtmlFragment("\r\n" + signature);
             }
         }
         return signature;
@@ -3071,7 +3071,7 @@ public class MessageCompose extends K9Activity implements OnClickListener,
         showOrHideQuotedText(quotedMode);
     }
 
-    /*
+    /**
      * Pull out the parts of the now loaded source message and apply them to the new message
      * depending on the type of message being composed.
      * @param message Source message
@@ -3096,13 +3096,13 @@ public class MessageCompose extends K9Activity implements OnClickListener,
 
                 // Regenerate the quoted text without our user content in it nor added newlines.
                 StringBuilder quotedText = new StringBuilder();
-                if (bodyOffset == 0 && text.substring(bodyLength, bodyLength + 2).equals("\n\n")) {
+                if (bodyOffset == 0 && text.substring(bodyLength, bodyLength + 4).equals("\r\n\r\n")) {
                     // top-posting: ignore two newlines at start of quote
-                    quotedText.append(text.substring(bodyLength + 2));
+                    quotedText.append(text.substring(bodyLength + 4));
                 } else if (bodyOffset + bodyLength == text.length() &&
-                        text.substring(bodyOffset - 1, bodyOffset).equals("\n")) {
+                        text.substring(bodyOffset - 2, bodyOffset).equals("\r\n")) {
                     // bottom-posting: ignore newline at end of quote
-                    quotedText.append(text.substring(0, bodyOffset - 1));
+                    quotedText.append(text.substring(0, bodyOffset - 2));
                 } else {
                     quotedText.append(text.substring(0, bodyOffset));   // stuff before the reply
                     quotedText.append(text.substring(bodyOffset + bodyLength));
@@ -3322,7 +3322,7 @@ public class MessageCompose extends K9Activity implements OnClickListener,
     private static final int FIND_INSERTION_POINT_FIRST_GROUP = 1;
     // HTML bits to insert as appropriate
     // TODO is it safe to assume utf-8 here?
-    private static final String FIND_INSERTION_POINT_HTML_CONTENT = "<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.0 Transitional//EN\">\n<html>";
+    private static final String FIND_INSERTION_POINT_HTML_CONTENT = "<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.0 Transitional//EN\">\r\n<html>";
     private static final String FIND_INSERTION_POINT_HTML_END_CONTENT = "</html>";
     private static final String FIND_INSERTION_POINT_HEAD_CONTENT = "<head><meta content=\"text/html; charset=utf-8\" http-equiv=\"Content-Type\"></head>";
     // Index of the start of the beginning of a String.
@@ -3715,24 +3715,24 @@ public class MessageCompose extends K9Activity implements OnClickListener,
             return quotedText.toString().replaceAll("\\\r", "");
         } else if (quoteStyle == QuoteStyle.HEADER) {
             StringBuilder quotedText = new StringBuilder(body.length() + QUOTE_BUFFER_LENGTH);
-            quotedText.append("\n");
+            quotedText.append("\r\n");
-            quotedText.append(getString(R.string.message_compose_quote_header_separator)).append("\n");
+            quotedText.append(getString(R.string.message_compose_quote_header_separator)).append("\r\n");
             if (originalMessage.getFrom() != null && Address.toString(originalMessage.getFrom()).length() != 0) {
-                quotedText.append(getString(R.string.message_compose_quote_header_from)).append(" ").append(Address.toString(originalMessage.getFrom())).append("\n");
+                quotedText.append(getString(R.string.message_compose_quote_header_from)).append(" ").append(Address.toString(originalMessage.getFrom())).append("\r\n");
             }
             if (originalMessage.getSentDate() != null) {
-                quotedText.append(getString(R.string.message_compose_quote_header_send_date)).append(" ").append(originalMessage.getSentDate()).append("\n");
+                quotedText.append(getString(R.string.message_compose_quote_header_send_date)).append(" ").append(originalMessage.getSentDate()).append("\r\n");
             }
             if (originalMessage.getRecipients(RecipientType.TO) != null && originalMessage.getRecipients(RecipientType.TO).length != 0) {
-                quotedText.append(getString(R.string.message_compose_quote_header_to)).append(" ").append(Address.toString(originalMessage.getRecipients(RecipientType.TO))).append("\n");
+                quotedText.append(getString(R.string.message_compose_quote_header_to)).append(" ").append(Address.toString(originalMessage.getRecipients(RecipientType.TO))).append("\r\n");
             }
             if (originalMessage.getRecipients(RecipientType.CC) != null && originalMessage.getRecipients(RecipientType.CC).length != 0) {
-                quotedText.append(getString(R.string.message_compose_quote_header_cc)).append(" ").append(Address.toString(originalMessage.getRecipients(RecipientType.CC))).append("\n");
+                quotedText.append(getString(R.string.message_compose_quote_header_cc)).append(" ").append(Address.toString(originalMessage.getRecipients(RecipientType.CC))).append("\r\n");
             }
             if (originalMessage.getSubject() != null) {
-                quotedText.append(getString(R.string.message_compose_quote_header_subject)).append(" ").append(originalMessage.getSubject()).append("\n");
+                quotedText.append(getString(R.string.message_compose_quote_header_subject)).append(" ").append(originalMessage.getSubject()).append("\r\n");
             }
-            quotedText.append("\n");
+            quotedText.append("\r\n");
 
             quotedText.append(body);
 
@@ -3763,7 +3763,7 @@ public class MessageCompose extends K9Activity implements OnClickListener,
                               Address.toString(originalMessage.getFrom()))
                                                           ));
             header.append("<blockquote class=\"gmail_quote\" " +
-                          "style=\"margin: 0pt 0pt 0pt 0.8ex; border-left: 1px solid rgb(204, 204, 204); padding-left: 1ex;\">\n");
+                          "style=\"margin: 0pt 0pt 0pt 0.8ex; border-left: 1px solid rgb(204, 204, 204); padding-left: 1ex;\">\r\n");
 
             String footer = "</blockquote></div>";
 
@@ -3772,35 +3772,35 @@ public class MessageCompose extends K9Activity implements OnClickListener,
         } else if (quoteStyle == QuoteStyle.HEADER) {
 
             StringBuilder header = new StringBuilder();
-            header.append("<div style='font-size:10.0pt;font-family:\"Tahoma\",\"sans-serif\";padding:3.0pt 0in 0in 0in'>\n");
+            header.append("<div style='font-size:10.0pt;font-family:\"Tahoma\",\"sans-serif\";padding:3.0pt 0in 0in 0in'>\r\n");
-            header.append("<hr style='border:none;border-top:solid #E1E1E1 1.0pt'>\n"); // This gets converted into a horizontal line during html to text conversion.
+            header.append("<hr style='border:none;border-top:solid #E1E1E1 1.0pt'>\r\n"); // This gets converted into a horizontal line during html to text conversion.
             if (mSourceMessage.getFrom() != null && Address.toString(mSourceMessage.getFrom()).length() != 0) {
                 header.append("<b>").append(getString(R.string.message_compose_quote_header_from)).append("</b> ")
                     .append(HtmlConverter.textToHtmlFragment(Address.toString(mSourceMessage.getFrom())))
-                    .append("<br>\n");
+                    .append("<br>\r\n");
             }
             if (mSourceMessage.getSentDate() != null) {
                 header.append("<b>").append(getString(R.string.message_compose_quote_header_send_date)).append("</b> ")
                     .append(mSourceMessage.getSentDate())
-                    .append("<br>\n");
+                    .append("<br>\r\n");
             }
             if (mSourceMessage.getRecipients(RecipientType.TO) != null && mSourceMessage.getRecipients(RecipientType.TO).length != 0) {
                 header.append("<b>").append(getString(R.string.message_compose_quote_header_to)).append("</b> ")
                     .append(HtmlConverter.textToHtmlFragment(Address.toString(mSourceMessage.getRecipients(RecipientType.TO))))
-                    .append("<br>\n");
+                    .append("<br>\r\n");
             }
             if (mSourceMessage.getRecipients(RecipientType.CC) != null && mSourceMessage.getRecipients(RecipientType.CC).length != 0) {
                 header.append("<b>").append(getString(R.string.message_compose_quote_header_cc)).append("</b> ")
                     .append(HtmlConverter.textToHtmlFragment(Address.toString(mSourceMessage.getRecipients(RecipientType.CC))))
-                    .append("<br>\n");
+                    .append("<br>\r\n");
             }
             if (mSourceMessage.getSubject() != null) {
                 header.append("<b>").append(getString(R.string.message_compose_quote_header_subject)).append("</b> ")
                     .append(HtmlConverter.textToHtmlFragment(mSourceMessage.getSubject()))
-                    .append("<br>\n");
+                    .append("<br>\r\n");
             }
-            header.append("</div>\n");
+            header.append("</div>\r\n");
-            header.append("<br>\n");
+            header.append("<br>\r\n");
 
             insertable.insertIntoQuotedHeader(header.toString());
         }

src/com/fsck/k9/helper/HtmlConverter.java

@@ -70,7 +70,7 @@ public class HtmlConverter {
             if (tag.equals("hr") && opening) {
                 // In the case of an <hr>, replace it with a bunch of underscores. This is roughly
                 // the behaviour of Outlook in Rich Text mode.
-                output.append("_____________________________________________\n");
+                output.append("_____________________________________________\r\n");
             } else if (TAGS_WITH_IGNORED_CONTENT.contains(tag)) {
                 handleIgnoredTag(opening, output);
             }
@@ -282,11 +282,16 @@ public class HtmlConverter {
         // Replace lines of -,= or _ with horizontal rules
         text = text.replaceAll("\\s*([-=_]{30,}+)\\s*", "<hr />");
 
-        // TODO: reverse engineer (or troll history) and document
+        /*
+         * Unwrap multi-line paragraphs into single line paragraphs that are
+         * wrapped when displayed. But try to avoid unwrapping consecutive lines
+         * of text that are not paragraphs, such as lists of system log entries
+         * or long URLs that are on their own line.
+         */
         text = text.replaceAll("(?m)^([^\r\n]{4,}[\\s\\w,:;+/])(?:\r\n|\n|\r)(?=[a-z]\\S{0,10}[\\s\\n\\r])", "$1 ");
 
         // Compress four or more newlines down to two newlines
-        text = text.replaceAll("(?m)(\r\n|\n|\r){4,}", "\n\n");
+        text = text.replaceAll("(?m)(\r\n|\n|\r){4,}", "\r\n\r\n");
 
         StringBuffer sb = new StringBuffer(text.length() + TEXT_TO_HTML_EXTRA_BUFFER_LENGTH);
 
@@ -1315,7 +1320,7 @@ public class HtmlConverter {
         //
         // For some reason, TextUtils.htmlEncode escapes ' into &apos;, which is technically part of the XHTML 1.0
         // standard, but Gmail doesn't recognize it as an HTML entity. We unescape that here.
-        return linkified.toString().replace("\n", "<br>\n").replace("&apos;", "&#39;");
+        return linkified.toString().replaceAll("\r?\n", "<br>\r\n").replace("&apos;", "&#39;");
     }
 
     /**
@@ -1345,10 +1350,10 @@ public class HtmlConverter {
                         lastChar = output.charAt(output.length() - 1);
                     }
                     if (lastChar != '\n') {
-                        output.append("\n");
+                        output.append("\r\n");
                     }
                 } else {
-                    output.append("\n");
+                    output.append("\r\n");
                 }
             }
 
@@ -1356,7 +1361,7 @@ public class HtmlConverter {
                 if (opening) {
                     output.append("\t•  ");
                 } else {
-                    output.append("\n");
+                    output.append("\r\n");
                 }
             }
         }

src/com/fsck/k9/helper/Utility.java

@@ -282,7 +282,7 @@ public class Utility {
         StringBuilder result = new StringBuilder();
         for (String piece : str.split(NEWLINE_REGEX)) {
             result.append(wrap(piece, wrapLength, null, false));
-            result.append("\n");
+            result.append("\r\n");
         }
         return result.toString();
     }
@@ -330,7 +330,7 @@ public class Utility {
             return null;
         }
         if (newLineStr == null) {
-            newLineStr = "\n";
+            newLineStr = "\r\n";
         }
         if (wrapLength < 1) {
             wrapLength = 1;

src/com/fsck/k9/mail/internet/MimeUtility.java

@@ -1732,7 +1732,7 @@ public class MimeUtility {
             text.append(context.getString(R.string.message_compose_quote_header_from));
             text.append(' ');
             text.append(Address.toString(from));
-            text.append("\n");
+            text.append("\r\n");
         }
 
         // To: <recipients>
@@ -1741,7 +1741,7 @@ public class MimeUtility {
             text.append(context.getString(R.string.message_compose_quote_header_to));
             text.append(' ');
             text.append(Address.toString(to));
-            text.append("\n");
+            text.append("\r\n");
         }
 
         // Cc: <recipients>
@@ -1750,7 +1750,7 @@ public class MimeUtility {
             text.append(context.getString(R.string.message_compose_quote_header_cc));
             text.append(' ');
             text.append(Address.toString(cc));
-            text.append("\n");
+            text.append("\r\n");
         }
 
         // Date: <date>
@@ -1759,7 +1759,7 @@ public class MimeUtility {
             text.append(context.getString(R.string.message_compose_quote_header_send_date));
             text.append(' ');
             text.append(date.toString());
-            text.append("\n");
+            text.append("\r\n");
         }
 
         // Subject: <subject>
@@ -1771,7 +1771,7 @@ public class MimeUtility {
         } else {
             text.append(subject);
         }
-        text.append("\n\n");
+        text.append("\r\n\r\n");
     }
 
     /**
@@ -1919,7 +1919,7 @@ public class MimeUtility {
         if (prependDivider) {
             String filename = getPartName(part);
 
-            text.append("\n\n");
+            text.append("\r\n\r\n");
             int len = filename.length();
             if (len > 0) {
                 if (len > TEXT_DIVIDER_LENGTH - FILENAME_PREFIX_LENGTH - FILENAME_SUFFIX_LENGTH) {
@@ -1934,7 +1934,7 @@ public class MimeUtility {
             } else {
                 text.append(TEXT_DIVIDER);
             }
-            text.append("\n\n");
+            text.append("\r\n\r\n");
         }
     }
 

src/com/fsck/k9/mail/store/ImapStore.java

@@ -1917,7 +1917,7 @@ public class ImapStore extends Store {
                      * of them.
                      */
                     for (int i = 0, count = bodyParams.size(); i < count; i += 2) {
-                        contentType.append(String.format(";\n %s=\"%s\"",
+                        contentType.append(String.format(";\r\n %s=\"%s\"",
                                            bodyParams.getString(i),
                                            bodyParams.getString(i + 1)));
                     }
@@ -1952,7 +1952,7 @@ public class ImapStore extends Store {
                          * about the attachment out.
                          */
                         for (int i = 0, count = bodyDispositionParams.size(); i < count; i += 2) {
-                            contentDisposition.append(String.format(";\n %s=\"%s\"",
+                            contentDisposition.append(String.format(";\r\n %s=\"%s\"",
                                                       bodyDispositionParams.getString(i).toLowerCase(Locale.US),
                                                       bodyDispositionParams.getString(i + 1)));
                         }
@@ -1960,7 +1960,7 @@ public class ImapStore extends Store {
                 }
 
                 if (MimeUtility.getHeaderParameter(contentDisposition.toString(), "size") == null) {
-                    contentDisposition.append(String.format(";\n size=%d", size));
+                    contentDisposition.append(String.format(";\r\n size=%d", size));
                 }
 
                 /*

src/com/fsck/k9/mail/store/LocalStore.java

@@ -1964,18 +1964,18 @@ public class LocalStore extends Store implements Serializable {
                                             bp.setEncoding(encoding);
                                             if (name != null) {
                                                 bp.setHeader(MimeHeader.HEADER_CONTENT_TYPE,
-                                                             String.format("%s;\n name=\"%s\"",
+                                                             String.format("%s;\r\n name=\"%s\"",
                                                                            type,
                                                                            name));
                                                 bp.setHeader(MimeHeader.HEADER_CONTENT_DISPOSITION,
-                                                             String.format("%s;\n filename=\"%s\";\n size=%d",
+                                                             String.format("%s;\r\n filename=\"%s\";\r\n size=%d",
                                                                            contentDisposition,
                                                                            name, // TODO: Should use encoded word defined in RFC 2231.
                                                                            size));
                                             } else {
                                                 bp.setHeader(MimeHeader.HEADER_CONTENT_TYPE, type);
                                                 bp.setHeader(MimeHeader.HEADER_CONTENT_DISPOSITION,
-                                                        String.format("%s;\n size=%d",
+                                                        String.format("%s;\r\n size=%d",
                                                                       contentDisposition,
                                                                       size));
                                             }

tests/src/com/fsck/k9/helper/HtmlConverterTest.java

@@ -12,16 +12,16 @@ public class HtmlConverterTest extends TestCase {
     private static final String OUTPUT_FILE = "C:/temp/parse.html";
 
     public void testTextQuoteToHtmlBlockquote() {
-        String message = "Panama!\n" +
+        String message = "Panama!\r\n" +
-            "\n" +
+            "\r\n" +
-            "Bob Barker <bob@aol.com> wrote:\n" +
+            "Bob Barker <bob@aol.com> wrote:\r\n" +
-            "> a canal\n" +
+            "> a canal\r\n" +
-            ">\n" +
+            ">\r\n" +
-            "> Dorothy Jo Gideon <dorothy@aol.com> espoused:\n" +
+            "> Dorothy Jo Gideon <dorothy@aol.com> espoused:\r\n" +
-            "> >A man, a plan...\n" +
+            "> >A man, a plan...\r\n" +
-            "> Too easy!\n" +
+            "> Too easy!\r\n" +
-            "\n" +
+            "\r\n" +
-            "Nice job :)\n" +
+            "Nice job :)\r\n" +
             ">> Guess!";
         String result = HtmlConverter.textToHtml(message);
         writeToFile(result);
@@ -49,13 +49,13 @@ public class HtmlConverterTest extends TestCase {
     }
 
     public void testTextQuoteToHtmlBlockquoteIndented() {
-        String message = "*facepalm*\n" +
+        String message = "*facepalm*\r\n" +
-            "\n" +
+            "\r\n" +
-            "Bob Barker <bob@aol.com> wrote:\n" +
+            "Bob Barker <bob@aol.com> wrote:\r\n" +
-            "> A wise man once said...\n" +
+            "> A wise man once said...\r\n" +
-            ">\n" +
+            ">\r\n" +
-            ">     LOL F1RST!!!!!\n" +
+            ">     LOL F1RST!!!!!\r\n" +
-            ">\n" +
+            ">\r\n" +
             "> :)";
         String result = HtmlConverter.textToHtml(message);
         writeToFile(result);
@@ -84,12 +84,12 @@ public class HtmlConverterTest extends TestCase {
         assertEquals(HtmlConverter.getQuoteColor(0), HtmlConverter.QUOTE_COLOR_DEFAULT);
         assertEquals(HtmlConverter.getQuoteColor(6), HtmlConverter.QUOTE_COLOR_DEFAULT);
 
-        String message = "zero\n" +
+        String message = "zero\r\n" +
-            "> one\n" +
+            "> one\r\n" +
-            ">> two\n" +
+            ">> two\r\n" +
-            ">>> three\n" +
+            ">>> three\r\n" +
-            ">>>> four\n" +
+            ">>>> four\r\n" +
-            ">>>>> five\n" +
+            ">>>>> five\r\n" +
             ">>>>>> six";
         String result = HtmlConverter.textToHtml(message);
         writeToFile(result);
@@ -136,9 +136,9 @@ public class HtmlConverterTest extends TestCase {
     }
 
     public void testPreserveSpacesAtFirst() {
-        String message = "foo\n"
+        String message = "foo\r\n"
-                + " bar\n"
+                + " bar\r\n"
-                + "  baz\n";
+                + "  baz\r\n";
         String result = HtmlConverter.textToHtml(message);
         writeToFile(result);
         assertEquals("<pre class=\"k9mail\">"
@@ -150,11 +150,11 @@ public class HtmlConverterTest extends TestCase {
 
     public void testPreserveSpacesAtFirstForSpecialCharacters() {
         String message =
-                  " \n"
+                  " \r\n"
-                + "  &\n"
+                + "  &\r\n"
-                + "    \r\n"
+                + "    \n"
-                + "   <\n"
+                + "   <\r\n"
-                + "  > \n";
+                + "  > \r\n";
         String result = HtmlConverter.textToHtml(message);
         writeToFile(result);
         assertEquals("<pre class=\"k9mail\">"

tests/src/com/fsck/k9/mail/internet/ViewablesTest.java

@@ -81,8 +81,8 @@ public class ViewablesTest extends AndroidTestCase {
         ViewableContainer container = MimeUtility.extractTextAndAttachments(getContext(), message);
 
         String expectedText =
-                bodyText1 + "\n\n" +
+                bodyText1 + "\r\n\r\n" +
-                "------------------------------------------------------------------------\n\n" +
+                "------------------------------------------------------------------------\r\n\r\n" +
                 bodyText2;
         String expectedHtml =
                 "<pre class=\"k9mail\">" +
@@ -138,14 +138,14 @@ public class ViewablesTest extends AndroidTestCase {
 
         String expectedText =
                 bodyText +
-                "\n\n" +
+                "\r\n\r\n" +
                 "----- message.eml ------------------------------------------------------" +
-                "\n\n" +
+                "\r\n\r\n" +
-                "From: from@example.com" + "\n" +
+                "From: from@example.com" + "\r\n" +
-                "To: to@example.com" + "\n" +
+                "To: to@example.com" + "\r\n" +
-                "Sent: Sat Mar 17 00:00:00 GMT+01:00 2012" + "\n" +
+                "Sent: Sat Mar 17 00:00:00 GMT+01:00 2012" + "\r\n" +
-                "Subject: Subject" + "\n" +
+                "Subject: Subject" + "\r\n" +
-                "\n" +
+                "\r\n" +
                 innerBodyText;
         String expectedHtml =
                 "<pre class=\"k9mail\">" +