moparisthebest/k-9
1
0
Fork 0
mirror of https://github.com/moparisthebest/k-9 synced 2024-11-27 03:32:16 -05:00
Code Issues Releases Wiki Activity

Write large message parts to file system

Browse Source 
Actually, we just move the temporary file to avoid having to copy the
data to a new file.
This commit is contained in:
cketti 2015-01-24 03:06:45 +01:00
parent 977d15c190
commit 378acbd313
7 changed files with 165 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
k9mail-library/src/main/java/com/fsck/k9/mail/internet/BinaryTempFileBody.java
View File

@ -1,6 +1,5 @@
package com.fsck.k9.mail.internet;
import com.fsck.k9.mail.Body;
import com.fsck.k9.mail.MessagingException;
import com.fsck.k9.mail.filter.Base64OutputStream;
import org.apache.commons.io.IOUtils;
@ -15,7 +14,7 @@ import java.io.*;
* and writeTo one time. After writeTo is called, or the InputStream returned from
* getInputStream is closed the file is deleted and the Body should be considered disposed of.
*/
public class BinaryTempFileBody implements RawDataBody {
public class BinaryTempFileBody implements RawDataBody, SizeAware {
private static File mTempDirectory;
private File mFile;
@ -26,6 +25,10 @@ public class BinaryTempFileBody implements RawDataBody {
mTempDirectory = tempDirectory;
}
public static File getTempDirectory() {
return mTempDirectory;
}
@Override
public String getEncoding() {
return mEncoding;
@ -101,6 +104,15 @@ public class BinaryTempFileBody implements RawDataBody {
}
}
@Override
public long getSize() {
return mFile.length();
}
public File getFile() {
return mFile;
}
class BinaryTempFileBodyInputStream extends FilterInputStream {
public BinaryTempFileBodyInputStream(InputStream in) {
super(in);

6
k9mail-library/src/main/java/com/fsck/k9/mail/internet/SizeAware.java Normal file
View File

@ -0,0 +1,6 @@
package com.fsck.k9.mail.internet;
public interface SizeAware {
long getSize();
}

32
k9mail-library/src/main/java/com/fsck/k9/mail/internet/TextBody.java
View File

@ -10,10 +10,11 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import com.fsck.k9.mail.filter.CountingOutputStream;
import org.apache.james.mime4j.codec.QuotedPrintableOutputStream;
import org.apache.james.mime4j.util.MimeUtil;
public class TextBody implements Body {
public class TextBody implements Body, SizeAware {
/**
* Immutable empty byte array
@ -98,4 +99,33 @@ public class TextBody implements Body {
public void setComposedMessageOffset(Integer composedMessageOffset) {
this.mComposedMessageOffset = composedMessageOffset;
}
@Override
public long getSize() {
try {
byte[] bytes = mBody.getBytes(mCharset);
if (MimeUtil.ENC_8BIT.equalsIgnoreCase(mEncoding)) {
return bytes.length;
} else {
return getLengthWhenQuotedPrintableEncoded(bytes);
}
} catch (IOException e) {
throw new RuntimeException("Couldn't get body size", e);
}
}
private long getLengthWhenQuotedPrintableEncoded(byte[] bytes) throws IOException {
CountingOutputStream countingOutputStream = new CountingOutputStream();
OutputStream quotedPrintableOutputStream = new QuotedPrintableOutputStream(countingOutputStream, false);
try {
quotedPrintableOutputStream.write(bytes);
} finally {
try {
quotedPrintableOutputStream.close();
} catch (IOException e) { /* ignore */ }
}
return countingOutputStream.getCount();
}
}

8
k9mail/src/main/java/com/fsck/k9/mailstore/BinaryMemoryBody.java
View File

@ -9,9 +9,10 @@ import java.io.OutputStream;
import com.fsck.k9.mail.Body;
import com.fsck.k9.mail.MessagingException;
import com.fsck.k9.mail.internet.RawDataBody;
import com.fsck.k9.mail.internet.SizeAware;
public class BinaryMemoryBody implements Body, RawDataBody {
public class BinaryMemoryBody implements Body, RawDataBody, SizeAware {
private final byte[] data;
private final String encoding;
@ -39,4 +40,9 @@ public class BinaryMemoryBody implements Body, RawDataBody {
public void writeTo(OutputStream out) throws IOException, MessagingException {
out.write(data);
}
@Override
public long getSize() {
return data.length;
}
}

120
k9mail/src/main/java/com/fsck/k9/mailstore/LocalFolder.java
View File

@ -4,8 +4,11 @@ package com.fsck.k9.mailstore;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
@ -43,9 +46,11 @@ import com.fsck.k9.mail.MessagingException;
import com.fsck.k9.mail.Multipart;
import com.fsck.k9.mail.Part;
import com.fsck.k9.mail.filter.CountingOutputStream;
import com.fsck.k9.mail.internet.BinaryTempFileBody;
import com.fsck.k9.mail.internet.MimeHeader;
import com.fsck.k9.mail.internet.MimeMessage;
import com.fsck.k9.mail.internet.MimeMultipart;
import com.fsck.k9.mail.internet.SizeAware;
import com.fsck.k9.mailstore.LockableDatabase.DbCallback;
import com.fsck.k9.mailstore.LockableDatabase.WrappedException;
import org.apache.commons.io.IOUtils;
@ -62,6 +67,8 @@ public class LocalFolder extends Folder<LocalMessage> implements Serializable {
private static final long serialVersionUID = -1973296520918624767L;
private static final Uri PLACEHOLDER_URI = Uri.EMPTY;
private static final int MAX_BODY_SIZE_FOR_DATABASE = 16 * 1024;
private static final long INVALID_MESSAGE_PART_ID = -1;
private final LocalStore localStore;
@ -1340,26 +1347,48 @@ public class LocalFolder extends Folder<LocalMessage> implements Serializable {
cv.put("seq", order);
cv.put("server_extra", part.getServerExtra());
partToContentValues(cv, part);
return db.insertOrThrow("message_parts", null, cv);
return updateOrInsertMessagePart(db, cv, part, INVALID_MESSAGE_PART_ID);
}
private void partToContentValues(ContentValues cv, Part part) throws IOException, MessagingException {
private void renameTemporaryFile(File file, String messagePartId) {
File destination = localStore.getAttachmentFile(messagePartId);
if (!file.renameTo(destination)) {
Log.w(K9.LOG_TAG, "Couldn't rename temporary file " + file.getAbsolutePath() +
" to " + destination.getAbsolutePath());
}
}
private long updateOrInsertMessagePart(SQLiteDatabase db, ContentValues cv, Part part, long existingMessagePartId)
throws IOException, MessagingException {
byte[] headerBytes = getHeaderBytes(part);
cv.put("mime_type", part.getMimeType());
cv.put("header", headerBytes);
cv.put("type", MessagePartType.UNKNOWN);
File file = null;
Body body = part.getBody();
if (body instanceof Multipart) {
multipartToContentValues(cv, (Multipart) body);
} else if (body == null) {
missingPartToContentValues(cv, part);
} else {
leafPartToContentValues(cv, part, body);
file = leafPartToContentValues(cv, part, body);
}
long messagePartId;
if (existingMessagePartId != INVALID_MESSAGE_PART_ID) {
messagePartId = existingMessagePartId;
db.update("message_parts", cv, "id = ?", new String[] { Long.toString(messagePartId) });
} else {
messagePartId = db.insertOrThrow("message_parts", null, cv);
}
if (file != null) {
renameTemporaryFile(file, Long.toString(messagePartId));
}
return messagePartId;
}
private void multipartToContentValues(ContentValues cv, Multipart multipart) {
@ -1376,21 +1405,64 @@ public class LocalFolder extends Folder<LocalMessage> implements Serializable {
cv.put("decoded_body_size", attachment.size);
}
private void leafPartToContentValues(ContentValues cv, Part part, Body body)
private File leafPartToContentValues(ContentValues cv, Part part, Body body)
throws MessagingException, IOException {
AttachmentViewInfo attachment = LocalMessageExtractor.extractAttachmentInfo(part, PLACEHOLDER_URI);
cv.put("display_name", attachment.displayName);
cv.put("data_location", DataLocation.IN_DATABASE);
byte[] bodyData = getBodyBytes(body);
String encoding = getTransferEncoding(part);
long size = decodeAndCountBytes(bodyData, encoding, bodyData.length);
cv.put("decoded_body_size", size);
if (!(body instanceof SizeAware)) {
throw new IllegalStateException("Body needs to implement SizeAware");
}
SizeAware sizeAwareBody = (SizeAware) body;
long fileSize = sizeAwareBody.getSize();
File file = null;
int dataLocation;
if (fileSize > MAX_BODY_SIZE_FOR_DATABASE) {
dataLocation = DataLocation.ON_DISK;
file = writeBodyToDiskIfNecessary(part);
long size = decodeAndCountBytes(file, encoding, fileSize);
cv.put("decoded_body_size", size);
} else {
dataLocation = DataLocation.IN_DATABASE;
byte[] bodyData = getBodyBytes(body);
cv.put("data", bodyData);
long size = decodeAndCountBytes(bodyData, encoding, bodyData.length);
cv.put("decoded_body_size", size);
}
cv.put("data_location", dataLocation);
cv.put("encoding", encoding);
cv.put("data", bodyData);
cv.put("content_id", part.getContentId());
return file;
}
private File writeBodyToDiskIfNecessary(Part part) throws MessagingException, IOException {
Body body = part.getBody();
if (body instanceof BinaryTempFileBody) {
return ((BinaryTempFileBody) body).getFile();
} else {
return writeBodyToDisk(body);
}
}
private File writeBodyToDisk(Body body) throws IOException, MessagingException {
File file = File.createTempFile("body", null, BinaryTempFileBody.getTempDirectory());
OutputStream out = new FileOutputStream(file);
try {
body.writeTo(out);
} finally {
out.close();
}
return file;
}
private long decodeAndCountBytes(byte[] bodyData, String encoding, long fallbackValue) {
@ -1398,7 +1470,17 @@ public class LocalFolder extends Folder<LocalMessage> implements Serializable {
return decodeAndCountBytes(rawInputStream, encoding, fallbackValue);
}
private long decodeAndCountBytes(ByteArrayInputStream rawInputStream, String encoding, long fallbackValue) {
private long decodeAndCountBytes(File file, String encoding, long fallbackValue)
throws MessagingException, IOException {
InputStream inputStream = new FileInputStream(file);
try {
return decodeAndCountBytes(inputStream, encoding, fallbackValue);
} finally {
inputStream.close();
}
}
private long decodeAndCountBytes(InputStream rawInputStream, String encoding, long fallbackValue) {
InputStream decodingInputStream = localStore.getDecodingInputStream(rawInputStream, encoding);
try {
CountingOutputStream countingOutputStream = new CountingOutputStream();
@ -1464,7 +1546,7 @@ public class LocalFolder extends Folder<LocalMessage> implements Serializable {
localStore.database.execute(false, new DbCallback<Void>() {
@Override
public Void doDbWork(final SQLiteDatabase db) throws WrappedException, UnavailableStorageException {
String messagePartId;
long messagePartId;
Cursor cursor = db.query("message_parts", new String[] { "id" }, "root = ? AND server_extra = ?",
new String[] { Long.toString(message.getMessagePartId()), part.getServerExtra() },
@ -1474,16 +1556,13 @@ public class LocalFolder extends Folder<LocalMessage> implements Serializable {
throw new IllegalStateException("Message part not found");
}
messagePartId = cursor.getString(0);
messagePartId = cursor.getLong(0);
} finally {
cursor.close();
}
try {
ContentValues cv = new ContentValues();
partToContentValues(cv, part);
db.update("message_parts", cv, "id = ?", new String[] { messagePartId });
updateOrInsertMessagePart(db, new ContentValues(), part, messagePartId);
} catch (Exception e) {
Log.e(K9.LOG_TAG, "Error writing message part", e);
}
@ -1686,16 +1765,13 @@ public class LocalFolder extends Folder<LocalMessage> implements Serializable {
}
private void deleteMessagePartsFromDisk(SQLiteDatabase db, long rootMessagePartId) {
File attachmentDirectory = StorageManager.getInstance(localStore.context)
.getAttachmentDirectory(getAccountUuid(), localStore.database.getStorageProviderId());
Cursor cursor = db.query("message_parts", new String[] { "id" },
"root = ? AND data_location = " + DataLocation.ON_DISK,
new String[] { Long.toString(rootMessagePartId) }, null, null, null);
try {
while (cursor.moveToNext()) {
String messagePartId = cursor.getString(0);
File file = new File(attachmentDirectory, messagePartId);
File file = localStore.getAttachmentFile(messagePartId);
if (file.exists()) {
if (!file.delete() && K9.DEBUG) {
Log.d(K9.LOG_TAG, "Couldn't delete message part file: " + file.getAbsolutePath());

2
k9mail/src/main/java/com/fsck/k9/mailstore/LocalStore.java
View File

@ -728,7 +728,7 @@ public class LocalStore extends Store implements Serializable {
return rawInputStream;
}
private File getAttachmentFile(String attachmentId) {
File getAttachmentFile(String attachmentId) {
final StorageManager storageManager = StorageManager.getInstance(context);
final File attachmentDirectory = storageManager.getAttachmentDirectory(uUid, database.getStorageProviderId());
return new File(attachmentDirectory, attachmentId);

9
k9mail/src/main/java/com/fsck/k9/mailstore/TempFileBody.java
View File

@ -7,11 +7,13 @@ import java.io.FileNotFoundException;
import java.io.InputStream;
import com.fsck.k9.mail.MessagingException;
import com.fsck.k9.mail.internet.SizeAware;
/**
* An attachment whose contents are contained in a file.
*/
public class TempFileBody extends BinaryAttachmentBody {
public class TempFileBody extends BinaryAttachmentBody implements SizeAware {
private final File mFile;
public TempFileBody(String filename) {
@ -26,4 +28,9 @@ public class TempFileBody extends BinaryAttachmentBody {
return new ByteArrayInputStream(LocalStore.EMPTY_BYTE_ARRAY);
}
}
@Override
public long getSize() {
return mFile.length();
}
}