diff --git a/src/com/fsck/k9/fragment/MessageListFragment.java b/src/com/fsck/k9/fragment/MessageListFragment.java
index 66dbc6691..f8cda0b0d 100644
--- a/src/com/fsck/k9/fragment/MessageListFragment.java
+++ b/src/com/fsck/k9/fragment/MessageListFragment.java
@@ -1923,8 +1923,8 @@ public class MessageListFragment extends SherlockFragment implements OnItemClick
             holder.position = cursor.getPosition();
 
             if (holder.contactBadge != null) {
-                holder.contactBadge.assignContactFromEmail(counterpartyAddress.getAddress(), true);
                 if (counterpartyAddress != null) {
+                    holder.contactBadge.assignContactFromEmail(counterpartyAddress.getAddress(), true);
                     /*
                      * At least in Android 2.2 a different background + padding is used when no
                      * email address is available. ListView reuses the views but QuickContactBadge