Start on unit tests for HMEF. Quite a bit is still stubbed out, and it shows that the LZW isn't quite right yet (so tests disabled)

git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1075955 13f79535-47bb-0310-9956-ffa450edef68

src/java/org/apache/poi/util/LZWDecompresser.java

@@ -35,10 +35,16 @@ public abstract class LZWDecompresser {
    /**
     * Does the mask bit mean it's compressed or uncompressed?
     */
-   private boolean maskMeansCompressed;
+   private final boolean maskMeansCompressed;
+   /**
+    * How much to append to the code length in the stream
+    *  to get the real code length? Normally 2 or 3
+    */
+   private final int codeLengthIncrease;
    
-   protected LZWDecompresser(boolean maskMeansCompressed) {
+   protected LZWDecompresser(boolean maskMeansCompressed, int codeLengthIncrease) {
       this.maskMeansCompressed = maskMeansCompressed;
+      this.codeLengthIncrease = codeLengthIncrease;
    }
    
    /**
@@ -135,7 +141,7 @@ public abstract class LZWDecompresser {
                //  what position of the code to start at
                // (The position is the first 12 bits, the
                //  length is the last 4 bits)
-               len = (dataIPt2 & 15) + 3;
+               len = (dataIPt2 & 15) + codeLengthIncrease;
                pntr = (dataIPt2 & 240)*16 + dataIPt1;
 
                // Adjust the pointer as needed

src/scratchpad/src/org/apache/poi/hdgf/HDGFLZW.java

@@ -38,7 +38,7 @@ import org.apache.poi.util.LZWDecompresser;
 public class HDGFLZW extends LZWDecompresser {
    public HDGFLZW() {
       // We're the wrong way round!
-      super(false);
+      super(false, 3);
    }
 
    /**

src/scratchpad/src/org/apache/poi/hmef/Attribute.java

@@ -254,4 +254,9 @@ public final class Attribute {
    public byte[] getData() {
       return data;
    }
+   
+   public String toString() {
+      return "Attachment " + getId().toString() + ", type=" + type + 
+             ", data length=" + data.length; 
+   }
 }

src/scratchpad/src/org/apache/poi/hmef/CompressedRTF.java

@@ -54,7 +54,7 @@ public final class CompressedRTF extends LZWDecompresser {
       "{\\colortbl\\red0\\green0\\blue0\n\r\\par \\pard\\plain\\f0\\fs20\\b\\i\\u\\tab\\tx";
    
    public CompressedRTF() {
-      super(true);
+      super(true, 2);
    }
 
    public void decompress(InputStream src, OutputStream res) throws IOException {

src/scratchpad/src/org/apache/poi/hmef/HMEFMessage.java

@@ -22,6 +22,8 @@ import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.poi.hmef.Attribute.AttributeID;
+import org.apache.poi.hsmf.datatypes.MAPIProperty;
 import org.apache.poi.util.LittleEndian;
 
 /**
@@ -103,4 +105,55 @@ public final class HMEFMessage {
       // Handle the next one down
       process(inp, level);
    }
+   
+   /**
+    * Returns all HMEF/TNEF attributes of the message. 
+    * Note - In a typical message, most of the interesting properties
+    *  are stored as {@link MAPIAttribute}s - see {@link #getMessageMAPIAttributes()} 
+    */
+   public List<Attribute> getMessageAttributes() {
+      return messageAttributes;
+   }
+   
+   /**
+    * Returns all MAPI attributes of the message.
+    * Note - A small number of HMEF/TNEF specific attributes normally
+    *  apply to most messages, see {@link #getMessageAttributes()}
+    */
+   public List<MAPIAttribute> getMessageMAPIAttributes() {
+      return mapiAttributes;
+   }
+   
+   /**
+    * Returns all the Attachments of the message.
+    */
+   public List<Attachment> getAttachments() {
+      return attachments;
+   }
+   
+   /**
+    * Return the message attribute with the given ID,
+    *  or null if there isn't one. 
+    */
+   public Attribute getMessageAttribute(AttributeID id) {
+      for(Attribute attr : messageAttributes) {
+         if(attr.getId() == id) {
+            return attr;
+         }
+      }
+      return null;
+   }
+   
+   /**
+    * Return the message MAPI Attribute with the given ID,
+    *  or null if there isn't one. 
+    */
+   public MAPIAttribute getMessageMAPIAttribute(MAPIProperty id) {
+      for(MAPIAttribute attr : mapiAttributes) {
+         if(attr.getProperty() == id) {
+            return attr;
+         }
+      }
+      return null;
+   }
 }

src/scratchpad/testcases/org/apache/poi/hmef/TestCompressedRTF.java

@@ -0,0 +1,155 @@
+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
+
+package org.apache.poi.hmef;
+
+import java.io.ByteArrayInputStream;
+
+import junit.framework.TestCase;
+
+import org.apache.poi.POIDataSamples;
+import org.apache.poi.hmef.Attribute.AttributeID;
+import org.apache.poi.hsmf.datatypes.MAPIProperty;
+import org.apache.poi.util.LittleEndian;
+import org.apache.poi.util.StringUtil;
+
+public final class TestCompressedRTF extends TestCase {
+    private static final POIDataSamples _samples = POIDataSamples.getHMEFInstance();
+
+    private static final String block1 = "{\\rtf1\\adeflang102";
+    private static final String block2 = block1 + "5\\ansi\\ansicpg1252";
+    
+    /**
+     * Check that things are as we expected. If this fails,
+     *  then decoding has no hope...  
+     */
+    public void testQuickBasics() throws Exception {
+       HMEFMessage msg = new HMEFMessage(
+             _samples.openResourceAsStream("quick-winmail.dat")
+       );
+       
+       MAPIAttribute rtfAttr = msg.getMessageMAPIAttribute(MAPIProperty.RTF_COMPRESSED);
+       assertNotNull(rtfAttr);
+       assertTrue(rtfAttr instanceof MAPIRtfAttribute);
+       
+       // Check the start of the compressed version
+       assertEquals(5907, rtfAttr.getData().length);
+       
+       // First 16 bytes is header stuff
+       // Check it has the length + compressed marker
+       assertEquals(5907-4, LittleEndian.getShort(rtfAttr.getData()));
+       assertEquals(
+             "LZFu", 
+             StringUtil.getFromCompressedUnicode(rtfAttr.getData(), 8, 4)
+       );
+             
+       
+       // Now Look at the code
+       assertEquals((byte)0x07, rtfAttr.getData()[16+0]);  // Flag: cccUUUUU
+       assertEquals((byte)0x00, rtfAttr.getData()[16+1]);  //  c1a: offset 0 / 0x000
+       assertEquals((byte)0x06, rtfAttr.getData()[16+2]);  //  c1b: length 6+2  -> {\rtf1\a
+       assertEquals((byte)0x01, rtfAttr.getData()[16+3]);  //  c2a: offset 16 / 0x010
+       assertEquals((byte)0x01, rtfAttr.getData()[16+4]);  //  c2b: length 1+2  ->  def
+       assertEquals((byte)0x0b, rtfAttr.getData()[16+5]);  //  c3a: offset 182 / 0xb6
+       assertEquals((byte)0x60, rtfAttr.getData()[16+6]);  //  c3b: length 0+2  -> la 
+       assertEquals((byte)0x6e, rtfAttr.getData()[16+7]);  // n
+       assertEquals((byte)0x67, rtfAttr.getData()[16+8]);  // g
+       assertEquals((byte)0x31, rtfAttr.getData()[16+9]);  // 1
+       assertEquals((byte)0x30, rtfAttr.getData()[16+10]); // 0
+       assertEquals((byte)0x32, rtfAttr.getData()[16+11]); // 2
+       
+       assertEquals((byte)0x66, rtfAttr.getData()[16+12]); // Flag:  UccUUccU
+       assertEquals((byte)0x35, rtfAttr.getData()[16+13]); // 5 
+       assertEquals((byte)0x00, rtfAttr.getData()[16+14]); //  c2a: offset 6 / 0x006
+       assertEquals((byte)0x64, rtfAttr.getData()[16+15]); //  c2b: length 4+2  -> \ansi\a
+       assertEquals((byte)0x00, rtfAttr.getData()[16+16]); //  c3a: offset 7 / 0x007
+       assertEquals((byte)0x72, rtfAttr.getData()[16+17]); //  c3b: length 2+2  -> nsi
+       assertEquals((byte)0x63, rtfAttr.getData()[16+18]); // c 
+       assertEquals((byte)0x70, rtfAttr.getData()[16+19]); // p
+       assertEquals((byte)0x0d, rtfAttr.getData()[16+20]); //  c6a: offset 221 / 0x0dd
+       assertEquals((byte)0xd0, rtfAttr.getData()[16+21]); //  c6b: length 0+2  -> g1
+       assertEquals((byte)0x0e, rtfAttr.getData()[16+22]); //  c7a: offset 224 / 0x0e0
+       assertEquals((byte)0x00, rtfAttr.getData()[16+23]); //  c7b: length 0+2  -> 25
+       assertEquals((byte)0x32, rtfAttr.getData()[16+24]); // 2
+    }
+
+    /**
+     * Check that we can decode the first 8 codes
+     * (1 flag byte + 8 codes)  
+     */
+    public void DISABLEDtestFirstBlock() throws Exception {
+       HMEFMessage msg = new HMEFMessage(
+             _samples.openResourceAsStream("quick-winmail.dat")
+       );
+       
+       MAPIAttribute rtfAttr = msg.getMessageMAPIAttribute(MAPIProperty.RTF_COMPRESSED);
+       assertNotNull(rtfAttr);
+
+       // Truncate to header + flag + data for flag
+       byte[] data = new byte[16+12];
+       System.arraycopy(rtfAttr.getData(), 0, data, 0, data.length);
+       
+       // Decompress it
+       CompressedRTF comp = new CompressedRTF();
+       byte[] decomp = comp.decompress(new ByteArrayInputStream(data));
+       String decompStr = new String(decomp, "ASCII");
+       
+       // Test
+System.err.println(decompStr);       
+       assertEquals(block1.length(), decomp.length);
+       assertEquals(block1, decompStr);
+    }
+
+    /**
+     * Check that we can decode the first 16 codes
+     * (flag + 8 codes, flag + 8 codes)  
+     */
+    public void DISABLEDtestFirstTwoBlocks() throws Exception {
+       HMEFMessage msg = new HMEFMessage(
+             _samples.openResourceAsStream("quick-winmail.dat")
+       );
+
+       MAPIAttribute rtfAttr = msg.getMessageMAPIAttribute(MAPIProperty.RTF_COMPRESSED);
+       assertNotNull(rtfAttr);
+
+       // Truncate to header + flag + data for flag + flag + data
+       byte[] data = new byte[16+12+13];
+       System.arraycopy(rtfAttr.getData(), 0, data, 0, data.length);
+       
+       // Decompress it
+       CompressedRTF comp = new CompressedRTF();
+       byte[] decomp = comp.decompress(new ByteArrayInputStream(data));
+       String decompStr = new String(decomp, "ASCII");
+       
+       // Test
+System.err.println(decompStr);       
+       assertEquals(block2.length(), decomp.length);
+       assertEquals(block2, decompStr);
+    }
+
+    /**
+     * Check that we can correctly decode the whole file
+     * @throws Exception
+     */
+    public void testFull() throws Exception {
+       HMEFMessage msg = new HMEFMessage(
+             _samples.openResourceAsStream("quick-winmail.dat")
+       );
+       
+       // TODO
+    }
+}

src/scratchpad/testcases/org/apache/poi/hmef/TestHMEFMessage.java

@@ -0,0 +1,103 @@
+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
+
+package org.apache.poi.hmef;
+
+import junit.framework.TestCase;
+
+import org.apache.poi.POIDataSamples;
+
+public final class TestHMEFMessage extends TestCase {
+    private static final POIDataSamples _samples = POIDataSamples.getHMEFInstance();
+
+	public void testOpen() throws Exception {
+		HMEFMessage msg = new HMEFMessage(
+                _samples.openResourceAsStream("quick-winmail.dat")
+		);
+
+		assertNotNull(msg);
+	}
+
+	public void testCounts() throws Exception {
+      HMEFMessage msg = new HMEFMessage(
+            _samples.openResourceAsStream("quick-winmail.dat")
+      );
+      
+      // Should have 4 attributes on the message
+      assertEquals(4, msg.getMessageAttributes().size());
+      
+      // And should have 54 MAPI attributes on it
+      assertEquals(54, msg.getMessageMAPIAttributes().size());
+      
+      
+      // Should have 5 attachments
+      assertEquals(5, msg.getAttachments().size());
+      
+      // Each attachment should have 6 normal attributes, and 
+      //  20 or so MAPI ones
+      for(Attachment attach : msg.getAttachments()) {
+         int attrCount = attach.getAttributes().size();
+         int mapiAttrCount = attach.getMAPIAttributes().size();
+         
+         assertEquals(6, attrCount);
+         // TODO
+//         assertTrue("Should be 3-4 attributes, found " + mapiAttrCount, mapiAttrCount >= 20);
+//         assertTrue("Should be 3-4 attributes, found " + mapiAttrCount, mapiAttrCount <= 25);
+      }
+      
+      
+      // TODO
+	}
+	
+	public void testBasicMessageAttributes() throws Exception {
+      HMEFMessage msg = new HMEFMessage(
+            _samples.openResourceAsStream("quick-winmail.dat")
+      );
+      
+      // Should have version, codepage, class and MAPI
+      assertEquals(4, msg.getMessageAttributes().size());
+      assertNotNull(msg.getMessageAttribute(Attribute.ID_TNEFVERSION));
+      assertNotNull(msg.getMessageAttribute(Attribute.ID_OEMCODEPAGE));
+      assertNotNull(msg.getMessageAttribute(Attribute.ID_MESSAGECLASS));
+      assertNotNull(msg.getMessageAttribute(Attribute.ID_MAPIPROPERTIES));
+      
+      // Check the order
+      assertEquals(Attribute.ID_TNEFVERSION, msg.getMessageAttributes().get(0).getId());
+      assertEquals(Attribute.ID_OEMCODEPAGE, msg.getMessageAttributes().get(1).getId());
+      assertEquals(Attribute.ID_MESSAGECLASS, msg.getMessageAttributes().get(2).getId());
+      assertEquals(Attribute.ID_MAPIPROPERTIES, msg.getMessageAttributes().get(3).getId());
+      
+      // Check some that aren't there
+      assertNull(msg.getMessageAttribute(Attribute.ID_AIDOWNER));
+      assertNull(msg.getMessageAttribute(Attribute.ID_ATTACHDATA));
+      
+      // Now check the details of one or two
+      // TODO
+	}
+   
+   public void testBasicMessageMAPIAttributes() throws Exception {
+      // TODO
+   }
+   
+   public void testBasicAttachments() throws Exception {
+      // TODO
+   }
+   
+   public void testMessageAttributeDetails() throws Exception {
+      // TODO
+   }
+}

src/testcases/org/apache/poi/POIDataSamples.java

@@ -35,6 +35,7 @@ public final class POIDataSamples {
     private static POIDataSamples _instOpenxml4j;
     private static POIDataSamples _instPOIFS;
     private static POIDataSamples _instDDF;
+    private static POIDataSamples _instHMEF;
     private static POIDataSamples _instHPSF;
     private static POIDataSamples _instHPBF;
     private static POIDataSamples _instHSMF;
@@ -99,6 +100,11 @@ public final class POIDataSamples {
         return _instHPBF;
     }
 
+    public static POIDataSamples getHMEFInstance(){
+        if(_instHMEF == null) _instHMEF = new POIDataSamples("hmef");
+        return _instHMEF;
+    }
+
     public static POIDataSamples getHSMFInstance(){
         if(_instHSMF == null) _instHSMF = new POIDataSamples("hsmf");
         return _instHSMF;

test-data/hmef/quick-contents/quick.html

@@ -0,0 +1,17 @@
+<html>
+
+<head>
+   <meta http-equiv=Content-Type content="text/html; charset=windows-1252">
+   <title>The quick brown fox jumps over the lazy dog</title>
+   <meta name="author" content="Nevin Nollop">
+   <meta name="keywords" content="Pangram, fox, dog">
+   <meta name="description" content="Gym class featuring a brown fox and lazy dog">
+</head>
+
+<body lang=EN-US>
+
+The quick brown fox jumps over the lazy dog
+
+</body>
+
+</html>

test-data/hmef/quick-contents/quick.txt

@@ -0,0 +1,7 @@
+The quick brown fox jumps over the lazy dog
+
+Le renard brun rapide saute par-dessus le chien paresseux
+
+Der schnelle braune Fuchs springt über den faulen Hund
+
+براون وكس السريع يقفز فوق الكلب كسالي

test-data/hmef/quick-contents/quick.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+
+<document>
+   <text>The quick brown fox jumps over the lazy dog</text>
+</document>