Basic support for a few key kinds of streams, and tests to go with this

git-svn-id: https://svn.apache.org/repos/asf/jakarta/poi/trunk@548511 13f79535-47bb-0310-9956-ffa450edef68

build.xml

@@ -495,6 +495,7 @@ under the License.
             <sysproperty key="HDF.testdata.path" file="${scratchpad.src.test}/org/apache/poi/hdf/data"/>
             <sysproperty key="HWPF.testdata.path" file="${scratchpad.src.test}/org/apache/poi/hwpf/data"/>
             <sysproperty key="HSLF.testdata.path" file="${scratchpad.src.test}/org/apache/poi/hslf/data"/>
+            <sysproperty key="HDGF.testdata.path" file="${scratchpad.src.test}/org/apache/poi/hdgf/data"/>
             <sysproperty key="java.awt.headless" value="true"/>
             <formatter type="plain"/>
             <formatter type="xml"/>
@@ -526,7 +527,8 @@ under the License.
 	            <sysproperty key="HSSF.testdata.path" file="${scratchpad.src.test}/org/apache/poi/hssf/data"/>
 	            <sysproperty key="HPSF.testdata.path" file="${scratchpad.src.test}/org/apache/poi/hpsf/data"/>
 	            <sysproperty key="HWPF.testdata.path" file="${scratchpad.src.test}/org/apache/poi/hwpf/data"/>
-            <sysproperty key="HSLF.testdata.path" file="${scratchpad.src.test}/org/apache/poi/hslf/data"/>
+                <sysproperty key="HSLF.testdata.path" file="${scratchpad.src.test}/org/apache/poi/hslf/data"/>
+                <sysproperty key="HDGF.testdata.path" file="${scratchpad.src.test}/org/apache/poi/hdgf/data"/>
 	            <sysproperty key="java.awt.headless" value="true"/>
 	                <sysproperty key="java.awt.headless" value="true"/>
 	            <formatter type="plain" usefile="no"/>

src/scratchpad/src/org/apache/poi/hdgf/streams/CompressedStreamStore.java

@@ -1,6 +1,19 @@
-/**
- * 
- */
+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
 package org.apache.poi.hdgf.streams;
 
 import java.io.ByteArrayInputStream;
@@ -8,6 +21,10 @@ import java.io.IOException;
 
 import org.apache.poi.hdgf.LZW4HDGF;
 
+/**
+ * A StreamStore where the data on-disk is compressed,
+ *  using the crazy Visio LZW
+ */
 class CompressedStreamStore extends StreamStore {
 	/** The raw, compressed contents */
 	private byte[] compressedContents;

src/scratchpad/src/org/apache/poi/hdgf/streams/PointerContainingStream.java

@@ -0,0 +1,94 @@
+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
+package org.apache.poi.hdgf.streams;
+
+import org.apache.poi.hdgf.pointers.Pointer;
+import org.apache.poi.hdgf.pointers.PointerFactory;
+import org.apache.poi.util.LittleEndian;
+
+/**
+ * A stream that holds pointers, possibly in addition to some
+ *  other data too.
+ */
+public class PointerContainingStream extends Stream {
+	private Pointer[] childPointers;
+	private Stream[] childStreams;
+	
+	private PointerFactory pointerFactory;
+	private int numPointersLocalOffset;
+	
+	protected PointerContainingStream(Pointer pointer, StreamStore store, PointerFactory pointerFactory) {
+		super(pointer, store);
+		this.pointerFactory = pointerFactory;
+		
+		// Find the offset to the number of child pointers we have
+		// This ought to be the first thing stored in us
+		numPointersLocalOffset = (int)LittleEndian.getUInt(
+				store.getContents(), 0
+		);
+		
+		// Generate the objects for the pointers we contain
+		int numPointers = (int)LittleEndian.getUInt(
+				store.getContents(), numPointersLocalOffset
+		);
+		childPointers = new Pointer[numPointers];
+		
+		// After the number of pointers is another (unknown)
+		//  4 byte value
+		int pos = numPointersLocalOffset + 4 + 4;
+		
+		// Now create the pointer objects
+		for(int i=0; i<numPointers; i++) {
+			childPointers[i] = pointerFactory.createPointer(
+					store.getContents(), pos
+			);
+			pos += childPointers[i].getSizeInBytes();
+		}
+	}
+	
+	/**
+	 * Returns all the pointers that we contain
+	 */
+	protected Pointer[] getChildPointers() { return childPointers; }
+	/**
+	 * Returns all the "child" streams.
+	 * These are all the streams pointed to by the pointers
+	 *  that we contain.
+	 */
+	public Stream[] getPointedToStreams() { return childStreams; } 
+	
+	/**
+	 * Performs a recursive search, identifying the pointers we contain,
+	 *  creating the Streams for where they point to, then searching
+	 *  those if appropriate.
+	 */
+	public void findChildren(byte[] documentData) {
+		// For each pointer, generate the Stream it points to
+		childStreams = new Stream[childPointers.length];
+		for(int i=0; i<childPointers.length; i++) {
+			Pointer ptr = childPointers[i];
+			childStreams[i] = Stream.createStream(ptr, documentData, pointerFactory);
+			
+			// Recurse if required
+			if(childStreams[i] instanceof PointerContainingStream) {
+				PointerContainingStream child = 
+					(PointerContainingStream)childStreams[i];
+				child.findChildren(documentData);
+			}
+		}
+	}
+}

src/scratchpad/src/org/apache/poi/hdgf/streams/Stream.java

@@ -19,6 +19,7 @@ package org.apache.poi.hdgf.streams;
 import java.io.IOException;
 
 import org.apache.poi.hdgf.pointers.Pointer;
+import org.apache.poi.hdgf.pointers.PointerFactory;
 
 /**
  * Base of all Streams within a HDGF document.
@@ -49,7 +50,7 @@ public abstract class Stream {
 	 * @param pointer The Pointer to create a stream for
 	 * @param documentData The raw document data
 	 */
-	public static Stream createStream(Pointer pointer, byte[] documentData) {
+	public static Stream createStream(Pointer pointer, byte[] documentData, PointerFactory pointerFactory) {
 		// Create the store
 		StreamStore store;
 		if(pointer.destinationCompressed()) {
@@ -67,7 +68,16 @@ public abstract class Stream {
 			);
 		}
 		
-		// Figure out what sort of Stream to create, and create it
+		// Figure out what sort of Stream to create, create and return it
+		if(pointer.getType() == 20) {
+			return new TrailerStream(pointer, store, pointerFactory);
+		}
+		else if(pointer.destinationHasPointers()) {
+			return new PointerContainingStream(pointer, store, pointerFactory);
+		}
+		else if(pointer.destinationHasStrings()) {
+			return new StringsStream(pointer, store);
+		}
 		
 		// Give up and return a generic one
 		return new UnknownStream(pointer, store);

src/scratchpad/src/org/apache/poi/hdgf/streams/StreamStore.java

@@ -1,3 +1,19 @@
+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
 package org.apache.poi.hdgf.streams;
 
 /**

src/scratchpad/src/org/apache/poi/hdgf/streams/StringsStream.java

@@ -0,0 +1,28 @@
+/* ====================================================================
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==================================================================== */
+package org.apache.poi.hdgf.streams;
+
+import org.apache.poi.hdgf.pointers.Pointer;
+
+/**
+ * A Stream which holds Strings
+ */
+public class StringsStream extends Stream {
+	protected StringsStream(Pointer pointer, StreamStore store) {
+		super(pointer, store);
+	}
+}

src/scratchpad/src/org/apache/poi/hdgf/streams/TrailerStream.java

@@ -0,0 +1,32 @@
+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
+package org.apache.poi.hdgf.streams;
+
+import org.apache.poi.hdgf.pointers.Pointer;
+import org.apache.poi.hdgf.pointers.PointerFactory;
+
+/**
+ * The TrailerStream is a special kind of Stream containing pointers,
+ *  and some core document settings.
+ * These is one of these in each document, and it's pointed to by
+ *  a special series of byte near the start of the file.
+ */
+public class TrailerStream extends PointerContainingStream {
+	protected TrailerStream(Pointer pointer, StreamStore store, PointerFactory pointerFactory) {
+		super(pointer, store, pointerFactory);
+	}
+}

src/scratchpad/testcases/org/apache/poi/hdgf/streams/StreamTest.java

@@ -0,0 +1,41 @@
+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
+package org.apache.poi.hdgf.streams;
+
+import org.apache.poi.hdgf.pointers.Pointer;
+
+import junit.framework.TestCase;
+
+public abstract class StreamTest extends TestCase {
+	public static class TestPointer extends Pointer {
+		private boolean compressed;
+		protected boolean hasPointers = false;
+		public TestPointer(boolean compressed, int offset, int length, int type, short format) {
+			this.compressed = compressed;
+			this.offset = offset;
+			this.length = length;
+			this.type = type;
+			this.format = format;
+		}
+
+		public boolean destinationCompressed() { return compressed; }
+		public boolean destinationHasChunks() { return false; }
+		public boolean destinationHasPointers() { return hasPointers; }
+		public boolean destinationHasStrings() { return false; }
+		public int getSizeInBytes() { return -1; }
+	}
+}

src/scratchpad/testcases/org/apache/poi/hdgf/streams/TestStreamBasics.java

@@ -1,11 +1,26 @@
+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
 package org.apache.poi.hdgf.streams;
 
 import org.apache.poi.hdgf.pointers.Pointer;
-import org.apache.poi.hdgf.pointers.PointerV6;
 
 import junit.framework.TestCase;
 
-public class TestStreamBasics extends TestCase {
+public class TestStreamBasics extends StreamTest {
 	/** The header from when compressedStream is decompressed */
 	public static final byte[] compressedStreamDCHeader = new byte[] {
 		-60, 2, 0, 0
@@ -74,7 +89,7 @@ public class TestStreamBasics extends TestCase {
 		// Create a fake pointer
 		Pointer ptr = new TestPointer(true, 0, compressedStream.length, -1, (short)-1);
 		// Now the stream
-		Stream stream = Stream.createStream(ptr, compressedStream);
+		Stream stream = Stream.createStream(ptr, compressedStream, null);
 		
 		// Check
 		assertNotNull(stream.getPointer());
@@ -98,7 +113,7 @@ public class TestStreamBasics extends TestCase {
 		// Create a fake pointer
 		Pointer ptr = new TestPointer(false, 0, uncompressedStream.length, -1, (short)-1);
 		// Now the stream
-		Stream stream = Stream.createStream(ptr, uncompressedStream);
+		Stream stream = Stream.createStream(ptr, uncompressedStream, null);
 		
 		// Check
 		assertNotNull(stream.getPointer());
@@ -107,21 +122,4 @@ public class TestStreamBasics extends TestCase {
 		assertFalse(stream.getStore() instanceof CompressedStreamStore);
 		assertTrue(stream instanceof UnknownStream);
 	}
-	
-	public static class TestPointer extends Pointer {
-		private boolean compressed;
-		public TestPointer(boolean compressed, int offset, int length, int type, short format) {
-			this.compressed = compressed;
-			this.offset = offset;
-			this.length = length;
-			this.type = type;
-			this.format = format;
-		}
-
-		public boolean destinationCompressed() { return compressed; }
-		public boolean destinationHasChunks() { return false; }
-		public boolean destinationHasPointers() { return false; }
-		public boolean destinationHasStrings() { return false; }
-		public int getSizeInBytes() { return -1; }
-	}
 }

src/scratchpad/testcases/org/apache/poi/hdgf/streams/TestStreamComplex.java

@@ -0,0 +1,184 @@
+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
+package org.apache.poi.hdgf.streams;
+
+import java.io.FileInputStream;
+
+import junit.framework.TestCase;
+
+import org.apache.poi.hdgf.pointers.Pointer;
+import org.apache.poi.hdgf.pointers.PointerFactory;
+import org.apache.poi.poifs.filesystem.DocumentEntry;
+import org.apache.poi.poifs.filesystem.POIFSFileSystem;
+
+public class TestStreamComplex extends StreamTest {
+	private byte[] contents;
+	private int trailerPointerAt = 0x24;
+	private int trailerDataAt = 0x8a94;
+	private PointerFactory ptrFactory;
+
+	protected void setUp() throws Exception {
+		String dirname = System.getProperty("HDGF.testdata.path");
+		String filename = dirname + "/Test_Visio-Some_Random_Text.vsd";
+		ptrFactory = new PointerFactory(6);
+
+		FileInputStream fin = new FileInputStream(filename);
+		POIFSFileSystem filesystem = new POIFSFileSystem(fin);
+		
+		DocumentEntry docProps =
+			(DocumentEntry)filesystem.getRoot().getEntry("VisioDocument");
+
+		// Grab the document stream
+		contents = new byte[docProps.getSize()];
+		filesystem.createDocumentInputStream("VisioDocument").read(contents);
+	}
+
+	/**
+	 * Test creating the trailer, but not looking for children
+	 */
+	public void testTrailer() {
+		// Find the trailer
+		Pointer trailerPtr = ptrFactory.createPointer(contents, trailerPointerAt);
+		
+		assertEquals(20, trailerPtr.getType());
+		assertEquals(trailerDataAt, trailerPtr.getOffset());
+		
+		Stream stream = Stream.createStream(trailerPtr, contents, ptrFactory);
+		assertTrue(stream instanceof TrailerStream);
+		TrailerStream ts = (TrailerStream)stream;
+		
+		assertNotNull(ts.getChildPointers());
+		assertNull(ts.getPointedToStreams());
+		
+		assertEquals(20, ts.getChildPointers().length);
+		assertEquals(0x16, ts.getChildPointers()[0].getType());
+		assertEquals(0x17, ts.getChildPointers()[1].getType());
+		assertEquals(0x17, ts.getChildPointers()[2].getType());
+		assertEquals(0xff, ts.getChildPointers()[3].getType());
+	}
+	
+	public void testStrings() {
+		Pointer trailerPtr = ptrFactory.createPointer(contents, trailerPointerAt);
+		TrailerStream ts = (TrailerStream)
+			Stream.createStream(trailerPtr, contents, ptrFactory);
+		
+		// Should be the 1st one
+		Pointer stringPtr = ts.getChildPointers()[0];
+		assertTrue(stringPtr.destinationHasStrings());
+		assertFalse(stringPtr.destinationHasChunks());
+		assertFalse(stringPtr.destinationHasPointers());
+		
+		Stream stream = Stream.createStream(stringPtr, contents, ptrFactory);
+		assertNotNull(stream);
+		assertTrue(stream instanceof StringsStream);
+	}
+	
+	public void testPointerToStrings() {
+		// The stream at 0x347f has strings
+		// The stream at 0x4312 has a pointer to 0x347f
+		// The stream at 0x44d3 has a pointer to 0x4312
+		//  (it's the 2nd one of 3, and the block is compressed)
+		
+		TestPointer ptr44d3 = new TestPointer(true, 0x44d3, 0x51, 0x4e, (short)0x56);
+		ptr44d3.hasPointers = true;
+		PointerContainingStream s44d3 = (PointerContainingStream)
+			Stream.createStream(ptr44d3, contents, ptrFactory);
+		
+		// Type: 0d  Addr: 014ff644  Offset: 4312  Len: 48  Format: 54  From: 44d3
+		Pointer ptr4312 = s44d3.getChildPointers()[1];
+		assertEquals(0x0d, ptr4312.getType());
+		assertEquals(0x4312, ptr4312.getOffset());
+		assertEquals(0x48, ptr4312.getLength());
+		assertEquals(0x54, ptr4312.getFormat());
+		assertTrue(ptr4312.destinationHasPointers());
+		assertFalse(ptr4312.destinationHasStrings());
+		
+		PointerContainingStream s4312 = (PointerContainingStream)
+			Stream.createStream(ptr4312, contents, ptrFactory);
+		
+		// Check it has 0x347f
+		// Type: 1f  Addr: 01540004  Offset: 347f  Len: 8e8  Format: 46  From: 4312
+		assertEquals(2, s4312.getChildPointers().length);
+		Pointer ptr347f = s4312.getChildPointers()[0];
+		assertEquals(0x1f, ptr347f.getType());
+		assertEquals(0x347f, ptr347f.getOffset());
+		assertEquals(0x8e8, ptr347f.getLength());
+		assertEquals(0x46, ptr347f.getFormat());
+		assertFalse(ptr347f.destinationHasPointers());
+		assertTrue(ptr347f.destinationHasStrings());
+		
+		// Find the children of 0x4312
+		assertNull(s4312.getPointedToStreams());
+		s4312.findChildren(contents);
+		// Should have two, both strings
+		assertNotNull(s4312.getPointedToStreams());
+		assertEquals(2, s4312.getPointedToStreams().length);
+		assertTrue(s4312.getPointedToStreams()[0] instanceof StringsStream);
+		assertTrue(s4312.getPointedToStreams()[1] instanceof StringsStream);
+	}
+	
+	public void testTrailerContents() {
+		Pointer trailerPtr = ptrFactory.createPointer(contents, trailerPointerAt);
+		TrailerStream ts = (TrailerStream)
+			Stream.createStream(trailerPtr, contents, ptrFactory);
+		
+		assertNotNull(ts.getChildPointers());
+		assertNull(ts.getPointedToStreams());
+		assertEquals(20, ts.getChildPointers().length);
+		
+		ts.findChildren(contents);
+		
+		assertNotNull(ts.getChildPointers());
+		assertNotNull(ts.getPointedToStreams());
+		assertEquals(20, ts.getChildPointers().length);
+		assertEquals(20, ts.getPointedToStreams().length);
+		
+		// Step down:
+		// 8 -> 4 -> 5 -> 1 -> 0 == String
+		assertNotNull(ts.getPointedToStreams()[8]);
+		assertTrue(ts.getPointedToStreams()[8] instanceof PointerContainingStream);
+		
+		PointerContainingStream s8 = 
+			(PointerContainingStream)ts.getPointedToStreams()[8];
+		assertNotNull(s8.getPointedToStreams());
+		
+		assertNotNull(s8.getPointedToStreams()[4]);
+		assertTrue(s8.getPointedToStreams()[4] instanceof PointerContainingStream);
+		
+		PointerContainingStream s84 = 
+			(PointerContainingStream)s8.getPointedToStreams()[4];
+		assertNotNull(s84.getPointedToStreams());
+		
+		assertNotNull(s84.getPointedToStreams()[5]);
+		assertTrue(s84.getPointedToStreams()[5] instanceof PointerContainingStream);
+		
+		PointerContainingStream s845 = 
+			(PointerContainingStream)s84.getPointedToStreams()[5];
+		assertNotNull(s845.getPointedToStreams());
+		
+		assertNotNull(s845.getPointedToStreams()[1]);
+		assertTrue(s845.getPointedToStreams()[1] instanceof PointerContainingStream);
+		
+		PointerContainingStream s8451 = 
+			(PointerContainingStream)s845.getPointedToStreams()[1];
+		assertNotNull(s8451.getPointedToStreams());
+		
+		assertNotNull(s8451.getPointedToStreams()[0]);
+		assertTrue(s8451.getPointedToStreams()[0] instanceof StringsStream);
+		assertTrue(s8451.getPointedToStreams()[1] instanceof StringsStream);
+	}
+}