Initial NPOIFS low level stream reader and writer. Still needs unit tests, but should allow reading and writing to a basic stream of (big) blocks

git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1051795 13f79535-47bb-0310-9956-ffa450edef68

src/java/org/apache/poi/poifs/filesystem/NPOIFSFileSystem.java

@@ -56,6 +56,7 @@ import org.apache.poi.poifs.storage.HeaderBlockWriter;
 import org.apache.poi.poifs.storage.RawDataBlockList;
 import org.apache.poi.poifs.storage.SmallBlockTableReader;
 import org.apache.poi.poifs.storage.SmallBlockTableWriter;
+import org.apache.poi.poifs.storage.BATBlock.BATBlockAndIndex;
 import org.apache.poi.util.CloseIgnoringInputStream;
 import org.apache.poi.util.IOUtils;
 import org.apache.poi.util.LongField;
@@ -81,10 +82,10 @@ public class NPOIFSFileSystem
        return new CloseIgnoringInputStream(is);
     }
    
-    private PropertyTable _property_table;
+    private PropertyTable  _property_table;
-	 private List<BATBlock> _blocks;
+	 private List<BATBlock> _bat_blocks;
-    private HeaderBlock   _header;
+    private HeaderBlock    _header;
-    private DirectoryNode _root;
+    private DirectoryNode  _root;
     
     private DataSource _data;
     
@@ -104,7 +105,7 @@ public class NPOIFSFileSystem
     {
         _header         = new HeaderBlock(bigBlockSize);
         _property_table = new PropertyTable(_header);// TODO Needs correct type
-        _blocks         = new ArrayList<BATBlock>();
+        _bat_blocks     = new ArrayList<BATBlock>();
         _root           = null;
     }
 
@@ -187,11 +188,7 @@ public class NPOIFSFileSystem
            // We need to buffer the whole file into memory when
            //  working with an InputStream.
            // The max possible size is when each BAT block entry is used
-           int maxSize = 
+           int maxSize = BATBlock.calculateMaximumSize(_header); 
-                 _header.getBATCount() * 
-                 _header.getBigBlockSize().getBATEntriesPerBlock() *
-                 _header.getBigBlockSize().getBigBlockSize()
-           ;
            ByteBuffer data = ByteBuffer.allocate(maxSize);
            // Copy in the header
            data.put(headerBuffer);
@@ -275,7 +272,7 @@ public class NPOIFSFileSystem
        for(int fatAt : _header.getBATArray()) {
           loopDetector.claim(fatAt);
           ByteBuffer fatData = getBlockAt(fatAt);
-          _blocks.add(BATBlock.createBATBlock(bigBlockSize, fatData));
+          _bat_blocks.add(BATBlock.createBATBlock(bigBlockSize, fatData));
        }
        
        // Now read the XFAT blocks
@@ -287,7 +284,7 @@ public class NPOIFSFileSystem
           xfat = BATBlock.createBATBlock(bigBlockSize, fatData);
           nextAt = xfat.getValueAt(bigBlockSize.getNextXBATChainOffset());
           
-          _blocks.add(xfat);
+          _bat_blocks.add(xfat);
        }
        
        // We're now able to load steams
@@ -305,10 +302,41 @@ public class NPOIFSFileSystem
        long startAt = (offset+1) * bigBlockSize.getBigBlockSize();
        return _data.read(bigBlockSize.getBigBlockSize(), startAt);
     }
+    
+    /**
+     * Returns the BATBlock that handles the specified offset,
+     *  and the relative index within it
+     */
+    protected BATBlockAndIndex getBATBlockAndIndex(final int offset) {
+       return BATBlock.getBATBlockAndIndex(
+             offset, _header, _bat_blocks
+       );
+    }
+    
     /**
      * Works out what block follows the specified one.
      */
     protected int getNextBlock(final int offset) {
+       BATBlockAndIndex bai = getBATBlockAndIndex(offset);
+       return bai.getBlock().getValueAt( bai.getIndex() );
+    }
+    
+    /**
+     * Changes the record of what block follows the specified one.
+     */
+    protected void setNextBlock(final int offset, final int nextBlock) {
+       BATBlockAndIndex bai = getBATBlockAndIndex(offset);
+       bai.getBlock().setValueAt(
+             bai.getIndex(), nextBlock
+       );
+    }
+    
+    /**
+     * Finds a free block, and returns its offset.
+     * This method will extend the file if needed, and if doing
+     *  so, allocate new FAT blocks to address the extra space.
+     */
+    protected int getFreeBlock() {
        // TODO
        return -1;
     }

src/java/org/apache/poi/poifs/filesystem/NPOIFSStream.java

@@ -0,0 +1,172 @@
+
+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
+
+
+package org.apache.poi.poifs.filesystem;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+
+import org.apache.poi.poifs.common.POIFSConstants;
+import org.apache.poi.poifs.property.Property;
+import org.apache.poi.poifs.storage.HeaderBlock;
+
+/**
+ * This handles reading and writing a stream within a
+ *  {@link NPOIFSFileSystem}. It can supply an iterator
+ *  to read blocks, and way to write out to existing and
+ *  new blocks.
+ * Most users will want a higher level version of this, 
+ *  which deals with properties to track which stream
+ *  this is.
+ * This only works on big block streams, it doesn't
+ *  handle small block ones.
+ * This uses the new NIO code
+ */
+
+public class NPOIFSStream implements Iterable<ByteBuffer>
+{
+	private NPOIFSFileSystem filesystem;
+	private int startBlock;
+	
+	/**
+	 * Constructor for an existing stream. It's up to you
+	 *  to know how to get the start block (eg from a 
+	 *  {@link HeaderBlock} or a {@link Property}) 
+	 */
+	public NPOIFSStream(NPOIFSFileSystem filesystem, int startBlock) {
+	   this.filesystem = filesystem;
+	   this.startBlock = startBlock;
+	}
+	
+	/**
+	 * Constructor for a new stream. A start block won't
+	 *  be allocated until you begin writing to it.
+	 */
+	public NPOIFSStream(NPOIFSFileSystem filesystem) {
+	   this.filesystem = filesystem;
+	   this.startBlock = POIFSConstants.END_OF_CHAIN;
+	}
+	
+	/**
+	 * What block does this stream start at?
+	 * Will be {@link POIFSConstants#END_OF_CHAIN} for a
+	 *  new stream that hasn't been written to yet.
+	 */
+	public int getStartBlock() {
+	   return startBlock;
+	}
+
+	/**
+	 * Returns an iterator that'll supply one {@link ByteBuffer}
+	 *  per block in the stream.
+	 */
+   public Iterator<ByteBuffer> iterator() {
+      return getBlockIterator();
+   }
+	
+   public Iterator<ByteBuffer> getBlockIterator() {
+      if(startBlock == POIFSConstants.END_OF_CHAIN) {
+         throw new IllegalStateException(
+               "Can't read from a new stream before it has been written to"
+         );
+      }
+      return new StreamBlockByteBufferIterator(startBlock);
+   }
+   
+   /**
+    * Updates the contents of the stream to the new
+    *  set of bytes.
+    * Note - if this is property based, you'll still
+    *  need to 
+    */
+   public void updateContents(byte[] contents) throws IOException {
+      // How many blocks are we going to need?
+      int blocks = (int)Math.ceil(contents.length / filesystem.getBigBlockSize());
+      
+      // Start writing
+      int prevBlock = POIFSConstants.END_OF_CHAIN;
+      int nextBlock = startBlock;
+      for(int i=0; i<blocks; i++) {
+         int thisBlock = nextBlock;
+         
+         // Allocate a block if needed, otherwise figure
+         //  out what the next block will be
+         if(thisBlock == POIFSConstants.END_OF_CHAIN) {
+            thisBlock = filesystem.getFreeBlock();
+            nextBlock = POIFSConstants.END_OF_CHAIN;
+            
+            // Mark the previous block as carrying on
+            if(prevBlock != POIFSConstants.END_OF_CHAIN) {
+               filesystem.setNextBlock(prevBlock, thisBlock);
+            }
+         } else {
+            nextBlock = filesystem.getNextBlock(thisBlock);
+         }
+         
+         // Write it
+         ByteBuffer buffer = filesystem.getBlockAt(thisBlock);
+         buffer.put(contents, i*filesystem.getBigBlockSize(), filesystem.getBigBlockSize());
+         
+         // Update pointers
+         prevBlock = thisBlock;
+      }
+      
+      // Mark the end of the stream
+      filesystem.setNextBlock(nextBlock, POIFSConstants.END_OF_CHAIN);
+   }
+   
+   // TODO Streaming write too
+   
+   /**
+    * Class that handles a streaming read of one stream
+    */
+   protected class StreamBlockByteBufferIterator implements Iterator<ByteBuffer> {
+      private int nextBlock;
+      protected StreamBlockByteBufferIterator(int firstBlock) {
+         nextBlock = firstBlock;
+      }
+
+      public boolean hasNext() {
+         if(nextBlock == POIFSConstants.END_OF_CHAIN) {
+            return false;
+         }
+         return true;
+      }
+
+      public ByteBuffer next() {
+         if(nextBlock == POIFSConstants.END_OF_CHAIN) {
+            throw new IndexOutOfBoundsException("Can't read past the end of the stream");
+         }
+         
+         try {
+            ByteBuffer data = filesystem.getBlockAt(nextBlock);
+            nextBlock = filesystem.getNextBlock(nextBlock);
+            return data;
+         } catch(IOException e) {
+            throw new RuntimeException(e);
+         }
+      }
+
+      public void remove() {
+         throw new UnsupportedOperationException();
+      }
+   }
+}
+

src/testcases/org/apache/poi/poifs/filesystem/TestNPOIFSFileSystem.java

@@ -17,20 +17,10 @@
 
 package org.apache.poi.poifs.filesystem;
 
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Iterator;
-
 import junit.framework.TestCase;
 
 import org.apache.poi.POIDataSamples;
-import org.apache.poi.hssf.HSSFTestDataSamples;
+import org.apache.poi.poifs.common.POIFSConstants;
-import org.apache.poi.poifs.common.POIFSBigBlockSize;
-import org.apache.poi.poifs.storage.HeaderBlock;
-import org.apache.poi.poifs.storage.RawDataBlockList;
 
 /**
  * Tests for the new NIO POIFSFileSystem implementation
@@ -81,4 +71,51 @@ public final class TestNPOIFSFileSystem extends TestCase {
          // TODO
       }
    }
+   
+   /**
+    * Check that for a given block, we can correctly figure
+    *  out what the next one is
+    */
+   public void testNextBlock() throws Exception {
+      NPOIFSFileSystem fs = new NPOIFSFileSystem(_inst.getFile("BlockSize512.zvi"));
+      
+      // 0 -> 21 are simple
+      for(int i=0; i<21; i++) {
+         assertEquals(i+1, fs.getNextBlock(i));
+      }
+      // 21 jumps to 89, then ends
+      assertEquals(89, fs.getNextBlock(21));
+      assertEquals(POIFSConstants.END_OF_CHAIN, fs.getNextBlock(89));
+      
+      // 22 -> 88 simple sequential stream
+      for(int i=22; i<88; i++) {
+         assertEquals(i+1, fs.getNextBlock(i));
+      }
+      assertEquals(POIFSConstants.END_OF_CHAIN, fs.getNextBlock(88));
+      
+      // 90 -> 96 is another stream
+      for(int i=90; i<96; i++) {
+         assertEquals(i+1, fs.getNextBlock(i));
+      }
+      assertEquals(POIFSConstants.END_OF_CHAIN, fs.getNextBlock(96));
+      
+      // 97+98 is another
+      assertEquals(98, fs.getNextBlock(97));
+      assertEquals(POIFSConstants.END_OF_CHAIN, fs.getNextBlock(98));
+      
+      // 99 is our FAT block
+      assertEquals(POIFSConstants.FAT_SECTOR_BLOCK, fs.getNextBlock(99));
+      
+      // 100 onwards is free
+      for(int i=100; i<fs.getBigBlockSizeDetails().getBATEntriesPerBlock(); i++) {
+         assertEquals(POIFSConstants.UNUSED_BLOCK, fs.getNextBlock(i));
+      }
+   }
+
+   /**
+    * Check we get the right data back for each block
+    */
+   public void testGetBlock() throws Exception {
+      // TODO
+   }
 }