PR: 19974

Obtained from:
Submitted by:
Reviewed by:

This patch fixes writeing VERY long > 8228 character strings. Existing SST serialisation had problem with calculation and writing to multiple continue records.

Also fixed some test case errors introduced by my last patch for DBCell and IndexRecords


git-svn-id: https://svn.apache.org/repos/asf/jakarta/poi/trunk@353607 13f79535-47bb-0310-9956-ffa450edef68

src/java/org/apache/poi/hssf/model/Sheet.java

@@ -763,7 +763,7 @@ public class Sheet implements Model
             if (record.getSid() == BOFRecord.sid) {
               //Can there be more than one BOF for a sheet? If not then we can
               //remove this guard. So be safe it is left here.
-              if (!haveSerializedIndex) {
+              if ((rows != null) && (!haveSerializedIndex)) {
                 haveSerializedIndex = true;
                 pos += serializeIndexRecord(k, pos, data);
               }
@@ -2029,6 +2029,7 @@ public class Sheet implements Model
             retval += (( Record ) records.get(k)).getRecordSize();
         }
         //Add space for the IndexRecord
+        if (rows != null) {
         final int blocks = rows.getRowBlockCount();
         retval += IndexRecord.getRecordSizeForBlockCount(blocks);
 
@@ -2043,6 +2044,7 @@ public class Sheet implements Model
           if (cells.rowHasCells(row.getRowNumber()))
             retval += 2;
         }
+        }
         return retval;
     }
 

src/java/org/apache/poi/hssf/record/SSTRecordSizeCalculator.java

@@ -30,6 +30,7 @@ import java.util.Map;
  * the SST serialization code needs to be rewritten.
  *
  * @author Glen Stampoultzis (glens at apache.org)
+ * @author Jason Height (jheight at apache.org)
  */
 class SSTRecordSizeCalculator
 {
@@ -52,180 +53,76 @@ class SSTRecordSizeCalculator
         this.strings = strings;
     }
 
-    /**
+    private boolean canFitStringInRecord(int recordLength) {
-     * Calculate the size in bytes of the SST record.  This will include continue
+      return (recordLength+SSTRecord.STRING_MINIMAL_OVERHEAD) < SSTRecord.MAX_RECORD_SIZE;
-     * records.
+                    }
-     *
-     * @return the size of the SST record.
-     */
-    public int getRecordSize()
-    {
-        initVars();
 
-        int retval;
+    public int getRecordSize() {
-        int totalStringSpaceRequired = SSTSerializer.calculateUnicodeSize(strings);
+       //Indicates how much of the current base or continue record has
+       //been written
+       int continueSize = SSTRecord.SST_RECORD_OVERHEAD;
+       int recordSize = 0;
+        for (int i=0; i < strings.size(); i++ )
+        {
+          Integer intunipos = new Integer(i);    
+          UnicodeString unistr = ( (UnicodeString) strings.get(intunipos));
+          final int stringLength = unistr.getRecordSize();
+          if ((continueSize + stringLength) <= SSTRecord.MAX_RECORD_SIZE) {
+            //String can fit within the bounds of the current record (SST or Continue)
+            continueSize += stringLength;
             
-        if ( totalStringSpaceRequired > SSTRecord.MAX_DATA_SPACE )
+            if ((i < (strings.size()-1)) && !canFitStringInRecord(continueSize)) {
-        {
+              //Start new continueRecord if there is another string              
-            retval = sizeOverContinuation( totalStringSpaceRequired );
+              recordLengths.add(new Integer(continueSize));
+              recordSize += continueSize;
+              //Minimum ammount of space for a new continue record.
+              continueSize = 4;   
+            }
+          } else {
+            int stringRemainder = stringLength;
+            while (stringRemainder != 0) {              
+              if ( (continueSize + stringRemainder) > SSTRecord.MAX_RECORD_SIZE) {
+                //Determine number of bytes that can be written in the space
+                //available
+                int bytesWritten = Math.min((SSTRecord.MAX_RECORD_SIZE - continueSize), stringRemainder);
+
+                //Ensure that the Unicode String writes both the high and low
+                //byte in the one action. Since the string overhead is 3 bytes
+                //if the bytes that can be written is even, then we need to 
+                //write one less byte to capture both the high and low bytes.
+                bytesWritten = unistr.maxBrokenLength(bytesWritten);
+                continueSize += bytesWritten;
+                stringRemainder -= bytesWritten;
+                recordLengths.add(new Integer(continueSize));
+                recordSize += continueSize;
+                //Minimum ammount of space for a new continue record.
+                continueSize = 4;
+                //Add one to the size of the string that is remaining, since the
+                //first byte for the next continue record will be compressed unicode indicator
+                stringRemainder++;
+              } else {
+                //Remainder of string can fit within the bounds of the current
+                //continue record
+                continueSize += stringRemainder;
+                stringRemainder = 0;
+                if ((i < (strings.size()-1)) && !canFitStringInRecord(continueSize)) {
+                  //Start new continueRecord if there is another string
+                  recordLengths.add(new Integer(continueSize));
+                  recordSize += continueSize;
+                  //Minimum ammount of space for a new continue record.
+                  continueSize = 4;          
         }
-        else
+    }
-        {
-            // short data: write one simple SST record
-            retval = SSTRecord.SST_RECORD_OVERHEAD + totalStringSpaceRequired;
-            recordLengths.add( new Integer( totalStringSpaceRequired ) );
         }
-        return retval;
+            }
+        }
+        recordLengths.add(new Integer(continueSize));
+        recordSize += continueSize;        
+        return recordSize;
     }
 
     public List getRecordLengths()
     {
         return recordLengths;
     }
-
-    private int sizeOverContinuation( int totalStringSpaceRequired )
-    {
-        int retval;
-
-        while ( !finished )
-        {
-            recordSize = 0;
-            pos = 0;
-
-            if ( firstRecord )
-            {
-                addMaxLengthRecordSize();
-            }
-            else
-            {
-
-                // writing continue record
-                pos = 0;
-                int toBeWritten = ( totalStringSpaceRequired - totalBytesWritten ) + ( isRemainingString ? 1 : 0 );
-                int size = Math.min( SSTRecord.MAX_RECORD_SIZE - SSTRecord.STD_RECORD_OVERHEAD, toBeWritten );
-
-                if ( size == toBeWritten )
-                {
-                    finished = true;
-                }
-                recordSize = size + SSTRecord.STD_RECORD_OVERHEAD;
-                recordLengths.add( new Integer( size ) );
-                pos = 4;
-            }
-            if ( isRemainingString )
-            {
-                calcReminant();
-            }
-            calcRemainingStrings();
-            totalWritten += recordSize;
-        }
-        retval = totalWritten;
-
-        return retval;
-    }
-
-    private void addMaxLengthRecordSize()
-    {
-        // writing SST record
-        recordSize = SSTRecord.MAX_RECORD_SIZE;
-        pos = 12;
-        firstRecord = false;
-        recordLengths.add( new Integer( recordSize - SSTRecord.STD_RECORD_OVERHEAD ) );
-    }
-
-    private void calcRemainingStrings()
-    {
-        for ( ; unipos < strings.size(); unipos++ )
-        {
-            int available = SSTRecord.MAX_RECORD_SIZE - pos;
-            Integer intunipos = new Integer( unipos );
-
-            unistr = ( (UnicodeString) strings.get( intunipos ) );
-            if ( unistr.getRecordSize() <= available )
-            {
-                totalBytesWritten += unistr.getRecordSize();
-                pos += unistr.getRecordSize();
-            }
-            else
-            {
-                if ( available >= SSTRecord.STRING_MINIMAL_OVERHEAD )
-                {
-                    int toBeWritten =
-                            unistr.maxBrokenLength( available );
-
-                    totalBytesWritten += toBeWritten;
-                    stringReminant =
-                            ( unistr.getRecordSize() - toBeWritten )
-                            + LittleEndianConsts.BYTE_SIZE;
-                    if ( available != toBeWritten )
-                    {
-                        int shortrecord = recordSize
-                                - ( available - toBeWritten );
-
-                        recordLengths.set(
-                                recordLengths.size() - 1,
-                                new Integer(
-                                        shortrecord - SSTRecord.STD_RECORD_OVERHEAD ) );
-                        recordSize = shortrecord;
-                    }
-                    isRemainingString = true;
-                    unipos++;
-                }
-                else
-                {
-                    int shortrecord = recordSize - available;
-
-                    recordLengths.set( recordLengths.size() - 1,
-                            new Integer( shortrecord - SSTRecord.STD_RECORD_OVERHEAD ) );
-                    recordSize = shortrecord;
-                }
-                break;
-            }
-        }
-    }
-
-    private void calcReminant()
-    {
-        int available = SSTRecord.MAX_RECORD_SIZE - pos;
-
-        if ( stringReminant <= available )
-        {
-
-            // write reminant
-            totalBytesWritten += stringReminant - 1;
-            pos += stringReminant;
-            isRemainingString = false;
-        }
-        else
-        {
-
-            // write as much of the remnant as possible
-            int toBeWritten = unistr.maxBrokenLength( available );
-
-            if ( available != toBeWritten )
-            {
-                int shortrecord = recordSize - ( available - toBeWritten );
-                recordLengths.set( recordLengths.size() - 1,
-                        new Integer( shortrecord - SSTRecord.STD_RECORD_OVERHEAD ) );
-                recordSize = shortrecord;
-            }
-            totalBytesWritten += toBeWritten - 1;
-            pos += toBeWritten;
-            stringReminant -= toBeWritten - 1;
-            isRemainingString = true;
-        }
-    }
-
-    private void initVars()
-    {
-        unistr = null;
-        stringReminant = 0;
-        unipos = 0;
-        isRemainingString = false;
-        totalBytesWritten = 0;
-        finished = false;
-        firstRecord = true;
-        totalWritten = 0;
-    }
-
 }

src/java/org/apache/poi/hssf/record/SSTSerializer.java

@@ -111,7 +111,7 @@ class SSTSerializer
     private void serializeSingleSSTRecord( byte[] data, int offset, int record_length_index )
     {
         int len = ( (Integer) recordLengths.get( record_length_index ) ).intValue();
-        int recordSize = SSTRecord.SST_RECORD_OVERHEAD + len - SSTRecord.STD_RECORD_OVERHEAD;
+        int recordSize = len - SSTRecord.STD_RECORD_OVERHEAD;
         sstRecordHeader.writeSSTHeader( data, 0 + offset, recordSize );
         int pos = SSTRecord.SST_RECORD_OVERHEAD;
 
@@ -149,13 +149,16 @@ class SSTSerializer
 
         while ( totalWritten != record_size )
         {
-            int recordLength = ( (Integer) recordLengths.get( record_length_index++ ) ).intValue();
+            //Total record length, including excel record header and sst/continue header
+            final int recordLength = ( (Integer) recordLengths.get( record_length_index++ ) ).intValue();
+            //Total available data length (minus the excel record header size)
+            final int recordDataLength = recordLength - 4;
             RecordProcessor recordProcessor = new RecordProcessor( buffer,
-                    recordLength, numStrings, numUniqueStrings );
+                    recordDataLength, numStrings, numUniqueStrings );
 
             // write the appropriate header
             startOfRecord = offset + totalWritten;
-            recordProcessor.writeRecordHeader( offset, totalWritten, recordLength, first_record );
+            recordProcessor.writeRecordHeader( offset, totalWritten, recordDataLength, first_record );
             first_record = false;
 
             // now, write the rest of the data into the current
@@ -166,6 +169,11 @@ class SSTSerializer
                 // the last string in the previous record was not written out completely
                 stringReminant = recordProcessor.writeStringRemainder( lastneedcontinue,
                         stringReminant, offset, totalWritten );
+                //Check to see if still not written out completely
+                if (lastneedcontinue) {
+                  totalWritten += recordLength;
+                  continue;
+                }
             }
 
             // last string's remnant, if any, is cleaned up as best as can be done ... now let's try and write
@@ -204,7 +212,7 @@ class SSTSerializer
                     break;
                 }
             }
-            totalWritten += recordLength + SSTRecord.STD_RECORD_OVERHEAD;
+            totalWritten += recordLength;
         }
     }
 

src/testcases/org/apache/poi/hssf/record/aggregates/TestValueRecordsAggregate.java

@@ -161,7 +161,8 @@ public class TestValueRecordsAggregate extends TestCase
         };
         List records = testData();
         valueRecord.construct( 0, records );
-        int bytesWritten = valueRecord.serialize( 0, actualArray );
+        int bytesWritten = valueRecord.serializeCellRow(1, 0, actualArray );
+        bytesWritten += valueRecord.serializeCellRow(2, bytesWritten, actualArray );
         assertEquals( 36, bytesWritten );
         for (int i = 0; i < 36; i++)
             assertEquals( expectedArray[i], actualArray[i] );