moparisthebest
/
poi
1
0
Fork 0
Code Issues Pull Requests Releases 1 Wiki Activity

PR: 19974 

Obtained from:
Submitted by:
Reviewed by:

This patch fixes writeing VERY long > 8228 character strings. Existing SST serialisation had problem with calculation and writing to multiple continue records.

Also fixed some test case errors introduced by my last patch for DBCell and IndexRecords


git-svn-id: https://svn.apache.org/repos/asf/jakarta/poi/trunk@353607 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jason Height 2004-10-13 03:37:56 +00:00
parent dbacc2b8f3
commit 6ab6d0a295
4 changed files with 81 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/java/org/apache/poi/hssf/model/Sheet.java
View File

@ -763,7 +763,7 @@ public class Sheet implements Model
if (record.getSid() == BOFRecord.sid) {
//Can there be more than one BOF for a sheet? If not then we can
//remove this guard. So be safe it is left here.
if (!haveSerializedIndex) {
if ((rows != null) && (!haveSerializedIndex)) {
haveSerializedIndex = true;
pos += serializeIndexRecord(k, pos, data);
}
@ -2029,6 +2029,7 @@ public class Sheet implements Model
retval += (( Record ) records.get(k)).getRecordSize();
}
//Add space for the IndexRecord
if (rows != null) {
final int blocks = rows.getRowBlockCount();
retval += IndexRecord.getRecordSizeForBlockCount(blocks);
@ -2043,6 +2044,7 @@ public class Sheet implements Model
if (cells.rowHasCells(row.getRowNumber()))
retval += 2;
}
}
return retval;
}

229
src/java/org/apache/poi/hssf/record/SSTRecordSizeCalculator.java
View File

@ -30,6 +30,7 @@ import java.util.Map;
* the SST serialization code needs to be rewritten.
*
* @author Glen Stampoultzis (glens at apache.org)
* @author Jason Height (jheight at apache.org)
*/
class SSTRecordSizeCalculator
{
@ -52,180 +53,76 @@ class SSTRecordSizeCalculator
this.strings = strings;
}
/**
* Calculate the size in bytes of the SST record. This will include continue
* records.
*
* @return the size of the SST record.
*/
public int getRecordSize()
{
initVars();
private boolean canFitStringInRecord(int recordLength) {
return (recordLength+SSTRecord.STRING_MINIMAL_OVERHEAD) < SSTRecord.MAX_RECORD_SIZE;
}
int retval;
int totalStringSpaceRequired = SSTSerializer.calculateUnicodeSize(strings);
public int getRecordSize() {
//Indicates how much of the current base or continue record has
//been written
int continueSize = SSTRecord.SST_RECORD_OVERHEAD;
int recordSize = 0;
for (int i=0; i < strings.size(); i++ )
{
Integer intunipos = new Integer(i);
UnicodeString unistr = ( (UnicodeString) strings.get(intunipos));
final int stringLength = unistr.getRecordSize();
if ((continueSize + stringLength) <= SSTRecord.MAX_RECORD_SIZE) {
//String can fit within the bounds of the current record (SST or Continue)
continueSize += stringLength;
if ((i < (strings.size()-1)) && !canFitStringInRecord(continueSize)) {
//Start new continueRecord if there is another string
recordLengths.add(new Integer(continueSize));
recordSize += continueSize;
//Minimum ammount of space for a new continue record.
continueSize = 4;
}
} else {
int stringRemainder = stringLength;
while (stringRemainder != 0) {
if ( (continueSize + stringRemainder) > SSTRecord.MAX_RECORD_SIZE) {
//Determine number of bytes that can be written in the space
//available
int bytesWritten = Math.min((SSTRecord.MAX_RECORD_SIZE - continueSize), stringRemainder);
if ( totalStringSpaceRequired > SSTRecord.MAX_DATA_SPACE )
{
retval = sizeOverContinuation( totalStringSpaceRequired );
//Ensure that the Unicode String writes both the high and low
//byte in the one action. Since the string overhead is 3 bytes
//if the bytes that can be written is even, then we need to
//write one less byte to capture both the high and low bytes.
bytesWritten = unistr.maxBrokenLength(bytesWritten);
continueSize += bytesWritten;
stringRemainder -= bytesWritten;
recordLengths.add(new Integer(continueSize));
recordSize += continueSize;
//Minimum ammount of space for a new continue record.
continueSize = 4;
//Add one to the size of the string that is remaining, since the
//first byte for the next continue record will be compressed unicode indicator
stringRemainder++;
} else {
//Remainder of string can fit within the bounds of the current
//continue record
continueSize += stringRemainder;
stringRemainder = 0;
if ((i < (strings.size()-1)) && !canFitStringInRecord(continueSize)) {
//Start new continueRecord if there is another string
recordLengths.add(new Integer(continueSize));
recordSize += continueSize;
//Minimum ammount of space for a new continue record.
continueSize = 4;
}
else
{
// short data: write one simple SST record
retval = SSTRecord.SST_RECORD_OVERHEAD + totalStringSpaceRequired;
recordLengths.add( new Integer( totalStringSpaceRequired ) );
}
}
return retval;
}
}
recordLengths.add(new Integer(continueSize));
recordSize += continueSize;
return recordSize;
}
public List getRecordLengths()
{
return recordLengths;
}
private int sizeOverContinuation( int totalStringSpaceRequired )
{
int retval;
while ( !finished )
{
recordSize = 0;
pos = 0;
if ( firstRecord )
{
addMaxLengthRecordSize();
}
else
{
// writing continue record
pos = 0;
int toBeWritten = ( totalStringSpaceRequired - totalBytesWritten ) + ( isRemainingString ? 1 : 0 );
int size = Math.min( SSTRecord.MAX_RECORD_SIZE - SSTRecord.STD_RECORD_OVERHEAD, toBeWritten );
if ( size == toBeWritten )
{
finished = true;
}
recordSize = size + SSTRecord.STD_RECORD_OVERHEAD;
recordLengths.add( new Integer( size ) );
pos = 4;
}
if ( isRemainingString )
{
calcReminant();
}
calcRemainingStrings();
totalWritten += recordSize;
}
retval = totalWritten;
return retval;
}
private void addMaxLengthRecordSize()
{
// writing SST record
recordSize = SSTRecord.MAX_RECORD_SIZE;
pos = 12;
firstRecord = false;
recordLengths.add( new Integer( recordSize - SSTRecord.STD_RECORD_OVERHEAD ) );
}
private void calcRemainingStrings()
{
for ( ; unipos < strings.size(); unipos++ )
{
int available = SSTRecord.MAX_RECORD_SIZE - pos;
Integer intunipos = new Integer( unipos );
unistr = ( (UnicodeString) strings.get( intunipos ) );
if ( unistr.getRecordSize() <= available )
{
totalBytesWritten += unistr.getRecordSize();
pos += unistr.getRecordSize();
}
else
{
if ( available >= SSTRecord.STRING_MINIMAL_OVERHEAD )
{
int toBeWritten =
unistr.maxBrokenLength( available );
totalBytesWritten += toBeWritten;
stringReminant =
( unistr.getRecordSize() - toBeWritten )
+ LittleEndianConsts.BYTE_SIZE;
if ( available != toBeWritten )
{
int shortrecord = recordSize
- ( available - toBeWritten );
recordLengths.set(
recordLengths.size() - 1,
new Integer(
shortrecord - SSTRecord.STD_RECORD_OVERHEAD ) );
recordSize = shortrecord;
}
isRemainingString = true;
unipos++;
}
else
{
int shortrecord = recordSize - available;
recordLengths.set( recordLengths.size() - 1,
new Integer( shortrecord - SSTRecord.STD_RECORD_OVERHEAD ) );
recordSize = shortrecord;
}
break;
}
}
}
private void calcReminant()
{
int available = SSTRecord.MAX_RECORD_SIZE - pos;
if ( stringReminant <= available )
{
// write reminant
totalBytesWritten += stringReminant - 1;
pos += stringReminant;
isRemainingString = false;
}
else
{
// write as much of the remnant as possible
int toBeWritten = unistr.maxBrokenLength( available );
if ( available != toBeWritten )
{
int shortrecord = recordSize - ( available - toBeWritten );
recordLengths.set( recordLengths.size() - 1,
new Integer( shortrecord - SSTRecord.STD_RECORD_OVERHEAD ) );
recordSize = shortrecord;
}
totalBytesWritten += toBeWritten - 1;
pos += toBeWritten;
stringReminant -= toBeWritten - 1;
isRemainingString = true;
}
}
private void initVars()
{
unistr = null;
stringReminant = 0;
unipos = 0;
isRemainingString = false;
totalBytesWritten = 0;
finished = false;
firstRecord = true;
totalWritten = 0;
}
}

18
src/java/org/apache/poi/hssf/record/SSTSerializer.java
View File

@ -111,7 +111,7 @@ class SSTSerializer
private void serializeSingleSSTRecord( byte[] data, int offset, int record_length_index )
{
int len = ( (Integer) recordLengths.get( record_length_index ) ).intValue();
int recordSize = SSTRecord.SST_RECORD_OVERHEAD + len - SSTRecord.STD_RECORD_OVERHEAD;
int recordSize = len - SSTRecord.STD_RECORD_OVERHEAD;
sstRecordHeader.writeSSTHeader( data, 0 + offset, recordSize );
int pos = SSTRecord.SST_RECORD_OVERHEAD;
@ -149,13 +149,16 @@ class SSTSerializer
while ( totalWritten != record_size )
{
int recordLength = ( (Integer) recordLengths.get( record_length_index++ ) ).intValue();
//Total record length, including excel record header and sst/continue header
final int recordLength = ( (Integer) recordLengths.get( record_length_index++ ) ).intValue();
//Total available data length (minus the excel record header size)
final int recordDataLength = recordLength - 4;
RecordProcessor recordProcessor = new RecordProcessor( buffer,
recordLength, numStrings, numUniqueStrings );
recordDataLength, numStrings, numUniqueStrings );
// write the appropriate header
startOfRecord = offset + totalWritten;
recordProcessor.writeRecordHeader( offset, totalWritten, recordLength, first_record );
recordProcessor.writeRecordHeader( offset, totalWritten, recordDataLength, first_record );
first_record = false;
// now, write the rest of the data into the current
@ -166,6 +169,11 @@ class SSTSerializer
// the last string in the previous record was not written out completely
stringReminant = recordProcessor.writeStringRemainder( lastneedcontinue,
stringReminant, offset, totalWritten );
//Check to see if still not written out completely
if (lastneedcontinue) {
totalWritten += recordLength;
continue;
}
}
// last string's remnant, if any, is cleaned up as best as can be done ... now let's try and write
@ -204,7 +212,7 @@ class SSTSerializer
break;
}
}
totalWritten += recordLength + SSTRecord.STD_RECORD_OVERHEAD;
totalWritten += recordLength;
}
}

3
src/testcases/org/apache/poi/hssf/record/aggregates/TestValueRecordsAggregate.java
View File

@ -161,7 +161,8 @@ public class TestValueRecordsAggregate extends TestCase
};
List records = testData();
valueRecord.construct( 0, records );
int bytesWritten = valueRecord.serialize( 0, actualArray );
int bytesWritten = valueRecord.serializeCellRow(1, 0, actualArray );
bytesWritten += valueRecord.serializeCellRow(2, bytesWritten, actualArray );
assertEquals( 36, bytesWritten );
for (int i = 0; i < 36; i++)
assertEquals( expectedArray[i], actualArray[i] );