PR: 19974
Obtained from: Submitted by: Reviewed by: This patch fixes writeing VERY long > 8228 character strings. Existing SST serialisation had problem with calculation and writing to multiple continue records. Also fixed some test case errors introduced by my last patch for DBCell and IndexRecords git-svn-id: https://svn.apache.org/repos/asf/jakarta/poi/trunk@353607 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
dbacc2b8f3
commit
6ab6d0a295
@ -763,7 +763,7 @@ public class Sheet implements Model
|
||||
if (record.getSid() == BOFRecord.sid) {
|
||||
//Can there be more than one BOF for a sheet? If not then we can
|
||||
//remove this guard. So be safe it is left here.
|
||||
if (!haveSerializedIndex) {
|
||||
if ((rows != null) && (!haveSerializedIndex)) {
|
||||
haveSerializedIndex = true;
|
||||
pos += serializeIndexRecord(k, pos, data);
|
||||
}
|
||||
@ -2029,6 +2029,7 @@ public class Sheet implements Model
|
||||
retval += (( Record ) records.get(k)).getRecordSize();
|
||||
}
|
||||
//Add space for the IndexRecord
|
||||
if (rows != null) {
|
||||
final int blocks = rows.getRowBlockCount();
|
||||
retval += IndexRecord.getRecordSizeForBlockCount(blocks);
|
||||
|
||||
@ -2043,6 +2044,7 @@ public class Sheet implements Model
|
||||
if (cells.rowHasCells(row.getRowNumber()))
|
||||
retval += 2;
|
||||
}
|
||||
}
|
||||
return retval;
|
||||
}
|
||||
|
||||
|
@ -30,6 +30,7 @@ import java.util.Map;
|
||||
* the SST serialization code needs to be rewritten.
|
||||
*
|
||||
* @author Glen Stampoultzis (glens at apache.org)
|
||||
* @author Jason Height (jheight at apache.org)
|
||||
*/
|
||||
class SSTRecordSizeCalculator
|
||||
{
|
||||
@ -52,180 +53,76 @@ class SSTRecordSizeCalculator
|
||||
this.strings = strings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the size in bytes of the SST record. This will include continue
|
||||
* records.
|
||||
*
|
||||
* @return the size of the SST record.
|
||||
*/
|
||||
public int getRecordSize()
|
||||
{
|
||||
initVars();
|
||||
private boolean canFitStringInRecord(int recordLength) {
|
||||
return (recordLength+SSTRecord.STRING_MINIMAL_OVERHEAD) < SSTRecord.MAX_RECORD_SIZE;
|
||||
}
|
||||
|
||||
int retval;
|
||||
int totalStringSpaceRequired = SSTSerializer.calculateUnicodeSize(strings);
|
||||
public int getRecordSize() {
|
||||
//Indicates how much of the current base or continue record has
|
||||
//been written
|
||||
int continueSize = SSTRecord.SST_RECORD_OVERHEAD;
|
||||
int recordSize = 0;
|
||||
for (int i=0; i < strings.size(); i++ )
|
||||
{
|
||||
Integer intunipos = new Integer(i);
|
||||
UnicodeString unistr = ( (UnicodeString) strings.get(intunipos));
|
||||
final int stringLength = unistr.getRecordSize();
|
||||
if ((continueSize + stringLength) <= SSTRecord.MAX_RECORD_SIZE) {
|
||||
//String can fit within the bounds of the current record (SST or Continue)
|
||||
continueSize += stringLength;
|
||||
|
||||
if ((i < (strings.size()-1)) && !canFitStringInRecord(continueSize)) {
|
||||
//Start new continueRecord if there is another string
|
||||
recordLengths.add(new Integer(continueSize));
|
||||
recordSize += continueSize;
|
||||
//Minimum ammount of space for a new continue record.
|
||||
continueSize = 4;
|
||||
}
|
||||
} else {
|
||||
int stringRemainder = stringLength;
|
||||
while (stringRemainder != 0) {
|
||||
if ( (continueSize + stringRemainder) > SSTRecord.MAX_RECORD_SIZE) {
|
||||
//Determine number of bytes that can be written in the space
|
||||
//available
|
||||
int bytesWritten = Math.min((SSTRecord.MAX_RECORD_SIZE - continueSize), stringRemainder);
|
||||
|
||||
if ( totalStringSpaceRequired > SSTRecord.MAX_DATA_SPACE )
|
||||
{
|
||||
retval = sizeOverContinuation( totalStringSpaceRequired );
|
||||
//Ensure that the Unicode String writes both the high and low
|
||||
//byte in the one action. Since the string overhead is 3 bytes
|
||||
//if the bytes that can be written is even, then we need to
|
||||
//write one less byte to capture both the high and low bytes.
|
||||
bytesWritten = unistr.maxBrokenLength(bytesWritten);
|
||||
continueSize += bytesWritten;
|
||||
stringRemainder -= bytesWritten;
|
||||
recordLengths.add(new Integer(continueSize));
|
||||
recordSize += continueSize;
|
||||
//Minimum ammount of space for a new continue record.
|
||||
continueSize = 4;
|
||||
//Add one to the size of the string that is remaining, since the
|
||||
//first byte for the next continue record will be compressed unicode indicator
|
||||
stringRemainder++;
|
||||
} else {
|
||||
//Remainder of string can fit within the bounds of the current
|
||||
//continue record
|
||||
continueSize += stringRemainder;
|
||||
stringRemainder = 0;
|
||||
if ((i < (strings.size()-1)) && !canFitStringInRecord(continueSize)) {
|
||||
//Start new continueRecord if there is another string
|
||||
recordLengths.add(new Integer(continueSize));
|
||||
recordSize += continueSize;
|
||||
//Minimum ammount of space for a new continue record.
|
||||
continueSize = 4;
|
||||
}
|
||||
else
|
||||
{
|
||||
// short data: write one simple SST record
|
||||
retval = SSTRecord.SST_RECORD_OVERHEAD + totalStringSpaceRequired;
|
||||
recordLengths.add( new Integer( totalStringSpaceRequired ) );
|
||||
}
|
||||
}
|
||||
return retval;
|
||||
}
|
||||
}
|
||||
recordLengths.add(new Integer(continueSize));
|
||||
recordSize += continueSize;
|
||||
return recordSize;
|
||||
}
|
||||
|
||||
public List getRecordLengths()
|
||||
{
|
||||
return recordLengths;
|
||||
}
|
||||
|
||||
private int sizeOverContinuation( int totalStringSpaceRequired )
|
||||
{
|
||||
int retval;
|
||||
|
||||
while ( !finished )
|
||||
{
|
||||
recordSize = 0;
|
||||
pos = 0;
|
||||
|
||||
if ( firstRecord )
|
||||
{
|
||||
addMaxLengthRecordSize();
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
// writing continue record
|
||||
pos = 0;
|
||||
int toBeWritten = ( totalStringSpaceRequired - totalBytesWritten ) + ( isRemainingString ? 1 : 0 );
|
||||
int size = Math.min( SSTRecord.MAX_RECORD_SIZE - SSTRecord.STD_RECORD_OVERHEAD, toBeWritten );
|
||||
|
||||
if ( size == toBeWritten )
|
||||
{
|
||||
finished = true;
|
||||
}
|
||||
recordSize = size + SSTRecord.STD_RECORD_OVERHEAD;
|
||||
recordLengths.add( new Integer( size ) );
|
||||
pos = 4;
|
||||
}
|
||||
if ( isRemainingString )
|
||||
{
|
||||
calcReminant();
|
||||
}
|
||||
calcRemainingStrings();
|
||||
totalWritten += recordSize;
|
||||
}
|
||||
retval = totalWritten;
|
||||
|
||||
return retval;
|
||||
}
|
||||
|
||||
private void addMaxLengthRecordSize()
|
||||
{
|
||||
// writing SST record
|
||||
recordSize = SSTRecord.MAX_RECORD_SIZE;
|
||||
pos = 12;
|
||||
firstRecord = false;
|
||||
recordLengths.add( new Integer( recordSize - SSTRecord.STD_RECORD_OVERHEAD ) );
|
||||
}
|
||||
|
||||
private void calcRemainingStrings()
|
||||
{
|
||||
for ( ; unipos < strings.size(); unipos++ )
|
||||
{
|
||||
int available = SSTRecord.MAX_RECORD_SIZE - pos;
|
||||
Integer intunipos = new Integer( unipos );
|
||||
|
||||
unistr = ( (UnicodeString) strings.get( intunipos ) );
|
||||
if ( unistr.getRecordSize() <= available )
|
||||
{
|
||||
totalBytesWritten += unistr.getRecordSize();
|
||||
pos += unistr.getRecordSize();
|
||||
}
|
||||
else
|
||||
{
|
||||
if ( available >= SSTRecord.STRING_MINIMAL_OVERHEAD )
|
||||
{
|
||||
int toBeWritten =
|
||||
unistr.maxBrokenLength( available );
|
||||
|
||||
totalBytesWritten += toBeWritten;
|
||||
stringReminant =
|
||||
( unistr.getRecordSize() - toBeWritten )
|
||||
+ LittleEndianConsts.BYTE_SIZE;
|
||||
if ( available != toBeWritten )
|
||||
{
|
||||
int shortrecord = recordSize
|
||||
- ( available - toBeWritten );
|
||||
|
||||
recordLengths.set(
|
||||
recordLengths.size() - 1,
|
||||
new Integer(
|
||||
shortrecord - SSTRecord.STD_RECORD_OVERHEAD ) );
|
||||
recordSize = shortrecord;
|
||||
}
|
||||
isRemainingString = true;
|
||||
unipos++;
|
||||
}
|
||||
else
|
||||
{
|
||||
int shortrecord = recordSize - available;
|
||||
|
||||
recordLengths.set( recordLengths.size() - 1,
|
||||
new Integer( shortrecord - SSTRecord.STD_RECORD_OVERHEAD ) );
|
||||
recordSize = shortrecord;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void calcReminant()
|
||||
{
|
||||
int available = SSTRecord.MAX_RECORD_SIZE - pos;
|
||||
|
||||
if ( stringReminant <= available )
|
||||
{
|
||||
|
||||
// write reminant
|
||||
totalBytesWritten += stringReminant - 1;
|
||||
pos += stringReminant;
|
||||
isRemainingString = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
// write as much of the remnant as possible
|
||||
int toBeWritten = unistr.maxBrokenLength( available );
|
||||
|
||||
if ( available != toBeWritten )
|
||||
{
|
||||
int shortrecord = recordSize - ( available - toBeWritten );
|
||||
recordLengths.set( recordLengths.size() - 1,
|
||||
new Integer( shortrecord - SSTRecord.STD_RECORD_OVERHEAD ) );
|
||||
recordSize = shortrecord;
|
||||
}
|
||||
totalBytesWritten += toBeWritten - 1;
|
||||
pos += toBeWritten;
|
||||
stringReminant -= toBeWritten - 1;
|
||||
isRemainingString = true;
|
||||
}
|
||||
}
|
||||
|
||||
private void initVars()
|
||||
{
|
||||
unistr = null;
|
||||
stringReminant = 0;
|
||||
unipos = 0;
|
||||
isRemainingString = false;
|
||||
totalBytesWritten = 0;
|
||||
finished = false;
|
||||
firstRecord = true;
|
||||
totalWritten = 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -111,7 +111,7 @@ class SSTSerializer
|
||||
private void serializeSingleSSTRecord( byte[] data, int offset, int record_length_index )
|
||||
{
|
||||
int len = ( (Integer) recordLengths.get( record_length_index ) ).intValue();
|
||||
int recordSize = SSTRecord.SST_RECORD_OVERHEAD + len - SSTRecord.STD_RECORD_OVERHEAD;
|
||||
int recordSize = len - SSTRecord.STD_RECORD_OVERHEAD;
|
||||
sstRecordHeader.writeSSTHeader( data, 0 + offset, recordSize );
|
||||
int pos = SSTRecord.SST_RECORD_OVERHEAD;
|
||||
|
||||
@ -149,13 +149,16 @@ class SSTSerializer
|
||||
|
||||
while ( totalWritten != record_size )
|
||||
{
|
||||
int recordLength = ( (Integer) recordLengths.get( record_length_index++ ) ).intValue();
|
||||
//Total record length, including excel record header and sst/continue header
|
||||
final int recordLength = ( (Integer) recordLengths.get( record_length_index++ ) ).intValue();
|
||||
//Total available data length (minus the excel record header size)
|
||||
final int recordDataLength = recordLength - 4;
|
||||
RecordProcessor recordProcessor = new RecordProcessor( buffer,
|
||||
recordLength, numStrings, numUniqueStrings );
|
||||
recordDataLength, numStrings, numUniqueStrings );
|
||||
|
||||
// write the appropriate header
|
||||
startOfRecord = offset + totalWritten;
|
||||
recordProcessor.writeRecordHeader( offset, totalWritten, recordLength, first_record );
|
||||
recordProcessor.writeRecordHeader( offset, totalWritten, recordDataLength, first_record );
|
||||
first_record = false;
|
||||
|
||||
// now, write the rest of the data into the current
|
||||
@ -166,6 +169,11 @@ class SSTSerializer
|
||||
// the last string in the previous record was not written out completely
|
||||
stringReminant = recordProcessor.writeStringRemainder( lastneedcontinue,
|
||||
stringReminant, offset, totalWritten );
|
||||
//Check to see if still not written out completely
|
||||
if (lastneedcontinue) {
|
||||
totalWritten += recordLength;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// last string's remnant, if any, is cleaned up as best as can be done ... now let's try and write
|
||||
@ -204,7 +212,7 @@ class SSTSerializer
|
||||
break;
|
||||
}
|
||||
}
|
||||
totalWritten += recordLength + SSTRecord.STD_RECORD_OVERHEAD;
|
||||
totalWritten += recordLength;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -161,7 +161,8 @@ public class TestValueRecordsAggregate extends TestCase
|
||||
};
|
||||
List records = testData();
|
||||
valueRecord.construct( 0, records );
|
||||
int bytesWritten = valueRecord.serialize( 0, actualArray );
|
||||
int bytesWritten = valueRecord.serializeCellRow(1, 0, actualArray );
|
||||
bytesWritten += valueRecord.serializeCellRow(2, bytesWritten, actualArray );
|
||||
assertEquals( 36, bytesWritten );
|
||||
for (int i = 0; i < 36; i++)
|
||||
assertEquals( expectedArray[i], actualArray[i] );
|
||||
|
Loading…
Reference in New Issue
Block a user