moparisthebest/poi
1
0
Fork 0
Code Issues Pull Requests Releases 1 Wiki Activity

Add a number of tests for the dev-tools. The tests iterate over all .xls

Browse Source 
files that are found under test-data, any newly added file or
code-change which breaks one of the dev-tools will cause the tests to
fail in the future. Known broken files can be excluded.

Also fixes two points where some files could cause BiffViewer to fail.

git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1535885 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Dominik Stadler 2013-10-25 21:57:48 +00:00
parent 616238f8e2
commit 780aec8ad2
12 changed files with 383 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/java/org/apache/poi/hssf/dev/BiffDrawingToXml.java
View File

@ -19,18 +19,23 @@
package org.apache.poi.hssf.dev;
import org.apache.poi.ddf.EscherRecord;
import org.apache.poi.hssf.model.InternalWorkbook;
import org.apache.poi.hssf.record.*;
import org.apache.poi.hssf.usermodel.HSSFPatriarch;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import java.io.*;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;
import org.apache.poi.ddf.EscherRecord;
import org.apache.poi.hssf.model.InternalWorkbook;
import org.apache.poi.hssf.record.DrawingGroupRecord;
import org.apache.poi.hssf.record.EscherAggregate;
import org.apache.poi.hssf.usermodel.HSSFPatriarch;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
/**
* Utility for representing drawings contained in a binary Excel file as a XML tree
*
@ -132,7 +137,7 @@ public class BiffDrawingToXml {
outputStream.close();
}
public static void writeToFile(FileOutputStream fos, InputStream xlsWorkbook, boolean excludeWorkbookRecords, String[] params) throws IOException {
public static void writeToFile(OutputStream fos, InputStream xlsWorkbook, boolean excludeWorkbookRecords, String[] params) throws IOException {
POIFSFileSystem fs = new POIFSFileSystem(xlsWorkbook);
HSSFWorkbook workbook = new HSSFWorkbook(fs);
InternalWorkbook internalWorkbook = getInternalWorkbook(workbook);

36
src/java/org/apache/poi/hssf/dev/BiffViewer.java
View File

@ -17,23 +17,20 @@
package org.apache.poi.hssf.dev;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.Writer;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
import org.apache.poi.hssf.record.*;
import org.apache.poi.hssf.record.RecordInputStream.LeftoverDataException;
import org.apache.poi.hssf.record.chart.*;
import org.apache.poi.hssf.record.pivottable.*;
import org.apache.poi.hssf.record.pivottable.DataItemRecord;
import org.apache.poi.hssf.record.pivottable.ExtendedPivotTableViewFieldsRecord;
import org.apache.poi.hssf.record.pivottable.PageItemRecord;
import org.apache.poi.hssf.record.pivottable.StreamIDRecord;
import org.apache.poi.hssf.record.pivottable.ViewDefinitionRecord;
import org.apache.poi.hssf.record.pivottable.ViewFieldsRecord;
import org.apache.poi.hssf.record.pivottable.ViewSourceRecord;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import org.apache.poi.util.HexDump;
import org.apache.poi.util.LittleEndian;
@ -413,9 +410,8 @@ public final class BiffViewer {
boolean dumpInterpretedRecords = cmdArgs.shouldDumpRecordInterpretations();
boolean dumpHex = cmdArgs.shouldDumpBiffHex();
boolean zeroAlignHexDump = dumpInterpretedRecords; // TODO - fix non-zeroAlign
BiffRecordListener recListener = new BiffRecordListener(dumpHex ? new OutputStreamWriter(ps) : null, zeroAlignHexDump, cmdArgs.suppressHeader());
is = new BiffDumpingStream(is, recListener);
createRecords(is, ps, recListener, dumpInterpretedRecords);
runBiffViewer(ps, is, dumpInterpretedRecords, dumpHex, zeroAlignHexDump,
cmdArgs.suppressHeader());
}
ps.close();
} catch (Exception e) {
@ -423,6 +419,14 @@ public final class BiffViewer {
}
}
protected static void runBiffViewer(PrintStream ps, InputStream is,
boolean dumpInterpretedRecords, boolean dumpHex, boolean zeroAlignHexDump,
boolean suppressHeader) {
BiffRecordListener recListener = new BiffRecordListener(dumpHex ? new OutputStreamWriter(ps) : null, zeroAlignHexDump, suppressHeader);
is = new BiffDumpingStream(is, recListener);
createRecords(is, ps, recListener, dumpInterpretedRecords);
}
private static final class BiffRecordListener implements IBiffRecordListener {
private final Writer _hexDumpWriter;
private final List<String> _headers;
@ -497,6 +501,7 @@ public final class BiffViewer {
_currentPos = 0;
}
@Override
public int read() throws IOException {
if (_currentPos >= _currentSize) {
fillNextBuffer();
@ -510,6 +515,7 @@ public final class BiffViewer {
formatBufferIfAtEndOfRec();
return result;
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (_currentPos >= _currentSize) {
fillNextBuffer();
@ -532,6 +538,7 @@ public final class BiffViewer {
return result;
}
@Override
public int available() throws IOException {
return _currentSize - _currentPos + _is.available();
}
@ -561,6 +568,7 @@ public final class BiffViewer {
int globalOffset = _overallStreamPos-_currentSize;
_listener.processRecord(globalOffset, _recordCounter, sid, dataSize, _data);
}
@Override
public void close() throws IOException {
_is.close();
}

6
src/java/org/apache/poi/hssf/dev/FormulaViewer.java
View File

@ -24,12 +24,12 @@ import org.apache.poi.hssf.model.HSSFFormulaParser;
import org.apache.poi.hssf.record.FormulaRecord;
import org.apache.poi.hssf.record.Record;
import org.apache.poi.hssf.record.RecordFactory;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import org.apache.poi.ss.formula.ptg.ExpPtg;
import org.apache.poi.ss.formula.ptg.FuncPtg;
import org.apache.poi.ss.formula.ptg.OperationPtg;
import org.apache.poi.ss.formula.ptg.Ptg;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
/**
* FormulaViewer - finds formulas in a BIFF8 file and attempts to read them/display
@ -68,7 +68,7 @@ public class FormulaViewer
for (int k = 0; k < records.size(); k++)
{
Record record = ( Record ) records.get(k);
Record record = records.get(k);
if (record.getSid() == FormulaRecord.sid)
{

13
src/java/org/apache/poi/hssf/record/ObjRecord.java
View File

@ -137,18 +137,22 @@ public final class ObjRecord extends Record {
return true;
}
@Override
public String toString() {
StringBuffer sb = new StringBuffer();
sb.append("[OBJ]\n");
for (int i = 0; i < subrecords.size(); i++) {
SubRecord record = subrecords.get(i);
sb.append("SUBRECORD: ").append(record.toString());
if(subrecords != null) { // there are special cases where this can be, see comments in constructor above
for (int i = 0; i < subrecords.size(); i++) {
SubRecord record = subrecords.get(i);
sb.append("SUBRECORD: ").append(record.toString());
}
}
sb.append("[/OBJ]\n");
return sb.toString();
}
@Override
public int getRecordSize() {
if (_uninterpretedData != null) {
return _uninterpretedData.length + 4;
@ -170,6 +174,7 @@ public final class ObjRecord extends Record {
return size + 4;
}
@Override
public int serialize(int offset, byte[] data) {
int recSize = getRecordSize();
int dataSize = recSize - 4;
@ -195,6 +200,7 @@ public final class ObjRecord extends Record {
return recSize;
}
@Override
public short getSid() {
return sid;
}
@ -215,6 +221,7 @@ public final class ObjRecord extends Record {
return subrecords.add(o);
}
@Override
public Object clone() {
ObjRecord rec = new ObjRecord();

3
src/java/org/apache/poi/hssf/record/chart/CatLabRecord.java
View File

@ -83,7 +83,8 @@ public final class CatLabRecord extends StandardRecord {
buffer.append(" .wOffset =").append(HexDump.shortToHex(wOffset)).append('\n');
buffer.append(" .at =").append(HexDump.shortToHex(at)).append('\n');
buffer.append(" .grbit =").append(HexDump.shortToHex(grbit)).append('\n');
buffer.append(" .unused =").append(HexDump.shortToHex(unused)).append('\n');
if(unused != null)
buffer.append(" .unused =").append(HexDump.shortToHex(unused)).append('\n');
buffer.append("[/CATLAB]\n");
return buffer.toString();

86
src/testcases/org/apache/poi/hssf/dev/BaseXLSIteratingTest.java Normal file
View File

@ -0,0 +1,86 @@
package org.apache.poi.hssf.dev;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import org.junit.Test;
/**
* Base class for integration-style tests which iterate over all test-files
* and execute the same action to find out if any change breaks these applications.
*/
public abstract class BaseXLSIteratingTest {
protected static final OutputStream NULL_OUTPUT_STREAM = new NullOutputStream();
protected static final List<String> EXCLUDED = new ArrayList<String>();
protected static final List<String> SILENT_EXCLUDED = new ArrayList<String>();
@Test
public void testMain() throws Exception {
int count = runWithDir("test-data/spreadsheet");
count += runWithDir("test-data/hpsf");
System.out.println("Had " + count + " files");
}
private int runWithDir(String dir) {
List<String> failed = new ArrayList<String>();
String[] files = new File(dir).list(new FilenameFilter() {
public boolean accept(File arg0, String arg1) {
return arg1.toLowerCase().endsWith(".xls");
}
});
runWithArrayOfFiles(files, dir, failed);
assertTrue("Expected to have no failed except the ones excluded, but had: " + failed,
failed.isEmpty());
return files.length;
}
private void runWithArrayOfFiles(String[] files, String dir, List<String> failed) {
for(String file : files) {
try {
runOneFile(dir, file, failed);
} catch (Exception e) {
System.out.println("Failed: " + file);
if(SILENT_EXCLUDED.contains(file)) {
continue;
}
e.printStackTrace();
if(!EXCLUDED.contains(file)) {
failed.add(file);
}
}
}
}
abstract void runOneFile(String dir, String file, List<String> failed) throws Exception;
/**
* Implementation of an OutputStream which does nothing, used
* to redirect stdout to avoid spamming the console with output
*/
private static class NullOutputStream extends OutputStream {
@Override
public void write(byte[] b, int off, int len) {
}
@Override
public void write(int b) {
}
@Override
public void write(byte[] b) throws IOException {
}
}
}

45
src/testcases/org/apache/poi/hssf/dev/TestBiffDrawingToXml.java Normal file
View File

@ -0,0 +1,45 @@
package org.apache.poi.hssf.dev;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.PrintStream;
import java.util.List;
import org.junit.Ignore;
import org.junit.Test;
public class TestBiffDrawingToXml extends BaseXLSIteratingTest {
static {
// TODO: is it ok to fail these?
// Look at the output of the test for the detailed stacktrace of the failures...
// EXCLUDED.add("password.xls");
// EXCLUDED.add("XRefCalc.xls");
// EXCLUDED.add("43493.xls");
// EXCLUDED.add("51832.xls");
};
@Override
@Ignore("Not yet done, nearly all files fail with various errors, remove this method when done to use the one from the abstract base class!...")
@Test
public void testMain() throws Exception {
}
@Override
void runOneFile(String dir, String file, List<String> failed)
throws Exception {
PrintStream save = System.out;
try {
//System.setOut(new PrintStream(TestBiffViewer.NULL_OUTPUT_STREAM));
// use a NullOutputStream to not write the bytes anywhere for best runtime
InputStream wb = new FileInputStream(new File(dir, file));
try {
BiffDrawingToXml.writeToFile(NULL_OUTPUT_STREAM, wb, false, new String[] {});
} finally {
wb.close();
}
} finally {
System.setOut(save);
}
}
}

45
src/testcases/org/apache/poi/hssf/dev/TestBiffViewer.java Normal file
View File

@ -0,0 +1,45 @@
package org.apache.poi.hssf.dev;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.util.List;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
public class TestBiffViewer extends BaseXLSIteratingTest {
static {
// TODO: is it ok to fail these?
// Look at the output of the test for the detailed stacktrace of the failures...
EXCLUDED.add("WORKBOOK_in_capitals.xls");
EXCLUDED.add("password.xls");
EXCLUDED.add("NoGutsRecords.xls");
EXCLUDED.add("BOOK_in_capitals.xls");
EXCLUDED.add("XRefCalc.xls");
EXCLUDED.add("50833.xls"); // probably a problem in BiffViewer
EXCLUDED.add("43493.xls");
EXCLUDED.add("51832.xls");
EXCLUDED.add("OddStyleRecord.xls");
SILENT_EXCLUDED.add("46904.xls");
};
@Override
void runOneFile(String dir, String file, List<String> failed) throws IOException {
FileInputStream inStream = new FileInputStream(new File(dir, file));
try {
POIFSFileSystem fs = new POIFSFileSystem(inStream);
InputStream is = fs.createDocumentInputStream("Workbook");
try {
// use a NullOutputStream to not write the bytes anywhere for best runtime
BiffViewer.runBiffViewer(new PrintStream(NULL_OUTPUT_STREAM), is, true, true, true, false);
} finally {
is.close();
}
} finally {
inStream.close();
}
}
}

30
src/testcases/org/apache/poi/hssf/dev/TestEFBiffViewer.java Normal file
View File

@ -0,0 +1,30 @@
package org.apache.poi.hssf.dev;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.util.List;
public class TestEFBiffViewer extends BaseXLSIteratingTest {
static {
// TODO: is it ok to fail these?
// Look at the output of the test for the detailed stacktrace of the failures...
EXCLUDED.add("password.xls");
EXCLUDED.add("XRefCalc.xls");
EXCLUDED.add("43493.xls");
EXCLUDED.add("51832.xls");
};
@Override
void runOneFile(String dir, String file, List<String> failed) throws IOException {
PrintStream save = System.out;
try {
// redirect standard out during the test to avoid spamming the console with output
System.setOut(new PrintStream(NULL_OUTPUT_STREAM));
EFBiffViewer.main(new String[] { new File(dir, file).getAbsolutePath() });
} finally {
System.setOut(save);
}
}
}

42
src/testcases/org/apache/poi/hssf/dev/TestFormulaViewer.java Normal file
View File

@ -0,0 +1,42 @@
package org.apache.poi.hssf.dev;
import java.io.File;
import java.io.PrintStream;
import java.util.List;
import org.junit.Ignore;
import org.junit.Test;
public class TestFormulaViewer extends BaseXLSIteratingTest {
static {
// TODO: is it ok to fail these?
// Look at the output of the test for the detailed stacktrace of the failures...
// EXCLUDED.add("WORKBOOK_in_capitals.xls");
// EXCLUDED.add("NoGutsRecords.xls");
// EXCLUDED.add("BOOK_in_capitals.xls");
// EXCLUDED.add("46904.xls");
// EXCLUDED.add("OddStyleRecord.xls");
};
@Override
@Ignore("Not yet done, nearly all files fail with various errors, remove this method when done to use the one from the abstract base class!...")
@Test
public void testMain() throws Exception {
}
@Override
void runOneFile(String dir, String file, List<String> failed) throws Exception {
PrintStream save = System.out;
try {
// redirect standard out during the test to avoid spamming the console with output
System.setOut(new PrintStream(NULL_OUTPUT_STREAM));
FormulaViewer viewer = new FormulaViewer();
viewer.setFile(new File(dir, file).getAbsolutePath());
viewer.setList(true);
viewer.run();
} finally {
System.setOut(save);
}
}
}

50
src/testcases/org/apache/poi/hssf/dev/TestReSave.java Normal file
View File

@ -0,0 +1,50 @@
package org.apache.poi.hssf.dev;
import java.io.File;
import java.io.PrintStream;
import java.util.List;
public class TestReSave extends BaseXLSIteratingTest {
static {
// TODO: is it ok to fail these?
// Look at the output of the test for the detailed stacktrace of the failures...
EXCLUDED.add("password.xls");
EXCLUDED.add("43493.xls");
EXCLUDED.add("51832.xls");
EXCLUDED.add("49219.xls");
EXCLUDED.add("49931.xls");
SILENT_EXCLUDED.add("46904.xls");
};
@Override
void runOneFile(String dir, String file, List<String> failed) throws Exception {
// avoid running on files leftover from previous failed runs
if(file.endsWith("-saved.xls")) {
return;
}
PrintStream save = System.out;
try {
// redirect standard out during the test to avoid spamming the console with output
System.setOut(new PrintStream(NULL_OUTPUT_STREAM));
try {
ReSave.main(new String[] { new File(dir, file).getAbsolutePath() });
try {
// had one case where the re-saved could not be re-saved!
ReSave.main(new String[] { new File(dir, file.replace(".xls", "-saved.xls")).getAbsolutePath() });
} finally {
// clean up the re-re-saved file
new File(dir, file.replace(".xls", "-saved.xls").replace(".xls", "-saved.xls")).delete();
}
} finally {
// clean up the re-saved file
new File(dir, file.replace(".xls", "-saved.xls")).delete();
}
} finally {
System.setOut(save);
}
}
}

34
src/testcases/org/apache/poi/hssf/dev/TestRecordLister.java Normal file
View File

@ -0,0 +1,34 @@
package org.apache.poi.hssf.dev;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.util.List;
public class TestRecordLister extends BaseXLSIteratingTest {
static {
// TODO: is it ok to fail these?
// Look at the output of the test for the detailed stacktrace of the failures...
EXCLUDED.add("WORKBOOK_in_capitals.xls");
EXCLUDED.add("NoGutsRecords.xls");
EXCLUDED.add("BOOK_in_capitals.xls");
EXCLUDED.add("OddStyleRecord.xls");
SILENT_EXCLUDED.add("46904.xls");
};
@Override
void runOneFile(String dir, String file, List<String> failed) throws IOException {
PrintStream save = System.out;
try {
// redirect standard out during the test to avoid spamming the console with output
System.setOut(new PrintStream(NULL_OUTPUT_STREAM));
RecordLister viewer = new RecordLister();
viewer.setFile(new File(dir, file).getAbsolutePath());
viewer.run();
} finally {
System.setOut(save);
}
}
}