From 30faafa87e78703afe5c1578788d54dfcdbb9eb5 Mon Sep 17 00:00:00 2001 From: Josh Micich Date: Thu, 22 May 2008 03:26:25 +0000 Subject: [PATCH] Follow on from bug 44675 - regenerated functionMetadata.txt from new ooo excelfileformat.odt git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@658986 13f79535-47bb-0310-9956-ffa450edef68 --- .../function/functionMetadata-asGenerated.txt | 10 +- .../formula/function/functionMetadata.txt | 6 +- .../ExcelFileFormatDocFunctionExtractor.java | 91 ++++++++++++------- .../TestParseMissingBuiltInFuncs.java | 5 +- .../function/TestReadMissingBuiltInFuncs.java | 12 +-- 5 files changed, 74 insertions(+), 50 deletions(-) diff --git a/src/resources/main/org/apache/poi/hssf/record/formula/function/functionMetadata-asGenerated.txt b/src/resources/main/org/apache/poi/hssf/record/formula/function/functionMetadata-asGenerated.txt index 475131e1c..8a85f4284 100644 --- a/src/resources/main/org/apache/poi/hssf/record/formula/function/functionMetadata-asGenerated.txt +++ b/src/resources/main/org/apache/poi/hssf/record/formula/function/functionMetadata-asGenerated.txt @@ -14,7 +14,7 @@ # limitations under the License. # Created by (org.apache.poi.hssf.record.formula.function.ExcelFileFormatDocFunctionExtractor) -# from source file 'excelfileformat.odt' (size=355750, crc=0x2FAEA65A) +# from source file 'excelfileformat.odt' (size=356107, md5=0x8f789cb6e75594caf068f8e193004ef4) # #Columns: (index, name, minParams, maxParams, returnClass, paramClasses, isVolatile, hasFootnote ) @@ -37,7 +37,7 @@ 15 SIN 1 1 V V 16 COS 1 1 V V 17 TAN 1 1 V V -18 ARCTAN 1 1 V V +18 ATAN 1 1 V V 19 PI 0 0 V - 20 SQRT 1 1 V V 21 EXP 1 1 V V @@ -141,8 +141,8 @@ 169 COUNTA 0 30 V R 183 PRODUCT 0 30 V R 184 FACT 1 1 V V -191 DPRODUCT 3 3 V R R R -192 ISNONTEXT 1 1 V V +189 DPRODUCT 3 3 V R R R +190 ISNONTEXT 1 1 V V 193 STDEVP 1 30 V R 194 VARP 1 30 V R 195 DSTDEVP 3 3 V R R R @@ -184,6 +184,8 @@ 244 INFO 1 1 V V # New Built-In Sheet Functions in BIFF4 14 FIXED 2 3 V V V V x +204 USDOLLAR 1 2 V V V x +215 DBCS 1 1 V V x 216 RANK 2 3 V V R V 247 DB 4 5 V V V V V V 252 FREQUENCY 2 2 A R R diff --git a/src/resources/main/org/apache/poi/hssf/record/formula/function/functionMetadata.txt b/src/resources/main/org/apache/poi/hssf/record/formula/function/functionMetadata.txt index 6902027de..8a85f4284 100644 --- a/src/resources/main/org/apache/poi/hssf/record/formula/function/functionMetadata.txt +++ b/src/resources/main/org/apache/poi/hssf/record/formula/function/functionMetadata.txt @@ -14,11 +14,9 @@ # limitations under the License. # Created by (org.apache.poi.hssf.record.formula.function.ExcelFileFormatDocFunctionExtractor) -# from source file 'excelfileformat.odt' (size=355750, crc=0x2FAEA65A) +# from source file 'excelfileformat.odt' (size=356107, md5=0x8f789cb6e75594caf068f8e193004ef4) # #Columns: (index, name, minParams, maxParams, returnClass, paramClasses, isVolatile, hasFootnote ) -# -# + some manual edits ! # Built-In Sheet Functions in BIFF2 0 COUNT 0 30 V R @@ -186,7 +184,7 @@ 244 INFO 1 1 V V # New Built-In Sheet Functions in BIFF4 14 FIXED 2 3 V V V V x -204 USDOLLAR 1 1 V V x +204 USDOLLAR 1 2 V V V x 215 DBCS 1 1 V V x 216 RANK 2 3 V V R V 247 DB 4 5 V V V V V V diff --git a/src/testcases/org/apache/poi/hssf/record/formula/function/ExcelFileFormatDocFunctionExtractor.java b/src/testcases/org/apache/poi/hssf/record/formula/function/ExcelFileFormatDocFunctionExtractor.java index 47137df4f..7702fce3d 100644 --- a/src/testcases/org/apache/poi/hssf/record/formula/function/ExcelFileFormatDocFunctionExtractor.java +++ b/src/testcases/org/apache/poi/hssf/record/formula/function/ExcelFileFormatDocFunctionExtractor.java @@ -26,9 +26,12 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.PrintStream; import java.io.UnsupportedEncodingException; +import java.math.BigInteger; import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -37,7 +40,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.Stack; -import java.util.zip.CRC32; import java.util.zip.ZipException; import java.util.zip.ZipFile; @@ -149,7 +151,6 @@ public final class ExcelFileFormatDocFunctionExtractor { private static final class FunctionDataCollector { - private final Map _allFunctionsByIndex; private final Map _allFunctionsByName; private final Set _groupFunctionIndexes; @@ -184,25 +185,29 @@ public final class ExcelFileFormatDocFunctionExtractor { _allFunctionsByName.put(funcName, fd); } + /** + * Some extra validation here. + * Any function which changes definition will have a footnote in the source document + */ private void checkRedefinedFunction(boolean hasNote, String funcName, Integer funcIxKey) { FunctionData fdPrev; + // check by index fdPrev = (FunctionData) _allFunctionsByIndex.get(funcIxKey); if(fdPrev != null) { - if(fdPrev.hasFootnote() && hasNote) { - // func def can change if both have a foot-note - _allFunctionsByName.remove(fdPrev.getName()); - } else { - throw new RuntimeException("changing function definition without foot-note"); + if(!fdPrev.hasFootnote() || !hasNote) { + throw new RuntimeException("changing function [" + + funcIxKey + "] definition without foot-note"); } + _allFunctionsByName.remove(fdPrev.getName()); } + // check by name fdPrev = (FunctionData) _allFunctionsByName.get(funcName); if(fdPrev != null) { - if(fdPrev.hasFootnote() && hasNote) { - // func def can change if both have a foot-note - _allFunctionsByIndex.remove(new Integer(fdPrev.getIndex())); - } else { - throw new RuntimeException("changing function definition without foot-note"); + if(!fdPrev.hasFootnote() || !hasNote) { + throw new RuntimeException("changing function '" + + funcName + "' definition without foot-note"); } + _allFunctionsByIndex.remove(new Integer(fdPrev.getIndex())); } } @@ -237,9 +242,13 @@ public final class ExcelFileFormatDocFunctionExtractor { private static final String[] TABLE_CELL_RELPATH_NAMES = { "table:table-row", "table:table-cell", "text:p", }; - private static final String[] NOTE_REF_RELPATH_NAMES = { + // after May 2008 there was one more style applied to the footnotes + private static final String[] NOTE_REF_RELPATH_NAMES_OLD = { "table:table-row", "table:table-cell", "text:p", "text:span", "text:note-ref", }; + private static final String[] NOTE_REF_RELPATH_NAMES = { + "table:table-row", "table:table-cell", "text:p", "text:span", "text:span", "text:note-ref", + }; private final Stack _elemNameStack; @@ -368,6 +377,8 @@ public final class ExcelFileFormatDocFunctionExtractor { } else if(matchesRelPath(TABLE_CELL_RELPATH_NAMES)) { _textNodeBuffer.setLength(0); _cellHasNote = false; + } else if(matchesRelPath(NOTE_REF_RELPATH_NAMES_OLD)) { + _cellHasNote = true; } else if(matchesRelPath(NOTE_REF_RELPATH_NAMES)) { _cellHasNote = true; } @@ -456,6 +467,9 @@ public final class ExcelFileFormatDocFunctionExtractor { } private static void processFile(File effDocFile, File outFile) { + if(!effDocFile.exists()) { + throw new RuntimeException("file '" + effDocFile.getAbsolutePath() + "' does not exist"); + } OutputStream os; try { os = new FileOutputStream(outFile); @@ -475,7 +489,7 @@ public final class ExcelFileFormatDocFunctionExtractor { ps.println("# Created by (" + genClass.getName() + ")"); // identify the source file ps.print("# from source file '" + SOURCE_DOC_FILE_NAME + "'"); - ps.println(" (size=" + effDocFile.length() + ", crc=" + getFileCRC(effDocFile) + ")"); + ps.println(" (size=" + effDocFile.length() + ", md5=" + getFileMD5(effDocFile) + ")"); ps.println("#"); ps.println("#Columns: (index, name, minParams, maxParams, returnClass, paramClasses, isVolatile, hasFootnote )"); ps.println(""); @@ -490,6 +504,14 @@ public final class ExcelFileFormatDocFunctionExtractor { throw new RuntimeException(e); } ps.close(); + + String canonicalOutputFileName; + try { + canonicalOutputFileName = outFile.getCanonicalPath(); + } catch (IOException e) { + throw new RuntimeException(e); + } + System.out.println("Successfully output to '" + canonicalOutputFileName + "'"); } private static void outputLicenseHeader(PrintStream ps) { @@ -519,8 +541,14 @@ public final class ExcelFileFormatDocFunctionExtractor { /** * Helps identify the source file */ - private static String getFileCRC(File f) { - CRC32 crc = new CRC32(); + private static String getFileMD5(File f) { + MessageDigest m; + try { + m = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + byte[]buf = new byte[2048]; try { InputStream is = new FileInputStream(f); @@ -529,21 +557,17 @@ public final class ExcelFileFormatDocFunctionExtractor { if(bytesRead<1) { break; } - crc.update(buf, 0, bytesRead); + m.update(buf, 0, bytesRead); } is.close(); } catch (IOException e) { throw new RuntimeException(e); } - return "0x" + Long.toHexString(crc.getValue()).toUpperCase(); + + return "0x" + new BigInteger(1, m.digest()).toString(16); } - private static File getSourceFile() { - if (false) { - File dir = new File("c:/temp"); - File effDocFile = new File(dir, SOURCE_DOC_FILE_NAME); - return effDocFile; - } + private static File downloadSourceFile() { URL url; try { url = new URL("http://sc.openoffice.org/" + SOURCE_DOC_FILE_NAME); @@ -557,7 +581,7 @@ public final class ExcelFileFormatDocFunctionExtractor { URLConnection conn = url.openConnection(); InputStream is = conn.getInputStream(); System.out.println("downloading " + url.toExternalForm()); - result = File.createTempFile("excelfileformat", "odt"); + result = File.createTempFile("excelfileformat", ".odt"); OutputStream os = new FileOutputStream(result); while(true) { int bytesRead = is.read(buf); @@ -577,12 +601,17 @@ public final class ExcelFileFormatDocFunctionExtractor { public static void main(String[] args) { - File effDocFile = getSourceFile(); - if(!effDocFile.exists()) { - throw new RuntimeException("file '" + effDocFile.getAbsolutePath() + "' does not exist"); - } - File outFile = new File("functionMetadata-asGenerated.txt"); - processFile(effDocFile, outFile); + + if (false) { // set true to use local file + File dir = new File("c:/temp"); + File effDocFile = new File(dir, SOURCE_DOC_FILE_NAME); + processFile(effDocFile, outFile); + return; + } + + File tempEFFDocFile = downloadSourceFile(); + processFile(tempEFFDocFile, outFile); + tempEFFDocFile.delete(); } } diff --git a/src/testcases/org/apache/poi/hssf/record/formula/function/TestParseMissingBuiltInFuncs.java b/src/testcases/org/apache/poi/hssf/record/formula/function/TestParseMissingBuiltInFuncs.java index 1e26fa706..7030c5a50 100644 --- a/src/testcases/org/apache/poi/hssf/record/formula/function/TestParseMissingBuiltInFuncs.java +++ b/src/testcases/org/apache/poi/hssf/record/formula/function/TestParseMissingBuiltInFuncs.java @@ -21,7 +21,6 @@ import junit.framework.AssertionFailedError; import junit.framework.TestCase; import org.apache.poi.hssf.model.FormulaParser; -import org.apache.poi.hssf.model.Workbook; import org.apache.poi.hssf.record.formula.AbstractFunctionPtg; import org.apache.poi.hssf.record.formula.FuncPtg; import org.apache.poi.hssf.record.formula.FuncVarPtg; @@ -29,7 +28,7 @@ import org.apache.poi.hssf.record.formula.Ptg; import org.apache.poi.hssf.usermodel.HSSFWorkbook; /** * Tests parsing of some built-in functions that were not properly - * registered in POI as bug #44675, #44733 (March/April 2008). + * registered in POI as of bug #44675, #44733 (March/April 2008). * * @author Josh Micich */ @@ -76,7 +75,7 @@ public final class TestParseMissingBuiltInFuncs extends TestCase { } public void testUsdollar() { - confirmFunc("USDOLLAR(1)", 2, false, 204); + confirmFunc("USDOLLAR(1)", 2, true, 204); } public void testDBCS() { diff --git a/src/testcases/org/apache/poi/hssf/record/formula/function/TestReadMissingBuiltInFuncs.java b/src/testcases/org/apache/poi/hssf/record/formula/function/TestReadMissingBuiltInFuncs.java index 6766f2fc0..0a62d64cd 100644 --- a/src/testcases/org/apache/poi/hssf/record/formula/function/TestReadMissingBuiltInFuncs.java +++ b/src/testcases/org/apache/poi/hssf/record/formula/function/TestReadMissingBuiltInFuncs.java @@ -17,22 +17,18 @@ package org.apache.poi.hssf.record.formula.function; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; import java.lang.reflect.InvocationTargetException; +import junit.framework.AssertionFailedError; +import junit.framework.TestCase; + import org.apache.poi.hssf.HSSFTestDataSamples; import org.apache.poi.hssf.record.RecordFormatException; import org.apache.poi.hssf.usermodel.HSSFSheet; import org.apache.poi.hssf.usermodel.HSSFWorkbook; - -import junit.framework.AssertionFailedError; -import junit.framework.TestCase; /** * Tests reading from a sample spreadsheet some built-in functions that were not properly - * registered in POI as bug #44675, #44733 (March/April 2008). + * registered in POI as of bug #44675, #44733 (March/April 2008). * * @author Josh Micich */