From 2ffc18f224c838b483131d9c08ea0ac74a56d964 Mon Sep 17 00:00:00 2001 From: ashley willis Date: Tue, 22 May 2012 22:16:24 -0500 Subject: [PATCH] Replaced android.test.InstrumentationTestRunner with com.zutubi.android.junitreport.JUnitReportTestRunner. Create javadoc/ directory when doing "ant javadoc". Fixed installing debug builds with ant. "ant -f tests/build.xml debug && ant -f tests/build.xml installt test test-report" compiles, installs, tests, and saves output to tests/junit-report.xml. --- build.xml | 9 +- tests/AndroidManifest.xml | 2 +- tests/ant.properties | 1 + tests/build.xml | 18 +- .../junitreport/JUnitReportListener.java | 296 ++++++++++++++++++ .../junitreport/JUnitReportTestRunner.java | 148 +++++++++ 6 files changed, 467 insertions(+), 7 deletions(-) create mode 100644 tests/src/com/zutubi/android/junitreport/JUnitReportListener.java create mode 100644 tests/src/com/zutubi/android/junitreport/JUnitReportTestRunner.java diff --git a/build.xml b/build.xml index 4e52cb9b6..925defe9f 100644 --- a/build.xml +++ b/build.xml @@ -162,19 +162,19 @@ - Installing ${out.debug.file} onto default emulator... + Installing ${out.final.file} onto default emulator... - + - Reinstalling ${out.debug.file} onto default emulator... + Reinstalling ${out.final.file} onto default emulator... - + @@ -214,6 +214,7 @@ + - diff --git a/tests/ant.properties b/tests/ant.properties index da0e730c5..448ca59c5 100644 --- a/tests/ant.properties +++ b/tests/ant.properties @@ -16,3 +16,4 @@ # The password will be asked during the build when you use the 'release' target. tested.project.dir=../ +test.runner=com.zutubi.android.junitreport.JUnitReportTestRunner diff --git a/tests/build.xml b/tests/build.xml index 7dd7a7b59..c3ee67e3c 100644 --- a/tests/build.xml +++ b/tests/build.xml @@ -79,7 +79,21 @@ In all cases you must update the value of version-tag below to read 'custom' instead of an integer, in order to avoid having your file be overridden by tools such as "android update project" --> - - + + + + + Downloading XML test report... + + + + + + + + + diff --git a/tests/src/com/zutubi/android/junitreport/JUnitReportListener.java b/tests/src/com/zutubi/android/junitreport/JUnitReportListener.java new file mode 100644 index 000000000..81e2d6c89 --- /dev/null +++ b/tests/src/com/zutubi/android/junitreport/JUnitReportListener.java @@ -0,0 +1,296 @@ +/* + * Copyright (C) 2010-2011 Zutubi Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.zutubi.android.junitreport; + +import android.content.Context; +import android.util.Log; +import android.util.Xml; + +import junit.framework.AssertionFailedError; +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestListener; + +import org.xmlpull.v1.XmlSerializer; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.io.Writer; +import java.util.Locale; + +/** + * Custom test listener that outputs test results to XML files. The files + * use a similar format to the Ant JUnit task XML formatter, with a few of + * caveats: + *
    + *
  • + * By default, multiple suites are all placed in a single file under a root + * <testsuites> element. In multiFile mode a separate file is + * created for each suite, which may be more compatible with existing + * tools. + *
    • + *
  • + * Redundant information about the number of nested cases within a suite is + * omitted. + *
    • + *
  • + * Durations are omitted from suites. + *
    • + *
  • + * Neither standard output nor system properties are included. + *
    • + *
+ * The differences mainly revolve around making this reporting as lightweight as + * possible. The report is streamed as the tests run, making it impossible to, + * e.g. include the case count in a <testsuite> element. + */ +public class JUnitReportListener implements TestListener { + private static final String LOG_TAG = "JUnitReportListener"; + + private static final String ENCODING_UTF_8 = "utf-8"; + + private static final String TAG_SUITES = "testsuites"; + private static final String TAG_SUITE = "testsuite"; + private static final String TAG_CASE = "testcase"; + private static final String TAG_ERROR = "error"; + private static final String TAG_FAILURE = "failure"; + + private static final String ATTRIBUTE_NAME = "name"; + private static final String ATTRIBUTE_CLASS = "classname"; + private static final String ATTRIBUTE_TYPE = "type"; + private static final String ATTRIBUTE_MESSAGE = "message"; + private static final String ATTRIBUTE_TIME = "time"; + + // With thanks to org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner. + // Trimmed some entries, added others for Android. + private static final String[] DEFAULT_TRACE_FILTERS = new String[] { + "junit.framework.TestCase", "junit.framework.TestResult", + "junit.framework.TestSuite", + "junit.framework.Assert.", // don't filter AssertionFailure + "java.lang.reflect.Method.invoke(", "sun.reflect.", + // JUnit 4 support: + "org.junit.", "junit.framework.JUnit4TestAdapter", " more", + // Added for Android + "android.test.", "android.app.Instrumentation", + "java.lang.reflect.Method.invokeNative", + }; + + private Context mContext; + private Context mTargetContext; + private String mReportFile; + private String mReportDir; + private boolean mFilterTraces; + private boolean mMultiFile; + private FileOutputStream mOutputStream; + private XmlSerializer mSerializer; + private String mCurrentSuite; + + // simple time tracking + private boolean mTimeAlreadyWritten = false; + private long mTestStartTime; + + /** + * Creates a new listener. + * + * @param context context of the test application + * @param targetContext context of the application under test + * @param reportFile name of the report file(s) to create + * @param reportDir path of the directory under which to write files + * (may be null in which case files are written under + * the context using {@link Context#openFileOutput(String, int)}). + * @param filterTraces if true, stack traces will have common noise (e.g. + * framework methods) omitted for clarity + * @param multiFile if true, use a separate file for each test suite + */ + public JUnitReportListener(Context context, Context targetContext, String reportFile, String reportDir, boolean filterTraces, boolean multiFile) { + this.mContext = context; + this.mTargetContext = targetContext; + this.mReportFile = reportFile; + this.mReportDir = reportDir; + this.mFilterTraces = filterTraces; + this.mMultiFile = multiFile; + } + + @Override + public void startTest(Test test) { + try { + if (test instanceof TestCase) { + TestCase testCase = (TestCase) test; + checkForNewSuite(testCase); + mSerializer.startTag("", TAG_CASE); + mSerializer.attribute("", ATTRIBUTE_CLASS, mCurrentSuite); + mSerializer.attribute("", ATTRIBUTE_NAME, testCase.getName()); + + mTimeAlreadyWritten = false; + mTestStartTime = System.currentTimeMillis(); + } + } catch (IOException e) { + Log.e(LOG_TAG, safeMessage(e)); + } + } + + private void checkForNewSuite(TestCase testCase) throws IOException { + String suiteName = testCase.getClass().getName(); + if (mCurrentSuite == null || !mCurrentSuite.equals(suiteName)) { + if (mCurrentSuite != null) { + if (mMultiFile) { + close(); + } else { + mSerializer.endTag("", TAG_SUITE); + } + } + + openIfRequired(suiteName); + + mSerializer.startTag("", TAG_SUITE); + mSerializer.attribute("", ATTRIBUTE_NAME, suiteName); + mCurrentSuite = suiteName; + } + } + + private void openIfRequired(String suiteName) throws IOException { + if (mSerializer == null) { + String fileName = mReportFile; + if (mMultiFile) { + fileName = fileName.replace("$(suite)", suiteName); + } + + if (mReportDir == null) { + if (mContext.getFilesDir() != null) { + mOutputStream = mContext.openFileOutput(fileName, 0); + } else { + mOutputStream = mTargetContext.openFileOutput(fileName, 0); + } + } else { + mOutputStream = new FileOutputStream(new File(mReportDir, fileName)); + } + + mSerializer = Xml.newSerializer(); + mSerializer.setOutput(mOutputStream, ENCODING_UTF_8); + mSerializer.startDocument(ENCODING_UTF_8, true); + if (!mMultiFile) { + mSerializer.startTag("", TAG_SUITES); + } + } + } + + @Override + public void addError(Test test, Throwable error) { + addProblem(TAG_ERROR, error); + } + + @Override + public void addFailure(Test test, AssertionFailedError error) { + addProblem(TAG_FAILURE, error); + } + + private void addProblem(String tag, Throwable error) { + try { + recordTestTime(); + + mSerializer.startTag("", tag); + mSerializer.attribute("", ATTRIBUTE_MESSAGE, safeMessage(error)); + mSerializer.attribute("", ATTRIBUTE_TYPE, error.getClass().getName()); + StringWriter w = new StringWriter(); + error.printStackTrace(mFilterTraces ? new FilteringWriter(w) : new PrintWriter(w)); + mSerializer.text(w.toString()); + mSerializer.endTag("", tag); + } catch (IOException e) { + Log.e(LOG_TAG, safeMessage(e)); + } + } + + private void recordTestTime() throws IOException { + if (!mTimeAlreadyWritten) { + mTimeAlreadyWritten = true; + mSerializer.attribute("", ATTRIBUTE_TIME, String.format(Locale.ENGLISH, "%.3f", + (System.currentTimeMillis() - mTestStartTime) / 1000.)); + } + } + + @Override + public void endTest(Test test) { + try { + if (test instanceof TestCase) { + recordTestTime(); + mSerializer.endTag("", TAG_CASE); + } + } catch (IOException e) { + Log.e(LOG_TAG, safeMessage(e)); + } + } + + /** + * Releases all resources associated with this listener. Must be called + * when the listener is finished with. + */ + public void close() { + if (mSerializer != null) { + try { + if (mCurrentSuite != null) { + mSerializer.endTag("", TAG_SUITE); + } + + if (!mMultiFile) { + mSerializer.endTag("", TAG_SUITES); + } + mSerializer.endDocument(); + mSerializer = null; + } catch (IOException e) { + Log.e(LOG_TAG, safeMessage(e)); + } + } + + if (mOutputStream != null) { + try { + mOutputStream.close(); + mOutputStream = null; + } catch (IOException e) { + Log.e(LOG_TAG, safeMessage(e)); + } + } + } + + private String safeMessage(Throwable error) { + String message = error.getMessage(); + return error.getClass().getName() + ": " + (message == null ? "" : message); + } + + /** + * Wrapper around a print writer that filters out common noise from stack + * traces, making it easier to see the actual failure. + */ + private static class FilteringWriter extends PrintWriter { + public FilteringWriter(Writer out) { + super(out); + } + + @Override + public void println(String s) { + for (String filtered : DEFAULT_TRACE_FILTERS) { + if (s.contains(filtered)) { + return; + } + } + + super.println(s); + } + } +} diff --git a/tests/src/com/zutubi/android/junitreport/JUnitReportTestRunner.java b/tests/src/com/zutubi/android/junitreport/JUnitReportTestRunner.java new file mode 100644 index 000000000..551bef9ea --- /dev/null +++ b/tests/src/com/zutubi/android/junitreport/JUnitReportTestRunner.java @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2010-2011 Zutubi Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.zutubi.android.junitreport; + +import android.os.Bundle; +import android.test.AndroidTestRunner; +import android.test.InstrumentationTestRunner; + +/** + * Custom test runner that adds a {@link JUnitReportListener} to the underlying + * test runner in order to capture test results in an XML report. You may use + * this class in place of {@link InstrumentationTestRunner} in your test + * project's manifest, and/or specify it to your Ant build using the test.runner + * property. + *

+ * This runner behaves identically to the default, with the added side-effect of + * producing JUnit XML reports. The report format is similar to that produced + * by the Ant JUnit task's XML formatter, making it compatible with existing + * tools that can process that format. See {@link JUnitReportListener} for + * further details. + *

+ * This runner accepts the following arguments: + *

    + *
  • + * reportFile: name of the file(s) to write the XML report to (default: + * junit-report.xml or junit-report-$(suite).xml depending on the value of + * multiFile). May contain $(suite), which will be replaced with the test + * suite name when using multiFile mode. See the reportDir argument for + * discussion of the file location. + *
    • + *
  • + * reportDir: if specified, absolute path to a directory in which to write + * the report file(s) (default: unset, in which case files are written to + * the test application's data area if possible, or the application under + * test's data area if that fails). + *
    • + *
  • + * multiFile: if true, write a separate XML file for each test suite; + * otherwise include all suites in a single XML file (default: false). + *
    • + *
  • + * filterTraces: if true, stack traces in test failure reports will be + * filtered to remove noise such as framework methods (default: true) + *
    • + *
+ * These arguments may be specified as follows: + * + * 
+ * {@code adb shell am instrument -w -e reportFile my-report-file.xml}
+ *
+ */ +public class JUnitReportTestRunner extends InstrumentationTestRunner { + /** + * Name of the report file(s) to write, may contain $(suite) in multiFile mode. + */ + private static final String ARG_REPORT_FILE = "reportFile"; + /** + * If specified, path of the directory to write report files to. If not set the files are + * written to the test application's data area. + */ + private static final String ARG_REPORT_DIR = "reportDir"; + /** + * If true, stack traces in the report will be filtered to remove common noise (e.g. framework + * methods). + */ + private static final String ARG_FILTER_TRACES = "filterTraces"; + /** + * If true, produce a separate file for each test suite. By default a single report is created + * for all suites. + */ + private static final String ARG_MULTI_FILE = "multiFile"; + /** + * Default name of the single report file. + */ + private static final String DEFAULT_SINGLE_REPORT_FILE = "junit-report.xml"; + /** + * Default name pattern for multiple report files. + */ + private static final String DEFAULT_MULTI_REPORT_FILE = "junit-report-$(suite).xml"; + + private JUnitReportListener mListener; + private String mReportFile; + private String mReportDir; + private boolean mFilterTraces = true; + private boolean mMultiFile = false; + + @Override + public void onCreate(Bundle arguments) { + if (arguments != null) { + mReportFile = arguments.getString(ARG_REPORT_FILE); + mReportDir = arguments.getString(ARG_REPORT_DIR); + mFilterTraces = getBooleanArgument(arguments, ARG_FILTER_TRACES, true); + mMultiFile = getBooleanArgument(arguments, ARG_MULTI_FILE, false); + } + + if (mReportFile == null) { + mReportFile = mMultiFile ? DEFAULT_MULTI_REPORT_FILE : DEFAULT_SINGLE_REPORT_FILE; + } + + super.onCreate(arguments); + } + + private boolean getBooleanArgument(Bundle arguments, String name, boolean defaultValue) + { + String value = arguments.getString(name); + if (value == null) { + return defaultValue; + } else { + return Boolean.parseBoolean(value); + } + } + + /** you can subclass and override this if you want to use a different TestRunner */ + protected AndroidTestRunner makeAndroidTestRunner() { + return new AndroidTestRunner(); + } + + @Override + protected AndroidTestRunner getAndroidTestRunner() { + AndroidTestRunner runner = makeAndroidTestRunner(); + mListener = new JUnitReportListener(getContext(), getTargetContext(), mReportFile, mReportDir, mFilterTraces, mMultiFile); + runner.addTestListener(mListener); + return runner; + } + + @Override + public void finish(int resultCode, Bundle results) { + if (mListener != null) { + mListener.close(); + } + + super.finish(resultCode, results); + } +}