/* * Copyright (C) 2014-2015 Daniel Dietsch (dietsch@informatik.uni-freiburg.de) * Copyright (C) 2015 University of Freiburg * * This file is part of the ULTIMATE UnitTest Library. * * The ULTIMATE UnitTest Library is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published * by the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * The ULTIMATE UnitTest Library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with the ULTIMATE UnitTest Library. If not, see . * * Additional permission under GNU GPL version 3 section 7: * If you modify the ULTIMATE UnitTest Library, or any covered work, by linking * or combining it with Eclipse RCP (or a modified version of Eclipse RCP), * containing parts covered by the terms of the Eclipse Public License, the * licensors of the ULTIMATE UnitTest Library grant you additional permission * to convey the resulting work. */ package de.uni_freiburg.informatik.ultimate.test; import java.util.List; import org.eclipse.core.runtime.IStatus; import de.uni_freiburg.informatik.ultimate.core.coreplugin.exceptions.LifecycleException; import de.uni_freiburg.informatik.ultimate.core.model.services.ILogger; import de.uni_freiburg.informatik.ultimate.core.model.services.IResultService; import de.uni_freiburg.informatik.ultimate.core.model.services.IUltimateServiceProvider; import de.uni_freiburg.informatik.ultimate.test.decider.ITestResultDecider; import de.uni_freiburg.informatik.ultimate.test.decider.ITestResultDecider.TestResult; import de.uni_freiburg.informatik.ultimate.test.junitextension.testfactory.FactoryTestMethod; import de.uni_freiburg.informatik.ultimate.test.junitextension.testfactory.FactoryTestRunner.SkipTestException; import de.uni_freiburg.informatik.ultimate.test.mocks.ConsoleLogger; import de.uni_freiburg.informatik.ultimate.test.reporting.IIncrementalLog; import de.uni_freiburg.informatik.ultimate.test.reporting.ITestLogfile; import de.uni_freiburg.informatik.ultimate.test.reporting.ITestSummary; import de.uni_freiburg.informatik.ultimate.util.CoreUtil; /** * @author Daniel Dietsch (dietsch@informatik.uni-freiburg.de) * */ public final class UltimateTestCase implements Comparable { private static final AfterTest NOOP = new AfterTest() { @Override public void afterTest() { // do nothing } }; private final String mName; private final UltimateRunDefinition mUltimateRunDefinition; private final ConsoleLogger mTestLogger; private boolean mHasStarted; private List mLogs; private ITestResultDecider mDecider; private AfterTest mFunAfterTest; public UltimateTestCase(final ITestResultDecider decider, final UltimateRunDefinition urd, final List logs) { if (urd == null) { throw new IllegalArgumentException("ultimateRunDefinition"); } mDecider = decider; mLogs = logs; mName = urd.toString(); mUltimateRunDefinition = urd; mTestLogger = new ConsoleLogger(); mHasStarted = false; if (urd.getAfterTestMethod() == null) { mFunAfterTest = NOOP; } else { mFunAfterTest = urd.getAfterTestMethod(); } } @FactoryTestMethod public void test() { if (mHasStarted) { throw new UltimateTestFailureException("Can run a test only once"); } mHasStarted = true; // call the garbage collector before starting a new test System.gc(); System.runFinalization(); System.gc(); Runtime.getRuntime().gc(); // start debug code: use this only in controlled situations! // try { // Thread.sleep(500); // } catch (final InterruptedException e1) { // } // HeapDumper.dumpHeap("C:\\Installed\\mat\\dumps\\", false); // end debug code Throwable th = null; TestResult result = TestResult.FAIL; boolean livecycleFailure = false; final UltimateStarter starter = new UltimateStarter(mUltimateRunDefinition); try { updateLogsPre(); final String deciderName = mDecider.getClass().getSimpleName(); final IStatus returnCode = starter.runUltimate(); // logging service is only available after runUltimate() has been called final ILogger logger = getLoggerFromStarter(starter); logger.info("Deciding this test: " + deciderName); result = mDecider.getTestResult(starter.getServices()); if (!returnCode.isOK() && result != TestResult.FAIL && result != TestResult.IGNORE) { logger.fatal("#################### Overwriting decision of " + deciderName + " and setting test status to FAIL ####################"); logger.fatal("Ultimate returned an unexpected status:"); logger.fatal("Code " + returnCode.getCode()); logger.fatal("Severity " + returnCode.getSeverity()); logger.fatal("Message " + returnCode.getMessage()); logger.fatal("Plugin ID " + returnCode.getPlugin()); if (returnCode.getException() != null) { logger.fatal("Exception:", returnCode.getException()); } result = TestResult.FAIL; } } catch (final LifecycleException lex) { // if this happens, mStarter, mLogger, etc. are not initialized th = lex; result = TestResult.FAIL; lex.printStackTrace(); livecycleFailure = true; } catch (final Throwable e) { th = e; result = mDecider.getTestResult(starter.getServices(), e); logWithFreshLogger(starter, e, "There was an exception during the execution of Ultimate"); } finally { boolean success = false; if (!livecycleFailure) { success = mDecider.getJUnitSuccess(result); } final String resultMessage = mDecider.getResultMessage(); final String resultCategory = mDecider.getResultCategory(); try { updateLogsPost(starter.getServices(), result, resultCategory, resultMessage); } catch (final Throwable ex) { logWithFreshLogger(starter, ex, "There was an exception during the writing of summary or log information"); throw ex; } starter.complete(); mDecider = null; mLogs = null; if (!success) { String message = null; if (!livecycleFailure) { message = resultMessage; } if (message == null) { message = "ITestResultDecider provided no message"; } if (th != null) { message += " (Ultimate threw an Exception: " + th.getMessage() + ")"; if (result == TestResult.IGNORE) { skipTest(message, th); } failTest(message, th); } else if (result == TestResult.IGNORE) { skipTest(message); } else { failTest(message); } } try { mFunAfterTest.afterTest(); } catch (final Throwable ex) { logWithFreshLogger(starter, ex, "There was an exception during execution of after-test method"); } mFunAfterTest = null; } } private static ILogger getLoggerFromStarter(final UltimateStarter starter) { if (starter.getServices() == null) { return new ConsoleLogger(); } return starter.getServices().getLoggingService().getLogger(UltimateStarter.class); } private static void logWithFreshLogger(final UltimateStarter starter, final Throwable ex, final String message) { getLoggerFromStarter(starter).fatal(String.format(message + ": %s%n%s", ex, CoreUtil.getStackTrace(ex))); } private void updateLogsPre() { if (mLogs == null) { return; } mLogs.stream().filter(a -> a instanceof IIncrementalLog).map(a -> (IIncrementalLog) a) .forEach(a -> a.addEntryPreStart(mUltimateRunDefinition, mTestLogger)); } private void updateLogsPost(final IUltimateServiceProvider services, final TestResult result, final String resultCategory, final String resultMessage) { if (mLogs == null) { return; } IResultService rservice = null; if (services != null) { rservice = services.getResultService(); assert rservice != null : "Could not retrieve ResultService"; } for (final ITestLogfile log : mLogs) { if (log instanceof IIncrementalLog) { ((IIncrementalLog) log).addEntryPostCompletion(mUltimateRunDefinition, result, resultCategory, resultMessage, services, mTestLogger); } if (log instanceof ITestSummary) { ((ITestSummary) log).addResult(mUltimateRunDefinition, result, resultCategory, resultMessage, mName, rservice); } } } public UltimateRunDefinition getUltimateRunDefinition() { return mUltimateRunDefinition; } @Override public String toString() { return mName; } private static void failTest(final String message) throws UltimateTestFailureException { final UltimateTestFailureException exception = new UltimateTestFailureException(message); final StackTraceElement elem = new StackTraceElement(UltimateTestCase.class.getPackage().getName(), "test", "UltimateTestCase.java", -1); exception.setStackTrace(new StackTraceElement[] { elem }); throw exception; } private static void failTest(final String message, final Throwable th) throws UltimateTestFailureException { final UltimateTestFailureException exception = new UltimateTestFailureException(message, th); final StackTraceElement elem = new StackTraceElement(UltimateTestCase.class.getPackage().getName(), "test", "UltimateTestCase.java", -1); exception.setStackTrace(new StackTraceElement[] { elem }); throw exception; } private static void skipTest(final String message) { throw new SkipTestException(message); } private static void skipTest(final String message, final Throwable cause) { throw new SkipTestException(message, cause); } @Override public int compareTo(final UltimateTestCase o) { return mUltimateRunDefinition.compareTo(o.mUltimateRunDefinition); } /** * A method that can be executed after a test. * * @author Daniel Dietsch (dietsch@informatik.uni-freiburg.de) * */ @FunctionalInterface public interface AfterTest { public void afterTest(); } }