/* * Copyright (C) 2014-2015 Daniel Dietsch (dietsch@informatik.uni-freiburg.de) * Copyright (C) 2015 University of Freiburg * * This file is part of the ULTIMATE UnitTest Library. * * The ULTIMATE UnitTest Library is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published * by the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * The ULTIMATE UnitTest Library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with the ULTIMATE UnitTest Library. If not, see . * * Additional permission under GNU GPL version 3 section 7: * If you modify the ULTIMATE UnitTest Library, or any covered work, by linking * or combining it with Eclipse RCP (or a modified version of Eclipse RCP), * containing parts covered by the terms of the Eclipse Public License, the * licensors of the ULTIMATE UnitTest Library grant you additional permission * to convey the resulting work. */ package de.uni_freiburg.informatik.ultimate.test.decider; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedList; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; import org.eclipse.core.runtime.IPath; import de.uni_freiburg.informatik.ultimate.core.lib.results.CounterExampleResult; import de.uni_freiburg.informatik.ultimate.core.lib.results.ExceptionOrErrorResult; import de.uni_freiburg.informatik.ultimate.core.lib.results.ExternalWitnessValidationResult; import de.uni_freiburg.informatik.ultimate.core.lib.results.ExternalWitnessValidationResult.WitnessVerificationStatus; import de.uni_freiburg.informatik.ultimate.core.lib.results.GenericResult; import de.uni_freiburg.informatik.ultimate.core.lib.results.SyntaxErrorResult; import de.uni_freiburg.informatik.ultimate.core.lib.results.TypeErrorResult; import de.uni_freiburg.informatik.ultimate.core.model.results.IResult; import de.uni_freiburg.informatik.ultimate.core.model.services.ILogger; import de.uni_freiburg.informatik.ultimate.core.model.services.IResultService; import de.uni_freiburg.informatik.ultimate.core.model.services.IUltimateServiceProvider; import de.uni_freiburg.informatik.ultimate.test.UltimateRunDefinition; import de.uni_freiburg.informatik.ultimate.test.util.TestUtil; import de.uni_freiburg.informatik.ultimate.util.CoreUtil; /** * @author dietsch@informatik.uni-freiburg.de */ public class BacktranslationTestResultDecider extends TestResultDecider { private final UltimateRunDefinition mRunDefinition; private final String mInputFilePath; public BacktranslationTestResultDecider(final UltimateRunDefinition runDefinition) { mRunDefinition = runDefinition; mInputFilePath = runDefinition.selectPrimaryInputFile().getAbsolutePath(); } @Override public TestResult getTestResult(final IUltimateServiceProvider services) { setResultCategory(""); setResultMessage(""); final ILogger log = services.getLoggingService().getLogger(BacktranslationTestResultDecider.class); final Collection customMessages = new LinkedList<>(); customMessages.add("Expecting results to not contain GenericResult \"Unhandled Backtranslation\" " + ", ExceptionOrErrorResult or TypeErrorResult, " + "and that there is a counter example result, and that the contained error trace " + "matches the given one."); final List> cex = new ArrayList<>(); final List witnesses = new ArrayList<>(); final IResultService resultService = services.getResultService(); final List results = resultService.getResults().entrySet().stream().flatMap(a -> a.getValue().stream()) .collect(Collectors.toList()); for (final IResult result : results) { if (result instanceof ExceptionOrErrorResult || result instanceof TypeErrorResult || result instanceof SyntaxErrorResult) { setCategoryAndMessageAndCustomMessage(result.getShortDescription(), customMessages); TestUtil.logResults(log, mInputFilePath, true, customMessages, resultService); return TestResult.FAIL; } else if (result instanceof GenericResult) { final GenericResult genRes = (GenericResult) result; if (genRes.getShortDescription().equals("Unfinished Backtranslation")) { setResultCategory(result.getShortDescription()); setResultMessage(result.getLongDescription()); customMessages.add(result.getShortDescription() + ": " + result.getLongDescription()); TestUtil.logResults(log, mInputFilePath, true, customMessages, resultService); return TestResult.FAIL; } } else if (result instanceof CounterExampleResult) { cex.add((CounterExampleResult) result); } else if (result instanceof ExternalWitnessValidationResult) { witnesses.add((ExternalWitnessValidationResult) result); } } if (cex.isEmpty()) { setCategoryAndMessageAndCustomMessage("No counter example found", customMessages); TestUtil.logResults(log, mInputFilePath, true, customMessages, resultService); return TestResult.FAIL; } if (cex.size() > 1) { setResultCategory("More than one counter example found"); final String errorMsg = "There were " + cex.size() + " counter examples, but we expected only one"; setResultMessage(errorMsg); customMessages.add(errorMsg); TestUtil.logResults(log, mInputFilePath, true, customMessages, resultService); return TestResult.FAIL; } final List witnessesWithCex = new ArrayList<>(); for (final IResult result : cex) { final Optional witness = witnesses.stream().filter(a -> a.getAffectedResult() == result).findAny(); if (witness.isPresent()) { witnessesWithCex.add(witness.get()); } } if (!witnessesWithCex.isEmpty()) { // we expect witness verification for .c files to succeed for (final ExternalWitnessValidationResult witness : witnessesWithCex) { if (witness.isEmpty()) { setResultCategory("Empty Witness"); final String errorMsg = "The witness is empty: " + witness.getShortDescription(); setResultMessage(errorMsg); customMessages.add(errorMsg); TestUtil.logResults(log, mInputFilePath, true, customMessages, resultService); return TestResult.FAIL; } else if (witness.getExpectedVerificationStatus() == WitnessVerificationStatus.VERIFIED && witness.getVerificationStatus() != WitnessVerificationStatus.VERIFIED) { setResultCategory("Witness failed to verify"); final String errorMsg = "The witness failed to verify: " + witness.getLongDescription(); setResultMessage(errorMsg); customMessages.add(errorMsg); TestUtil.logResults(log, mInputFilePath, true, customMessages, resultService); return TestResult.FAIL; } } } // so far so good, now we compare the error path with the expected // error path boolean fail = false; String desiredCounterExample = null; try { desiredCounterExample = getDesiredCounterExample(log, mRunDefinition); } catch (final IOException e) { setResultCategory(e.getMessage()); setResultMessage(e.toString()); e.printStackTrace(); TestUtil.logResults(log, mInputFilePath, true, customMessages, resultService); return TestResult.FAIL; } final String actualCounterExample = cex.get(0).getProgramExecutionAsString(); if (desiredCounterExample == null) { setResultCategory("No .errorpath file for comparison"); final String errorMsg = String.format("There is no .errorpath file for %s!", mInputFilePath); tryWritingActualResultToFile(actualCounterExample); setResultMessage(errorMsg); customMessages.add(errorMsg); fail = true; } else { final StringCounterexample desired = new StringCounterexample(desiredCounterExample); final StringCounterexample actual = new StringCounterexample(actualCounterExample); final Difference difference = desired.getFirstDifference(actual); fail = difference != null; if (fail) { tryWritingActualResultToFile(actualCounterExample); setCategoryAndMessageAndCustomMessage("Desired error trace does not match actual error trace.", customMessages); customMessages.add("Lengths: Desired=" + desired.mLines.size() + " Actual=" + actual.mLines.size()); customMessages.addAll(difference.getCustomDifferenceMessage()); customMessages .add("Desired error trace:" + CoreUtil.getPlatformLineSeparator() + desiredCounterExample); customMessages.add("Actual error trace:" + CoreUtil.getPlatformLineSeparator() + actualCounterExample); } else { setResultCategory("Success"); } } TestUtil.logResults(log, mInputFilePath, fail, customMessages, resultService); return fail ? TestResult.FAIL : TestResult.SUCCESS; } private static String getDesiredCounterExample(final ILogger log, final UltimateRunDefinition rundefinition) throws IOException { final File inputFile = rundefinition.selectPrimaryInputFile(); final String inputDir = inputFile.getParentFile().getAbsolutePath(); final String inputFileNameWithoutEnding = removeFileEnding(inputFile); final String settingsFileNameWithoutEnding = removeFileEnding(rundefinition.getSettings()); final String toolchainFileNameWithoutEnding = removeFileEnding(rundefinition.getToolchain()); // order some candidates which we would like, we take the first that // matches final List candidates = new ArrayList<>(); candidates.add(getDesiredCounterExampleCandidate(inputDir, inputFileNameWithoutEnding + "_" + toolchainFileNameWithoutEnding + "_" + settingsFileNameWithoutEnding)); candidates.add(getDesiredCounterExampleCandidate(inputDir, inputFileNameWithoutEnding + "_" + settingsFileNameWithoutEnding)); candidates.add(getDesiredCounterExampleCandidate(inputDir, inputFileNameWithoutEnding)); log.info("Considering desired error paths:"); candidates.stream().forEach(log::info); for (final File candidate : candidates) { if (candidate.canRead()) { log.info("Using " + candidate); return CoreUtil.readFile(candidate); } } return null; } private static File getDesiredCounterExampleCandidate(final String inputDir, final String filename) { return new File(String.format("%s%s%s%s", inputDir, IPath.SEPARATOR, filename, ".errorpath")); } private static String removeFileEnding(final File file) { return file.getName().replaceAll("\\..*", ""); } @Override public TestResult getTestResult(final IUltimateServiceProvider services, final Throwable e) { setResultCategory("Unexpected exception"); setResultMessage("Unexpected exception: " + e.getMessage()); TestUtil.logResults(BacktranslationTestResultDecider.class, mInputFilePath, true, new ArrayList(), services); return TestResult.FAIL; } private void setCategoryAndMessageAndCustomMessage(final String msg, final Collection customMessages) { setResultCategory(msg); setResultMessage(msg); customMessages.add(msg); } private boolean tryWritingActualResultToFile(final String actualCounterExample) { final String[] actualLines = actualCounterExample.split(CoreUtil.getPlatformLineSeparator()); try { final File input = new File(mInputFilePath); final String path = input.getParentFile().getAbsolutePath(); final String name = removeFileEnding(input) + "_" + removeFileEnding(mRunDefinition.getToolchain()) + "_" + removeFileEnding(mRunDefinition.getSettings()); final String target = new File(String.format("%s%s%s%s", path, IPath.SEPARATOR, name, ".errorpath-actual")) .getAbsolutePath(); CoreUtil.writeFile(target, actualLines); return true; } catch (final IOException e) { return false; } } /** * @author Daniel Dietsch (dietsch@informatik.uni-freiburg.de) */ private static final class StringCounterexample { private final List mLines; private StringCounterexample(final String counterexampleAsString) { mLines = transformCex(counterexampleAsString); } private static List transformCex(final String counterexampleAsString) { final String platformLineSeparator = CoreUtil.getPlatformLineSeparator(); final String[] lines = counterexampleAsString.split(platformLineSeparator); final List rtr = new ArrayList<>(lines.length); for (int i = 0; i < lines.length; ++i) { final int j = i + 1; final String current = lines[i]; if (current.trim().startsWith("IVAL")) { rtr.add(new StringCounterexampleLine(null, current)); continue; } if (j < lines.length) { final String next = lines[j]; if (next.trim().startsWith("VAL")) { rtr.add(new StringCounterexampleLine(current, next)); i = j; continue; } } rtr.add(new StringCounterexampleLine(current, null)); } return rtr; } public Difference getFirstDifference(final StringCounterexample actual) { int i = 0; for (; i < mLines.size(); ++i) { final StringCounterexampleLine desiredLine = mLines.get(i); if (i >= actual.mLines.size()) { // other has less lines than this return new Difference(desiredLine, null); } final StringCounterexampleLine actualLine = actual.mLines.get(i); if (desiredLine.isSimilar(actualLine)) { continue; } return new Difference(desiredLine, actualLine); } if (i < actual.mLines.size()) { // other has more lines than this final StringCounterexampleLine actualLine = actual.mLines.get(i); return new Difference(null, actualLine); } return null; } } /** * @author Daniel Dietsch (dietsch@informatik.uni-freiburg.de) */ private static final class StringCounterexampleLine { private final String mLine; private final String mValueLine; /** * Expects Strings that represent a line of a counterexample and the following value line. Both can be null: the * value line if none is present, and the line if it is the initial value line. * */ private StringCounterexampleLine(final String line, final String valueLine) { if (line == null) { mLine = null; mValueLine = Objects.requireNonNull(valueLine); return; } mLine = Objects.requireNonNull(line); mValueLine = valueLine; } public boolean isSimilar(final StringCounterexampleLine actual) { if (Objects.equals(actual.mLine, mLine)) { if (mValueLine == null && actual.mValueLine == null) { return true; } else if (mValueLine == null || actual.mValueLine == null) { return false; } return isValueLineOk(mValueLine, actual.mValueLine); } return false; } /** * @param desiredValue * A line from the desired error trace * @param actualValue * The corresponding line from the actual error trace * @return true iff it is a value line and the values do not differ too much (i.e. there is the same number of * the same variables, but the values do not match) */ private static boolean isValueLineOk(final String desiredValueLine, final String actualValueLine) { final String desiredValue = desiredValueLine.trim(); final String actualValue = actualValueLine.trim(); if ((!desiredValue.startsWith("VAL") || !actualValue.startsWith("VAL")) && (!desiredValue.startsWith("IVAL") || !actualValue.startsWith("IVAL"))) { // not a valid value line return false; } final String[] curDesVals = getValues(desiredValue); final String[] curActVals = getValues(actualValue); if (curDesVals.length != curActVals.length) { // not the same amount of variables return false; } for (int i = 0; i < curDesVals.length; ++i) { final String[] singleDesVal = curDesVals[i].split("="); final String[] singleActVal = curActVals[i].split("="); if (singleDesVal.length != singleActVal.length) { return false; } // check for the name of the var if (!singleDesVal[0].equals(singleActVal[0])) { return false; } // check if the value is a pointer, we allow all values for the actual value if (singleDesVal.length != 2 || "*".equals(singleDesVal[1]) || singleDesVal[1].contains(":")) { continue; } if (!singleDesVal[1].equals(singleActVal[1])) { return false; } } return true; } private static String[] getValues(final String value) { final int lbracketIdx = value.indexOf('['); final int rbracketIdx = value.lastIndexOf(']'); final String values = value.substring(lbracketIdx + 1, rbracketIdx); return values.split(","); } @Override public String toString() { if (mLine == null) { return mValueLine; } else if (mValueLine == null) { return mLine; } else { return mLine + " followed by " + mValueLine; } } } private static final class Difference { private final StringCounterexampleLine mDesired; private final StringCounterexampleLine mActual; private Difference(final StringCounterexampleLine desired, final StringCounterexampleLine actual) { mDesired = desired; mActual = actual; } private List getCustomDifferenceMessage() { final List rtr = new ArrayList<>(); if (mDesired == null) { rtr.add("Counterexample should have been shorter, but was identical up to the following line."); rtr.add(mActual.toString()); } else if (mActual == null) { rtr.add("Counterexample should have been longer. The next missing line is the following."); rtr.add(mDesired.toString()); } else { rtr.add("Counterexamples first differ at the following lines."); rtr.add("Desired: " + mDesired.toString()); rtr.add("Actual: " + mActual.toString()); } return rtr; } } }