/*
* Copyright (C) 2015 Daniel Dietsch (dietsch@informatik.uni-freiburg.de)
* Copyright (C) 2015 Jan Leike (leike@informatik.uni-freiburg.de)
* Copyright (C) 2015 University of Freiburg
*
* This file is part of the ULTIMATE UnitTest Library.
*
* The ULTIMATE UnitTest Library is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* The ULTIMATE UnitTest Library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with the ULTIMATE UnitTest Library. If not, see .
*
* Additional permission under GNU GPL version 3 section 7:
* If you modify the ULTIMATE UnitTest Library, or any covered work, by linking
* or combining it with Eclipse RCP (or a modified version of Eclipse RCP),
* containing parts covered by the terms of the Eclipse Public License, the
* licensors of the ULTIMATE UnitTest Library grant you additional permission
* to convey the resulting work.
*/
package de.uni_freiburg.informatik.ultimate.test.decider;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import de.uni_freiburg.informatik.ultimate.core.lib.results.NoResult;
import de.uni_freiburg.informatik.ultimate.core.lib.results.NonTerminationArgumentResult;
import de.uni_freiburg.informatik.ultimate.core.lib.results.SyntaxErrorResult;
import de.uni_freiburg.informatik.ultimate.core.lib.results.TerminationArgumentResult;
import de.uni_freiburg.informatik.ultimate.core.model.results.IResult;
import de.uni_freiburg.informatik.ultimate.core.model.services.ILogger;
import de.uni_freiburg.informatik.ultimate.core.model.services.IResultService;
import de.uni_freiburg.informatik.ultimate.core.model.services.IUltimateServiceProvider;
import de.uni_freiburg.informatik.ultimate.test.util.TestUtil;
/**
* Read the predefined expected result from the input file and compare it to ULTIMATE's output
*
* @author Jan Leike
*/
public class LassoRankerTestResultDecider extends TestResultDecider {
/**
* Types of results that can be specified by test examples
*
* @author Jan Leike
*/
public enum ExpectedResult {
IGNORE, TERMINATION, TERMINATIONDERIVABLE, NONTERMINATION, NONTERMINATIONDERIVABLE, UNKNOWN, ERROR, UNSPECIFIED
}
private final String minput_file_name;
private final ExpectedResult mexpected_result;
public LassoRankerTestResultDecider(final File inputFile) {
minput_file_name = inputFile.getName();
mexpected_result = checkExpectedResult(inputFile);
}
/**
* Return the expected result
*/
public ExpectedResult getExpectedResult() {
return mexpected_result;
}
/**
* Read the expected result from an input file
*
* Expected results are expected to be specified in an input file's first line and start with '//#r'.
*/
private static ExpectedResult checkExpectedResult(final File inputFile) {
BufferedReader br;
String line = null;
try {
br = new BufferedReader(new FileReader(inputFile));
line = br.readLine();
br.close();
} catch (final IOException e) {
line = null;
}
if (line != null && line.startsWith("//#r")) {
final String expected = line.substring(4).toLowerCase();
if (expected.equals("ignore")) {
return ExpectedResult.IGNORE;
} else if (expected.equals("termination")) {
return ExpectedResult.TERMINATION;
} else if (expected.equals("terminationderivable")) {
return ExpectedResult.TERMINATIONDERIVABLE;
} else if (expected.equals("nontermination")) {
return ExpectedResult.NONTERMINATION;
} else if (expected.equals("nonterminationderivable")) {
return ExpectedResult.NONTERMINATIONDERIVABLE;
} else if (expected.equals("unknown")) {
return ExpectedResult.UNKNOWN;
} else if (expected.equals("error")) {
return ExpectedResult.ERROR;
} else {
return null;
}
}
if (line != null) {
return ExpectedResult.UNSPECIFIED;
}
return null;
}
@Override
public TestResult getTestResult(final IUltimateServiceProvider services) {
final ILogger logger = services.getLoggingService().getLogger(LassoRankerTestResultDecider.class);
final IResultService resultService = services.getResultService();
final Collection customMessages = new LinkedList<>();
boolean fail = false;
String result = "";
if (mexpected_result == null) {
customMessages.add("Could not understand the specification of the " + "results.");
fail = true;
} else if (mexpected_result == ExpectedResult.UNSPECIFIED) {
customMessages.add("No expected results defined in the input file");
} else {
final Map> resultMap = resultService.getResults();
final List results =
resultMap.get("de.uni_freiburg.informatik.ultimate.plugins.analysis.lassoranker");
if (results != null) {
final IResult lastResult = results.get(results.size() - 1);
customMessages.add("Expected Result: " + mexpected_result.toString());
customMessages.add("Results: " + results.toString());
switch (mexpected_result) {
case TERMINATION:
fail = lastResult instanceof NonTerminationArgumentResult;
break;
case TERMINATIONDERIVABLE:
fail = !(lastResult instanceof TerminationArgumentResult);
break;
case NONTERMINATION:
fail = lastResult instanceof TerminationArgumentResult;
break;
case NONTERMINATIONDERIVABLE:
fail = !(lastResult instanceof NonTerminationArgumentResult);
break;
case UNKNOWN:
fail = !(lastResult instanceof NoResult);
break;
case ERROR:
fail = !(lastResult instanceof SyntaxErrorResult);
break;
default:
fail = true;
break;
}
result = lastResult.getClass().getSimpleName();
} else {
fail = true;
}
}
setResultCategory(result + " (Expected: " + mexpected_result + ")");
setResultMessage(customMessages.toString());
TestUtil.logResults(logger, minput_file_name, fail, customMessages, resultService);
return fail ? TestResult.FAIL : TestResult.SUCCESS;
}
@Override
public TestResult getTestResult(final IUltimateServiceProvider service, final Throwable e) {
return TestResult.FAIL;
}
}