#! /usr/bin/env python3
#
# Wrapper around CTest for DUNE
#
# CTest returns with an error status not only when tests failed, but also
# when tests were only skipped. This wrapper checks the log and returns
# successfully if no tests failed; skipped tests do not result in an error.
# This behaviour is needed in a continuous integration environment, when
# building binary packages or in other cases where the testsuite should be
# run automatically.
#
# Moreover, this script also converts the XML test report generated by CTest
# into a JUnit report file that can be consumed by a lot of reporting
# software.
#
# Author: Ansgar Burchardt <Ansgar.Burchardt@tu-dresden.de>
# Author: Steffen Müthing <steffen.muething@iwr.uni-heidelberg.de> (for the JUnit part)
import errno
import glob
import os.path
import shutil
import subprocess
import sys
import xml.etree.ElementTree as et
from pathlib import Path
import os
import re
class CTestParser:
def findCTestOutput(self):
files = glob.glob("Testing/*/Test.xml")
if len(files) != 1:
fn = files.join(", ")
raise Exception("Found multiple CTest output files: {}".format(files.join(", ")))
return files[0]
def printTest(self,test,output=None):
status = test.get("Status")
name = test.find("Name").text
fullName = test.find("FullName").text
if output is not None:
output = test.find("Results").find("Measurement").find("Value").text
print("======================================================================")
print("Name: {}".format(name))
print("FullName: {}".format(fullName))
print("Status: {}".format(status.upper()))
if output:
print("Output:")
for line in output.splitlines():
print(" ", line)
print()
def __init__(self,junitpath=None):
self.inputpath = self.findCTestOutput()
if junitpath is None:
if "CI_PROJECT_DIR" in os.environ:
buildroot = Path(os.environ["CI_PROJECT_DIR"])
# create a slug from the project name
name = os.environ["CI_PROJECT_NAME"].lower()
name = re.sub(r"[^-a-z0-9]","-",name);
junitbasename = "{}-".format(name)
else:
buildroot = Path.cwd()
junitbasename = ""
junitdir = buildroot / "junit"
junitdir.mkdir(parents=True,exist_ok=True)
self.junitpath = junitdir / "{}cmake.xml".format(junitbasename)
else:
self.junitpath = Path(junitpath)
junitdir = junitpath.resolve().parent
junitdir.mkdir(parents=True,exist_ok=True)
self.tests = 0
self.passed = 0
self.failures = 0
self.skipped = 0
self.errors = 0
self.skipped = 0
self.time = 0.0
def createJUnitSkeleton(self):
self.testsuites = et.Element("testsuites")
self.testsuite = et.SubElement(self.testsuites,"testsuite")
self.properties = et.SubElement(self.testsuite,"properties")
def fillJUnitStatistics(self):
self.testsuite.set("name","cmake")
self.testsuite.set("tests",str(self.tests))
self.testsuite.set("disabled","0")
self.testsuite.set("errors",str(self.errors))
self.testsuite.set("failures",str(self.failures))
self.testsuite.set("skipped",str(self.skipped))
self.testsuite.set("time",str(self.time))
def processTest(self,test):
testcase = et.SubElement(self.testsuite,"testcase")
testcase.set("name",test.find("Name").text)
testcase.set("assertions","1")
testcase.set("classname","cmake")
time = test.find("./Results/NamedMeasurement[@name='Execution Time']/Value")
if time is not None:
self.time += float(time.text)
testcase.set("time",time.text)
self.tests += 1
outcome = test.get("Status")
if outcome == "passed":
testcase.set("status","passed")
self.passed += 1
elif outcome == "failed":
self.failures += 1
testcase.set("status","failure")
failure = et.SubElement(testcase,"failure")
failure.set("message","program execution failed")
failure.text = test.find("./Results/Measurement/Value").text
self.printTest(test)
elif outcome == "notrun":
# This does not exit on older CMake versions, so work around that
try:
status = test.find("./Results/NamedMeasurement[@name='Completion Status']/Value").text
if status == "SKIP_RETURN_CODE=77":
self.skipped += 1
et.SubElement(testcase,"skipped")
elif status == "Required Files Missing":
self.errors += 1
error = et.SubElement(testcase,"error")
error.set("message","compilation failed")
error.set("type","compilation error")
self.printTest(test,output="Compilation error")
else:
error = et.SubElement(testcase,"error")
error.set("message","unknown error during test execution")
error.set("type","unknown")
error.text = test.find("./Results/Measurement/Value").text
self.errors += 1
self.printTest(test)
except AttributeError:
output_tag = test.find("./Results/Measurement/Value")
if output_tag is not None:
msg = output_tag.text
if "skipped" in msg:
self.skipped += 1
et.SubElement(testcase,"skipped")
elif "Unable to find required file" in msg:
self.errors += 1
error = et.SubElement(testcase,"error")
error.set("message","compilation failed")
error.set("type","compilation error")
self.printTest(test,output="Compilation error")
else:
error = et.SubElement(testcase,"error")
error.set("message","unknown error during test execution")
error.set("type","unknown")
error.text = msg
self.errors += 1
self.printTest(test)
else:
error = et.SubElement(testcase,"error")
error.set("message","unknown error during test execution")
error.set("type","unknown")
error.text = "no message"
self.errors += 1
self.printTest(test)
output_tag = test.find("./Results/Measurement/Value")
if output_tag is not None:
out = et.SubElement(testcase,"system-out")
out.text = output_tag.text
def process(self):
with open(self.inputpath, "r", encoding="utf-8") as fh:
tree = et.parse(fh)
root = tree.getroot()
self.createJUnitSkeleton()
for test in root.findall(".//Testing/Test"):
self.processTest(test)
self.fillJUnitStatistics()
with self.junitpath.open("wb") as fh:
fh.write(et.tostring(self.testsuites,encoding="utf-8"))
print("JUnit report for CTest results written to {}".format(self.junitpath))
return self.errors + self.failures
def runCTest(argv=[]):
cmd = ["ctest",
"--output-on-failure",
"--dashboard", "ExperimentalTest",
"--no-compress-output",
]
cmd.extend(argv)
subprocess.call(cmd)
def checkDirectory():
if not os.path.exists("CMakeCache.txt"):
raise Exception("ERROR: dune-ctest must be run in a cmake build directory")
def removeCTestOutput():
try:
shutil.rmtree("Testing")
except OSError as e:
if e.errno != errno.ENOENT:
raise
def main():
try:
checkDirectory()
removeCTestOutput()
runCTest(argv=sys.argv[1:])
parser = CTestParser()
errors = parser.process()
status = 0 if errors == 0 else 1
sys.exit(status)
except Exception as e:
print("Internal error: {}".format(e))
sys.exit(127)
if __name__ == "__main__":
main()