Commit cc81f07d authored by Steffen Müthing's avatar Steffen Müthing

[!34] Output JUnit test results and improve skipped test detection log

Merge branch 'feature/add-junit-support' into 'master'

ref:docker/ci

-   dune-ctest now writes a JUnit XML result file to Testing/cmake.xml. This
    can be picked up by GitLab for better error reporting.
-   Right now, tests that fail because compilation failed are not detected as
    an error. This is fixed now.

See merge request [!34]

  [!34]: gitlab.dune-project.org/docker/ci/merge_requests/34
parents 751b9722 f334b1f8
......@@ -17,12 +17,23 @@ import os.path
import shutil
import subprocess
import sys
import xml.etree.ElementTree
import xml.etree.ElementTree as et
def printTest(test):
class CTestParser:
def findCTestOutput(self):
files = glob.glob("Testing/*/Test.xml")
if len(files) != 1:
fn = files.join(", ")
raise Exception("Found multiple CTest output files: {}".format(files.join(", ")))
return files[0]
def printTest(self,test,output=None):
status = test.get("Status")
name = test.find("Name").text
fullName = test.find("FullName").text
if output is not None:
output = test.find("Results").find("Measurement").find("Value").text
print("======================================================================")
......@@ -35,39 +46,91 @@ def printTest(test):
print(" ", line)
print()
def handleTest(test):
status = test.get("Status")
name = test.find("Path").text
if status == "passed":
passed = True
elif status == "notrun":
printTest(test)
passed = True
def __init__(self):
self.inputpath = self.findCTestOutput()
self.tests = 0
self.passed = 0
self.failures = 0
self.skipped = 0
self.errors = 0
self.skipped = 0
self.time = 0.0
def createJUnitSkeleton(self):
self.testsuites = et.Element("testsuites")
self.testsuite = et.SubElement(self.testsuites,"testsuite")
self.properties = et.SubElement(self.testsuite,"properties")
def fillJUnitStatistics(self):
self.testsuite.set("name","cmake")
self.testsuite.set("tests",str(self.tests))
self.testsuite.set("disabled","0")
self.testsuite.set("errors",str(self.errors))
self.testsuite.set("failures",str(self.failures))
self.testsuite.set("skipped",str(self.skipped))
self.testsuite.set("time",str(self.time))
def processTest(self,test):
testcase = et.SubElement(self.testsuite,"testcase")
testcase.set("name",test.find("Name").text)
testcase.set("assertions","1")
testcase.set("classname","cmake")
time = test.find("./Results/NamedMeasurement[@name='Execution Time']/Value")
if time is not None:
self.time += float(time.text)
testcase.set("time",time.text)
self.tests += 1
outcome = test.get("Status")
if outcome == "passed":
testcase.set("status","passed")
self.passed += 1
elif outcome == "failed":
self.failures += 1
testcase.set("status","failure")
failure = et.SubElement(testcase,"failure")
failure.set("message","program execution failed")
failure.text = test.find("./Results/Measurement/Value").text
self.printTest(test)
elif outcome == "notrun":
status = test.find("./Results/NamedMeasurement[@name='Completion Status']/Value").text
if status == "SKIP_RETURN_CODE=77":
self.skipped += 1
et.SubElement(testcase,"skipped")
elif status == "Required Files Missing":
self.errors += 1
error = et.SubElement(testcase,"error")
error.set("message","compilation failed")
error.set("type","compilation error")
self.printTest(test,output="Compilation error")
else:
printTest(test)
passed = False
return passed
error = et.SubElement(testcase,"error")
error.message("unknown error during test execution")
error.type("unknown")
error.text = test.find("./Results/Measurement/Value").text
self.errors += 1
self.printTest(test)
out = et.SubElement(testcase,"system-out")
out.text = test.find("./Results/Measurement/Value").text
def findCTestOutput():
files = glob.glob("Testing/*/Test.xml")
if len(files) != 1:
fn = files.join(", ")
raise Exception("Found multiple CTest output files: {}".format(files.join(", ")))
return files[0]
def process(self):
with open(self.inputpath, "r", encoding="utf-8") as fh:
tree = et.parse(fh)
def handleCTestOutput():
path = findCTestOutput()
with open(path, "r", encoding="latin-1") as fh:
tree = xml.etree.ElementTree.parse(fh)
root = tree.getroot()
testing = root.find("Testing")
passed = True
for test in testing.findall("Test"):
testPassed = handleTest(test)
passed = passed and testPassed
self.createJUnitSkeleton()
for test in root.findall(".//Testing/Test"):
self.processTest(test)
self.fillJUnitStatistics()
with open("Testing/cmake.xml", "wb") as fh:
fh.write(et.tostring(self.testsuites,encoding="utf-8"))
return self.errors + self.failures
return passed
def runCTest(argv=[]):
cmd = ["ctest",
......@@ -94,8 +157,9 @@ def main():
checkDirectory()
removeCTestOutput()
runCTest(argv=sys.argv[1:])
passed = handleCTestOutput()
status = 0 if passed else 1
parser = CTestParser()
errors = parser.process()
status = 0 if errors == 0 else 1
sys.exit(status)
except Exception as e:
print("Internal error: {}".format(e))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment