dune-ctest

#! /usr/bin/env python3
#
# Wrapper around CTest for DUNE
#
# CTest returns with an error status not only when tests failed, but also
# when tests were only skipped.  This wrapper checks the log and returns
# successfully if no tests failed; skipped tests do not result in an error.
# This behaviour is needed in a continuous integration environment, when
# building binary packages or in other cases where the testsuite should be
# run automatically.
#
# Moreover, this script also converts the XML test report generated by CTest
# into a JUnit report file that can be consumed by a lot of reporting
# software.
#
# Author: Ansgar Burchardt <Ansgar.Burchardt@tu-dresden.de>
# Author: Steffen Müthing <steffen.muething@iwr.uni-heidelberg.de> (for the JUnit part)

import errno
import glob
import os.path
import shutil
import subprocess
import sys
import xml.etree.ElementTree as et
from pathlib import Path
import os
import re


class CTestParser:

    def findCTestOutput(self):
        files = glob.glob("Testing/*/Test.xml")
        if len(files) != 1:
            fn = files.join(", ")
            raise Exception("Found multiple CTest output files: {}".format(files.join(", ")))
        return files[0]

    def printTest(self,test,output=None):
        status = test.get("Status")
        name = test.find("Name").text
        fullName = test.find("FullName").text
        if output is not None:
            output = test.find("Results").find("Measurement").find("Value").text

        print("======================================================================")
        print("Name:      {}".format(name))
        print("FullName:  {}".format(fullName))
        print("Status:    {}".format(status.upper()))
        if output:
            print("Output:")
            for line in output.splitlines():
                print("          ", line)
        print()

    def __init__(self,junitpath=None):
        self.inputpath = self.findCTestOutput()
        if junitpath is None:
            if "CI_PROJECT_DIR" in os.environ:
                buildroot = Path(os.environ["CI_PROJECT_DIR"])
                # create a slug from the project name
                name = os.environ["CI_PROJECT_NAME"].lower()
                name = re.sub(r"[^-a-z0-9]","-",name);
                junitbasename = "{}-".format(name)
            else:
                buildroot = Path.cwd()
                junitbasename = ""
            junitdir = buildroot / "junit"
            junitdir.mkdir(parents=True,exist_ok=True)
            self.junitpath = junitdir / "{}cmake.xml".format(junitbasename)
        else:
            self.junitpath = Path(junitpath)
            junitdir = junitpath.resolve().parent
            junitdir.mkdir(parents=True,exist_ok=True)
        self.tests = 0
        self.passed = 0
        self.failures = 0
        self.skipped = 0
        self.errors = 0
        self.skipped = 0
        self.time = 0.0

    def createJUnitSkeleton(self):
        self.testsuites = et.Element("testsuites")
        self.testsuite = et.SubElement(self.testsuites,"testsuite")
        self.properties = et.SubElement(self.testsuite,"properties")

    def fillJUnitStatistics(self):
        self.testsuite.set("name","cmake")
        self.testsuite.set("tests",str(self.tests))
        self.testsuite.set("disabled","0")
        self.testsuite.set("errors",str(self.errors))
        self.testsuite.set("failures",str(self.failures))
        self.testsuite.set("skipped",str(self.skipped))
        self.testsuite.set("time",str(self.time))

    def processTest(self,test):
        testcase = et.SubElement(self.testsuite,"testcase")
        testcase.set("name",test.find("Name").text)
        testcase.set("assertions","1")
        testcase.set("classname","cmake")
        time = test.find("./Results/NamedMeasurement[@name='Execution Time']/Value")
        if time is not None:
            self.time += float(time.text)
            testcase.set("time",time.text)
        self.tests += 1
        outcome = test.get("Status")
        if outcome == "passed":
            testcase.set("status","passed")
            self.passed += 1
        elif outcome == "failed":
            self.failures += 1
            testcase.set("status","failure")
            failure = et.SubElement(testcase,"failure")
            failure.set("message","program execution failed")
            failure.text = test.find("./Results/Measurement/Value").text
            self.printTest(test)
        elif outcome == "notrun":
            # This does not exit on older CMake versions, so work around that
            try:
                status = test.find("./Results/NamedMeasurement[@name='Completion Status']/Value").text
                if status == "SKIP_RETURN_CODE=77":
                    self.skipped += 1
                    et.SubElement(testcase,"skipped")
                elif status == "Required Files Missing":
                    self.errors += 1
                    error = et.SubElement(testcase,"error")
                    error.set("message","compilation failed")
                    error.set("type","compilation error")
                    self.printTest(test,output="Compilation error")
                else:
                    error = et.SubElement(testcase,"error")
                    error.set("message","unknown error during test execution")
                    error.set("type","unknown")
                    error.text = test.find("./Results/Measurement/Value").text
                    self.errors += 1
                    self.printTest(test)
            except AttributeError:
                output_tag = test.find("./Results/Measurement/Value")
                if output_tag is not None:
                    msg = output_tag.text
                    if "skipped" in msg:
                        self.skipped += 1
                        et.SubElement(testcase,"skipped")
                    elif "Unable to find required file" in msg:
                        self.errors += 1
                        error = et.SubElement(testcase,"error")
                        error.set("message","compilation failed")
                        error.set("type","compilation error")
                        self.printTest(test,output="Compilation error")
                    else:
                        error = et.SubElement(testcase,"error")
                        error.set("message","unknown error during test execution")
                        error.set("type","unknown")
                        error.text = msg
                        self.errors += 1
                        self.printTest(test)
                else:
                    error = et.SubElement(testcase,"error")
                    error.set("message","unknown error during test execution")
                    error.set("type","unknown")
                    error.text = "no message"
                    self.errors += 1
                    self.printTest(test)

        output_tag = test.find("./Results/Measurement/Value")
        if output_tag is not None:
            out = et.SubElement(testcase,"system-out")
            out.text = output_tag.text

    def process(self):

        with open(self.inputpath, "r", encoding="utf-8") as fh:
            tree = et.parse(fh)

        root = tree.getroot()

        self.createJUnitSkeleton()

        for test in root.findall(".//Testing/Test"):
            self.processTest(test)

        self.fillJUnitStatistics()

        with self.junitpath.open("wb") as fh:
            fh.write(et.tostring(self.testsuites,encoding="utf-8"))
            print("JUnit report for CTest results written to {}".format(self.junitpath))

        return self.errors + self.failures


def runCTest(argv=[]):
    cmd = ["ctest",
           "--output-on-failure",
           "--dashboard", "ExperimentalTest",
           "--no-compress-output",
    ]
    cmd.extend(argv)
    subprocess.call(cmd)

def checkDirectory():
    if not os.path.exists("CMakeCache.txt"):
        raise Exception("ERROR: dune-ctest must be run in a cmake build directory")

def removeCTestOutput():
    try:
        shutil.rmtree("Testing")
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise

def main():
    try:
        checkDirectory()
        removeCTestOutput()
        runCTest(argv=sys.argv[1:])
        parser = CTestParser()
        errors = parser.process()
        status = 0 if errors == 0 else 1
        sys.exit(status)
    except Exception as e:
        print("Internal error: {}".format(e))
        sys.exit(127)

if __name__ == "__main__":
    main()