bin/check-log-store.in

-#!/bin/sh
-
-set -e
-
-# DB format
-#
-# <tag> <revision> <host> <mode> <module> <path> <errors> <warnings> <log>
-
-# parameter:
-# $(LOG_FILE) $(LOG_DIR) "build" "$$target" "$$path"
-
-host="@hostid@"
-tag="@tag@"
-revision="@revision@"
-
-logfile="$1"
-logdir="$2"
-mode="$3"
-module="`basename $4`"
-path="$5"
-errors="$6"
-warnings="$7"
-
-test -d $logdir || mkdir $logdir
-
-storelog=`tempfile -d $logdir`
-
-trap "rm -f $storelog" EXIT
-
-if true; then
-	echo "TAG: $tag"
-	echo "REVISION: $revision"
-	echo "HOST: $host"
-	echo "MODE: $mode"
-	echo "MODULE: $module"
-	echo "PATH: $path"
-	if test "x$errors" != x; then
-		echo "ERRORS: $errors"
-	fi
-	if test "x$warnings" != x; then
-		echo "WARNINGS: $warnings"
-	fi
-	echo "LOG:"
-	cat $logfile
-fi > $storelog
-
-trap - EXIT

bin/dune-add-spdx

+#! /usr/bin/env python3
+# SPDX-FileCopyrightInfo: Copyright © DUNE Project contributors, see file LICENSE.md in module root
+# SPDX-License-Identifier: LicenseRef-GPL-2.0-only-with-DUNE-exception
+
+import argparse
+import dataclasses
+import errno
+import logging
+import re
+
+# REUSE-IgnoreStart
+parser = argparse.ArgumentParser(
+    description='add SPDX header to DUNE project modules'
+)
+parser.add_argument(
+    '--license', '-l', dest='license',
+    help='SPDX license name',
+    default='LicenseRef-GPL-2.0-only-with-DUNE-exception',
+)
+parser.add_argument(
+    '--copyright', '-c', dest='copyright',
+    help='copyright information',
+    default='Copyright © DUNE Project contributors, see file LICENSE.md in module root',
+)
+parser.add_argument(
+    '--like', dest='like',
+    help='handle files like a file named like this',
+),
+parser.add_argument(
+    '--log', dest='log',
+    help='set log level',
+),
+parser.add_argument(
+    'files', nargs='+',
+    help='files to add copyright and license headers to',
+)
+
+
+@dataclasses.dataclass
+class Rule:
+    pattern: str
+    prefix: str = ""
+    suffix: str = ""
+    line_prefix: str = ""
+    line_suffix: str = ""
+    use_license_file: bool = False
+    skip_lines: re.Pattern = None
+
+    def match(self, filename) -> bool:
+        return bool(re.search(self.pattern, filename))
+
+
+rules = [
+    Rule(
+        pattern=r"\.(?:c|cc|cc\.in|h|hh|hh\.in|ct|ht|t|geo)$|config\.h\.cmake$",
+        line_prefix="// ",
+    ),
+    Rule(
+        pattern=r"\.(?:rst|rst\.in)$",
+        prefix="::\n",
+        line_prefix="  ",
+        suffix="\n",
+    ),
+    Rule(
+        pattern=r"\.bib$",
+        line_prefix="@Comment ",
+        suffix="\n",
+    ),
+    Rule(
+        pattern=r"\.dgf$",
+        line_prefix="% ",
+        skip_lines=re.compile("^DGF"),
+    ),
+    Rule(
+        pattern=r"\.tex$",
+        line_prefix="% ",
+        suffix="\n",
+    ),
+    Rule(
+        pattern=r"(?:\.amc|\.cmake|\.cmake\.in|\.gitignore|\.ini|\.pc\.in"
+                r"|\.pl|\.py|\.py\.in|\.sh|\.toml|\.toml\.in|\.yml"
+                r"|CMakeLists.txt|Doxylocal|dune.module|MANIFEST\.in)$",
+        line_prefix="# ",
+        suffix="\n",
+    ),
+    Rule(
+        pattern=r"/doxygen/.*\.txt$",
+        line_prefix="// ",
+        suffix="\n",
+    ),
+    Rule(
+        pattern=r"(?:\.md|INSTALL)$",
+        prefix="<!--\n",
+        suffix="-->\n\n",
+    ),
+    Rule(
+        pattern=r"\.(?:cel|eps|fig|pdf|png|svg|vrt)$",
+        use_license_file=True,
+    ),
+    Rule(
+        # gmsh's MSH file format support comments via unknown sections
+        # ($Comment ... $EndComment), but DUNE does not handle that.
+        # Reference: https://gmsh.info/doc/texinfo/gmsh.html#MSH-file-format
+        pattern=r"\.msh",
+        use_license_file=True,
+    ),
+]
+
+
+skip_lines = re.compile(r'-\*-|vi(?:m)?:|^#!')
+
+
+class Notice:
+    def __init__(self, copyright, license):
+        self.copyright = f'SPDX-FileCopyrightInfo: {copyright}'
+        self.license = f'SPDX-License-Identifier: {license}'
+
+    def match(self, lines) -> bool:
+        pattern = re.compile(f'(?:{re.escape(self.copyright)}|{re.escape(self.license)})')
+        return any(pattern.search(line) for line in lines)
+
+
+def rule_for_file(filename) -> Rule:
+    for rule in rules:
+        if rule.match(filename):
+            return rule
+    return None
+
+
+def line_for_notice(lines, rule: Rule) -> int:
+    for index, line in enumerate(lines):
+        if not skip_lines.search(line) \
+           and (rule.skip_lines is None or not rule.skip_lines.search(line)):
+            return index
+    return len(lines)
+
+
+def apply_rule_to_file(filename, rule: Rule, notice: Notice):
+    if rule.use_license_file:
+        try:
+            with open(f"{filename}.license", "xt") as fh:
+                logging.debug(f"{filename}: create separate .license file")
+                print(notice.copyright, file=fh)
+                print(notice.license, file=fh)
+        except OSError as e:
+            if e.errno == errno.EEXIST:
+                logging.info(f"{filename}: separate .license file already exists")
+            else:
+                raise
+    else:
+        with open(filename, "rt") as fh:
+            lines = fh.readlines()
+        if notice.match(lines):
+            logging.info(f"{filename}: already contains a notice")
+            return
+        index = line_for_notice(lines, rule)
+        logging.debug(f"{filename}: Will insert notice at line {index}")
+        if rule.suffix:
+            lines.insert(index, rule.suffix)
+        lines.insert(index, f"{rule.line_prefix}{notice.license}{rule.line_suffix}\n")
+        lines.insert(index, f"{rule.line_prefix}{notice.copyright}{rule.line_suffix}\n")
+        if rule.prefix:
+            lines.insert(index, rule.prefix)
+        with open(filename, "wt") as fh:
+            print(*lines, sep='', end='', file=fh)
+
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+    if args.log:
+        logging.basicConfig(level=getattr(logging, args.log.upper()))
+    notice = Notice(args.copyright, args.license)
+    for filename in args.files:
+        logging.debug(f"Processing {filename}")
+        if args.like:
+            rule = rule_for_file(args.like)
+        else:
+            rule = rule_for_file(filename)
+        if rule is None:
+            logging.warning(f"{filename}: No rule found for this file. Add a rule or try --like.")
+            continue
+        apply_rule_to_file(filename, rule, notice)
+
+
+# add-spdx() {
+#     local file line_prefix prefix suffix
+#     local copying_file="LICENSE.md"
+#     local license="LicenseRef-GPL-2.0-only-with-DUNE-exception"
+#     xxlicense="LGPL-2.1-or-later"
+#     for file in "$@"; do
+#         prefix=""
+#         line_prefix=""
+#         suffix=""
+#         license_file="${file}"
+#         case "${file}" in
+#             *.cc|*.hh|*config.h.cmake|*.cc.in|*.ct|*.ht|*.t) line_prefix="// " ;;
+#             */doxygen/*.txt) line_prefix="// "; suffix=$'\n' ;;
+#             *CMakeLists.txt|*.py|*.gitignore|*.yml|*Doxylocal|*.toml|*.pc.in|*dune.module|*.cmake|*.ini) line_prefix="# "; suffix=$'\n' ;;
+#             *.md|*INSTALL) prefix=$'<!--\n'; suffix=$'\n-->\n' ;;
+#             *.tex) line_prefix="% "; suffix=$'\n' ;;
+#             *.svg|*.png|*.eps|*.pdf) license_file="${file}.license"; file="" ;;
+#         esac
+#
+#         ed ${file} <<EOT
+# 0i
+# ${prefix}${line_prefix}SPDX-FileCopyrightText: Copyright © DUNE Project contributors, see file ${copying_file} in module root
+# ${line_prefix}SPDX-License-Identifier: ${license}${suffix}
+# .
+# w ${license_file}
+# q
+# EOT
+#     done
+# }
+# REUSE-IgnoreEnd

bin/dune-autogen

-#!/bin/bash
-# $Id: autogen.sh 5054 2008-01-08 15:06:55Z christi $
-
-# barf on errors
-set -e
-
-usage () {
-    echo "Usage: dune-autogen DUNE_MODULE_PATH_LIST [options]"
-    echo "  --ac=, --acversion=VERSION   use a specific VERSION of autoconf"
-    echo "  --am=, --amversion=VERSION   use a specific VERSION of automake"
-    echo "  -h,    --help                you already found this :-)"
-}
-
-## get my name...
-grep '^Module:' dune.module >/dev/null || echo "Parser Error: Module entry missing in dune.module"
-name=
-while read head name rest
-do case "$head" in
-   Module:) break;;
-   Module:*) name="${head#Module:}"; break;;
-   esac
-   name=
-done <dune.module
-
-## dune-all.m4
-rm -f dune-all.m4
-rm -f $name.m4
-
-# add current dir to PATH
-PATH=`dirname $0`:$PATH
-
-# guess libtool prefix
-if test -n "$LIBTOOLIZE"; then
-    LIBTOOL_prefix=`dirname \`dirname $LIBTOOLIZE\``
-    PATH=$LIBTOOL_prefix:$PATH
-    ACLOCAL_FLAGS="$ACLOCAL_FLAGS -I $LIBTOOL_prefix/share/aclocal"
-fi
-
-for OPT in "$@"; do
-    set +e
-    # stolen from configure...
-    # when no option is set, this returns an error code
-    arg=`expr "x$OPT" : 'x[^=]*=\(.*\)'`
-    set -e
-
-    case "$OPT" in
-	--ac=*|--acversion=*)
-			if test "x$arg" = "x"; then
-				usage; 
-				exit 1;
-			fi
-			ACVERSION=$arg
-			;;
-	--am=*|--amversion=*)
-			if test "x$arg" = "x"; then
-				usage; 
-				exit 1;
-			fi
-			AMVERSION=$arg
-			;;
-	-h|--help) usage ; exit 0 ;;
-	*)
-            if test -d "$OPT/m4"; then
-              ACLOCAL_FLAGS="$ACLOCAL_FLAGS -I $OPT/m4"
-            fi
-            if test -f "$OPT/dune-common.pc.in" ; then
-#            if test \( -d "$OPT/am" \) -a ! \( -h "$OPT/am" \) ; then
-              echo "Found am directory $OPT/am"
-              am_dir="$OPT/am"
-            fi
-            if test -d "$OPT/share/aclocal"; then
-              ACLOCAL_FLAGS="$ACLOCAL_FLAGS -I $OPT/share/aclocal"
-            fi
-            if test -d "$OPT/share/dune-common/am"; then
-              echo "Found am directory $OPT/share/dune-common/am"
-              am_dir="$OPT/share/dune-common/am"
-            fi
-			      PATH=$OPT/bin:$PATH
-            ;;
-    esac
-done
-
-## report parameters
-if test "x$ACVERSION" != "x"; then
-	echo "Forcing autoconf version $ACVERSION"
-	if ! which autoconf$ACVERSION > /dev/null; then
-		echo
-		echo "Error: Could not find autoconf$ACVERSION"
-		echo "       Did you specify a wrong version?"
-		exit 1
-	fi
-fi
-if test "x$AMVERSION" != "x"; then
-	echo "Forcing automake version $AMVERSION"
-	if ! which automake$AMVERSION > /dev/null; then
-		echo
-		echo "Error: Could not find automake$AMVERSION"
-		echo "       Did you specify a wrong version?"
-		exit 1
-	fi
-fi
-
-## run autotools
-
-echo "--> dunedoxynize..."
-dunedoxynize
-
-echo "--> libtoolize..."
-# this script won't rewrite the files if they already exist. This is a
-# PITA when you want to upgrade libtool, thus I'm setting --force
-if [ x`type -t glibtoolize` = xfile ]; then
-  LIBTOOLIZE=glibtoolize
-fi
-${LIBTOOLIZE-libtoolize} --force 
-
-# writing privat m4 file
-echo -n "--> "
-dunecontrol --only=$name m4create
-
-# prepare everything
-echo "--> aclocal..."
-rm -f aclocal.m4
-rm -rf autom4te.cache
-aclocal$AMVERSION -I . $ACLOCAL_FLAGS
-
-# create a link to the dune-common am directory
-if [ "$name" != "dune-common" ]; then
-  if [ -n "$am_dir" ] && [ -d $am_dir ]; then
-    echo "--> linking dune-common/am..."
-    rm -f am
-    ln -s $am_dir am
-  else
-    echo
-    echo "Error: Could not find dune-common/am!"
-    usage
-    exit 1
-  fi
-fi
-
-# applications should provide a config.h for now
-echo "--> autoheader..."
-autoheader$ACVERSION
-
-echo "--> automake..."
-automake$AMVERSION -W all --add-missing
-
-echo "--> autoconf..."
-autoconf$ACVERSION
-
-## tell the user what to do next
-echo "Now run ./configure to setup $name"

bin/dune-ctest

+#! /usr/bin/env python3
+# SPDX-FileCopyrightInfo: Copyright © DUNE Project contributors, see file LICENSE.md in module root
+# SPDX-License-Identifier: LicenseRef-GPL-2.0-only-with-DUNE-exception
+#
+# Wrapper around CTest for DUNE
+#
+# CTest returns with an error status not only when tests failed, but also
+# when tests were only skipped.  This wrapper checks the log and returns
+# successfully if no tests failed; skipped tests do not result in an error.
+# This behaviour is needed in a continuous integration environment, when
+# building binary packages or in other cases where the testsuite should be
+# run automatically.
+#
+# Moreover, this script also converts the XML test report generated by CTest
+# into a JUnit report file that can be consumed by a lot of reporting
+# software.
+#
+# Author: Ansgar Burchardt <Ansgar.Burchardt@tu-dresden.de>
+# Author: Steffen Müthing <steffen.muething@iwr.uni-heidelberg.de> (for the JUnit part)
+
+import errno
+import glob
+import os.path
+import shutil
+import subprocess
+import sys
+import xml.etree.ElementTree as et
+from pathlib import Path
+import os
+import re
+
+
+class CTestParser:
+
+    def findCTestOutput(self):
+        files = glob.glob("Testing/*/Test.xml")
+        if len(files) != 1:
+            fn = files.join(", ")
+            raise Exception("Found multiple CTest output files: {}".format(files.join(", ")))
+        return files[0]
+
+    def printTest(self,test,output=None):
+        status = test.get("Status")
+        name = test.find("Name").text
+        fullName = test.find("FullName").text
+        if output is not None:
+            output = test.find("Results").find("Measurement").find("Value").text
+
+        print("======================================================================")
+        print("Name:      {}".format(name))
+        print("FullName:  {}".format(fullName))
+        print("Status:    {}".format(status.upper()))
+        if output:
+            print("Output:")
+            for line in output.splitlines():
+                print("          ", line)
+        print()
+
+    def __init__(self,junitpath=None):
+        self.inputpath = self.findCTestOutput()
+        if junitpath is None:
+            if "CI_PROJECT_DIR" in os.environ:
+                buildroot = Path(os.environ["CI_PROJECT_DIR"])
+                # create a slug from the project name
+                name = os.environ["CI_PROJECT_NAME"].lower()
+                name = re.sub(r"[^-a-z0-9]","-",name);
+                junitbasename = "{}-".format(name)
+            else:
+                buildroot = Path.cwd()
+                junitbasename = ""
+            junitdir = buildroot / "junit"
+            junitdir.mkdir(parents=True,exist_ok=True)
+            self.junitpath = junitdir / "{}cmake.xml".format(junitbasename)
+        else:
+            self.junitpath = Path(junitpath)
+            junitdir = junitpath.resolve().parent
+            junitdir.mkdir(parents=True,exist_ok=True)
+        self.tests = 0
+        self.passed = 0
+        self.failures = 0
+        self.skipped = 0
+        self.errors = 0
+        self.skipped = 0
+        self.time = 0.0
+
+    def createJUnitSkeleton(self):
+        self.testsuites = et.Element("testsuites")
+        self.testsuite = et.SubElement(self.testsuites,"testsuite")
+        self.properties = et.SubElement(self.testsuite,"properties")
+
+    def fillJUnitStatistics(self):
+        self.testsuite.set("name","cmake")
+        self.testsuite.set("tests",str(self.tests))
+        self.testsuite.set("disabled","0")
+        self.testsuite.set("errors",str(self.errors))
+        self.testsuite.set("failures",str(self.failures))
+        self.testsuite.set("skipped",str(self.skipped))
+        self.testsuite.set("time",str(self.time))
+
+    def processTest(self,test):
+        testcase = et.SubElement(self.testsuite,"testcase")
+        testcase.set("name",test.find("Name").text)
+        testcase.set("assertions","1")
+        testcase.set("classname","cmake")
+        time = test.find("./Results/NamedMeasurement[@name='Execution Time']/Value")
+        if time is not None:
+            self.time += float(time.text)
+            testcase.set("time",time.text)
+        self.tests += 1
+        outcome = test.get("Status")
+        if outcome == "passed":
+            testcase.set("status","passed")
+            self.passed += 1
+        elif outcome == "failed":
+            self.failures += 1
+            testcase.set("status","failure")
+            failure = et.SubElement(testcase,"failure")
+            failure.set("message","program execution failed")
+            failure.text = test.find("./Results/Measurement/Value").text
+            self.printTest(test)
+        elif outcome == "notrun":
+            # This does not exit on older CMake versions, so work around that
+            try:
+                status = test.find("./Results/NamedMeasurement[@name='Completion Status']/Value").text
+                if status == "SKIP_RETURN_CODE=77":
+                    self.skipped += 1
+                    et.SubElement(testcase,"skipped")
+                elif status == "Required Files Missing":
+                    self.errors += 1
+                    error = et.SubElement(testcase,"error")
+                    error.set("message","compilation failed")
+                    error.set("type","compilation error")
+                    self.printTest(test,output="Compilation error")
+                else:
+                    error = et.SubElement(testcase,"error")
+                    error.set("message","unknown error during test execution")
+                    error.set("type","unknown")
+                    error.text = test.find("./Results/Measurement/Value").text
+                    self.errors += 1
+                    self.printTest(test)
+            except AttributeError:
+                output_tag = test.find("./Results/Measurement/Value")
+                if output_tag is not None:
+                    msg = output_tag.text
+                    if "skipped" in msg:
+                        self.skipped += 1
+                        et.SubElement(testcase,"skipped")
+                    elif "Unable to find required file" in msg:
+                        self.errors += 1
+                        error = et.SubElement(testcase,"error")
+                        error.set("message","compilation failed")
+                        error.set("type","compilation error")
+                        self.printTest(test,output="Compilation error")
+                    else:
+                        error = et.SubElement(testcase,"error")
+                        error.set("message","unknown error during test execution")
+                        error.set("type","unknown")
+                        error.text = msg
+                        self.errors += 1
+                        self.printTest(test)
+                else:
+                    error = et.SubElement(testcase,"error")
+                    error.set("message","unknown error during test execution")
+                    error.set("type","unknown")
+                    error.text = "no message"
+                    self.errors += 1
+                    self.printTest(test)
+
+        output_tag = test.find("./Results/Measurement/Value")
+        if output_tag is not None:
+            out = et.SubElement(testcase,"system-out")
+            out.text = output_tag.text
+
+    def process(self):
+
+        with open(self.inputpath, "r", encoding="utf-8") as fh:
+            tree = et.parse(fh)
+
+        root = tree.getroot()
+
+        self.createJUnitSkeleton()
+
+        for test in root.findall(".//Testing/Test"):
+            self.processTest(test)
+
+        self.fillJUnitStatistics()
+
+        with self.junitpath.open("wb") as fh:
+            fh.write(et.tostring(self.testsuites,encoding="utf-8"))
+            print("JUnit report for CTest results written to {}".format(self.junitpath))
+
+        return self.errors + self.failures
+
+
+def runCTest(argv=[]):
+    cmd = ["ctest",
+           "--output-on-failure",
+           "--dashboard", "ExperimentalTest",
+           "--no-compress-output",
+    ]
+    cmd.extend(argv)
+    subprocess.call(cmd)
+
+def checkDirectory():
+    if not os.path.exists("CMakeCache.txt"):
+        raise Exception("ERROR: dune-ctest must be run in a cmake build directory")
+
+def removeCTestOutput():
+    try:
+        shutil.rmtree("Testing")
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+
+def main():
+    try:
+        checkDirectory()
+        removeCTestOutput()
+        runCTest(argv=sys.argv[1:])
+        parser = CTestParser()
+        errors = parser.process()
+        status = 0 if errors == 0 else 1
+        sys.exit(status)
+    except Exception as e:
+        print("Internal error: {}".format(e))
+        sys.exit(127)
+
+if __name__ == "__main__":
+    main()

bin/find_modules.py

+# SPDX-FileCopyrightInfo: Copyright © DUNE Project contributors, see file LICENSE.md in module root
+# SPDX-License-Identifier: LicenseRef-GPL-2.0-only-with-DUNE-exception
+
+try:
+    from dune.common.module import resolve_dependencies, resolve_order, select_modules
+
+except ImportError:
+    import os
+    here = os.path.dirname(os.path.abspath(__file__))
+    mods = os.path.join(os.path.dirname(here), "python", "dune", "common")
+    if os.path.exists(os.path.join(mods, "module.py")):
+        import sys
+        sys.path.append(mods)
+        from module import resolve_dependencies, resolve_order, select_modules
+    else:
+        raise
+
+print("Found Modules:")
+print("--------------")
+modules, _ = select_modules()
+for description in modules.values():
+    print(repr(description))
+    print()
+
+print()
+print("Resolved Dependencies:")
+print("----------------------")
+deps = resolve_dependencies(modules)
+for mod_name, mod_deps in deps.items():
+    print(mod_name + ": " + " ".join(mod_deps))
+
+print()
+print("Build Order:")
+print("------------")
+print(" ".join(resolve_order(deps)))

bin/xfail-compile-tests

-#!/bin/bash
-
-make compile_XFAIL

cmake/.gitignore

-Makefile.in
-Makefile

cmake/CMakeLists.txt

+# SPDX-FileCopyrightInfo: Copyright © DUNE Project contributors, see file LICENSE.md in module root
+# SPDX-License-Identifier: LicenseRef-GPL-2.0-only-with-DUNE-exception
+
+add_subdirectory(modules)
+add_subdirectory(scripts)
+add_subdirectory(test)
\ No newline at end of file

cmake/Makefile.am

-SUBDIRS=modules pkg scripts

cmake/modules/.gitignore

-Makefile.in
-Makefile