improve locking mechanism when generating dune-py

use clang 10-20 for python test since some other parallel tests fail with the clang 10-17 toolchain

fix an issue with a double installation of the dune module in the case
of using packages (there was a 2.8.0 version and a dev version being added to the venv)

minor improvement to plotting

.gitlab-ci.yml

@@ -13,6 +13,7 @@ before_script:
 variables:
   DUNECI_TEST_LABELS: quick
   DUNE_TEST_EXPECTED_VC_IMPLEMENTATION: SSE2
+  PIP_DEFAULT_TIMEOUT: 0
 
 debian:10 gcc-7-17--expensive:
   # This image has Vc
@@ -39,20 +40,20 @@ debian-11-gcc-9-17-python:
   variables:
     DUNECI_TOOLCHAIN:  gcc-9-17
     # the CMAKE_FLAGS used to build the modules - set python bindings to on
-    DUNE_CMAKE_FLAGS:   '-DDUNE_ENABLE_PYTHONBINDINGS=ON'
+    DUNE_CMAKE_FLAGS:   '-DDUNE_ENABLE_PYTHONBINDINGS=ON -DALLOW_CXXFLAGS_OVERWRITE=ON'
     # cmake flags we use for all dune module
     DUNECI_CMAKE_FLAGS: $DUNE_CMAKE_FLAGS
     # add some logging output from the python bindings
     DUNE_LOG_LEVEL:     DEBUG
   tags: [duneci]
-ubuntu-20.04-clang-10-17-python:
+ubuntu-20.04-clang-10-20-python:
   image: registry.dune-project.org/docker/ci/ubuntu:20.04
   script: duneci-standard-test
   stage: test
   variables:
-    DUNECI_TOOLCHAIN: clang-10-17
+    DUNECI_TOOLCHAIN: clang-10-20
     # the CMAKE_FLAGS used to build the modules - set python bindings to on
-    DUNE_CMAKE_FLAGS:   '-DDUNE_ENABLE_PYTHONBINDINGS=ON'
+    DUNE_CMAKE_FLAGS:   '-DDUNE_ENABLE_PYTHONBINDINGS=ON -DALLOW_CXXFLAGS_OVERWRITE=ON'
     # cmake flags we use for all dune module
     DUNECI_CMAKE_FLAGS: $DUNE_CMAKE_FLAGS
     # add some logging output from the python bindings

cmake/modules/DunePythonInstallPackage.cmake

@@ -101,7 +101,6 @@ function(dune_python_install_package)
     COMMENT "Installing Python package at ${PYINST_FULLPATH} into Dune virtual environment..."
     DEPENDS ${PYINST_DEPENDS}
   )
-
   #
   # Now define rules for `make install_python`.
   #
@@ -126,7 +125,11 @@ function(dune_python_install_package)
                       COMMENT "Installing the python package at ${PYINST_FULLPATH}"
                       )
 
-    add_dependencies(install_python ${targetname})
+    # during package installation we don't want the package to be installed
+    # since skbuild takes care of that.
+    if(NOT SKBUILD)
+      add_dependencies(install_python ${targetname})
+    endif()
 
     # Define rules for `make install` that install a wheel into a central wheelhouse
     #
@@ -170,30 +173,52 @@ function(dune_python_install_package)
       string(APPEND _export_builddirs "\;${${mod}_DIR}")
     endforeach()
 
-    # issue: with the cxx override the CMAKE_CXX_COMPILER is replace by the
-    # script name and the original compiler is lost (available as DEFAULT_CXXFLAGS)
-    # so this case needs to be checked
     # todo: fix the leading ';'
     set(_cmake_flags "")
-    FOREACH(cmd_line_loop IN ITEMS ${PYINST_FLAGS})
-      if(${cmd_line_loop})
-        string(APPEND _cmake_flags "\;${cmd_line_loop}:=\"${${cmd_line_loop}}\"")
+    FOREACH(flags_loop IN ITEMS ${PYINST_FLAGS})
+      if(${flags_loop})
+        set(value ${${flags_loop}})
+        # need to make sure not to use the script generated by the CXX_OVERWRITE
+        # because dune-py should not depend on a file in the build dir
+        if(DEFINED DEFAULT_CXX_COMPILER AND "${flags_loop}" STREQUAL "CMAKE_CXX_COMPILER")
+          message("CXX: ${flags_loop} ${value} ${DEFAULT_CXX_COMPILER}")
+          set(value "${DEFAULT_CXX_COMPILER}")
+        endif()
+        if(DEFINED DEFAULT_CXXFLAGS AND "${flags_loop}" STREQUAL "CMAKE_CXX_FLAGS")
+          message("FLAGS: ${flags_loop} ${value} ${DEFAULT_CXXFLAGS}")
+          set(value "${DEFAULT_CXXFLAGS}")
+        endif()
+        string(APPEND _cmake_flags "\;${flags_loop}:=\"${value}\"")
       endif()
-    ENDFOREACH(cmd_line_loop)
+    ENDFOREACH(flags_loop)
 
     # Make sure to generate the metadata for the build stage
     # todo: possible issue with empty _cmake_flags - missing separator error
-    add_custom_target(
-      metadata_${envtargetname}
-      COMMAND ${CMAKE_COMMAND}
-        -Dmetadatafile=${metadatafile}
-        -DDEPBUILDDIRS="${_export_builddirs}"
-        -DDEPS="${PROJECT_NAME};${ALL_DEPENDENCIES}"
-        -DMODULENAME=${PROJECT_NAME}
-        -DCMAKE_FLAGS="${_cmake_flags}"
-        -P ${scriptdir}/WritePythonCMakeMetadata.cmake
-      COMMENT "Generating the CMake metadata file at ${PYINST_CMAKE_METADATA_FILE}"
-    )
+    if(SKBUILD)
+      add_custom_target(
+        metadata_${envtargetname}
+        COMMAND ${CMAKE_COMMAND}
+          -Dmetadatafile=${metadatafile}
+          -DDEPBUILDDIRS=
+          -DDEPS="${PROJECT_NAME};${ALL_DEPENDENCIES}"
+          -DMODULENAME=${PROJECT_NAME}
+          -DCMAKE_FLAGS="${_cmake_flags}"
+          -P ${scriptdir}/WritePythonCMakeMetadata.cmake
+        COMMENT "Generating the CMake metadata file at ${PYINST_CMAKE_METADATA_FILE}"
+      )
+    else()
+      add_custom_target(
+        metadata_${envtargetname}
+        COMMAND ${CMAKE_COMMAND}
+          -Dmetadatafile=${metadatafile}
+          -DDEPBUILDDIRS="${_export_builddirs}"
+          -DDEPS="${PROJECT_NAME};${ALL_DEPENDENCIES}"
+          -DMODULENAME=${PROJECT_NAME}
+          -DCMAKE_FLAGS="${_cmake_flags}"
+          -P ${scriptdir}/WritePythonCMakeMetadata.cmake
+        COMMENT "Generating the CMake metadata file at ${PYINST_CMAKE_METADATA_FILE}"
+      )
+    endif()
     add_dependencies(${envtargetname} metadata_${envtargetname})
     # Add a custom command that triggers the configuration of dune-py
     add_custom_command(TARGET metadata_${envtargetname} POST_BUILD
@@ -210,7 +235,6 @@ function(dune_python_install_package)
             -DDEPS="${PROJECT_NAME};${ALL_DEPENDENCIES}"
             -DDEPBUILDDIRS=
             -DMODULENAME=${PROJECT_NAME}
-            -DINSTALL_PREFIX=
             -DCMAKE_FLAGS="${_cmake_flags}"
             -P ${scriptdir}/WritePythonCMakeMetadata.cmake
           COMMENT "Generating the CMake metadata file at ${PYINST_CMAKE_METADATA_FILE}"

python/CMakeLists.txt

@@ -2,9 +2,9 @@ add_subdirectory(dune)
 
 # installing the python package mpi4py fails if mpi is not available on the
 # system so we only add it here to the python requires in setup.py if mpi was found
-if(MPI_C_FOUND)
-    set(ProjectPythonRequires "${ProjectPythonRequires} mpi4py")
-endif()
+# if(MPI_C_FOUND)
+#     set(ProjectPythonRequires "${ProjectPythonRequires} mpi4py")
+# endif()
 
 configure_file(MANIFEST.in MANIFEST.in)
 configure_file(setup.py.in setup.py)
@@ -14,5 +14,5 @@ dune_python_install_package(
   PATH "."
   DEPENDS _common _typeregistry
   CMAKE_METADATA_FILE dune/common/metadata.cmake
-  FLAGS CMAKE_BUILD_TYPE CMAKE_CXX_COMPILER CMAKE_CXX_FLAGS CMAKE_C_COMPILER CMAKE_C_FLAGS CMAKE_PREFIX_PATH
+  FLAGS CMAKE_BUILD_TYPE CMAKE_CXX_COMPILER CMAKE_CXX_FLAGS CMAKE_C_COMPILER CMAKE_C_FLAGS CMAKE_PREFIX_PATH ALLOW_CXXFLAGS_OVERWRITE
 )

python/dune/generator/cmakebuilder.py

@@ -166,35 +166,28 @@ class Builder:
             self.savedOutput = [sys.stdout, sys.stderr]
         else:
             self.savedOutput = None
+        self.dune_py_dir = dune.common.module.get_dune_py_dir()
+        self.build_args = dune.common.module.get_default_build_args()
+        self.generated_dir = os.path.join(self.dune_py_dir, 'python', 'dune', 'generated')
         self.initialized = False
 
     def initialize(self):
-        self.dune_py_dir = dune.common.module.get_dune_py_dir()
-        os.makedirs(self.dune_py_dir, exist_ok=True)
-
         if comm.rank == 0:
-            # Trigger the generation of dune-py
-            from dune.common.locking import Lock, LOCK_EX,LOCK_SH
-            dunepy = get_dune_py_dir()
-            tagfile = os.path.join(dunepy, ".noconfigure")
-            if os.path.exists(dunepy):
-                # check for existence of .noconfigure tag file
-                with Lock(os.path.join(dunepy, 'lock-module.lock'), flags=LOCK_EX):
-                    if not os.path.isfile(tagfile):
-                        generateDunePy = True
-                    else:
-                        generateDunePy = False
-                        logger.debug('Using existing dune-py module in'+dunepy)
-            else:
-                generateDunePy = True
-            if generateDunePy:
-                Builder.dunepy_from_template(get_dune_py_dir())
-                subprocess.check_call("cmake .".split(), cwd=get_dune_py_dir())
-                open(tagfile, 'a').close()
+            os.makedirs(self.dune_py_dir, exist_ok=True)
+            # need to lock here so that multiple procersses don't try to
+            # generate a new dune-py at the same time
+            with Lock(os.path.join(self.dune_py_dir, '..', 'lock-module.lock'), flags=LOCK_EX):
+                # check for existence of tag file - this is removed if a
+                # new dune package is added
+                tagfile = os.path.join(self.dune_py_dir, ".noconfigure")
+                if not os.path.isfile(tagfile):
+                    logger.info('Generating dune-py module in '+self.dune_py_dir)
+                    Builder.dunepy_from_template(get_dune_py_dir())
+                    subprocess.check_call("cmake .".split(), cwd=get_dune_py_dir())
+                    open(tagfile, 'a').close()
+                else:
+                    logger.debug('Using existing dune-py module in '+self.dune_py_dir)
         comm.barrier()
-
-        self.build_args = dune.common.module.get_default_build_args()
-        self.generated_dir = os.path.join(self.dune_py_dir, 'python', 'dune', 'generated')
         try:
             dune.__path__._path.insert(0,os.path.join(self.dune_py_dir, 'python', 'dune'))
         except:
@@ -249,7 +242,7 @@ class Builder:
             module = sys.modules.get("dune.generated." + moduleName)
             if module is None:
                 # make sure nothing (compilation, generating and building) is # taking place
-                with Lock(os.path.join(self.dune_py_dir, 'lock-module.lock'), flags=LOCK_EX):
+                with Lock(os.path.join(self.dune_py_dir, '..', 'lock-module.lock'), flags=LOCK_EX):
                     # module must be generated so lock the source file
                     with Lock(os.path.join(self.dune_py_dir, 'lock-'+moduleName+'.lock'), flags=LOCK_EX):
                         sourceFileName = os.path.join(self.generated_dir, moduleName + ".cc")
@@ -295,7 +288,7 @@ class Builder:
                 # end of exclusive dune-py lock
 
                 # for compilation a shared lock is enough
-                with Lock(os.path.join(self.dune_py_dir, 'lock-module.lock'), flags=LOCK_SH):
+                with Lock(os.path.join(self.dune_py_dir, '..', 'lock-module.lock'), flags=LOCK_SH):
                     # lock generated module
                     with Lock(os.path.join(self.dune_py_dir, 'lock-'+moduleName+'.lock'), flags=LOCK_EX):
                         logger.debug("Now compiling "+moduleName)

python/dune/plotting.py

@@ -15,17 +15,16 @@ if addPlot:
 
     try:
         s = os.environ['DUNEPY_BLOCK_PLOTTING']
-        block = s in ['TRUE','true', '1', 't', 'y', 'yes']
+        block = s in ['TRUE','True','true', '1', 't', 'y', 'yes']
     except KeyError:
         block = True
     try:
         s = os.environ['DUNEPY_DISABLE_PLOTTING']
-        disable = s in ['TRUE','true', '1', 't', 'y', 'yes']
+        disable = s in ['TRUE','True','true', '1', 't', 'y', 'yes']
     except KeyError:
         disable = False
     block = block and (not disable)
 
-
     def _plotGrid(fig, grid, gridLines="black"):
         for p in grid.polygons():
             coll = PolyCollection(p, facecolor='none', edgecolor=gridLines, linewidth=0.5, zorder=2)