use new approach in dune-common for handling python bindings

fix ci

pep 42: use implicit namespace packages which make it possible to combine subpackages using editable (source) and non editable (pypi)

only use a cmake install a python module in the SKBUILD setting - for an install from build dir pip will take take of this since it is part of the setup.py file

add a stopgap solution unti ci repo up and running again

Update requirements.

add dune packages to requires in setup.py.in

[testing]

only add dune modules that have been found as requirements in setup.py.in

removed setup.py.in since the default is provided in dune-common

adapt ci yml file to python bindings on by default

disable python bindings for clang 5 (ubuntu 18) since no numpy matching numpy version can be installed from the gitlab registry

[bug fix python bindings]
fix an issue with 'entry' being used in implicit build mode but ready==build

.gitlab-ci.yml

@@ -19,29 +19,29 @@ variables:
 # than one thread.
   OMP_NUM_THREADS: 1
 
-debian-11-gcc-9-17-python:
+debian-11-gcc-9-17-with-checking:
   image: registry.dune-project.org/docker/ci/debian:11
   script: duneci-standard-test
   variables:
     DUNECI_TOOLCHAIN:   gcc-9-17
-    # so we need some variables to build the dune-py module during execution of the first python test:
-    # we need to find the dune mdoule
-    DUNE_CONTROL_PATH:  /duneci/modules:$CI_PROJECT_DIR
-    # the position for the dune-py module
-    DUNE_PY_DIR:        /duneci/modules/dune-py
-    # during dune-py build this variable is used - do know a way to access
-    # the CMAKE_FLAGS used to build the modules...
-    DUNE_CMAKE_FLAGS:   "-DCMAKE_CXX_COMPILER=g++-9 -DCMAKE_C_COMPILER=gcc-9 -DCMAKE_CXX_FLAGS='-std=c++17 -O2 -g -Wall -fdiagnostics-color=always' -DDUNE_ENABLE_PYTHONBINDINGS=ON -DDUNE_MAX_TEST_CORES=4 -DBUILD_SHARED_LIBS=TRUE -DDUNE_PYTHON_INSTALL_LOCATION=none -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE -DCMAKE_DISABLE_FIND_PACKAGE_LATEX=TRUE -DCMAKE_DISABLE_FIND_PACKAGE_Alberta=TRUE -DCMAKE_DISABLE_FIND_PACKAGE_Vc=TRUE -DCMAKE_DISABLE_DOCUMENTATION=TRUE"
-    # cmake flags we use for all dune moudle - important is that build shared libs is set (need some better way of doing this)
-    DUNECI_CMAKE_FLAGS: $DUNE_CMAKE_FLAGS
-    # finally set the python path to all modules
-    PYTHONPATH:         /duneci/modules/dune-common/build-cmake/python:$CI_PROJECT_DIR/build-cmake/python
+    DUNECI_CMAKE_FLAGS:   "CC=gcc-9 CXX=g++-9 -DCMAKE_CXX_FLAGS='-std=c++17 -O2 -g -Wall -fdiagnostics-color=always -DDUNE_ISTL_WITH_CHECKING' -DDUNE_MAX_TEST_CORES=4 -DCMAKE_DISABLE_FIND_PACKAGE_LATEX=TRUE -DCMAKE_DISABLE_FIND_PACKAGE_Alberta=TRUE -DCMAKE_DISABLE_DOCUMENTATION=TRUE"
   tags: [duneci]
 
-debian-11-gcc-9-17-with-checking:
-  image: registry.dune-project.org/docker/ci/debian:11
+# no numpy version available for this image on the gitlab registry
+# disable python bindings for now
+ubuntu:18.04 clang-6-17:
+  image: registry.dune-project.org/docker/ci/ubuntu:18.04
   script: duneci-standard-test
+  stage: test
   variables:
-    DUNECI_TOOLCHAIN:   gcc-9-17
-    DUNECI_CMAKE_FLAGS:   "CC=gcc-9 CXX=g++-9 -DCMAKE_CXX_FLAGS='-std=c++17 -O2 -g -Wall -fdiagnostics-color=always -DDUNE_ISTL_WITH_CHECKING' -DDUNE_MAX_TEST_CORES=4 -DCMAKE_DISABLE_FIND_PACKAGE_LATEX=TRUE -DCMAKE_DISABLE_FIND_PACKAGE_Alberta=TRUE -DCMAKE_DISABLE_DOCUMENTATION=TRUE"
+    DUNECI_TOOLCHAIN: clang-6-17
+    DUNECI_CMAKE_FLAGS: "-DDUNE_ENABLE_PYTHONBINDINGS=OFF"
+  tags: [duneci]
+ubuntu:18.04 clang-5-17:
+  image: registry.dune-project.org/docker/ci/ubuntu:18.04
+  script: duneci-standard-test
+  stage: test
+  variables:
+    DUNECI_TOOLCHAIN: clang-5-17
+    DUNECI_CMAKE_FLAGS: "-DDUNE_ENABLE_PYTHONBINDINGS=OFF"
   tags: [duneci]

CMakeLists.txt

@@ -30,7 +30,6 @@ add_subdirectory(doc)
 # if Python bindings are enabled, include necessary sub directories.
 if( DUNE_ENABLE_PYTHONBINDINGS )
   add_subdirectory(python)
-  dune_python_install_package(PATH python)
 endif()
 
 # finalize the dune project, e.g. generating config.h etc.

dune/python/istl/bcrsmatrix.hh

@@ -185,7 +185,7 @@ namespace Dune
           if( col >= self.M() )
             throw pybind11::index_error( "No such column: " + std::to_string( col ) );
 
-          if( self.buildMode() != BCRSMatrix::implicit )
+          if( self.buildMode() != BCRSMatrix::implicit || self.buildStage() == BCRSMatrix::built )
           {
             auto pos = self[ row ].find( col );
             if( pos != self[ row ].end() )
@@ -212,7 +212,7 @@ namespace Dune
           if( col >= self.M() )
             throw pybind11::index_error( "No such column: " + std::to_string( col ) );
 
-          if( self.buildMode() != BCRSMatrix::implicit )
+          if( self.buildMode() != BCRSMatrix::implicit || self.buildStage() == BCRSMatrix::built )
           {
             auto pos = self[ row ].find( col );
             if( pos != self[ row ].end() )
@@ -231,7 +231,7 @@ namespace Dune
           std::size_t cstart, cstop, cstep, clength;
           std::get< 1 >( index ).compute( self.M(), &cstart, &cstop, &cstep, &clength );
 
-          if( self.buildMode() == BCRSMatrix::implicit )
+          if( self.buildMode() != BCRSMatrix::implicit || self.buildStage() == BCRSMatrix::built )
           {
             for( auto v : value )
             {

pyproject.toml

 [build-system]
-requires = ['setuptools', 'wheel', 'scikit-build', 'cmake', 'ninja', 'requests', 'dune-common>=2.8.0.dev0']
+requires = ['dune-common>=2.8.0.dev0', 'numpy', 'pip', 'setuptools', 'wheel', 'scikit-build', 'cmake', 'ninja', 'requests']
 build-backend = 'setuptools.build_meta'

python/CMakeLists.txt

 add_subdirectory(dune)
 
-configure_file(setup.py.in setup.py)
+dune_python_install_package(
+  PATH "."
+  CMAKE_METADATA_FILE dune/istl/metadata.cmake
+  DEPENDS _istl
+)

python/dune/CMakeLists.txt

 add_subdirectory(istl)
-
-add_python_targets(dune
-  __init__
-)

python/dune/__init__.py

-__import__('pkg_resources').declare_namespace(__name__)

python/dune/istl/CMakeLists.txt

@@ -3,3 +3,6 @@ add_python_targets(istl
 )
 dune_add_pybind11_module(NAME _istl)
 set_property(TARGET _istl PROPERTY LINK_LIBRARIES dunecommon APPEND)
+if(SKBUILD)
+  install(TARGETS _istl LIBRARY DESTINATION python/dune/istl)
+endif()

python/setup.py.in

-from setuptools import setup, find_packages
-
-pkg = [m for m in "${ProjectPythonRequires}".split(' ') if "dune" not in m]
-
-setup(name="${ProjectName}",
-    namespace_packages=['dune'],
-    description="${ProjectDescription}",
-    version="${ProjectVersionString}",
-    author="${ProjectAuthor}",
-    author_email="${ProjectMaintainerEmail}",
-    packages = find_packages(),
-    zip_safe = 0,
-    package_data = {'': ['*.so']},
-    install_requires = pkg
-  )

setup.py

+import os, sys
 try:
     from dune.packagemetadata import metaData
 except ImportError: