fix dune/python

move bin files and cmake modules from dune-python
add sphinx support from dune-python

CMakeLists.txt

@@ -24,6 +24,8 @@ add_subdirectory("bin")
 add_subdirectory("doc")
 add_subdirectory("cmake/modules")
 add_subdirectory("cmake/scripts")
+add_subdirectory(python)
+dune_python_install_package(PATH "python")
 
 # finalize the dune project, e.g. generating config.h etc.
 finalize_dune_project()

bin/CMakeLists.txt

@@ -3,4 +3,6 @@ install(PROGRAMS
   duneproject
   dunecontrol
   dune-git-whitespace-hook
+  rmgenerated.py
+  setup-dunepy.py
   DESTINATION ${CMAKE_INSTALL_BINDIR})

bin/find_modules.py

+from __future__ import print_function
+
+try:
+    from dune.common.module import resolve_dependencies, resolve_order, select_modules
+
+except ImportError:
+    import os
+    here = os.path.dirname(os.path.abspath(__file__))
+    mods = os.path.join(os.path.dirname(here), "python", "dune", "common")
+    if os.path.exists(os.path.join(mods, "module.py")):
+        import sys
+        sys.path.append(mods)
+        from module import resolve_dependencies, resolve_order, select_modules
+    else:
+        raise
+
+print("Found Modules:")
+print("--------------")
+modules, _ = select_modules()
+for description in modules.values():
+    print(repr(description))
+    print()
+
+print()
+print("Resolved Dependencies:")
+print("----------------------")
+deps = resolve_dependencies(modules)
+for mod_name, mod_deps in deps.items():
+    print(mod_name + ": " + " ".join(mod_deps))
+
+print()
+print("Build Order:")
+print("------------")
+print(" ".join(resolve_order(deps)))

bin/rmgenerated.py

+#!/usr/bin/env python
+
+import glob, os, sys, re, fileinput
+
+import dune.common.module
+dune_py_dir = dune.common.module.get_dune_py_dir()
+generated_dir = os.path.join(dune_py_dir, 'python', 'dune', 'generated')
+
+from argparse import ArgumentParser
+parser = ArgumentParser(description='Removing generated module from dune-py')
+parser.add_argument('-a', '--all', help='remove all modules', action='store_true', default=False)
+parser.add_argument('modules', metavar='M', nargs='*',
+                    help='base of the modules to remove')
+
+try:
+    args = parser.parse_args()
+except:
+    sys.exit(0)
+
+moduleFiles = set()
+
+if args.all:
+    base = os.path.join(generated_dir, '*.so')
+    for filename in glob.iglob( base ):
+        os.remove( filename )
+        os.remove( os.path.splitext(filename)[0]+'.cc' )
+        moduleFiles.update( [os.path.splitext(os.path.basename(filename))[0]] )
+elif len(args.modules)>0:
+    for m in args.modules:
+        base = os.path.join(generated_dir, m+'*')
+        for filename in glob.iglob( base ):
+            os.remove( filename )
+            moduleFiles.update( [os.path.splitext(os.path.basename(filename))[0]] )
+else:
+    parser.print_help()
+    sys.exit(0)
+
+for line in fileinput.input( os.path.join(generated_dir, 'CMakeLists.txt'), inplace = True):
+    if not any( [m in line for m in moduleFiles] ):
+         print(line, end="")

bin/setup-dunepy.py

+#!/usr/bin/env python
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import getopt
+import os
+import shlex
+import subprocess
+import sys
+import shutil
+import logging
+
+logger = logging.getLogger(__name__)
+
+try:
+    from dune.common.module import build_dune_py_module, get_dune_py_dir, make_dune_py_module, select_modules
+except ImportError:
+    import os
+    here = os.path.dirname(os.path.abspath(__file__))
+    mods = os.path.join(os.path.dirname(here), "python", "dune", "common")
+    if os.path.exists(os.path.join(mods, "module.py")):
+        sys.path.append(mods)
+        from module import build_dune_py_module, get_dune_py_dir, make_dune_py_module, select_modules
+    else:
+        raise
+
+def buffer_to_str(b):
+    return b if sys.version_info.major == 2 else b.decode('utf-8')
+
+def toBuildDir(builddir, moddir, module):
+    if os.path.isabs(builddir):
+        return os.path.join(builddir ,module)
+    else:
+        return os.path.join(moddir, builddir)
+
+def main(argv):
+    try:
+        opts, args = getopt.getopt(argv,"ho",["opts=","builddir="])
+    except getopt.GetoptError:
+        print('usage: setup-dunepy.py [-o config.opts | --opts=config.opts | --builddir] [install]')
+        sys.exit(2)
+
+    optsfile = None
+    builddir = None
+    for opt, arg in opts:
+        if opt == '-h':
+            print('usage: setup-dunepy.py [-o config.opts | --opts=config.opts] [install]')
+            sys.exit(2)
+        elif opt in ("-o", "--opts"):
+            optsfile = arg
+        elif opt in ("--builddir"):
+            builddir = arg
+    if len(args) > 0:
+        execute = args[0]
+    else:
+        execute = ""
+
+    if optsfile is not None:
+        definitions = {}
+        command = ['bash', '-c', 'source ' + optsfile + ' && echo "$CMAKE_FLAGS"']
+        proc = subprocess.Popen(command, stdout = subprocess.PIPE)
+        stdout, _ = proc.communicate()
+        for arg in shlex.split(buffer_to_str(stdout)):
+            key, value = arg.split('=', 1)
+            if key.startswith('-D'):
+                key = key[2:]
+            definitions[key] = value
+        if builddir is None:
+            # get the build dir (check for BUILDDIR, DUNE_BUILDDIR in opts file
+            # and then DUNE_BUILDDIR in environment variable
+            command = ['bash', '-c', 'source ' + optsfile + ' && echo "$BUILDDIR"']
+            proc = subprocess.Popen(command, stdout = subprocess.PIPE)
+            stdout, _ = proc.communicate()
+            builddir = buffer_to_str(stdout).strip()
+            if not builddir:
+                command = ['bash', '-c', 'source ' + optsfile + ' && echo "$DUNE_BUILDDIR"']
+                proc = subprocess.Popen(command, stdout = subprocess.PIPE)
+                stdout, _ = proc.communicate()
+                builddir = buffer_to_str(stdout).strip()
+                if not builddir:
+                    builddir = os.environ.get('DUNE_BUILDDIR', 'build-cmake')
+    else:
+        definitions = None
+        if builddir is None:
+            builddir = os.environ.get('DUNE_BUILDDIR', 'build-cmake')
+
+    dunepy = get_dune_py_dir()
+
+    if os.path.exists(dunepy):
+        shutil.rmtree(dunepy)
+    foundModule = make_dune_py_module(dunepy)
+
+    output = build_dune_py_module(dunepy, definitions, None, builddir)
+
+    print("CMake output")
+    print(output)
+
+    # set a tag file to avoid automatic reconfiguration in builder
+    tagfile = os.path.join(dunepy, ".noconfigure")
+    f = open(tagfile, 'w')
+    f.close()
+
+    if execute == "install":
+        duneModules = select_modules()
+        moddir = duneModules[1]["dune-python"]
+        for m,depends in duneModules[0].items():
+            moddir = duneModules[1][m]
+            pythonModule = toBuildDir(builddir,moddir,m)
+            print("calling install_python in",moddir)
+            command = ['cmake', '--build', '.', '--target', 'install_python']
+            proc = subprocess.Popen(command, cwd=pythonModule, stdout = subprocess.PIPE)
+            stdout, stderr = proc.communicate()
+            logger.debug(buffer_to_str(stdout))
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

cmake/modules/CMakeLists.txt

@@ -9,6 +9,7 @@ install(FILES
   AddVcFlags.cmake
   CheckCXXFeatures.cmake
   CMakeBuiltinFunctionsDocumentation.cmake
+  DuneAddPybind11Module.cmake
   DuneCMakeCompat.cmake
   DuneCommonMacros.cmake
   DuneCxaDemangle.cmake
@@ -24,9 +25,11 @@ install(FILES
   DunePythonCommonMacros.cmake
   DunePythonFindPackage.cmake
   DunePythonInstallPackage.cmake
+  DunePythonMacros.cmake
   DunePythonRequireVersion.cmake
   DunePythonTestCommand.cmake
   DunePythonVirtualenv.cmake
+  DuneSphinxDoc.cmake
   DuneSphinxCMakeDoc.cmake
   DuneStreams.cmake
   DuneSymlinkOrCopy.cmake

cmake/modules/DuneAddPybind11Module.cmake

+# This cmake module provides infrastructure for building modules using Pybind11
+#
+# .. cmake_function:: dune_add_pybind11_module
+#
+#    .. cmake_param:: NAME
+#       :required:
+#       :single:
+#
+#       name of the Python module
+#
+#    .. cmake_param:: SOURCES
+#       :multi:
+#
+#       source files to build shared library
+#
+#       If this parameter is omitted, <name>.cc will be used if it exists.
+#
+#    .. cmake_param:: EXCLUDE_FROM_ALL
+#       :option:
+#
+#       exclude this module from the all target
+#
+#    .. cmake_param:: COMPILE_DEFINITIONS
+#       :multi:
+#       :argname: def
+#
+#       A set of compile definitions to add to the target.
+#       Only definitions beyond the application of :ref:`add_dune_all_flags`
+#       have to be stated.
+#
+#    .. cmake_param:: CMAKE_GUARD
+#       :multi:
+#       :argname: condition
+#
+#       A number of conditions that CMake should evaluate before adding this
+#       module. Use this feature instead of guarding the call to
+#       :code:`dune_add_pybind11_module` with an :code:`if` clause.
+#
+#       The passed condition can be a complex expression like
+#       `( A OR B ) AND ( C OR D )`. Mind the spaces around the parentheses.
+#
+#       Example: Write CMAKE_GUARD dune-foo_FOUND if you want your module to only
+#       build when the dune-foo module is present.
+#
+
+function(dune_add_pybind11_module)
+  include(CMakeParseArguments)
+  cmake_parse_arguments(PYBIND11_MODULE "EXCLUDE_FROM_ALL" "NAME" "SOURCES;COMPILE_DEFINITIONS;CMAKE_GUARD" ${ARGN})
+  if(PYBIND11_MODULE_UNPARSED_ARGUMENTS)
+    message(WARNING "dune_add_pybind11_module: extra arguments provided (typos in named arguments?)")
+  endif()
+
+  if(NOT PYBIND11_MODULE_NAME)
+    message(FATAL_ERROR "dune_add_pybind11_module: module name not specified")
+  endif()
+
+  if(NOT PYBIND11_MODULE_SOURCES)
+    if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${PYBIND11_MODULE_NAME}.cc)
+      set(PYBIND11_MODULE_SOURCES ${PYBIND11_MODULE_NAME}.cc)
+    else()
+      message(FATAL_ERROR "dune_add_pybind11_module: no source files specified")
+    endif()
+  endif()
+
+  foreach(condition ${PYBIND11_MODULE_CMAKE_GUARD})
+    separate_arguments(condition)
+    if(NOT (${condition}))
+      message(STATUS "not building ${PYBIND11_MODULE_NAME}, because condition ${condition} failed.")
+      return()
+    endif()
+  endforeach()
+
+  add_library(${PYBIND11_MODULE_NAME} SHARED ${PYBIND11_MODULE_SOURCES})
+  set_target_properties(${PYBIND11_MODULE_NAME} PROPERTIES PREFIX "")
+  #set_target_properties(${PYBIND11_MODULE_NAME} PROPERTIES CXX_VISIBILITY_PRESET hidden)
+  #set_target_properties(${PYBIND11_MODULE_NAME} PROPERTIES VISIBILITY_INLINES_HIDDEN TRUE)
+  target_compile_definitions(${PYBIND11_MODULE_NAME} PRIVATE ${PYBIND11_MODULE_COMPILE_DEFINITIONS})
+
+  if(PYBIND11_MODULE_EXCLUDE_FROM_ALL)
+    set_property(TARGET ${PYBIND11_MODULE_NAME} PROPERTY EXCLUDE_FROM_ALL 1)
+  endif()
+endfunction()

cmake/modules/DuneMacros.cmake

@@ -421,7 +421,7 @@ macro(dune_process_dependency_leafs modules versions is_required next_level_deps
     math(EXPR length "${mlength}-1")
     foreach(i RANGE 0 ${length})
       list(GET mmodules ${i} _mod)
-      list(GET mversions ${i} _ver)
+        list(GET mversions ${i} _ver)
       find_dune_package(${_mod} ${is_required} VERSION "${_ver}")
       set(${_mod}_SEARCHED ON)
       if(NOT "${is_required}" STREQUAL "")

cmake/modules/DunePythonCommonMacros.cmake

@@ -148,3 +148,5 @@ endfunction()
 if(DUNE_PYTHON_VIRTUALENV_SETUP)
   include(DunePythonVirtualenv)
 endif()
+
+include(DunePythonMacros)

cmake/modules/DunePythonMacros.cmake

+if(NOT PYTHONINTERP_FOUND)
+  message(FATAL_ERROR "Module dune-python requires a Python interpreter")
+endif()
+if(NOT PYTHONLIBS_FOUND)
+  message(FATAL_ERROR "Found a Python interpreter but module dune-python also requires the Python libraries (e.g. a python-dev package)")
+endif()
+
+include_directories("${PYTHON_INCLUDE_DIRS}")
+
+function(add_python_targets base)
+  include(DuneSymlinkOrCopy)
+  if( "${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}" )
+    message(WARNING "Source and binary dir are the same, skipping symlink!")
+  else()
+    foreach(file ${ARGN})
+      dune_symlink_to_source_files(FILES ${file}.py)
+    endforeach()
+  endif()
+endfunction()
+
+include(DuneAddPybind11Module)

cmake/modules/DuneSphinxDoc.cmake

+find_package(Sphinx)
+find_package(PythonInterp)
+
+function(dune_sphinx_doc)
+  # Only proceed if Sphinx was found on the system
+  if(NOT SPHINX_FOUND)
+    message("-- Skipping building Sphinx documentation (Sphinx was not found!)")
+    return()
+  endif()
+
+  # Only proceed if the python interpreter was found by cmake
+  if(NOT PYTHONINTERP_FOUND)
+    message("-- Skipping building Sphinx documentation (Python interpreter was not found!)")
+    return()
+  endif()
+
+  # Parse Arguments
+  include(CMakeParseArguments)
+  cmake_parse_arguments(SPHINX_DOC "" "CONF" "BUILDTYPE" ${ARGN})
+  if(SPHINX_DOC_UNPARSED_ARGUMENTS)
+    message(WARNING "Unparsed arguments in dune_sphinx_doc")
+  endif()
+
+  # copy conf.py into build directory
+  if(NOT SPHINX_DOC_CONF)
+    set(SPHINX_DOC_CONF conf.py)
+  endif()
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${SPHINX_DOC_CONF}.in)
+    configure_file(${SPHINX_DOC_CONF}.in ${CMAKE_CURRENT_BINARY_DIR}/conf.py)
+  elseif(EXISTS ${CMAKE_CURRENT_SOUREC_DIR}/${SPHINX_DOC_CONF})
+    configure_file(${SPHINX_DOC_CONF} ${CMAKE_CURRENT_BINARY_DIR}/conf.py COPYONLY)
+    # elseif(EXISTS ${CMAKE_SOURCE_DIR}/${SPHINX_DOC_CONF}.in)
+    #  configure_file(${CMAKE_SOURCE_DIR}/${SPHINX_DOC_CONF}.in ${CMAKE_BINARY_DIR}/conf.py)
+    # elseif(EXISTS ${CMAKE_SOUREC_DIR}/${SPHINX_DOC_CONF})
+    #  configure_file(${CMAKE_SOURCE_DIR}/${SPHINX_DOC_CONF} ${CMAKE_BINARY_DIR}/conf.py COPYONLY)
+  else()
+    message(SEND_ERROR "Sphinx configuration '${SPHINX_DOC_CONF}' not found.")
+  endif()
+
+  # call Sphinx for each requested build type
+  if(NOT SPHINX_DOC_BUILDTYPE)
+    set(SPHINX_DOC_BUILDTYPE html)
+  endif()
+  foreach(type ${SPHINX_DOC_BUILDTYPE})
+    add_custom_target(sphinx_doc_${type}
+                      COMMAND ${SPHINX_EXECUTABLE}
+                                -b ${type}
+                                -w ${CMAKE_BINARY_DIR}/Sphinx-${type}.log
+                                -c ${CMAKE_CURRENT_BINARY_DIR}
+                                ${CMAKE_CURRENT_BINARY_DIR}
+                                ${CMAKE_CURRENT_BINARY_DIR}/${type}
+                      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+                      DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/conf.py
+                     )
+    add_dependencies(sphinx_doc_${type} sphinx_files)
+    add_dependencies(doc sphinx_doc_${type})
+  endforeach()
+endfunction()
+
+function(add_sphinx_target base file)
+  find_program(JUPYTER jupyter)
+  get_filename_component(extension ${file} EXT)
+  set(SPHINXDIR ${CMAKE_BINARY_DIR}/doc/sphinx)
+  set(OUT ${SPHINXDIR}/${file})
+  set(IN ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+  string(REGEX REPLACE "\\.[^.]*$" "" filebase ${file})
+  set(TARGET ${base}.${file})
+  add_custom_target(${TARGET} DEPENDS ${OUT})
+  add_dependencies(sphinx_files ${TARGET})
+  add_custom_command(
+    OUTPUT ${OUT}
+    DEPENDS ${IN}
+    COMMAND ${CMAKE_COMMAND} -E copy ${IN} ${OUT}
+    VERBATIM
+  )
+  if ("${extension}" STREQUAL ".ipynb")
+    if (JUPYTER)
+      set(TARGET ${base}.${filebase}.rst)
+      set(OUTRST ${SPHINXDIR}/${filebase}.rst)
+      add_custom_target(${TARGET} DEPENDS ${OUTRST})
+      add_dependencies(sphinx_files ${TARGET})
+      add_custom_command(
+        OUTPUT ${OUTRST}
+        DEPENDS ${OUT}
+        COMMAND jupyter nbconvert --ExecutePreprocessor.timeout=-1 --execute --allow-errors --to="rst" ${OUT} --output ${filebase}.rst
+        COMMAND sed -i "s/raw:: latex/math::/g" ${OUTRST}
+        WORKING_DIRECTORY ${SPHINXDIR}
+        VERBATIM
+      )
+    endif()
+  endif()
+endfunction()
+
+function(add_sphinx_files base)
+  foreach(file ${ARGN})
+    add_sphinx_target(${base} ${file})
+  endforeach()
+endfunction()
+function(add_sphinx_targets base)
+  add_custom_target(sphinx_files)
+  add_sphinx_files(${base} ${ARGN})
+  dune_sphinx_doc()
+endfunction()

dune/CMakeLists.txt

 add_subdirectory("common")
+add_subdirectory("python")

dune/python/CMakeLists.txt

+add_subdirectory(common)
+add_subdirectory(pybind11)

dune/python/utility/CMakeLists.txt

-set(HEADERS
-  numpycommdatahandle.hh
-)
-
-install(FILES ${HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/dune/python/utility)

python/CMakeLists.txt

+add_subdirectory(dune)
+
+if((${DUNE_COMMON_VERSION} VERSION_LESS 2.6))
+  configure_file(setup.py.in setup.py)
+endif()

python/dune/CMakeLists.txt

+add_subdirectory(common)
+add_subdirectory(generator)
+add_subdirectory(typeregistry)
+
+add_python_targets(dune
+  __init__
+  create
+  plotting
+  deprecate
+  utility
+)

python/dune/__init__.py

+__import__('pkg_resources').declare_namespace(__name__)

python/dune/common/CMakeLists.txt

@@ -8,3 +8,5 @@ add_python_targets(common
   pickle
   project
 )
+dune_add_pybind11_module(NAME _common)
+set_property(TARGET _common PROPERTY LINK_LIBRARIES dunecommon)

python/dune/common/__init__.py

@@ -22,7 +22,7 @@ try:
 except ImportError:
     logger.info('mpi4py not found, MPI not initialized')
 
-from .._common import *
+from ._common import *
 from .deprecated import DeprecatedObject
 
 import numpy