[!536] Try to install a virtualenv with pip before annoying the user

When the virtualenv support was added to the build system,
https://bugs.launchpad.net/debian/+source/python3.4/+bug/1290847# made us bail out
completely on building a virtualenv with preinstalled pip. The situation has improved
a lot since then; even Ubuntu 14.04 gets this right now as long as you install the
right packages. So let's be nice to our users and try whether installing pip works
before forcing them to do it manually or use `get-pip`. If a user insists on `get-pip`
by setting `DUNE_PYTHON_ALLOW_GET_PIP`, we skip this additional step and keep using
`get-pip`.

This shouldn't break anything, so I'll backport it to 2.6 as well, as it should make
it much easier to use the virtualenv in CI images (e.g. for
`dune-pdelab-systemtesting`, which requires `dune-testtools`).

See merge request !536

(cherry picked from commit 7e856817)

99f05bb7 Try to install Python virtualenv with pip before forcing get-pip onto users
92bf4520 Update documentation
1e9ae690 Create virtualenv during CI tests

.gitlab-ci.yml

@@ -9,6 +9,7 @@ debian:10 gcc-8-17:
   script: duneci-standard-test
   variables:
     DUNECI_TOOLCHAIN: gcc-8-17
+    DUNECI_CMAKE_FLAGS: "-DDUNE_PYTHON_VIRTUALENV_SETUP=1"
   tags: [duneci]
 
 debian:10 clang-6-libcpp-17:
@@ -23,6 +24,7 @@ debian:9 gcc-6-14:
   script: duneci-standard-test
   variables:
     DUNECI_TOOLCHAIN: gcc-6-14
+    DUNECI_CMAKE_FLAGS: "-DDUNE_PYTHON_VIRTUALENV_SETUP=1"
   tags: [duneci]
 
 debian:9 clang-3.8-14:
@@ -37,6 +39,7 @@ ubuntu:16.04 gcc-5-14:
   script: duneci-standard-test
   variables:
     DUNECI_TOOLCHAIN: gcc-5-14
+    DUNECI_CMAKE_FLAGS: "-DDUNE_PYTHON_VIRTUALENV_SETUP=1"
   tags: [duneci]
 
 ubuntu:16.04 clang-3.8-14:
@@ -51,4 +54,5 @@ ubuntu:18.04 clang-6-17:
   script: duneci-standard-test
   variables:
     DUNECI_TOOLCHAIN: clang-6-17
+    DUNECI_CMAKE_FLAGS: "-DDUNE_PYTHON_VIRTUALENV_SETUP=1"
   tags: [duneci]

cmake/modules/DunePythonVirtualenv.cmake

@@ -34,16 +34,18 @@
 #
 # .. cmake_variable:: DUNE_PYTHON_ALLOW_GET_PIP
 #
-#    Set this variable to allow the Dune build system to download :code:`get-pip.py`
-#    from https://bootstrap.pypa.io/get-pip.py at configure time and execute it
-#    to install pip into the freshly set up virtual environment. This step became
-#    necessary because of a debian bug:
-#    https://bugs.launchpad.net/debian/+source/python3.4/+bug/1290847
+#    The Dune build system will try to build a virtualenv with pip installed into it,
+#    but this can fail in some situations, in particular on Debian and Ubuntu distributions.
+#    In this case, you will se a warning message in the CMake output. If you are on Debian
+#    or Ubuntu, try installing the :code:`python3-venv` (for Python 3) and / or
+#    :code:`python-virtualenv` packages, delete your build directory and try configuring
+#    again.
 #
-#    If you do not want the Dune build system to download :code:`get-pip.py`, you can
-#    manually activate the virtual environment (sourcing the activate script
-#    symlinked into the build directories), install pip through your favorite
-#    method and reconfigure.
+#    If that still does not help, set this variable to allow the Dune build system to download
+#    :code:`get-pip.py` from https://bootstrap.pypa.io/get-pip.py at configure time and execute
+#    it to install pip into the freshly set up virtual environment. While this should normally
+#    not be necessary anymore, see https://bugs.launchpad.net/debian/+source/python3.4/+bug/1290847
+#    for more information about the underlying distribution bug.
 #
 
 # First, we look through the dependency tree of this module for a build directory
@@ -80,12 +82,34 @@ if(NOT DUNE_PYTHON_VIRTUALENV_PATH)
 
   # Set up the env itself
   message("-- Building a virtual env in ${CMAKE_BINARY_DIR}/dune-env...")
-  dune_execute_process(COMMAND ${PYTHON_EXECUTABLE}
-                                -m ${VIRTUALENV_PACKAGE_NAME}
-                                ${NOPIP_OPTION}
-                                ${CMAKE_BINARY_DIR}/dune-env
-                       ERROR_MESSAGE "Fatal error when setting up a virtualenv."
-                       )
+  # First, try to build it with pip installed, but only if the user has not set DUNE_PYTHON_ALLOW_GET_PIP
+  if(NOT DUNE_PYTHON_ALLOW_GET_PIP)
+    dune_execute_process(COMMAND ${PYTHON_EXECUTABLE}
+                                  -m ${VIRTUALENV_PACKAGE_NAME}
+                                  "${CMAKE_BINARY_DIR}/dune-env"
+                         RESULT_VARIABLE venv_install_result
+                         )
+  endif()
+
+  if(NOT "${venv_install_result}" STREQUAL "0")
+
+    if(NOT DUNE_PYTHON_ALLOW_GET_PIP)
+      # we attempted the default installation before, so issue a warning
+      message("-- WARNING: Failed to build a virtual env with pip installed, trying again without pip")
+      message("-- If you are using Debian or Ubuntu, consider installing python3-venv and / or python-virtualenv")
+    endif()
+
+    # remove the remainder of a potential first attempt
+    file(REMOVE_RECURSE "${CMAKE_BINARY_DIR}/dune-env")
+
+    # try to build the env without pip
+    dune_execute_process(COMMAND ${PYTHON_EXECUTABLE}
+                                  -m ${VIRTUALENV_PACKAGE_NAME}
+                                  ${NOPIP_OPTION}
+                                  "${CMAKE_BINARY_DIR}/dune-env"
+                         ERROR_MESSAGE "Fatal error when setting up a virtualenv."
+                         )
+  endif()
 
   # And set the path to it
   set(DUNE_PYTHON_VIRTUALENV_PATH ${CMAKE_BINARY_DIR}/dune-env)
@@ -113,11 +137,11 @@ else()
   message(WARNING "Writing script 'run-in-dune-env' not implemented on your platform!")
 endif()
 
-# We previously omitted pip from the env, because of this Debian bug:
+# The virtualenv might not contain pip due to the distribution bug described in
 # https://bugs.launchpad.net/debian/+source/python3.4/+bug/1290847
-# We now, need to install pip. Easiest way is to download the get-pip
-# script. We ask users for permission to do so, or we allow them to
-# set it up themselves.
+# We need to install pip, so if pip is missing, we offer to download and run the get-pip
+# script. We ask users for permission to do so, or we allow them to set it up themselves.
+
 dune_python_find_package(PACKAGE pip
                          RESULT pippresent
                          INTERPRETER ${DUNE_PYTHON_VIRTUALENV_EXECUTABLE}
@@ -148,4 +172,4 @@ if(NOT pippresent)
                          the CMake variable DUNE_PYTHON_ALLOW_GET_PIP to allow Dune to use get-pip.py
                          from https://bootstrap.pypa.io/get-pip.py")
   endif()
-endif()
\ No newline at end of file
+endif()