move locking mechanism from dune.generator to dune.common and use in setup-dunepy

[bugfix]

remove some debug output

bin/setup-dunepy.py

@@ -11,6 +11,7 @@ logger = logging.getLogger(__name__)
 
 try:
     from dune.common.module import build_dune_py_module, get_dune_py_dir, make_dune_py_module, select_modules, resolve_dependencies, resolve_order
+    from dune.common.locking import Lock, LOCK_EX
 except ImportError:
     import os
     here = os.path.dirname(os.path.abspath(__file__))
@@ -20,6 +21,7 @@ except ImportError:
     sys.path.append(modsA)
     if os.path.exists(os.path.join(modsB, "module.py")):
         from module import build_dune_py_module, get_dune_py_dir, make_dune_py_module, select_modules, resolve_dependencies, resolve_order
+        from locking import Lock
     else:
         raise
 
@@ -82,11 +84,6 @@ def main(argv):
         if builddir is None:
             builddir = os.environ.get('DUNE_BUILDDIR', 'build-cmake')
 
-    dunepy = get_dune_py_dir()
-
-    if os.path.exists(dunepy):
-        shutil.rmtree(dunepy)
-
     # Generate list of all modules
     duneModules = select_modules()
 
@@ -101,9 +98,15 @@ def main(argv):
         deps = resolve_order(deps)
         deps += [masterModule]
 
-    foundModule = make_dune_py_module(dunepy, deps)
 
-    output = build_dune_py_module(dunepy, cmake_args, None, builddir, deps, writetagfile=True)
+    dunepy = get_dune_py_dir()
+    dunepyBase = os.path.realpath( os.path.join(dunepy,"..") )
+    with Lock(os.path.join(dunepyBase, 'lock-module.lock'), flags=LOCK_EX):
+        if os.path.exists(dunepy):
+            shutil.rmtree(dunepy)
+        os.makedirs(dunepy)
+        foundModule = make_dune_py_module(dunepy, deps)
+        output = build_dune_py_module(dunepy, cmake_args, None, builddir, deps, writetagfile=True)
 
     print("CMake output")
     print(output)

python/dune/common/CMakeLists.txt

@@ -4,6 +4,7 @@ add_python_targets(common
   compatibility
   deprecated            # deprecated 2.8
   hashit
+  locking
   module
   pickle                # deprecated 2.8
   project

python/dune/common/locking.py

+try:
+    from portalocker import Lock as _Lock
+    from portalocker.constants import LOCK_EX, LOCK_SH
+    class Lock(_Lock):
+        def __init__(self, path, flags, *args, **kwargs):
+            _Lock.__init__(self,path,*args,flags=flags,timeout=None,**kwargs)
+except ModuleNotFoundError:
+    import os
+    import fcntl
+    from fcntl import LOCK_EX, LOCK_SH
+    # file locking from fcntl
+    def lock_file(f, cmd=fcntl.LOCK_EX):
+        fcntl.flock(f, cmd)
+        return f
+    def unlock_file(f):
+        fcntl.flock(f, fcntl.LOCK_UN)
+        return f
+
+    # This file opener *must* be used in a "with" block.
+    class Lock:
+        # Open the file with arguments provided by user. Then acquire
+        # a lock on that file object (WARNING: Advisory locking).
+        def __init__(self, path, flags, *args, **kwargs):
+            # Open the file and acquire a lock on the file before operating
+            self.file = open(path, mode='w+', *args, **kwargs)
+            # Lock the opened file
+            self.file = lock_file(self.file, flags) # flags are either LOCK_EX or LOCK_SH
+
+        # Return the opened file object (knowing a lock has been obtained).
+        def __enter__(self, *args, **kwargs): return self.file
+
+        # Unlock the file and close the file object.
+        def __exit__(self, exc_type=None, exc_value=None, traceback=None):
+            # Flush to make sure all buffered contents are written to file.
+            self.file.flush()
+            os.fsync(self.file.fileno())
+            # Release the lock on the file.
+            self.file = unlock_file(self.file)
+            self.file.close()
+            # Handle exceptions that may have come up during execution, by
+            # default any exceptions are raised to the user.
+            return exc_type == None

python/dune/generator/builder.py

@@ -5,49 +5,8 @@ import subprocess
 import os
 import sys
 
-try:
-    from portalocker import Lock as _Lock
-    from portalocker.constants import LOCK_EX, LOCK_SH
-    class Lock(_Lock):
-        def __init__(self, path, flags, *args, **kwargs):
-            _Lock.__init__(self,path,*args,flags=flags,timeout=None,**kwargs)
-except ModuleNotFoundError:
-    import fcntl
-    from fcntl import LOCK_EX, LOCK_SH
-    # file locking from fcntl
-    def lock_file(f, cmd=fcntl.LOCK_EX):
-        fcntl.flock(f, cmd)
-        return f
-    def unlock_file(f):
-        fcntl.flock(f, fcntl.LOCK_UN)
-        return f
-
-    # This file opener *must* be used in a "with" block.
-    class Lock:
-        # Open the file with arguments provided by user. Then acquire
-        # a lock on that file object (WARNING: Advisory locking).
-        def __init__(self, path, flags, *args, **kwargs):
-            # Open the file and acquire a lock on the file before operating
-            self.file = open(path, mode='w+', *args, **kwargs)
-            # Lock the opened file
-            self.file = lock_file(self.file, flags) # flags are either LOCK_EX or LOCK_SH
-
-        # Return the opened file object (knowing a lock has been obtained).
-        def __enter__(self, *args, **kwargs): return self.file
-
-        # Unlock the file and close the file object.
-        def __exit__(self, exc_type=None, exc_value=None, traceback=None):
-            # Flush to make sure all buffered contents are written to file.
-            self.file.flush()
-            os.fsync(self.file.fileno())
-            # Release the lock on the file.
-            self.file = unlock_file(self.file)
-            self.file.close()
-            # Handle exceptions that may have come up during execution, by
-            # default any exceptions are raised to the user.
-            return exc_type == None
-
 from dune.common import comm
+from dune.common.locking import Lock, LOCK_EX,LOCK_SH
 from dune.common.utility import buffer_to_str, isString, reload_module
 from dune.generator.exceptions import CompileError, ConfigurationError
 import dune.common.module

python/dune/packagemetadata.py

@@ -265,18 +265,21 @@ def cmakeFlags():
     ]))
     # test environment for additional flags
     cmakeFlags = os.environ.get('DUNE_CMAKE_FLAGS')
-    print("@@@@@ cmakeFlags=",cmakeFlags,flush=True)
     # split cmakeFlags and add them to flags
     if cmakeFlags is not None:
         flags += shlex.split(cmakeFlags)
     cmakeFlags = os.environ.get('CMAKE_FLAGS')
-    print("@@@@@ cmakeFlags=",cmakeFlags,flush=True)
     if cmakeFlags is not None:
         flags += shlex.split(cmakeFlags)
-    print("@@@@@ flags=",flags,flush=True)
     return flags
 
 def inVEnv():
+    # check whether we are in a anaconda environment
+    # were the checks based on prefix and base_prefix
+    # seem to fail
+    if "CONDA_DEFAULT_ENV" in os.environ:
+        return 1
+
     # If sys.real_prefix exists, this is a virtualenv set up with the virtualenv package
     real_prefix = hasattr(sys, 'real_prefix')
     if real_prefix: