From f40b9821be2674bd442f97b838789b378b0ac8a1 Mon Sep 17 00:00:00 2001
From: dedner <a.s.dedner@warwick.ac.uk>
Date: Tue, 9 Feb 2021 09:52:37 +0000
Subject: [PATCH] move locking mechanism from dune.generator to dune.common and
use in setup-dunepy
[bugfix]
remove some debug output
---
bin/setup-dunepy.py | 17 +++++++-----
python/dune/common/CMakeLists.txt | 1 +
python/dune/common/locking.py | 42 ++++++++++++++++++++++++++++++
python/dune/generator/builder.py | 43 +------------------------------
python/dune/packagemetadata.py | 9 ++++---
5 files changed, 60 insertions(+), 52 deletions(-)
create mode 100644 python/dune/common/locking.py
diff --git a/bin/setup-dunepy.py b/bin/setup-dunepy.py
index 3b4366037..6384838a6 100755
--- a/bin/setup-dunepy.py
+++ b/bin/setup-dunepy.py
@@ -11,6 +11,7 @@ logger = logging.getLogger(__name__)
try:
from dune.common.module import build_dune_py_module, get_dune_py_dir, make_dune_py_module, select_modules, resolve_dependencies, resolve_order
+ from dune.common.locking import Lock, LOCK_EX
except ImportError:
import os
here = os.path.dirname(os.path.abspath(__file__))
@@ -20,6 +21,7 @@ except ImportError:
sys.path.append(modsA)
if os.path.exists(os.path.join(modsB, "module.py")):
from module import build_dune_py_module, get_dune_py_dir, make_dune_py_module, select_modules, resolve_dependencies, resolve_order
+ from locking import Lock
else:
raise
@@ -82,11 +84,6 @@ def main(argv):
if builddir is None:
builddir = os.environ.get('DUNE_BUILDDIR', 'build-cmake')
- dunepy = get_dune_py_dir()
-
- if os.path.exists(dunepy):
- shutil.rmtree(dunepy)
-
# Generate list of all modules
duneModules = select_modules()
@@ -101,9 +98,15 @@ def main(argv):
deps = resolve_order(deps)
deps += [masterModule]
- foundModule = make_dune_py_module(dunepy, deps)
- output = build_dune_py_module(dunepy, cmake_args, None, builddir, deps, writetagfile=True)
+ dunepy = get_dune_py_dir()
+ dunepyBase = os.path.realpath( os.path.join(dunepy,"..") )
+ with Lock(os.path.join(dunepyBase, 'lock-module.lock'), flags=LOCK_EX):
+ if os.path.exists(dunepy):
+ shutil.rmtree(dunepy)
+ os.makedirs(dunepy)
+ foundModule = make_dune_py_module(dunepy, deps)
+ output = build_dune_py_module(dunepy, cmake_args, None, builddir, deps, writetagfile=True)
print("CMake output")
print(output)
diff --git a/python/dune/common/CMakeLists.txt b/python/dune/common/CMakeLists.txt
index 0fcbcdf9c..fe114b9c7 100644
--- a/python/dune/common/CMakeLists.txt
+++ b/python/dune/common/CMakeLists.txt
@@ -4,6 +4,7 @@ add_python_targets(common
compatibility
deprecated # deprecated 2.8
hashit
+ locking
module
pickle # deprecated 2.8
project
diff --git a/python/dune/common/locking.py b/python/dune/common/locking.py
new file mode 100644
index 000000000..6858d43fe
--- /dev/null
+++ b/python/dune/common/locking.py
@@ -0,0 +1,42 @@
+try:
+ from portalocker import Lock as _Lock
+ from portalocker.constants import LOCK_EX, LOCK_SH
+ class Lock(_Lock):
+ def __init__(self, path, flags, *args, **kwargs):
+ _Lock.__init__(self,path,*args,flags=flags,timeout=None,**kwargs)
+except ModuleNotFoundError:
+ import os
+ import fcntl
+ from fcntl import LOCK_EX, LOCK_SH
+ # file locking from fcntl
+ def lock_file(f, cmd=fcntl.LOCK_EX):
+ fcntl.flock(f, cmd)
+ return f
+ def unlock_file(f):
+ fcntl.flock(f, fcntl.LOCK_UN)
+ return f
+
+ # This file opener *must* be used in a "with" block.
+ class Lock:
+ # Open the file with arguments provided by user. Then acquire
+ # a lock on that file object (WARNING: Advisory locking).
+ def __init__(self, path, flags, *args, **kwargs):
+ # Open the file and acquire a lock on the file before operating
+ self.file = open(path, mode='w+', *args, **kwargs)
+ # Lock the opened file
+ self.file = lock_file(self.file, flags) # flags are either LOCK_EX or LOCK_SH
+
+ # Return the opened file object (knowing a lock has been obtained).
+ def __enter__(self, *args, **kwargs): return self.file
+
+ # Unlock the file and close the file object.
+ def __exit__(self, exc_type=None, exc_value=None, traceback=None):
+ # Flush to make sure all buffered contents are written to file.
+ self.file.flush()
+ os.fsync(self.file.fileno())
+ # Release the lock on the file.
+ self.file = unlock_file(self.file)
+ self.file.close()
+ # Handle exceptions that may have come up during execution, by
+ # default any exceptions are raised to the user.
+ return exc_type == None
diff --git a/python/dune/generator/builder.py b/python/dune/generator/builder.py
index 2e3793316..5e0c185b9 100644
--- a/python/dune/generator/builder.py
+++ b/python/dune/generator/builder.py
@@ -5,49 +5,8 @@ import subprocess
import os
import sys
-try:
- from portalocker import Lock as _Lock
- from portalocker.constants import LOCK_EX, LOCK_SH
- class Lock(_Lock):
- def __init__(self, path, flags, *args, **kwargs):
- _Lock.__init__(self,path,*args,flags=flags,timeout=None,**kwargs)
-except ModuleNotFoundError:
- import fcntl
- from fcntl import LOCK_EX, LOCK_SH
- # file locking from fcntl
- def lock_file(f, cmd=fcntl.LOCK_EX):
- fcntl.flock(f, cmd)
- return f
- def unlock_file(f):
- fcntl.flock(f, fcntl.LOCK_UN)
- return f
-
- # This file opener *must* be used in a "with" block.
- class Lock:
- # Open the file with arguments provided by user. Then acquire
- # a lock on that file object (WARNING: Advisory locking).
- def __init__(self, path, flags, *args, **kwargs):
- # Open the file and acquire a lock on the file before operating
- self.file = open(path, mode='w+', *args, **kwargs)
- # Lock the opened file
- self.file = lock_file(self.file, flags) # flags are either LOCK_EX or LOCK_SH
-
- # Return the opened file object (knowing a lock has been obtained).
- def __enter__(self, *args, **kwargs): return self.file
-
- # Unlock the file and close the file object.
- def __exit__(self, exc_type=None, exc_value=None, traceback=None):
- # Flush to make sure all buffered contents are written to file.
- self.file.flush()
- os.fsync(self.file.fileno())
- # Release the lock on the file.
- self.file = unlock_file(self.file)
- self.file.close()
- # Handle exceptions that may have come up during execution, by
- # default any exceptions are raised to the user.
- return exc_type == None
-
from dune.common import comm
+from dune.common.locking import Lock, LOCK_EX,LOCK_SH
from dune.common.utility import buffer_to_str, isString, reload_module
from dune.generator.exceptions import CompileError, ConfigurationError
import dune.common.module
diff --git a/python/dune/packagemetadata.py b/python/dune/packagemetadata.py
index 692fbac33..19cb490dc 100755
--- a/python/dune/packagemetadata.py
+++ b/python/dune/packagemetadata.py
@@ -265,18 +265,21 @@ def cmakeFlags():
]))
# test environment for additional flags
cmakeFlags = os.environ.get('DUNE_CMAKE_FLAGS')
- print("@@@@@ cmakeFlags=",cmakeFlags,flush=True)
# split cmakeFlags and add them to flags
if cmakeFlags is not None:
flags += shlex.split(cmakeFlags)
cmakeFlags = os.environ.get('CMAKE_FLAGS')
- print("@@@@@ cmakeFlags=",cmakeFlags,flush=True)
if cmakeFlags is not None:
flags += shlex.split(cmakeFlags)
- print("@@@@@ flags=",flags,flush=True)
return flags
def inVEnv():
+ # check whether we are in a anaconda environment
+ # were the checks based on prefix and base_prefix
+ # seem to fail
+ if "CONDA_DEFAULT_ENV" in os.environ:
+ return 1
+
# If sys.real_prefix exists, this is a virtualenv set up with the virtualenv package
real_prefix = hasattr(sys, 'real_prefix')
if real_prefix:
--
GitLab