Commit b10c4162 authored by Andreas Dedner's avatar Andreas Dedner

[!131] Feature/dune python add cache lock

Merge branch 'feature/dune-python-add_cache_lock' into 'master'

ref:staging/dune-python Moved source branch from [!122]

See merge request [!131]

  [!122]: gitlab.dune-project.org/NoneNone/merge_requests/122
  [!131]: gitlab.dune-project.org/staging/dune-python/merge_requests/131
parents fb787bca 635b36ee
Pipeline #21537 passed with stage
in 8 minutes and 34 seconds
......@@ -40,6 +40,9 @@ def reset():
unsetFlags()
setDependencyCheck()
def path(f):
return os.path.dirname(os.path.realpath(f))+"/"
class Constructor(object):
def __init__(self, args, body=None, extra=None):
self.args = args
......
......@@ -7,6 +7,28 @@ from . import builder
from dune.common.compatibility import isString
def load(functionName, includes, *args):
'''Just in time compile an algorithm.
Generates binding for a single (template) function. The name of the
function and the C++ types of the arguments passed to this function are
used to generate a static type used in the bindings. The file(s)
containing the code for the function are passed in either as single
string or as a list of strings. Note that these files will be copied
into the generated module. The file name can include a path name. So in
the simples case `includes="header.hh" will include the file from the
current working directory. To include a file from the directory
containing the calling script use
`includes=dune.generator.path(__file__)+"header.hh"`.
Args:
functionName: name of the C++ function to provide bindings for
includes: single or list of files to add to the generated module
*args: list of arguments that will be passed to the generated module
Returns:
Callalble object
'''
source = '#include <config.h>\n\n'
source += '#define USING_DUNE_PYTHON 1\n\n'
if isString(includes):
......@@ -78,4 +100,16 @@ def load(functionName, includes, *args):
def run(functionName, includes, *args):
'''Just in time compile and run an algorithm.
For details see the help for `dune.algorithm.load`.
Args:
functionName: name of the C++ function to provide bindings for
includes: single or list of files to add to the generated module
*args: list of arguments that will be passed to the generated module
Returns:
return value of `functionName(*args)`
'''
return load(functionName, includes, *args)(*args)
......@@ -16,6 +16,10 @@ logger = logging.getLogger(__name__)
cxxFlags = None
noDepCheck = False
import portalocker
from portalocker import Lock
from portalocker.constants import LOCK_EX, LOCK_SH
class Builder:
def __init__(self, force=False):
self.force = force
......@@ -61,25 +65,39 @@ class Builder:
module = sys.modules.get("dune.generated." + moduleName)
if module is None:
if comm.rank == 0:
sourceFileName = os.path.join(self.generated_dir, moduleName + ".cc")
if not os.path.isfile(sourceFileName):
logger.info("Loading " + pythonName + " (new)")
code = str(source)
with open(os.path.join(sourceFileName), 'w') as out:
out.write(code)
with open(os.path.join(self.generated_dir, "CMakeLists.txt"), 'a') as out:
out.write("dune_add_pybind11_module(NAME " + moduleName + " EXCLUDE_FROM_ALL)\n")
# update build system
self.compile()
elif isString(source) and not source == open(os.path.join(sourceFileName), 'r').read():
logger.info("Loading " + pythonName + " (updated)")
code = str(source)
with open(os.path.join(sourceFileName), 'w') as out:
out.write(code)
else:
logger.info("Loading " + pythonName)
# lock the source file
with Lock(os.path.join(self.dune_py_dir, 'lock-'+moduleName+'-source.lock'), flags=LOCK_EX):
sourceFileName = os.path.join(self.generated_dir, moduleName + ".cc")
if not os.path.isfile(sourceFileName):
logger.info("Loading " + pythonName + " (new)")
code = str(source)
# the CMakeLists.txt needs changing and cmake rerun - lock folder
with Lock(os.path.join(self.dune_py_dir, 'lock-all.lock'), flags=LOCK_EX):
with open(os.path.join(sourceFileName), 'w') as out:
out.write(code)
line = "dune_add_pybind11_module(NAME " + moduleName + " EXCLUDE_FROM_ALL)"
# first check if trhis line is already present in the CMakeLists file
# (possible if a previous script was stopped by user before module was compiled)
with open(os.path.join(self.generated_dir, "CMakeLists.txt"), 'r') as out:
found = line in out.read()
if not found:
with open(os.path.join(self.generated_dir, "CMakeLists.txt"), 'a') as out:
out.write(line+"\n")
# update build system
self.compile()
elif isString(source) and not source == open(os.path.join(sourceFileName), 'r').read():
logger.info("Loading " + pythonName + " (updated)")
code = str(source)
with open(os.path.join(sourceFileName), 'w') as out:
out.write(code)
else:
logger.info("Loading " + pythonName)
self.compile(moduleName)
# lock generated module and make sure the folder isn't
# locked due to CMakeLists.txt changed being made
with Lock(os.path.join(self.dune_py_dir, 'lock-'+moduleName+'.lock'), flags=LOCK_EX):
with Lock(os.path.join(self.dune_py_dir, 'lock-all.lock'), flags=LOCK_SH):
self.compile(moduleName)
comm.barrier()
module = importlib.import_module("dune.generated." + moduleName)
......
......@@ -7,5 +7,6 @@ setup(name="dune.python",
author="Andreas Dedner and Martin Nolte",
packages = find_packages(),
zip_safe = 0,
package_data = {'': ['*.so']}
package_data = {'': ['*.so']},
install_requires = ['portalocker']
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment