make locking work correctly - should fix CI issue

python/dune/generator/builder.py

@@ -5,14 +5,12 @@ import subprocess
 import os
 import sys
 
-
-import subprocess
-import os
-import sys
-
 try:
-    from portalocker import Lock
+    from portalocker import Lock as _Lock
     from portalocker.constants import LOCK_EX, LOCK_SH
+    class Lock(_Lock):
+        def __init__(self, path, flags, *args, **kwargs):
+            _Lock.__init__(self,path,*args,flags=flags,timeout=None,**kwargs)
 except ModuleNotFoundError:
     import fcntl
     from fcntl import LOCK_EX, LOCK_SH
@@ -61,11 +59,12 @@ noDepCheck = False
 class Builder:
     def __init__(self, force=False, saveOutput=False):
         self.force = force
-
         self.dune_py_dir = dune.common.module.get_dune_py_dir()
         os.makedirs(self.dune_py_dir, exist_ok=True)
 
         if comm.rank == 0:
+            # lock the whole dune-py module exclusively to possibly
+            # generate and then build the module
             with Lock(os.path.join(self.dune_py_dir, 'lock-module.lock'), flags=LOCK_EX):
                 dune.common.module.make_dune_py_module(self.dune_py_dir)
                 tagfile = os.path.join(self.dune_py_dir, ".noconfigure")
@@ -121,17 +120,18 @@ class Builder:
                 self.savedOutput[1].write("\n###############################\n")
 
     def load(self, moduleName, source, pythonName):
-        module = sys.modules.get("dune.generated." + moduleName)
-        if module is None:
-            if comm.rank == 0:
-                # lock the source file
-                with Lock(os.path.join(self.dune_py_dir, 'lock-'+moduleName+'-source.lock'), flags=LOCK_EX):
-                    sourceFileName = os.path.join(self.generated_dir, moduleName + ".cc")
-                    if not os.path.isfile(sourceFileName):
-                        logger.info("Loading " + pythonName + " (new)")
-                        code = str(source)
-                        # the CMakeLists.txt needs changing and cmake rerun - lock folder
-                        with Lock(os.path.join(self.dune_py_dir, 'lock-all.lock'), flags=LOCK_EX):
+        if comm.rank == 0:
+            module = sys.modules.get("dune.generated." + moduleName)
+            if module is None:
+                # make sure nothing (compilation, generating and building) is # taking place
+                with Lock(os.path.join(self.dune_py_dir, 'lock-module.lock'), flags=LOCK_EX):
+                    # module must be generated so lock the source file
+                    with Lock(os.path.join(self.dune_py_dir, 'lock-'+moduleName+'.lock'), flags=LOCK_EX):
+                        sourceFileName = os.path.join(self.generated_dir, moduleName + ".cc")
+                        if not os.path.isfile(sourceFileName):
+                            logger.info("Loading " + pythonName + " (new)")
+                            code = str(source)
+                            # the CMakeLists.txt needs changing and cmake rerun - lock folder
                             with open(os.path.join(sourceFileName), 'w') as out:
                                 out.write(code)
                             line = "dune_add_pybind11_module(NAME " + moduleName + " EXCLUDE_FROM_ALL)"
@@ -144,24 +144,25 @@ class Builder:
                                     out.write(line+"\n")
                                 # update build system
                                 self.compile()
-                    elif isString(source) and not source == open(os.path.join(sourceFileName), 'r').read():
-                        logger.info("Loading " + pythonName + " (updated)")
-                        code = str(source)
-                        with open(os.path.join(sourceFileName), 'w') as out:
-                            out.write(code)
-                    else:
-                        logger.info("Loading " + pythonName)
-
-                # lock generated module and make sure the folder isn't
-                # locked due to CMakeLists.txt changed being made
+                        elif isString(source) and not source == open(os.path.join(sourceFileName), 'r').read():
+                            logger.info("Loading " + pythonName + " (updated)")
+                            code = str(source)
+                            with open(os.path.join(sourceFileName), 'w') as out:
+                                out.write(code)
+                        else:
+                            logger.info("Loading " + pythonName)
+                # end of exclusive dune-py lock
+            # for compilation a shared lock is enough
+            with Lock(os.path.join(self.dune_py_dir, 'lock-module.lock'), flags=LOCK_SH):
+                # lock generated module
                 with Lock(os.path.join(self.dune_py_dir, 'lock-'+moduleName+'.lock'), flags=LOCK_EX):
-                    with Lock(os.path.join(self.dune_py_dir, 'lock-all.lock'), flags=LOCK_SH):
-                        self.compile(moduleName)
+                    self.compile(moduleName)
 
-            comm.barrier()
-            module = importlib.import_module("dune.generated." + moduleName)
+        comm.barrier()
+        module = importlib.import_module("dune.generated." + moduleName)
 
         if self.force:
             logger.info("Reloading " + pythonName)
             module = reload_module(module)
+
         return module