Commit a3bf5967 authored by Martin Nolte's avatar Martin Nolte
Browse files

introduce new builder: `HTTPBuilder`

The `HTTPBuilder` uses a server process to do the actual compilation of the
just-in-time compiled Python modules. For client-server communication, the
established HTTP protocol is used.

Having a separate build process allows to run several Python programs using the
same `dune-py` module. Any conflicts are managed by the single server process
performing the actual build.
parent 23cfe597
Pipeline #5965 passed with stage
in 14 minutes and 44 seconds
......@@ -4,4 +4,5 @@ add_python_targets(generator
builder
exceptions
generator
)
httpbuilder
)
from __future__ import absolute_import, division, print_function, unicode_literals
import importlib
import logging
import time
from http.client import HTTPConnection
from dune.common import comm
from dune.generator.exceptions import CompileError, ConfigurationError
logger = logging.getLogger(__name__)
class HTTPBuilder:
def __init__(self, force=False):
self.force = force
self.build_args = dune.common.module.get_default_build_args()
self.dune_py_dir = dune.common.module.get_dune_py_dir()
self.generated_dir = os.path.join(self.dune_py_dir, 'python', 'dune', 'generated')
try:
dune.__path__._path.insert(0,os.path.join(self.dune_py_dir, 'python', 'dune'))
except:
dune.__path__.insert(0,os.path.join(self.dune_py_dir, 'python', 'dune'))
if comm.rank == 0:
dune.common.module.make_dune_py_module(self.dune_py_dir)
tagfile = os.path.join(self.dune_py_dir, ".noconfigure")
if not os.path.isfile(tagfile):
dune.common.module.build_dune_py_module(self.dune_py_dir)
else:
logger.info('using pre configured dune-py module')
comm.barrier()
def request(self, method, url, body=None, headers={}):
conn HTTPConnection(self.host, self.port)
conn.request(method, url, body=body, headers=headers)
response = conn.getresponse()
result = (response.status, response.read())
conn.close()
return data
def sendSource(self, moduleName, source):
result, _ = self.requst("POST", moduleName, body=str(source).encode('utf-8'))
if result not in 202:
raise IOError("Server did not accept compile request.")
def getModule(self, moduleName):
result, data = self.request("GET", moduleName)
if result == 200:
# module has successfully been returned
return data
elif result == 204:
# module is being compiled
return None
elif result == 404:
# module not found; data contains last error message (may be empty)
raise CompileError(data)
else:
raise IOError("Invalid server response.")
def load(self, moduleName, source, pythonName):
module = sys.modules.get("dune.generated." + moduleName)
if module is None:
if comm.rank == 0:
logger.info("Loading " + pythonName)
# try to get module; ignore compile errors on first try
try:
data = self.getModule(moduleName)
except CompileError:
data = None
if data is None:
self.sendSource(moduleName, source)
else:
data = None
comm.barrier()
while data is None:
data = self.getModule(moduleName)
time.sleep(self.delay)
# need to load module, here
module = importlib.import_module("dune.generated." + moduleName)
if self.force:
logger.info("Reloading " + pythonName)
module = reload_module(module)
return module
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment