diff --git a/python/dune/geometry/CMakeLists.txt b/python/dune/geometry/CMakeLists.txt
index 73a871d54895601d5184a2064cf569fd1efa59cd..263f6c1c6f9a40e678f8cab4af3402a7ea0eac20 100644
--- a/python/dune/geometry/CMakeLists.txt
+++ b/python/dune/geometry/CMakeLists.txt
@@ -11,14 +11,15 @@ add_python_targets(geometry
 dune_add_pybind11_module(NAME _geometry SOURCES _geometry.cc)
 target_link_libraries(_geometry PUBLIC Dune::Geometry)
 if( DUNE_ENABLE_PYTHONMODULE_PRECOMPILE )
-  set(DIMS 0 1 2 3 4)
-  foreach( DIM ${DIMS} )
-    dune_add_pybind11_submodule(NAME registerrefelem_${DIM}
-                                MODULE _geometry SOURCES registerrefelem.cc
-                                COMPILE_DEFINITIONS DIM=${DIM})
+  foreach(DIM RANGE 0 4)
+    add_library(registerrefelem_${DIM} OBJECT registerrefelem.cc)
+    target_compile_definitions(registerrefelem_${DIM} PRIVATE DIM=${DIM})
+    target_link_libraries(registerrefelem_${DIM} PRIVATE Dune::Geometry)
+
+    # Register the object in the pybind11 module _geometry
+    target_link_libraries(_geometry PRIVATE registerrefelem_${DIM})
   endforeach()
 endif()
-set_property(TARGET _geometry PROPERTY LINK_LIBRARIES Dune::Geometry APPEND)
 
 if(SKBUILD)
   install(TARGETS _geometry LIBRARY DESTINATION python/dune/geometry)