Thread safety of AMG has regressed
Some years ago, I worked with @markus.blatt to enable the usage of the AMG in dune-istl on multiple threads at the same time. This has regressed, starting with dune 2.4. It's a bit hard to find the real reason, I have obtained various backtraces, e.g.
0 _int_malloc (av=av@entry=0x7fff20000020, bytes=bytes@entry=144) at malloc.c:3516
#1 0x00007ffff5016184 in __GI___libc_malloc (bytes=144) at malloc.c:2913
#2 0x00007ffff5a3ee78 in operator new(unsigned long) ()
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#3 0x0000000000acb1ec in __gnu_cxx::new_allocator<Dune::FieldVector<double, 1> >::allocate (this=0x7fff4546f650, __n=18) at /usr/include/c++/5/ext/new_allocator.h:104
#4 0x0000000000ab7074 in std::allocator_traits<std::allocator<Dune::FieldVector<double, 1> > >::allocate (__a=..., __n=18) at /usr/include/c++/5/bits/alloc_traits.h:491
#5 0x0000000000a791a2 in std::_Vector_base<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >::_M_allocate (this=0x7fff4546f650, __n=18)
at /usr/include/c++/5/bits/stl_vector.h:170
#6 0x0000000000a4250e in std::vector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >::_M_default_append (this=0x7fff4546f650, __n=18)
at /usr/include/c++/5/bits/vector.tcc:557
#7 0x0000000000a26317 in std::vector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >::resize (this=0x7fff4546f650, __new_size=18)
at /usr/include/c++/5/bits/stl_vector.h:676
#8 0x0000000000a1eb66 in Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >::resize (this=0x7fff4546f640, size=18)
at /home/akva/kode/opm/dune/install/include/dune/istl/bvector.hh:495
#9 0x0000000000a1e450 in Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >::resize (this=0x7fff4546f640, size=18,
copyOldValues=false)
at /home/akva/kode/opm/dune/install/include/dune/istl/bvector.hh:519
#10 0x0000000000a1a567 in Opm::Elasticity::MortarUtils::extractBlock (x=..., y=...,
len=18, start=42)
at /home/akva/kode/opm/super-build/opm-upscaling/opm/elasticity/mortar_utils.hpp:27
#11 0x0000000000a1aa05 in Opm::Elasticity::MortarEvaluator::applyscaleadd (
this=0x17fa650, alpha=-1, x=..., y=...)
at /home/akva/kode/opm/super-build/opm-upscaling/opm/elasticity/mortar_evaluator.hpp:70
#12 0x0000000000b659d5 in Dune::MINRESSolver<Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > > >::apply (this=0x17fa690, x=...,
b=..., res=...) at /home/akva/kode/opm/dune/install/include/dune/istl/solvers.hh:794
#13 0x0000000000a76f23 in Opm::Elasticity::ElasticityUpscale<Dune::CpGrid, Opm::Elasticity::AMG1<Dune::SeqSSOR<Dune::BCRSMatrix<Dune::FieldMatrix<double, 1, 1>, std::allocator<Dune::FieldMatrix<double, 1, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, 1> > >::solve (this=0x7fffffffd5e0,
loadcase=2)
at /home/akva/kode/opm/super-build/opm-upscaling/opm/elasticity/elasticity_upscale_impl.hpp:1104
#14 0x0000000000a0cccf in run<Dune::CpGrid, Opm::Elasticity::AMG1<Dune::SeqSSOR<Dune::BCRSMatrix<Dune::FieldMatrix<double, 1, 1>, std::allocator<Dune::FieldMatrix<double, 1, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, 1> > > ()
at /home/akva/kode/opm/super-build/opm-upscaling/examples/upscale_elasticity.cpp:403
#15 0x00007ffff579e43e in ?? () from /usr/lib/x86_64-linux-gnu/libgomp.so.1
#16 0x00007ffff53636ba in start_thread (arg=0x7fff45470700) at pthread_create.c:333
#17 0x00007ffff509941d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
which is hard to decode. I also get
x0000000000b134d4 in Dune::Amg::Hierarchy<Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, std::allocator<Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > > > >::finest (
this=0x0)
at /home/akva/kode/opm/dune/install/include/dune/istl/paamg/hierarchy.hh:1365
1365 return Iterator(finest_);
(gdb) where
#0 0x0000000000b134d4 in Dune::Amg::Hierarchy<Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, std::allocator<Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > > > >::finest
(this=0x0)
at /home/akva/kode/opm/dune/install/include/dune/istl/paamg/hierarchy.hh:1365
#1 0x0000000000b72d4b in Dune::Amg::AMG<Dune::MatrixAdapter<Dune::BCRSMatrix<Dune::FieldMatrix<double, 1, 1>, std::allocator<Dune::FieldMatrix<double, 1, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, Dune::SeqSSOR<Dune::BCRSMatrix<Dune::FieldMatrix<double, 1, 1>, std::allocator<Dune::FieldMatrix<double, 1, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, 1>, Dune::Amg::SequentialInformation, std::allocator<Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > > > >::initIteratorsWithFineLevel (this=0x1829ca0,
levelContext=...)
at /home/akva/kode/opm/dune/install/include/dune/istl/paamg/amg.hh:688
#2 0x0000000000b6a0d1 in Dune::Amg::AMG<Dune::MatrixAdapter<Dune::BCRSMatrix<Dune::FieldMatrix<double, 1, 1>, std::allocator<Dune::FieldMatrix<double, 1, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, Dune::SeqSSOR<Dune::BCRSMatrix<Dune::FieldMatrix<double, 1, 1>, std::allocator<Dune::FieldMatrix<double, 1, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, 1>, Dune::Amg::SequentialInformation, std::allocator<Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > > > >::apply (this=0x1829ca0, v=..., d=...)
at /home/akva/kode/opm/dune/install/include/dune/istl/paamg/amg.hh:661
#3 0x0000000000b5e76a in Opm::Elasticity::MortarSchurPre<Dune::Amg::AMG<Dune::MatrixAdapter<Dune::BCRSMatrix<Dune::FieldMatrix<double, 1, 1>, std::allocator<Dune::FieldMatrix<double, 1, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, Dune::SeqSSOR<Dune::BCRSMatrix<Dune::FieldMatrix<double, 1, 1>, std::allocator<Dune::FieldMatrix<double, 1, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, 1>, Dune::Amg::SequentialInformation, std::allocator<Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > > > > >::apply (this=0x182f590,
v=..., d=...)
at /home/akva/kode/opm/super-build/opm-upscaling/opm/elasticity/mortar_schur_precond.hpp:97
#4 0x0000000000b66212 in Dune::MINRESSolver<Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > > >::apply (this=0x17fa690, x=...,
b=..., res=...) at /home/akva/kode/opm/dune/install/include/dune/istl/solvers.hh:885
#5 0x0000000000a76f23 in Opm::Elasticity::ElasticityUpscale<Dune::CpGrid, Opm::Elasticity::AMG1<Dune::SeqSSOR<Dune::BCRSMatrix<Dune::FieldMatrix<double, 1, 1>, std::allocator<Dune::FieldMatrix<double, 1, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, 1> > >::solve (this=0x7fffffffd5e0,
loadcase=0)
at /home/akva/kode/opm/super-build/opm-upscaling/opm/elasticity/elasticity_upscale_impl.hpp:1104
#6 0x0000000000a0cccf in run<Dune::CpGrid, Opm::Elasticity::AMG1<Dune::SeqSSOR<Dune::BCRSMatrix<Dune::FieldMatrix<double, 1, 1>, std::allocator<Dune::FieldMatrix<double, 1, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, 1> > > ()
at /home/akva/kode/opm/super-build/opm-upscaling/examples/upscale_elasticity.cpp:403
#7 0x00007ffff579acbf in GOMP_parallel () from /usr/lib/x86_64-linux-gnu/libgomp.so.1
#8 0x0000000000a3e81c in run<Dune::CpGrid, Opm::Elasticity::AMG1<Dune::SeqSSOR<Dune::BCRSMatrix<Dune::FieldMatrix<double, 1, 1>, std::allocator<Dune::FieldMatrix<double, 1, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, Dune::BlockVector<Dune::FieldVector<double, 1>, std::allocator<Dune::FieldVector<double, 1> > >, 1> > > (p=...)
---Type <return> to continue, or q <return> to quit---
at /home/akva/kode/opm/super-build/opm-upscaling/examples/upscale_elasticity.cpp:387
#9 0x0000000000a2274e in runAMG<Opm::Elasticity::AMG1> (p=...)
at /home/akva/kode/opm/super-build/opm-upscaling/examples/upscale_elasticity.cpp:471
#10 0x00000000009feb58 in main (argc=2, argv=0x7fffffffe2d8)
at /home/akva/kode/opm/super-build/opm-upscaling/examples/upscale_elasticity.cpp:503
which is easier to spot the failure reason (ie this=0) for the hierarchy.
Likewise, the valgrind output doesn't really show the real reason, it points to the same null pointer dereference, val.log
This is against HEAD as of now, i.e.
dune-common=76cf27a3db89f21b47cbbaa2425dd40b766ceb01
dune-geometry=5e6c08ecc4ddcbd9e855f2a093e25c63f275d103
dune-grid=67c9cb8e42ddab4981ea1353ce6603ba8038256b
dune-istl=d74bd9c2
dune-localfunctions=6e57616d38482ae2923a4bcc36f8d2bc5e0a9b76