6 #include "Numeric/Stencil.H"
11 const Vector<BoxArray>& a_grids,
12 const Vector<DistributionMapping>& a_dmap,
15 BL_PROFILE(
"Operator::Elastic::Elastic()");
17 define(a_geom, a_grids, a_dmap, a_info);
27 const Vector<BoxArray>& a_grids,
28 const Vector<DistributionMapping>& a_dmap,
30 const Vector<FabFactory<FArrayBox>
const*>& a_factory)
32 BL_PROFILE(
"Operator::Elastic::define()");
34 Operator::define(a_geom, a_grids, a_dmap, a_info, a_factory);
38 m_ddw_mf.resize(m_num_amr_levels);
39 m_psi_mf.resize(m_num_amr_levels);
40 for (
int amrlev = 0; amrlev < m_num_amr_levels; ++amrlev)
42 m_ddw_mf[amrlev].resize(m_num_mg_levels[amrlev]);
43 m_psi_mf[amrlev].resize(m_num_mg_levels[amrlev]);
44 for (
int mglev = 0; mglev < m_num_mg_levels[amrlev]; ++mglev)
46 m_ddw_mf[amrlev][mglev].reset(
new MultiTab(amrex::convert(m_grids[amrlev][mglev],
47 amrex::IntVect::TheNodeVector()),
48 m_dmap[amrlev][mglev], 1, model_nghost));
49 m_psi_mf[amrlev][mglev].reset(
new MultiFab(m_grids[amrlev][mglev],
50 m_dmap[amrlev][mglev], 1, model_nghost));
52 if (!m_psi_set) m_psi_mf[amrlev][mglev]->setVal(1.0);
61 for (
int amrlev = 0; amrlev < m_num_amr_levels; amrlev++)
63 amrex::Box domain(m_geom[amrlev][0].Domain());
64 domain.convert(amrex::IntVect::TheNodeVector());
66 int nghost = m_ddw_mf[amrlev][0]->nGrow();
68 for (MFIter mfi(*m_ddw_mf[amrlev][0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi)
70 Box bx = mfi.tilebox();
74 amrex::Array4<MATRIX4>
const& ddw = (*(m_ddw_mf[amrlev][0])).array(mfi);
76 amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(
int i,
int j,
int k) {
77 ddw(i, j, k) = a_model;
80 if (ddw(i, j, k).contains_nan())
Util::Abort(
INFO,
"model is nan at (", i,
",", j,
",", k,
"), amrlev=", amrlev);
92 BL_PROFILE(
"Operator::Elastic::SetModel()");
94 amrex::Box domain(m_geom[amrlev][0].Domain());
95 domain.convert(amrex::IntVect::TheNodeVector());
97 if (a_model.boxArray() != m_ddw_mf[amrlev][0]->boxArray())
Util::Abort(
INFO,
"Inconsistent box arrays\n",
"a_model.boxArray()=\n", a_model.boxArray(),
"\n but the current box array is \n", m_ddw_mf[amrlev][0]->boxArray());
98 if (a_model.DistributionMap() != m_ddw_mf[amrlev][0]->DistributionMap())
Util::Abort(
INFO,
"Inconsistent distribution maps");
99 if (a_model.nComp() != m_ddw_mf[amrlev][0]->nComp())
Util::Abort(
INFO,
"Inconsistent # of components - should be ", m_ddw_mf[amrlev][0]->nComp());
100 if (a_model.nGrow() != m_ddw_mf[amrlev][0]->nGrow())
Util::Abort(
INFO,
"Inconsistent # of ghost nodes, should be ", m_ddw_mf[amrlev][0]->nGrow());
103 int nghost = m_ddw_mf[amrlev][0]->nGrow();
105 for (MFIter mfi(a_model, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi)
107 Box bx = mfi.tilebox();
111 amrex::Array4<MATRIX4>
const& C = (*(m_ddw_mf[amrlev][0])).array(mfi);
112 amrex::Array4<const MATRIX4>
const& a_C = a_model.array(mfi);
114 amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(
int i,
int j,
int k) {
115 C(i, j, k) = a_C(i, j, k);
128 BL_PROFILE(
"Operator::Elastic::SetPsi()");
129 amrex::Box domain(m_geom[amrlev][0].Domain());
131 for (MFIter mfi(a_psi_mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi)
133 Box bx = mfi.growntilebox() & domain;
135 amrex::Array4<Set::Scalar>
const& m_psi = (*(m_psi_mf[amrlev][0])).array(mfi);
136 amrex::Array4<const Set::Scalar>
const& a_psi = a_psi_mf.array(mfi);
138 amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(
int i,
int j,
int k) {
139 m_psi(i, j, k) = a_psi(i, j, k);
149 BL_PROFILE(
"Operator::Elastic::Fapply()");
151 amrex::Box domain(m_geom[amrlev][mglev].Domain());
152 domain.convert(amrex::IntVect::TheNodeVector());
154 const Real* DX = m_geom[amrlev][mglev].CellSize();
156 for (MFIter mfi(a_f, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi)
158 Box bx = mfi.validbox().grow(1) & domain;
159 amrex::Box tilebox = mfi.grownnodaltilebox() & bx;
161 amrex::Array4<MATRIX4>
const& DDW = (*(m_ddw_mf[amrlev][mglev])).array(mfi);
162 amrex::Array4<const amrex::Real>
const& U = a_u.array(mfi);
163 amrex::Array4<amrex::Real>
const&
F = a_f.array(mfi);
164 amrex::Array4<Set::Scalar>
const& psi = m_psi_mf[amrlev][mglev]->array(mfi);
166 const Dim3 lo = amrex::lbound(domain), hi = amrex::ubound(domain);
168 amrex::ParallelFor(tilebox, [=] AMREX_GPU_DEVICE(
int i,
int j,
int k) {
173 for (
int p = 0; p < AMREX_SPACEDIM; p++) u(p) = U(i, j, k, p);
176 bool AMREX_D_DECL(xmin = (i == lo.x), ymin = (j == lo.y), zmin = (k == lo.z)),
177 AMREX_D_DECL(xmax = (i == hi.x), ymax = (j == hi.y), zmax = (k == hi.z));
180 std::array<Numeric::StencilType, AMREX_SPACEDIM>
187 for (
int p = 0; p < AMREX_SPACEDIM; p++)
189 AMREX_D_TERM(
gradu(p, 0) = (
Numeric::Stencil<Set::Scalar, 1, 0, 0>::D(U, i, j, k, p, DX, sten));,
190 gradu(p, 1) = (
Numeric::Stencil<Set::Scalar, 0, 1, 0>::D(U, i, j, k, p, DX, sten));,
191 gradu(p, 2) = (
Numeric::Stencil<Set::Scalar, 0, 0, 1>::D(U, i, j, k, p, DX, sten)););
203 if (AMREX_D_TERM(xmax || xmin, || ymax || ymin, || zmax || zmin))
205 f = (*m_bc)(u,
gradu, sig, i, j, k, domain);
217 for (
int p = 0; p < AMREX_SPACEDIM; p++)
220 AMREX_D_TERM(gradgradu(p, 0, 0) = (
Numeric::Stencil<Set::Scalar, 2, 0, 0>::D(U, i, j, k, p, DX));,
221 gradgradu(p, 1, 1) = (
Numeric::Stencil<Set::Scalar, 0, 2, 0>::D(U, i, j, k, p, DX));,
222 gradgradu(p, 2, 2) = (
Numeric::Stencil<Set::Scalar, 0, 0, 2>::D(U, i, j, k, p, DX)););
226 gradgradu(p, 0, 1) = (
Numeric::Stencil<Set::Scalar, 1, 1, 0>::D(U, i, j, k, p, DX));
227 gradgradu(p, 1, 0) = gradgradu(p, 0, 1);
229 gradgradu(p, 0, 2) = (
Numeric::Stencil<Set::Scalar, 1, 0, 1>::D(U, i, j, k, p, DX));
230 gradgradu(p, 1, 2) = (
Numeric::Stencil<Set::Scalar, 0, 1, 1>::D(U, i, j, k, p, DX));
231 gradgradu(p, 2, 0) = gradgradu(p, 0, 2);
232 gradgradu(p, 2, 1) = gradgradu(p, 1, 2););
244 f = (DDW(i, j, k) * gradgradu) * psi_avg;
249 AMREX_D_DECL(Cgrad1 = (
Numeric::Stencil<MATRIX4, 1, 0, 0>::D(DDW, i, j, k, 0, DX, sten)),
250 Cgrad2 = (
Numeric::Stencil<MATRIX4, 0, 1, 0>::D(DDW, i, j, k, 0, DX, sten)),
252 f += (AMREX_D_TERM((Cgrad1 *
gradu).col(0),
253 +(Cgrad2 *
gradu).col(1),
254 +(Cgrad3 *
gradu).col(2))) * (psi_avg);
259 gradpsi *= (1.0 - m_psi_small);
260 f += (DDW(i, j, k) *
gradu) * gradpsi;
263 AMREX_D_TERM(
F(i, j, k, 0) = f[0];,
F(i, j, k, 1) = f[1];,
F(i, j, k, 2) = f[2];);
274 BL_PROFILE(
"Operator::Elastic::Diagonal()");
276 amrex::Box domain(m_geom[amrlev][mglev].Domain());
277 domain.convert(amrex::IntVect::TheNodeVector());
278 const Real* DX = m_geom[amrlev][mglev].CellSize();
280 for (MFIter mfi(a_diag, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi)
282 Box bx = mfi.validbox().grow(1) & domain;
283 amrex::Box tilebox = mfi.grownnodaltilebox() & bx;
285 amrex::Array4<MATRIX4>
const& DDW = (*(m_ddw_mf[amrlev][mglev])).array(mfi);
286 amrex::Array4<Set::Scalar>
const& diag = a_diag.array(mfi);
287 amrex::Array4<Set::Scalar>
const& psi = m_psi_mf[amrlev][mglev]->array(mfi);
289 const Dim3 lo = amrex::lbound(domain), hi = amrex::ubound(domain);
291 amrex::ParallelFor(tilebox, [=] AMREX_GPU_DEVICE(
int i,
int j,
int k) {
295 bool AMREX_D_DECL(xmin = (i == lo.x), ymin = (j == lo.y), zmin = (k == lo.z)),
296 AMREX_D_DECL(xmax = (i == hi.x), ymax = (j == hi.y), zmax = (k == hi.z));
304 for (
int p = 0; p < AMREX_SPACEDIM; p++)
307 diag(i, j, k, p) = 0.0;
308 for (
int q = 0; q < AMREX_SPACEDIM; q++)
311 gradu(q, 0) = ((!xmax ? 0.0 : (p == q ? 1.0 : 0.0)) - (!xmin ? 0.0 : (p == q ? 1.0 : 0.0))) / ((xmin || xmax ? 1.0 : 2.0) * DX[0]);,
312 gradu(q, 1) = ((!ymax ? 0.0 : (p == q ? 1.0 : 0.0)) - (!ymin ? 0.0 : (p == q ? 1.0 : 0.0))) / ((ymin || ymax ? 1.0 : 2.0) * DX[1]);,
313 gradu(q, 2) = ((!zmax ? 0.0 : (p == q ? 1.0 : 0.0)) - (!zmin ? 0.0 : (p == q ? 1.0 : 0.0))) / ((zmin || zmax ? 1.0 : 2.0) * DX[2]););
316 gradgradu(q, 0, 0) = (p == q ? -2.0 : 0.0) / DX[0] / DX[0];
318 gradgradu(q, 0, 1) = 0.0;
319 gradgradu(q, 1, 0) = 0.0;
320 gradgradu(q, 1, 1) = (p == q ? -2.0 : 0.0) / DX[1] / DX[1];
322 gradgradu(q, 0, 2) = 0.0;
323 gradgradu(q, 1, 2) = 0.0;
324 gradgradu(q, 2, 0) = 0.0;
325 gradgradu(q, 2, 1) = 0.0;
326 gradgradu(q, 2, 2) = (p == q ? -2.0 : 0.0) / DX[2] / DX[2]);
331 if (AMREX_D_TERM(xmax || xmin, || ymax || ymin, || zmax || zmin))
336 f = (*m_bc)(u,
gradu, sig, i, j, k, domain);
337 diag(i, j, k, p) = f(p);
341 Set::Vector f = (DDW(i, j, k) * gradgradu) * psi_avg;
342 diag(i, j, k, p) += f(p);
346 if (std::isnan(diag(i, j, k, p)))
Util::Abort(
INFO,
"diagonal is nan at (", i,
",", j,
",", k,
"), amrlev=", amrlev,
", mglev=", mglev);
347 if (std::isinf(diag(i, j, k, p)))
Util::Abort(
INFO,
"diagonal is inf at (", i,
",", j,
",", k,
"), amrlev=", amrlev,
", mglev=", mglev);
348 if (diag(i, j, k, p) == 0)
Util::Abort(
INFO,
"diagonal is zero at (", i,
",", j,
",", k,
"), amrlev=", amrlev,
", mglev=", mglev);
361 BL_PROFILE(
"Operator::Elastic::Error0x()");
364 int ncomp = x.nComp();
365 int nghost = x.nGrow();
367 if (!m_diagonal_computed)
368 Util::Abort(
INFO,
"Operator::Diagonal() must be called before using normalize");
370 amrex::MultiFab D0x(x.boxArray(), x.DistributionMap(), ncomp, nghost);
371 amrex::MultiFab AD0x(x.boxArray(), x.DistributionMap(), ncomp, nghost);
373 amrex::MultiFab::Copy(D0x, x, 0, 0, ncomp, nghost);
374 amrex::MultiFab::Divide(D0x, *m_diag[amrlev][mglev], 0, 0, ncomp, 0);
375 amrex::MultiFab::Copy(AD0x, D0x, 0, 0, ncomp, nghost);
377 Fapply(amrlev, mglev, AD0x, D0x);
379 amrex::MultiFab::Copy(R0x, x, 0, 0, ncomp, nghost);
380 amrex::MultiFab::Subtract(R0x, AD0x, 0, 0, ncomp, nghost);
387 const std::array<FArrayBox*, AMREX_SPACEDIM>& sigmafab,
388 const FArrayBox& ,
const int )
const
390 BL_PROFILE(
"Operator::Elastic::FFlux()");
392 amrex::BaseFab<amrex::Real>
AMREX_D_DECL(&fxfab = *sigmafab[0],
393 &fyfab = *sigmafab[1],
394 &fzfab = *sigmafab[2]);
395 AMREX_D_TERM(fxfab.setVal(0.0);,
404 amrex::MultiFab& a_eps,
405 const amrex::MultiFab& a_u,
408 BL_PROFILE(
"Operator::Elastic::Strain()");
410 const amrex::Real* DX = m_geom[amrlev][0].CellSize();
411 amrex::Box domain(m_geom[amrlev][0].Domain());
412 domain.convert(amrex::IntVect::TheNodeVector());
415 for (MFIter mfi(a_u, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi)
417 const Box& bx = mfi.tilebox();
418 amrex::Array4<amrex::Real>
const&
epsilon = a_eps.array(mfi);
419 amrex::Array4<const amrex::Real>
const& u = a_u.array(mfi);
420 amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(
int i,
int j,
int k)
424 std::array<Numeric::StencilType, AMREX_SPACEDIM> sten
428 for (
int p = 0; p < AMREX_SPACEDIM; p++)
430 AMREX_D_TERM(
gradu(p, 0) = (
Numeric::Stencil<Set::Scalar, 1, 0, 0>::D(u, i, j, k, p, DX, sten));,
431 gradu(p, 1) = (
Numeric::Stencil<Set::Scalar, 0, 1, 0>::D(u, i, j, k, p, DX, sten));,
432 gradu(p, 2) = (
Numeric::Stencil<Set::Scalar, 0, 0, 1>::D(u, i, j, k, p, DX, sten)););
439 AMREX_D_PICK(
epsilon(i, j, k, 0) = eps(0, 0);
441 epsilon(i, j, k, 0) = eps(0, 0);
epsilon(i, j, k, 1) = eps(1, 1);
epsilon(i, j, k, 2) = eps(0, 1);
443 epsilon(i, j, k, 0) = eps(0, 0);
epsilon(i, j, k, 1) = eps(1, 1);
epsilon(i, j, k, 2) = eps(2, 2);
444 epsilon(i, j, k, 3) = eps(1, 2);
epsilon(i, j, k, 4) = eps(2, 0);
epsilon(i, j, k, 5) = eps(0, 1););
448 AMREX_D_PICK(
epsilon(i, j, k, 0) = eps(0, 0);
453 epsilon(i, j, k, 0) = eps(0, 0);
epsilon(i, j, k, 1) = eps(0, 1);
epsilon(i, j, k, 2) = eps(0, 2);
454 epsilon(i, j, k, 3) = eps(1, 0);
epsilon(i, j, k, 4) = eps(1, 1);
epsilon(i, j, k, 5) = eps(1, 2);
455 epsilon(i, j, k, 6) = eps(2, 0);
epsilon(i, j, k, 7) = eps(2, 1);
epsilon(i, j, k, 8) = eps(2, 2););
465 amrex::MultiFab& a_sigma,
466 const amrex::MultiFab& a_u,
467 bool voigt,
bool a_homogeneous)
469 BL_PROFILE(
"Operator::Elastic::Stress()");
470 SetHomogeneous(a_homogeneous);
472 const amrex::Real* DX = m_geom[amrlev][0].CellSize();
473 amrex::Box domain(m_geom[amrlev][0].Domain());
474 domain.convert(amrex::IntVect::TheNodeVector());
476 for (MFIter mfi(a_u, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi)
478 const Box& bx = mfi.tilebox();
479 amrex::Array4<Set::Matrix4<AMREX_SPACEDIM, SYM>>
const& DDW = (*(m_ddw_mf[amrlev][0])).array(mfi);
480 amrex::Array4<amrex::Real>
const& sigma = a_sigma.array(mfi);
481 amrex::Array4<Set::Scalar>
const& psi = m_psi_mf[amrlev][0]->array(mfi);
482 amrex::Array4<const amrex::Real>
const& u = a_u.array(mfi);
483 amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(
int i,
int j,
int k)
487 std::array<Numeric::StencilType, AMREX_SPACEDIM> sten
491 for (
int p = 0; p < AMREX_SPACEDIM; p++)
493 AMREX_D_TERM(
gradu(p, 0) = (
Numeric::Stencil<Set::Scalar, 1, 0, 0>::D(u, i, j, k, p, DX, sten));,
494 gradu(p, 1) = (
Numeric::Stencil<Set::Scalar, 0, 1, 0>::D(u, i, j, k, p, DX, sten));,
495 gradu(p, 2) = (
Numeric::Stencil<Set::Scalar, 0, 0, 1>::D(u, i, j, k, p, DX, sten)););
504 AMREX_D_PICK(sigma(i, j, k, 0) = sig(0, 0);
506 sigma(i, j, k, 0) = sig(0, 0); sigma(i, j, k, 1) = sig(1, 1); sigma(i, j, k, 2) = sig(0, 1);
508 sigma(i, j, k, 0) = sig(0, 0); sigma(i, j, k, 1) = sig(1, 1); sigma(i, j, k, 2) = sig(2, 2);
509 sigma(i, j, k, 3) = sig(1, 2); sigma(i, j, k, 4) = sig(2, 0); sigma(i, j, k, 5) = sig(0, 1););
513 AMREX_D_PICK(sigma(i, j, k, 0) = sig(0, 0);
515 sigma(i, j, k, 0) = sig(0, 0); sigma(i, j, k, 1) = sig(0, 1);
516 sigma(i, j, k, 2) = sig(1, 0); sigma(i, j, k, 3) = sig(1, 1);
518 sigma(i, j, k, 0) = sig(0, 0); sigma(i, j, k, 1) = sig(0, 1); sigma(i, j, k, 2) = sig(0, 2);
519 sigma(i, j, k, 3) = sig(1, 0); sigma(i, j, k, 4) = sig(1, 1); sigma(i, j, k, 5) = sig(1, 2);
520 sigma(i, j, k, 6) = sig(2, 0); sigma(i, j, k, 7) = sig(2, 1); sigma(i, j, k, 8) = sig(2, 2););
530 amrex::MultiFab& a_energy,
531 const amrex::MultiFab& a_u,
bool a_homogeneous)
533 BL_PROFILE(
"Operator::Elastic::Energy()");
534 SetHomogeneous(a_homogeneous);
536 amrex::Box domain(m_geom[amrlev][0].Domain());
537 domain.convert(amrex::IntVect::TheNodeVector());
539 const amrex::Real* DX = m_geom[amrlev][0].CellSize();
541 for (MFIter mfi(a_u, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi)
543 const Box& bx = mfi.tilebox();
544 amrex::Array4<Set::Matrix4<AMREX_SPACEDIM, SYM>>
const& DDW = (*(m_ddw_mf[amrlev][0])).array(mfi);
545 amrex::Array4<amrex::Real>
const& energy = a_energy.array(mfi);
546 amrex::Array4<const amrex::Real>
const& u = a_u.array(mfi);
547 amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(
int i,
int j,
int k)
551 std::array<Numeric::StencilType, AMREX_SPACEDIM> sten
555 for (
int p = 0; p < AMREX_SPACEDIM; p++)
557 AMREX_D_TERM(
gradu(p, 0) = (
Numeric::Stencil<Set::Scalar, 1, 0, 0>::D(u, i, j, k, p, DX, sten));,
558 gradu(p, 1) = (
Numeric::Stencil<Set::Scalar, 0, 1, 0>::D(u, i, j, k, p, DX, sten));,
559 gradu(p, 2) = (
Numeric::Stencil<Set::Scalar, 0, 0, 1>::D(u, i, j, k, p, DX, sten)););
569 for (
int m = 0; m < AMREX_SPACEDIM; m++)
571 for (
int n = 0; n < AMREX_SPACEDIM; n++)
573 energy(i, j, k) += .5 * sig(m, n) * eps(m, n);
584 BL_PROFILE(
"Elastic::averageDownCoeffs()");
586 if (m_average_down_coeffs)
587 for (
int amrlev = m_num_amr_levels - 1; amrlev > 0; --amrlev)
588 averageDownCoeffsDifferentAmrLevels(amrlev);
590 averageDownCoeffsSameAmrLevel(0);
591 for (
int amrlev = 0; amrlev < m_num_amr_levels; ++amrlev)
593 for (
int mglev = 0; mglev < m_num_mg_levels[amrlev]; ++mglev)
595 if (m_ddw_mf[amrlev][mglev]) {
596 FillBoundaryCoeff(*m_ddw_mf[amrlev][mglev], m_geom[amrlev][mglev]);
597 FillBoundaryCoeff(*m_psi_mf[amrlev][mglev], m_geom[amrlev][mglev]);
607 BL_PROFILE(
"Operator::Elastic::averageDownCoeffsDifferentAmrLevels()");
610 const int crse_amrlev = fine_amrlev - 1;
613 MultiTab& crse_ddw = *m_ddw_mf[crse_amrlev][0];
614 MultiTab& fine_ddw = *m_ddw_mf[fine_amrlev][0];
616 amrex::Box cdomain(m_geom[crse_amrlev][0].Domain());
617 cdomain.convert(amrex::IntVect::TheNodeVector());
619 const Geometry& cgeom = m_geom[crse_amrlev][0];
621 const BoxArray& fba = fine_ddw.boxArray();
622 const DistributionMapping& fdm = fine_ddw.DistributionMap();
624 MultiTab fine_ddw_for_coarse(amrex::coarsen(fba, 2), fdm, ncomp, 2);
625 fine_ddw_for_coarse.ParallelCopy(crse_ddw, 0, 0, ncomp, 0, 0, cgeom.periodicity());
627 const int coarse_fine_node = 1;
628 const int fine_fine_node = 2;
630 amrex::iMultiFab nodemask(amrex::coarsen(fba, 2), fdm, 1, 2);
631 nodemask.ParallelCopy(*m_nd_fine_mask[crse_amrlev], 0, 0, 1, 0, 0, cgeom.periodicity());
633 amrex::iMultiFab cellmask(amrex::convert(amrex::coarsen(fba, 2), amrex::IntVect::TheCellVector()), fdm, 1, 2);
634 cellmask.ParallelCopy(*m_cc_fine_mask[crse_amrlev], 0, 0, 1, 1, 1, cgeom.periodicity());
636 for (MFIter mfi(fine_ddw_for_coarse,
false); mfi.isValid(); ++mfi)
638 const Box& bx = mfi.validbox();
640 amrex::Array4<const int>
const& nmask = nodemask.array(mfi);
643 amrex::Array4<MATRIX4>
const& cdata = fine_ddw_for_coarse.array(mfi);
644 amrex::Array4<const MATRIX4>
const& fdata = fine_ddw.array(mfi);
646 const Dim3 lo = amrex::lbound(cdomain), hi = amrex::ubound(cdomain);
648 for (
int n = 0; n < fine_ddw.nComp(); n++)
652 amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(
int I,
int J,
int K) {
653 int i = I * 2, j = J * 2, k = K * 2;
655 if (nmask(I, J, K) == fine_fine_node || nmask(I, J, K) == coarse_fine_node)
657 if ((I == lo.x || I == hi.x) &&
658 (J == lo.y || J == hi.y) &&
659 (K == lo.z || K == hi.z))
660 cdata(I, J, K, n) = fdata(i, j, k, n);
661 else if ((J == lo.y || J == hi.y) &&
662 (K == lo.z || K == hi.z))
663 cdata(I, J, K, n) = fdata(i - 1, j, k, n) * 0.25 + fdata(i, j, k, n) * 0.5 + fdata(i + 1, j, k, n) * 0.25;
664 else if ((K == lo.z || K == hi.z) &&
665 (I == lo.x || I == hi.x))
666 cdata(I, J, K, n) = fdata(i, j - 1, k, n) * 0.25 + fdata(i, j, k, n) * 0.5 + fdata(i, j + 1, k, n) * 0.25;
667 else if ((I == lo.x || I == hi.x) &&
668 (J == lo.y || J == hi.y))
669 cdata(I, J, K, n) = fdata(i, j, k - 1, n) * 0.25 + fdata(i, j, k, n) * 0.5 + fdata(i, j, k + 1, n) * 0.25;
670 else if (I == lo.x || I == hi.x)
672 (fdata(i, j - 1, k - 1, n) + fdata(i, j, k - 1, n) * 2.0 + fdata(i, j + 1, k - 1, n)
673 + fdata(i, j - 1, k, n) * 2.0 + fdata(i, j, k, n) * 4.0 + fdata(i, j + 1, k, n) * 2.0
674 + fdata(i, j - 1, k + 1, n) + fdata(i, j, k + 1, n) * 2.0 + fdata(i, j + 1, k + 1, n)) / 16.0;
675 else if (J == lo.y || J == hi.y)
677 (fdata(i - 1, j, k - 1, n) + fdata(i - 1, j, k, n) * 2.0 + fdata(i - 1, j, k + 1, n)
678 + fdata(i, j, k - 1, n) * 2.0 + fdata(i, j, k, n) * 4.0 + fdata(i, j, k + 1, n) * 2.0
679 + fdata(i + 1, j, k - 1, n) + fdata(i + 1, j, k, n) * 2.0 + fdata(i + 1, j, k + 1, n)) / 16.0;
680 else if (K == lo.z || K == hi.z)
682 (fdata(i - 1, j - 1, k, n) + fdata(i, j - 1, k, n) * 2.0 + fdata(i + 1, j - 1, k, n)
683 + fdata(i - 1, j, k, n) * 2.0 + fdata(i, j, k, n) * 4.0 + fdata(i + 1, j, k, n) * 2.0
684 + fdata(i - 1, j + 1, k, n) + fdata(i, j + 1, k, n) * 2.0 + fdata(i + 1, j + 1, k, n)) / 16.0;
687 (fdata(i - 1, j - 1, k - 1, n) + fdata(i - 1, j - 1, k + 1, n) + fdata(i - 1, j + 1, k - 1, n) + fdata(i - 1, j + 1, k + 1, n) +
688 fdata(i + 1, j - 1, k - 1, n) + fdata(i + 1, j - 1, k + 1, n) + fdata(i + 1, j + 1, k - 1, n) + fdata(i + 1, j + 1, k + 1, n)) / 64.0
690 (fdata(i, j - 1, k - 1, n) + fdata(i, j - 1, k + 1, n) + fdata(i, j + 1, k - 1, n) + fdata(i, j + 1, k + 1, n) +
691 fdata(i - 1, j, k - 1, n) + fdata(i + 1, j, k - 1, n) + fdata(i - 1, j, k + 1, n) + fdata(i + 1, j, k + 1, n) +
692 fdata(i - 1, j - 1, k, n) + fdata(i - 1, j + 1, k, n) + fdata(i + 1, j - 1, k, n) + fdata(i + 1, j + 1, k, n)) / 32.0
694 (fdata(i - 1, j, k, n) + fdata(i, j - 1, k, n) + fdata(i, j, k - 1, n) +
695 fdata(i + 1, j, k, n) + fdata(i, j + 1, k, n) + fdata(i, j, k + 1, n)) / 16.0
697 fdata(i, j, k, n) / 8.0;
700 if (cdata(i, j, k).contains_nan())
Util::Abort(
INFO,
"restricted model is nan at (", i,
",", j,
",", k,
"), fine_amrlev=", fine_amrlev);
710 crse_ddw.ParallelCopy(fine_ddw_for_coarse, 0, 0, ncomp, 0, 0, cgeom.periodicity());
722 BL_PROFILE(
"Elastic::averageDownCoeffsSameAmrLevel()");
724 for (
int mglev = 1; mglev < m_num_mg_levels[amrlev]; ++mglev)
726 amrex::Box cdomain(m_geom[amrlev][mglev].Domain());
727 cdomain.convert(amrex::IntVect::TheNodeVector());
728 amrex::Box fdomain(m_geom[amrlev][mglev - 1].Domain());
729 fdomain.convert(amrex::IntVect::TheNodeVector());
731 MultiTab& crse = *m_ddw_mf[amrlev][mglev];
732 MultiTab& fine = *m_ddw_mf[amrlev][mglev - 1];
734 amrex::BoxArray crseba = crse.boxArray();
735 amrex::BoxArray fineba = fine.boxArray();
737 BoxArray newba = crseba;
740 fine_on_crseba.define(newba, crse.DistributionMap(), 1, 4);
741 fine_on_crseba.ParallelCopy(fine, 0, 0, 1, 2, 4, m_geom[amrlev][mglev].periodicity());
743 for (MFIter mfi(crse, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi)
745 Box bx = mfi.tilebox();
748 amrex::Array4<const Set::Matrix4<AMREX_SPACEDIM, SYM>>
const& fdata = fine_on_crseba.array(mfi);
749 amrex::Array4<Set::Matrix4<AMREX_SPACEDIM, SYM>>
const& cdata = crse.array(mfi);
751 const Dim3 lo = amrex::lbound(cdomain), hi = amrex::ubound(cdomain);
755 amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(
int I,
int J,
int K) {
756 int i = 2 * I, j = 2 * J, k = 2 * K;
758 if ((I == lo.x || I == hi.x) &&
759 (J == lo.y || J == hi.y) &&
760 (K == lo.z || K == hi.z))
761 cdata(I, J, K) = fdata(i, j, k);
762 else if ((J == lo.y || J == hi.y) &&
763 (K == lo.z || K == hi.z))
764 cdata(I, J, K) = fdata(i - 1, j, k) * 0.25 + fdata(i, j, k) * 0.5 + fdata(i + 1, j, k) * 0.25;
765 else if ((K == lo.z || K == hi.z) &&
766 (I == lo.x || I == hi.x))
767 cdata(I, J, K) = fdata(i, j - 1, k) * 0.25 + fdata(i, j, k) * 0.5 + fdata(i, j + 1, k) * 0.25;
768 else if ((I == lo.x || I == hi.x) &&
769 (J == lo.y || J == hi.y))
770 cdata(I, J, K) = fdata(i, j, k - 1) * 0.25 + fdata(i, j, k) * 0.5 + fdata(i, j, k + 1) * 0.25;
771 else if (I == lo.x || I == hi.x)
773 (fdata(i, j - 1, k - 1) + fdata(i, j, k - 1) * 2.0 + fdata(i, j + 1, k - 1)
774 + fdata(i, j - 1, k) * 2.0 + fdata(i, j, k) * 4.0 + fdata(i, j + 1, k) * 2.0
775 + fdata(i, j - 1, k + 1) + fdata(i, j, k + 1) * 2.0 + fdata(i, j + 1, k + 1)) / 16.0;
776 else if (J == lo.y || J == hi.y)
778 (fdata(i - 1, j, k - 1) + fdata(i - 1, j, k) * 2.0 + fdata(i - 1, j, k + 1)
779 + fdata(i, j, k - 1) * 2.0 + fdata(i, j, k) * 4.0 + fdata(i, j, k + 1) * 2.0
780 + fdata(i + 1, j, k - 1) + fdata(i + 1, j, k) * 2.0 + fdata(i + 1, j, k + 1)) / 16.0;
781 else if (K == lo.z || K == hi.z)
783 (fdata(i - 1, j - 1, k) + fdata(i, j - 1, k) * 2.0 + fdata(i + 1, j - 1, k)
784 + fdata(i - 1, j, k) * 2.0 + fdata(i, j, k) * 4.0 + fdata(i + 1, j, k) * 2.0
785 + fdata(i - 1, j + 1, k) + fdata(i, j + 1, k) * 2.0 + fdata(i + 1, j + 1, k)) / 16.0;
788 (fdata(i - 1, j - 1, k - 1) + fdata(i - 1, j - 1, k + 1) + fdata(i - 1, j + 1, k - 1) + fdata(i - 1, j + 1, k + 1) +
789 fdata(i + 1, j - 1, k - 1) + fdata(i + 1, j - 1, k + 1) + fdata(i + 1, j + 1, k - 1) + fdata(i + 1, j + 1, k + 1)) / 64.0
791 (fdata(i, j - 1, k - 1) + fdata(i, j - 1, k + 1) + fdata(i, j + 1, k - 1) + fdata(i, j + 1, k + 1) +
792 fdata(i - 1, j, k - 1) + fdata(i + 1, j, k - 1) + fdata(i - 1, j, k + 1) + fdata(i + 1, j, k + 1) +
793 fdata(i - 1, j - 1, k) + fdata(i - 1, j + 1, k) + fdata(i + 1, j - 1, k) + fdata(i + 1, j + 1, k)) / 32.0
795 (fdata(i - 1, j, k) + fdata(i, j - 1, k) + fdata(i, j, k - 1) +
796 fdata(i + 1, j, k) + fdata(i, j + 1, k) + fdata(i, j, k + 1)) / 16.0
798 fdata(i, j, k) / 8.0;
801 if (cdata(I, J, K).contains_nan())
Util::Abort(
INFO,
"restricted model is nan at crse coordinates (I=", I,
",J=", J,
",K=", k,
"), amrlev=", amrlev,
" interpolating from mglev", mglev - 1,
" to ", mglev);
805 FillBoundaryCoeff(crse, m_geom[amrlev][mglev]);
808 if (!m_psi_set)
continue;
810 amrex::Box cdomain_cell(m_geom[amrlev][mglev].Domain());
811 amrex::Box fdomain_cell(m_geom[amrlev][mglev - 1].Domain());
812 MultiFab& crse_psi = *m_psi_mf[amrlev][mglev];
813 MultiFab& fine_psi = *m_psi_mf[amrlev][mglev - 1];
814 MultiFab fine_psi_on_crseba;
815 fine_psi_on_crseba.define(newba.convert(amrex::IntVect::TheCellVector()), crse_psi.DistributionMap(), 1, 1);
816 fine_psi_on_crseba.ParallelCopy(fine_psi, 0, 0, 1, 1, 1, m_geom[amrlev][mglev].periodicity());
818 for (MFIter mfi(crse_psi, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi)
820 Box bx = mfi.tilebox();
821 bx = bx & cdomain_cell;
823 amrex::Array4<const Set::Scalar>
const& fdata = fine_psi_on_crseba.array(mfi);
824 amrex::Array4<Set::Scalar>
const& cdata = crse_psi.array(mfi);
826 const Dim3 lo = amrex::lbound(cdomain), hi = amrex::ubound(cdomain);
830 amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(
int I,
int J,
int K) {
831 int i = 2 * I, j = 2 * J, k = 2 * K;
833 if ((I == lo.x || I == hi.x) &&
834 (J == lo.y || J == hi.y) &&
835 (K == lo.z || K == hi.z))
836 cdata(I, J, K) = fdata(i, j, k);
837 else if ((J == lo.y || J == hi.y) &&
838 (K == lo.z || K == hi.z))
839 cdata(I, J, K) = fdata(i - 1, j, k) * 0.25 + fdata(i, j, k) * 0.5 + fdata(i + 1, j, k) * 0.25;
840 else if ((K == lo.z || K == hi.z) &&
841 (I == lo.x || I == hi.x))
842 cdata(I, J, K) = fdata(i, j - 1, k) * 0.25 + fdata(i, j, k) * 0.5 + fdata(i, j + 1, k) * 0.25;
843 else if ((I == lo.x || I == hi.x) &&
844 (J == lo.y || J == hi.y))
845 cdata(I, J, K) = fdata(i, j, k - 1) * 0.25 + fdata(i, j, k) * 0.5 + fdata(i, j, k + 1) * 0.25;
846 else if (I == lo.x || I == hi.x)
848 (fdata(i, j - 1, k - 1) + fdata(i, j, k - 1) * 2.0 + fdata(i, j + 1, k - 1)
849 + fdata(i, j - 1, k) * 2.0 + fdata(i, j, k) * 4.0 + fdata(i, j + 1, k) * 2.0
850 + fdata(i, j - 1, k + 1) + fdata(i, j, k + 1) * 2.0 + fdata(i, j + 1, k + 1)) / 16.0;
851 else if (J == lo.y || J == hi.y)
853 (fdata(i - 1, j, k - 1) + fdata(i - 1, j, k) * 2.0 + fdata(i - 1, j, k + 1)
854 + fdata(i, j, k - 1) * 2.0 + fdata(i, j, k) * 4.0 + fdata(i, j, k + 1) * 2.0
855 + fdata(i + 1, j, k - 1) + fdata(i + 1, j, k) * 2.0 + fdata(i + 1, j, k + 1)) / 16.0;
856 else if (K == lo.z || K == hi.z)
858 (fdata(i - 1, j - 1, k) + fdata(i, j - 1, k) * 2.0 + fdata(i + 1, j - 1, k)
859 + fdata(i - 1, j, k) * 2.0 + fdata(i, j, k) * 4.0 + fdata(i + 1, j, k) * 2.0
860 + fdata(i - 1, j + 1, k) + fdata(i, j + 1, k) * 2.0 + fdata(i + 1, j + 1, k)) / 16.0;
863 (fdata(i - 1, j - 1, k - 1) + fdata(i - 1, j - 1, k + 1) + fdata(i - 1, j + 1, k - 1) + fdata(i - 1, j + 1, k + 1) +
864 fdata(i + 1, j - 1, k - 1) + fdata(i + 1, j - 1, k + 1) + fdata(i + 1, j + 1, k - 1) + fdata(i + 1, j + 1, k + 1)) / 64.0
866 (fdata(i, j - 1, k - 1) + fdata(i, j - 1, k + 1) + fdata(i, j + 1, k - 1) + fdata(i, j + 1, k + 1) +
867 fdata(i - 1, j, k - 1) + fdata(i + 1, j, k - 1) + fdata(i - 1, j, k + 1) + fdata(i + 1, j, k + 1) +
868 fdata(i - 1, j - 1, k) + fdata(i - 1, j + 1, k) + fdata(i + 1, j - 1, k) + fdata(i + 1, j + 1, k)) / 32.0
870 (fdata(i - 1, j, k) + fdata(i, j - 1, k) + fdata(i, j, k - 1) +
871 fdata(i + 1, j, k) + fdata(i, j + 1, k) + fdata(i, j, k + 1)) / 16.0
873 fdata(i, j, k) / 8.0;
876 FillBoundaryCoeff(crse_psi, m_geom[amrlev][mglev]);
885 BL_PROFILE(
"Elastic::FillBoundaryCoeff()");
886 for (
int i = 0; i < 2; i++)
889 mf.FillBoundary(geom.periodicity());
890 const int ncomp = mf.nComp();
893 MultiTab tmpmf(mf.boxArray(), mf.DistributionMap(), ncomp, ng1);
894 tmpmf.ParallelCopy(mf, 0, 0, ncomp, ng2, ng1, geom.periodicity());
895 mf.ParallelCopy(tmpmf, 0, 0, ncomp, ng1, ng2, geom.periodicity());
903 BL_PROFILE(
"Elastic::FillBoundaryCoeff()");
904 for (
int i = 0; i < 2; i++)
907 mf.FillBoundary(geom.periodicity());
908 const int ncomp = mf.nComp();
911 MultiFab tmpmf(mf.boxArray(), mf.DistributionMap(), ncomp, ng1);
912 tmpmf.ParallelCopy(mf, 0, 0, ncomp, ng2, ng1, geom.periodicity());
913 mf.ParallelCopy(tmpmf, 0, 0, ncomp, ng1, ng2, geom.periodicity());