My Project
Loading...
Searching...
No Matches
ParentToChildCellToPointGlobalIdHandle.hpp
1//===========================================================================
2//
3// File: ParentToChildCellToPointGlobalIdHandle.hpp
4//
5// Created: November 19 2024
6//
7// Author(s): Antonella Ritorto <antonella.ritorto@opm-op.com>
8// Markus Blatt <markus.blatt@opm-op.com>
9//
10// $Date$
11//
12// $Revision$
13//
14//===========================================================================
15
16/*
17 Copyright 2024 Equinor ASA
18 This file is part of The Open Porous Media project (OPM).
19 OPM is free software: you can redistribute it and/or modify
20 it under the terms of the GNU General Public License as published by
21 the Free Software Foundation, either version 3 of the License, or
22 (at your option) any later version.
23 OPM is distributed in the hope that it will be useful,
24 but WITHOUT ANY WARRANTY; without even the implied warranty of
25 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26 GNU General Public License for more details.
27 You should have received a copy of the GNU General Public License
28 along with OPM. If not, see <http://www.gnu.org/licenses/>.
29*/
30
31#ifndef OPM_PARENTTOCHILDCELLTOPOINTGLOBALIDHANDLE_HEADER
32#define OPM_PARENTTOCHILDCELLTOPOINTGLOBALIDHANDLE_HEADER
33
34#include <opm/grid/common/CommunicationUtils.hpp>
35#include <opm/grid/cpgrid/Entity.hpp>
36
37#include <array>
38#include <tuple>
39#include <vector>
40
41
42namespace
43{
44#if HAVE_MPI
45
47struct ParentToChildCellToPointGlobalIdHandle {
48 // - The container used for gather and scatter contains "candidates" point global ids for interior elements of the refined level grids (LGRs).
49 // Access is done with the local index of its parent cell index and its parent cell list of children.
50 // level_point_global_ids[ level-1 ][ child_cell_local_index [ corner ] ] = "candidate" point global id,
51 // when child_cell_local_index belongs to the children_local_index_list:
52 // parent_to_children_[ element.index() ] = parent_to_children_[ element.index() ] = { level, children_local_index_list }
53 // and corner = 0, ...,7.
54 // To decide which "candidate" point global id wins, we use the rank. The smallest ranks wins,
55 // i.e., the other non-selected candidates get rewritten with the values from the smallest (winner) rank.
56 // - In the scatter method, the "winner" rank and the 8 point global ids of each number of children) get rewritten.
57
58 using DataType = int;
59
67 ParentToChildCellToPointGlobalIdHandle(const Dune::CpGrid::Communication& comm,
68 const std::vector<std::tuple<int, std::vector<int>>>& parent_to_children,
69 const std::vector<std::vector<std::array<int,8>>>& level_cell_to_point,
70 std::vector<std::vector<DataType>>& level_winning_ranks,
71 std::vector<std::vector<DataType>>& level_point_global_ids)
72 : comm_(comm)
73 , parent_to_children_(parent_to_children)
74 , level_cell_to_point_(level_cell_to_point)
75 , level_winning_ranks_(level_winning_ranks)
76 , level_point_global_ids_(level_point_global_ids)
77 {}
78
79 bool fixedSize(std::size_t, std::size_t)
80 {
81 // Not every cell has children. When they have children, the amount might vary.
82 return false;
83 }
84
85 bool contains(std::size_t, std::size_t codim)
86 {
87 // Only communicate values attached to cells.
88 return codim == 0;
89 }
90
91 template <class T> // T = Entity<0>
92 std::size_t size(const T& element)
93 {
94 // Communicate variable size: 1 (rank) + (8* amount of child cells) from an interior parent cell from level zero grid.
95 // Skip values that are not interior, or have no children (in that case, 'invalid' level = -1)
96 const auto& [level, children] = parent_to_children_[element.index()];
97 // [Bug in dune-common] VariableSizeCommunicator will deadlock if a process attempts to send a message of size zero.
98 // This can happen if the size method returns zero for all entities that are shared with another process.
99 // Therefore, when skipping cells without children or for overlap cells, we set the size to 1.
100 if ( (element.partitionType() != Dune::InteriorEntity) || (level == -1))
101 return 1;
102 return 1 + ( 8*children.size()); // rank + 8 "winner" point global ids per child cell
103 }
104
105 // Gather global ids of child cells of a coarse interior parent cell
106 template <class B, class T> // T = Entity<0>
107 void gather(B& buffer, const T& element)
108 {
109 // Skip values that are not interior, or have no children (in that case, 'invalid level' = -1)
110 const auto& [level, children] = parent_to_children_[element.index()];
111 // [Bug in dune-common] VariableSizeCommunicator will deadlock if a process tries to send a message with size zero.
112 // To avoid this, for cells without children or for overlap cells, we set the size to 1 and write a single DataType
113 // value (e.g., '42').
114 if ( (element.partitionType() != Dune::InteriorEntity) || (level==-1)) {
115 buffer.write(42);
116 return;
117 }
118 // Store the children's corner global ids in the buffer when the element is interior and has children.
119 // Write the rank first, for example via the "corner 0" of cell_to_point_ of the first child:
120 // First child: children[0]
121 // First corner of first child: level_cell_to_point_[ level -1 ][children[0]] [0]
122 buffer.write( comm_.rank() );
123 for (const auto& child : children)
124 for (const auto& corner : level_cell_to_point_[level -1][child])
125 buffer.write(level_point_global_ids_[level-1][corner]);
126 }
127
128 // Scatter global ids of child cells of a coarse overlap parent cell
129 template <class B, class T> // T = Entity<0>
130 void scatter(B& buffer, const T& element, std::size_t size)
131 {
132 const auto& [level, children] = parent_to_children_[element.index()];
133 // Read all values to advance the pointer used by the buffer to the correct index.
134 // (Skip overlap-cells-without-children and interior-cells).
135 if ( ( (element.partitionType() == Dune::OverlapEntity) && (level==-1) ) || (element.partitionType() == Dune::InteriorEntity ) ) {
136 // Read all values to advance the pointer used by the buffer
137 // to the correct index
138 for (std::size_t received_int = 0; received_int < size; ++received_int) {
139 DataType tmp;
140 buffer.read(tmp);
141 }
142 }
143 else { // Overlap cell with children.
144 // Read and store the values in the correct location directly.
145 // The order of the children is the same on each process.
146 assert(children.size()>0);
147 assert(level>0);
148 // Read and store the values in the correct location directly.
149 DataType tmp_rank;
150 buffer.read(tmp_rank);
151 for (const auto& child : children) {
152 for (const auto& corner : level_cell_to_point_[level -1][child]) {
153 auto& min_rank = level_winning_ranks_[level-1][corner];
154 // Rewrite the rank (smaller rank wins)
155 if (tmp_rank < min_rank) {
156 min_rank = tmp_rank;
157 auto& target_entry = level_point_global_ids_[level-1][corner];
158 buffer.read(target_entry);
159 } else {
160 DataType rubbish;
161 buffer.read(rubbish);
162 }
163 }
164 }
165 }
166 }
167
168private:
169 const Dune::CpGrid::Communication& comm_;
170 const std::vector<std::tuple<int, std::vector<int>>>& parent_to_children_;
171 const std::vector<std::vector<std::array<int,8>>>& level_cell_to_point_;
172 std::vector<std::vector<DataType>>& level_winning_ranks_;
173 std::vector<std::vector<DataType>>& level_point_global_ids_;
174};
175#endif // HAVE_MPI
176} // namespace
177#endif