Peano 4
Loading...
Searching...
No Matches
STDVectorStackOverSmartPointers.cpph
Go to the documentation of this file.
2
3template <class T>
5 "STDVectorStackOverSmartPointers<T>");
6
7template <class T>
9 _data(),
10 _gatheredData(nullptr),
11 _gatheredMetaData(nullptr),
12 _gatheredDebugMetaData(nullptr),
13 _ioMode(IOMode::None),
14 _ioRank(-1),
15 _ioTag(-1)
16#ifdef Parallel
17 ,
18 _ioMPIRequest(nullptr),
19 _ioMetaDataMPIRequest(nullptr),
20 _ioDebugMetaDataMPIRequest(nullptr)
21#endif
22{
23}
24
25template <class T>
28):
29 _data(),
30 _gatheredData(nullptr),
31 _gatheredMetaData(nullptr),
32 _gatheredDebugMetaData(nullptr),
33 _ioMode(IOMode::None)
34#ifdef Parallel
35 ,
36 _ioMPIRequest(nullptr),
37 _ioMetaDataMPIRequest(nullptr),
38 _ioDebugMetaDataMPIRequest(nullptr)
39#endif
40{
41 assertionMsg(_data.empty(), "may not copy non-empty stack");
42 assertionMsg(copy._data.empty(), "may not copy non-empty stack");
43#ifdef Parallel
44 assertionMsg(_ioMPIRequest == nullptr, "may not copy sending/receiving stack");
45 assertionMsg(_ioMetaDataMPIRequest == nullptr, "may not copy sending/receiving stack");
46 assertionMsg(_ioDebugMetaDataMPIRequest == nullptr, "may not copy sending/receiving stack");
47 assertionMsg(copy._ioMPIRequest == nullptr, "may not copy sending/receiving stack");
48 assertionMsg(copy._ioMetaDataMPIRequest == nullptr, "may not copy sending/receiving stack");
49 assertionMsg(copy._ioDebugMetaDataMPIRequest == nullptr, "may not copy sending/receiving stack");
50#endif
51}
52
53template <class T>
55 assertion1(_data.empty(), toString());
56
57 _data.clear();
58 _data.resize(data._data.size(), T(T::ObjectConstruction::NoData));
59
60 logDebug("clone()", "copy over stack of size " << data.size());
61
62 for (int stackNumber = 0; stackNumber < size(); stackNumber++) {
63 _data[stackNumber].clone(data._data[stackNumber]);
64 }
65}
66
67template <class T>
69 prepareMetaDataBuffers();
70
71 assertion( _gatheredMetaData!=nullptr );
72 assertion( _gatheredDebugMetaData!=nullptr );
73 assertion( T::Cardinality>0 );
74
75 for (int i = 0; i < size(); i++) {
76 _gatheredMetaData[i] = _data[i].value==nullptr ? 0 : T::Cardinality;
77 }
78
79 logDebug( "gatherMetaData()", "gathered meta data which identifies nullptrs vs real data" );
80
81#if PeanoDebug >= 1
82 for (int i = 0; i < size(); i++) {
84 "gatherMetaData()",
85 i << "th entry: copy " << (size() * 2 * Dimensions)
86 << " doubles over into temporary local buffer. x=" << _data[i].getDebugX() << ", h=" << _data[i].getDebugH()
87 );
88 for (int d = 0; d < Dimensions; d++) {
89 _gatheredDebugMetaData[i * Dimensions * 2 + d] = _data[i].getDebugX()(d);
90 _gatheredDebugMetaData[i * Dimensions * 2 + d + Dimensions] = _data[i].getDebugH()(d);
91 }
92 }
93#endif
94}
95
96template <class T>
98 _gatheredMetaData = tarch::allocateMemory<int>(size(), tarch::MemoryLocation::Heap);
99#if PeanoDebug >= 1
100 _gatheredDebugMetaData = tarch::allocateMemory(Dimensions * 2 * size(), tarch::MemoryLocation::Heap);
101#endif
102 logDebug( "prepareMetaDataBuffers()", "prepared meta data buffer with " << size() << " entries and debug meta data buffer with " << (Dimensions*2*size()) << " entries (if debug info available)" );
103}
104
105template <class T>
107#if PeanoDebug >= 1
108 for (int i = 0; i < size(); i++) {
109#if Dimensions == 2
110 _data[i].setDebugX({_gatheredDebugMetaData[i * Dimensions * 2 + 0], _gatheredDebugMetaData[i * Dimensions * 2 + 1]});
111 _data[i].setDebugH({_gatheredDebugMetaData[i * Dimensions * 2 + 2], _gatheredDebugMetaData[i * Dimensions * 2 + 3]});
112#else
113 _data[i].setDebugX(
114 {_gatheredDebugMetaData[i * Dimensions * 2 + 0],
115 _gatheredDebugMetaData[i * Dimensions * 2 + 1],
116 _gatheredDebugMetaData[i * Dimensions * 2 + 2]}
117 );
118 _data[i].setDebugH(
119 {_gatheredDebugMetaData[i * Dimensions * 2 + 3],
120 _gatheredDebugMetaData[i * Dimensions * 2 + 4],
121 _gatheredDebugMetaData[i * Dimensions * 2 + 5]}
122 );
123#endif
124 logDebug(
125 "scatterDebugMetaData()",
126 "copied "
127 << (size() * 2 * Dimensions) << " doubles over into meta data. x=" << _data[i].getDebugX()
128 << ", h=" << _data[i].getDebugH()
129 );
130 }
131#endif
132}
133
134template <class T>
136 logDebug(
137 "scatter()",
138 "copy "
139 << size() << " data entries over into real smart pointer data structure. One entry hosts " << (T::Cardinality)
140 << " unknowns"
141 );
142 assertion( _gatheredMetaData!=nullptr );
143 for (int i = 0; i < size(); i++) {
144 if ( _gatheredMetaData[i]==0 ) {
145 assertion( _data[i].value==nullptr );
146 // Not required, as the object should already accept only such entries
147 _data[i] = T(T::ObjectConstruction::NoData);
148 }
149 else {
150 _data[i] = T();
151 std::copy_n(_gatheredData + T::Cardinality * i, T::Cardinality, _data[i].value);
152 }
153 }
154}
155
156template <class T>
158 prepareGatheredDataBuffer();
159
160 assertion(_gatheredMetaData!=nullptr);
161
162 for (int i = 0; i < size(); i++) {
163 if ( _gatheredMetaData[i]!=0 ) {
164 assertion( _data[i].value!=nullptr );
165 logDebug( "gather()", "gather data from " << _data[i].toString() );
166 std::copy_n(_data[i].value, T::Cardinality, _gatheredData + T::Cardinality * i);
167 }
168 }
169}
170
171template <class T>
173 logDebug(
174 "prepareGatheredDataBuffer()",
175 "allocate (temporary) receive of size "
176 << (size() * T::Cardinality) << ". " << size() << " local entries with " << T::Cardinality << " values per entry"
177 );
178 _gatheredData = tarch::allocateMemory<typename T::DoFType>(size() * T::Cardinality, tarch::MemoryLocation::Heap);
179}
180
181template <class T>
183 peano4::grid::TraversalObserver::SendReceiveContext context, int rank, int tag, MPI_Comm comm
184) {
185#ifdef Parallel
186 assertion(_ioMode == IOMode::None);
187 assertion(_ioMPIRequest == nullptr);
188
189 _ioMode = IOMode::MPISend;
190 _ioTag = tag;
191 _ioRank = rank;
192
193 gatherMetaData();
194 gather();
195
196 // backup data as clear() below might change this value
197 const int stackEntriesToSend = size();
198 clear();
199
200#if PeanoDebug >= 1
201 _ioDebugMetaDataMPIRequest = new MPI_Request();
202 MPI_Isend(
203 _gatheredDebugMetaData, stackEntriesToSend * 2 * Dimensions, MPI_DOUBLE, _ioRank, _ioTag, comm, _ioDebugMetaDataMPIRequest
204 );
205#endif
206
207 _ioMetaDataMPIRequest = new MPI_Request();
208 MPI_Isend(
209 _gatheredMetaData, stackEntriesToSend, MPI_INT, _ioRank, _ioTag, comm, _ioMetaDataMPIRequest
210 );
211
212 _ioMPIRequest = new MPI_Request;
213 int result = MPI_Isend(
214 _gatheredData, stackEntriesToSend, translateContextIntoDatatype<T>(context), _ioRank, _ioTag, comm, _ioMPIRequest
215 );
216 if (result != MPI_SUCCESS) {
217 logError(
218 "startSend(int,int,bool)",
219 "was not able to send to node " << rank << " on tag " << tag << ": " << tarch::mpi::MPIReturnValueToString(result)
220 );
221 }
222
223 logDebug("startSend(...)", "sent out real data with " << stackEntriesToSend << " entries");
224#endif
225}
226
227template <class T>
229 peano4::grid::TraversalObserver::SendReceiveContext context, int rank, int tag, MPI_Comm comm, int numberOfElements
230) {
231#ifdef Parallel
232 assertion3(_ioMode == IOMode::None, rank, tag, numberOfElements);
233 assertion3(numberOfElements > 0, rank, tag, numberOfElements);
234
235 _ioMode = IOMode::MPIReceive;
236 _ioTag = tag;
237 _ioRank = rank;
238
239 _data.clear();
240 _data.resize(numberOfElements, T(T::ObjectConstruction::NoData));
241
242 prepareMetaDataBuffers();
243 prepareGatheredDataBuffer();
244
245#if PeanoDebug >= 1
246 _ioDebugMetaDataMPIRequest = new MPI_Request();
247 MPI_Irecv(
248 _gatheredDebugMetaData, numberOfElements * 2 * Dimensions, MPI_DOUBLE, _ioRank, _ioTag, comm, _ioDebugMetaDataMPIRequest
249 );
250#endif
251
252 _ioMetaDataMPIRequest = new MPI_Request();
253 MPI_Irecv(
254 _gatheredMetaData, numberOfElements, MPI_INT, _ioRank, _ioTag, comm, _ioMetaDataMPIRequest
255 );
256
257 assertion(_ioMPIRequest == nullptr);
258 _ioMPIRequest = new MPI_Request;
259
260 int result = MPI_Irecv(
261 _gatheredData, numberOfElements, translateContextIntoDatatype<T>(context), _ioRank, _ioTag, comm, _ioMPIRequest
262 );
263 if (result != MPI_SUCCESS) {
264 logError(
265 "startReceive(int,int,int)",
266 "was not able to receive "
267 << numberOfElements << " values from node " << rank << " on tag " << tag << ": "
269 );
270 }
271
272 logDebug("startReceive()", toString());
273#endif
274}
275
276template <class T>
278#ifdef Parallel
279 logTraceInWith4Arguments("tryToFinishSendOrReceive()", ::toString(_ioMode), size(), _ioRank, _ioTag);
280 bool result = true;
281 if (_ioMode == IOMode::MPISend or _ioMode == IOMode::MPIReceive) {
282 assertion(_ioMPIRequest != nullptr);
283
284 int flag = 0;
285 MPI_Test(_ioMPIRequest, &flag, MPI_STATUS_IGNORE);
286 if (flag) {
287 result = true;
288 logDebug("tryToFinishSendOrReceive()", "send/receive complete, free MPI request: " << toString());
289
290 delete _ioMPIRequest;
291 _ioMPIRequest = nullptr;
292
293#if PeanoDebug >= 1
294 MPI_Wait(_ioDebugMetaDataMPIRequest, MPI_STATUS_IGNORE);
295 delete _ioDebugMetaDataMPIRequest;
296 _ioDebugMetaDataMPIRequest = nullptr;
297#endif
298
299 MPI_Wait(_ioMetaDataMPIRequest, MPI_STATUS_IGNORE);
300 delete _ioMetaDataMPIRequest;
301 _ioMetaDataMPIRequest = nullptr;
302
303 if (_ioMode == IOMode::MPIReceive) {
304 scatterDebugMetaData();
305 scatter();
306 }
307
310#if PeanoDebug >= 1
311 tarch::freeMemory(_gatheredDebugMetaData, tarch::MemoryLocation::Heap);
312#endif
313
314 _ioMode = IOMode::None;
315 } else {
316 result = false;
317 }
318 }
319 logTraceOutWith1Argument("tryToFinishSendOrReceive()", result);
320 return result;
321#else
322 return true;
323#endif
324}
325
326template <class T>
328 return _data.size();
329}
330
331template <class T>
333 return _data.empty();
334}
335
336template <class T>
340
341template <class T>
343 return _ioMode == IOMode::None ? -1 : _ioRank;
344}
345
346template <class T>
348 std::reverse(std::begin(_data), std::end(_data));
349}
350
351template <class T>
353 std::ostringstream msg;
354 msg << "(size=" << size() << ",io-mode=" << ::toString(_ioMode) << ",rank=" << _ioRank << ",tag=" << _ioTag << ")";
355 return msg.str();
356}
357
358template <class T>
360 assertion(not empty());
361 T result = std::move(_data.back());
362 _data.pop_back();
363 return result;
364}
365
366template <class T>
368 assertion(not empty());
369 return _data[_data.size() - 1 - shift];
370}
371
372template <class T>
374 assertion(not empty());
375 return _data[_data.size() - 1 - shift];
376}
377
378template <class T>
380 _data.push_back(element);
381}
382
383template <class T>
386 return PopBlockStackView(numberOfElements, this);
387}
388
389template <class T>
393
394 _data.resize(_data.size() + numberOfElements, T(T::ObjectConstruction::NoData));
395
396 return result;
397}
#define assertion3(expr, param0, param1, param2)
#define assertion1(expr, param)
#define assertionMsg(expr, message)
#define assertion(expr)
#define logError(methodName, logMacroMessageStream)
Wrapper macro around tarch::tarch::logging::Log to improve logging.
Definition Log.h:464
#define logDebug(methodName, logMacroMessageStream)
Definition Log.h:50
#define logTraceOutWith1Argument(methodName, argument0)
Definition Log.h:380
#define logTraceInWith4Arguments(methodName, argument0, argument1, argument2, argument3)
Definition Log.h:373
SendReceiveContext
There are three different scenarios when we merge data:
This stack hosts data of constant size which are stored on the heap and subject to smart pointers.
void startReceive(peano4::grid::TraversalObserver::SendReceiveContext context, int rank, int tag, MPI_Comm comm, int numberOfElements)
Counterpart of startSend()
void push(const T &element)
Pushes element to a stack.
PushBlockStackView pushBlock(int numberOfElements)
Push a block on the output stack.
bool tryToFinishSendOrReceive()
Semantics are alost an exact 1:1 copy of the superclass implementation.
PopBlockStackView popBlock(int numberOfElements)
This operation grabs numberOfElements from the input stack en block and returns a view to it.
void clone(const STDVectorStackOverSmartPointers< T > &data)
Clone data into the current object on which clone() is called.
void startSend(peano4::grid::TraversalObserver::SendReceiveContext context, int rank, int tag, MPI_Comm comm)
Start the actual send process.
int sendingOrReceiving() const
I need this one to find out whether I'm waiting for data.
T & top(int shift=0)
Get top element or shiftth top element.
void reverse()
Reversing a stream is something I need extremely rarely.
std::vector< T > _data
This is the attribute holding all the temporary stacks.
Log Device.
Definition Log.h:516
std::string toString(Filter filter)
Definition convert.cpp:170
std::string MPIReturnValueToString(int result)
Definition Rank.cpp:204
void freeMemory(void *data, MemoryLocation location)
@ Heap
Create data on the heap of the local device.
T * allocateMemory(int size, MemoryLocation location, int device=-1)
Definition accelerator.h:82