Peano 4
Loading...
Searching...
No Matches
SpacetreeSet.cpph
Go to the documentation of this file.
1#include "Node.h"
2
5
8
9#ifdef UseSmartMPI
10#include "smartmpi.h"
11#endif
12
13template <class Container>
15 [[maybe_unused]] Container& stackContainer,
16 [[maybe_unused]] int master,
17 [[maybe_unused]] int worker
18) {}
19
20template <class Container>
22 Container& stackContainer,
23 int spacetreeId
24) {
25 assertion( Node::getInstance().getRank( spacetreeId ) == tarch::mpi::Rank::getInstance().getRank() );
26 logDebug( "deleteAllStacks()", "delete all stacks of " << spacetreeId );
27 stackContainer.clear(spacetreeId);
28}
29
30template <class Container>
32 Container& stackContainer, int master, int worker
33) {
34 logTraceInWith2Arguments( "streamDataFromSplittingTreeToNewTree(...)", master, worker );
35 const int sourceRank = Node::getInstance().getRank( master );
36 const int destinationRank = Node::getInstance().getRank( worker );
37 const int sourceStack = Node::getOutputStackNumberForVerticalDataExchange(worker);
38
39 if (
40 sourceRank==tarch::mpi::Rank::getInstance().getRank()
41 and
42 destinationRank==tarch::mpi::Rank::getInstance().getRank()
43 ) {
44 int destinationStack = peano4::grid::PeanoCurve::getOutputStackNumber( getInstance().getSpacetree(worker)._root );
45
46 if (
47 not stackContainer.holdsStack( master, sourceStack )
48 ) {
50 "streamDataFromSplittingTreeToNewTree()",
51 "source stack for target stack " << destinationStack << " is empty, so skip copying"
52 );
53 }
54 else if (
55 not stackContainer.getForPush(worker,destinationStack)->empty()
56 ) {
58 "streamDataFromSplittingTreeToNewTree()",
59 "target stack " << destinationStack << " of tree " << worker << " contains already " << stackContainer.getForPush(worker,destinationStack)->size() << " entries, so skip copying"
60 );
61 }
62 else {
64 "streamDataFromSplittingTreeToNewTree()",
65 "copy stack " << sourceStack << " from tree " << master << " into stack " << destinationStack << " from tree " << worker <<
66 " with a stack size of " << stackContainer.getForPush(master,sourceStack)->size() << ". Can be done directly as both stacks reside on same machine"
67 );
68 assertion4( stackContainer.getForPush(worker,destinationStack)->empty(), master, worker, sourceStack, destinationStack );
69 stackContainer.getForPush(worker,destinationStack)->clone( *stackContainer.getForPop(master,sourceStack) );
70 }
71
72 if ( stackContainer.holdsStack( master, sourceStack ) ) {
74 "streamDataFromSplittingTreeToNewTree()",
75 "clear stack " << sourceStack << " on tree " << master
76 );
77 stackContainer.getForPush(master,sourceStack)->clear();
78 }
79 }
80 else if ( sourceRank==tarch::mpi::Rank::getInstance().getRank() ) {
81 // send stuff to the new worker via MPI
82 #ifdef Parallel
84 const int messageSize =
85 stackContainer.holdsStack( master, sourceStack ) ?
86 stackContainer.getForPop(master,sourceStack)->size() : 0;
87
89 "streamDataFromSplittingTreeToNewTree()",
90 "send stack " << sourceStack << " from tree " << master << " on rank " << sourceRank << " through tag " << meta.first <<
91 " to tree " << worker << " on rank " << destinationRank << ". size=" << messageSize
92 );
93 tarch::mpi::IntegerMessage message( messageSize );
94 tarch::mpi::IntegerMessage::send(message, destinationRank, meta.first, meta.second);
95
96 if (messageSize>0) {
97 stackContainer.getForPush(master,sourceStack)->startSend(
99 destinationRank, meta.first, meta.second
100 );
101 }
102 #else
103 assertionMsg( false, "should never be entered" );
104 #endif
105 }
106 else if ( destinationRank==tarch::mpi::Rank::getInstance().getRank() ) {
107 #ifdef Parallel
108 int destinationStack = peano4::grid::PeanoCurve::getOutputStackNumber( getInstance().getSpacetree(worker)._root );
110
112 tarch::mpi::IntegerMessage::receiveAndPollDanglingMessages(message, sourceRank, metaInfo.first, metaInfo.second);
113 logDebug(
114 "streamDataFromSplittingTreeToNewTree()",
115 "receive " << message.getValue() << " entries from tree " << master << " on rank " << sourceRank << " (used tag " << metaInfo.first << ")"
116 );
117 if (message.getValue()>0) {
118 stackContainer.getForPush(worker,destinationStack)->startReceive(
120 sourceRank, metaInfo.first, metaInfo.second, message.getValue()
121 );
122 }
123 #else
124 assertionMsg( false, "should never be entered" );
125 #endif
126 }
127 logTraceOutWith2Arguments( "streamDataFromSplittingTreeToNewTree(...)", master, worker );
128}
129
130template <class Container>
132 [[maybe_unused]] Container& stackContainer,
133 [[maybe_unused]] int spacetreeId,
134 [[maybe_unused]] int parentId
135) {
136 logTraceInWith2Arguments( "exchangeAllVerticalDataExchangeStacks(...)", spacetreeId, parentId );
137 logTraceOutWith2Arguments( "exchangeAllVerticalDataExchangeStacks(...)", spacetreeId, parentId );
138}
139
140template <class Container>
142 [[maybe_unused]] Container& stackContainer,
143 [[maybe_unused]] int spacetreeId,
144 [[maybe_unused]] bool symmetricDataCardinality
145) {
146 logTraceInWith2Arguments( "exchangeAllHorizontalDataExchangeStacks(...)", spacetreeId, symmetricDataCardinality );
147 assertionMsg( symmetricDataCardinality, "haven't implemented the asymmetric case yet, but would be simple: Just need the integer messages as I do for the vertical data flow" );
148 // Trigger all send and receives required
149 // --------------------------------------
150 // We exploit all the symmetries
151 std::set< peano4::maps::StackKey > keys = stackContainer.getKeys();
152 for (auto& sourceStackKey: keys) {
153 if (
154 sourceStackKey.first==spacetreeId
155 and
157 and
158 not stackContainer.getForPop(sourceStackKey)->empty()
159 and
160 Node::getInstance().getRank( Node::getInstance().getTreeNumberTiedToExchangeStackNumber(sourceStackKey.second) )!=tarch::mpi::Rank::getInstance().getRank()
161 ) {
162 int targetId = Node::getInstance().getTreeNumberTiedToExchangeStackNumber(sourceStackKey.second);
163 int rank = Node::getInstance().getRank( targetId );
164 int count = stackContainer.getForPush(sourceStackKey)->size();
166
168 logDebug( "exchangeAllHorizontalDataExchangeStacks(...)", "send stack " << sourceStackKey.second << " of tree " << sourceStackKey.first << " to rank " << rank << " with tag " << sendMetaInfo.first << ": " << count << " element(s)");
169
170 stackContainer.getForPush(sourceStackKey)->startSend(
172 rank,sendMetaInfo.first,sendMetaInfo.second);
173
175
176 logDebug( "exchangeAllHorizontalDataExchangeStacks(...)", "in return, receive " << count << " element(s) from rank " << rank << " with tag " << receiveMetaInfo.first << " into stack " << inStack );
177 assertion(not Node::getInstance().isHorizontalDataExchangeOutputStackNumber(inStack));
178 // This one likely creates new entries in the stackContainer. At this
179 // point, we have already copied the set of keys, i.e. we are fine as
180 // we have a valid overview.
181 stackContainer.getForPush(spacetreeId,inStack)->startReceive(
183 rank,
184 receiveMetaInfo.first,
185 receiveMetaInfo.second,count);
186 }
187 }
188
189 // All local boundary stacks
190 // -------------------------
191 for (auto sourceStackKey: keys) {
192 if (
193 sourceStackKey.first==spacetreeId
194 and
196 and
197 not stackContainer.getForPop(sourceStackKey)->empty()
198 and
199 Node::getInstance().getRank( Node::getInstance().getTreeNumberTiedToExchangeStackNumber(sourceStackKey.second) )==tarch::mpi::Rank::getInstance().getRank()
200 ) {
201 const int targetId = Node::getInstance().getTreeNumberTiedToExchangeStackNumber(sourceStackKey.second);
202
203 const int targetStack = Node::getInstance().getInputStackNumberForHorizontalDataExchange(spacetreeId);
204 logDebug(
205 "exchangeAllHorizontalDataExchangeStacks(...)",
206 "map output stream " << sourceStackKey.second << " of tree " <<
207 spacetreeId << " onto input stream " << targetStack <<
208 " of tree " << targetId <<
209 ". Copy " << stackContainer.getForPush(sourceStackKey)->size() << " entries"
210 );
211
212 assertion4( stackContainer.getForPush(targetId,targetStack)->empty(), spacetreeId, targetId, sourceStackKey.second, targetStack );
213 stackContainer.getForPush(targetId,targetStack)->clone( *stackContainer.getForPop(sourceStackKey) );
214
215 #if PeanoDebug>0
216 [[maybe_unused]] const int comparisonStackForTarget = Node::getInstance().getOutputStackNumberForHorizontalDataExchange( spacetreeId );
217
219 stackContainer.getForPush(targetId,targetStack)->size() == stackContainer.getForPush(spacetreeId,comparisonStackForTarget)->size()
220 or
221 stackContainer.getForPush(spacetreeId,comparisonStackForTarget)->empty(),
222 stackContainer.getForPush(targetId,targetStack)->size(),
223 stackContainer.getForPush(spacetreeId,comparisonStackForTarget)->size(),
224 stackContainer.getForPush(targetId,targetStack)->toString(),
225 stackContainer.getForPush(spacetreeId,comparisonStackForTarget)->toString(),
226 targetStack, comparisonStackForTarget, spacetreeId,
227 "target stack is what I have already sent over"
228 );
229 #endif
230
231 stackContainer.getForPush(sourceStackKey)->clear();
232 }
233 }
234 logTraceOut( "exchangeAllHorizontalDataExchangeStacks(...)" );
235}
236
237template <class Container>
238void peano4::parallel::SpacetreeSet::finishAllOutstandingSendsAndReceives( Container& stackContainer, int spacetreeId ) {
239 logTraceInWith1Argument( "finishAllOutstandingSendsAndReceives(...)", spacetreeId );
240 // Finalise data exchange
241 // ----------------------
242 std::set< peano4::maps::StackKey > keys = stackContainer.getKeys();
243
246
247 bool allSendReceivesFinished = false;
248
249 #ifdef UseSmartMPI
250 tarch::timing::Watch dataExchangeTime("peano4::parallel::SpacetreeSet", "traverse", false);
251 #endif
252
253 while (not allSendReceivesFinished) {
254 allSendReceivesFinished = true;
255 for (auto& sourceStackKey: keys) {
256 if ( sourceStackKey.first==spacetreeId ) {
257 logDebug( "finishAllOutstandingSendsAndReceives(...)", "check stack no " << sourceStackKey.first << " x " << sourceStackKey.second );
258 allSendReceivesFinished &= stackContainer.getForPush(sourceStackKey)->tryToFinishSendOrReceive();
259
260 #ifdef UseSmartMPI
261 const int rank = stackContainer.getForPush(sourceStackKey)->sendingOrReceiving();
262 if ( rank>=0 ) {
263 dataExchangeTime.stop();
264 smartmpi::reportWaitTime(dataExchangeTime.getCPUTime(),rank);
265 }
266 #endif
267 }
268 }
269
271 "peano4::parallel::SpacetreeSet",
272 "finishAllOutstandingSendsAndReceives(...)", spacetreeId, -1, -1
273 );
274
275 /* for (auto& pp: keys) {
276 if ( pp.first==spacetreeId and not stackContainer.getForPush(pp)->tryToFinishSendOrReceive()) {
277 logWarning( "finishAllOutstandingSendsAndReceives(...)", "exchange with tree " << pp.second << " is still not complete (likely no corresponding send or receive posted): " << stackContainer.getForPush(pp)->toString() );
278 }
279 }*/
280
282 "peano4::parallel::SpacetreeSet",
283 "finishAllOutstandingSendsAndReceives(...)", spacetreeId, -1, -1
284 );
285
287 }
288
289 logDebug( "finishAllOutstandingSendsAndReceives(...)", "all data transfer is done, trigger garbage collection" );
290
291 stackContainer.garbageCollection(spacetreeId);
292 logTraceOutWith1Argument( "finishAllOutstandingSendsAndReceives(...)", spacetreeId );
293}
294
295template <class Container>
296void peano4::parallel::SpacetreeSet::exchangeAllPeriodicBoundaryDataStacks( Container& stackContainer, int spacetreeId ) {
297 std::set< peano4::maps::StackKey > keys = stackContainer.getKeys();
298 logTraceInWith2Arguments( "exchangeAllPeriodicBoundaryDataStacks(...)",keys.size(),spacetreeId);
299 for (auto& sourceStackKey: keys) {
300 if (
301 sourceStackKey.first==spacetreeId
302 and
304 and
305 not stackContainer.getForPush(sourceStackKey)->empty()
306 ) {
307 const int targetStack = Node::mapPeriodicBoundaryExchangeOutputStackOntoInputStack(sourceStackKey.second);
308 logDebug(
309 "exchangeAllPeriodicBoundaryDataStacks(...)",
310 "map output stream " << sourceStackKey.second << " onto input stream " << targetStack <<
311 " to realise periodic boundary conditions. Copy/clone " << stackContainer.getForPush(sourceStackKey)->size() << " entries"
312 );
313
314 assertion4( stackContainer.getForPush(sourceStackKey.first,targetStack)->empty(), sourceStackKey.first, sourceStackKey.second, targetStack, stackContainer.getForPush(sourceStackKey.first,targetStack)->size() );
315 stackContainer.getForPush(sourceStackKey.first,targetStack)->clone( *stackContainer.getForPush(sourceStackKey) );
316 stackContainer.getForPush(sourceStackKey)->clear();
317 }
318 }
319 logTraceOut( "exchangeAllPeriodicBoundaryDataStacks(...)");
320}
#define assertion4(expr, param0, param1, param2, param3)
#define assertion8(expr, param0, param1, param2, param3, param4, param5, param6, param7)
#define assertionMsg(expr, message)
#define assertion(expr)
#define logDebug(methodName, logMacroMessageStream)
Definition Log.h:50
#define logTraceOutWith1Argument(methodName, argument0)
Definition Log.h:380
#define logTraceOut(methodName)
Definition Log.h:379
#define logTraceOutWith2Arguments(methodName, argument0, argument1)
Definition Log.h:381
#define logTraceInWith1Argument(methodName, argument0)
Definition Log.h:370
#define logTraceInWith2Arguments(methodName, argument0, argument1)
Definition Log.h:371
static int getOutputStackNumber(const AutomatonState &state)
static int getTreeNumberTiedToExchangeStackNumber(int number)
Gives you back the id of a communication partner, i.e.
Definition Node.cpp:413
static bool isPeriodicBoundaryExchangeOutputStackNumber(int number)
Definition Node.cpp:377
static int getOutputStackNumberForHorizontalDataExchange(int id)
Hand in a spacetree id and get back the number that we should use to send something to this tree.
Definition Node.cpp:211
static int mapPeriodicBoundaryExchangeOutputStackOntoInputStack(int outputStack)
Definition Node.cpp:203
int getRank(int treeId) const
You hand in a tree number and the node tells you on which rank such a tree is hosted.
Definition Node.cpp:119
static int getInputStackNumberForHorizontalDataExchange(int id)
Counterpart of getOutputStackNumberOfBoundaryExchange(int)
Definition Node.cpp:216
@ VerticalData
Vertical data is data running from the master to the worker and the other way round.
static bool isHorizontalDataExchangeOutputStackNumber(int number)
See getOutputStackNumberOfBoundaryExchange().
Definition Node.cpp:385
std::pair< int, MPI_Comm > GridDataExchangeMetaInformation
Definition Node.h:382
static int getOutputStackNumberForVerticalDataExchange(int id)
Definition Node.cpp:221
GridDataExchangeMetaInformation getGridDataExchangeMetaInformation(int sendingTreeId, int receivingTreeId, ExchangeMode exchange) const
I use two tags per spacetree per rank: one for boundary data (horizontal) and one for up-down and syn...
Definition Node.cpp:476
static Node & getInstance()
This operation returns the singleton instance.
Definition Node.cpp:108
static void streamDataFromSplittingTreeToNewTree(Container &stackContainer, int master, int worker)
Copies (streams) data from the master to the worker.
static void exchangeAllPeriodicBoundaryDataStacks(Container &stackContainer, int spacetreeId)
Exchange periodic BC data.
static void deleteAllStacks(Container &stackContainer, int spacetreeId)
static void exchangeAllVerticalDataExchangeStacks(Container &stackContainer, int spacetreeId, int parentId)
static void finishAllOutstandingSendsAndReceives(Container &stackContainer, int spacetreeId)
This routine finishes all the sends and receives that are still active, i.e.
static void exchangeAllHorizontalDataExchangeStacks(Container &stackContainer, int spacetreeId, bool symmetricDataCardinality)
Realise domain boundary exchange (over multiple scales)
static void streamDataFromJoiningTreeToMasterTree(Container &stackContainer, int master, int worker)
void triggerDeadlockTimeOut(const std::string &className, const std::string &methodName, int communicationPartnerRank, int tag, int numberOfExpectedMessages=1, const std::string &comment="")
Triggers a time out and shuts down the cluster if a timeout is violated.
Definition Rank.cpp:119
void setDeadlockWarningTimeStamp()
Memorise global timeout.
Definition Rank.cpp:188
void writeTimeOutWarning(const std::string &className, const std::string &methodName, int communicationPartnerRank, int tag, int numberOfExpectedMessages=1)
Writes a warning if relevant.
Definition Rank.cpp:143
void setDeadlockTimeOutTimeStamp()
Definition Rank.cpp:193
static Rank & getInstance()
This operation returns the singleton instance.
Definition Rank.cpp:538
int getRank() const
Return rank of this node.
Definition Rank.cpp:528
virtual void receiveDanglingMessages() override
Answer to MPI Messages.
static ServiceRepository & getInstance()
A simple class that has to be included to measure the clock ticks required for an operation.
Definition Watch.h:45
double getCPUTime()
Return CPU Time in Seconds.
Definition Watch.cpp:64
void stop()
Stop timer.
Definition Watch.cpp:55
static void receiveAndPollDanglingMessages(tarch::mpi::IntegerMessage &message, int source, int tag, MPI_Comm communicator=tarch::mpi::Rank::getInstance().getCommunicator())
static void send(const tarch::mpi::IntegerMessage &buffer, int destination, int tag, MPI_Comm communicator)
In DaStGen (the first version), I had a non-static version of the send as well as the receive.