Peano 4
Loading...
Searching...
No Matches
Node.cpp
Go to the documentation of this file.
1#include "Node.h"
4
5
10
12
13#include <thread> // std::this_thread::sleep_for
14#include <chrono> // std::chrono::seconds
15#include <unordered_set>
16
17#include "tarch/Assertions.h"
18#include "tarch/mpi/Rank.h"
21
22
25
26
42
43
59
60
63
64
66 #if !defined(UseSmartMPI)
68 tarch::mpi::Rank::getInstance().getNumberOfRanks()==1
69 or
70 _currentProgramStep==Terminate,
71 "forgot to terminate node properly through peano4::parallel::Node::getInstance().shutdown()"
72 );
73 #endif
74}
75
76
78 _currentProgramStep = UndefProgramStep;
79 _rankOrchestrationTag = tarch::mpi::Rank::reserveFreeTag("peano4::parallel::Node - rank orchestration");
80 if (tarch::mpi::Rank::getInstance().isGlobalMaster()) {
81 registerId( 0, -1);
82 }
83
84 #ifdef Parallel
85 for (int i=0; i<MaxSpacetreesPerRank; i++) {
86 MPI_Comm_dup(tarch::mpi::Rank::getInstance().getCommunicator(), &(_dataExchangeCommunicators[i]));
87 }
88 #endif
89}
90
91
93 switch (mode) {
94 case ExchangeMode::HorizontalData:
95 return "horizontal-data";
96 case ExchangeMode::VerticalData:
97 return "vertical-data";
98 }
99 return "undef";
100}
101
102
104 return treeId==0;
105}
106
107
111
112
113int peano4::parallel::Node::getId(int rank, int localTreeId) const {
114 const int numberOfRanks = tarch::mpi::Rank::getInstance().getNumberOfRanks();
115 return numberOfRanks * localTreeId + rank;
116}
117
118
120 const int numberOfRanks = tarch::mpi::Rank::getInstance().getNumberOfRanks();
121 return id % numberOfRanks;
122}
123
124
126 const int numberOfRanks = tarch::mpi::Rank::getInstance().getNumberOfRanks();
127 return treeId / numberOfRanks;
128}
129
130
132 const int numberOfRanks = tarch::mpi::Rank::getInstance().getNumberOfRanks();
133 return treeId * numberOfRanks + tarch::mpi::Rank::getInstance().getRank();
134}
135
136
137int peano4::parallel::Node::reserveId(int rank, int forTreeId) {
138 int localThread = 0;
139 int result = -1;
140 while (result==-1 and localThread<MaxSpacetreesPerRank) {
141 if ( _treeEntries.count( getId(rank,localThread) )==0 ) {
142 result = getId(rank,localThread);
143 }
144 else {
145 logDebug( "reserveId(int,int)", "local tree " << localThread << " (global id=" << getId(rank,localThread) << ") on rank " << rank << " is already in use" );
146 }
147 localThread++;
148 }
149
150 if (localThread==MaxSpacetreesPerRank-1) {
151 logWarning( "reserveId(int,int)", "gave out " << (localThread+1) << " trees on rank " << tarch::mpi::Rank::getInstance().getRank() << ". Max trees per rank=" << MaxSpacetreesPerRank );
152 }
153
154 if (localThread<MaxSpacetreesPerRank) {
155 registerId( result, forTreeId );
156 }
157 else {
158 result = -1;
159 }
160 return result;
161}
162
163
164void peano4::parallel::Node::registerId(int id, int masterId) {
165 tarch::multicore::Lock lock(_semaphore);
166 assertion2( _treeEntries.count(id)==0, id, masterId );
167 assertion2( id!=masterId, id, masterId );
168 #ifndef Parallel
169 assertion( isGlobalMaster(id) or _treeEntries.count(masterId)==1 );
170 #endif
171
172 logTraceInWith2Arguments( "registerId(int,int)", id, masterId );
173 TreeEntry newEntry;
174
175 newEntry.setId( id );
176 newEntry.setMaster( masterId );
177
178 _treeEntries.insert( std::pair<int,TreeEntry>(id, newEntry) );
179 logTraceOut( "registerId(int,int)" );
180}
181
182
184 return _treeEntries.size();
185}
186
187
189 assertion1( _treeEntries.count(id)==1, id );
190
191 tarch::multicore::Lock lock(_semaphore);
192 _treeEntries.erase(id);
193
194 logDebug( "deregisterId(int)", "removed tree " << id << " from list of trees" );
195}
196
197
201
202
205 int outputStackMinusOtherStacks = outputStack - firstPeriodicBCStack;
206 int mirroredStack = peano4::grid::PeanoCurve::NumberOfPeriodicBoundaryConditionStacks - outputStackMinusOtherStacks - 1;
207 return mirroredStack + firstPeriodicBCStack;
208}
209
210
214
215
219
220
224
225
229
230
232 std::bitset<2*Dimensions> result;
233
234 for (int d=0; d<2*Dimensions; d++) {
235 bool allFlagsSet = true;
236 dfor2(k)
238 entry = k;
239 entry( d%Dimensions ) = d/Dimensions;
242 if (allFlagsSet) {
243 result.set(d);
244 }
245 }
246 return result;
247}
248
249
252 int normal = faceNumber % Dimensions;
253 bool lookRight = faceNumber < Dimensions;
254 neighbour(normal) += lookRight ? 1 : -1;
255 int linearisedIndex = peano4::utils::dLinearised(neighbour,3);
256 if (linearisedIndex>=ThreePowerD/2) linearisedIndex--;
258 return baseIndex + linearisedIndex;
259}
260
261
262std::string peano4::parallel::Node::toString( const std::set< PeriodicBoundaryStackIdentifier >& data ) {
263 std::ostringstream msg;
264
265 msg << "{";
266 bool firstEntry = true;
267 for (auto& p: data) {
268 if (firstEntry) {
269 firstEntry = false;
270 }
271 else {
272 msg << ",";
273 }
274 msg << "(stack=" << p.first << ",bnd=" << std::bitset<Dimensions>(p.second) << ")";
275 }
276 msg << "}";
277
278 return msg.str();
279}
280
281
282std::set<peano4::parallel::Node::PeriodicBoundaryStackIdentifier> peano4::parallel::Node::getOutputStacksForPeriodicBoundaryExchange(const tarch::la::Vector<TwoPowerD,int>& flags) {
283 std::bitset<2*Dimensions> boundaryNumbers = getPeriodicBoundaryNumber(flags);
284 logTraceInWith2Arguments( "getOutputStacksForPeriodicBoundaryExchange(...)", flags, boundaryNumbers );
285
286 std::unordered_set< std::bitset<2*Dimensions> > powerSet;
287 for (int i=0; i<2*Dimensions; i++) {
288 if ( boundaryNumbers[i] ) {
289 std::bitset<2*Dimensions> newEntry(0);
290 newEntry.set(i);
291 powerSet.insert( newEntry );
292
293 for (auto p: powerSet) {
294 newEntry = p;
295 newEntry.set(i);
296 powerSet.insert( newEntry );
297 }
298 }
299 }
300
301 std::set<peano4::parallel::Node::PeriodicBoundaryStackIdentifier> result;
302 int currentStackNumber = 0;
303 dfor3(neighbour)
304 if ( neighbour!=tarch::la::Vector<Dimensions,int>(1)) {
305 std::bitset<2*Dimensions> requiredEntryForThisDirection(0);
306 std::bitset<Dimensions> direction(0);
307 for (int d=0; d<Dimensions; d++) {
308 if (neighbour(d)>1) {
309 requiredEntryForThisDirection[d] = true;
310 direction[d] = true;
311 }
312 if (neighbour(d)<1) {
313 requiredEntryForThisDirection[d+Dimensions] = true;
314 direction[d] = true;
315 }
316 }
317 if (powerSet.count(requiredEntryForThisDirection)>0) {
320 direction.to_ulong()
321 );
323 isPeriodicBoundaryExchangeOutputStackNumber(newEntry.first),
324 newEntry.first, newEntry.second, flags,
326 );
327 result.insert( newEntry );
328 }
329 currentStackNumber++;
330 }
332
333
334/*
335 for (auto p: powerSet) {
336 std::bitset<Dimensions> stackAnnotation(0);
337 for (int i=0; i<Dimensions; i++) {
338 stackAnnotation.set( i, std::bitset<2*Dimensions>(p)[i] or std::bitset<2*Dimensions>(p)[i+Dimensions] );
339 }
340 peano4::parallel::Node::PeriodicBoundaryStackIdentifier newEntry(getOutputStackForPeriodicBoundaryExchange(p),stackAnnotation.to_ulong());
341 assertion5(
342 isPeriodicBoundaryExchangeOutputStackNumber(newEntry.first),
343 newEntry.first, newEntry.second, flags,
344 peano4::grid::PeanoCurve::MaxNumberOfCoreStacksPerSpacetreeInstance, peano4::grid::PeanoCurve::NumberOfPeriodicBoundaryConditionStacks
345 );
346 result.insert( newEntry );
347 }
348*/
349
350 logTraceOutWith2Arguments( "getOutputStacksForPeriodicBoundaryExchange(...)", result.size(), (result.size()==1 ? std::to_string(result.begin()->first) : "" ) );
351 return result;
352}
353
354
357 assertion( isPeriodicBoundaryExchangeOutputStackNumber(outputStackNumber) );
358
360 logTraceOutWith1Argument( "getPeriodicBoundaryExchangeInputStackNumberForOutputStack(int)", result );
361 return result;
362}
363
364
367 assertion( isPeriodicBoundaryExchangeOutputStackNumber(outputStackIdentifier.first) );
368
370 result.first = outputStackIdentifier.first + peano4::grid::PeanoCurve::NumberOfPeriodicBoundaryConditionStacks/2;
371 result.second = outputStackIdentifier.second;
372 logTraceOutWith2Arguments( "getPeriodicBoundaryExchangeInputStackNumberForOutputStack(int)", result.first, result.second );
373 return result;
374}
375
376
383
384
387 and ( (id-peano4::grid::PeanoCurve::MaxNumberOfCoreStacksPerSpacetreeInstance) % StacksPerCommunicationPartner == 0 );
388
389 return domainBoundary;
390}
391
392
395 and ( (id-peano4::grid::PeanoCurve::MaxNumberOfCoreStacksPerSpacetreeInstance) % StacksPerCommunicationPartner == 1 );
396
397 return domainBoundary;
398}
399
400
405
406
411
412
416
417
419 #ifdef Parallel
420 logTraceIn( "continueToRun()" );
421 if (tarch::mpi::Rank::getInstance().isGlobalMaster()) {
422 for (int i=1; i<tarch::mpi::Rank::getInstance().getNumberOfRanks(); i++ ) {
423 StartTraversalMessage message;
424 message.setStepIdentifier(_currentProgramStep);
425 logDebug( "continueToRun()", "send out " << message.toString() << " to rank " << i);
426 StartTraversalMessage::sendAndPollDanglingMessages(message, i,_rankOrchestrationTag);
427 }
428 }
429 else {
430 StartTraversalMessage message;
432 logDebug( "continueToRun()", "received message " << message.toString() );
433 _currentProgramStep = message.getStepIdentifier();
434 }
435 logTraceOutWith1Argument( "continueToRun()", _currentProgramStep );
436 #endif
437 return _currentProgramStep!=Terminate;
438}
439
440
442 assertion1( number==Terminate or number>=0, number);
443 _currentProgramStep = number;
444}
445
446
448 return _currentProgramStep;
449}
450
451
453 logTraceIn( "shutdown()" );
454 if (tarch::mpi::Rank::getInstance().isGlobalMaster()) {
455 setNextProgramStep(peano4::parallel::Node::Terminate);
456 continueToRun();
457 }
458
460 [&]() -> void {
462 }
463 );
464
465 #ifdef Parallel
466 for (int i=0; i<MaxSpacetreesPerRank; i++) {
467 MPI_Comm_free(&(_dataExchangeCommunicators[i]));
468 }
469 #endif
470
471 logTraceOut( "shutdown()" );
472}
473
474
475
477 logTraceInWith3Arguments( "getGridDataExchangeTag(int,int,ExchangeMode)", sendingTreeId, receivingTreeId, toString(exchange) );
478
479 assertion(sendingTreeId>=0);
480 assertion(sendingTreeId<MaxSpacetreesPerRank);
481
482 int tag = StacksPerCommunicationPartner/2 * getLocalTreeId(receivingTreeId);
483 switch (exchange) {
484 case ExchangeMode::HorizontalData:
485 tag += 0;
486 break;
487 case ExchangeMode::VerticalData:
488 tag += 1;
489 break;
490 }
491
492 logTraceOutWith1Argument( "getGridDataExchangeTag(int,int,ExchangeMode)", tag );
493 return peano4::parallel::Node::GridDataExchangeMetaInformation(tag,_dataExchangeCommunicators[getLocalTreeId(sendingTreeId)]);
494}
495
496
498 if (tag==getInstance()._rankOrchestrationTag) {
499 return "rank orchestration tag";
500 }
501
502 assertion(tag>=0);
503
504 std::string result = "data exchange tag for ";
505
506 switch ( tag % (StacksPerCommunicationPartner/2)) {
507 case 0: result += "horizontal data"; break;
508 case 1: result += "vertical data"; break;
509 }
510
511 return result;
512}
513
#define assertion2(expr, param0, param1)
#define assertion1(expr, param)
#define assertionMsg(expr, message)
#define assertion(expr)
#define assertion5(expr, param0, param1, param2, param3, param4)
we integrate over each cell and then take the sum across each of the cells We also consider the terms that enter the f$ k
#define ThreePowerD
Definition Globals.h:24
#define logDebug(methodName, logMacroMessageStream)
Definition Log.h:50
#define logTraceOutWith1Argument(methodName, argument0)
Definition Log.h:380
#define logTraceOut(methodName)
Definition Log.h:379
#define logWarning(methodName, logMacroMessageStream)
Wrapper macro around tarch::tarch::logging::Log to improve logging.
Definition Log.h:440
#define logTraceInWith4Arguments(methodName, argument0, argument1, argument2, argument3)
Definition Log.h:373
#define logTraceInWith5Arguments(methodName, argument0, argument1, argument2, argument3, argument4)
Definition Log.h:374
#define logTraceInWith3Arguments(methodName, argument0, argument1, argument2)
Definition Log.h:372
#define logTraceOutWith2Arguments(methodName, argument0, argument1)
Definition Log.h:381
#define logTraceIn(methodName)
Definition Log.h:369
#define logTraceInWith2Arguments(methodName, argument0, argument1)
Definition Log.h:371
#define dfor2(counter)
Shortcut For dfor(counter,2)
Definition Loop.h:906
#define dfor3(counter)
Shortcut For dfor(counter,3)
Definition Loop.h:642
#define enddforx
I prefer to use this macro for dforx instead of a closing bracket as many syntax parser fail otherwis...
Definition Loop.h:928
static constexpr int MaxNumberOfCoreStacksPerSpacetreeInstance
Standard (serial) number of stacks required per spacetree.
Definition PeanoCurve.h:86
static constexpr int NumberOfPeriodicBoundaryConditionStacks
In principle, there are Dimensions axes along which we can have periodic boundary conditions.
Definition PeanoCurve.h:50
static constexpr int RankOfPeriodicBoundaryCondition
Periodic boundary conditions are technically realised as domain decomposition, i.e.
Definition Spacetree.h:54
Node is Peano's abstraction for the hybrid of MPI/threads.
Definition Node.h:36
static void shutdownMPIDatatypes()
Definition Node.cpp:44
static std::bitset< 2 *Dimensions > getPeriodicBoundaryNumber(const tarch::la::Vector< TwoPowerD, int > &flags)
Analyse to which of the 2d faces it is adjacent.
Definition Node.cpp:231
static int getOutputStackForPeriodicBoundaryExchange(int faceNumber)
Identify output stack for periodic boundary data written by face.
Definition Node.cpp:250
Node()
The standard constructor assigns the attributes default values and checks whether the program is comp...
Definition Node.cpp:61
void shutdown()
The shutdown is invoked by peano4::shutdownSingletons()
Definition Node.cpp:452
static int getTreeNumberTiedToExchangeStackNumber(int number)
Gives you back the id of a communication partner, i.e.
Definition Node.cpp:413
static tarch::logging::Log _log
Logging device.
Definition Node.h:51
static bool isPeriodicBoundaryExchangeOutputStackNumber(int number)
Definition Node.cpp:377
bool continueToRun()
You should call this operation only on the ranks >0 to find out whether you should do more iteration/...
Definition Node.cpp:418
int reserveId(int rank, int forTreeId)
This operation is not const as it does some internal bookkeeping.
Definition Node.cpp:137
static bool isHorizontalDataExchangeInputStackNumber(int number)
Definition Node.cpp:393
static int getOutputStackNumberForHorizontalDataExchange(int id)
Hand in a spacetree id and get back the number that we should use to send something to this tree.
Definition Node.cpp:211
int getLocalTreeId(int treeId) const
Definition Node.cpp:125
static int mapPeriodicBoundaryExchangeOutputStackOntoInputStack(int outputStack)
Definition Node.cpp:203
int getRank(int treeId) const
You hand in a tree number and the node tells you on which rank such a tree is hosted.
Definition Node.cpp:119
static int getInputStackNumberForHorizontalDataExchange(int id)
Counterpart of getOutputStackNumberOfBoundaryExchange(int)
Definition Node.cpp:216
static constexpr int Terminate
Definition Node.h:38
int getCurrentProgramStep() const
Definition Node.cpp:447
void deregisterId(int id)
Only the SpacetreeSet should call this operation.
Definition Node.cpp:188
void registerId(int id, int masterId)
The operation is not thread-safe as we call it only internally, i.e.
Definition Node.cpp:164
static void initMPIDatatypes()
I originally wanted to embed these guys into the singleton's constructor.
Definition Node.cpp:27
std::pair< int, int > PeriodicBoundaryStackIdentifier
A periodic boundary stack is basically a stack (an integer), but I do augment it by a bitset which id...
Definition Node.h:216
int getGlobalTreeId(int treeId) const
Definition Node.cpp:131
static bool isVerticalDataExchangeOutputStackNumber(int number)
Definition Node.cpp:401
static bool isHorizontalDataExchangeOutputStackNumber(int number)
See getOutputStackNumberOfBoundaryExchange().
Definition Node.cpp:385
std::pair< int, MPI_Comm > GridDataExchangeMetaInformation
Definition Node.h:382
static std::string toString(const std::set< PeriodicBoundaryStackIdentifier > &data)
Definition Node.cpp:262
void setNextProgramStep(int number)
The user tells the set which program step to use, i.e.
Definition Node.cpp:441
static int getOutputStackNumberForVerticalDataExchange(int id)
Definition Node.cpp:221
static Node _singleton
Definition Node.h:53
int getNumberOfRegisteredTrees() const
Definition Node.cpp:183
virtual ~Node()
The standard destructor calls MPI_Finalize().
Definition Node.cpp:65
static std::set< PeriodicBoundaryStackIdentifier > getOutputStacksForPeriodicBoundaryExchange(const tarch::la::Vector< TwoPowerD, int > &flags)
You hand in a the flags of a vertex and you get the boundary stack identifiers including their direct...
Definition Node.cpp:282
static int getPeriodicBoundaryExchangeInputStackNumberForOutputStack(int outputStackNumber)
Hand in an output stack number of a face and you get back the input stack number.
Definition Node.cpp:355
static bool isVerticalDataExchangeInputStackNumber(int number)
Definition Node.cpp:407
static std::string getSemanticsForTag(int tag)
I use this for debugging.
Definition Node.cpp:497
static bool isGlobalMaster(int treeId)
Is this the global master?
Definition Node.cpp:103
static bool isStorageStackNumber(int number)
There are communication stacks and storage stacks.
Definition Node.cpp:198
GridDataExchangeMetaInformation getGridDataExchangeMetaInformation(int sendingTreeId, int receivingTreeId, ExchangeMode exchange) const
I use two tags per spacetree per rank: one for boundary data (horizontal) and one for up-down and syn...
Definition Node.cpp:476
static int getInputStackNumberForVerticalDataExchange(int id)
Get the input stack where a tree writes all of its vertical data to/from when it exchanges informatio...
Definition Node.cpp:226
int getId(int rank, int localTreeId) const
Peano maps grid instance threads + mpi ranks onto global IDs through this routine.
Definition Node.cpp:113
static Node & getInstance()
This operation returns the singleton instance.
Definition Node.cpp:108
Log Device.
Definition Log.h:516
static int getGlobalMasterRank()
Get the global master.
Definition Rank.cpp:414
int getNumberOfRanks() const
Definition Rank.cpp:551
static Rank & getInstance()
This operation returns the singleton instance.
Definition Rank.cpp:538
int getRank() const
Return rank of this node.
Definition Rank.cpp:528
void barrier(std::function< void()> waitor=[]() -> void {})
Definition Rank.cpp:347
static int reserveFreeTag(const std::string &fullQualifiedMessageName, int numberOfTags=1)
Return a Free Tag.
Definition Rank.cpp:39
Create a lock around a boolean semaphore region.
Definition Lock.h:19
virtual void receiveDanglingMessages() override
Answer to MPI Messages.
static ServiceRepository & getInstance()
std::string toString(Filter filter)
Definition convert.cpp:170
CPUGPUMethod int dLinearised(const tarch::la::Vector< Dimensions, int > &counter, int max)
Map d-dimensional vector onto integer index.
Definition Loop.cpp:106
static void initDatatype()
To be called prior to any MPI usage of this class.
static void shutdownDatatype()
Free the underlying MPI datatype.
static void initDatatype()
Wrapper around getDatatype() to trigger lazy evaluation if we use the lazy initialisation.
static void initDatatype()
Wrapper around getDatatype() to trigger lazy evaluation if we use the lazy initialisation.
static void shutdownDatatype()
Free the underlying MPI datatype.
static void shutdownDatatype()
Free the underlying MPI datatype.
static void initDatatype()
Wrapper around getDatatype() to trigger lazy evaluation if we use the lazy initialisation.
static void shutdownDatatype()
Free the underlying MPI datatype.
static void initDatatype()
Wrapper around getDatatype() to trigger lazy evaluation if we use the lazy initialisation.
static void receiveAndPollDanglingMessages(peano4::parallel::StartTraversalMessage &message, int source, int tag, MPI_Comm communicator=tarch::mpi::Rank::getInstance().getCommunicator())
static void sendAndPollDanglingMessages(const peano4::parallel::StartTraversalMessage &message, int destination, int tag, MPI_Comm communicator=tarch::mpi::Rank::getInstance().getCommunicator())
static void shutdownDatatype()
Free the underlying MPI datatype.
static void initDatatype()
Wrapper around getDatatype() to trigger lazy evaluation if we use the lazy initialisation.
void setMaster(int value)
Definition TreeEntry.cpp:46
static void shutdownDatatype()
Free the underlying MPI datatype.
static void initDatatype()
Wrapper around getDatatype() to trigger lazy evaluation if we use the lazy initialisation.
Simple vector class.
Definition Vector.h:134