Peano
Loading...
Searching...
No Matches
SeparateSweeps.EnclaveTask.template.cpp
Go to the documentation of this file.
1// **********************************************************************************************
2// *** !!!WARNING!!! ***
3// *** WARNING: AUTO GENERATED FILE! DO NOT MODIFY BY HAND! YOUR CHANGES WILL BE OVERWRITTEN! ***
4// *** !!!WARNING!!! ***
5// *** Generated by Peano's Python API: www.peano-framework.org ***
6// **********************************************************************************************
7#include "{{CLASSNAME}}.h"
8
9// user includes
10{{SOLVER_INCLUDES}}
11
12#include "config.h"
13
14
15#include "exahype2/CellData.h"
16#include "exahype2/EnclaveBookkeeping.h"
17#include "exahype2/EnclaveTask.h"
18
19#include "exahype2/enumerator/enumerator.h"
20
21#include "exahype2/fd/PatchUtils.h"
22
23#include "peano4/parallel/parallel.h"
24
25#include "peano4/utils/Loop.h"
26
27#include "tarch/compiler/CompilerSpecificSettings.h"
28
29#include "tarch/multicore/smartScheduling.h"
30#include "tarch/multicore/otter.h"
31
32#include "tarch/mpi/DoubleMessage.h"
33#include "tarch/mpi/IntegerMessage.h"
34
35#if defined(UseSmartMPI)
36#include "communication/Tags.h"
37#endif
38
39#include <string.h>
40#include <memory>
41
42tarch::logging::Log {{NAMESPACE | join("::")}}::{{CLASSNAME}}::_log( "{{NAMESPACE | join("::")}}::{{CLASSNAME}}" );
43int {{NAMESPACE | join("::")}}::{{CLASSNAME}}::_enclaveTaskTypeId(peano4::parallel::getTaskType("{{NAMESPACE | join("::")}}::{{CLASSNAME}}"));
44
45
46int {{NAMESPACE | join("::")}}::{{CLASSNAME}}::getEnclaveTaskTypeId() {
47 return _enclaveTaskTypeId;
48}
49
50
51double {{NAMESPACE | join("::")}}::{{CLASSNAME}}::applyKernelToCell(
52 const ::peano4::datamanagement::CellMarker& marker,
53 double t,
54 double dt,
55 double* __restrict__ reconstructedPatch,
56 double* __restrict__ targetPatch
57) {
58 ::exahype2::CellData patchData( reconstructedPatch, marker.x(), marker.h(), t, dt, targetPatch );
59
60 {% if STATELESS_PDE_TERMS %}
61 if ( repositories::{{SOLVER_INSTANCE}}.patchCanUseStatelessPDETerms(
62 marker.x(),
63 marker.h(),
64 t,
65 dt
66 )) {
67 ::exahype2::fd::{{KERNEL_NAMESPACE}}::{{FUSED_COMPUTE_KERNEL_CALL_CPU}}
68 }
69 else
70 {% endif %}
71
72 ::exahype2::fd::{{KERNEL_NAMESPACE}}::{{COMPUTE_KERNEL_CALL}}
73
74 ::exahype2::fd::validatePatch(
75 targetPatch,
76 {{NUMBER_OF_UNKNOWNS}},
77 {{NUMBER_OF_AUXILIARY_VARIABLES}},
78 {{NUMBER_OF_GRID_CELLS_PER_PATCH_PER_AXIS}},
79 0, // halo
80 std::string(__FILE__) + "(" + std::to_string(__LINE__) + "): " + marker.toString()
81 ); // outcome has to be valid
82
83 tarch::freeMemory(reconstructedPatch,tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );
84
85 {% if COMPUTE_MAX_EIGENVALUE %}
86 return patchData.maxEigenvalue[0];
87 {% else %}
88 return 0.0;
89 {% endif %}
90}
91
92
93//_inputValues,_outputValues,_marker,_t,_dt,_maxEigenvalue
94
95{{NAMESPACE | join("::")}}::{{CLASSNAME}}::{{CLASSNAME}}(
96 const ::peano4::datamanagement::CellMarker& marker,
97 double t,
98 double dt,
99 double* __restrict__ reconstructedPatch,
100 double* __restrict__ output
101):
102 ::exahype2::EnclaveTask(
103 _enclaveTaskTypeId,
104 marker,
105 t,
106 dt,
107 reconstructedPatch,
108 output,
109 #if Dimensions==2
110 ({{NUMBER_OF_GRID_CELLS_PER_PATCH_PER_AXIS}}+2*{{OVERLAP}}) * ({{NUMBER_OF_GRID_CELLS_PER_PATCH_PER_AXIS}}+2*{{OVERLAP}}) * ({{NUMBER_OF_UNKNOWNS}}+{{NUMBER_OF_AUXILIARY_VARIABLES}}),
111 {{NUMBER_OF_GRID_CELLS_PER_PATCH_PER_AXIS}} * {{NUMBER_OF_GRID_CELLS_PER_PATCH_PER_AXIS}} * {{NUMBER_OF_UNKNOWNS}},
112 #else
113 ({{NUMBER_OF_GRID_CELLS_PER_PATCH_PER_AXIS}}+2*{{OVERLAP}}) * ({{NUMBER_OF_GRID_CELLS_PER_PATCH_PER_AXIS}}+2*{{OVERLAP}}) * ({{NUMBER_OF_GRID_CELLS_PER_PATCH_PER_AXIS}}+2*{{OVERLAP}}) * ({{NUMBER_OF_UNKNOWNS}}+{{NUMBER_OF_AUXILIARY_VARIABLES}}),
114 {{NUMBER_OF_GRID_CELLS_PER_PATCH_PER_AXIS}} * {{NUMBER_OF_GRID_CELLS_PER_PATCH_PER_AXIS}} * {{NUMBER_OF_GRID_CELLS_PER_PATCH_PER_AXIS}} * {{NUMBER_OF_UNKNOWNS}},
115 #endif
116 [&]() -> void {
117 _maxEigenvalue = applyKernelToCell(
118 _marker,
119 _t,
120 _dt,
121 _inputValues,
122 _outputValues
123 );
124 }
125 )
126 #ifdef UseSmartMPI
127 , smartmpi::Task(_enclaveTaskTypeId)
128 #endif
129{
130 setPriority( {{ENCLAVE_TASK_PRIORITY}} );
131}
132
133
134#ifdef UseSmartMPI
135bool {{NAMESPACE | join("::")}}::{{CLASSNAME}}::isSmartMPITask() const {
136 #ifdef UseSmartMPI
137 return true;
138 #else
139 return false;
140 #endif
141}
142
143
144void {{NAMESPACE | join("::")}}::{{CLASSNAME}}::runLocally() {
145 computeTask();
146 if (_remoteTaskId != -1) {
147 ::exahype2::EnclaveBookkeeping::getInstance().finishedTask(_remoteTaskId,_numberOfResultValues,_outputValues,_maxEigenvalue);
148 } else {
149 ::exahype2::EnclaveBookkeeping::getInstance().finishedTask(getTaskId(),_numberOfResultValues,_outputValues,_maxEigenvalue);
150 }
151}
152
153
154void {{NAMESPACE | join("::")}}::{{CLASSNAME}}::moveTask(int rank, int tag, MPI_Comm communicator) {
155 ::tarch::mpi::DoubleMessage tMessage(_t);
156 ::tarch::mpi::DoubleMessage dtMessage(_dt);
157 ::tarch::mpi::IntegerMessage taskIdMessage;
158
159 if ( tag != smartmpi::communication::MoveTaskToMyServerForEvaluationTag &&
160 tag != smartmpi::communication::MoveTaskToComputesComputeRankTag ) {
161 taskIdMessage.setValue(_remoteTaskId);
162 } else {
163 taskIdMessage.setValue(getTaskId());
164 }
165
166 ::peano4::datamanagement::CellMarker::send( _marker, rank, tag, communicator );
167 ::tarch::mpi::DoubleMessage::send( tMessage, rank, tag, communicator );
168 ::tarch::mpi::DoubleMessage::send( dtMessage, rank, tag, communicator );
169 ::tarch::mpi::IntegerMessage::send( taskIdMessage, rank, tag, communicator );
170
171 MPI_Request request;
172 MPI_Isend( _inputValues, _numberOfInputValues, MPI_DOUBLE, rank, tag, communicator, &request );
173
174 logInfo(
175 "moveTask(...)",
176 "sent (" << _marker.toString() << "," << tMessage.toString() << "," << dtMessage.toString() << "," << _numberOfInputValues <<
177 "," << taskIdMessage.toString() << ") to rank " << rank <<
178 " via tag " << tag
179 );
180}
181
182
183smartmpi::Task* {{NAMESPACE | join("::")}}::{{CLASSNAME}}::receiveTask(int rank, int tag, MPI_Comm communicator) {
184 peano4::grid::GridTraversalEvent dummyEvent;
185 const int NumberOfInputValues =
186 #if Dimensions==2
187 {{NUMBER_OF_DOUBLE_VALUES_IN_PATCH_PLUS_HALO_2D}};
188 #else
189 {{NUMBER_OF_DOUBLE_VALUES_IN_PATCH_PLUS_HALO_3D}};
190 #endif
191
192 ::tarch::mpi::DoubleMessage tMessage;
193 ::tarch::mpi::DoubleMessage dtMessage;
194 ::tarch::mpi::IntegerMessage taskIdMessage;
195 ::peano4::datamanagement::CellMarker markerMessage(dummyEvent);
196 double* inputValues = tarch::allocateMemory( NumberOfInputValues, tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );
197
198 ::peano4::datamanagement::CellMarker::receive( markerMessage, rank, tag, communicator );
199 ::tarch::mpi::DoubleMessage::receive( tMessage, rank, tag, communicator );
200 ::tarch::mpi::DoubleMessage::receive( dtMessage, rank, tag, communicator );
201 ::tarch::mpi::IntegerMessage::receive( taskIdMessage, rank, tag, communicator );
202
203 logInfo(
204 "receiveTask(...)",
205 "received (" << markerMessage.toString() << "," << tMessage.toString() << "," << dtMessage.toString() << "," << taskIdMessage.toString() << ") from rank " << rank <<
206 " via tag " << tag << " and will now receive " << NumberOfInputValues << " doubles"
207 );
208
209 MPI_Recv( inputValues, NumberOfInputValues, MPI_DOUBLE, rank, tag, communicator,
210 MPI_STATUS_IGNORE
211 );
212
213 {{CLASSNAME}}* result = new {{CLASSNAME}}(
214 markerMessage,
215 tMessage.getValue(),
216 dtMessage.getValue(),
217 inputValues
218 );
219 result->_remoteTaskId = taskIdMessage.getValue();
220 return result;
221}
222
223
224void {{NAMESPACE | join("::")}}::{{CLASSNAME}}::runLocallyAndSendTaskOutputToRank(int rank, int tag, MPI_Comm communicator) {
225 _outputValues = tarch::allocateMemory( _numberOfResultValues, tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );
226
227// _functor(_inputValues,_outputValues,_marker,_t,_dt,_maxEigenvalue);
228// tarch::freeMemory(_inputValues,tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );
229 _functor();
230
231 logInfo(
232 "runLocallyAndSendTaskOutputToRank(...)",
233 "executed remote task on this rank. Will start to send result back"
234 );
235
236 forwardTaskOutputToRank(rank, tag, communicator);
237}
238
239
240void {{NAMESPACE | join("::")}}::{{CLASSNAME}}::forwardTaskOutputToRank(int rank, int tag, MPI_Comm communicator) {
241 logInfo(
242 "forwardTaskOutputToRank(...)",
243 "will start to forward task output (which has already been computed)"
244 );
245
246 ::tarch::mpi::DoubleMessage tMessage(_t);
247 ::tarch::mpi::DoubleMessage dtMessage(_dt);
248 ::tarch::mpi::DoubleMessage eValueMessage(_maxEigenvalue);
249 ::tarch::mpi::IntegerMessage taskIdMessage(_remoteTaskId);
250
251 ::peano4::datamanagement::CellMarker::send( _marker, rank, tag, communicator );
252 ::tarch::mpi::DoubleMessage::send( tMessage, rank, tag, communicator );
253 ::tarch::mpi::DoubleMessage::send( dtMessage, rank, tag, communicator );
254 ::tarch::mpi::DoubleMessage::send( eValueMessage, rank, tag, communicator );
255 ::tarch::mpi::IntegerMessage::send( taskIdMessage, rank, tag, communicator );
256
257 MPI_Request request;
258 MPI_Isend( _outputValues, _numberOfResultValues, MPI_DOUBLE, rank, tag, communicator, &request );
259
260 logInfo(
261 "forwardTaskOutputToRank(...)",
262 "sent (" << _marker.toString() << "," << tMessage.toString() << "," << dtMessage.toString() << "," << _numberOfResultValues <<
263 "," << taskIdMessage.toString() << ") to rank " << rank <<
264 " via tag " << tag
265 );
266
267 // tarch::freeMemory(_outputValues,tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );
268}
269
270
271smartmpi::Task* {{NAMESPACE | join("::")}}::{{CLASSNAME}}::receiveOutcome(int rank, int tag, MPI_Comm communicator, const bool intentionToForward) {
272 logInfo( "receiveOutcome(...)", "rank=" << rank << ", tag=" << tag );
273 peano4::grid::GridTraversalEvent dummyEvent;
274 const int NumberOfResultValues =
275 #if Dimensions==2
276 {{NUMBER_OF_DOUBLE_VALUES_IN_PATCH_2D}};
277 #else
278 {{NUMBER_OF_DOUBLE_VALUES_IN_PATCH_3D}};
279 #endif
280
281 ::tarch::mpi::DoubleMessage tMessage;
282 ::tarch::mpi::DoubleMessage dtMessage;
283 ::tarch::mpi::DoubleMessage eValueMessage;
284 ::tarch::mpi::IntegerMessage taskIdMessage;
285 ::peano4::datamanagement::CellMarker markerMessage(dummyEvent);
286 double* outputValues = tarch::allocateMemory( NumberOfResultValues, tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );
287
288 ::peano4::datamanagement::CellMarker::receive( markerMessage, rank, tag, communicator );
289 ::tarch::mpi::DoubleMessage::receive( tMessage, rank, tag, communicator );
290 ::tarch::mpi::DoubleMessage::receive( dtMessage, rank, tag, communicator );
291 ::tarch::mpi::DoubleMessage::receive( eValueMessage, rank, tag, communicator );
292 ::tarch::mpi::IntegerMessage::receive( taskIdMessage, rank, tag, communicator );
293
294 logInfo(
295 "receiveOutcome(...)",
296 "received (" << markerMessage.toString() << "," << tMessage.toString() << "," << dtMessage.toString() << "," << taskIdMessage.toString() << ") from rank " << rank <<
297 " via tag " << tag << " and will now receive " << NumberOfResultValues << " doubles"
298 );
299
300 MPI_Recv( outputValues, NumberOfResultValues, MPI_DOUBLE, rank, tag, communicator,
301 MPI_STATUS_IGNORE
302 );
303
310 if(intentionToForward) {
311 double* inputValues = nullptr; // no input as already computed
312
313 {{CLASSNAME}}* result = new {{CLASSNAME}}(
314 markerMessage,
315 tMessage.getValue(),
316 dtMessage.getValue(),
317 inputValues
318 );
319 result->_remoteTaskId = taskIdMessage.getValue();
320 result->_outputValues = outputValues;
321 result->_maxEigenvalue = eValueMessage.getValue();
322 return result;
323 }
324 logInfo(
325 "receiveOutcome(...)",
326 "bookmark outcome of task " << taskIdMessage.getValue()
327 );
328 ::exahype2::EnclaveBookkeeping::getInstance().finishedTask(taskIdMessage.getValue(),NumberOfResultValues,outputValues,eValueMessage.getValue());
329 return nullptr;
330}
331#endif
332
333
334{% if STATELESS_PDE_TERMS %}
335bool {{NAMESPACE | join("::")}}::{{CLASSNAME}}::canFuse() const {
336 return repositories::{{SOLVER_INSTANCE}}.patchCanUseStatelessPDETerms(
337 _marker.x(),
338 _marker.h(),
339 _t,
340 _dt
341 );
342}
343
344
348void {{NAMESPACE | join("::")}}::{{CLASSNAME}}::fuse( const std::list<Task*>& otherTasks, int targetDevice ) {
349 logDebug( "fuse(...)", "asked to fuse " << (otherTasks.size()+1) << " tasks into one large GPU task" );
350
351 ::exahype2::CellData patchData(otherTasks.size() + 1, tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory, targetDevice);
352 int currentTask = 0;
353 for (auto& p: otherTasks) {
354 patchData.QIn[currentTask] = static_cast<{{NAMESPACE | join("::")}}::{{CLASSNAME}}*>(p)->_inputValues;
355 patchData.cellCentre[currentTask] = static_cast<{{NAMESPACE | join("::")}}::{{CLASSNAME}}*>(p)->_marker.x();
356 patchData.cellSize[currentTask] = static_cast<{{NAMESPACE | join("::")}}::{{CLASSNAME}}*>(p)->_marker.h();
357 patchData.t[currentTask] = static_cast<{{NAMESPACE | join("::")}}::{{CLASSNAME}}*>(p)->_t;
358 patchData.dt[currentTask] = static_cast<{{NAMESPACE | join("::")}}::{{CLASSNAME}}*>(p)->_dt;
359 patchData.id[currentTask] = static_cast<{{NAMESPACE | join("::")}}::{{CLASSNAME}}*>(p)->_taskNumber;
360 patchData.QOut[currentTask] = tarch::allocateMemory(_numberOfResultValues, tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory, targetDevice);
361 currentTask++;
362 }
363 patchData.QIn[currentTask] = _inputValues;
364 patchData.cellCentre[currentTask] = _marker.x();
365 patchData.cellSize[currentTask] = _marker.h();
366 patchData.t[currentTask] = _t;
367 patchData.dt[currentTask] = _dt;
368 patchData.id[currentTask] = _taskNumber;
369 patchData.QOut[currentTask] = tarch::allocateMemory(_numberOfResultValues, tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory, targetDevice);
370
371
372 //
373 // ==============
374 // Invoke kernels
375 // ==============
376 //
377 bool foundOffloadingBranch = false;
378
379
380 #if defined(GPUOffloadingOMP)
381 if (targetDevice>=0) {
382 foundOffloadingBranch = true;
383 ::exahype2::fd::{{KERNEL_NAMESPACE}}::omp::{{FUSED_COMPUTE_KERNEL_CALL_GPU}}
384 }
385 #endif
386
387 #if defined(GPUOffloadingHIP)
388 if (targetDevice>=0) {
389 foundOffloadingBranch = true;
390 ::exahype2::fd::{{KERNEL_NAMESPACE}}::hip::{{FUSED_COMPUTE_KERNEL_CALL_GPU}}
391 }
392 #endif
393
394 #if defined(GPUOffloadingSYCL)
395 if (targetDevice>=0 or targetDevice==Host) {
396 foundOffloadingBranch = true;
397
398 ::exahype2::fd::{{KERNEL_NAMESPACE}}::sycl::{{FUSED_COMPUTE_KERNEL_CALL_GPU}}
399 }
400 #endif
401
402 if (not foundOffloadingBranch) {
403 ::exahype2::fd::{{KERNEL_NAMESPACE}}::{{FUSED_COMPUTE_KERNEL_CALL_CPU}}
404 }
405
406
407 //
408 // ==================
409 // Bring back results
410 // ==================
411 //
412 for (int i=0; i<patchData.numberOfCells; i++) {
413 tarch::freeMemory(patchData.QIn[i], tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory, targetDevice);
414 ::exahype2::EnclaveBookkeeping::getInstance().finishedTask(patchData.id[i], _numberOfResultValues, patchData.QOut[i], patchData.maxEigenvalue[i]);
415 }
416}
417{% endif %}
logDebug
#define logDebug(methodName, logMacroMessageStream)
Definition Log.h:50
logInfo
#define logInfo(methodName, logMacroMessageStream)
Wrapper macro around tarch::tarch::logging::Log to improve logging.
Definition Log.h:411
exahype2::EnclaveBookkeeping::getInstance
static EnclaveBookkeeping & getInstance()
Definition EnclaveBookkeeping.cpp:21
exahype2::EnclaveBookkeeping::finishedTask
void finishedTask(int taskNumber, int numberOfResultValues, double *data, double maxEigenvalue)
Usually called directly by EnclaveTask.
Definition EnclaveBookkeeping.cpp:102
tarch::logging::Log
Log Device.
Definition Log.h:516
_log
tarch::logging::Log _log("exahype2::fv")
DSL_test.dt
float dt
Definition DSL_test.py:5
exahype2::fd::validatePatch
void validatePatch(const double *__restrict__ Q, int unknowns, int auxiliaryVariables, int numberOfGridCellsPerPatchPerAxis, int haloSize, const std::string &location="", bool triggerNonCriticalAssertion=true, double *minValues=nullptr, double *maxValues=nullptr)
Just runs over the patch and ensures that no entry is non or infinite.
Definition PatchUtils.cpp:79
exahype2
For the generic kernels that I use here most of the time.
Definition CellAccess.h:13
peano4::parallel::getTaskType
int getTaskType(const std::string &className)
Get unique number (id) for task.
Definition parallel.cpp:7
plots_llvm_paper_likwid.marker
marker
Definition plots_llvm_paper_likwid.py:172
tarch::freeMemory
void freeMemory(void *data, MemoryLocation location, int device=accelerator::Device::HostDevice)
Free memory.
Definition accelerator.cpp:30
tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory
@ ManagedSharedAcceleratorDeviceMemory
To be used on host only.
tarch::allocateMemory
T * allocateMemory(std::size_t count, MemoryLocation location, int device=accelerator::Device::HostDevice)
Allocates memory on the specified memory location.
Definition accelerator.h:99
exahype2::CellData
Representation of a number of cells which contains all information that's required to process the sto...
Definition CellData.h:77
peano4::datamanagement::CellMarker::receive
static void receive(CellMarker &buffer, int source, int tag, MPI_Comm communicator)
Definition CellMarker.cpp:259
peano4::datamanagement::CellMarker::send
static void send(const CellMarker &buffer, int destination, int tag, MPI_Comm communicator)
In DaStGen (the first version), I had a non-static version of the send as well as the receive.
Definition CellMarker.cpp:253
tarch::mpi::DoubleMessage::toString
std::string toString() const
Definition DoubleMessage.cpp:16
tarch::mpi::DoubleMessage::receive
static void receive(tarch::mpi::DoubleMessage &buffer, int source, int tag, MPI_Comm communicator)
Definition DoubleMessage.cpp:202
tarch::mpi::DoubleMessage::send
static void send(const tarch::mpi::DoubleMessage &buffer, int destination, int tag, MPI_Comm communicator)
In DaStGen (the first version), I had a non-static version of the send as well as the receive.
Definition DoubleMessage.cpp:197
tarch::mpi::IntegerMessage::toString
std::string toString() const
Definition IntegerMessage.cpp:16
tarch::mpi::IntegerMessage::receive
static void receive(tarch::mpi::IntegerMessage &buffer, int source, int tag, MPI_Comm communicator)
Definition IntegerMessage.cpp:202
tarch::mpi::IntegerMessage::send
static void send(const tarch::mpi::IntegerMessage &buffer, int destination, int tag, MPI_Comm communicator)
In DaStGen (the first version), I had a non-static version of the send as well as the receive.
Definition IntegerMessage.cpp:197