Peano
Loading...
Searching...
No Matches
SolveVolumeIntegral.EnclaveTask.template.cpp
Go to the documentation of this file.
1// **********************************************************************************************
2// *** !!!WARNING!!! ***
3// *** WARNING: AUTO GENERATED FILE! DO NOT MODIFY BY HAND! YOUR CHANGES WILL BE OVERWRITTEN! ***
4// *** !!!WARNING!!! ***
5// *** Generated by Peano's Python API: www.peano-framework.org ***
6// **********************************************************************************************
7#include "{{CLASSNAME}}.h"
8
9{{SOLVER_INCLUDES}}
10
11#include "config.h"
12
13#if GPUOffloadingSYCL>0
14#include "tarch/multicore/sycl/SYCL.h"
15#endif
16
18#include "exahype2/CellData.h"
21
23
25
26#include "peano4/utils/Loop.h"
27
29
32
35
36#if defined(UseSmartMPI)
37#include "communication/Tags.h"
38#endif
39
40#include <string.h>
41#include <memory>
42
43tarch::logging::Log {{NAMESPACE | join("::")}}::{{CLASSNAME}}::_log( "{{NAMESPACE | join("::")}}::{{CLASSNAME}}" );
44int {{NAMESPACE | join("::")}}::{{CLASSNAME}}::_enclaveTaskTypeId(peano4::parallel::getTaskType("{{NAMESPACE | join("::")}}::{{CLASSNAME}}"));
45
46
47int {{NAMESPACE | join("::")}}::{{CLASSNAME}}::getEnclaveTaskTypeId() {
48 return _enclaveTaskTypeId;
49}
50
51
52void {{NAMESPACE | join("::")}}::{{CLASSNAME}}::applyKernelToCell(
53 const ::peano4::datamanagement::CellMarker& marker,
54 double timeStamp,
55 double timeStepSize,
56 double* __restrict__ QIn,
57 double* __restrict__ QOut
58) {
59 ::exahype2::CellData cellData(
60 QIn,
61 marker.x(),
62 marker.h(),
63 timeStamp,
64 timeStepSize,
65 QOut
66 );
67
68 {% if STATELESS_PDE_TERMS %}
69 if (repositories::{{SOLVER_INSTANCE}}.cellCanUseStatelessPDETerms(
70 marker.x(),
71 marker.h(),
72 timeStamp,
73 timeStepSize
74 )) {
75 ::exahype2::dg::{{KERNEL_NAMESPACE}}::{{VOLUMETRIC_COMPUTE_KERNEL_CALL_STATELESS}}
76 } else
77 {% endif %}
78
79 ::exahype2::dg::{{KERNEL_NAMESPACE}}::{{VOLUMETRIC_COMPUTE_KERNEL_CALL}}
80}
81
82
83{{NAMESPACE | join("::")}}::{{CLASSNAME}}::{{CLASSNAME}}(
84 const ::peano4::datamanagement::CellMarker& marker,
85 double t,
86 double dt,
87 const double* __restrict__ linearCombinationOfPreviousShots
88):
89 ::exahype2::EnclaveTask(
90 _enclaveTaskTypeId,
91 marker,
92 t,
93 dt,
94 #if Dimensions==2
95 tarch::allocateMemory({{NUMBER_OF_DOFS_PER_CELL_2D}}*({{NUMBER_OF_UNKNOWNS}}+{{NUMBER_OF_AUXILIARY_VARIABLES}}), tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory),
96 nullptr,
97 {{NUMBER_OF_DOFS_PER_CELL_2D}}*({{NUMBER_OF_UNKNOWNS}}+{{NUMBER_OF_AUXILIARY_VARIABLES}}),
98 {{NUMBER_OF_DOFS_PER_CELL_2D}}* {{NUMBER_OF_UNKNOWNS}},
99 #else
100 tarch::allocateMemory({{NUMBER_OF_DOFS_PER_CELL_3D}}*({{NUMBER_OF_UNKNOWNS}}+{{NUMBER_OF_AUXILIARY_VARIABLES}}), tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory),
101 nullptr,
102 {{NUMBER_OF_DOFS_PER_CELL_3D}}*({{NUMBER_OF_UNKNOWNS}}+{{NUMBER_OF_AUXILIARY_VARIABLES}}),
103 {{NUMBER_OF_DOFS_PER_CELL_3D}}* {{NUMBER_OF_UNKNOWNS}},
104 #endif
105 [&]() -> void {
106 applyKernelToCell(
107 _marker,
108 _t,
109 _dt,
110 _inputValues,
111 _outputValues
112 );
113
115 }
116 ),
117#ifdef UseSmartMPI
118 smartmpi::Task(_enclaveTaskTypeId),
119#endif
121 // @todo Name is not good. Should be SOLVE_VOLUME_INTEGRAL_ENCLAVE_TASK, so we can
122 // distinguish different task types once we map other steps onto tasks, too
123 setPriority({{ENCLAVE_TASK_PRIORITY}});
124 std::copy_n(linearCombinationOfPreviousShots, _numberOfInputValues, _inputValues);
125}
126
127
128{{NAMESPACE | join("::")}}::{{CLASSNAME}}::{{CLASSNAME}}(
129 const ::peano4::datamanagement::CellMarker& marker,
130 double t,
131 double dt,
132 std::shared_ptr< double[] > linearCombinationOfPreviousShots,
133 double* __restrict__ output
134):
135 ::exahype2::EnclaveTask(
136 _enclaveTaskTypeId,
137 marker,
138 t,
139 dt,
140 linearCombinationOfPreviousShots.get(),
141 output,
142 #if Dimensions==2
143 {{NUMBER_OF_DOFS_PER_CELL_2D}}*({{NUMBER_OF_UNKNOWNS}}+{{NUMBER_OF_AUXILIARY_VARIABLES}}),
144 {{NUMBER_OF_DOFS_PER_CELL_2D}}* {{NUMBER_OF_UNKNOWNS}},
145 #else
146 {{NUMBER_OF_DOFS_PER_CELL_3D}}*({{NUMBER_OF_UNKNOWNS}}+{{NUMBER_OF_AUXILIARY_VARIABLES}}),
147 {{NUMBER_OF_DOFS_PER_CELL_3D}}* {{NUMBER_OF_UNKNOWNS}},
148 #endif
149 [&]() -> void {
150 applyKernelToCell(
151 _marker,
152 _t,
153 _dt,
154 _inputValues,
155 _outputValues
156 );
157 }
158 ),
159#ifdef UseSmartMPI
160 smartmpi::Task(_enclaveTaskTypeId),
161#endif
162 _linearCombinationOfPreviousShots(linearCombinationOfPreviousShots) {
163 // @todo Name is not good. Should be SOLVE_VOLUME_INTEGRAL_ENCLAVE_TASK, so we can
164 // distinguish different task types once we map other steps onto tasks, too
165 setPriority({{ENCLAVE_TASK_PRIORITY}});
166}
167
168
169#ifdef UseSmartMPI
170bool {{NAMESPACE | join("::")}}::{{CLASSNAME}}::isSmartMPITask() const {
171#ifdef UseSmartMPI
172 return true;
173#else
174 return false;
175#endif
176}
177
178
179void {{NAMESPACE | join("::")}}::{{CLASSNAME}}::runLocally() {
180 computeTask();
181 if (_remoteTaskId != -1) {
182 ::exahype2::EnclaveBookkeeping::getInstance().finishedTask(_remoteTaskId,_numberOfResultValues,_outputValues,_maxEigenvalue);
183 } else {
184 ::exahype2::EnclaveBookkeeping::getInstance().finishedTask(getTaskId(),_numberOfResultValues,_outputValues,_maxEigenvalue);
185 }
186}
187
188
189void {{NAMESPACE | join("::")}}::{{CLASSNAME}}::moveTask(int rank, int tag, MPI_Comm communicator) {
190 ::tarch::mpi::DoubleMessage tMessage(_t);
191 ::tarch::mpi::DoubleMessage dtMessage(_dt);
192 ::tarch::mpi::IntegerMessage taskIdMessage;
193
194 if ( tag != smartmpi::communication::MoveTaskToMyServerForEvaluationTag &&
195 tag != smartmpi::communication::MoveTaskToComputesComputeRankTag ) {
196 taskIdMessage.setValue(_remoteTaskId);
197 } else {
198 taskIdMessage.setValue(getTaskId());
199 }
200
201 ::peano4::datamanagement::CellMarker::send( _marker, rank, tag, communicator );
202 ::tarch::mpi::DoubleMessage::send( tMessage, rank, tag, communicator );
203 ::tarch::mpi::DoubleMessage::send( dtMessage, rank, tag, communicator );
204 ::tarch::mpi::IntegerMessage::send( taskIdMessage, rank, tag, communicator );
205
206 MPI_Request request;
207 MPI_Isend( _inputValues, _numberOfInputValues, MPI_DOUBLE, rank, tag, communicator, &request );
208
209 logInfo(
210 "moveTask(...)",
211 "sent (" << _marker.toString() << "," << tMessage.toString() << "," << dtMessage.toString() << "," << _numberOfInputValues <<
212 "," << taskIdMessage.toString() << ") to rank " << rank <<
213 " via tag " << tag
214 );
215}
216
217
218smartmpi::Task* {{NAMESPACE | join("::")}}::{{CLASSNAME}}::receiveTask(int rank, int tag, MPI_Comm communicator) {
220 const int NumberOfInputValues =
221 #if Dimensions==2
222 {{NUMBER_OF_DOUBLE_VALUES_IN_PATCH_PLUS_HALO_2D}};
223 #else
224 {{NUMBER_OF_DOUBLE_VALUES_IN_PATCH_PLUS_HALO_3D}};
225 #endif
226
229 ::tarch::mpi::IntegerMessage taskIdMessage;
230 ::peano4::datamanagement::CellMarker markerMessage(dummyEvent);
232
233 ::peano4::datamanagement::CellMarker::receive( markerMessage, rank, tag, communicator );
234 ::tarch::mpi::DoubleMessage::receive( tMessage, rank, tag, communicator );
235 ::tarch::mpi::DoubleMessage::receive( dtMessage, rank, tag, communicator );
236 ::tarch::mpi::IntegerMessage::receive( taskIdMessage, rank, tag, communicator );
237
238 logInfo(
239 "receiveTask(...)",
240 "received (" << markerMessage.toString() << "," << tMessage.toString() << "," << dtMessage.toString() << "," << taskIdMessage.toString() << ") from rank " << rank <<
241 " via tag " << tag << " and will now receive " << NumberOfInputValues << " doubles"
242 );
243
244 MPI_Recv( inputValues, NumberOfInputValues, MPI_DOUBLE, rank, tag, communicator,
245 MPI_STATUS_IGNORE
246 );
247
248 {{CLASSNAME}}* result = new {{CLASSNAME}}(
249 markerMessage,
250 tMessage.getValue(),
251 dtMessage.getValue(),
252 inputValues
253 );
254 result->_remoteTaskId = taskIdMessage.getValue();
255 return result;
256}
257
258
259void {{NAMESPACE | join("::")}}::{{CLASSNAME}}::runLocallyAndSendTaskOutputToRank(int rank, int tag, MPI_Comm communicator) {
261
262// _functor(_inputValues,_outputValues,_marker,_t,_dt,_maxEigenvalue);
263// tarch::freeMemory(_inputValues,tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );
264 _functor();
265
266 logInfo(
267 "runLocallyAndSendTaskOutputToRank(...)",
268 "executed remote task on this rank. Will start to send result back"
269 );
270
271 forwardTaskOutputToRank(rank, tag, communicator);
272}
273
274
275void {{NAMESPACE | join("::")}}::{{CLASSNAME}}::forwardTaskOutputToRank(int rank, int tag, MPI_Comm communicator) {
276 logInfo(
277 "forwardTaskOutputToRank(...)",
278 "will start to forward task output (which has already been computed)"
279 );
280
281 ::tarch::mpi::DoubleMessage tMessage(_t);
282 ::tarch::mpi::DoubleMessage dtMessage(_dt);
283 ::tarch::mpi::DoubleMessage eValueMessage(_maxEigenvalue);
284 ::tarch::mpi::IntegerMessage taskIdMessage(_remoteTaskId);
285
286 ::peano4::datamanagement::CellMarker::send( _marker, rank, tag, communicator );
287 ::tarch::mpi::DoubleMessage::send( tMessage, rank, tag, communicator );
288 ::tarch::mpi::DoubleMessage::send( dtMessage, rank, tag, communicator );
289 ::tarch::mpi::DoubleMessage::send( eValueMessage, rank, tag, communicator );
290 ::tarch::mpi::IntegerMessage::send( taskIdMessage, rank, tag, communicator );
291
292 MPI_Request request;
293 MPI_Isend( _outputValues, _numberOfResultValues, MPI_DOUBLE, rank, tag, communicator, &request );
294
295 logInfo(
296 "forwardTaskOutputToRank(...)",
297 "sent (" << _marker.toString() << "," << tMessage.toString() << "," << dtMessage.toString() << "," << _numberOfResultValues <<
298 "," << taskIdMessage.toString() << ") to rank " << rank <<
299 " via tag " << tag
300 );
301
302 // tarch::freeMemory(_outputValues,tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );
303}
304
305
306smartmpi::Task* {{NAMESPACE | join("::")}}::{{CLASSNAME}}::receiveOutcome(int rank, int tag, MPI_Comm communicator, const bool intentionToForward) {
307 logInfo( "receiveOutcome(...)", "rank=" << rank << ", tag=" << tag );
309 const int NumberOfResultValues =
310 #if Dimensions==2
311 {{NUMBER_OF_DOUBLE_VALUES_IN_PATCH_2D}};
312 #else
313 {{NUMBER_OF_DOUBLE_VALUES_IN_PATCH_3D}};
314 #endif
315
318 ::tarch::mpi::DoubleMessage eValueMessage;
319 ::tarch::mpi::IntegerMessage taskIdMessage;
320 ::peano4::datamanagement::CellMarker markerMessage(dummyEvent);
321 double* outputValues = tarch::allocateMemory( NumberOfResultValues, tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );
322
323 ::peano4::datamanagement::CellMarker::receive( markerMessage, rank, tag, communicator );
324 ::tarch::mpi::DoubleMessage::receive( tMessage, rank, tag, communicator );
325 ::tarch::mpi::DoubleMessage::receive( dtMessage, rank, tag, communicator );
326 ::tarch::mpi::DoubleMessage::receive( eValueMessage, rank, tag, communicator );
327 ::tarch::mpi::IntegerMessage::receive( taskIdMessage, rank, tag, communicator );
328
329 logInfo(
330 "receiveOutcome(...)",
331 "received (" << markerMessage.toString() << "," << tMessage.toString() << "," << dtMessage.toString() << "," << taskIdMessage.toString() << ") from rank " << rank <<
332 " via tag " << tag << " and will now receive " << NumberOfResultValues << " doubles"
333 );
334
335 MPI_Recv( outputValues, NumberOfResultValues, MPI_DOUBLE, rank, tag, communicator,
336 MPI_STATUS_IGNORE
337 );
338
345 if(intentionToForward) {
346 double* inputValues = nullptr; // no input as already computed
347
348 {{CLASSNAME}}* result = new {{CLASSNAME}}(
349 markerMessage,
350 tMessage.getValue(),
351 dtMessage.getValue(),
352 inputValues
353 );
354 result->_remoteTaskId = taskIdMessage.getValue();
355 result->_outputValues = outputValues;
356 result->_maxEigenvalue = eValueMessage.getValue();
357 return result;
358 }
359 logInfo(
360 "receiveOutcome(...)",
361 "bookmark outcome of task " << taskIdMessage.getValue()
362 );
363 ::exahype2::EnclaveBookkeeping::getInstance().finishedTask(taskIdMessage.getValue(),NumberOfResultValues,outputValues,eValueMessage.getValue());
364 return nullptr;
365}
366#endif
367
368
369{% if STATELESS_PDE_TERMS %}
370
371bool {{NAMESPACE | join("::")}}::{{CLASSNAME}}::canFuse() const {
372 return repositories::{{SOLVER_INSTANCE}}.cellCanUseStatelessPDETerms(
373 _marker.x(),
374 _marker.h(),
375 _t,
376 _dt
377 );
378}
379
380
384void {{NAMESPACE | join("::")}}::{{CLASSNAME}}::fuse( const std::list<Task*>& otherTasks, int targetDevice ) {
385 logDebug("fuse(...)", "rank " << tarch::mpi::Rank::getInstance().getRank()
386 << " asked to fuse " << (otherTasks.size() + 1) << " tasks into one large GPU task on device " << targetDevice);
387
388 ::exahype2::CellData cellData(otherTasks.size() + 1, tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory, targetDevice);
389 int currentTask = 0;
390 for (auto& p: otherTasks) {
391 cellData.QIn[currentTask] = static_cast<{{NAMESPACE | join("::")}}::{{CLASSNAME}}*>(p)->_inputValues;
392 cellData.cellCentre[currentTask] = static_cast<{{NAMESPACE | join("::")}}::{{CLASSNAME}}*>(p)->_marker.x();
393 cellData.cellSize[currentTask] = static_cast<{{NAMESPACE | join("::")}}::{{CLASSNAME}}*>(p)->_marker.h();
394 cellData.t[currentTask] = static_cast<{{NAMESPACE | join("::")}}::{{CLASSNAME}}*>(p)->_t;
395 cellData.dt[currentTask] = static_cast<{{NAMESPACE | join("::")}}::{{CLASSNAME}}*>(p)->_dt;
396 cellData.id[currentTask] = static_cast<{{NAMESPACE | join("::")}}::{{CLASSNAME}}*>(p)->_taskNumber;
397 cellData.QOut[currentTask] = tarch::allocateMemory(_numberOfResultValues, tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory, targetDevice);
398 currentTask++;
399 }
400 cellData.QIn[currentTask] = _inputValues;
401 cellData.cellCentre[currentTask] = _marker.x();
402 cellData.cellSize[currentTask] = _marker.h();
403 cellData.t[currentTask] = _t;
404 cellData.dt[currentTask] = _dt;
405 cellData.id[currentTask] = _taskNumber;
406 cellData.QOut[currentTask] = tarch::allocateMemory(_numberOfResultValues, tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory, targetDevice);
407
408 //
409 // ==============
410 // Invoke kernels
411 // ==============
412 //
413 bool foundOffloadingBranch = false;
414
415 #if defined(GPUOffloadingOMP)
416 if (targetDevice>=0) {
417 foundOffloadingBranch = true;
418 ::exahype2::dg::{{KERNEL_NAMESPACE}}::omp::{{FUSED_VOLUMETRIC_COMPUTE_KERNEL_CALL_STATELESS_GPU}}
419 }
420 #endif
421
422 #if defined(GPUOffloadingHIP)
423 if (targetDevice>=0) {
424 foundOffloadingBranch = true;
425 ::exahype2::dg::{{KERNEL_NAMESPACE}}::hip::{{FUSED_VOLUMETRIC_COMPUTE_KERNEL_CALL_STATELESS_GPU}}
426 }
427 #endif
428
429 #if defined(GPUOffloadingSYCL)
430 if (targetDevice>=0 or targetDevice==Host) {
431 foundOffloadingBranch = true;
432 ::exahype2::dg::{{KERNEL_NAMESPACE}}::sycl::{{FUSED_VOLUMETRIC_COMPUTE_KERNEL_CALL_STATELESS_GPU}}
433 }
434 #endif
435
436 if (not foundOffloadingBranch) {
437 logDebug("fuse(...)", "rank " << tarch::mpi::Rank::getInstance().getRank()
438 << " cannot find offloading branch for device " << targetDevice << ". Process fused tasks on the CPU.");
439 ::exahype2::dg::{{KERNEL_NAMESPACE}}::{{FUSED_VOLUMETRIC_COMPUTE_KERNEL_CALL_STATELESS_CPU}}
440 }
441
442 //
443 // ==================
444 // Bring back results
445 // ==================
446 //
447 for (int i=0; i<cellData.numberOfCells; i++) {
448 //tarch::freeMemory(cellData.QIn[i], tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory, targetDevice);
449 ::exahype2::EnclaveBookkeeping::getInstance().finishedTask(cellData.id[i], _numberOfResultValues, cellData.QOut[i], cellData.maxEigenvalue[i]);
450 }
451}
452
453{% endif %}
#define logDebug(methodName, logMacroMessageStream)
Definition Log.h:50
#define logInfo(methodName, logMacroMessageStream)
Wrapper macro around tarch::tarch::logging::Log to improve logging.
Definition Log.h:411
static EnclaveBookkeeping & getInstance()
void finishedTask(int taskNumber, int numberOfResultValues, double *data, double maxEigenvalue)
Usually called directly by EnclaveTask.
Log Device.
Definition Log.h:516
static Rank & getInstance()
This operation returns the singleton instance.
Definition Rank.cpp:539
tarch::logging::Log _log("exahype2::fv")
float dt
Definition DSL_test.py:5
For the generic kernels that I use here most of the time.
Definition CellAccess.h:13
int getTaskType(const std::string &className)
Get unique number (id) for task.
Definition parallel.cpp:13
Have to include this header, as I need access to the SYCL_EXTERNAL keyword.
Definition accelerator.h:19
void freeMemory(void *data, MemoryLocation location, int device=accelerator::Device::HostDevice)
Free memory.
@ ManagedSharedAcceleratorDeviceMemory
To be used on host only.
T * allocateMemory(std::size_t count, MemoryLocation location, int device=accelerator::Device::HostDevice)
Allocates memory on the specified memory location.
Definition accelerator.h:99
Representation of a number of cells which contains all information that's required to process the sto...
Definition CellData.h:77
static void receive(CellMarker &buffer, int source, int tag, MPI_Comm communicator)
static void send(const CellMarker &buffer, int destination, int tag, MPI_Comm communicator)
In DaStGen (the first version), I had a non-static version of the send as well as the receive.
std::string toString() const
static void receive(tarch::mpi::DoubleMessage &buffer, int source, int tag, MPI_Comm communicator)
static void send(const tarch::mpi::DoubleMessage &buffer, int destination, int tag, MPI_Comm communicator)
In DaStGen (the first version), I had a non-static version of the send as well as the receive.
std::string toString() const
static void receive(tarch::mpi::IntegerMessage &buffer, int source, int tag, MPI_Comm communicator)
static void send(const tarch::mpi::IntegerMessage &buffer, int destination, int tag, MPI_Comm communicator)
In DaStGen (the first version), I had a non-static version of the send as well as the receive.