Line data Source code
1 : #ifndef SPTASKGRAPH_HPP
2 : #define SPTASKGRAPH_HPP
3 :
4 : #include <functional>
5 : #include <list>
6 : #include <map>
7 : #include <unordered_map>
8 : #include <memory>
9 : #include <unistd.h>
10 : #include <cmath>
11 : #include <type_traits>
12 : #include <initializer_list>
13 :
14 : #include "Config/SpConfig.hpp"
15 : #include "Utils/SpUtils.hpp"
16 : #include "Task/SpAbstractTask.hpp"
17 : #include "Task/SpTask.hpp"
18 : #include "Data/SpDependence.hpp"
19 : #include "Data/SpDataHandle.hpp"
20 : #include "Utils/SpArrayView.hpp"
21 : #include "Task/SpPriority.hpp"
22 : #include "Task/SpProbability.hpp"
23 : #include "Utils/SpTimePoint.hpp"
24 : #include "Random/SpMTGenerator.hpp"
25 : #include "Scheduler/SpTaskManager.hpp"
26 : #include "Speculation/SpSpecTaskGroup.hpp"
27 : #include "Utils/small_vector.hpp"
28 : #include "Speculation/SpSpeculativeModel.hpp"
29 : #include "SpAbstractTaskGraph.hpp"
30 : #include "Compute/SpComputeEngine.hpp"
31 : #include "Scheduler/SpTaskManagerListener.hpp"
32 :
33 : template <const bool isSpeculativeTaskGraph>
34 : class SpTaskGraphCommon : public SpAbstractTaskGraph {
35 : protected:
36 : //! Map of all data handles
37 : std::unordered_map<void*, std::unique_ptr<SpDataHandle>> allDataHandles;
38 :
39 : ///////////////////////////////////////////////////////////////////////////
40 : /// Data handle management
41 : ///////////////////////////////////////////////////////////////////////////
42 :
43 : template <class ParamsType>
44 9769 : SpDataHandle* getDataHandleCore(ParamsType& inParam){
45 9769 : auto iterHandleFound = allDataHandles.find(const_cast<std::remove_const_t<ParamsType>*>(&inParam));
46 9769 : if(iterHandleFound == allDataHandles.end()){
47 497 : SpDebugPrint() << "SpTaskGraph -- => Not found";
48 : // Create a new data handle
49 497 : auto newHandle = std::make_unique<SpDataHandle>(const_cast<std::remove_const_t<ParamsType>*>(&inParam));
50 : // Save the ptr
51 497 : SpDataHandle* newHandlePtr = newHandle.get();
52 : // Move in the map
53 497 : allDataHandles[const_cast<std::remove_const_t<ParamsType>*>(&inParam)] = std::move(newHandle);
54 : // Return the ptr
55 497 : return newHandlePtr;
56 : }
57 9272 : SpDebugPrint() << "SpTaskGraph -- => Found";
58 : // Exists, return the pointer
59 9272 : return iterHandleFound->second.get();
60 : }
61 :
62 : template <class ParamsType>
63 : typename std::enable_if_t<ParamsType::IsScalar, small_vector<SpDataHandle*,1>>
64 3386 : getDataHandle(ParamsType& scalarData){
65 : static_assert(std::tuple_size<decltype(scalarData.getAllData())>::value, "Size must be one for scalar");
66 3386 : typename ParamsType::HandleTypePtr ptrToObject = scalarData.getAllData()[0];
67 6772 : SpDebugPrint() << "SpTaskGraph -- Look for " << ptrToObject << " with mode provided " << SpModeToStr(ParamsType::AccessMode)
68 3386 : << " scalarData address " << &scalarData;
69 3386 : return small_vector<SpDataHandle*, 1>{getDataHandleCore(*ptrToObject)};
70 : }
71 :
72 : template <class ParamsType>
73 : typename std::enable_if_t<!ParamsType::IsScalar,small_vector<SpDataHandle*>>
74 246 : getDataHandle(ParamsType& containerData){
75 246 : small_vector<SpDataHandle*> handles;
76 1526 : for(typename ParamsType::HandleTypePtr ptrToObject : containerData.getAllData()){
77 1280 : SpDebugPrint() << "SpTaskGraph -- Look for " << ptrToObject << " with array view in getDataHandleExtra";
78 1280 : SpDataHandle* handle = getDataHandleCore(*ptrToObject);
79 1280 : handles.emplace_back(handle);
80 : }
81 246 : return handles;
82 : }
83 :
84 : ///////////////////////////////////////////////////////////////////////////
85 : /// Task method call argument partitioning and dispatch
86 : ///////////////////////////////////////////////////////////////////////////
87 :
88 : template <typename T>
89 : using is_prio_or_proba = typename std::disjunction<std::is_same<std::decay_t<T>, SpPriority>, std::is_same<std::decay_t<T>, SpProbability>>;
90 :
91 : template <typename T>
92 : inline static constexpr bool is_prio_or_proba_v = is_prio_or_proba<T>::value;
93 :
94 : template <class T>
95 : using is_data_dependency = std::conjunction<has_getView<T>, has_getAllData<T>>;
96 :
97 : template <class T>
98 : inline static constexpr bool is_data_dependency_v = is_data_dependency<T>::value;
99 :
100 : template <class... ParamsTy>
101 : using contains_potential_write_dependencies = std::disjunction<access_modes_are_equal<SpDataAccessMode::POTENTIAL_WRITE, ParamsTy>...>;
102 :
103 : template <class... ParamsTy>
104 : inline static constexpr bool contains_potential_write_dependencies_v = contains_potential_write_dependencies<ParamsTy...>::value;
105 :
106 : template <typename Func, class T0, class T1, class... ParamsTy,
107 : typename=std::enable_if_t<
108 : std::conjunction_v<is_prio_or_proba<T0>, is_prio_or_proba<T1>, std::negation<std::is_same<std::decay_t<T0>, std::decay_t<T1>>>>>>
109 : auto callWithPartitionedArgs(Func&& f, T0&& t0, T1&& t1, ParamsTy&&... inParams) {
110 : static_assert(isSpeculativeTaskGraph, "SpTaskGraph::task of non speculative task graph should not be given a probability.");
111 : if constexpr(std::is_same_v<std::decay_t<T0>, SpProbability>) {
112 : return callWithPartitionedArgsStage2<true>(std::forward<Func>(f), std::forward<T1>(t1), std::forward<T0>(t0), std::forward<ParamsTy>(inParams)...);
113 : } else {
114 : return callWithPartitionedArgsStage2<true>(std::forward<Func>(f), std::forward<T0>(t0), std::forward<T1>(t1), std::forward<ParamsTy>(inParams)...);
115 : }
116 : }
117 :
118 : template <typename Func, class T0, class... ParamsTy,
119 : typename=std::enable_if_t<is_prio_or_proba_v<T0>>>
120 603 : auto callWithPartitionedArgs(Func&& f, T0&& t0, ParamsTy&&... inParams) {
121 : if constexpr(std::is_same_v<std::decay_t<T0>, SpProbability>) {
122 : static_assert(isSpeculativeTaskGraph, "SpTaskGraph::task of non speculative task graph should not be given a probability.");
123 : return callWithPartitionedArgsStage2<true>(std::forward<Func>(f), SpPriority(0), std::forward<T0>(t0), std::forward<ParamsTy>(inParams)...);
124 : }else {
125 603 : return callWithPartitionedArgsStage2<false>(std::forward<Func>(f), std::forward<T0>(t0), SpProbability(), std::forward<ParamsTy>(inParams)...);
126 : }
127 : }
128 :
129 : template <typename Func, class... ParamsTy>
130 361 : auto callWithPartitionedArgs(Func&& f, ParamsTy&&... inParams) {
131 : static_assert(sizeof...(inParams) > 0, "SpTaskGraph::task should be given at least a callable.");
132 361 : return callWithPartitionedArgsStage2<false>(std::forward<Func>(f), SpPriority(0), SpProbability(), std::forward<ParamsTy>(inParams)...);
133 : }
134 :
135 : template <class Tuple, size_t n, typename = std::make_index_sequence<n>>
136 : struct dispatchRotate {};
137 :
138 : template <class Tuple, size_t n, size_t... Is>
139 : struct dispatchRotate<Tuple, n, std::index_sequence<Is...>> {
140 : template <bool probabilityArgWasGivenByUser, typename Func, class T0, class T1, class T2, class T3>
141 961 : static auto doDispatch(SpTaskGraphCommon& tg, Func&& f, T0&& t0, T1&& t1, typename std::tuple_element<Is, Tuple>::type... args, T2&& t2, T3&& t3) {
142 : if constexpr(is_data_dependency_v<std::remove_reference_t<T2>>) {
143 : return tg.callWithPartitionedArgsStage3<probabilityArgWasGivenByUser>(std::forward<Func>(f), std::forward<T0>(t0), std::forward<T1>(t1),
144 957 : std::forward<T3>(t3), std::forward<typename std::tuple_element<Is, Tuple>::type>(args)..., std::forward<T2>(t2));
145 : } else {
146 : return tg.callWithPartitionedArgsStage3<probabilityArgWasGivenByUser>(std::forward<Func>(f), std::forward<T0>(t0), std::forward<T1>(t1),
147 4 : std::forward<T2>(t2), std::forward<T3>(t3), std::forward<typename std::tuple_element<Is, Tuple>::type>(args)...);
148 : }
149 : }
150 : };
151 :
152 : template <bool probabilityArgWasGivenByUser, typename Func, class T0, class T1, class... ParamsAndTask>
153 964 : auto callWithPartitionedArgsStage2(Func&& f, T0&& t0, T1&& t1, ParamsAndTask&&... inParamsAndTask){
154 : static_assert(sizeof...(ParamsAndTask) > 0, "SpTaskGraph::task should be given at least a callable.");
155 :
156 : using TupleTy = decltype(std::forward_as_tuple(std::forward<ParamsAndTask>(inParamsAndTask)...));
157 :
158 : if constexpr(sizeof...(ParamsAndTask) >= 2) {
159 1606 : return dispatchRotate<TupleTy, std::tuple_size<TupleTy>::value-2>::template doDispatch<probabilityArgWasGivenByUser>(*this,
160 1606 : std::forward<Func>(f), std::forward<T0>(t0), std::forward<T1>(t1), std::forward<ParamsAndTask>(inParamsAndTask)...);
161 : } else {
162 3 : return callWithPartitionedArgsStage3<probabilityArgWasGivenByUser>(std::forward<Func>(f), std::forward<T0>(t0), std::forward<T1>(t1),
163 3 : std::forward<ParamsAndTask>(inParamsAndTask)...);
164 : }
165 : }
166 :
167 : template <typename T>
168 968 : decltype(auto) wrapIfNotAlreadyWrapped(T&& callable) {
169 : if constexpr(is_instantiation_of_callable_wrapper_v<std::remove_reference_t<T>>) {
170 7 : return std::forward<T>(callable);
171 : } else {
172 961 : return SpCpu(std::forward<T>(callable));
173 : }
174 : }
175 :
176 : template <bool probabilityArgWasGivenByUser, typename Func, class T0, class T1, class T2, class T3, class... ParamsTy,
177 : typename = std::enable_if_t<std::conjunction_v<std::negation<is_data_dependency<std::remove_reference_t<T2>>>,
178 : std::negation<is_data_dependency<std::remove_reference_t<T3>>>>>>
179 4 : auto callWithPartitionedArgsStage3(Func&& f, T0&& t0, T1&& t1, T2&& t2, T3&& t3, ParamsTy&&... params) {
180 :
181 8 : auto dispatchStage4 = [&, this](auto&& c1, auto&& c2) {
182 : return this->callWithPartitionedArgsStage4<probabilityArgWasGivenByUser>(
183 12 : std::forward<Func>(f), std::forward<T0>(t0), std::forward<T1>(t1),
184 : std::forward<decltype(c1)>(c1), std::forward<decltype(c2)>(c2),
185 4 : std::forward<ParamsTy>(params)...);
186 : };
187 :
188 4 : auto&& c1 = wrapIfNotAlreadyWrapped(std::forward<T2>(t2));
189 4 : auto&& c2 = wrapIfNotAlreadyWrapped(std::forward<T3>(t3));
190 :
191 : if constexpr(is_instantiation_of_callable_wrapper_with_type_v<std::remove_reference_t<T2>, SpCallableType::GPU>) {
192 2 : dispatchStage4(std::forward<decltype(c2)>(c2), std::forward<decltype(c1)>(c1));
193 : } else {
194 2 : dispatchStage4(std::forward<decltype(c1)>(c1), std::forward<decltype(c2)>(c2));
195 : }
196 4 : }
197 :
198 : template <bool probabilityArgWasGivenByUser, typename Func, class T0, class T1, class T2, class... ParamsTy,
199 : typename = std::enable_if_t<std::negation_v<is_data_dependency<std::remove_reference_t<T2>>>>>
200 960 : auto callWithPartitionedArgsStage3(Func&& f, T0&& t0, T1&& t1, T2&& t2, ParamsTy&&... params) {
201 960 : auto&& c1 = wrapIfNotAlreadyWrapped(std::forward<T2>(t2));
202 :
203 960 : return callWithPartitionedArgsStage4<probabilityArgWasGivenByUser>(
204 : std::forward<Func>(f), std::forward<T0>(t0), std::forward<T1>(t1),
205 1919 : std::forward<decltype(c1)>(c1), std::forward<ParamsTy>(params)...);
206 : }
207 :
208 : template <bool probabilityArgWasGivenByUser, typename Func, class T0, class T1, class T2, class T3, class... ParamsTy,
209 : typename = std::enable_if_t<std::conjunction_v<is_instantiation_of_callable_wrapper<T2>, is_instantiation_of_callable_wrapper<T3>>>>
210 4 : auto callWithPartitionedArgsStage4(Func&& f, T0&& t0, T1&& t1, T2&& t2, [[maybe_unused]] T3&& t3, ParamsTy&&...params) {
211 : static_assert(std::conjunction_v<is_instantiation_of_callable_wrapper_with_type<std::remove_reference_t<T2>, SpCallableType::CPU>,
212 : is_instantiation_of_callable_wrapper_with_type<std::remove_reference_t<T3>, SpCallableType::GPU>>,
213 : "SpTaskGraph::task when providing two callables to a task one should be a CPU callable and the other a GPU callable");
214 :
215 : static_assert(std::conjunction_v<has_getView<ParamsTy>..., has_getAllData<ParamsTy>...>,
216 : "SpTaskGraph::task some data dependencies don't have a getView() and/or a getAllData method.");
217 :
218 : static_assert(std::is_invocable_v<decltype(t2.getCallableRef()), decltype(params.getView())...>,
219 : "SpTaskGraph::task Cpu callable is not invocable with data dependencies.");
220 :
221 4 : constexpr bool isPotentialTask = contains_potential_write_dependencies_v<ParamsTy...>;
222 :
223 : static_assert(isSpeculativeTaskGraph || !isPotentialTask, "SpTaskGraph::task of non speculative task graph should not be given potential-write data dependencies.");
224 :
225 : static_assert(!(probabilityArgWasGivenByUser && !isPotentialTask),
226 : "SpTaskGraph::task no probability should be specified for normal tasks.");
227 :
228 4 : auto dataDepTuple = std::forward_as_tuple(std::forward<ParamsTy>(params)...);
229 4 : auto callableTuple =
230 4 : [&](){
231 : if constexpr(SpConfig::CompileWithCuda) {
232 : static_assert(std::is_invocable_v<decltype(t3.getCallableRef()), decltype(params.getView())...>,
233 : "SpTaskGraph::task Gpu callable is not invocable with data dependencies.");
234 : return std::forward_as_tuple(std::forward<T2>(t2), std::forward<T3>(t3));
235 : } else {
236 4 : return std::forward_as_tuple(std::forward<T2>(t2));
237 : }
238 : }();
239 :
240 : if constexpr(isSpeculativeTaskGraph) {
241 4 : return std::invoke(std::forward<Func>(f), std::bool_constant<isPotentialTask>{}, std::forward<T0>(t0), std::forward<T1>(t1), dataDepTuple, callableTuple);
242 : } else {
243 : return std::invoke(std::forward<Func>(f), std::forward<T0>(t0), dataDepTuple, callableTuple);
244 : }
245 : }
246 :
247 : template <bool probabilityArgWasGivenByUser, typename Func, class T0, class T1, class T2, class... ParamsTy>
248 960 : auto callWithPartitionedArgsStage4(Func&& f, T0&& t0, T1&& t1, T2&& t2, ParamsTy&&...params) {
249 : static_assert(!(!SpConfig::CompileWithCuda && is_instantiation_of_callable_wrapper_with_type_v<std::remove_reference_t<T2>, SpCallableType::GPU>),
250 : "SpTaskGraph::task : SPETABARU_USE_CUDA macro is undefined. Unable to compile tasks for which only a GPU callable has been provided.");
251 :
252 : static_assert(std::conjunction_v<has_getView<ParamsTy>..., has_getAllData<ParamsTy>...>,
253 : "SpTaskGraph::task some data dependencies don't have a getView() and/or a getAllData method.");
254 :
255 : static_assert(std::is_invocable_v<decltype(t2.getCallableRef()), decltype(params.getView())...>,
256 : "SpTaskGraph::task callable is not invocable with data dependencies.");
257 :
258 960 : constexpr bool isPotentialTask = contains_potential_write_dependencies_v<ParamsTy...>;
259 :
260 : static_assert(isSpeculativeTaskGraph || !isPotentialTask, "SpTaskGraph::task of non speculative task graph should not be given potential-write data dependencies.");
261 :
262 : static_assert(!(probabilityArgWasGivenByUser && !isPotentialTask),
263 : "SpTaskGraph::task no probability should be specified for normal tasks.");
264 :
265 957 : auto dataDepTuple = std::forward_as_tuple(std::forward<ParamsTy>(params)...);
266 960 : auto callableTuple = std::forward_as_tuple(std::forward<T2>(t2));
267 :
268 : if constexpr(isSpeculativeTaskGraph) {
269 957 : return std::invoke(std::forward<Func>(f), std::bool_constant<isPotentialTask>{}, std::forward<T0>(t0), std::forward<T1>(t1), dataDepTuple, callableTuple);
270 : } else {
271 6 : return std::invoke(std::forward<Func>(f), std::forward<T0>(t0), dataDepTuple, callableTuple);
272 : }
273 : }
274 :
275 : template <typename CallableTy, typename TupleTy>
276 : static auto apply_on_data_dep_tuple(CallableTy&& c, TupleTy&& t) {
277 : return std::apply([&c](auto&&... elt) {
278 : return std::invoke(std::forward<CallableTy>(c), std::forward<decltype(elt)>(elt).getView()...);
279 : }, std::forward<TupleTy>(t));
280 : }
281 :
282 52 : explicit SpTaskGraphCommon() : allDataHandles() {}
283 :
284 : // No copy and no move
285 : SpTaskGraphCommon(const SpTaskGraphCommon&) = delete;
286 : SpTaskGraphCommon(SpTaskGraphCommon&&) = delete;
287 : SpTaskGraphCommon& operator=(const SpTaskGraphCommon&) = delete;
288 : SpTaskGraphCommon& operator=(SpTaskGraphCommon&&) = delete;
289 : };
290 :
291 : template <SpSpeculativeModel SpecModel = SpSpeculativeModel::SP_MODEL_1>
292 : class SpTaskGraph : public SpTaskGraphCommon<true>, public SpTaskManagerListener {
293 :
294 : static_assert(SpecModel == SpSpeculativeModel::SP_MODEL_1
295 : || SpecModel == SpSpeculativeModel::SP_MODEL_2
296 : || SpecModel == SpSpeculativeModel::SP_MODEL_3, "Should not happen");
297 :
298 : //=====----------------------------=====
299 : // Private API - members
300 : //=====----------------------------=====
301 : private:
302 :
303 : class SpAbstractDeleter{
304 : public:
305 282 : virtual ~SpAbstractDeleter(){}
306 : virtual void deleteObject(void* ptr) = 0;
307 : virtual void createDeleteTaskForObject(SpTaskGraph& tg, void* ptr) = 0;
308 : };
309 :
310 : template <class ObjectType>
311 : class SpDeleter : public SpAbstractDeleter{
312 : public:
313 282 : ~SpDeleter() = default;
314 :
315 108 : void deleteObject(void* ptr) override final{
316 108 : delete reinterpret_cast<ObjectType*>(ptr);
317 108 : }
318 :
319 21 : void createDeleteTaskForObject(SpTaskGraph<SpecModel>& tg, void* ptr) override final{
320 42 : tg.taskInternal(std::array<CopyMapPtrTy, 1>{std::addressof(tg.emptyCopyMap)}, SpTaskActivation::ENABLE, SpPriority(0),
321 21 : SpWrite(*reinterpret_cast<ObjectType*>(ptr)),
322 21 : [](ObjectType& output){
323 21 : delete &output;
324 62 : }).setTaskName("sp-delete");
325 21 : }
326 : };
327 :
328 : struct SpCurrentCopy{
329 300 : SpCurrentCopy()
330 : : originAdress(nullptr), sourceAdress(nullptr), latestAdress(nullptr), lastestSpecGroup(nullptr),
331 300 : latestCopyTask(nullptr), usedInRead(false), isUniquePtr(std::make_shared<bool>(true)) {
332 300 : }
333 :
334 282 : SpCurrentCopy(void* inOriginAddress, void* inSourceAddress, void* inLatestAddress, SpGeneralSpecGroup<SpecModel>* inLatestSpecGroup,
335 : SpAbstractTask* inLatestCopyTask, std::shared_ptr<SpAbstractDeleter> inDeleter)
336 : : originAdress(inOriginAddress), sourceAdress(inSourceAddress), latestAdress(inLatestAddress), lastestSpecGroup(inLatestSpecGroup),
337 282 : latestCopyTask(inLatestCopyTask), usedInRead(false), isUniquePtr(std::make_shared<bool>(true)), deleter(inDeleter) {}
338 :
339 580 : bool isDefined() const{
340 580 : return originAdress != nullptr;
341 : }
342 :
343 : void* originAdress;
344 : void* sourceAdress;
345 : void* latestAdress;
346 : SpGeneralSpecGroup<SpecModel>* lastestSpecGroup;
347 : SpAbstractTask* latestCopyTask;
348 : bool usedInRead;
349 : std::shared_ptr<bool> isUniquePtr;
350 : std::shared_ptr<SpAbstractDeleter> deleter;
351 : };
352 :
353 : using CopyMapTy = std::unordered_map<const void*, SpCurrentCopy>;
354 : using CopyMapPtrTy = CopyMapTy*;
355 :
356 : //! Current speculation group
357 : SpGeneralSpecGroup<SpecModel>* currentSpecGroup;
358 :
359 : //! List of all speculation groups that have been created
360 : std::list<std::unique_ptr<SpGeneralSpecGroup<SpecModel>>> specGroups;
361 :
362 : //! Mutex for spec group list
363 : std::mutex specGroupMutex;
364 :
365 : using ExecutionPathWeakPtrTy = std::weak_ptr<small_vector<CopyMapTy>>;
366 : using ExecutionPathSharedPtrTy = std::shared_ptr<small_vector<CopyMapTy>>;
367 :
368 : //! Map mapping original addresses to execution paths
369 : std::unordered_map<const void*, ExecutionPathSharedPtrTy> hashMap;
370 :
371 : //! Predicate function used to decide if a speculative task and any of its
372 : // dependent speculative tasks should be allowed to run
373 : std::function<bool(int,const SpProbability&)> specFormula;
374 :
375 : //! Empty map of copies
376 : CopyMapTy emptyCopyMap;
377 :
378 : //=====----------------------------=====
379 : // Private API - methods
380 : //=====----------------------------=====
381 : private:
382 :
383 : ///////////////////////////////////////////////////////////////////////////
384 : /// Cleanup
385 : ///////////////////////////////////////////////////////////////////////////
386 :
387 77 : void releaseCopies(small_vector_base<CopyMapTy>& copyMaps){
388 128 : for(auto& copyMapIt : copyMaps){
389 159 : for(auto& iter : copyMapIt) {
390 108 : assert(iter.second.latestAdress);
391 108 : assert(iter.second.deleter);
392 108 : if(iter.second.isUniquePtr.use_count() == 1) {
393 108 : iter.second.deleter->deleteObject(iter.second.latestAdress);
394 : }
395 : }
396 51 : copyMapIt.clear();
397 : }
398 77 : copyMaps.clear();
399 77 : }
400 :
401 : template <bool isSpeculative, class TaskCoreTy, std::size_t N>
402 1070 : void setDataHandlesOfTaskGeneric(TaskCoreTy* aTask, const std::array<CopyMapPtrTy, N>& copyMapsToLookInto){
403 1070 : auto& args = aTask->getDataDependencyTupleRef();
404 2938 : SpUtils::foreach_in_tuple(
405 1868 : [&, this, aTask](auto index, auto&& scalarOrContainerData) {
406 : using ScalarOrContainerType = std::remove_reference_t<decltype(scalarOrContainerData)>;
407 : using TargetParamType = typename ScalarOrContainerType::RawHandleType;
408 :
409 : static_assert(!isSpeculative || (std::is_default_constructible_v<TargetParamType> && std::is_copy_assignable_v<TargetParamType>),
410 : "They should all be default constructible here");
411 1868 : constexpr const SpDataAccessMode accessMode = ScalarOrContainerType::AccessMode;
412 :
413 3736 : auto hh = this->getDataHandle(scalarOrContainerData);
414 1752 : assert(ScalarOrContainerType::IsScalar == false || std::size(hh) == 1);
415 1868 : long int indexHh = 0;
416 :
417 4250 : for([[maybe_unused]] typename ScalarOrContainerType::HandleTypePtr ptr : scalarOrContainerData.getAllData()){
418 2382 : assert(ptr == this->getDataHandleCore(*ptr)->template castPtr<typename ScalarOrContainerType::RawHandleType>());
419 2382 : assert(ptr == hh[indexHh]->template castPtr<typename ScalarOrContainerType::RawHandleType>());
420 2382 : SpDataHandle* h = hh[indexHh];
421 :
422 2382 : bool foundAddressInCopies = false;
423 2382 : [[maybe_unused]] void *cpLatestAddress = nullptr;
424 :
425 : if constexpr(isSpeculative){
426 1269 : for(auto me : copyMapsToLookInto) {
427 1119 : if(auto found = me->find(ptr); found != me->end()){
428 290 : const SpCurrentCopy& cp = found->second;
429 290 : assert(cp.isDefined());
430 290 : assert(cp.originAdress == ptr);
431 290 : assert(cp.latestAdress != nullptr);
432 290 : h = this->getDataHandleCore(*reinterpret_cast<TargetParamType*>(cp.latestAdress));
433 290 : cpLatestAddress = cp.latestAdress;
434 290 : foundAddressInCopies = true;
435 290 : break;
436 : }
437 : }
438 : }
439 :
440 2382 : if(!foundAddressInCopies) {
441 2092 : SpDebugPrint() << "accessMode in runtime to add dependence -- => " << SpModeToStr(accessMode);
442 : }
443 :
444 2382 : const long int handleKey = h->addDependence(aTask, accessMode);
445 2382 : if(indexHh == 0){
446 1868 : aTask->template setDataHandle<index>(h, handleKey);
447 : }
448 : else{
449 0 : assert(ScalarOrContainerType::IsScalar == false);
450 514 : aTask->template addDataHandleExtra<index>(h, handleKey);
451 : }
452 :
453 : if constexpr(isSpeculative) {
454 440 : if(foundAddressInCopies) {
455 290 : aTask->template updatePtr<index>(indexHh, reinterpret_cast<TargetParamType*>(cpLatestAddress));
456 : }
457 : }
458 :
459 2382 : indexHh += 1;
460 : }
461 : }
462 : , args);
463 1070 : }
464 :
465 : template <class TaskCoreTy, std::size_t N>
466 961 : inline void setDataHandlesOfTask(TaskCoreTy* aTask, const std::array<CopyMapPtrTy, N>& copyMapsToLookInto){
467 961 : setDataHandlesOfTaskGeneric<false>(aTask, copyMapsToLookInto);
468 961 : }
469 :
470 : template <class TaskCoreTy, std::size_t N>
471 109 : inline void setDataHandlesOfTaskAndUpdateDataDepTupleOfTask(TaskCoreTy* aTask, const std::array<CopyMapPtrTy, N>& copyMapsToLookInto){
472 109 : setDataHandlesOfTaskGeneric<true>(aTask, copyMapsToLookInto);
473 109 : }
474 :
475 : ///////////////////////////////////////////////////////////////////////////
476 : /// Core task creation and task submission to scheduler
477 : ///////////////////////////////////////////////////////////////////////////
478 :
479 : //! Convert tuple to data and call the function
480 : //! Args is a value to allow for move or pass a rvalue reference
481 : template <template<typename...> typename TaskType, const bool isSpeculative, class DataDependencyTupleTy, class CallableTupleTy,
482 : std::size_t N, typename... T>
483 1070 : auto coreTaskCreationAux(const SpTaskActivation inActivation, const SpPriority& inPriority, DataDependencyTupleTy& dataDepTuple,
484 : CallableTupleTy& callableTuple, [[maybe_unused]] const std::array<CopyMapPtrTy, N>& copyMapsToLookInto, T... additionalArgs){
485 1070 : SpDebugPrint() << "SpTaskGraph -- coreTaskCreation";
486 :
487 : auto createCopyTupleFunc =
488 2140 : [](auto& tupleOfRefs) {
489 2140 : return [&tupleOfRefs]() {
490 : // Notice we create a tuple from lvalue references (we don't perfect forward to the
491 : // std::tuple_element_t<Is, std::remove_reference_t<DataDependencyTupleTy> type) because we want
492 : // to copy all data dependency objects into the new tuple (i.e. we don't want to move from the object even
493 : // if it is an rvalue because we might need to reference it again later on when we insert yet another
494 : // speculative version of the same task).
495 2143 : return std::apply([](const auto&...elt) {
496 2140 : return std::make_tuple(elt...);
497 2143 : }, tupleOfRefs);
498 4280 : };
499 : };
500 :
501 1070 : auto dataDependencyTupleCopyFunc = createCopyTupleFunc(dataDepTuple);
502 1070 : auto callableTupleCopyFunc = createCopyTupleFunc(callableTuple);
503 :
504 : static_assert(0 < std::tuple_size<decltype(callableTupleCopyFunc())>::value );
505 :
506 : using DataDependencyTupleCopyTy = std::remove_reference_t<decltype(dataDependencyTupleCopyFunc())>;
507 : using CallableTupleCopyTy = std::remove_reference_t<decltype(callableTupleCopyFunc())>;
508 : using RetTypeRef = decltype(apply_on_data_dep_tuple(std::get<0>(callableTuple).getCallableRef(), dataDepTuple));
509 : using RetType = std::remove_reference_t<RetTypeRef>;
510 : using TaskTy = TaskType<RetType, DataDependencyTupleCopyTy, CallableTupleCopyTy>;
511 :
512 : // Create a task with a copy of the callables and the data dependency objects
513 1070 : auto aTask = new TaskTy(this, inActivation, inPriority, dataDependencyTupleCopyFunc(), callableTupleCopyFunc(), additionalArgs...);
514 :
515 : // Lock the task
516 1070 : aTask->takeControl();
517 :
518 : // Add the handles
519 : if constexpr(!isSpeculative) {
520 961 : setDataHandlesOfTask(aTask, copyMapsToLookInto);
521 : } else {
522 109 : setDataHandlesOfTaskAndUpdateDataDepTupleOfTask(aTask, copyMapsToLookInto);
523 : }
524 :
525 : // Check coherency
526 1070 : assert(aTask->areDepsCorrect());
527 :
528 : // The task has been initialized
529 1070 : aTask->setState(SpTaskState::INITIALIZED);
530 :
531 : // Get the view
532 1070 : auto descriptor = aTask->getViewer();
533 :
534 1070 : aTask->setState(SpTaskState::WAITING_TO_BE_READY);
535 :
536 1070 : if(currentSpecGroup){
537 282 : currentSpecGroup->addCopyTask(aTask);
538 282 : aTask->setSpecGroup(currentSpecGroup);
539 : }
540 :
541 1070 : aTask->releaseControl();
542 :
543 1070 : SpDebugPrint() << "SpTaskGraph -- coreTaskCreation => " << aTask << " of id " << aTask->getId();
544 :
545 : // Push to the scheduler
546 1070 : this->scheduler.addNewTask(aTask);
547 :
548 : // Return the view
549 1070 : return descriptor;
550 : }
551 :
552 : template <std::size_t N, class DataDependencyTupleTy, class CallableTupleTy>
553 358 : inline auto coreTaskCreation(const std::array<CopyMapPtrTy, N>& copyMapsToLookInto,
554 : const SpTaskActivation inActivation,
555 : const SpPriority& inPriority,
556 : DataDependencyTupleTy&& dataDepTuple,
557 : CallableTupleTy&& callableTuple){
558 358 : return coreTaskCreationAux<SpTask, false>(inActivation, inPriority,
559 358 : std::forward<DataDependencyTupleTy>(dataDepTuple), std::forward<CallableTupleTy>(callableTuple), copyMapsToLookInto);
560 : }
561 :
562 : template <std::size_t N, class DataDependencyTupleTy, class CallableTupleTy>
563 109 : inline auto coreTaskCreationSpeculative(const std::array<CopyMapPtrTy, N>& copyMapsToLookInto,
564 : const SpTaskActivation inActivation,
565 : const SpPriority& inPriority,
566 : DataDependencyTupleTy&& dataDepTuple,
567 : CallableTupleTy&& callableTuple) {
568 109 : return coreTaskCreationAux<SpTask, true>(inActivation, inPriority,
569 109 : std::forward<DataDependencyTupleTy>(dataDepTuple), std::forward<CallableTupleTy>(callableTuple), copyMapsToLookInto);
570 : }
571 :
572 : ///////////////////////////////////////////////////////////////////////////
573 : /// Core logic (task insertion for the different speculative models)
574 : ///////////////////////////////////////////////////////////////////////////
575 :
576 : template <typename T>
577 : struct SpRange{
578 : T beginIt;
579 : T endIt;
580 : T currentIt;
581 12 : SpRange(T inBeginIt, T inEndIt, T inCurrentIt) : beginIt(inBeginIt), endIt(inEndIt), currentIt(inCurrentIt) {}
582 : };
583 :
584 : template <const bool isPotentialTask, class CallableTupleTy, class DataDependencyTupleTy>
585 358 : auto preCoreTaskCreation(const SpPriority& inPriority, const SpProbability& inProbability, DataDependencyTupleTy&& dataDepTuple, CallableTupleTy&& callableTuple) {
586 716 : std::unique_lock<std::mutex> lock(specGroupMutex);
587 :
588 358 : bool isPathResultingFromMerge = false;
589 716 : auto executionPaths = getCorrespondingExecutionPaths(std::forward<DataDependencyTupleTy>(dataDepTuple));
590 :
591 358 : ExecutionPathSharedPtrTy e;
592 :
593 716 : small_vector<const void *> originalAddresses;
594 :
595 358 : constexpr const auto potentialWriteFlags = SpDataAccessModeField(1) << static_cast<SpDataAccessModeField>(SpDataAccessMode::POTENTIAL_WRITE);
596 :
597 358 : constexpr const auto writeFlags = SpDataAccessModeField(1) << static_cast<SpDataAccessModeField>(SpDataAccessMode::WRITE)
598 : | SpDataAccessModeField(1) << static_cast<SpDataAccessModeField>(SpDataAccessMode::COMMUTATIVE_WRITE)
599 : | SpDataAccessModeField(1) << static_cast<SpDataAccessModeField>(SpDataAccessMode::PARALLEL_WRITE);
600 :
601 716 : auto originalAddressesOfPotentiallyWrittenHandles = getOriginalAddressesOfHandlesWithAccessModes<potentialWriteFlags>(std::forward<DataDependencyTupleTy>(dataDepTuple));
602 358 : auto originalAddressesOfWrittenHandles = getOriginalAddressesOfHandlesWithAccessModes<writeFlags>(std::forward<DataDependencyTupleTy>(dataDepTuple));
603 :
604 358 : if(executionPaths.empty()) {
605 280 : e = std::make_shared<small_vector<CopyMapTy>>();
606 78 : }else if(executionPaths.size() == 1){
607 72 : e = executionPaths[0].lock();
608 : }else{ // merge case
609 6 : isPathResultingFromMerge = true;
610 6 : e = std::make_shared<small_vector<CopyMapTy>>();
611 :
612 : using ExecutionPathIteratorVectorTy = small_vector<SpRange<typename small_vector_base<CopyMapTy>::iterator>>;
613 :
614 12 : ExecutionPathIteratorVectorTy vectorExecutionPaths;
615 :
616 18 : for(auto &ep : executionPaths) {
617 12 : vectorExecutionPaths.push_back({ep.lock()->begin(), ep.lock()->end(), ep.lock()->begin()});
618 : }
619 :
620 6 : auto it = vectorExecutionPaths.begin();
621 :
622 6 : while(true) {
623 :
624 12 : e->emplace_back();
625 12 : auto speculationBranchIt = e->end()-1;
626 :
627 36 : for(auto& ep : vectorExecutionPaths) {
628 24 : if(ep.currentIt != ep.endIt) {
629 35 : for(auto mIt = ep.currentIt->cbegin(); mIt != ep.currentIt->cend();) {
630 : if constexpr(SpecModel == SpSpeculativeModel::SP_MODEL_3) {
631 9 : if(isUsedByTask(mIt->first, std::forward<DataDependencyTupleTy>(dataDepTuple))) {
632 8 : (*speculationBranchIt)[mIt->first] = mIt->second;
633 8 : originalAddresses.push_back(mIt->first);
634 8 : mIt++;
635 : }else{
636 1 : originalAddresses.push_back(mIt->first);
637 1 : mIt->second.deleter->createDeleteTaskForObject(*this, mIt->second.latestAdress);
638 1 : mIt = ep.currentIt->erase(mIt);
639 : }
640 : }else {
641 10 : (*speculationBranchIt)[mIt->first] = mIt->second;
642 10 : originalAddresses.push_back(mIt->first);
643 10 : mIt++;
644 : }
645 : }
646 : }
647 : }
648 :
649 12 : if(speculationBranchIt->empty()) {
650 2 : e->pop_back();
651 : }
652 :
653 : if constexpr(SpecModel != SpSpeculativeModel::SP_MODEL_3) {
654 4 : break;
655 : }
656 :
657 14 : while(it != vectorExecutionPaths.end() && it->currentIt == it->endIt) {
658 6 : it->currentIt = it->beginIt;
659 6 : it++;
660 : }
661 :
662 8 : if(it != vectorExecutionPaths.end()) {
663 6 : it->currentIt++;
664 6 : it = vectorExecutionPaths.begin();
665 : }else {
666 2 : break;
667 : }
668 : }
669 :
670 6 : std::sort(originalAddresses.begin(), originalAddresses.end());
671 6 : originalAddresses.erase(std::unique(originalAddresses.begin(), originalAddresses.end()), originalAddresses.end());
672 :
673 : if constexpr(SpecModel == SpSpeculativeModel::SP_MODEL_1) {
674 2 : setExecutionPathForOriginalAddressesInHashMap(e, originalAddresses);
675 : }else if constexpr(SpecModel == SpSpeculativeModel::SP_MODEL_3) {
676 2 : removeOriginalAddressesFromHashMap(originalAddresses);
677 : }
678 : }
679 :
680 : if constexpr(SpecModel == SpSpeculativeModel::SP_MODEL_2) {
681 47 : removeOriginalAddressesFromHashMap(originalAddresses);
682 : }
683 :
684 358 : auto res = preCoreTaskCreationAux<isPotentialTask>(isPathResultingFromMerge, *e, inPriority, inProbability,
685 : std::forward<DataDependencyTupleTy>(dataDepTuple), std::forward<CallableTupleTy>(callableTuple));
686 :
687 : if constexpr(isPotentialTask) {
688 98 : setExecutionPathForOriginalAddressesInHashMap(e, originalAddressesOfPotentiallyWrittenHandles);
689 : }
690 :
691 : if constexpr(SpecModel == SpSpeculativeModel::SP_MODEL_1) {
692 266 : removeOriginalAddressesFromHashMap(originalAddressesOfWrittenHandles);
693 : }else if constexpr(SpecModel == SpSpeculativeModel::SP_MODEL_3) {
694 45 : if(executionPaths.size() == 1) {
695 23 : removeOriginalAddressesFromHashMap(originalAddressesOfWrittenHandles);
696 : }
697 : }
698 :
699 716 : return res;
700 : }
701 :
702 : template <const bool isPotentialTask, class DataDependencyTupleTy, class CallableTupleTy>
703 358 : auto preCoreTaskCreationAux([[maybe_unused]] bool pathResultsFromMerge, small_vector_base<CopyMapTy> ©Maps, const SpPriority& inPriority,
704 : const SpProbability& inProbability, DataDependencyTupleTy &&dataDepTuple, CallableTupleTy &&callableTuple) {
705 :
706 : static_assert(SpecModel == SpSpeculativeModel::SP_MODEL_1
707 : || SpecModel == SpSpeculativeModel::SP_MODEL_2
708 : || SpecModel == SpSpeculativeModel::SP_MODEL_3, "Should not happen");
709 :
710 358 : auto it = copyMaps.begin();
711 :
712 : if constexpr (!isPotentialTask && allAreCopieableAndDeletable<DataDependencyTupleTy>() == false){
713 131 : for(; it != copyMaps.end(); it++){
714 : if constexpr(SpecModel == SpSpeculativeModel::SP_MODEL_2) {
715 : for(auto& e : *it) {
716 : e.second.deleter->createDeleteTaskForObject(*this, e.second.latestAdress);
717 : }
718 : it->clear();
719 : }else {
720 0 : manageReadDuplicate(*it, std::forward<DataDependencyTupleTy>(dataDepTuple));
721 0 : removeAllCorrespondingCopies(*it, std::forward<DataDependencyTupleTy>(dataDepTuple));
722 :
723 : if constexpr(SpecModel == SpSpeculativeModel::SP_MODEL_3) {
724 : if(pathResultsFromMerge) {
725 : removeAllCopiesReadFrom(*it, std::forward<DataDependencyTupleTy>(dataDepTuple));
726 : }
727 : }
728 : }
729 : }
730 131 : return coreTaskCreation(std::array<CopyMapPtrTy, 1>{std::addressof(emptyCopyMap)},
731 : SpTaskActivation::ENABLE,
732 : inPriority,
733 : std::forward<DataDependencyTupleTy>(dataDepTuple),
734 262 : std::forward<CallableTupleTy>(callableTuple));
735 : } else {
736 : static_assert(allAreCopieableAndDeletable<DataDependencyTupleTy>() == true,
737 : "Add data passed to a potential task must be copiable");
738 :
739 : using TaskViewTy = decltype(coreTaskCreation(std::array<CopyMapPtrTy, 1>{std::addressof(emptyCopyMap)},
740 : std::declval<SpTaskActivation>(), inPriority,
741 : std::forward<DataDependencyTupleTy>(dataDepTuple),
742 : std::forward<CallableTupleTy>(callableTuple)));
743 :
744 454 : small_vector<TaskViewTy> speculativeTasks;
745 454 : small_vector<std::function<void()>> selectTaskCreationFunctions;
746 :
747 454 : std::shared_ptr<std::atomic<size_t>> numberOfSpeculativeSiblingSpecGroupsCounter = std::make_shared<std::atomic<size_t>>(0);
748 :
749 338 : for(; it != copyMaps.end(); ++it) {
750 :
751 222 : std::unordered_map<const void*, SpCurrentCopy> l1, l2, l1p;
752 :
753 222 : auto groups = getCorrespondingCopyGroups(*it, std::forward<DataDependencyTupleTy>(dataDepTuple));
754 111 : bool oneGroupDisableOrFailed = false;
755 :
756 288 : for(auto gp : groups){
757 177 : if(gp->isSpeculationDisable() || gp->didSpeculationFailed() || gp->didParentSpeculationFailed()){
758 0 : oneGroupDisableOrFailed = true;
759 0 : break;
760 : }
761 : }
762 :
763 111 : const bool taskAlsoSpeculateOnOther = (groups.size() != 0 && !oneGroupDisableOrFailed);
764 :
765 111 : if(!taskAlsoSpeculateOnOther) {
766 2 : manageReadDuplicate(*it, std::forward<DataDependencyTupleTy>(dataDepTuple));
767 2 : removeAllCorrespondingCopies(*it, std::forward<DataDependencyTupleTy>(dataDepTuple));
768 :
769 : if constexpr(SpecModel == SpSpeculativeModel::SP_MODEL_3) {
770 0 : if(pathResultsFromMerge) {
771 0 : removeAllCopiesReadFrom(*it, std::forward<DataDependencyTupleTy>(dataDepTuple));
772 : }
773 : }
774 : } else {
775 109 : (*numberOfSpeculativeSiblingSpecGroupsCounter)++;
776 109 : std::unique_ptr<SpGeneralSpecGroup<SpecModel>> specGroupSpecTask = std::make_unique<SpGeneralSpecGroup<SpecModel>>
777 : (
778 109 : true,
779 : numberOfSpeculativeSiblingSpecGroupsCounter
780 : );
781 109 : specGroupSpecTask->addParents(groups);
782 :
783 : if constexpr(isPotentialTask) {
784 85 : specGroupSpecTask->setProbability(inProbability);
785 85 : currentSpecGroup = specGroupSpecTask.get();
786 85 : l1 = copyIfPotentialWriteAndNotDuplicateOrUsedInRead(std::array<CopyMapPtrTy, 1>{std::addressof(*it)},
787 : specGroupSpecTask->getActivationStateForCopyTasks(),
788 : inPriority,
789 : std::forward<DataDependencyTupleTy>(dataDepTuple));
790 85 : assert(taskAlsoSpeculateOnOther == true || l1.size());
791 85 : currentSpecGroup = nullptr;
792 : }
793 :
794 109 : currentSpecGroup = specGroupSpecTask.get();
795 109 : l2 = copyIfWriteAndNotDuplicateOrUsedInRead(std::array<CopyMapPtrTy, 1>{std::addressof(*it)},
796 : specGroupSpecTask->getActivationStateForCopyTasks(),
797 : inPriority,
798 : std::forward<DataDependencyTupleTy>(dataDepTuple));
799 109 : currentSpecGroup = nullptr;
800 :
801 : if constexpr(isPotentialTask && SpecModel != SpSpeculativeModel::SP_MODEL_2) {
802 65 : currentSpecGroup = specGroupSpecTask.get();
803 65 : l1p = copyIfPotentialWriteAndDuplicate(std::array<CopyMapPtrTy, 3>{std::addressof(l1), std::addressof(l2), std::addressof(*it)},
804 : specGroupSpecTask->getActivationStateForCopyTasks(),
805 : inPriority, std::forward<DataDependencyTupleTy>(dataDepTuple));
806 65 : currentSpecGroup = nullptr;
807 : }
808 :
809 109 : TaskViewTy taskViewSpec = coreTaskCreationSpeculative(std::array<CopyMapPtrTy, 3>{std::addressof(l1), std::addressof(l2), std::addressof(*it)},
810 : specGroupSpecTask->getActivationStateForSpeculativeTask(),
811 : inPriority,
812 : std::forward<DataDependencyTupleTy>(dataDepTuple),
813 : std::forward<CallableTupleTy>(callableTuple));
814 :
815 109 : specGroupSpecTask->setSpecTask(taskViewSpec.getTaskPtr());
816 109 : taskViewSpec.getTaskPtr()->setSpecGroup(specGroupSpecTask.get());
817 109 : speculativeTasks.push_back(taskViewSpec);
818 :
819 218 : auto functions = createSelectTaskCreationFunctions(std::array<CopyMapPtrTy, 3>{std::addressof(l1), std::addressof(l2), std::addressof(*it)},
820 : specGroupSpecTask.get(),
821 : inPriority,
822 : std::forward<DataDependencyTupleTy>(dataDepTuple));
823 :
824 109 : selectTaskCreationFunctions.reserve(selectTaskCreationFunctions.size() + functions.size());
825 109 : selectTaskCreationFunctions.insert(selectTaskCreationFunctions.end(), functions.begin(), functions.end());
826 :
827 109 : manageReadDuplicate(*it, std::forward<DataDependencyTupleTy>(dataDepTuple));
828 109 : removeAllCorrespondingCopies(*it, std::forward<DataDependencyTupleTy>(dataDepTuple));
829 :
830 : if constexpr(SpecModel == SpSpeculativeModel::SP_MODEL_3) {
831 58 : if(pathResultsFromMerge) {
832 6 : removeAllCopiesReadFrom(*it, std::forward<DataDependencyTupleTy>(dataDepTuple));
833 : }
834 : }
835 :
836 109 : specGroups.emplace_back(std::move(specGroupSpecTask));
837 :
838 : if constexpr(isPotentialTask && SpecModel != SpSpeculativeModel::SP_MODEL_2) {
839 65 : it->merge(l1p);
840 : }
841 : }
842 : }
843 :
844 : if constexpr(SpecModel != SpSpeculativeModel::SP_MODEL_3) {
845 182 : if(copyMaps.empty()) {
846 129 : copyMaps.emplace_back();
847 129 : it = copyMaps.end()-1;
848 : } else {
849 53 : it = copyMaps.begin();
850 : }
851 : }else {
852 45 : copyMaps.emplace_back();
853 45 : it = copyMaps.end()-1;
854 : }
855 :
856 : if constexpr(SpecModel == SpSpeculativeModel::SP_MODEL_2) {
857 67 : for(auto& e : *it) {
858 20 : e.second.deleter->createDeleteTaskForObject(*this, e.second.latestAdress);
859 : }
860 47 : it->clear();
861 : }
862 :
863 0 : std::unique_ptr<SpGeneralSpecGroup<SpecModel>> specGroupNormalTask = std::make_unique<SpGeneralSpecGroup<SpecModel>>
864 : (
865 227 : !speculativeTasks.empty(),
866 : numberOfSpeculativeSiblingSpecGroupsCounter
867 : );
868 :
869 227 : specGroupNormalTask->setSpeculationActivation(true);
870 :
871 : if constexpr(isPotentialTask) {
872 98 : specGroupNormalTask->setProbability(inProbability);
873 :
874 98 : std::unordered_map<const void*, SpCurrentCopy> l1p;
875 :
876 98 : currentSpecGroup = specGroupNormalTask.get();
877 : if constexpr(SpecModel == SpSpeculativeModel::SP_MODEL_1) {
878 32 : if(speculativeTasks.empty()) {
879 13 : l1p = copyIfPotentialWriteAndDuplicate(std::array<CopyMapPtrTy, 1>{std::addressof(emptyCopyMap)},
880 : specGroupNormalTask->getActivationStateForCopyTasks(),
881 : inPriority,
882 : std::forward<DataDependencyTupleTy>(dataDepTuple));
883 13 : it->merge(l1p);
884 : }
885 : }else{
886 66 : l1p = copyIfPotentialWriteAndDuplicate(std::array<CopyMapPtrTy, 1>{std::addressof(emptyCopyMap)},
887 : specGroupNormalTask->getActivationStateForCopyTasks(),
888 : inPriority,
889 : std::forward<DataDependencyTupleTy>(dataDepTuple));
890 66 : it->merge(l1p);
891 : }
892 98 : currentSpecGroup = nullptr;
893 : }
894 :
895 227 : TaskViewTy result = coreTaskCreation(std::array<CopyMapPtrTy, 1>{std::addressof(emptyCopyMap)},
896 : specGroupNormalTask->getActivationStateForMainTask(),
897 : inPriority,
898 : std::forward<DataDependencyTupleTy>(dataDepTuple),
899 : std::forward<CallableTupleTy>(callableTuple));
900 :
901 227 : specGroupNormalTask->setMainTask(result.getTaskPtr());
902 227 : result.getTaskPtr()->setSpecGroup(specGroupNormalTask.get());
903 :
904 336 : for(auto& t : speculativeTasks) {
905 109 : SpGeneralSpecGroup<SpecModel>* sg = t.getTaskPtr()->template getSpecGroup<SpGeneralSpecGroup<SpecModel>>();
906 109 : sg->setMainTask(result.getTaskPtr());
907 109 : t.setOriginalTask(result.getTaskPtr());
908 : }
909 :
910 374 : for(auto& f : selectTaskCreationFunctions) {
911 147 : f();
912 : }
913 :
914 : if constexpr(isPotentialTask) {
915 183 : for(auto& t : speculativeTasks) {
916 85 : SpGeneralSpecGroup<SpecModel>* sg = t.getTaskPtr()->template getSpecGroup<SpGeneralSpecGroup<SpecModel>>();
917 85 : addCallbackToTask(t, sg);
918 : }
919 :
920 98 : addCallbackToTask(result, specGroupNormalTask.get());
921 : }
922 :
923 227 : specGroups.emplace_back(std::move(specGroupNormalTask));
924 :
925 454 : return result;
926 : }
927 : }
928 :
929 : ///////////////////////////////////////////////////////////////////////////
930 : /// Select task creation
931 : ///////////////////////////////////////////////////////////////////////////
932 :
933 : template <class Tuple, std::size_t N>
934 109 : auto createSelectTaskCreationFunctions(const std::array<CopyMapPtrTy, N>& copyMapsToLookInto,
935 : [[maybe_unused]] SpGeneralSpecGroup<SpecModel>* sg,
936 : const SpPriority& inPriority,
937 : Tuple& args){
938 109 : small_vector<std::function<void()>> res;
939 :
940 349 : SpUtils::foreach_in_tuple(
941 240 : [&, this](auto&& scalarOrContainerData) {
942 : using ScalarOrContainerType = std::remove_reference_t<decltype(scalarOrContainerData)>;
943 :
944 : using TargetParamType = typename ScalarOrContainerType::RawHandleType;
945 :
946 : static_assert(std::is_default_constructible<TargetParamType>::value && std::is_copy_assignable<TargetParamType>::value,
947 : "They should all be default constructible here");
948 :
949 240 : constexpr const SpDataAccessMode accessMode = ScalarOrContainerType::AccessMode;
950 :
951 480 : auto hh = this->getDataHandle(scalarOrContainerData);
952 :
953 190 : assert(ScalarOrContainerType::IsScalar == false || std::size(hh) == 1);
954 :
955 240 : long int indexHh = 0;
956 :
957 680 : for(typename ScalarOrContainerType::HandleTypePtr ptr : scalarOrContainerData.getAllData()){
958 440 : assert(ptr == this->getDataHandleCore(*ptr)->template castPtr<typename ScalarOrContainerType::RawHandleType>());
959 440 : assert(ptr == hh[indexHh]->template castPtr<typename ScalarOrContainerType::RawHandleType>());
960 440 : [[maybe_unused]] SpDataHandle* h1 = hh[indexHh];
961 1269 : for(auto me : copyMapsToLookInto) {
962 1119 : if(auto found = me->find(ptr); found != me->end()){
963 290 : const SpCurrentCopy& cp = found->second;
964 290 : assert(cp.isDefined());
965 290 : assert(cp.originAdress == ptr);
966 290 : assert(cp.latestAdress != nullptr);
967 :
968 147 : assert(accessMode == SpDataAccessMode::READ || found->second.usedInRead == false);
969 :
970 147 : if constexpr(accessMode != SpDataAccessMode::READ){
971 147 : const bool isCarryingSurelyWrittenValuesOver = accessMode == SpDataAccessMode::WRITE;
972 147 : void* const cpLatestAddress = cp.latestAdress;
973 :
974 588 : auto s = [this, &inPriority, h1, cpLatestAddress, sg]() {
975 :
976 147 : SpDataHandle* h1copy = this->getDataHandleCore(*reinterpret_cast<TargetParamType*>(cpLatestAddress));
977 :
978 294 : auto taskViewSelect = this->taskInternalSpSelect(
979 147 : std::array<CopyMapPtrTy, 1>{std::addressof(emptyCopyMap)},
980 : isCarryingSurelyWrittenValuesOver,
981 : sg->getActivationStateForSelectTask(isCarryingSurelyWrittenValuesOver),
982 : inPriority,
983 147 : SpWrite(*h1->castPtr<TargetParamType>()),
984 147 : SpWrite(*h1copy->castPtr<TargetParamType>()),
985 43 : [](TargetParamType& output, TargetParamType& input){
986 43 : output = std::move(input);
987 : }
988 : );
989 :
990 147 : taskViewSelect.setTaskName("sp-select");
991 147 : taskViewSelect.getTaskPtr()->setSpecGroup(sg);
992 147 : sg->addSelectTask(taskViewSelect.getTaskPtr());
993 :
994 : // delete copied data carried over by select task
995 147 : auto taskViewDelete = this->taskInternal(std::array<CopyMapPtrTy, 1>{std::addressof(emptyCopyMap)},
996 : SpTaskActivation::ENABLE,
997 : SpPriority(0),
998 147 : SpWrite(*reinterpret_cast<TargetParamType*>(cpLatestAddress)),
999 147 : [](TargetParamType& output){
1000 147 : delete &output;
1001 : });
1002 :
1003 147 : taskViewDelete.setTaskName("sp-delete");
1004 147 : taskViewDelete.getTaskPtr()->setSpecGroup(sg);
1005 : };
1006 :
1007 147 : res.push_back(s);
1008 :
1009 : // delete copy from copy map
1010 147 : me->erase(found);
1011 : } else{
1012 143 : found->second.usedInRead = true;
1013 : }
1014 :
1015 290 : break;
1016 : }
1017 :
1018 : }
1019 :
1020 440 : indexHh += 1;
1021 : }
1022 : }
1023 : , args);
1024 :
1025 109 : return res;
1026 : }
1027 :
1028 : ///////////////////////////////////////////////////////////////////////////
1029 : /// Copy task creation
1030 : ///////////////////////////////////////////////////////////////////////////
1031 :
1032 : //! Copy an object and return this related info (the task is created and submited)
1033 : template <class ObjectType>
1034 282 : SpCurrentCopy coreCopyCreationCore(std::unordered_map<const void*, SpCurrentCopy>& copyMapToLookInto,
1035 : const SpTaskActivation initialActivationState,
1036 : const SpPriority& inPriority,
1037 : ObjectType& objectToCopy) {
1038 :
1039 : using TargetParamType = typename std::remove_reference<ObjectType>::type;
1040 :
1041 : static_assert(std::is_default_constructible<TargetParamType>::value && std::is_copy_assignable<TargetParamType>::value,
1042 : "Potentially written to data must be copiable");
1043 :
1044 282 : TargetParamType* ptr = new TargetParamType();
1045 282 : const TargetParamType* originPtr = &objectToCopy;
1046 282 : const TargetParamType* sourcePtr = originPtr;
1047 :
1048 : // Use the latest version of the data
1049 282 : if(auto found = copyMapToLookInto.find(originPtr) ; found != copyMapToLookInto.end()){
1050 91 : assert(found->second.latestAdress);
1051 91 : sourcePtr = reinterpret_cast<TargetParamType*>(found->second.latestAdress);
1052 : }
1053 :
1054 282 : SpDebugPrint() << "SpTaskGraph -- coreCopyCreationCore -- setup copy from " << sourcePtr << " to " << &ptr;
1055 561 : SpAbstractTaskWithReturn<void>::SpTaskViewer taskView = taskInternal(std::array<CopyMapPtrTy, 1>{std::addressof(copyMapToLookInto)},
1056 : initialActivationState,
1057 : inPriority,
1058 282 : SpWrite(*ptr),
1059 282 : SpRead(*sourcePtr),
1060 282 : [](TargetParamType& output, const TargetParamType& input){
1061 282 : SpDebugPrint() << "SpTaskGraph -- coreCopyCreationCore -- execute copy from " << &input << " to " << &output;
1062 282 : output = input;
1063 : });
1064 :
1065 282 : taskView.setTaskName("sp-copy");
1066 :
1067 : return SpCurrentCopy(
1068 : const_cast<TargetParamType*>(originPtr), // .originAdress
1069 : const_cast<TargetParamType*>(sourcePtr), // .sourceAdress
1070 : ptr, // .latestAdress
1071 : currentSpecGroup, // .lastestSpecGroup
1072 : taskView.getTaskPtr(), // .latestCopyTask
1073 : std::make_shared<SpDeleter<TargetParamType>>() // .deleter
1074 282 : );
1075 : }
1076 :
1077 : template <bool copyIfAlreadyDuplicate, bool copyIfUsedInRead, SpDataAccessMode targetMode, class Tuple, std::size_t... Is, std::size_t N>
1078 338 : auto copyIfAccess(const std::array<CopyMapPtrTy, N>& copyMapsToLookInto,
1079 : const SpTaskActivation initialActivationState,
1080 : const SpPriority& inPriority,
1081 : const Tuple& args){
1082 : static_assert(N > 0, "coreCopyIfAccess -- You must provide at least one copy map for inspection.");
1083 :
1084 338 : std::unordered_map<const void*, SpCurrentCopy> res;
1085 :
1086 661 : SpUtils::foreach_in_tuple(
1087 723 : [&, this, initialActivationState](auto&& scalarOrContainerData) {
1088 : using ScalarOrContainerType = std::remove_reference_t<decltype(scalarOrContainerData)>;
1089 : using TargetParamType = typename ScalarOrContainerType::RawHandleType;
1090 :
1091 723 : constexpr const SpDataAccessMode accessMode = ScalarOrContainerType::AccessMode;
1092 :
1093 282 : if constexpr (accessMode == targetMode){
1094 : static_assert(std::is_default_constructible<TargetParamType>::value
1095 : && std::is_copy_assignable<TargetParamType>::value,
1096 : "Data must be copiable");
1097 :
1098 646 : auto hh = this->getDataHandle(scalarOrContainerData);
1099 323 : assert(ScalarOrContainerType::IsScalar == false || std::size(hh) == 1);
1100 :
1101 323 : long int indexHh = 0;
1102 646 : for([[maybe_unused]] typename ScalarOrContainerType::HandleTypePtr ptr : scalarOrContainerData.getAllData()){
1103 323 : assert(ptr == this->getDataHandleCore(*ptr)->template castPtr<typename ScalarOrContainerType::RawHandleType>());
1104 323 : assert(hh[indexHh]->template castPtr<typename ScalarOrContainerType::RawHandleType>() == ptr);
1105 323 : SpDataHandle* h1 = hh[indexHh];
1106 :
1107 323 : bool doCopy = false;
1108 323 : std::unordered_map<const void*, SpCurrentCopy>* mPtr = nullptr;
1109 :
1110 : if constexpr (copyIfAlreadyDuplicate) { // always copy regardless of the fact that the data might have been previously copied
1111 176 : doCopy = true;
1112 283 : for(auto me : copyMapsToLookInto) {
1113 188 : mPtr = me;
1114 188 : if(me->find(h1->castPtr<TargetParamType>()) != me->end()) {
1115 81 : break;
1116 : }
1117 : }
1118 : }else if constexpr (copyIfUsedInRead){ // if data has already been previously copied then only copy if the previous copy is used in read
1119 147 : bool hasBeenFound = false;
1120 243 : for(auto me : copyMapsToLookInto) {
1121 147 : mPtr = me;
1122 :
1123 147 : if(auto found = me->find(h1->castPtr<TargetParamType>()); found != me->end()) {
1124 51 : hasBeenFound = true;
1125 51 : doCopy = found->second.usedInRead || found->second.isUniquePtr.use_count() > 1 || *(found->second.isUniquePtr) == false;
1126 51 : break;
1127 : }
1128 : }
1129 147 : if(!hasBeenFound) {
1130 96 : doCopy = true;
1131 : }
1132 : }else{ // if none of the above has been triggered, copy the data only if it has not already been duplicated
1133 : doCopy = true;
1134 : for(auto me : copyMapsToLookInto) {
1135 : doCopy &= me->find(h1->castPtr<TargetParamType>()) == me->end();
1136 : }
1137 :
1138 : mPtr = copyMapsToLookInto.back();
1139 : }
1140 :
1141 323 : if(doCopy) {
1142 282 : auto copy = this->coreCopyCreationCore(*mPtr, initialActivationState, inPriority, *h1->castPtr<TargetParamType>());
1143 282 : res[copy.originAdress] = copy;
1144 : }
1145 :
1146 323 : indexHh += 1;
1147 : }
1148 :
1149 : }
1150 : }
1151 : , args);
1152 :
1153 338 : return res;
1154 : }
1155 :
1156 : template <class Tuple, std::size_t N>
1157 85 : inline auto copyIfPotentialWriteAndNotDuplicateOrUsedInRead(const std::array<CopyMapPtrTy, N>& copyMapsToLookInto,
1158 : const SpTaskActivation initialActivationState,
1159 : const SpPriority& inPriority,
1160 : Tuple& args){
1161 85 : return copyIfAccess<false, true, SpDataAccessMode::POTENTIAL_WRITE>(copyMapsToLookInto, initialActivationState, inPriority, args);
1162 : }
1163 :
1164 : template <class Tuple, std::size_t N>
1165 144 : inline auto copyIfPotentialWriteAndDuplicate(const std::array<CopyMapPtrTy, N>& copyMapsToLookInto,
1166 : const SpTaskActivation initialActivationState,
1167 : const SpPriority& inPriority,
1168 : Tuple& args){
1169 144 : return copyIfAccess<true, false, SpDataAccessMode::POTENTIAL_WRITE>(copyMapsToLookInto, initialActivationState, inPriority, args);
1170 : }
1171 :
1172 : template <class Tuple, std::size_t N>
1173 109 : inline auto copyIfWriteAndNotDuplicateOrUsedInRead(const std::array<CopyMapPtrTy, N>& copyMapsToLookInto,
1174 : const SpTaskActivation initialActivationState,
1175 : const SpPriority& inPriority,
1176 : Tuple& args){
1177 109 : return copyIfAccess<false, true, SpDataAccessMode::WRITE>(copyMapsToLookInto, initialActivationState, inPriority, args);
1178 : }
1179 :
1180 : ///////////////////////////////////////////////////////////////////////////
1181 : /// Core logic helper functions
1182 : ///////////////////////////////////////////////////////////////////////////
1183 : template <typename T>
1184 183 : void addCallbackToTask(T& inTaskView, SpGeneralSpecGroup<SpecModel>* inSg) {
1185 671 : inTaskView.addCallback([this, aTaskPtr = inTaskView.getTaskPtr(), specGroupPtr = inSg]
1186 : (const bool alreadyDone, const bool& taskRes, SpAbstractTaskWithReturn<bool>::SpTaskViewer& /*view*/,
1187 183 : const bool isEnabled){
1188 183 : if(isEnabled){
1189 110 : if(!alreadyDone){
1190 101 : specGroupMutex.lock();
1191 : }
1192 :
1193 110 : specGroupPtr->setSpeculationCurrentResult(!taskRes);
1194 :
1195 110 : if(!alreadyDone){
1196 101 : specGroupMutex.unlock();
1197 : }
1198 : }
1199 : });
1200 183 : }
1201 :
1202 : template <class Tuple>
1203 358 : auto getCorrespondingExecutionPaths(Tuple& args){
1204 :
1205 358 : small_vector<ExecutionPathWeakPtrTy> res;
1206 :
1207 358 : if(hashMap.empty()) {
1208 271 : return res;
1209 : }
1210 :
1211 267 : SpUtils::foreach_in_tuple(
1212 180 : [this, &res](auto&& scalarOrContainerData) {
1213 : using ScalarOrContainerType = std::remove_reference_t<decltype(scalarOrContainerData)>;
1214 :
1215 360 : [[maybe_unused]] auto hh = this->getDataHandle(scalarOrContainerData);
1216 150 : assert(ScalarOrContainerType::IsScalar == false || std::size(hh) == 1);
1217 180 : long int indexHh = 0;
1218 480 : for(typename ScalarOrContainerType::HandleTypePtr ptr : scalarOrContainerData.getAllData()){
1219 300 : assert(ptr == this->getDataHandleCore(*ptr)->template castPtr<typename ScalarOrContainerType::RawHandleType>());
1220 300 : assert(hh[indexHh]->template castPtr<typename ScalarOrContainerType::RawHandleType>() == ptr);
1221 :
1222 300 : if(auto found = hashMap.find(ptr); found != hashMap.end()){
1223 167 : res.push_back(found->second);
1224 : }
1225 :
1226 300 : indexHh += 1;
1227 : }
1228 : }
1229 : , args);
1230 :
1231 175 : auto sortLambda = [] (ExecutionPathWeakPtrTy& a, ExecutionPathWeakPtrTy& b) {
1232 175 : return a.lock() < b.lock();
1233 : };
1234 :
1235 89 : auto uniqueLambda = [] (ExecutionPathWeakPtrTy& a, ExecutionPathWeakPtrTy& b) {
1236 89 : return a.lock() == b.lock();
1237 : };
1238 :
1239 87 : std::sort(res.begin(), res.end(), sortLambda);
1240 87 : res.erase(std::unique(res.begin(), res.end(), uniqueLambda), res.end());
1241 :
1242 87 : return res;
1243 : }
1244 :
1245 : template <const SpDataAccessModeField flags, class Tuple>
1246 716 : auto getOriginalAddressesOfHandlesWithAccessModes(Tuple& args) {
1247 :
1248 716 : small_vector<const void*> res;
1249 :
1250 1191 : SpUtils::foreach_in_tuple(
1251 1192 : [&, this](auto&& scalarOrContainerData) {
1252 : using ScalarOrContainerType = std::remove_reference_t<decltype(scalarOrContainerData)>;
1253 :
1254 1192 : constexpr const SpDataAccessMode accessMode = ScalarOrContainerType::AccessMode;
1255 :
1256 : if constexpr((flags & (SpDataAccessModeField(1) << static_cast<SpDataAccessModeField>(accessMode))) != 0) {
1257 890 : [[maybe_unused]] auto hh = this->getDataHandle(scalarOrContainerData);
1258 445 : assert(ScalarOrContainerType::IsScalar == false || std::size(hh) == 1);
1259 445 : long int indexHh = 0;
1260 890 : for(typename ScalarOrContainerType::HandleTypePtr ptr : scalarOrContainerData.getAllData()){
1261 445 : assert(ptr == this->getDataHandleCore(*ptr)->template castPtr<typename ScalarOrContainerType::RawHandleType>());
1262 445 : assert(hh[indexHh]->template castPtr<typename ScalarOrContainerType::RawHandleType>() == ptr);
1263 :
1264 445 : res.push_back(ptr);
1265 :
1266 445 : indexHh += 1;
1267 : }
1268 : }
1269 : }
1270 : , args);
1271 :
1272 716 : return res;
1273 : }
1274 :
1275 100 : void setExecutionPathForOriginalAddressesInHashMap(ExecutionPathSharedPtrTy &ep, small_vector_base<const void*>& originalAddresses){
1276 224 : for(auto oa : originalAddresses) {
1277 124 : hashMap[oa] = ep;
1278 : }
1279 100 : }
1280 :
1281 338 : void removeOriginalAddressesFromHashMap(small_vector_base<const void*>& originalAddresses){
1282 651 : for(auto oa : originalAddresses) {
1283 313 : hashMap.erase(oa);
1284 : }
1285 338 : }
1286 :
1287 : template <class Tuple>
1288 9 : bool isUsedByTask(const void* inPtr, Tuple& args) {
1289 9 : bool res = false;
1290 :
1291 73 : SpUtils::foreach_in_tuple(
1292 14 : [this, &res, inPtr](auto&& scalarOrContainerData) {
1293 : using ScalarOrContainerType = std::remove_reference_t<decltype(scalarOrContainerData)>;
1294 :
1295 14 : [[maybe_unused]] auto hh = this->getDataHandle(scalarOrContainerData);
1296 14 : assert(ScalarOrContainerType::IsScalar == false || std::size(hh) == 1);
1297 14 : [[maybe_unused]] long int indexHh = 0;
1298 20 : for(typename ScalarOrContainerType::HandleTypePtr ptr : scalarOrContainerData.getAllData()){
1299 14 : assert(ptr == this->getDataHandleCore(*ptr)->template castPtr<typename ScalarOrContainerType::RawHandleType>());
1300 14 : assert(hh[indexHh]->template castPtr<typename ScalarOrContainerType::RawHandleType>() == ptr);
1301 14 : if(ptr == inPtr) {
1302 8 : res = true;
1303 8 : break;
1304 : }
1305 : }
1306 :
1307 28 : return res;
1308 : }
1309 : , args);
1310 :
1311 9 : return res;
1312 : }
1313 :
1314 : template <class Tuple>
1315 111 : void manageReadDuplicate(std::unordered_map<const void*, SpCurrentCopy>& copyMap, Tuple& args){
1316 262 : SpUtils::foreach_in_tuple(
1317 244 : [&, this](auto&& scalarOrContainerData) {
1318 : using ScalarOrContainerType = std::remove_reference_t<decltype(scalarOrContainerData)>;
1319 : using TargetParamType = typename ScalarOrContainerType::RawHandleType;
1320 :
1321 244 : constexpr const SpDataAccessMode accessMode = ScalarOrContainerType::AccessMode;
1322 :
1323 306 : if constexpr (std::is_destructible<TargetParamType>::value && accessMode != SpDataAccessMode::READ){
1324 302 : [[maybe_unused]] auto hh = this->getDataHandle(scalarOrContainerData);
1325 151 : assert(ScalarOrContainerType::IsScalar == false || std::size(hh) == 1);
1326 151 : long int indexHh = 0;
1327 302 : for(typename ScalarOrContainerType::HandleTypePtr ptr : scalarOrContainerData.getAllData()){
1328 151 : assert(ptr == this->getDataHandleCore(*ptr)->template castPtr<typename ScalarOrContainerType::RawHandleType>());
1329 151 : assert(hh[indexHh]->template castPtr<typename ScalarOrContainerType::RawHandleType>() == ptr);
1330 :
1331 151 : if(auto found = copyMap.find(ptr); found != copyMap.end()
1332 151 : && found->second.usedInRead){
1333 0 : assert(std::is_copy_assignable<TargetParamType>::value);
1334 :
1335 2 : SpCurrentCopy& cp = found->second;
1336 2 : assert(cp.latestAdress != ptr);
1337 2 : assert(cp.latestAdress);
1338 2 : if(found->second.isUniquePtr.use_count() == 1) {
1339 4 : this->taskInternal(std::array<CopyMapPtrTy, 1>{std::addressof(copyMap)}, SpTaskActivation::ENABLE, SpPriority(0),
1340 2 : SpWrite(*reinterpret_cast<TargetParamType*>(cp.latestAdress)),
1341 2 : [](TargetParamType& output){
1342 :
1343 2 : delete &output;
1344 8 : }).setTaskName("sp-delete");
1345 : }else {
1346 0 : *(found->second.isUniquePtr) = false;
1347 : }
1348 2 : copyMap.erase(found);
1349 : }
1350 :
1351 151 : indexHh += 1;
1352 : }
1353 : }
1354 : }
1355 : , args);
1356 111 : }
1357 :
1358 : template <class Tuple>
1359 111 : auto getCorrespondingCopyGroups(std::unordered_map<const void*, SpCurrentCopy>& copyMap, Tuple& args){
1360 111 : small_vector<SpGeneralSpecGroup<SpecModel>*> res;
1361 :
1362 355 : SpUtils::foreach_in_tuple(
1363 244 : [this, ©Map, &res](auto &&scalarOrContainerData) {
1364 : using ScalarOrContainerType = std::remove_reference_t<decltype(scalarOrContainerData)>;
1365 :
1366 488 : [[maybe_unused]] auto hh = this->getDataHandle(scalarOrContainerData);
1367 194 : assert(ScalarOrContainerType::IsScalar == false || std::size(hh) == 1);
1368 244 : long int indexHh = 0;
1369 688 : for(typename ScalarOrContainerType::HandleTypePtr ptr : scalarOrContainerData.getAllData()){
1370 444 : assert(ptr == this->getDataHandleCore(*ptr)->template castPtr<typename ScalarOrContainerType::RawHandleType>());
1371 444 : assert(hh[indexHh]->template castPtr<typename ScalarOrContainerType::RawHandleType>() == ptr);
1372 :
1373 444 : if(auto found = copyMap.find(ptr); found != copyMap.end()){
1374 194 : assert(found->second.lastestSpecGroup);
1375 194 : res.emplace_back(found->second.lastestSpecGroup);
1376 : }
1377 :
1378 444 : indexHh += 1;
1379 : }
1380 : }
1381 : , args);
1382 :
1383 111 : std::sort(res.begin(), res.end());
1384 111 : res.erase(std::unique(res.begin(), res.end()), res.end());
1385 :
1386 111 : return res;
1387 : }
1388 :
1389 : template <bool updateIsUniquePtr, class Tuple>
1390 117 : void removeAllGeneric(std::unordered_map<const void*, SpCurrentCopy>& copyMapToLookInto, Tuple& args){
1391 280 : SpUtils::foreach_in_tuple(
1392 256 : [&, this](auto&& scalarOrContainerData) {
1393 : using ScalarOrContainerType = std::remove_reference_t<decltype(scalarOrContainerData)>;
1394 : using TargetParamType = typename ScalarOrContainerType::RawHandleType;
1395 :
1396 256 : constexpr const SpDataAccessMode accessMode = ScalarOrContainerType::AccessMode;
1397 :
1398 338 : if constexpr (std::is_destructible<TargetParamType>::value && accessMode != SpDataAccessMode::READ){
1399 :
1400 326 : [[maybe_unused]] auto hh = this->getDataHandle(scalarOrContainerData);
1401 163 : assert(ScalarOrContainerType::IsScalar == false || std::size(hh) == 1);
1402 163 : long int indexHh = 0;
1403 326 : for(typename ScalarOrContainerType::HandleTypePtr ptr : scalarOrContainerData.getAllData()){
1404 163 : assert(ptr == this->getDataHandleCore(*ptr)->template castPtr<typename ScalarOrContainerType::RawHandleType>());
1405 163 : assert(hh[indexHh]->template castPtr<typename ScalarOrContainerType::RawHandleType>() == ptr);
1406 :
1407 163 : if(auto found = copyMapToLookInto.find(ptr); found != copyMapToLookInto.end()){
1408 0 : assert(std::is_copy_assignable<TargetParamType>::value);
1409 8 : SpCurrentCopy& cp = found->second;
1410 8 : if(found->second.isUniquePtr.use_count() == 1) {
1411 4 : assert(cp.latestAdress);
1412 : #ifndef NDEBUG
1413 8 : this->taskInternal(std::array<CopyMapPtrTy, 1>{std::addressof(copyMapToLookInto)},
1414 : SpTaskActivation::ENABLE,
1415 : SpPriority(0),
1416 0 : SpWrite(*reinterpret_cast<TargetParamType*>(cp.latestAdress)),
1417 8 : [ptr = cp.latestAdress](TargetParamType& output){
1418 0 : assert(ptr == &output);
1419 4 : delete &output;
1420 16 : }).setTaskName("sp-delete");
1421 : #else
1422 : this->taskInternal(std::array<CopyMapPtrTy, 1>{std::addressof(copyMapToLookInto)},
1423 : SpTaskActivation::ENABLE,
1424 : SpPriority(0),
1425 : SpWrite(*reinterpret_cast<TargetParamType*>(cp.latestAdress)),
1426 : [](TargetParamType& output){
1427 : delete &output;
1428 : }).setTaskName("sp-delete");
1429 : #endif
1430 : }else {
1431 : if constexpr(updateIsUniquePtr) {
1432 4 : *(found->second.isUniquePtr) = false;
1433 : }
1434 : }
1435 8 : copyMapToLookInto.erase(found);
1436 : }
1437 :
1438 163 : indexHh += 1;
1439 : }
1440 : }
1441 : }
1442 : , args);
1443 117 : }
1444 :
1445 : template <class Tuple>
1446 6 : void removeAllCopiesReadFrom(std::unordered_map<const void*, SpCurrentCopy>& copyMapToLookInto, Tuple& args){
1447 6 : removeAllGeneric<false>(copyMapToLookInto, args);
1448 6 : }
1449 :
1450 : template <class Tuple>
1451 111 : void removeAllCorrespondingCopies(std::unordered_map<const void*, SpCurrentCopy>& copyMapToLookInto, Tuple& args){
1452 111 : removeAllGeneric<true>(copyMapToLookInto, args);
1453 111 : }
1454 :
1455 : template <class Tuple, std::size_t... Is>
1456 : static constexpr bool coreAllAreCopieableAndDeletable(std::index_sequence<Is...>){
1457 : return ([]() {
1458 : using ScalarOrContainerType = std::remove_reference_t<std::tuple_element_t<Is, Tuple>>;
1459 : using TargetParamType = typename ScalarOrContainerType::RawHandleType;
1460 :
1461 : return std::conjunction_v<std::is_default_constructible<TargetParamType>,
1462 : std::is_copy_assignable<TargetParamType>,
1463 : std::is_destructible<TargetParamType>>;
1464 : }() && ...);
1465 : }
1466 :
1467 : template <class Tuple>
1468 : static constexpr bool allAreCopieableAndDeletable(){
1469 : using TupleWithoutRef = std::remove_reference_t<Tuple>;
1470 : return coreAllAreCopieableAndDeletable<TupleWithoutRef>(std::make_index_sequence<std::tuple_size_v<TupleWithoutRef>>{});
1471 : }
1472 :
1473 : ///////////////////////////////////////////////////////////////////////////
1474 : /// Internal task creation
1475 : ///////////////////////////////////////////////////////////////////////////
1476 :
1477 : template <template<typename...> typename TaskType, class... ParamsAndTask, std::size_t N>
1478 456 : auto taskInternal(const std::array<CopyMapPtrTy, N>& copyMapsToLookInto,
1479 : const SpTaskActivation inActivation,
1480 : ParamsAndTask&&... inParamsAndTask){
1481 1368 : auto f = [&, this, inActivation] (auto isPotentialTask, auto&& priority, auto&& /* unused probability */, auto&&... partitionedParams) {
1482 174 : return this->coreTaskCreationAux<TaskType, isPotentialTask>(inActivation, std::forward<decltype(priority)>(priority),
1483 630 : std::forward<decltype(partitionedParams)>(partitionedParams)..., copyMapsToLookInto);
1484 : };
1485 :
1486 912 : return this->callWithPartitionedArgs(f, std::forward<ParamsAndTask>(inParamsAndTask)...);
1487 : }
1488 :
1489 :
1490 : template <class... ParamsAndTask, std::size_t N>
1491 456 : inline auto taskInternal(const std::array<CopyMapPtrTy, N>& copyMapsToLookInto,
1492 : const SpTaskActivation inActivation,
1493 : ParamsAndTask&&... inParamsAndTask){
1494 456 : return taskInternal<SpTask>(copyMapsToLookInto, inActivation, std::forward<ParamsAndTask>(inParamsAndTask)...);
1495 : }
1496 :
1497 : template <class... ParamsAndTask, std::size_t N>
1498 147 : auto taskInternalSpSelect(const std::array<CopyMapPtrTy, N>& copyMapsToLookInto,
1499 : bool isCarryingSurelyWrittenValuesOver,
1500 : const SpTaskActivation inActivation,
1501 : ParamsAndTask&&... inParamsAndTask){
1502 441 : auto f =
1503 : [&, this, isCarryingSurelyWrittenValuesOver, inActivation]
1504 147 : (auto isPotentialTask, auto&& priority, auto&& /* unused probability */, auto&&... partitionedParams) {
1505 : return this->coreTaskCreationAux<SpSelectTask, isPotentialTask>(inActivation, std::forward<decltype(priority)>(priority),
1506 147 : std::forward<decltype(partitionedParams)>(partitionedParams)..., copyMapsToLookInto,
1507 147 : isCarryingSurelyWrittenValuesOver);
1508 : };
1509 :
1510 294 : return this->callWithPartitionedArgs(f, std::forward<ParamsAndTask>(inParamsAndTask)...);
1511 : }
1512 :
1513 : ///////////////////////////////////////////////////////////////////////////
1514 : /// Notify function (called by scheduler when a task is ready to run)
1515 : ///////////////////////////////////////////////////////////////////////////
1516 :
1517 1070 : void thisTaskIsReady(SpAbstractTask* aTask, const bool isNotCalledInAContextOfTaskCreation) final {
1518 1070 : SpDebugPrint() << "SpTaskGraph -- thisTaskIsReady -- will test ";
1519 :
1520 1070 : if(isNotCalledInAContextOfTaskCreation){
1521 734 : specGroupMutex.lock();
1522 : }
1523 :
1524 1070 : SpGeneralSpecGroup<SpecModel>* specGroup = aTask->getSpecGroup<SpGeneralSpecGroup<SpecModel>>();
1525 :
1526 1070 : if(specGroup && specGroup->isSpeculationNotSet()){
1527 0 : if(specFormula){
1528 0 : if(specFormula(this->scheduler.getNbReadyTasks(), specGroup->getAllProbability())){
1529 0 : SpDebugPrint() << "SpTaskGraph -- thisTaskIsReady -- enableSpeculation ";
1530 0 : specGroup->setSpeculationActivation(true);
1531 : }
1532 : else{
1533 0 : specGroup->setSpeculationActivation(false);
1534 : }
1535 : }
1536 0 : else if(this->scheduler.getNbReadyTasks() == 0){
1537 0 : SpDebugPrint() << "SpTaskGraph -- thisTaskIsReady -- enableSpeculation ";
1538 0 : specGroup->setSpeculationActivation(true);
1539 : }
1540 : else{
1541 0 : specGroup->setSpeculationActivation(false);
1542 : }
1543 :
1544 0 : assert(!specGroup->isSpeculationNotSet());
1545 : }
1546 :
1547 1070 : if(isNotCalledInAContextOfTaskCreation){
1548 734 : specGroupMutex.unlock();
1549 : }
1550 1070 : }
1551 :
1552 : //=====--------------------=====
1553 : // Public API
1554 : //=====--------------------=====
1555 : public:
1556 : ///////////////////////////////////////////////////////////////////////////
1557 : /// Constructor
1558 : ///////////////////////////////////////////////////////////////////////////
1559 :
1560 51 : explicit SpTaskGraph() : currentSpecGroup(nullptr) {
1561 51 : this->scheduler.setListener(this);
1562 51 : }
1563 :
1564 : ///////////////////////////////////////////////////////////////////////////
1565 : /// Destructor
1566 : ///////////////////////////////////////////////////////////////////////////
1567 :
1568 : //! Destructor waits for tasks to finish
1569 51 : ~SpTaskGraph(){
1570 51 : this->waitAllTasks();
1571 : // free copies
1572 128 : for(auto &e : hashMap) {
1573 77 : releaseCopies(*(e.second));
1574 : }
1575 102 : }
1576 :
1577 : // No copy and no move
1578 : SpTaskGraph(const SpTaskGraph&) = delete;
1579 : SpTaskGraph(SpTaskGraph&&) = delete;
1580 : SpTaskGraph& operator=(const SpTaskGraph&) = delete;
1581 : SpTaskGraph& operator=(SpTaskGraph&&) = delete;
1582 :
1583 : ///////////////////////////////////////////////////////////////////////////
1584 : /// Task creation method
1585 : ///////////////////////////////////////////////////////////////////////////
1586 :
1587 : template <class... ParamsTy>
1588 358 : auto task(ParamsTy&&...params) {
1589 716 : auto f = [this](auto isPotentialTask, auto&&... partitionedParams) {
1590 358 : return this->preCoreTaskCreation<isPotentialTask>(std::forward<decltype(partitionedParams)>(partitionedParams)...);
1591 : };
1592 716 : return this->callWithPartitionedArgs(f, std::forward<ParamsTy>(params)...);
1593 : }
1594 :
1595 : ///////////////////////////////////////////////////////////////////////////
1596 : /// Getters/actions
1597 : ///////////////////////////////////////////////////////////////////////////
1598 :
1599 31 : void setSpeculationTest(std::function<bool(int,const SpProbability&)> inFormula){
1600 31 : specFormula = std::move(inFormula);
1601 31 : }
1602 : };
1603 :
1604 : template<>
1605 : class SpTaskGraph<SpSpeculativeModel::SP_NO_SPEC> : public SpTaskGraphCommon<false> {
1606 :
1607 : private:
1608 :
1609 : template <class TaskCoreTy>
1610 3 : void setDataHandlesOfTask(TaskCoreTy* aTask) {
1611 3 : auto& args = aTask->getDataDependencyTupleRef();
1612 7 : SpUtils::foreach_in_tuple(
1613 4 : [&, this, aTask](auto index, auto&& scalarOrContainerData) {
1614 : using ScalarOrContainerType = std::remove_reference_t<decltype(scalarOrContainerData)>;
1615 4 : constexpr const SpDataAccessMode accessMode = ScalarOrContainerType::AccessMode;
1616 :
1617 8 : auto hh = this->getDataHandle(scalarOrContainerData);
1618 4 : assert(ScalarOrContainerType::IsScalar == false || std::size(hh) == 1);
1619 4 : long int indexHh = 0;
1620 :
1621 8 : for([[maybe_unused]] typename ScalarOrContainerType::HandleTypePtr ptr : scalarOrContainerData.getAllData()){
1622 4 : assert(ptr == this->getDataHandleCore(*ptr)->template castPtr<typename ScalarOrContainerType::RawHandleType>());
1623 4 : assert(ptr == hh[indexHh]->template castPtr<typename ScalarOrContainerType::RawHandleType>());
1624 4 : SpDataHandle* h = hh[indexHh];
1625 :
1626 4 : SpDebugPrint() << "accessMode in runtime to add dependence -- => " << SpModeToStr(accessMode);
1627 :
1628 4 : const long int handleKey = h->addDependence(aTask, accessMode);
1629 4 : if(indexHh == 0){
1630 4 : aTask->template setDataHandle<index>(h, handleKey);
1631 : }
1632 : else{
1633 0 : assert(ScalarOrContainerType::IsScalar == false);
1634 : aTask->template addDataHandleExtra<index>(h, handleKey);
1635 : }
1636 4 : indexHh += 1;
1637 : }
1638 : }
1639 : , args);
1640 3 : }
1641 :
1642 : template <typename CallableTy, typename TupleTy, std::size_t... Is>
1643 6 : static auto apply_with_forward_impl(CallableTy&& c, TupleTy&& t, std::index_sequence<Is...>) {
1644 : using TupleTyWithoutRef = std::remove_reference_t<TupleTy>;
1645 :
1646 : return std::invoke(std::forward<CallableTy>(c),
1647 6 : std::forward<std::tuple_element_t<Is, TupleTyWithoutRef>>(std::get<Is>(t))...);
1648 : }
1649 :
1650 : template <typename CallableTy, typename TupleTy>
1651 6 : static auto apply_with_forward(CallableTy&& c, TupleTy&& t) {
1652 : using TupleTyWithoutRef = std::remove_reference_t<TupleTy>;
1653 : return apply_with_forward_impl(std::forward<CallableTy>(c),
1654 : std::forward<TupleTy>(t),
1655 6 : std::make_index_sequence<std::tuple_size_v<TupleTyWithoutRef>>{});
1656 : }
1657 :
1658 : template <class DataDependencyTupleTy, class CallableTupleTy>
1659 3 : auto coreTaskCreation(const SpPriority& inPriority, DataDependencyTupleTy& dataDepTuple, CallableTupleTy& callableTuple) {
1660 3 : SpDebugPrint() << "SpTaskGraph -- coreTaskCreation";
1661 :
1662 : auto createCopyTupleFunc =
1663 6 : [](auto& tupleOfRefs) {
1664 6 : return [&tupleOfRefs]() {
1665 : // Here we forward to std::tuple_element_t<Is, std::remove_reference_t<DataDependencyTupleTy>> in order
1666 : // to move the data dependency object into the new tuple when the data dependency object is an rvalue.
1667 : // Since this is a non speculative task graph we will only insert one instance of each task and so we can safely
1668 : // move from the data dependency object since we won't reference it again afterwards.
1669 6 : return apply_with_forward([](auto&&...elt) {
1670 6 : return std::make_tuple(std::forward<decltype(elt)>(elt)...);
1671 6 : }, tupleOfRefs);
1672 12 : };
1673 : };
1674 :
1675 3 : auto dataDependencyTupleCopyFunc = createCopyTupleFunc(dataDepTuple);
1676 3 : auto callableTupleCopyFunc = createCopyTupleFunc(callableTuple);
1677 :
1678 : static_assert(0 < std::tuple_size<decltype(callableTupleCopyFunc())>::value );
1679 :
1680 : using DataDependencyTupleCopyTy = std::remove_reference_t<decltype(dataDependencyTupleCopyFunc())>;
1681 : using CallableTupleCopyTy = std::remove_reference_t<decltype(callableTupleCopyFunc())>;
1682 : using RetTypeRef = decltype(apply_on_data_dep_tuple(std::get<0>(callableTuple).getCallableRef(), dataDepTuple));
1683 : using RetType = std::remove_reference_t<RetTypeRef>;
1684 : using TaskTy = SpTask<RetType, DataDependencyTupleCopyTy, CallableTupleCopyTy>;
1685 :
1686 : // Create a task with a copy of the callables and the data dependency objects
1687 3 : auto aTask = new TaskTy(this, SpTaskActivation::ENABLE, inPriority, dataDependencyTupleCopyFunc(), callableTupleCopyFunc());
1688 :
1689 : // Lock the task
1690 3 : aTask->takeControl();
1691 :
1692 : // Add the handles
1693 3 : setDataHandlesOfTask(aTask);
1694 :
1695 : // Check coherency
1696 3 : assert(aTask->areDepsCorrect());
1697 :
1698 : // The task has been initialized
1699 3 : aTask->setState(SpTaskState::INITIALIZED);
1700 :
1701 : // Get the view
1702 3 : auto descriptor = aTask->getViewer();
1703 :
1704 3 : aTask->setState(SpTaskState::WAITING_TO_BE_READY);
1705 :
1706 3 : aTask->releaseControl();
1707 :
1708 3 : SpDebugPrint() << "SpTaskGraph -- coreTaskCreation => " << aTask << " of id " << aTask->getId();
1709 :
1710 : // Push to the scheduler
1711 3 : this->scheduler.addNewTask(aTask);
1712 :
1713 : // Return the view
1714 3 : return descriptor;
1715 : }
1716 :
1717 : //=====--------------------=====
1718 : // Public API
1719 : //=====--------------------=====
1720 : public:
1721 : ///////////////////////////////////////////////////////////////////////////
1722 : /// Constructor
1723 : ///////////////////////////////////////////////////////////////////////////
1724 :
1725 1 : explicit SpTaskGraph() {}
1726 :
1727 : ///////////////////////////////////////////////////////////////////////////
1728 : /// Destructor
1729 : ///////////////////////////////////////////////////////////////////////////
1730 :
1731 : //! Destructor waits for tasks to finish
1732 1 : ~SpTaskGraph(){
1733 1 : this->waitAllTasks();
1734 1 : }
1735 :
1736 : // No copy and no move
1737 : SpTaskGraph(const SpTaskGraph&) = delete;
1738 : SpTaskGraph(SpTaskGraph&&) = delete;
1739 : SpTaskGraph& operator=(const SpTaskGraph&) = delete;
1740 : SpTaskGraph& operator=(SpTaskGraph&&) = delete;
1741 :
1742 : ///////////////////////////////////////////////////////////////////////////
1743 : /// Task creation method
1744 : ///////////////////////////////////////////////////////////////////////////
1745 :
1746 : template <class... ParamsTy>
1747 3 : auto task(ParamsTy&&...params) {
1748 6 : auto f = [this](auto&&... partitionedParams) {
1749 3 : return this->coreTaskCreation(std::forward<decltype(partitionedParams)>(partitionedParams)...);
1750 : };
1751 6 : return this->callWithPartitionedArgs(f, std::forward<ParamsTy>(params)...);
1752 : }
1753 : };
1754 :
1755 : #endif
|
