49 static void __kmp_enable_tasking( kmp_task_team_t *task_team, kmp_info_t *this_thr );
50 static void __kmp_alloc_task_deque( kmp_info_t *thread, kmp_thread_data_t *thread_data );
51 static int __kmp_realloc_task_threads_data( kmp_info_t *thread, kmp_task_team_t *task_team );
54 # define __kmp_static_delay( arg )
58 __kmp_static_delay(
int arg )
61 # if KMP_ARCH_X86_64 && KMP_OS_LINUX
62 KMP_ASSERT( arg != 0 );
64 KMP_ASSERT( arg >= 0 );
70 __kmp_static_yield(
int arg )
75 #ifdef BUILD_TIED_TASK_STACK
87 __kmp_trace_task_stack( kmp_int32 gtid, kmp_thread_data_t *thread_data,
int threshold,
char *location )
89 kmp_task_stack_t *task_stack = & thread_data->td.td_susp_tied_tasks;
90 kmp_taskdata_t **stack_top = task_stack -> ts_top;
91 kmp_int32 entries = task_stack -> ts_entries;
92 kmp_taskdata_t *tied_task;
94 KA_TRACE(threshold, (
"__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, "
95 "first_block = %p, stack_top = %p \n",
96 location, gtid, entries, task_stack->ts_first_block, stack_top ) );
98 KMP_DEBUG_ASSERT( stack_top != NULL );
99 KMP_DEBUG_ASSERT( entries > 0 );
101 while ( entries != 0 )
103 KMP_DEBUG_ASSERT( stack_top != & task_stack->ts_first_block.sb_block[0] );
105 if ( entries & TASK_STACK_INDEX_MASK == 0 )
107 kmp_stack_block_t *stack_block = (kmp_stack_block_t *) (stack_top) ;
109 stack_block = stack_block -> sb_prev;
110 stack_top = & stack_block -> sb_block[TASK_STACK_BLOCK_SIZE];
117 tied_task = * stack_top;
119 KMP_DEBUG_ASSERT( tied_task != NULL );
120 KMP_DEBUG_ASSERT( tied_task -> td_flags.tasktype == TASK_TIED );
122 KA_TRACE(threshold, (
"__kmp_trace_task_stack(%s): gtid=%d, entry=%d, "
123 "stack_top=%p, tied_task=%p\n",
124 location, gtid, entries, stack_top, tied_task ) );
126 KMP_DEBUG_ASSERT( stack_top == & task_stack->ts_first_block.sb_block[0] );
128 KA_TRACE(threshold, (
"__kmp_trace_task_stack(exit): location = %s, gtid = %d\n",
141 __kmp_init_task_stack( kmp_int32 gtid, kmp_thread_data_t *thread_data )
143 kmp_task_stack_t *task_stack = & thread_data->td.td_susp_tied_tasks;
144 kmp_stack_block_t *first_block;
147 first_block = & task_stack -> ts_first_block;
148 task_stack -> ts_top = (kmp_taskdata_t **) first_block;
149 memset( (
void *) first_block,
'\0', TASK_STACK_BLOCK_SIZE *
sizeof(kmp_taskdata_t *));
152 task_stack -> ts_entries = TASK_STACK_EMPTY;
153 first_block -> sb_next = NULL;
154 first_block -> sb_prev = NULL;
165 __kmp_free_task_stack( kmp_int32 gtid, kmp_thread_data_t *thread_data )
167 kmp_task_stack_t *task_stack = & thread_data->td.td_susp_tied_tasks;
168 kmp_stack_block_t *stack_block = & task_stack -> ts_first_block;
170 KMP_DEBUG_ASSERT( task_stack -> ts_entries == TASK_STACK_EMPTY );
172 while ( stack_block != NULL ) {
173 kmp_stack_block_t *next_block = (stack_block) ? stack_block -> sb_next : NULL;
175 stack_block -> sb_next = NULL;
176 stack_block -> sb_prev = NULL;
177 if (stack_block != & task_stack -> ts_first_block) {
178 __kmp_thread_free( thread, stack_block );
180 stack_block = next_block;
183 task_stack -> ts_entries = 0;
184 task_stack -> ts_top = NULL;
197 __kmp_push_task_stack( kmp_int32 gtid, kmp_info_t *thread, kmp_taskdata_t * tied_task )
200 kmp_thread_data_t *thread_data = & thread -> th.th_task_team ->
201 tt.tt_threads_data[ __kmp_tid_from_gtid( gtid ) ];
202 kmp_task_stack_t *task_stack = & thread_data->td.td_susp_tied_tasks ;
204 if ( tied_task->td_flags.team_serial || tied_task->td_flags.tasking_ser ) {
208 KMP_DEBUG_ASSERT( tied_task -> td_flags.tasktype == TASK_TIED );
209 KMP_DEBUG_ASSERT( task_stack -> ts_top != NULL );
211 KA_TRACE(20, (
"__kmp_push_task_stack(enter): GTID: %d; THREAD: %p; TASK: %p\n",
212 gtid, thread, tied_task ) );
214 * (task_stack -> ts_top) = tied_task;
217 task_stack -> ts_top++;
218 task_stack -> ts_entries++;
220 if ( task_stack -> ts_entries & TASK_STACK_INDEX_MASK == 0 )
223 kmp_stack_block_t *stack_block =
224 (kmp_stack_block_t *) (task_stack -> ts_top - TASK_STACK_BLOCK_SIZE);
227 if ( stack_block -> sb_next != NULL )
229 task_stack -> ts_top = & stack_block -> sb_next -> sb_block[0];
233 kmp_stack_block_t *new_block = (kmp_stack_block_t *)
234 __kmp_thread_calloc(thread,
sizeof(kmp_stack_block_t));
236 task_stack -> ts_top = & new_block -> sb_block[0];
237 stack_block -> sb_next = new_block;
238 new_block -> sb_prev = stack_block;
239 new_block -> sb_next = NULL;
241 KA_TRACE(30, (
"__kmp_push_task_stack(): GTID: %d; TASK: %p; Alloc new block: %p\n",
242 gtid, tied_task, new_block ) );
245 KA_TRACE(20, (
"__kmp_push_task_stack(exit): GTID: %d; TASK: %p\n", gtid, tied_task ) );
258 __kmp_pop_task_stack( kmp_int32 gtid, kmp_info_t *thread, kmp_taskdata_t *ending_task )
261 kmp_thread_data_t *thread_data = & thread -> th.th_task_team -> tt_threads_data[ __kmp_tid_from_gtid( gtid ) ];
262 kmp_task_stack_t *task_stack = & thread_data->td.td_susp_tied_tasks ;
263 kmp_taskdata_t *tied_task;
265 if ( ending_task->td_flags.team_serial || ending_task->td_flags.tasking_ser ) {
269 KMP_DEBUG_ASSERT( task_stack -> ts_top != NULL );
270 KMP_DEBUG_ASSERT( task_stack -> ts_entries > 0 );
272 KA_TRACE(20, (
"__kmp_pop_task_stack(enter): GTID: %d; THREAD: %p\n", gtid, thread ) );
275 if ( task_stack -> ts_entries & TASK_STACK_INDEX_MASK == 0 )
277 kmp_stack_block_t *stack_block =
278 (kmp_stack_block_t *) (task_stack -> ts_top) ;
280 stack_block = stack_block -> sb_prev;
281 task_stack -> ts_top = & stack_block -> sb_block[TASK_STACK_BLOCK_SIZE];
285 task_stack -> ts_top--;
286 task_stack -> ts_entries--;
288 tied_task = * (task_stack -> ts_top );
290 KMP_DEBUG_ASSERT( tied_task != NULL );
291 KMP_DEBUG_ASSERT( tied_task -> td_flags.tasktype == TASK_TIED );
292 KMP_DEBUG_ASSERT( tied_task == ending_task );
294 KA_TRACE(20, (
"__kmp_pop_task_stack(exit): GTID: %d; TASK: %p\n", gtid, tied_task ) );
303 __kmp_push_task(kmp_int32 gtid, kmp_task_t * task )
305 kmp_info_t * thread = __kmp_threads[ gtid ];
306 kmp_taskdata_t * taskdata = KMP_TASK_TO_TASKDATA(task);
307 kmp_task_team_t * task_team = thread->th.th_task_team;
308 kmp_int32 tid = __kmp_tid_from_gtid( gtid );
309 kmp_thread_data_t * thread_data;
311 KA_TRACE(20, (
"__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata ) );
314 if ( taskdata->td_flags.task_serial ) {
315 KA_TRACE(20, (
"__kmp_push_task: T#%d team serialized; returning TASK_NOT_PUSHED for task %p\n",
317 return TASK_NOT_PUSHED;
321 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
322 if ( ! KMP_TASKING_ENABLED( task_team, thread->th.th_task_state ) ) {
323 __kmp_enable_tasking( task_team, thread );
325 KMP_DEBUG_ASSERT( TCR_4(task_team -> tt.tt_found_tasks) == TRUE );
326 KMP_DEBUG_ASSERT( TCR_PTR(task_team -> tt.tt_threads_data) != NULL );
329 thread_data = & task_team -> tt.tt_threads_data[ tid ];
332 if (thread_data -> td.td_deque == NULL ) {
333 __kmp_alloc_task_deque( thread, thread_data );
337 if ( TCR_4(thread_data -> td.td_deque_ntasks) >= TASK_DEQUE_SIZE )
339 KA_TRACE(20, (
"__kmp_push_task: T#%d deque is full; returning TASK_NOT_PUSHED for task %p\n",
341 return TASK_NOT_PUSHED;
345 __kmp_acquire_bootstrap_lock( & thread_data -> td.td_deque_lock );
348 KMP_DEBUG_ASSERT( TCR_4(thread_data -> td.td_deque_ntasks) < TASK_DEQUE_SIZE );
350 thread_data -> td.td_deque[ thread_data -> td.td_deque_tail ] = taskdata;
352 thread_data -> td.td_deque_tail = ( thread_data -> td.td_deque_tail + 1 ) & TASK_DEQUE_MASK;
353 TCW_4(thread_data -> td.td_deque_ntasks, TCR_4(thread_data -> td.td_deque_ntasks) + 1);
355 __kmp_release_bootstrap_lock( & thread_data -> td.td_deque_lock );
357 KA_TRACE(20, (
"__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: "
358 "task=%p ntasks=%d head=%u tail=%u\n",
359 gtid, taskdata, thread_data->td.td_deque_ntasks,
360 thread_data->td.td_deque_tail, thread_data->td.td_deque_head) );
362 return TASK_SUCCESSFULLY_PUSHED;
371 __kmp_pop_current_task_from_thread( kmp_info_t *this_thr )
373 KF_TRACE( 10, (
"__kmp_pop_current_task_from_thread(enter): T#%d this_thread=%p, curtask=%p, "
374 "curtask_parent=%p\n",
375 0, this_thr, this_thr -> th.th_current_task,
376 this_thr -> th.th_current_task -> td_parent ) );
378 this_thr -> th.th_current_task = this_thr -> th.th_current_task -> td_parent;
380 KF_TRACE( 10, (
"__kmp_pop_current_task_from_thread(exit): T#%d this_thread=%p, curtask=%p, "
381 "curtask_parent=%p\n",
382 0, this_thr, this_thr -> th.th_current_task,
383 this_thr -> th.th_current_task -> td_parent ) );
394 __kmp_push_current_task_to_thread( kmp_info_t *this_thr, kmp_team_t *team,
int tid )
397 KF_TRACE( 10, (
"__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p curtask=%p "
399 tid, this_thr, this_thr->th.th_current_task,
400 team->t.t_implicit_task_taskdata[tid].td_parent ) );
402 KMP_DEBUG_ASSERT (this_thr != NULL);
405 if( this_thr->th.th_current_task != & team -> t.t_implicit_task_taskdata[ 0 ] ) {
406 team -> t.t_implicit_task_taskdata[ 0 ].td_parent = this_thr->th.th_current_task;
407 this_thr->th.th_current_task = & team -> t.t_implicit_task_taskdata[ 0 ];
410 team -> t.t_implicit_task_taskdata[ tid ].td_parent = team -> t.t_implicit_task_taskdata[ 0 ].td_parent;
411 this_thr->th.th_current_task = & team -> t.t_implicit_task_taskdata[ tid ];
414 KF_TRACE( 10, (
"__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p curtask=%p "
416 tid, this_thr, this_thr->th.th_current_task,
417 team->t.t_implicit_task_taskdata[tid].td_parent ) );
428 __kmp_task_start( kmp_int32 gtid, kmp_task_t * task, kmp_taskdata_t * current_task )
430 kmp_taskdata_t * taskdata = KMP_TASK_TO_TASKDATA(task);
431 kmp_info_t * thread = __kmp_threads[ gtid ];
433 KA_TRACE(10, (
"__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n",
434 gtid, taskdata, current_task) );
436 KMP_DEBUG_ASSERT( taskdata -> td_flags.tasktype == TASK_EXPLICIT );
441 current_task -> td_flags.executing = 0;
444 #ifdef BUILD_TIED_TASK_STACK
445 if ( taskdata -> td_flags.tiedness == TASK_TIED )
447 __kmp_push_task_stack( gtid, thread, taskdata );
452 thread -> th.th_current_task = taskdata;
454 KMP_DEBUG_ASSERT( taskdata -> td_flags.started == 0 );
455 KMP_DEBUG_ASSERT( taskdata -> td_flags.executing == 0 );
456 taskdata -> td_flags.started = 1;
457 taskdata -> td_flags.executing = 1;
458 KMP_DEBUG_ASSERT( taskdata -> td_flags.complete == 0 );
459 KMP_DEBUG_ASSERT( taskdata -> td_flags.freed == 0 );
466 KA_TRACE(10, (
"__kmp_task_start(exit): T#%d task=%p\n",
480 __kmpc_omp_task_begin_if0(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * task )
482 kmp_taskdata_t * taskdata = KMP_TASK_TO_TASKDATA(task);
483 kmp_taskdata_t * current_task = __kmp_threads[ gtid ] -> th.th_current_task;
485 KA_TRACE(10, (
"__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p current_task=%p\n",
486 gtid, loc_ref, taskdata, current_task ) );
488 taskdata -> td_flags.task_serial = 1;
489 __kmp_task_start( gtid, task, current_task );
491 KA_TRACE(10, (
"__kmpc_omp_task_begin_if0(exit): T#%d loc=%p task=%p,\n",
492 gtid, loc_ref, taskdata ) );
503 __kmpc_omp_task_begin(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * task )
505 kmp_taskdata_t * current_task = __kmp_threads[ gtid ] -> th.th_current_task;
507 KA_TRACE(10, (
"__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n",
508 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task), current_task ) );
510 __kmp_task_start( gtid, task, current_task );
512 KA_TRACE(10, (
"__kmpc_omp_task_begin(exit): T#%d loc=%p task=%p,\n",
513 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task) ) );
517 #endif // TASK_UNUSED
527 __kmp_free_task( kmp_int32 gtid, kmp_taskdata_t * taskdata, kmp_info_t * thread )
529 KA_TRACE(30, (
"__kmp_free_task: T#%d freeing data from task %p\n",
533 KMP_DEBUG_ASSERT( taskdata->td_flags.tasktype == TASK_EXPLICIT );
534 KMP_DEBUG_ASSERT( taskdata->td_flags.executing == 0 );
535 KMP_DEBUG_ASSERT( taskdata->td_flags.complete == 1 );
536 KMP_DEBUG_ASSERT( taskdata->td_flags.freed == 0 );
537 KMP_DEBUG_ASSERT( TCR_4(taskdata->td_allocated_child_tasks) == 0 || taskdata->td_flags.task_serial == 1);
538 KMP_DEBUG_ASSERT( TCR_4(taskdata->td_incomplete_child_tasks) == 0 );
540 taskdata->td_flags.freed = 1;
543 __kmp_fast_free( thread, taskdata );
545 __kmp_thread_free( thread, taskdata );
548 KA_TRACE(20, (
"__kmp_free_task: T#%d freed task %p\n",
560 __kmp_free_task_and_ancestors( kmp_int32 gtid, kmp_taskdata_t * taskdata, kmp_info_t * thread )
562 kmp_int32 children = 0;
563 kmp_int32 team_or_tasking_serialized = taskdata -> td_flags.team_serial || taskdata -> td_flags.tasking_ser;
565 KMP_DEBUG_ASSERT( taskdata -> td_flags.tasktype == TASK_EXPLICIT );
567 if ( !team_or_tasking_serialized ) {
568 children = KMP_TEST_THEN_DEC32( (kmp_int32 *)(& taskdata -> td_allocated_child_tasks) ) - 1;
569 KMP_DEBUG_ASSERT( children >= 0 );
573 while ( children == 0 )
575 kmp_taskdata_t * parent_taskdata = taskdata -> td_parent;
577 KA_TRACE(20, (
"__kmp_free_task_and_ancestors(enter): T#%d task %p complete "
578 "and freeing itself\n", gtid, taskdata) );
581 __kmp_free_task( gtid, taskdata, thread );
583 taskdata = parent_taskdata;
587 if ( team_or_tasking_serialized || taskdata -> td_flags.tasktype == TASK_IMPLICIT )
590 if ( !team_or_tasking_serialized ) {
592 children = KMP_TEST_THEN_DEC32( (kmp_int32 *)(& taskdata -> td_allocated_child_tasks) ) - 1;
593 KMP_DEBUG_ASSERT( children >= 0 );
597 KA_TRACE(20, (
"__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; "
598 "not freeing it yet\n", gtid, taskdata, children) );
608 __kmp_task_finish( kmp_int32 gtid, kmp_task_t *task, kmp_taskdata_t *resumed_task )
610 kmp_taskdata_t * taskdata = KMP_TASK_TO_TASKDATA(task);
611 kmp_info_t * thread = __kmp_threads[ gtid ];
612 kmp_int32 children = 0;
614 KA_TRACE(10, (
"__kmp_task_finish(enter): T#%d finishing task %p and resuming task %p\n",
615 gtid, taskdata, resumed_task) );
617 KMP_DEBUG_ASSERT( taskdata -> td_flags.tasktype == TASK_EXPLICIT );
620 #ifdef BUILD_TIED_TASK_STACK
621 if ( taskdata -> td_flags.tiedness == TASK_TIED )
623 __kmp_pop_task_stack( gtid, thread, taskdata );
627 KMP_DEBUG_ASSERT( taskdata -> td_flags.executing == 1 );
628 KMP_DEBUG_ASSERT( taskdata -> td_flags.complete == 0 );
629 taskdata -> td_flags.executing = 0;
630 taskdata -> td_flags.complete = 1;
631 KMP_DEBUG_ASSERT( taskdata -> td_flags.started == 1 );
632 KMP_DEBUG_ASSERT( taskdata -> td_flags.freed == 0 );
635 if ( !( taskdata -> td_flags.team_serial || taskdata -> td_flags.tasking_ser ) ) {
637 children = KMP_TEST_THEN_DEC32( (kmp_int32 *)(& taskdata -> td_parent -> td_incomplete_child_tasks) ) - 1;
638 KMP_DEBUG_ASSERT( children >= 0 );
640 if ( taskdata->td_taskgroup )
641 KMP_TEST_THEN_DEC32( (kmp_int32 *)(& taskdata->td_taskgroup->count) );
642 __kmp_release_deps(gtid,taskdata);
646 KA_TRACE(20, (
"__kmp_task_finish: T#%d finished task %p, %d incomplete children\n",
647 gtid, taskdata, children) );
657 if (taskdata->td_flags.destructors_thunk) {
658 kmp_routine_entry_t destr_thunk = task->destructors;
659 KMP_ASSERT(destr_thunk);
660 destr_thunk(gtid, task);
662 #endif // OMP_40_ENABLED
666 KMP_DEBUG_ASSERT( (taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) ==
667 taskdata->td_flags.task_serial);
668 if ( taskdata->td_flags.task_serial )
670 if (resumed_task == NULL) {
671 resumed_task = taskdata->td_parent;
675 KMP_DEBUG_ASSERT( resumed_task == taskdata->td_parent );
679 KMP_DEBUG_ASSERT( resumed_task != NULL );
683 __kmp_free_task_and_ancestors(gtid, taskdata, thread);
685 __kmp_threads[ gtid ] -> th.th_current_task = resumed_task;
689 resumed_task->td_flags.executing = 1;
691 KA_TRACE(10, (
"__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n",
692 gtid, taskdata, resumed_task) );
704 __kmpc_omp_task_complete_if0(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task )
706 KA_TRACE(10, (
"__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n",
707 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task) ) );
709 __kmp_task_finish( gtid, task, NULL );
711 KA_TRACE(10, (
"__kmpc_omp_task_complete_if0(exit): T#%d loc=%p task=%p\n",
712 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task) ) );
723 __kmpc_omp_task_complete(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task )
725 KA_TRACE(10, (
"__kmpc_omp_task_complete(enter): T#%d loc=%p task=%p\n",
726 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task) ) );
728 __kmp_task_finish( gtid, task, NULL );
730 KA_TRACE(10, (
"__kmpc_omp_task_complete(exit): T#%d loc=%p task=%p\n",
731 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task) ) );
734 #endif // TASK_UNUSED
749 __kmp_init_implicit_task(
ident_t *loc_ref, kmp_info_t *this_thr, kmp_team_t *team,
int tid,
int set_curr_task )
751 kmp_taskdata_t * task = & team->t.t_implicit_task_taskdata[ tid ];
753 KF_TRACE(10, (
"__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n",
754 tid, team, task, set_curr_task ?
"TRUE" :
"FALSE" ) );
756 task->td_task_id = KMP_GEN_TASK_ID();
757 task->td_team = team;
759 task->td_ident = loc_ref;
760 task->td_taskwait_ident = NULL;
761 task->td_taskwait_counter = 0;
762 task->td_taskwait_thread = 0;
764 task->td_flags.tiedness = TASK_TIED;
765 task->td_flags.tasktype = TASK_IMPLICIT;
767 task->td_flags.task_serial = 1;
768 task->td_flags.tasking_ser = ( __kmp_tasking_mode == tskm_immediate_exec );
769 task->td_flags.team_serial = ( team->t.t_serialized ) ? 1 : 0;
771 task->td_flags.started = 1;
772 task->td_flags.executing = 1;
773 task->td_flags.complete = 0;
774 task->td_flags.freed = 0;
777 task->td_dephash = NULL;
778 task->td_depnode = NULL;
782 task->td_incomplete_child_tasks = 0;
783 task->td_allocated_child_tasks = 0;
785 task->td_taskgroup = NULL;
787 __kmp_push_current_task_to_thread( this_thr, team, tid );
789 KMP_DEBUG_ASSERT(task->td_incomplete_child_tasks == 0);
790 KMP_DEBUG_ASSERT(task->td_allocated_child_tasks == 0);
793 KF_TRACE(10, (
"__kmp_init_implicit_task(exit): T#:%d team=%p task=%p\n",
800 __kmp_round_up_to_val(
size_t size,
size_t val ) {
801 if ( size & ( val - 1 ) ) {
802 size &= ~ ( val - 1 );
803 if ( size <= KMP_SIZE_T_MAX - val ) {
824 __kmp_task_alloc(
ident_t *loc_ref, kmp_int32 gtid, kmp_tasking_flags_t *flags,
825 size_t sizeof_kmp_task_t,
size_t sizeof_shareds,
826 kmp_routine_entry_t task_entry )
829 kmp_taskdata_t *taskdata;
830 kmp_info_t *thread = __kmp_threads[ gtid ];
831 kmp_team_t *team = thread->th.th_team;
832 kmp_taskdata_t *parent_task = thread->th.th_current_task;
833 size_t shareds_offset;
835 KA_TRACE(10, (
"__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) "
836 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
837 gtid, loc_ref, *((kmp_int32 *)flags), sizeof_kmp_task_t,
838 sizeof_shareds, task_entry) );
840 if ( parent_task->td_flags.final ) {
841 if (flags->merged_if0) {
848 shareds_offset =
sizeof( kmp_taskdata_t ) + sizeof_kmp_task_t;
849 shareds_offset = __kmp_round_up_to_val( shareds_offset,
sizeof(
void * ));
852 KA_TRACE(30, (
"__kmp_task_alloc: T#%d First malloc size: %ld\n",
853 gtid, shareds_offset) );
854 KA_TRACE(30, (
"__kmp_task_alloc: T#%d Second malloc size: %ld\n",
855 gtid, sizeof_shareds) );
859 taskdata = (kmp_taskdata_t *) __kmp_fast_allocate( thread, shareds_offset + sizeof_shareds );
861 taskdata = (kmp_taskdata_t *) __kmp_thread_malloc( thread, shareds_offset + sizeof_shareds );
864 task = KMP_TASKDATA_TO_TASK(taskdata);
868 KMP_DEBUG_ASSERT( ( ((kmp_uintptr_t)taskdata) & (
sizeof(
double)-1) ) == 0 );
869 KMP_DEBUG_ASSERT( ( ((kmp_uintptr_t)task) & (
sizeof(
double)-1) ) == 0 );
871 KMP_DEBUG_ASSERT( ( ((kmp_uintptr_t)taskdata) & (
sizeof(_Quad)-1) ) == 0 );
872 KMP_DEBUG_ASSERT( ( ((kmp_uintptr_t)task) & (
sizeof(_Quad)-1) ) == 0 );
874 if (sizeof_shareds > 0) {
876 task->shareds = & ((
char *) taskdata)[ shareds_offset ];
878 KMP_DEBUG_ASSERT( ( ((kmp_uintptr_t)task->shareds) & (
sizeof(
void *)-1) ) == 0 );
880 task->shareds = NULL;
882 task->routine = task_entry;
885 taskdata->td_task_id = KMP_GEN_TASK_ID();
886 taskdata->td_team = team;
887 taskdata->td_alloc_thread = thread;
888 taskdata->td_parent = parent_task;
889 taskdata->td_level = parent_task->td_level + 1;
890 taskdata->td_ident = loc_ref;
891 taskdata->td_taskwait_ident = NULL;
892 taskdata->td_taskwait_counter = 0;
893 taskdata->td_taskwait_thread = 0;
894 KMP_DEBUG_ASSERT( taskdata->td_parent != NULL );
895 copy_icvs( &taskdata->td_icvs, &taskdata->td_parent->td_icvs );
897 taskdata->td_flags.tiedness = flags->tiedness;
898 taskdata->td_flags.final = flags->final;
899 taskdata->td_flags.merged_if0 = flags->merged_if0;
901 taskdata->td_flags.destructors_thunk = flags->destructors_thunk;
902 #endif // OMP_40_ENABLED
903 taskdata->td_flags.tasktype = TASK_EXPLICIT;
906 taskdata->td_flags.tasking_ser = ( __kmp_tasking_mode == tskm_immediate_exec );
909 taskdata->td_flags.team_serial = ( team->t.t_serialized ) ? 1 : 0;
914 taskdata->td_flags.task_serial = ( taskdata->td_flags.final
915 || taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser );
917 taskdata->td_flags.started = 0;
918 taskdata->td_flags.executing = 0;
919 taskdata->td_flags.complete = 0;
920 taskdata->td_flags.freed = 0;
922 taskdata->td_flags.native = flags->native;
924 taskdata->td_incomplete_child_tasks = 0;
925 taskdata->td_allocated_child_tasks = 1;
927 taskdata->td_taskgroup = parent_task->td_taskgroup;
928 taskdata->td_dephash = NULL;
929 taskdata->td_depnode = NULL;
932 if ( !( taskdata -> td_flags.team_serial || taskdata -> td_flags.tasking_ser ) ) {
933 KMP_TEST_THEN_INC32( (kmp_int32 *)(& parent_task->td_incomplete_child_tasks) );
935 if ( parent_task->td_taskgroup )
936 KMP_TEST_THEN_INC32( (kmp_int32 *)(& parent_task->td_taskgroup->count) );
939 if ( taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT ) {
940 KMP_TEST_THEN_INC32( (kmp_int32 *)(& taskdata->td_parent->td_allocated_child_tasks) );
944 KA_TRACE(20, (
"__kmp_task_alloc(exit): T#%d created task %p parent=%p\n",
945 gtid, taskdata, taskdata->td_parent) );
952 __kmpc_omp_task_alloc(
ident_t *loc_ref, kmp_int32 gtid, kmp_int32 flags,
953 size_t sizeof_kmp_task_t,
size_t sizeof_shareds,
954 kmp_routine_entry_t task_entry )
957 kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *) & flags;
959 input_flags->native = FALSE;
962 KA_TRACE(10, (
"__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s) "
963 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
964 gtid, loc_ref, input_flags->tiedness ?
"tied " :
"untied",
965 sizeof_kmp_task_t, sizeof_shareds, task_entry) );
967 retval = __kmp_task_alloc( loc_ref, gtid, input_flags, sizeof_kmp_task_t,
968 sizeof_shareds, task_entry );
970 KA_TRACE(20, (
"__kmpc_omp_task_alloc(exit): T#%d retval %p\n", gtid, retval) );
983 __kmp_invoke_task( kmp_int32 gtid, kmp_task_t *task, kmp_taskdata_t * current_task )
985 kmp_taskdata_t * taskdata = KMP_TASK_TO_TASKDATA(task);
989 KA_TRACE(30, (
"__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n",
990 gtid, taskdata, current_task) );
992 __kmp_task_start( gtid, task, current_task );
998 if (__kmp_omp_cancellation) {
999 kmp_info_t *this_thr = __kmp_threads [ gtid ];
1000 kmp_team_t * this_team = this_thr->th.th_team;
1001 kmp_taskgroup_t * taskgroup = taskdata->td_taskgroup;
1002 if ((taskgroup && taskgroup->cancel_request) || (this_team->t.t_cancel_request == cancel_parallel)) {
1013 #endif // OMP_40_ENABLED
1014 #ifdef KMP_GOMP_COMPAT
1015 if (taskdata->td_flags.native) {
1016 ((void (*)(
void *))(*(task->routine)))(task->shareds);
1021 (*(task->routine))(gtid, task);
1025 #endif // OMP_40_ENABLED
1027 __kmp_task_finish( gtid, task, current_task );
1029 KA_TRACE(30, (
"__kmp_inovke_task(exit): T#%d completed task %p, resuming task %p\n",
1030 gtid, taskdata, current_task) );
1045 __kmpc_omp_task_parts(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * new_task)
1047 kmp_taskdata_t * new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1049 KA_TRACE(10, (
"__kmpc_omp_task_parts(enter): T#%d loc=%p task=%p\n",
1050 gtid, loc_ref, new_taskdata ) );
1055 if ( __kmp_push_task( gtid, new_task ) == TASK_NOT_PUSHED )
1057 kmp_taskdata_t * current_task = __kmp_threads[ gtid ] -> th.th_current_task;
1058 new_taskdata->td_flags.task_serial = 1;
1059 __kmp_invoke_task( gtid, new_task, current_task );
1062 KA_TRACE(10, (
"__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: "
1063 "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n", gtid, loc_ref,
1066 return TASK_CURRENT_NOT_QUEUED;
1081 __kmpc_omp_task(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * new_task)
1083 kmp_taskdata_t * new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1086 KA_TRACE(10, (
"__kmpc_omp_task(enter): T#%d loc=%p task=%p\n",
1087 gtid, loc_ref, new_taskdata ) );
1092 if ( __kmp_push_task( gtid, new_task ) == TASK_NOT_PUSHED )
1094 kmp_taskdata_t * current_task = __kmp_threads[ gtid ] -> th.th_current_task;
1095 new_taskdata -> td_flags.task_serial = 1;
1096 __kmp_invoke_task( gtid, new_task, current_task );
1099 KA_TRACE(10, (
"__kmpc_omp_task(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n",
1100 gtid, loc_ref, new_taskdata ) );
1102 return TASK_CURRENT_NOT_QUEUED;
1110 __kmpc_omp_taskwait(
ident_t *loc_ref, kmp_int32 gtid )
1112 kmp_taskdata_t * taskdata;
1113 kmp_info_t * thread;
1114 int thread_finished = FALSE;
1116 KA_TRACE(10, (
"__kmpc_omp_taskwait(enter): T#%d loc=%p\n",
1119 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
1122 thread = __kmp_threads[ gtid ];
1123 taskdata = thread -> th.th_current_task;
1127 taskdata->td_taskwait_counter += 1;
1128 taskdata->td_taskwait_ident = loc_ref;
1129 taskdata->td_taskwait_thread = gtid + 1;
1132 void * itt_sync_obj = __kmp_itt_taskwait_object( gtid );
1133 if ( itt_sync_obj != NULL )
1134 __kmp_itt_taskwait_starting( gtid, itt_sync_obj );
1137 if ( ! taskdata->td_flags.team_serial ) {
1139 while ( TCR_4(taskdata -> td_incomplete_child_tasks) != 0 ) {
1140 __kmp_execute_tasks( thread, gtid, &(taskdata->td_incomplete_child_tasks),
1141 0, FALSE, &thread_finished
1142 USE_ITT_BUILD_ARG(itt_sync_obj),
1143 __kmp_task_stealing_constraint );
1147 if ( itt_sync_obj != NULL )
1148 __kmp_itt_taskwait_finished( gtid, itt_sync_obj );
1152 taskdata->td_taskwait_thread = - taskdata->td_taskwait_thread;
1155 KA_TRACE(10, (
"__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, "
1156 "returning TASK_CURRENT_NOT_QUEUED\n", gtid, taskdata) );
1158 return TASK_CURRENT_NOT_QUEUED;
1166 __kmpc_omp_taskyield(
ident_t *loc_ref, kmp_int32 gtid,
int end_part )
1168 kmp_taskdata_t * taskdata;
1169 kmp_info_t * thread;
1170 int thread_finished = FALSE;
1172 KA_TRACE(10, (
"__kmpc_omp_taskyield(enter): T#%d loc=%p end_part = %d\n",
1173 gtid, loc_ref, end_part) );
1175 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
1178 thread = __kmp_threads[ gtid ];
1179 taskdata = thread -> th.th_current_task;
1184 taskdata->td_taskwait_counter += 1;
1185 taskdata->td_taskwait_ident = loc_ref;
1186 taskdata->td_taskwait_thread = gtid + 1;
1189 void * itt_sync_obj = __kmp_itt_taskwait_object( gtid );
1190 if ( itt_sync_obj != NULL )
1191 __kmp_itt_taskwait_starting( gtid, itt_sync_obj );
1193 if ( ! taskdata->td_flags.team_serial ) {
1194 __kmp_execute_tasks( thread, gtid, NULL, 0, FALSE, &thread_finished
1195 USE_ITT_BUILD_ARG(itt_sync_obj),
1196 __kmp_task_stealing_constraint );
1200 if ( itt_sync_obj != NULL )
1201 __kmp_itt_taskwait_finished( gtid, itt_sync_obj );
1205 taskdata->td_taskwait_thread = - taskdata->td_taskwait_thread;
1208 KA_TRACE(10, (
"__kmpc_omp_taskyield(exit): T#%d task %p resuming, "
1209 "returning TASK_CURRENT_NOT_QUEUED\n", gtid, taskdata) );
1211 return TASK_CURRENT_NOT_QUEUED;
1220 __kmpc_taskgroup(
ident_t* loc,
int gtid )
1222 kmp_info_t * thread = __kmp_threads[ gtid ];
1223 kmp_taskdata_t * taskdata = thread->th.th_current_task;
1224 kmp_taskgroup_t * tg_new =
1225 (kmp_taskgroup_t *)__kmp_thread_malloc( thread,
sizeof( kmp_taskgroup_t ) );
1226 KA_TRACE(10, (
"__kmpc_taskgroup: T#%d loc=%p group=%p\n", gtid, loc, tg_new) );
1228 tg_new->cancel_request = cancel_noreq;
1229 tg_new->parent = taskdata->td_taskgroup;
1230 taskdata->td_taskgroup = tg_new;
1239 __kmpc_end_taskgroup(
ident_t* loc,
int gtid )
1241 kmp_info_t * thread = __kmp_threads[ gtid ];
1242 kmp_taskdata_t * taskdata = thread->th.th_current_task;
1243 kmp_taskgroup_t * taskgroup = taskdata->td_taskgroup;
1244 int thread_finished = FALSE;
1246 KA_TRACE(10, (
"__kmpc_end_taskgroup(enter): T#%d loc=%p\n", gtid, loc) );
1247 KMP_DEBUG_ASSERT( taskgroup != NULL );
1249 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
1252 void * itt_sync_obj = __kmp_itt_taskwait_object( gtid );
1253 if ( itt_sync_obj != NULL )
1254 __kmp_itt_taskwait_starting( gtid, itt_sync_obj );
1257 if ( ! taskdata->td_flags.team_serial ) {
1258 while ( TCR_4(taskgroup->count) != 0 ) {
1259 __kmp_execute_tasks( thread, gtid, &(taskgroup->count),
1260 0, FALSE, &thread_finished
1261 USE_ITT_BUILD_ARG(itt_sync_obj),
1262 __kmp_task_stealing_constraint );
1267 if ( itt_sync_obj != NULL )
1268 __kmp_itt_taskwait_finished( gtid, itt_sync_obj );
1271 KMP_DEBUG_ASSERT( taskgroup->count == 0 );
1274 taskdata->td_taskgroup = taskgroup->parent;
1275 __kmp_thread_free( thread, taskgroup );
1277 KA_TRACE(10, (
"__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n", gtid, taskdata) );
1286 __kmp_remove_my_task( kmp_info_t * thread, kmp_int32 gtid, kmp_task_team_t *task_team,
1287 kmp_int32 is_constrained )
1290 kmp_taskdata_t * taskdata;
1291 kmp_thread_data_t *thread_data;
1294 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
1295 KMP_DEBUG_ASSERT( task_team -> tt.tt_threads_data != NULL );
1297 thread_data = & task_team -> tt.tt_threads_data[ __kmp_tid_from_gtid( gtid ) ];
1299 KA_TRACE(10, (
"__kmp_remove_my_task(enter): T#%d ntasks=%d head=%u tail=%u\n",
1300 gtid, thread_data->td.td_deque_ntasks, thread_data->td.td_deque_head,
1301 thread_data->td.td_deque_tail) );
1303 if (TCR_4(thread_data -> td.td_deque_ntasks) == 0) {
1304 KA_TRACE(10, (
"__kmp_remove_my_task(exit #1): T#%d No tasks to remove: ntasks=%d head=%u tail=%u\n",
1305 gtid, thread_data->td.td_deque_ntasks, thread_data->td.td_deque_head,
1306 thread_data->td.td_deque_tail) );
1310 __kmp_acquire_bootstrap_lock( & thread_data -> td.td_deque_lock );
1312 if (TCR_4(thread_data -> td.td_deque_ntasks) == 0) {
1313 __kmp_release_bootstrap_lock( & thread_data -> td.td_deque_lock );
1314 KA_TRACE(10, (
"__kmp_remove_my_task(exit #2): T#%d No tasks to remove: ntasks=%d head=%u tail=%u\n",
1315 gtid, thread_data->td.td_deque_ntasks, thread_data->td.td_deque_head,
1316 thread_data->td.td_deque_tail) );
1320 tail = ( thread_data -> td.td_deque_tail - 1 ) & TASK_DEQUE_MASK;
1321 taskdata = thread_data -> td.td_deque[ tail ];
1323 if (is_constrained) {
1326 kmp_taskdata_t * current = thread->th.th_current_task;
1327 kmp_int32 level = current->td_level;
1328 kmp_taskdata_t * parent = taskdata->td_parent;
1329 while ( parent != current && parent->td_level > level ) {
1330 parent = parent->td_parent;
1331 KMP_DEBUG_ASSERT(parent != NULL);
1333 if ( parent != current ) {
1335 __kmp_release_bootstrap_lock( & thread_data -> td.td_deque_lock );
1336 KA_TRACE(10, (
"__kmp_remove_my_task(exit #2): T#%d No tasks to remove: ntasks=%d head=%u tail=%u\n",
1337 gtid, thread_data->td.td_deque_ntasks, thread_data->td.td_deque_head,
1338 thread_data->td.td_deque_tail) );
1343 thread_data -> td.td_deque_tail = tail;
1344 TCW_4(thread_data -> td.td_deque_ntasks, thread_data -> td.td_deque_ntasks - 1);
1346 __kmp_release_bootstrap_lock( & thread_data->td.td_deque_lock );
1348 KA_TRACE(10, (
"__kmp_remove_my_task(exit #2): T#%d task %p removed: ntasks=%d head=%u tail=%u\n",
1349 gtid, taskdata, thread_data->td.td_deque_ntasks, thread_data->td.td_deque_head,
1350 thread_data->td.td_deque_tail) );
1352 task = KMP_TASKDATA_TO_TASK( taskdata );
1363 __kmp_steal_task( kmp_info_t *victim, kmp_int32 gtid, kmp_task_team_t *task_team,
1364 volatile kmp_uint32 *unfinished_threads,
int *thread_finished,
1365 kmp_int32 is_constrained )
1368 kmp_taskdata_t * taskdata;
1369 kmp_thread_data_t *victim_td, *threads_data;
1370 kmp_int32 victim_tid, thread_tid;
1372 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
1374 threads_data = task_team -> tt.tt_threads_data;
1375 KMP_DEBUG_ASSERT( threads_data != NULL );
1377 victim_tid = victim->th.th_info.ds.ds_tid;
1378 victim_td = & threads_data[ victim_tid ];
1380 KA_TRACE(10, (
"__kmp_steal_task(enter): T#%d try to steal from T#%d: task_team=%p ntasks=%d "
1381 "head=%u tail=%u\n",
1382 gtid, __kmp_gtid_from_thread( victim ), task_team, victim_td->td.td_deque_ntasks,
1383 victim_td->td.td_deque_head, victim_td->td.td_deque_tail) );
1385 if ( (TCR_4(victim_td -> td.td_deque_ntasks) == 0) ||
1386 (TCR_PTR(victim->th.th_task_team) != task_team))
1388 KA_TRACE(10, (
"__kmp_steal_task(exit #1): T#%d could not steal from T#%d: task_team=%p "
1389 "ntasks=%d head=%u tail=%u\n",
1390 gtid, __kmp_gtid_from_thread( victim ), task_team, victim_td->td.td_deque_ntasks,
1391 victim_td->td.td_deque_head, victim_td->td.td_deque_tail) );
1395 __kmp_acquire_bootstrap_lock( & victim_td -> td.td_deque_lock );
1398 if ( (TCR_4(victim_td -> td.td_deque_ntasks) == 0) ||
1399 (TCR_PTR(victim->th.th_task_team) != task_team))
1401 __kmp_release_bootstrap_lock( & victim_td -> td.td_deque_lock );
1402 KA_TRACE(10, (
"__kmp_steal_task(exit #2): T#%d could not steal from T#%d: task_team=%p "
1403 "ntasks=%d head=%u tail=%u\n",
1404 gtid, __kmp_gtid_from_thread( victim ), task_team, victim_td->td.td_deque_ntasks,
1405 victim_td->td.td_deque_head, victim_td->td.td_deque_tail) );
1409 KMP_DEBUG_ASSERT( victim_td -> td.td_deque != NULL );
1411 if ( !is_constrained ) {
1412 taskdata = victim_td -> td.td_deque[ victim_td -> td.td_deque_head ];
1414 victim_td -> td.td_deque_head = ( victim_td -> td.td_deque_head + 1 ) & TASK_DEQUE_MASK;
1417 kmp_int32 tail = ( victim_td -> td.td_deque_tail - 1 ) & TASK_DEQUE_MASK;
1418 taskdata = victim_td -> td.td_deque[ tail ];
1421 kmp_taskdata_t * current = __kmp_threads[ gtid ]->th.th_current_task;
1422 kmp_int32 level = current->td_level;
1423 kmp_taskdata_t * parent = taskdata->td_parent;
1424 while ( parent != current && parent->td_level > level ) {
1425 parent = parent->td_parent;
1426 KMP_DEBUG_ASSERT(parent != NULL);
1428 if ( parent != current ) {
1430 __kmp_release_bootstrap_lock( & victim_td -> td.td_deque_lock );
1431 KA_TRACE(10, (
"__kmp_steal_task(exit #2): T#%d could not steal from T#%d: task_team=%p "
1432 "ntasks=%d head=%u tail=%u\n",
1433 gtid, __kmp_gtid_from_thread( threads_data[victim_tid].td.td_thr ),
1434 task_team, victim_td->td.td_deque_ntasks,
1435 victim_td->td.td_deque_head, victim_td->td.td_deque_tail) );
1438 victim_td -> td.td_deque_tail = tail;
1440 if (*thread_finished) {
1444 kmp_uint32 count = KMP_TEST_THEN_INC32( (kmp_int32 *)unfinished_threads );
1446 KA_TRACE(20, (
"__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n",
1447 gtid, count + 1, task_team) );
1449 *thread_finished = FALSE;
1451 TCW_4(victim_td -> td.td_deque_ntasks, TCR_4(victim_td -> td.td_deque_ntasks) - 1);
1453 __kmp_release_bootstrap_lock( & victim_td -> td.td_deque_lock );
1455 KA_TRACE(10, (
"__kmp_steal_task(exit #3): T#%d stole task %p from T#d: task_team=%p "
1456 "ntasks=%d head=%u tail=%u\n",
1457 gtid, taskdata, __kmp_gtid_from_thread( victim ), task_team,
1458 victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head,
1459 victim_td->td.td_deque_tail) );
1461 task = KMP_TASKDATA_TO_TASK( taskdata );
1477 __kmp_execute_tasks( kmp_info_t *thread,
1479 volatile kmp_uint *spinner,
1482 int *thread_finished
1483 USE_ITT_BUILD_ARG(
void * itt_sync_obj),
1484 kmp_int32 is_constrained )
1486 kmp_task_team_t * task_team;
1488 kmp_thread_data_t * threads_data;
1490 kmp_taskdata_t * current_task = thread -> th.th_current_task;
1491 volatile kmp_uint32 * unfinished_threads;
1492 kmp_int32 nthreads, last_stolen, k, tid;
1494 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
1495 KMP_DEBUG_ASSERT( thread == __kmp_threads[ gtid ] );
1497 task_team = thread -> th.th_task_team;
1498 KMP_DEBUG_ASSERT( task_team != NULL );
1500 KA_TRACE(15, (
"__kmp_execute_tasks(enter): T#%d final_spin=%d *thread_finished=%d\n",
1501 gtid, final_spin, *thread_finished) );
1503 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team -> tt.tt_threads_data);
1504 KMP_DEBUG_ASSERT( threads_data != NULL );
1506 nthreads = task_team -> tt.tt_nproc;
1507 unfinished_threads = &(task_team -> tt.tt_unfinished_threads);
1508 KMP_DEBUG_ASSERT( nthreads > 1 );
1509 KMP_DEBUG_ASSERT( TCR_4((
int)*unfinished_threads) >= 0 );
1513 while (( task = __kmp_remove_my_task( thread, gtid, task_team, is_constrained )) != NULL ) {
1514 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1515 if ( __itt_sync_create_ptr || KMP_ITT_DEBUG ) {
1516 if ( itt_sync_obj == NULL ) {
1518 itt_sync_obj = __kmp_itt_barrier_object( gtid, bs_forkjoin_barrier );
1520 __kmp_itt_task_starting( itt_sync_obj );
1523 __kmp_invoke_task( gtid, task, current_task );
1525 if ( itt_sync_obj != NULL )
1526 __kmp_itt_task_finished( itt_sync_obj );
1534 if ((spinner == NULL) || ((!final_spin) && (TCR_4(*spinner) == checker))) {
1535 KA_TRACE(15, (
"__kmp_execute_tasks(exit #1): T#%d spin condition satisfied\n", gtid) );
1538 KMP_YIELD( __kmp_library == library_throughput );
1547 if (! *thread_finished) {
1548 kmp_uint32 count = KMP_TEST_THEN_DEC32( (kmp_int32 *)unfinished_threads ) - 1;
1549 KA_TRACE(20, (
"__kmp_execute_tasks(dec #1): T#%d dec unfinished_threads to %d task_team=%p\n",
1550 gtid, count, task_team) );
1551 *thread_finished = TRUE;
1559 if ((spinner != NULL) && (TCR_4(*spinner) == checker)) {
1560 KA_TRACE(15, (
"__kmp_execute_tasks(exit #2): T#%d spin condition satisfied\n", gtid) );
1566 tid = thread -> th.th_info.ds.ds_tid;
1567 last_stolen = threads_data[ tid ].td.td_deque_last_stolen;
1569 if (last_stolen != -1) {
1570 kmp_info_t *other_thread = threads_data[last_stolen].td.td_thr;
1572 while ((task = __kmp_steal_task( other_thread, gtid, task_team, unfinished_threads,
1573 thread_finished, is_constrained )) != NULL)
1575 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1576 if ( __itt_sync_create_ptr || KMP_ITT_DEBUG ) {
1577 if ( itt_sync_obj == NULL ) {
1579 itt_sync_obj = __kmp_itt_barrier_object( gtid, bs_forkjoin_barrier );
1581 __kmp_itt_task_starting( itt_sync_obj );
1584 __kmp_invoke_task( gtid, task, current_task );
1586 if ( itt_sync_obj != NULL )
1587 __kmp_itt_task_finished( itt_sync_obj );
1591 if ((spinner == NULL) || ((!final_spin) && (TCR_4(*spinner) == checker))) {
1592 KA_TRACE(15, (
"__kmp_execute_tasks(exit #3): T#%d spin condition satisfied\n",
1597 KMP_YIELD( __kmp_library == library_throughput );
1600 if (TCR_4(threads_data[ tid ].td.td_deque_ntasks) != 0) {
1601 KA_TRACE(20, (
"__kmp_execute_tasks: T#%d stolen task spawned other tasks, restart\n",
1608 threads_data[ tid ].td.td_deque_last_stolen = -1;
1616 if (! *thread_finished) {
1617 kmp_uint32 count = KMP_TEST_THEN_DEC32( (kmp_int32 *)unfinished_threads ) - 1;
1618 KA_TRACE(20, (
"__kmp_execute_tasks(dec #2): T#%d dec unfinished_threads to %d "
1619 "task_team=%p\n", gtid, count, task_team) );
1620 *thread_finished = TRUE;
1629 if ((spinner != NULL) && (TCR_4(*spinner) == checker)) {
1630 KA_TRACE(15, (
"__kmp_execute_tasks(exit #4): T#%d spin condition satisfied\n",
1643 k = __kmp_get_random( thread ) % (nthreads - 1);
1644 if ( k >= thread -> th.th_info.ds.ds_tid ) {
1648 kmp_info_t *other_thread = threads_data[k].td.td_thr;
1658 if ( ( __kmp_tasking_mode == tskm_task_teams ) &&
1659 (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) &&
1660 (TCR_PTR(other_thread->th.th_sleep_loc) != NULL))
1662 __kmp_resume( __kmp_gtid_from_thread( other_thread ), NULL );
1674 while ((task = __kmp_steal_task( other_thread, gtid, task_team, unfinished_threads,
1675 thread_finished, is_constrained )) != NULL)
1677 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1678 if ( __itt_sync_create_ptr || KMP_ITT_DEBUG ) {
1679 if ( itt_sync_obj == NULL ) {
1681 itt_sync_obj = __kmp_itt_barrier_object( gtid, bs_forkjoin_barrier );
1683 __kmp_itt_task_starting( itt_sync_obj );
1686 __kmp_invoke_task( gtid, task, current_task );
1688 if ( itt_sync_obj != NULL )
1689 __kmp_itt_task_finished( itt_sync_obj );
1694 threads_data[ tid ].td.td_deque_last_stolen = k;
1699 if ((spinner == NULL) || ((!final_spin) && (TCR_4(*spinner) == checker))) {
1700 KA_TRACE(15, (
"__kmp_execute_tasks(exit #5): T#%d spin condition satisfied\n",
1704 KMP_YIELD( __kmp_library == library_throughput );
1708 if (TCR_4(threads_data[ tid ].td.td_deque_ntasks) != 0) {
1709 KA_TRACE(20, (
"__kmp_execute_tasks: T#%d stolen task spawned other tasks, restart\n",
1724 if (! *thread_finished) {
1725 kmp_uint32 count = KMP_TEST_THEN_DEC32( (kmp_int32 *)unfinished_threads ) - 1;
1726 KA_TRACE(20, (
"__kmp_execute_tasks(dec #3): T#%d dec unfinished_threads to %d; "
1728 gtid, count, task_team) );
1729 *thread_finished = TRUE;
1738 if ((spinner != NULL) && (TCR_4(*spinner) == checker)) {
1739 KA_TRACE(15, (
"__kmp_execute_tasks(exit #6): T#%d spin condition satisfied\n",
1746 KA_TRACE(15, (
"__kmp_execute_tasks(exit #7): T#%d can't find work\n", gtid) );
1757 __kmp_enable_tasking( kmp_task_team_t *task_team, kmp_info_t *this_thr )
1759 kmp_team_t *team = this_thr->th.th_team;
1760 kmp_thread_data_t *threads_data;
1761 int nthreads, i, is_init_thread;
1763 KA_TRACE( 10, (
"__kmp_enable_tasking(enter): T#%d\n",
1764 __kmp_gtid_from_thread( this_thr ) ) );
1766 KMP_DEBUG_ASSERT(task_team != NULL);
1767 KMP_DEBUG_ASSERT(team != NULL);
1769 nthreads = task_team->tt.tt_nproc;
1770 KMP_DEBUG_ASSERT(nthreads > 0);
1771 KMP_DEBUG_ASSERT(nthreads == team->t.t_nproc);
1774 is_init_thread = __kmp_realloc_task_threads_data( this_thr, task_team );
1776 if (!is_init_thread) {
1778 KA_TRACE( 20, (
"__kmp_enable_tasking(exit): T#%d: threads array already set up.\n",
1779 __kmp_gtid_from_thread( this_thr ) ) );
1782 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team -> tt.tt_threads_data);
1783 KMP_DEBUG_ASSERT( threads_data != NULL );
1785 if ( ( __kmp_tasking_mode == tskm_task_teams ) &&
1786 ( __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ) )
1791 for (i = 0; i < nthreads; i++) {
1792 volatile kmp_uint *sleep_loc;
1793 kmp_info_t *thread = threads_data[i].td.td_thr;
1795 if (i == this_thr->th.th_info.ds.ds_tid) {
1805 if ( ( sleep_loc = (
volatile kmp_uint *)
1806 TCR_PTR( thread -> th.th_sleep_loc) ) != NULL )
1808 KF_TRACE( 50, (
"__kmp_enable_tasking: T#%d waking up thread T#%d\n",
1809 __kmp_gtid_from_thread( this_thr ),
1810 __kmp_gtid_from_thread( thread ) ) );
1811 __kmp_resume( __kmp_gtid_from_thread( thread ), sleep_loc );
1814 KF_TRACE( 50, (
"__kmp_enable_tasking: T#%d don't wake up thread T#%d\n",
1815 __kmp_gtid_from_thread( this_thr ),
1816 __kmp_gtid_from_thread( thread ) ) );
1821 KA_TRACE( 10, (
"__kmp_enable_tasking(exit): T#%d\n",
1822 __kmp_gtid_from_thread( this_thr ) ) );
1861 static kmp_task_team_t *__kmp_free_task_teams = NULL;
1863 static kmp_bootstrap_lock_t __kmp_task_team_lock = KMP_BOOTSTRAP_LOCK_INITIALIZER( __kmp_task_team_lock );
1875 __kmp_alloc_task_deque( kmp_info_t *thread, kmp_thread_data_t *thread_data )
1877 __kmp_init_bootstrap_lock( & thread_data -> td.td_deque_lock );
1878 KMP_DEBUG_ASSERT( thread_data -> td.td_deque == NULL );
1881 thread_data -> td.td_deque_last_stolen = -1;
1883 KMP_DEBUG_ASSERT( TCR_4(thread_data -> td.td_deque_ntasks) == 0 );
1884 KMP_DEBUG_ASSERT( thread_data -> td.td_deque_head == 0 );
1885 KMP_DEBUG_ASSERT( thread_data -> td.td_deque_tail == 0 );
1887 KE_TRACE( 10, (
"__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n",
1888 __kmp_gtid_from_thread( thread ), TASK_DEQUE_SIZE, thread_data ) );
1892 thread_data -> td.td_deque = (kmp_taskdata_t **)
1893 __kmp_allocate( TASK_DEQUE_SIZE *
sizeof(kmp_taskdata_t *));
1903 __kmp_free_task_deque( kmp_thread_data_t *thread_data )
1905 __kmp_acquire_bootstrap_lock( & thread_data -> td.td_deque_lock );
1907 if ( thread_data -> td.td_deque != NULL ) {
1908 TCW_4(thread_data -> td.td_deque_ntasks, 0);
1909 __kmp_free( thread_data -> td.td_deque );
1910 thread_data -> td.td_deque = NULL;
1912 __kmp_release_bootstrap_lock( & thread_data -> td.td_deque_lock );
1914 #ifdef BUILD_TIED_TASK_STACK
1916 if ( thread_data -> td.td_susp_tied_tasks.ts_entries != TASK_STACK_EMPTY ) {
1917 __kmp_free_task_stack( __kmp_thread_from_gtid( gtid ), thread_data );
1919 #endif // BUILD_TIED_TASK_STACK
1933 __kmp_realloc_task_threads_data( kmp_info_t *thread, kmp_task_team_t *task_team )
1935 kmp_thread_data_t ** threads_data_p;
1936 kmp_int32 nthreads, maxthreads;
1937 int is_init_thread = FALSE;
1939 if ( TCR_4(task_team -> tt.tt_found_tasks) ) {
1944 threads_data_p = & task_team -> tt.tt_threads_data;
1945 nthreads = task_team -> tt.tt_nproc;
1946 maxthreads = task_team -> tt.tt_max_threads;
1950 __kmp_acquire_bootstrap_lock( & task_team -> tt.tt_threads_lock );
1952 if ( ! TCR_4(task_team -> tt.tt_found_tasks) ) {
1954 kmp_team_t *team = thread -> th.th_team;
1957 is_init_thread = TRUE;
1958 if ( maxthreads < nthreads ) {
1960 if ( *threads_data_p != NULL ) {
1961 kmp_thread_data_t *old_data = *threads_data_p;
1962 kmp_thread_data_t *new_data = NULL;
1964 KE_TRACE( 10, (
"__kmp_realloc_task_threads_data: T#%d reallocating "
1965 "threads data for task_team %p, new_size = %d, old_size = %d\n",
1966 __kmp_gtid_from_thread( thread ), task_team,
1967 nthreads, maxthreads ) );
1972 new_data = (kmp_thread_data_t *)
1973 __kmp_allocate( nthreads *
sizeof(kmp_thread_data_t) );
1975 memcpy( (
void *) new_data, (
void *) old_data,
1976 maxthreads *
sizeof(kmp_taskdata_t *) );
1978 #ifdef BUILD_TIED_TASK_STACK
1980 for (i = maxthreads; i < nthreads; i++) {
1981 kmp_thread_data_t *thread_data = & (*threads_data_p)[i];
1982 __kmp_init_task_stack( __kmp_gtid_from_thread( thread ), thread_data );
1984 #endif // BUILD_TIED_TASK_STACK
1986 (*threads_data_p) = new_data;
1987 __kmp_free( old_data );
1990 KE_TRACE( 10, (
"__kmp_realloc_task_threads_data: T#%d allocating "
1991 "threads data for task_team %p, size = %d\n",
1992 __kmp_gtid_from_thread( thread ), task_team, nthreads ) );
1996 *threads_data_p = (kmp_thread_data_t *)
1997 __kmp_allocate( nthreads *
sizeof(kmp_thread_data_t) );
1998 #ifdef BUILD_TIED_TASK_STACK
2000 for (i = 0; i < nthreads; i++) {
2001 kmp_thread_data_t *thread_data = & (*threads_data_p)[i];
2002 __kmp_init_task_stack( __kmp_gtid_from_thread( thread ), thread_data );
2004 #endif // BUILD_TIED_TASK_STACK
2006 task_team -> tt.tt_max_threads = nthreads;
2010 KMP_DEBUG_ASSERT( *threads_data_p != NULL );
2014 for (i = 0; i < nthreads; i++) {
2015 kmp_thread_data_t *thread_data = & (*threads_data_p)[i];
2016 thread_data -> td.td_thr = team -> t.t_threads[i];
2018 if ( thread_data -> td.td_deque_last_stolen >= nthreads) {
2022 thread_data -> td.td_deque_last_stolen = -1;
2027 TCW_SYNC_4(task_team -> tt.tt_found_tasks, TRUE);
2030 __kmp_release_bootstrap_lock( & task_team -> tt.tt_threads_lock );
2031 return is_init_thread;
2041 __kmp_free_task_threads_data( kmp_task_team_t *task_team )
2043 __kmp_acquire_bootstrap_lock( & task_team -> tt.tt_threads_lock );
2044 if ( task_team -> tt.tt_threads_data != NULL ) {
2046 for (i = 0; i < task_team->tt.tt_max_threads; i++ ) {
2047 __kmp_free_task_deque( & task_team -> tt.tt_threads_data[i] );
2049 __kmp_free( task_team -> tt.tt_threads_data );
2050 task_team -> tt.tt_threads_data = NULL;
2052 __kmp_release_bootstrap_lock( & task_team -> tt.tt_threads_lock );
2061 static kmp_task_team_t *
2062 __kmp_allocate_task_team( kmp_info_t *thread, kmp_team_t *team )
2064 kmp_task_team_t *task_team = NULL;
2067 KA_TRACE( 20, (
"__kmp_allocate_task_team: T#%d entering; team = %p\n",
2068 (thread ? __kmp_gtid_from_thread( thread ) : -1), team ) );
2070 if (TCR_PTR(__kmp_free_task_teams) != NULL) {
2072 __kmp_acquire_bootstrap_lock( &__kmp_task_team_lock );
2073 if (__kmp_free_task_teams != NULL) {
2074 task_team = __kmp_free_task_teams;
2075 TCW_PTR(__kmp_free_task_teams, task_team -> tt.tt_next);
2076 task_team -> tt.tt_next = NULL;
2078 __kmp_release_bootstrap_lock( &__kmp_task_team_lock );
2081 if (task_team == NULL) {
2082 KE_TRACE( 10, (
"__kmp_allocate_task_team: T#%d allocating "
2083 "task team for team %p\n",
2084 __kmp_gtid_from_thread( thread ), team ) );
2088 task_team = (kmp_task_team_t *) __kmp_allocate(
sizeof(kmp_task_team_t) );
2089 __kmp_init_bootstrap_lock( & task_team -> tt.tt_threads_lock );
2095 TCW_4(task_team -> tt.tt_found_tasks, FALSE);
2096 task_team -> tt.tt_nproc = nthreads = team->t.t_nproc;
2098 task_team -> tt.tt_state = 0;
2099 TCW_4( task_team -> tt.tt_unfinished_threads, nthreads );
2100 TCW_4( task_team -> tt.tt_active, TRUE );
2101 TCW_4( task_team -> tt.tt_ref_ct, nthreads - 1);
2103 KA_TRACE( 20, (
"__kmp_allocate_task_team: T#%d exiting; task_team = %p\n",
2104 (thread ? __kmp_gtid_from_thread( thread ) : -1), task_team ) );
2116 __kmp_free_task_team( kmp_info_t *thread, kmp_task_team_t *task_team )
2118 KA_TRACE( 20, (
"__kmp_free_task_team: T#%d task_team = %p\n",
2119 thread ? __kmp_gtid_from_thread( thread ) : -1, task_team ) );
2121 KMP_DEBUG_ASSERT( TCR_4(task_team -> tt.tt_ref_ct) == 0 );
2124 __kmp_acquire_bootstrap_lock( & __kmp_task_team_lock );
2126 KMP_DEBUG_ASSERT( task_team -> tt.tt_next == NULL );
2127 task_team -> tt.tt_next = __kmp_free_task_teams;
2128 TCW_4(task_team -> tt.tt_found_tasks, FALSE);
2129 TCW_PTR(__kmp_free_task_teams, task_team);
2131 __kmp_release_bootstrap_lock( & __kmp_task_team_lock );
2142 __kmp_reap_task_teams(
void )
2144 kmp_task_team_t *task_team;
2146 if ( TCR_PTR(__kmp_free_task_teams) != NULL ) {
2148 __kmp_acquire_bootstrap_lock( &__kmp_task_team_lock );
2149 while ( ( task_team = __kmp_free_task_teams ) != NULL ) {
2150 __kmp_free_task_teams = task_team -> tt.tt_next;
2151 task_team -> tt.tt_next = NULL;
2154 if ( task_team -> tt.tt_threads_data != NULL ) {
2155 __kmp_free_task_threads_data( task_team );
2157 __kmp_free( task_team );
2159 __kmp_release_bootstrap_lock( &__kmp_task_team_lock );
2171 __kmp_unref_task_team( kmp_task_team_t *task_team, kmp_info_t *thread )
2175 ref_ct = KMP_TEST_THEN_DEC32( (kmp_int32 *)(& task_team->tt.tt_ref_ct) ) - 1;
2177 KA_TRACE( 20, (
"__kmp_unref_task_team: T#%d task_team = %p ref_ct = %d\n",
2178 __kmp_gtid_from_thread( thread ), task_team, ref_ct ) );
2181 if ( ref_ct == 0 ) {
2182 __kmp_free_task_team( thread, task_team );
2185 TCW_PTR( *((
volatile kmp_task_team_t **)(&thread->th.th_task_team)), NULL );
2195 __kmp_wait_to_unref_task_teams(
void)
2201 KMP_INIT_YIELD( spins );
2211 for (thread = (kmp_info_t *)__kmp_thread_pool;
2213 thread = thread->th.th_next_pool)
2215 volatile kmp_uint *sleep_loc;
2219 if ( TCR_PTR(thread->th.th_task_team) == NULL ) {
2220 KA_TRACE( 10, (
"__kmp_wait_to_unref_task_team: T#%d task_team == NULL\n",
2221 __kmp_gtid_from_thread( thread ) ) );
2226 if (!__kmp_is_thread_alive(thread, &exit_val)) {
2227 if (TCR_PTR(thread->th.th_task_team) != NULL) {
2228 __kmp_unref_task_team( thread->th.th_task_team, thread );
2236 KA_TRACE( 10, (
"__kmp_wait_to_unref_task_team: Waiting for T#%d to unreference task_team\n",
2237 __kmp_gtid_from_thread( thread ) ) );
2239 if ( __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ) {
2241 if ( ( sleep_loc = (
volatile kmp_uint *) TCR_PTR( thread->th.th_sleep_loc) ) != NULL ) {
2242 KA_TRACE( 10, (
"__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n",
2243 __kmp_gtid_from_thread( thread ), __kmp_gtid_from_thread( thread ) ) );
2244 __kmp_resume( __kmp_gtid_from_thread( thread ), sleep_loc );
2255 KMP_YIELD( TCR_4(__kmp_nth) > __kmp_avail_proc );
2256 KMP_YIELD_SPIN( spins );
2269 __kmp_task_team_setup( kmp_info_t *this_thr, kmp_team_t *team )
2271 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
2273 if ( ( team->t.t_task_team == NULL ) && ( team->t.t_nproc > 1 ) ) {
2281 team->t.t_task_team = __kmp_allocate_task_team( this_thr, team );
2282 KA_TRACE( 20, (
"__kmp_task_team_setup: Master T#%d created new "
2283 "task_team %p for team %d\n",
2284 __kmp_gtid_from_thread( this_thr ), team->t.t_task_team,
2285 ((team != NULL) ? team->t.t_id : -1)) );
2293 if ( team->t.t_task_team != NULL ) {
2296 team->t.t_task_team->tt.tt_state = 1 - this_thr->th.th_task_state;
2307 __kmp_task_team_sync( kmp_info_t *this_thr, kmp_team_t *team )
2309 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
2313 if ( this_thr->th.th_task_team != NULL ) {
2314 if ( ! TCR_SYNC_4( this_thr->th.th_task_team->tt.tt_active ) ) {
2315 KMP_DEBUG_ASSERT( ! KMP_MASTER_TID( __kmp_tid_from_gtid( __kmp_gtid_from_thread( this_thr ) ) ) );
2316 __kmp_unref_task_team( this_thr->th.th_task_team, this_thr );
2321 KMP_DEBUG_ASSERT( this_thr->th.th_task_team == team->t.t_task_team );
2329 TCW_PTR(this_thr->th.th_task_team, team->t.t_task_team);
2330 if ( this_thr->th.th_task_team != NULL ) {
2337 this_thr->th.th_task_state = 1 - this_thr->th.th_task_state;
2338 KMP_DEBUG_ASSERT( this_thr->th.th_task_state == TCR_4(team->t.t_task_team->tt.tt_state) );
2340 KA_TRACE( 20, (
"__kmp_task_team_sync: Thread T#%d task team assigned pointer (%p) from Team #%d task team\n",
2341 __kmp_gtid_from_thread( this_thr ), &this_thr->th.th_task_team,
2342 this_thr->th.th_task_team, ((team != NULL) ? (team->t.t_id) : -1) ) );
2352 __kmp_task_team_wait( kmp_info_t *this_thr,
2354 USE_ITT_BUILD_ARG(
void * itt_sync_obj)
2357 kmp_task_team_t *task_team = team->t.t_task_team;
2359 KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
2360 KMP_DEBUG_ASSERT( task_team == this_thr->th.th_task_team );
2362 if ( ( task_team != NULL ) && KMP_TASKING_ENABLED( task_team, this_thr->th.th_task_state ) ) {
2363 KA_TRACE( 20, (
"__kmp_task_team_wait: Master T#%d waiting for all tasks: task_team = %p\n",
2364 __kmp_gtid_from_thread( this_thr ), task_team ) );
2372 __kmp_wait_sleep( this_thr, &task_team->tt.tt_unfinished_threads, 0, TRUE
2373 USE_ITT_BUILD_ARG(itt_sync_obj)
2382 KA_TRACE( 20, (
"__kmp_task_team_wait: Master T#%d deactivating task_team %p\n",
2383 __kmp_gtid_from_thread( this_thr ), task_team ) );
2384 KMP_DEBUG_ASSERT( task_team->tt.tt_nproc > 1 );
2385 TCW_SYNC_4( task_team->tt.tt_active, FALSE );
2388 TCW_PTR(this_thr->th.th_task_team, NULL);
2389 team->t.t_task_team = NULL;
2403 __kmp_tasking_barrier( kmp_team_t *team, kmp_info_t *thread,
int gtid )
2405 volatile kmp_uint32 *spin = &team->t.t_task_team->tt.tt_unfinished_threads;
2407 KMP_DEBUG_ASSERT( __kmp_tasking_mode == tskm_extra_barrier );
2410 KMP_FSYNC_SPIN_INIT( spin, (kmp_uint32*) NULL );
2412 while (! __kmp_execute_tasks( thread, gtid, spin, 0, TRUE, &flag
2413 USE_ITT_BUILD_ARG(NULL), 0 ) ) {
2416 KMP_FSYNC_SPIN_PREPARE( spin );
2419 if( TCR_4(__kmp_global.g.g_done) ) {
2420 if( __kmp_global.g.g_abort )
2421 __kmp_abort_thread( );
2427 KMP_FSYNC_SPIN_ACQUIRED( (
void*) spin );
2431 #endif // OMP_30_ENABLED