39 #include "kmp_error.h"
40 #include "kmp_stats.h"
43 #include "ompt-internal.h"
44 #include "ompt-specific.h"
47 #define MAX_MESSAGE 512
68 if (__kmp_ignore_mppbeg() == FALSE) {
69 __kmp_internal_begin();
71 KC_TRACE( 10, (
"__kmpc_begin: called\n" ) );
89 if (__kmp_ignore_mppend() == FALSE) {
90 KC_TRACE( 10, (
"__kmpc_end: called\n" ) );
91 KA_TRACE( 30, (
"__kmpc_end\n" ));
93 __kmp_internal_end_thread( -1 );
119 kmp_int32 gtid = __kmp_entry_gtid();
121 KC_TRACE( 10, (
"__kmpc_global_thread_num: T#%d\n", gtid ) );
142 KC_TRACE( 10, (
"__kmpc_global_num_threads: num_threads = %d\n", __kmp_nth ) );
144 return TCR_4(__kmp_nth);
156 KC_TRACE( 10, (
"__kmpc_bound_thread_num: called\n" ) );
157 return __kmp_tid_from_gtid( __kmp_entry_gtid() );
168 KC_TRACE( 10, (
"__kmpc_bound_num_threads: called\n" ) );
170 return __kmp_entry_thread() -> th.th_team -> t.t_nproc;
192 if (__kmp_par_range == 0) {
199 semi2 = strchr(semi2,
';');
203 semi2 = strchr(semi2 + 1,
';');
207 if (__kmp_par_range_filename[0]) {
208 const char *name = semi2 - 1;
209 while ((name > loc->
psource) && (*name !=
'/') && (*name !=
';')) {
212 if ((*name ==
'/') || (*name ==
';')) {
215 if (strncmp(__kmp_par_range_filename, name, semi2 - name)) {
216 return __kmp_par_range < 0;
219 semi3 = strchr(semi2 + 1,
';');
220 if (__kmp_par_range_routine[0]) {
221 if ((semi3 != NULL) && (semi3 > semi2)
222 && (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
223 return __kmp_par_range < 0;
226 if (KMP_SSCANF(semi3 + 1,
"%d", &line_no) == 1) {
227 if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
228 return __kmp_par_range > 0;
230 return __kmp_par_range < 0;
246 return __kmp_entry_thread() -> th.th_root -> r.r_active;
261 KA_TRACE( 20, (
"__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
262 global_tid, num_threads ) );
264 __kmp_push_num_threads( loc, global_tid, num_threads );
268 __kmpc_pop_num_threads(
ident_t *loc, kmp_int32 global_tid )
270 KA_TRACE( 20, (
"__kmpc_pop_num_threads: enter\n" ) );
279 __kmpc_push_proc_bind(
ident_t *loc, kmp_int32 global_tid, kmp_int32 proc_bind )
281 KA_TRACE( 20, (
"__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n",
282 global_tid, proc_bind ) );
284 __kmp_push_proc_bind( loc, global_tid, (kmp_proc_bind_t)proc_bind );
304 int gtid = __kmp_entry_gtid();
308 va_start( ap, microtask );
311 kmp_info_t *master_th = __kmp_threads[ gtid ];
312 kmp_team_t *parent_team = master_th->th.th_team;
313 int tid = __kmp_tid_from_gtid( gtid );
314 parent_team->t.t_implicit_task_taskdata[tid].
315 ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(0);
318 #if INCLUDE_SSC_MARKS
321 __kmp_fork_call( loc, gtid, fork_context_intel,
324 VOLATILE_CAST(
void *) microtask,
326 VOLATILE_CAST(microtask_t) microtask,
327 VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
329 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
335 #if INCLUDE_SSC_MARKS
338 __kmp_join_call( loc, gtid );
343 if (ompt_status & ompt_status_track) {
344 parent_team->t.t_implicit_task_taskdata[tid].
345 ompt_task_info.frame.reenter_runtime_frame = 0;
367 KA_TRACE( 20, (
"__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
368 global_tid, num_teams, num_threads ) );
370 __kmp_push_num_teams( loc, global_tid, num_teams, num_threads );
385 int gtid = __kmp_entry_gtid();
386 kmp_info_t *this_thr = __kmp_threads[ gtid ];
388 va_start( ap, microtask );
391 this_thr->th.th_teams_microtask = microtask;
392 this_thr->th.th_teams_level = this_thr->th.th_team->t.t_level;
395 if ( this_thr->th.th_teams_size.nteams == 0 ) {
396 __kmp_push_num_teams( loc, gtid, 0, 0 );
398 KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
399 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nteams >= 1);
400 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nth >= 1);
402 __kmp_fork_call( loc, gtid, fork_context_intel,
405 VOLATILE_CAST(
void *) microtask,
407 VOLATILE_CAST(microtask_t) __kmp_teams_master,
408 VOLATILE_CAST(launch_t) __kmp_invoke_teams_master,
409 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
415 __kmp_join_call( loc, gtid );
416 this_thr->th.th_teams_microtask = NULL;
417 this_thr->th.th_teams_level = 0;
418 *(kmp_int64*)(&this_thr->th.th_teams_size) = 0L;
431 __kmpc_invoke_task_func(
int gtid )
433 return __kmp_invoke_task_func( gtid );
451 __kmp_serialized_parallel(loc, global_tid);
466 kmp_internal_control_t *top;
467 kmp_info_t *this_thr;
468 kmp_team_t *serial_team;
470 KC_TRACE( 10, (
"__kmpc_end_serialized_parallel: called by T#%d\n", global_tid ) );
478 if( ! TCR_4( __kmp_init_parallel ) )
479 __kmp_parallel_initialize();
481 this_thr = __kmp_threads[ global_tid ];
482 serial_team = this_thr->th.th_serial_team;
485 kmp_task_team_t * task_team = this_thr->th.th_task_team;
488 if ( task_team != NULL && task_team->tt.tt_found_proxy_tasks )
489 __kmp_task_team_wait(this_thr, serial_team, NULL );
493 KMP_DEBUG_ASSERT( serial_team );
494 KMP_ASSERT( serial_team -> t.t_serialized );
495 KMP_DEBUG_ASSERT( this_thr -> th.th_team == serial_team );
496 KMP_DEBUG_ASSERT( serial_team != this_thr->th.th_root->r.r_root_team );
497 KMP_DEBUG_ASSERT( serial_team -> t.t_threads );
498 KMP_DEBUG_ASSERT( serial_team -> t.t_threads[0] == this_thr );
501 top = serial_team -> t.t_control_stack_top;
502 if ( top && top -> serial_nesting_level == serial_team -> t.t_serialized ) {
503 copy_icvs( &serial_team -> t.t_threads[0] -> th.th_current_task -> td_icvs, top );
504 serial_team -> t.t_control_stack_top = top -> next;
509 serial_team -> t.t_level--;
512 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
514 dispatch_private_info_t * disp_buffer = serial_team->t.t_dispatch->th_disp_buffer;
515 serial_team->t.t_dispatch->th_disp_buffer =
516 serial_team->t.t_dispatch->th_disp_buffer->next;
517 __kmp_free( disp_buffer );
520 -- serial_team -> t.t_serialized;
521 if ( serial_team -> t.t_serialized == 0 ) {
525 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
526 if ( __kmp_inherit_fp_control && serial_team->t.t_fp_control_saved ) {
527 __kmp_clear_x87_fpu_status_word();
528 __kmp_load_x87_fpu_control_word( &serial_team->t.t_x87_fpu_control_word );
529 __kmp_load_mxcsr( &serial_team->t.t_mxcsr );
533 this_thr -> th.th_team = serial_team -> t.t_parent;
534 this_thr -> th.th_info.ds.ds_tid = serial_team -> t.t_master_tid;
537 this_thr -> th.th_team_nproc = serial_team -> t.t_parent -> t.t_nproc;
538 this_thr -> th.th_team_master = serial_team -> t.t_parent -> t.t_threads[0];
539 this_thr -> th.th_team_serialized = this_thr -> th.th_team -> t.t_serialized;
542 this_thr -> th.th_dispatch = & this_thr -> th.th_team ->
543 t.t_dispatch[ serial_team -> t.t_master_tid ];
545 __kmp_pop_current_task_from_thread( this_thr );
547 KMP_ASSERT( this_thr -> th.th_current_task -> td_flags.executing == 0 );
548 this_thr -> th.th_current_task -> td_flags.executing = 1;
550 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
552 this_thr->th.th_task_team = this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
553 KA_TRACE(20, (
"__kmpc_end_serialized_parallel: T#%d restoring task_team %p / team %p\n",
554 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
557 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
558 KA_TRACE( 20, (
"__kmpc_end_serialized_parallel: T#%d decreasing nesting depth of serial team %p to %d\n",
559 global_tid, serial_team, serial_team -> t.t_serialized ) );
564 kmp_uint64 cur_time = 0;
566 if ( __itt_get_timestamp_ptr ) {
567 cur_time = __itt_get_timestamp();
570 if ( this_thr->th.th_team->t.t_level == 0
572 && this_thr->th.th_teams_microtask == NULL
576 this_thr->th.th_ident = loc;
577 if ( ( __itt_frame_submit_v3_ptr || KMP_ITT_DEBUG ) &&
578 ( __kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 1 ) )
580 __kmp_itt_frame_submit( global_tid, this_thr->th.th_frame_time_serialized,
581 cur_time, 0, loc, this_thr->th.th_team_nproc, 0 );
582 if ( __kmp_forkjoin_frames_mode == 3 )
584 __kmp_itt_frame_submit( global_tid, serial_team->t.t_region_time,
585 cur_time, 0, loc, this_thr->th.th_team_nproc, 2 );
586 }
else if ( ( __itt_frame_end_v3_ptr || KMP_ITT_DEBUG ) &&
587 ! __kmp_forkjoin_frames_mode && __kmp_forkjoin_frames )
589 __kmp_itt_region_joined( global_tid, 1 );
593 if ( __kmp_env_consistency_check )
594 __kmp_pop_parallel( global_tid, NULL );
608 KC_TRACE( 10, (
"__kmpc_flush: called\n" ) );
613 #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 )
626 if ( ! __kmp_cpuinfo.initialized ) {
627 __kmp_query_cpuid( & __kmp_cpuinfo );
629 if ( ! __kmp_cpuinfo.sse2 ) {
632 #if KMP_COMPILER_ICC || KMP_COMPILER_MSVC
635 __sync_synchronize();
636 #endif // KMP_COMPILER_ICC
639 #elif (KMP_ARCH_ARM || KMP_ARCH_AARCH64)
655 #error Unknown or unsupported architecture
676 int explicit_barrier_flag;
677 KC_TRACE( 10, (
"__kmpc_barrier: called T#%d\n", global_tid ) );
679 if (! TCR_4(__kmp_init_parallel))
680 __kmp_parallel_initialize();
682 if ( __kmp_env_consistency_check ) {
684 KMP_WARNING( ConstructIdentInvalid );
687 __kmp_check_barrier( global_tid, ct_barrier, loc );
690 __kmp_threads[ global_tid ]->th.th_ident = loc;
698 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
714 KC_TRACE( 10, (
"__kmpc_master: called T#%d\n", global_tid ) );
716 if( ! TCR_4( __kmp_init_parallel ) )
717 __kmp_parallel_initialize();
719 if( KMP_MASTER_GTID( global_tid ))
722 #if OMPT_SUPPORT && OMPT_TRACE
724 kmp_info_t *this_thr = __kmp_threads[ global_tid ];
725 kmp_team_t *team = this_thr -> th.th_team;
726 if ((ompt_status == ompt_status_track_callback) &&
727 ompt_callbacks.ompt_callback(ompt_event_master_begin)) {
728 int tid = __kmp_tid_from_gtid( global_tid );
729 ompt_callbacks.ompt_callback(ompt_event_master_begin)(
730 team->t.ompt_team_info.parallel_id,
731 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
736 if ( __kmp_env_consistency_check ) {
737 #if KMP_USE_DYNAMIC_LOCK
739 __kmp_push_sync( global_tid, ct_master, loc, NULL, 0 );
741 __kmp_check_sync( global_tid, ct_master, loc, NULL, 0 );
744 __kmp_push_sync( global_tid, ct_master, loc, NULL );
746 __kmp_check_sync( global_tid, ct_master, loc, NULL );
764 KC_TRACE( 10, (
"__kmpc_end_master: called T#%d\n", global_tid ) );
766 KMP_DEBUG_ASSERT( KMP_MASTER_GTID( global_tid ));
768 #if OMPT_SUPPORT && OMPT_TRACE
769 kmp_info_t *this_thr = __kmp_threads[ global_tid ];
770 kmp_team_t *team = this_thr -> th.th_team;
771 if ((ompt_status == ompt_status_track_callback) &&
772 ompt_callbacks.ompt_callback(ompt_event_master_end)) {
773 int tid = __kmp_tid_from_gtid( global_tid );
774 ompt_callbacks.ompt_callback(ompt_event_master_end)(
775 team->t.ompt_team_info.parallel_id,
776 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
780 if ( __kmp_env_consistency_check ) {
782 KMP_WARNING( ThreadIdentInvalid );
784 if( KMP_MASTER_GTID( global_tid ))
785 __kmp_pop_sync( global_tid, ct_master, loc );
801 KMP_DEBUG_ASSERT( __kmp_init_serial );
803 KC_TRACE( 10, (
"__kmpc_ordered: called T#%d\n", gtid ));
805 if (! TCR_4(__kmp_init_parallel))
806 __kmp_parallel_initialize();
809 __kmp_itt_ordered_prep( gtid );
813 th = __kmp_threads[ gtid ];
815 #if OMPT_SUPPORT && OMPT_TRACE
816 if (ompt_status & ompt_status_track) {
818 th->th.ompt_thread_info.wait_id = (uint64_t) loc;
819 th->th.ompt_thread_info.state = ompt_state_wait_ordered;
822 if ((ompt_status == ompt_status_track_callback) &&
823 ompt_callbacks.ompt_callback(ompt_event_wait_ordered)) {
824 ompt_callbacks.ompt_callback(ompt_event_wait_ordered)(
825 th->th.ompt_thread_info.wait_id);
830 if ( th -> th.th_dispatch -> th_deo_fcn != 0 )
831 (*th->th.th_dispatch->th_deo_fcn)( & gtid, & cid, loc );
833 __kmp_parallel_deo( & gtid, & cid, loc );
835 #if OMPT_SUPPORT && OMPT_TRACE
836 if (ompt_status & ompt_status_track) {
838 th->th.ompt_thread_info.state = ompt_state_work_parallel;
839 th->th.ompt_thread_info.wait_id = 0;
842 if ((ompt_status == ompt_status_track_callback) &&
843 ompt_callbacks.ompt_callback(ompt_event_acquired_ordered)) {
844 ompt_callbacks.ompt_callback(ompt_event_acquired_ordered)(
845 th->th.ompt_thread_info.wait_id);
851 __kmp_itt_ordered_start( gtid );
868 KC_TRACE( 10, (
"__kmpc_end_ordered: called T#%d\n", gtid ) );
871 __kmp_itt_ordered_end( gtid );
875 th = __kmp_threads[ gtid ];
877 if ( th -> th.th_dispatch -> th_dxo_fcn != 0 )
878 (*th->th.th_dispatch->th_dxo_fcn)( & gtid, & cid, loc );
880 __kmp_parallel_dxo( & gtid, & cid, loc );
882 #if OMPT_SUPPORT && OMPT_BLAME
883 if ((ompt_status == ompt_status_track_callback) &&
884 ompt_callbacks.ompt_callback(ompt_event_release_ordered)) {
885 ompt_callbacks.ompt_callback(ompt_event_release_ordered)(
886 th->th.ompt_thread_info.wait_id);
891 #if KMP_USE_DYNAMIC_LOCK
893 static __forceinline kmp_indirect_lock_t *
894 __kmp_get_indirect_csptr(kmp_critical_name * crit,
ident_t const * loc, kmp_int32 gtid, kmp_dyna_lockseq_t seq)
898 kmp_indirect_lock_t **lck, *ret;
899 lck = (kmp_indirect_lock_t **)crit;
900 ret = (kmp_indirect_lock_t *)TCR_PTR(*lck);
903 kmp_indirect_locktag_t tag = DYNA_GET_I_TAG(seq);
904 kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag);
906 DYNA_I_LOCK_FUNC(ilk, init)(ilk->lock);
907 DYNA_SET_I_LOCK_LOCATION(ilk, loc);
908 DYNA_SET_I_LOCK_FLAGS(ilk, kmp_lf_critical_section);
909 KA_TRACE(20, (
"__kmp_get_indirect_csptr: initialized indirect lock #%d\n", tag));
911 __kmp_itt_critical_creating(ilk->lock, loc);
913 int status = KMP_COMPARE_AND_STORE_PTR(lck, 0, ilk);
916 __kmp_itt_critical_destroyed(ilk->lock);
920 ret = (kmp_indirect_lock_t *)TCR_PTR(*lck);
921 KMP_DEBUG_ASSERT(ret != NULL);
928 #define DYNA_ACQUIRE_TAS_LOCK(lock, gtid) { \
929 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
930 if (l->lk.poll != DYNA_LOCK_FREE(tas) || \
931 ! KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), DYNA_LOCK_FREE(tas), DYNA_LOCK_BUSY(gtid+1, tas))) { \
933 KMP_FSYNC_PREPARE(l); \
934 KMP_INIT_YIELD(spins); \
935 if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \
938 KMP_YIELD_SPIN(spins); \
940 while (l->lk.poll != DYNA_LOCK_FREE(tas) || \
941 ! KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), DYNA_LOCK_FREE(tas), DYNA_LOCK_BUSY(gtid+1, tas))) { \
942 if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \
945 KMP_YIELD_SPIN(spins); \
949 KMP_FSYNC_ACQUIRED(l); \
953 #define DYNA_TEST_TAS_LOCK(lock, gtid, rc) { \
954 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
955 rc = l->lk.poll == DYNA_LOCK_FREE(tas) && \
956 KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), DYNA_LOCK_FREE(tas), DYNA_LOCK_BUSY(gtid+1, tas)); \
960 #define DYNA_RELEASE_TAS_LOCK(lock, gtid) { \
961 TCW_4(((kmp_tas_lock_t *)lock)->lk.poll, DYNA_LOCK_FREE(tas)); \
968 # include <sys/syscall.h>
970 # define FUTEX_WAIT 0
973 # define FUTEX_WAKE 1
977 #define DYNA_ACQUIRE_FUTEX_LOCK(lock, gtid) { \
978 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
979 kmp_int32 gtid_code = (gtid+1) << 1; \
981 KMP_FSYNC_PREPARE(ftx); \
982 kmp_int32 poll_val; \
983 while ((poll_val = KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), DYNA_LOCK_FREE(futex), \
984 DYNA_LOCK_BUSY(gtid_code, futex))) != DYNA_LOCK_FREE(futex)) { \
985 kmp_int32 cond = DYNA_LOCK_STRIP(poll_val) & 1; \
987 if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, poll_val | DYNA_LOCK_BUSY(1, futex))) { \
990 poll_val |= DYNA_LOCK_BUSY(1, futex); \
993 if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, NULL, NULL, 0)) != 0) { \
998 KMP_FSYNC_ACQUIRED(ftx); \
1002 #define DYNA_TEST_FUTEX_LOCK(lock, gtid, rc) { \
1003 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
1004 if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), DYNA_LOCK_FREE(futex), DYNA_LOCK_BUSY(gtid+1, futex) << 1)) { \
1005 KMP_FSYNC_ACQUIRED(ftx); \
1013 #define DYNA_RELEASE_FUTEX_LOCK(lock, gtid) { \
1014 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
1016 KMP_FSYNC_RELEASING(ftx); \
1017 kmp_int32 poll_val = KMP_XCHG_FIXED32(&(ftx->lk.poll), DYNA_LOCK_FREE(futex)); \
1018 if (DYNA_LOCK_STRIP(poll_val) & 1) { \
1019 syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, DYNA_LOCK_BUSY(1, futex), NULL, NULL, 0); \
1022 KMP_YIELD(TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)); \
1025 #endif // DYNA_HAS_FUTEX
1027 #else // KMP_USE_DYNAMIC_LOCK
1029 static kmp_user_lock_p
1030 __kmp_get_critical_section_ptr( kmp_critical_name * crit,
ident_t const * loc, kmp_int32 gtid )
1032 kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit;
1038 kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR( *lck_pp );
1040 if ( lck == NULL ) {
1045 lck = __kmp_user_lock_allocate( &idx, gtid, kmp_lf_critical_section );
1046 __kmp_init_user_lock_with_checks( lck );
1047 __kmp_set_user_lock_location( lck, loc );
1049 __kmp_itt_critical_creating( lck );
1062 int status = KMP_COMPARE_AND_STORE_PTR( lck_pp, 0, lck );
1064 if ( status == 0 ) {
1067 __kmp_itt_critical_destroyed( lck );
1071 __kmp_destroy_user_lock_with_checks( lck );
1072 __kmp_user_lock_free( &idx, gtid, lck );
1073 lck = (kmp_user_lock_p)TCR_PTR( *lck_pp );
1074 KMP_DEBUG_ASSERT( lck != NULL );
1080 #endif // KMP_USE_DYNAMIC_LOCK
1095 kmp_user_lock_p lck;
1097 KC_TRACE( 10, (
"__kmpc_critical: called T#%d\n", global_tid ) );
1099 #if KMP_USE_DYNAMIC_LOCK
1102 if (DYNA_IS_D_LOCK(__kmp_user_lock_seq)) {
1103 lck = (kmp_user_lock_p)crit;
1105 if (*((kmp_dyna_lock_t *)lck) == 0) {
1106 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)lck, 0, DYNA_GET_D_TAG(__kmp_user_lock_seq));
1108 if (__kmp_env_consistency_check) {
1109 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
1112 __kmp_itt_critical_acquiring(lck);
1114 # if DYNA_USE_FAST_TAS
1115 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1116 DYNA_ACQUIRE_TAS_LOCK(lck, global_tid);
1118 # elif DYNA_USE_FAST_FUTEX
1119 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1120 DYNA_ACQUIRE_FUTEX_LOCK(lck, global_tid);
1124 DYNA_D_LOCK_FUNC(lck, set)((kmp_dyna_lock_t *)lck, global_tid);
1127 kmp_indirect_lock_t *ilk = __kmp_get_indirect_csptr(crit, loc, global_tid, __kmp_user_lock_seq);
1129 if (__kmp_env_consistency_check) {
1130 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
1133 __kmp_itt_critical_acquiring(lck);
1135 DYNA_I_LOCK_FUNC(ilk, set)(lck, global_tid);
1138 #else // KMP_USE_DYNAMIC_LOCK
1142 KMP_CHECK_USER_LOCK_INIT();
1144 if ( ( __kmp_user_lock_kind == lk_tas )
1145 && (
sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1146 lck = (kmp_user_lock_p)crit;
1148 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1149 else if ( ( __kmp_user_lock_kind == lk_futex )
1150 && (
sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1151 lck = (kmp_user_lock_p)crit;
1155 lck = __kmp_get_critical_section_ptr( crit, loc, global_tid );
1158 if ( __kmp_env_consistency_check )
1159 __kmp_push_sync( global_tid, ct_critical, loc, lck );
1168 __kmp_itt_critical_acquiring( lck );
1171 __kmp_acquire_user_lock_with_checks( lck, global_tid );
1173 #endif // KMP_USE_DYNAMIC_LOCK
1176 __kmp_itt_critical_acquired( lck );
1179 KA_TRACE( 15, (
"__kmpc_critical: done T#%d\n", global_tid ));
1194 kmp_user_lock_p lck;
1196 KC_TRACE( 10, (
"__kmpc_end_critical: called T#%d\n", global_tid ));
1198 #if KMP_USE_DYNAMIC_LOCK
1199 if (DYNA_IS_D_LOCK(__kmp_user_lock_seq)) {
1200 lck = (kmp_user_lock_p)crit;
1201 KMP_ASSERT(lck != NULL);
1202 if (__kmp_env_consistency_check) {
1203 __kmp_pop_sync(global_tid, ct_critical, loc);
1206 __kmp_itt_critical_releasing( lck );
1208 # if DYNA_USE_FAST_TAS
1209 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1210 DYNA_RELEASE_TAS_LOCK(lck, global_tid);
1212 # elif DYNA_USE_FAST_FUTEX
1213 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1214 DYNA_RELEASE_FUTEX_LOCK(lck, global_tid);
1218 DYNA_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
1221 kmp_indirect_lock_t *ilk = (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
1222 KMP_ASSERT(ilk != NULL);
1224 if (__kmp_env_consistency_check) {
1225 __kmp_pop_sync(global_tid, ct_critical, loc);
1228 __kmp_itt_critical_releasing( lck );
1230 DYNA_I_LOCK_FUNC(ilk, unset)(lck, global_tid);
1233 #else // KMP_USE_DYNAMIC_LOCK
1235 if ( ( __kmp_user_lock_kind == lk_tas )
1236 && (
sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1237 lck = (kmp_user_lock_p)crit;
1239 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1240 else if ( ( __kmp_user_lock_kind == lk_futex )
1241 && (
sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1242 lck = (kmp_user_lock_p)crit;
1246 lck = (kmp_user_lock_p) TCR_PTR(*((kmp_user_lock_p *)crit));
1249 KMP_ASSERT(lck != NULL);
1251 if ( __kmp_env_consistency_check )
1252 __kmp_pop_sync( global_tid, ct_critical, loc );
1255 __kmp_itt_critical_releasing( lck );
1258 __kmp_release_user_lock_with_checks( lck, global_tid );
1260 #if OMPT_SUPPORT && OMPT_BLAME
1261 if ((ompt_status == ompt_status_track_callback) &&
1262 ompt_callbacks.ompt_callback(ompt_event_release_critical)) {
1263 ompt_callbacks.ompt_callback(ompt_event_release_critical)(
1268 #endif // KMP_USE_DYNAMIC_LOCK
1270 KA_TRACE( 15, (
"__kmpc_end_critical: done T#%d\n", global_tid ));
1286 KC_TRACE( 10, (
"__kmpc_barrier_master: called T#%d\n", global_tid ) );
1288 if (! TCR_4(__kmp_init_parallel))
1289 __kmp_parallel_initialize();
1291 if ( __kmp_env_consistency_check )
1292 __kmp_check_barrier( global_tid, ct_barrier, loc );
1295 __kmp_threads[global_tid]->th.th_ident = loc;
1297 status = __kmp_barrier( bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL );
1299 return (status != 0) ? 0 : 1;
1314 KC_TRACE( 10, (
"__kmpc_end_barrier_master: called T#%d\n", global_tid ));
1316 __kmp_end_split_barrier ( bs_plain_barrier, global_tid );
1334 KC_TRACE( 10, (
"__kmpc_barrier_master_nowait: called T#%d\n", global_tid ));
1336 if (! TCR_4(__kmp_init_parallel))
1337 __kmp_parallel_initialize();
1339 if ( __kmp_env_consistency_check ) {
1341 KMP_WARNING( ConstructIdentInvalid );
1343 __kmp_check_barrier( global_tid, ct_barrier, loc );
1347 __kmp_threads[global_tid]->th.th_ident = loc;
1349 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
1353 if ( __kmp_env_consistency_check ) {
1357 if ( global_tid < 0 ) {
1358 KMP_WARNING( ThreadIdentInvalid );
1364 __kmp_pop_sync( global_tid, ct_master, loc );
1387 kmp_int32 rc = __kmp_enter_single( global_tid, loc, TRUE );
1389 #if OMPT_SUPPORT && OMPT_TRACE
1390 kmp_info_t *this_thr = __kmp_threads[ global_tid ];
1391 kmp_team_t *team = this_thr -> th.th_team;
1392 int tid = __kmp_tid_from_gtid( global_tid );
1394 if ((ompt_status == ompt_status_track_callback)) {
1396 if (ompt_callbacks.ompt_callback(ompt_event_single_in_block_begin)) {
1397 ompt_callbacks.ompt_callback(ompt_event_single_in_block_begin)(
1398 team->t.ompt_team_info.parallel_id,
1399 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id,
1400 team->t.ompt_team_info.microtask);
1403 if (ompt_callbacks.ompt_callback(ompt_event_single_others_begin)) {
1404 ompt_callbacks.ompt_callback(ompt_event_single_others_begin)(
1405 team->t.ompt_team_info.parallel_id,
1406 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
1408 this_thr->th.ompt_thread_info.state = ompt_state_wait_single;
1428 __kmp_exit_single( global_tid );
1430 #if OMPT_SUPPORT && OMPT_TRACE
1431 kmp_info_t *this_thr = __kmp_threads[ global_tid ];
1432 kmp_team_t *team = this_thr -> th.th_team;
1433 int tid = __kmp_tid_from_gtid( global_tid );
1435 if ((ompt_status == ompt_status_track_callback) &&
1436 ompt_callbacks.ompt_callback(ompt_event_single_in_block_end)) {
1437 ompt_callbacks.ompt_callback(ompt_event_single_in_block_end)(
1438 team->t.ompt_team_info.parallel_id,
1439 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
1454 KE_TRACE( 10, (
"__kmpc_for_static_fini called T#%d\n", global_tid));
1456 #if OMPT_SUPPORT && OMPT_TRACE
1457 kmp_info_t *this_thr = __kmp_threads[ global_tid ];
1458 kmp_team_t *team = this_thr -> th.th_team;
1459 int tid = __kmp_tid_from_gtid( global_tid );
1461 if ((ompt_status == ompt_status_track_callback) &&
1462 ompt_callbacks.ompt_callback(ompt_event_loop_end)) {
1463 ompt_callbacks.ompt_callback(ompt_event_loop_end)(
1464 team->t.ompt_team_info.parallel_id,
1465 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
1469 if ( __kmp_env_consistency_check )
1470 __kmp_pop_workshare( global_tid, ct_pdo, loc );
1479 ompc_set_num_threads(
int arg )
1482 __kmp_set_num_threads( arg, __kmp_entry_gtid() );
1486 ompc_set_dynamic(
int flag )
1491 thread = __kmp_entry_thread();
1493 __kmp_save_internal_controls( thread );
1495 set__dynamic( thread, flag ? TRUE : FALSE );
1499 ompc_set_nested(
int flag )
1504 thread = __kmp_entry_thread();
1506 __kmp_save_internal_controls( thread );
1508 set__nested( thread, flag ? TRUE : FALSE );
1512 ompc_set_max_active_levels(
int max_active_levels )
1518 __kmp_set_max_active_levels( __kmp_entry_gtid(), max_active_levels );
1522 ompc_set_schedule( omp_sched_t kind,
int modifier )
1525 __kmp_set_schedule( __kmp_entry_gtid(), ( kmp_sched_t ) kind, modifier );
1529 ompc_get_ancestor_thread_num(
int level )
1531 return __kmp_get_ancestor_thread_num( __kmp_entry_gtid(), level );
1535 ompc_get_team_size(
int level )
1537 return __kmp_get_team_size( __kmp_entry_gtid(), level );
1541 kmpc_set_stacksize(
int arg )
1544 __kmp_aux_set_stacksize( arg );
1548 kmpc_set_stacksize_s(
size_t arg )
1551 __kmp_aux_set_stacksize( arg );
1555 kmpc_set_blocktime(
int arg )
1560 gtid = __kmp_entry_gtid();
1561 tid = __kmp_tid_from_gtid(gtid);
1562 thread = __kmp_thread_from_gtid(gtid);
1564 __kmp_aux_set_blocktime( arg, thread, tid );
1568 kmpc_set_library(
int arg )
1571 __kmp_user_set_library( (
enum library_type)arg );
1575 kmpc_set_defaults(
char const * str )
1578 __kmp_aux_set_defaults( str, KMP_STRLEN( str ) );
1582 kmpc_set_affinity_mask_proc(
int proc,
void **mask )
1584 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
1587 if ( ! TCR_4(__kmp_init_middle) ) {
1588 __kmp_middle_initialize();
1590 return __kmp_aux_set_affinity_mask_proc( proc, mask );
1595 kmpc_unset_affinity_mask_proc(
int proc,
void **mask )
1597 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
1600 if ( ! TCR_4(__kmp_init_middle) ) {
1601 __kmp_middle_initialize();
1603 return __kmp_aux_unset_affinity_mask_proc( proc, mask );
1608 kmpc_get_affinity_mask_proc(
int proc,
void **mask )
1610 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
1613 if ( ! TCR_4(__kmp_init_middle) ) {
1614 __kmp_middle_initialize();
1616 return __kmp_aux_get_affinity_mask_proc( proc, mask );
1667 KC_TRACE( 10, (
"__kmpc_copyprivate: called T#%d\n", gtid ));
1671 data_ptr = & __kmp_team_from_gtid( gtid )->t.t_copypriv_data;
1673 if ( __kmp_env_consistency_check ) {
1675 KMP_WARNING( ConstructIdentInvalid );
1681 if (didit) *data_ptr = cpy_data;
1685 __kmp_threads[gtid]->th.th_ident = loc;
1687 __kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL );
1689 if (! didit) (*cpy_func)( cpy_data, *data_ptr );
1695 __kmp_threads[gtid]->th.th_ident = loc;
1697 __kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL );
1702 #define INIT_LOCK __kmp_init_user_lock_with_checks
1703 #define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks
1704 #define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks
1705 #define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed
1706 #define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks
1707 #define ACQUIRE_NESTED_LOCK_TIMED __kmp_acquire_nested_user_lock_with_checks_timed
1708 #define RELEASE_LOCK __kmp_release_user_lock_with_checks
1709 #define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks
1710 #define TEST_LOCK __kmp_test_user_lock_with_checks
1711 #define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks
1712 #define DESTROY_LOCK __kmp_destroy_user_lock_with_checks
1713 #define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks
1722 __kmpc_init_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1723 #if KMP_USE_DYNAMIC_LOCK
1724 KMP_DEBUG_ASSERT(__kmp_init_serial);
1725 if (__kmp_env_consistency_check && user_lock == NULL) {
1726 KMP_FATAL(LockIsUninitialized,
"omp_init_lock");
1728 if (DYNA_IS_D_LOCK(__kmp_user_lock_seq)) {
1729 DYNA_INIT_D_LOCK(user_lock, __kmp_user_lock_seq);
1731 __kmp_itt_lock_creating((kmp_user_lock_p)user_lock, NULL);
1734 DYNA_INIT_I_LOCK(user_lock, __kmp_user_lock_seq);
1735 kmp_indirect_lock_t *ilk = DYNA_LOOKUP_I_LOCK(user_lock);
1736 DYNA_SET_I_LOCK_LOCATION(ilk, loc);
1738 __kmp_itt_lock_creating(ilk->lock, loc);
1742 #else // KMP_USE_DYNAMIC_LOCK
1744 static char const *
const func =
"omp_init_lock";
1745 kmp_user_lock_p lck;
1746 KMP_DEBUG_ASSERT( __kmp_init_serial );
1748 if ( __kmp_env_consistency_check ) {
1749 if ( user_lock == NULL ) {
1750 KMP_FATAL( LockIsUninitialized, func );
1754 KMP_CHECK_USER_LOCK_INIT();
1756 if ( ( __kmp_user_lock_kind == lk_tas )
1757 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1758 lck = (kmp_user_lock_p)user_lock;
1760 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1761 else if ( ( __kmp_user_lock_kind == lk_futex )
1762 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1763 lck = (kmp_user_lock_p)user_lock;
1767 lck = __kmp_user_lock_allocate( user_lock, gtid, 0 );
1770 __kmp_set_user_lock_location( lck, loc );
1773 __kmp_itt_lock_creating( lck );
1776 #endif // KMP_USE_DYNAMIC_LOCK
1781 __kmpc_init_nest_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1782 #if KMP_USE_DYNAMIC_LOCK
1784 KMP_DEBUG_ASSERT(__kmp_init_serial);
1785 if (__kmp_env_consistency_check && user_lock == NULL) {
1786 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock");
1789 kmp_dyna_lockseq_t nested_seq;
1790 switch (__kmp_user_lock_seq) {
1791 case lockseq_tas: nested_seq = lockseq_nested_tas;
break;
1793 case lockseq_futex: nested_seq = lockseq_nested_futex;
break;
1795 case lockseq_ticket: nested_seq = lockseq_nested_ticket;
break;
1796 case lockseq_queuing: nested_seq = lockseq_nested_queuing;
break;
1797 case lockseq_drdpa: nested_seq = lockseq_nested_drdpa;
break;
1798 default: nested_seq = lockseq_nested_queuing;
break;
1801 DYNA_INIT_I_LOCK(user_lock, nested_seq);
1803 kmp_indirect_lock_t *ilk = DYNA_LOOKUP_I_LOCK(user_lock);
1804 DYNA_SET_I_LOCK_LOCATION(ilk, loc);
1806 __kmp_itt_lock_creating(ilk->lock, loc);
1809 #else // KMP_USE_DYNAMIC_LOCK
1811 static char const *
const func =
"omp_init_nest_lock";
1812 kmp_user_lock_p lck;
1813 KMP_DEBUG_ASSERT( __kmp_init_serial );
1815 if ( __kmp_env_consistency_check ) {
1816 if ( user_lock == NULL ) {
1817 KMP_FATAL( LockIsUninitialized, func );
1821 KMP_CHECK_USER_LOCK_INIT();
1823 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1824 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1825 lck = (kmp_user_lock_p)user_lock;
1827 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1828 else if ( ( __kmp_user_lock_kind == lk_futex )
1829 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1830 <= OMP_NEST_LOCK_T_SIZE ) ) {
1831 lck = (kmp_user_lock_p)user_lock;
1835 lck = __kmp_user_lock_allocate( user_lock, gtid, 0 );
1838 INIT_NESTED_LOCK( lck );
1839 __kmp_set_user_lock_location( lck, loc );
1842 __kmp_itt_lock_creating( lck );
1845 #endif // KMP_USE_DYNAMIC_LOCK
1849 __kmpc_destroy_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1850 #if KMP_USE_DYNAMIC_LOCK
1853 kmp_user_lock_p lck;
1854 if (DYNA_EXTRACT_D_TAG(user_lock) == 0) {
1855 lck = ((kmp_indirect_lock_t *)DYNA_LOOKUP_I_LOCK(user_lock))->lock;
1857 lck = (kmp_user_lock_p)user_lock;
1859 __kmp_itt_lock_destroyed(lck);
1861 DYNA_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
1863 kmp_user_lock_p lck;
1865 if ( ( __kmp_user_lock_kind == lk_tas )
1866 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1867 lck = (kmp_user_lock_p)user_lock;
1869 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1870 else if ( ( __kmp_user_lock_kind == lk_futex )
1871 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1872 lck = (kmp_user_lock_p)user_lock;
1876 lck = __kmp_lookup_user_lock( user_lock,
"omp_destroy_lock" );
1880 __kmp_itt_lock_destroyed( lck );
1882 DESTROY_LOCK( lck );
1884 if ( ( __kmp_user_lock_kind == lk_tas )
1885 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1888 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1889 else if ( ( __kmp_user_lock_kind == lk_futex )
1890 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1895 __kmp_user_lock_free( user_lock, gtid, lck );
1897 #endif // KMP_USE_DYNAMIC_LOCK
1902 __kmpc_destroy_nest_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1903 #if KMP_USE_DYNAMIC_LOCK
1906 kmp_indirect_lock_t *ilk = DYNA_LOOKUP_I_LOCK(user_lock);
1907 __kmp_itt_lock_destroyed(ilk->lock);
1909 DYNA_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
1911 #else // KMP_USE_DYNAMIC_LOCK
1913 kmp_user_lock_p lck;
1915 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1916 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1917 lck = (kmp_user_lock_p)user_lock;
1919 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1920 else if ( ( __kmp_user_lock_kind == lk_futex )
1921 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1922 <= OMP_NEST_LOCK_T_SIZE ) ) {
1923 lck = (kmp_user_lock_p)user_lock;
1927 lck = __kmp_lookup_user_lock( user_lock,
"omp_destroy_nest_lock" );
1931 __kmp_itt_lock_destroyed( lck );
1934 DESTROY_NESTED_LOCK( lck );
1936 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1937 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1940 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1941 else if ( ( __kmp_user_lock_kind == lk_futex )
1942 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1943 <= OMP_NEST_LOCK_T_SIZE ) ) {
1948 __kmp_user_lock_free( user_lock, gtid, lck );
1950 #endif // KMP_USE_DYNAMIC_LOCK
1954 __kmpc_set_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1956 #if KMP_USE_DYNAMIC_LOCK
1957 int tag = DYNA_EXTRACT_D_TAG(user_lock);
1959 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
1961 # if DYNA_USE_FAST_TAS
1962 if (tag == locktag_tas && !__kmp_env_consistency_check) {
1963 DYNA_ACQUIRE_TAS_LOCK(user_lock, gtid);
1965 # elif DYNA_USE_FAST_FUTEX
1966 if (tag == locktag_futex && !__kmp_env_consistency_check) {
1967 DYNA_ACQUIRE_FUTEX_LOCK(user_lock, gtid);
1971 __kmp_direct_set_ops[tag]((kmp_dyna_lock_t *)user_lock, gtid);
1974 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
1977 #else // KMP_USE_DYNAMIC_LOCK
1979 kmp_user_lock_p lck;
1981 if ( ( __kmp_user_lock_kind == lk_tas )
1982 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1983 lck = (kmp_user_lock_p)user_lock;
1985 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1986 else if ( ( __kmp_user_lock_kind == lk_futex )
1987 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1988 lck = (kmp_user_lock_p)user_lock;
1992 lck = __kmp_lookup_user_lock( user_lock,
"omp_set_lock" );
1996 __kmp_itt_lock_acquiring( lck );
1999 ACQUIRE_LOCK( lck, gtid );
2002 __kmp_itt_lock_acquired( lck );
2005 #endif // KMP_USE_DYNAMIC_LOCK
2009 __kmpc_set_nest_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
2010 #if KMP_USE_DYNAMIC_LOCK
2013 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2015 DYNA_D_LOCK_FUNC(user_lock, set)((kmp_dyna_lock_t *)user_lock, gtid);
2017 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2020 #else // KMP_USE_DYNAMIC_LOCK
2021 kmp_user_lock_p lck;
2023 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
2024 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
2025 lck = (kmp_user_lock_p)user_lock;
2027 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
2028 else if ( ( __kmp_user_lock_kind == lk_futex )
2029 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
2030 <= OMP_NEST_LOCK_T_SIZE ) ) {
2031 lck = (kmp_user_lock_p)user_lock;
2035 lck = __kmp_lookup_user_lock( user_lock,
"omp_set_nest_lock" );
2039 __kmp_itt_lock_acquiring( lck );
2042 ACQUIRE_NESTED_LOCK( lck, gtid );
2045 __kmp_itt_lock_acquired( lck );
2047 #endif // KMP_USE_DYNAMIC_LOCK
2051 __kmpc_unset_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
2053 #if KMP_USE_DYNAMIC_LOCK
2055 int tag = DYNA_EXTRACT_D_TAG(user_lock);
2057 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2059 # if DYNA_USE_FAST_TAS
2060 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2061 DYNA_RELEASE_TAS_LOCK(user_lock, gtid);
2063 # elif DYNA_USE_FAST_FUTEX
2064 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2065 DYNA_RELEASE_FUTEX_LOCK(user_lock, gtid);
2069 __kmp_direct_unset_ops[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2072 #else // KMP_USE_DYNAMIC_LOCK
2074 kmp_user_lock_p lck;
2079 if ( ( __kmp_user_lock_kind == lk_tas )
2080 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2081 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
2084 __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );
2086 TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0);
2090 lck = (kmp_user_lock_p)user_lock;
2093 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
2094 else if ( ( __kmp_user_lock_kind == lk_futex )
2095 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2096 lck = (kmp_user_lock_p)user_lock;
2100 lck = __kmp_lookup_user_lock( user_lock,
"omp_unset_lock" );
2104 __kmp_itt_lock_releasing( lck );
2107 RELEASE_LOCK( lck, gtid );
2109 #if OMPT_SUPPORT && OMPT_BLAME
2110 if ((ompt_status == ompt_status_track_callback) &&
2111 ompt_callbacks.ompt_callback(ompt_event_release_lock)) {
2112 ompt_callbacks.ompt_callback(ompt_event_release_lock)((uint64_t) lck);
2116 #endif // KMP_USE_DYNAMIC_LOCK
2121 __kmpc_unset_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
2123 #if KMP_USE_DYNAMIC_LOCK
2126 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2128 DYNA_D_LOCK_FUNC(user_lock, unset)((kmp_dyna_lock_t *)user_lock, gtid);
2130 #else // KMP_USE_DYNAMIC_LOCK
2132 kmp_user_lock_p lck;
2136 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
2137 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
2138 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
2140 kmp_tas_lock_t *tl = (kmp_tas_lock_t*)user_lock;
2142 __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );
2144 if ( --(tl->lk.depth_locked) == 0 ) {
2145 TCW_4(tl->lk.poll, 0);
2150 lck = (kmp_user_lock_p)user_lock;
2153 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
2154 else if ( ( __kmp_user_lock_kind == lk_futex )
2155 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
2156 <= OMP_NEST_LOCK_T_SIZE ) ) {
2157 lck = (kmp_user_lock_p)user_lock;
2161 lck = __kmp_lookup_user_lock( user_lock,
"omp_unset_nest_lock" );
2165 __kmp_itt_lock_releasing( lck );
2168 int release_status = RELEASE_NESTED_LOCK( lck, gtid );
2169 #if OMPT_SUPPORT && OMPT_BLAME
2170 if (ompt_status == ompt_status_track_callback) {
2171 if (release_status == KMP_LOCK_RELEASED) {
2172 if (ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_last)) {
2173 ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_last)(
2176 }
else if (ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_prev)) {
2177 ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_prev)(
2183 #endif // KMP_USE_DYNAMIC_LOCK
2188 __kmpc_test_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
2193 #if KMP_USE_DYNAMIC_LOCK
2195 int tag = DYNA_EXTRACT_D_TAG(user_lock);
2197 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2199 # if DYNA_USE_FAST_TAS
2200 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2201 DYNA_TEST_TAS_LOCK(user_lock, gtid, rc);
2203 # elif DYNA_USE_FAST_FUTEX
2204 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2205 DYNA_TEST_FUTEX_LOCK(user_lock, gtid, rc);
2209 rc = __kmp_direct_test_ops[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2213 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2218 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
2223 #else // KMP_USE_DYNAMIC_LOCK
2225 kmp_user_lock_p lck;
2228 if ( ( __kmp_user_lock_kind == lk_tas )
2229 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2230 lck = (kmp_user_lock_p)user_lock;
2232 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
2233 else if ( ( __kmp_user_lock_kind == lk_futex )
2234 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2235 lck = (kmp_user_lock_p)user_lock;
2239 lck = __kmp_lookup_user_lock( user_lock,
"omp_test_lock" );
2243 __kmp_itt_lock_acquiring( lck );
2246 rc = TEST_LOCK( lck, gtid );
2249 __kmp_itt_lock_acquired( lck );
2251 __kmp_itt_lock_cancelled( lck );
2254 return ( rc ? FTN_TRUE : FTN_FALSE );
2258 #endif // KMP_USE_DYNAMIC_LOCK
2263 __kmpc_test_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
2265 #if KMP_USE_DYNAMIC_LOCK
2268 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2270 rc = DYNA_D_LOCK_FUNC(user_lock, test)((kmp_dyna_lock_t *)user_lock, gtid);
2273 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2275 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
2280 #else // KMP_USE_DYNAMIC_LOCK
2282 kmp_user_lock_p lck;
2285 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
2286 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
2287 lck = (kmp_user_lock_p)user_lock;
2289 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
2290 else if ( ( __kmp_user_lock_kind == lk_futex )
2291 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
2292 <= OMP_NEST_LOCK_T_SIZE ) ) {
2293 lck = (kmp_user_lock_p)user_lock;
2297 lck = __kmp_lookup_user_lock( user_lock,
"omp_test_nest_lock" );
2301 __kmp_itt_lock_acquiring( lck );
2304 rc = TEST_NESTED_LOCK( lck, gtid );
2307 __kmp_itt_lock_acquired( lck );
2309 __kmp_itt_lock_cancelled( lck );
2316 #endif // KMP_USE_DYNAMIC_LOCK
2329 #define __KMP_SET_REDUCTION_METHOD(gtid,rmethod) \
2330 ( ( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method ) = ( rmethod ) )
2332 #define __KMP_GET_REDUCTION_METHOD(gtid) \
2333 ( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method )
2339 static __forceinline
void
2340 __kmp_enter_critical_section_reduce_block(
ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) {
2346 kmp_user_lock_p lck;
2348 #if KMP_USE_DYNAMIC_LOCK
2350 if (DYNA_IS_D_LOCK(__kmp_user_lock_seq)) {
2351 lck = (kmp_user_lock_p)crit;
2352 if (*((kmp_dyna_lock_t *)lck) == 0) {
2353 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)lck, 0, DYNA_GET_D_TAG(__kmp_user_lock_seq));
2355 KMP_DEBUG_ASSERT(lck != NULL);
2356 if (__kmp_env_consistency_check) {
2357 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
2359 DYNA_D_LOCK_FUNC(lck, set)((kmp_dyna_lock_t *)lck, global_tid);
2361 kmp_indirect_lock_t *ilk = __kmp_get_indirect_csptr(crit, loc, global_tid, __kmp_user_lock_seq);
2362 KMP_DEBUG_ASSERT(ilk != NULL);
2363 if (__kmp_env_consistency_check) {
2364 __kmp_push_sync(global_tid, ct_critical, loc, ilk->lock, __kmp_user_lock_seq);
2366 DYNA_I_LOCK_FUNC(ilk, set)(ilk->lock, global_tid);
2369 #else // KMP_USE_DYNAMIC_LOCK
2374 if ( __kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE ) {
2375 lck = (kmp_user_lock_p)crit;
2378 lck = __kmp_get_critical_section_ptr( crit, loc, global_tid );
2380 KMP_DEBUG_ASSERT( lck != NULL );
2382 if ( __kmp_env_consistency_check )
2383 __kmp_push_sync( global_tid, ct_critical, loc, lck );
2385 __kmp_acquire_user_lock_with_checks( lck, global_tid );
2387 #endif // KMP_USE_DYNAMIC_LOCK
2391 static __forceinline
void
2392 __kmp_end_critical_section_reduce_block(
ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) {
2394 kmp_user_lock_p lck;
2396 #if KMP_USE_DYNAMIC_LOCK
2398 if (DYNA_IS_D_LOCK(__kmp_user_lock_seq)) {
2399 lck = (kmp_user_lock_p)crit;
2400 if (__kmp_env_consistency_check)
2401 __kmp_pop_sync(global_tid, ct_critical, loc);
2402 DYNA_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
2404 kmp_indirect_lock_t *ilk = (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
2405 if (__kmp_env_consistency_check)
2406 __kmp_pop_sync(global_tid, ct_critical, loc);
2407 DYNA_I_LOCK_FUNC(ilk, unset)(ilk->lock, global_tid);
2410 #else // KMP_USE_DYNAMIC_LOCK
2414 if ( __kmp_base_user_lock_size > 32 ) {
2415 lck = *( (kmp_user_lock_p *) crit );
2416 KMP_ASSERT( lck != NULL );
2418 lck = (kmp_user_lock_p) crit;
2421 if ( __kmp_env_consistency_check )
2422 __kmp_pop_sync( global_tid, ct_critical, loc );
2424 __kmp_release_user_lock_with_checks( lck, global_tid );
2426 #endif // KMP_USE_DYNAMIC_LOCK
2446 ident_t *loc, kmp_int32 global_tid,
2447 kmp_int32 num_vars,
size_t reduce_size,
void *reduce_data,
void (*reduce_func)(
void *lhs_data,
void *rhs_data),
2448 kmp_critical_name *lck ) {
2452 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2456 int teams_swapped = 0, task_state;
2458 KA_TRACE( 10, (
"__kmpc_reduce_nowait() enter: called T#%d\n", global_tid ) );
2465 if( ! TCR_4( __kmp_init_parallel ) )
2466 __kmp_parallel_initialize();
2469 #if KMP_USE_DYNAMIC_LOCK
2470 if ( __kmp_env_consistency_check )
2471 __kmp_push_sync( global_tid, ct_reduce, loc, NULL, 0 );
2473 if ( __kmp_env_consistency_check )
2474 __kmp_push_sync( global_tid, ct_reduce, loc, NULL );
2478 th = __kmp_thread_from_gtid(global_tid);
2479 if( th->th.th_teams_microtask ) {
2480 team = th->th.th_team;
2481 if( team->t.t_level == th->th.th_teams_level ) {
2483 KMP_DEBUG_ASSERT(!th->th.th_info.ds.ds_tid);
2486 th->th.th_info.ds.ds_tid = team->t.t_master_tid;
2487 th->th.th_team = team->t.t_parent;
2488 th->th.th_team_nproc = th->th.th_team->t.t_nproc;
2489 th->th.th_task_team = th->th.th_team->t.t_task_team[0];
2490 task_state = th->th.th_task_state;
2491 th->th.th_task_state = 0;
2494 #endif // OMP_40_ENABLED
2505 packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck );
2506 __KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method );
2508 if( packed_reduction_method == critical_reduce_block ) {
2510 __kmp_enter_critical_section_reduce_block( loc, global_tid, lck );
2513 }
else if( packed_reduction_method == empty_reduce_block ) {
2518 }
else if( packed_reduction_method == atomic_reduce_block ) {
2525 if ( __kmp_env_consistency_check )
2526 __kmp_pop_sync( global_tid, ct_reduce, loc );
2528 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2541 __kmp_threads[global_tid]->th.th_ident = loc;
2543 retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, FALSE, reduce_size, reduce_data, reduce_func );
2544 retval = ( retval != 0 ) ? ( 0 ) : ( 1 );
2548 if ( __kmp_env_consistency_check ) {
2550 __kmp_pop_sync( global_tid, ct_reduce, loc );
2561 if( teams_swapped ) {
2563 th->th.th_info.ds.ds_tid = 0;
2564 th->th.th_team = team;
2565 th->th.th_team_nproc = team->t.t_nproc;
2566 th->th.th_task_team = team->t.t_task_team[task_state];
2567 th->th.th_task_state = task_state;
2570 KA_TRACE( 10, (
"__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) );
2586 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2588 KA_TRACE( 10, (
"__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid ) );
2590 packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid );
2592 if( packed_reduction_method == critical_reduce_block ) {
2594 __kmp_end_critical_section_reduce_block( loc, global_tid, lck );
2596 }
else if( packed_reduction_method == empty_reduce_block ) {
2600 }
else if( packed_reduction_method == atomic_reduce_block ) {
2607 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2618 if ( __kmp_env_consistency_check )
2619 __kmp_pop_sync( global_tid, ct_reduce, loc );
2621 KA_TRACE( 10, (
"__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) );
2643 ident_t *loc, kmp_int32 global_tid,
2644 kmp_int32 num_vars,
size_t reduce_size,
void *reduce_data,
2645 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
2646 kmp_critical_name *lck )
2650 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2652 KA_TRACE( 10, (
"__kmpc_reduce() enter: called T#%d\n", global_tid ) );
2659 if( ! TCR_4( __kmp_init_parallel ) )
2660 __kmp_parallel_initialize();
2663 #if KMP_USE_DYNAMIC_LOCK
2664 if ( __kmp_env_consistency_check )
2665 __kmp_push_sync( global_tid, ct_reduce, loc, NULL, 0 );
2667 if ( __kmp_env_consistency_check )
2668 __kmp_push_sync( global_tid, ct_reduce, loc, NULL );
2671 packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck );
2672 __KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method );
2674 if( packed_reduction_method == critical_reduce_block ) {
2676 __kmp_enter_critical_section_reduce_block( loc, global_tid, lck );
2679 }
else if( packed_reduction_method == empty_reduce_block ) {
2684 }
else if( packed_reduction_method == atomic_reduce_block ) {
2688 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2694 __kmp_threads[global_tid]->th.th_ident = loc;
2696 retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, TRUE, reduce_size, reduce_data, reduce_func );
2697 retval = ( retval != 0 ) ? ( 0 ) : ( 1 );
2701 if ( __kmp_env_consistency_check ) {
2703 __kmp_pop_sync( global_tid, ct_reduce, loc );
2714 KA_TRACE( 10, (
"__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) );
2731 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2733 KA_TRACE( 10, (
"__kmpc_end_reduce() enter: called T#%d\n", global_tid ) );
2735 packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid );
2740 if( packed_reduction_method == critical_reduce_block ) {
2742 __kmp_end_critical_section_reduce_block( loc, global_tid, lck );
2746 __kmp_threads[global_tid]->th.th_ident = loc;
2748 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
2750 }
else if( packed_reduction_method == empty_reduce_block ) {
2756 __kmp_threads[global_tid]->th.th_ident = loc;
2758 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
2760 }
else if( packed_reduction_method == atomic_reduce_block ) {
2764 __kmp_threads[global_tid]->th.th_ident = loc;
2766 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
2768 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2771 __kmp_end_split_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid );
2780 if ( __kmp_env_consistency_check )
2781 __kmp_pop_sync( global_tid, ct_reduce, loc );
2783 KA_TRACE( 10, (
"__kmpc_end_reduce() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) );
2788 #undef __KMP_GET_REDUCTION_METHOD
2789 #undef __KMP_SET_REDUCTION_METHOD
2794 __kmpc_get_taskid() {
2797 kmp_info_t * thread;
2799 gtid = __kmp_get_gtid();
2803 thread = __kmp_thread_from_gtid( gtid );
2804 return thread->th.th_current_task->td_task_id;
2810 __kmpc_get_parent_taskid() {
2813 kmp_info_t * thread;
2814 kmp_taskdata_t * parent_task;
2816 gtid = __kmp_get_gtid();
2820 thread = __kmp_thread_from_gtid( gtid );
2821 parent_task = thread->th.th_current_task->td_parent;
2822 return ( parent_task == NULL ? 0 : parent_task->td_task_id );
2826 void __kmpc_place_threads(
int nS,
int sO,
int nC,
int cO,
int nT)
2828 if ( ! __kmp_init_serial ) {
2829 __kmp_serial_initialize();
2831 __kmp_place_num_sockets = nS;
2832 __kmp_place_socket_offset = sO;
2833 __kmp_place_num_cores = nC;
2834 __kmp_place_core_offset = cO;
2835 __kmp_place_num_threads_per_core = nT;
kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid)
void(* kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
kmp_int32 __kmpc_global_thread_num(ident_t *loc)
void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid)
#define KMP_START_EXPLICIT_TIMER(name)
"Starts" an explicit timer which will need a corresponding KMP_STOP_EXPLICIT_TIMER() macro...
#define KMP_STOP_EXPLICIT_TIMER(name)
"Stops" an explicit timer.
#define KMP_TIME_BLOCK(name)
Uses specified timer (name) to time code block.
void __kmpc_flush(ident_t *loc)
kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end(ident_t *loc)
void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid)
void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
#define KMP_IDENT_AUTOPAR
void __kmpc_begin(ident_t *loc, kmp_int32 flags)
kmp_int32 __kmpc_bound_thread_num(ident_t *loc)
kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), kmp_int32 didit)
void __kmpc_ordered(ident_t *loc, kmp_int32 gtid)
#define KMP_COUNT_BLOCK(name)
Increments specified counter (name).
void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)
void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
kmp_int32 __kmpc_in_parallel(ident_t *loc)
kmp_int32 __kmpc_ok_to_fork(ident_t *loc)
kmp_int32 __kmpc_global_num_threads(ident_t *loc)
kmp_int32 __kmpc_bound_num_threads(ident_t *loc)
void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads)
kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid)
void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)