42 #include "kmp_debug.h"
52 #define KMP_PAD(type, sz) (sizeof(type) + (sz - ((sizeof(type) - 1) % (sz)) - 1))
53 #define KMP_GTID_DNE (-2)
74 #if KMP_OS_LINUX && defined(KMP_GOMP_COMPAT)
75 # define OMP_LOCK_T_SIZE sizeof(int)
76 # define OMP_NEST_LOCK_T_SIZE sizeof(void *)
78 # define OMP_LOCK_T_SIZE sizeof(void *)
79 # define OMP_NEST_LOCK_T_SIZE sizeof(void *)
87 #define OMP_CRITICAL_SIZE sizeof(void *)
88 #define INTEL_CRITICAL_SIZE 32
93 typedef kmp_uint32 kmp_lock_flags_t;
95 #define kmp_lf_critical_section 1
100 typedef kmp_uint32 kmp_lock_index_t;
106 struct kmp_lock_pool {
107 union kmp_user_lock *next;
108 kmp_lock_index_t index;
111 typedef struct kmp_lock_pool kmp_lock_pool_t;
114 extern void __kmp_validate_locks(
void );
158 struct kmp_base_tas_lock {
159 volatile kmp_int32 poll;
161 kmp_int32 depth_locked;
164 typedef struct kmp_base_tas_lock kmp_base_tas_lock_t;
167 kmp_base_tas_lock_t lk;
168 kmp_lock_pool_t pool;
173 typedef union kmp_tas_lock kmp_tas_lock_t;
179 #define KMP_TAS_LOCK_INITIALIZER( lock ) { { 0, 0 } }
181 extern void __kmp_acquire_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
182 extern int __kmp_test_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
183 extern int __kmp_release_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
184 extern void __kmp_init_tas_lock( kmp_tas_lock_t *lck );
185 extern void __kmp_destroy_tas_lock( kmp_tas_lock_t *lck );
187 extern void __kmp_acquire_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
188 extern int __kmp_test_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
189 extern int __kmp_release_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
190 extern void __kmp_init_nested_tas_lock( kmp_tas_lock_t *lck );
191 extern void __kmp_destroy_nested_tas_lock( kmp_tas_lock_t *lck );
193 #define KMP_LOCK_RELEASED 1
194 #define KMP_LOCK_STILL_HELD 0
196 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
210 struct kmp_base_futex_lock {
211 volatile kmp_int32 poll;
214 kmp_int32 depth_locked;
217 typedef struct kmp_base_futex_lock kmp_base_futex_lock_t;
219 union kmp_futex_lock {
220 kmp_base_futex_lock_t lk;
221 kmp_lock_pool_t pool;
226 typedef union kmp_futex_lock kmp_futex_lock_t;
232 #define KMP_FUTEX_LOCK_INITIALIZER( lock ) { { 0, 0 } }
234 extern void __kmp_acquire_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
235 extern int __kmp_test_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
236 extern int __kmp_release_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
237 extern void __kmp_init_futex_lock( kmp_futex_lock_t *lck );
238 extern void __kmp_destroy_futex_lock( kmp_futex_lock_t *lck );
240 extern void __kmp_acquire_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
241 extern int __kmp_test_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
242 extern int __kmp_release_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
243 extern void __kmp_init_nested_futex_lock( kmp_futex_lock_t *lck );
244 extern void __kmp_destroy_nested_futex_lock( kmp_futex_lock_t *lck );
246 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
253 struct kmp_base_ticket_lock {
255 volatile union kmp_ticket_lock * initialized;
257 volatile kmp_uint32 next_ticket;
258 volatile kmp_uint32 now_serving;
259 volatile kmp_int32 owner_id;
260 kmp_int32 depth_locked;
261 kmp_lock_flags_t flags;
264 typedef struct kmp_base_ticket_lock kmp_base_ticket_lock_t;
266 union KMP_ALIGN_CACHE kmp_ticket_lock {
267 kmp_base_ticket_lock_t lk;
268 kmp_lock_pool_t pool;
270 char lk_pad[ KMP_PAD( kmp_base_ticket_lock_t, CACHE_LINE ) ];
273 typedef union kmp_ticket_lock kmp_ticket_lock_t;
280 #define KMP_TICKET_LOCK_INITIALIZER( lock ) { { (kmp_ticket_lock_t *) & (lock), NULL, 0, 0, 0, -1 } }
282 extern void __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
283 extern int __kmp_test_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
284 extern int __kmp_test_ticket_lock_with_cheks( kmp_ticket_lock_t *lck, kmp_int32 gtid );
285 extern int __kmp_release_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
286 extern void __kmp_init_ticket_lock( kmp_ticket_lock_t *lck );
287 extern void __kmp_destroy_ticket_lock( kmp_ticket_lock_t *lck );
289 extern void __kmp_acquire_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
290 extern int __kmp_test_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
291 extern int __kmp_release_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
292 extern void __kmp_init_nested_ticket_lock( kmp_ticket_lock_t *lck );
293 extern void __kmp_destroy_nested_ticket_lock( kmp_ticket_lock_t *lck );
300 #if KMP_USE_ADAPTIVE_LOCKS
302 struct kmp_adaptive_lock_info;
304 typedef struct kmp_adaptive_lock_info kmp_adaptive_lock_info_t;
306 #if KMP_DEBUG_ADAPTIVE_LOCKS
308 struct kmp_adaptive_lock_statistics {
310 kmp_adaptive_lock_info_t * next;
311 kmp_adaptive_lock_info_t * prev;
314 kmp_uint32 successfulSpeculations;
315 kmp_uint32 hardFailedSpeculations;
316 kmp_uint32 softFailedSpeculations;
317 kmp_uint32 nonSpeculativeAcquires;
318 kmp_uint32 nonSpeculativeAcquireAttempts;
319 kmp_uint32 lemmingYields;
322 typedef struct kmp_adaptive_lock_statistics kmp_adaptive_lock_statistics_t;
324 extern void __kmp_print_speculative_stats();
325 extern void __kmp_init_speculative_stats();
327 #endif // KMP_DEBUG_ADAPTIVE_LOCKS
329 struct kmp_adaptive_lock_info
336 kmp_uint32
volatile badness;
337 kmp_uint32
volatile acquire_attempts;
339 kmp_uint32 max_badness;
340 kmp_uint32 max_soft_retries;
342 #if KMP_DEBUG_ADAPTIVE_LOCKS
343 kmp_adaptive_lock_statistics_t
volatile stats;
347 #endif // KMP_USE_ADAPTIVE_LOCKS
350 struct kmp_base_queuing_lock {
353 volatile union kmp_queuing_lock *initialized;
359 volatile kmp_int32 tail_id;
361 volatile kmp_int32 head_id;
364 volatile kmp_uint32 next_ticket;
365 volatile kmp_uint32 now_serving;
366 volatile kmp_int32 owner_id;
367 kmp_int32 depth_locked;
369 kmp_lock_flags_t flags;
372 typedef struct kmp_base_queuing_lock kmp_base_queuing_lock_t;
374 KMP_BUILD_ASSERT( offsetof( kmp_base_queuing_lock_t, tail_id ) % 8 == 0 );
376 union KMP_ALIGN_CACHE kmp_queuing_lock {
377 kmp_base_queuing_lock_t lk;
378 kmp_lock_pool_t pool;
380 char lk_pad[ KMP_PAD( kmp_base_queuing_lock_t, CACHE_LINE ) ];
383 typedef union kmp_queuing_lock kmp_queuing_lock_t;
385 extern void __kmp_acquire_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
386 extern int __kmp_test_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
387 extern int __kmp_release_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
388 extern void __kmp_init_queuing_lock( kmp_queuing_lock_t *lck );
389 extern void __kmp_destroy_queuing_lock( kmp_queuing_lock_t *lck );
391 extern void __kmp_acquire_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
392 extern int __kmp_test_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
393 extern int __kmp_release_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
394 extern void __kmp_init_nested_queuing_lock( kmp_queuing_lock_t *lck );
395 extern void __kmp_destroy_nested_queuing_lock( kmp_queuing_lock_t *lck );
397 #if KMP_USE_ADAPTIVE_LOCKS
402 struct kmp_base_adaptive_lock {
403 kmp_base_queuing_lock qlk;
404 KMP_ALIGN(CACHE_LINE)
405 kmp_adaptive_lock_info_t adaptive;
408 typedef struct kmp_base_adaptive_lock kmp_base_adaptive_lock_t;
410 union KMP_ALIGN_CACHE kmp_adaptive_lock {
411 kmp_base_adaptive_lock_t lk;
412 kmp_lock_pool_t pool;
414 char lk_pad[ KMP_PAD(kmp_base_adaptive_lock_t, CACHE_LINE) ];
416 typedef union kmp_adaptive_lock kmp_adaptive_lock_t;
418 # define GET_QLK_PTR(l) ((kmp_queuing_lock_t *) & (l)->lk.qlk)
420 #endif // KMP_USE_ADAPTIVE_LOCKS
426 struct kmp_base_drdpa_lock {
437 volatile union kmp_drdpa_lock * initialized;
439 volatile struct kmp_lock_poll {
442 volatile kmp_uint64 mask;
443 kmp_uint64 cleanup_ticket;
444 volatile struct kmp_lock_poll * old_polls;
445 kmp_uint32 num_polls;
453 volatile kmp_uint64 next_ticket;
472 kmp_uint64 now_serving;
473 volatile kmp_uint32 owner_id;
474 kmp_int32 depth_locked;
475 kmp_lock_flags_t flags;
478 typedef struct kmp_base_drdpa_lock kmp_base_drdpa_lock_t;
480 union KMP_ALIGN_CACHE kmp_drdpa_lock {
481 kmp_base_drdpa_lock_t lk;
482 kmp_lock_pool_t pool;
484 char lk_pad[ KMP_PAD( kmp_base_drdpa_lock_t, CACHE_LINE ) ];
487 typedef union kmp_drdpa_lock kmp_drdpa_lock_t;
489 extern void __kmp_acquire_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
490 extern int __kmp_test_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
491 extern int __kmp_release_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
492 extern void __kmp_init_drdpa_lock( kmp_drdpa_lock_t *lck );
493 extern void __kmp_destroy_drdpa_lock( kmp_drdpa_lock_t *lck );
495 extern void __kmp_acquire_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
496 extern int __kmp_test_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
497 extern int __kmp_release_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
498 extern void __kmp_init_nested_drdpa_lock( kmp_drdpa_lock_t *lck );
499 extern void __kmp_destroy_nested_drdpa_lock( kmp_drdpa_lock_t *lck );
517 typedef kmp_ticket_lock_t kmp_bootstrap_lock_t;
519 #define KMP_BOOTSTRAP_LOCK_INITIALIZER( lock ) KMP_TICKET_LOCK_INITIALIZER( (lock) )
522 __kmp_acquire_bootstrap_lock( kmp_bootstrap_lock_t *lck )
524 __kmp_acquire_ticket_lock( lck, KMP_GTID_DNE );
528 __kmp_test_bootstrap_lock( kmp_bootstrap_lock_t *lck )
530 return __kmp_test_ticket_lock( lck, KMP_GTID_DNE );
534 __kmp_release_bootstrap_lock( kmp_bootstrap_lock_t *lck )
536 __kmp_release_ticket_lock( lck, KMP_GTID_DNE );
540 __kmp_init_bootstrap_lock( kmp_bootstrap_lock_t *lck )
542 __kmp_init_ticket_lock( lck );
546 __kmp_destroy_bootstrap_lock( kmp_bootstrap_lock_t *lck )
548 __kmp_destroy_ticket_lock( lck );
564 typedef kmp_ticket_lock_t kmp_lock_t;
567 __kmp_acquire_lock( kmp_lock_t *lck, kmp_int32 gtid )
569 __kmp_acquire_ticket_lock( lck, gtid );
573 __kmp_test_lock( kmp_lock_t *lck, kmp_int32 gtid )
575 return __kmp_test_ticket_lock( lck, gtid );
579 __kmp_release_lock( kmp_lock_t *lck, kmp_int32 gtid )
581 __kmp_release_ticket_lock( lck, gtid );
585 __kmp_init_lock( kmp_lock_t *lck )
587 __kmp_init_ticket_lock( lck );
591 __kmp_destroy_lock( kmp_lock_t *lck )
593 __kmp_destroy_ticket_lock( lck );
612 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
618 #if KMP_USE_ADAPTIVE_LOCKS
620 #endif // KMP_USE_ADAPTIVE_LOCKS
623 typedef enum kmp_lock_kind kmp_lock_kind_t;
625 extern kmp_lock_kind_t __kmp_user_lock_kind;
627 union kmp_user_lock {
629 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
630 kmp_futex_lock_t futex;
632 kmp_ticket_lock_t ticket;
633 kmp_queuing_lock_t queuing;
634 kmp_drdpa_lock_t drdpa;
635 #if KMP_USE_ADAPTIVE_LOCKS
636 kmp_adaptive_lock_t adaptive;
637 #endif // KMP_USE_ADAPTIVE_LOCKS
638 kmp_lock_pool_t pool;
641 typedef union kmp_user_lock *kmp_user_lock_p;
643 #if ! KMP_USE_DYNAMIC_LOCK
645 extern size_t __kmp_base_user_lock_size;
646 extern size_t __kmp_user_lock_size;
648 extern kmp_int32 ( *__kmp_get_user_lock_owner_ )( kmp_user_lock_p lck );
650 static inline kmp_int32
651 __kmp_get_user_lock_owner( kmp_user_lock_p lck )
653 KMP_DEBUG_ASSERT( __kmp_get_user_lock_owner_ != NULL );
654 return ( *__kmp_get_user_lock_owner_ )( lck );
657 extern void ( *__kmp_acquire_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
659 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
661 #define __kmp_acquire_user_lock_with_checks(lck,gtid) \
662 if (__kmp_user_lock_kind == lk_tas) { \
663 if ( __kmp_env_consistency_check ) { \
664 char const * const func = "omp_set_lock"; \
665 if ( ( sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE ) \
666 && lck->tas.lk.depth_locked != -1 ) { \
667 KMP_FATAL( LockNestableUsedAsSimple, func ); \
669 if ( ( gtid >= 0 ) && ( lck->tas.lk.poll - 1 == gtid ) ) { \
670 KMP_FATAL( LockIsAlreadyOwned, func ); \
673 if ( ( lck->tas.lk.poll != 0 ) || \
674 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \
676 KMP_FSYNC_PREPARE( lck ); \
677 KMP_INIT_YIELD( spins ); \
678 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \
681 KMP_YIELD_SPIN( spins ); \
683 while ( ( lck->tas.lk.poll != 0 ) || \
684 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \
685 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \
688 KMP_YIELD_SPIN( spins ); \
692 KMP_FSYNC_ACQUIRED( lck ); \
694 KMP_DEBUG_ASSERT( __kmp_acquire_user_lock_with_checks_ != NULL ); \
695 ( *__kmp_acquire_user_lock_with_checks_ )( lck, gtid ); \
700 __kmp_acquire_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
702 KMP_DEBUG_ASSERT( __kmp_acquire_user_lock_with_checks_ != NULL );
703 ( *__kmp_acquire_user_lock_with_checks_ )( lck, gtid );
707 extern int ( *__kmp_test_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
709 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
711 #include "kmp_i18n.h"
712 extern int __kmp_env_consistency_check;
714 __kmp_test_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
716 if ( __kmp_user_lock_kind == lk_tas ) {
717 if ( __kmp_env_consistency_check ) {
718 char const *
const func =
"omp_test_lock";
719 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
720 && lck->tas.lk.depth_locked != -1 ) {
721 KMP_FATAL( LockNestableUsedAsSimple, func );
724 return ( ( lck->tas.lk.poll == 0 ) &&
725 KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) );
727 KMP_DEBUG_ASSERT( __kmp_test_user_lock_with_checks_ != NULL );
728 return ( *__kmp_test_user_lock_with_checks_ )( lck, gtid );
733 __kmp_test_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
735 KMP_DEBUG_ASSERT( __kmp_test_user_lock_with_checks_ != NULL );
736 return ( *__kmp_test_user_lock_with_checks_ )( lck, gtid );
740 extern int ( *__kmp_release_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
743 __kmp_release_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
745 KMP_DEBUG_ASSERT( __kmp_release_user_lock_with_checks_ != NULL );
746 ( *__kmp_release_user_lock_with_checks_ ) ( lck, gtid );
749 extern void ( *__kmp_init_user_lock_with_checks_ )( kmp_user_lock_p lck );
752 __kmp_init_user_lock_with_checks( kmp_user_lock_p lck )
754 KMP_DEBUG_ASSERT( __kmp_init_user_lock_with_checks_ != NULL );
755 ( *__kmp_init_user_lock_with_checks_ )( lck );
762 extern void ( *__kmp_destroy_user_lock_ )( kmp_user_lock_p lck );
765 __kmp_destroy_user_lock( kmp_user_lock_p lck )
767 KMP_DEBUG_ASSERT( __kmp_destroy_user_lock_ != NULL );
768 ( *__kmp_destroy_user_lock_ )( lck );
771 extern void ( *__kmp_destroy_user_lock_with_checks_ )( kmp_user_lock_p lck );
774 __kmp_destroy_user_lock_with_checks( kmp_user_lock_p lck )
776 KMP_DEBUG_ASSERT( __kmp_destroy_user_lock_with_checks_ != NULL );
777 ( *__kmp_destroy_user_lock_with_checks_ )( lck );
780 extern void ( *__kmp_acquire_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
782 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
784 #define __kmp_acquire_nested_user_lock_with_checks(lck,gtid) \
785 if (__kmp_user_lock_kind == lk_tas) { \
786 if ( __kmp_env_consistency_check ) { \
787 char const * const func = "omp_set_nest_lock"; \
788 if ( ( sizeof ( kmp_tas_lock_t ) <= OMP_NEST_LOCK_T_SIZE ) \
789 && lck->tas.lk.depth_locked == -1 ) { \
790 KMP_FATAL( LockSimpleUsedAsNestable, func ); \
793 if ( lck->tas.lk.poll - 1 == gtid ) { \
794 lck->tas.lk.depth_locked += 1; \
796 if ( ( lck->tas.lk.poll != 0 ) || \
797 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \
799 KMP_FSYNC_PREPARE( lck ); \
800 KMP_INIT_YIELD( spins ); \
801 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \
804 KMP_YIELD_SPIN( spins ); \
806 while ( ( lck->tas.lk.poll != 0 ) || \
807 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \
808 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \
811 KMP_YIELD_SPIN( spins ); \
815 lck->tas.lk.depth_locked = 1; \
817 KMP_FSYNC_ACQUIRED( lck ); \
819 KMP_DEBUG_ASSERT( __kmp_acquire_nested_user_lock_with_checks_ != NULL ); \
820 ( *__kmp_acquire_nested_user_lock_with_checks_ )( lck, gtid ); \
825 __kmp_acquire_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
827 KMP_DEBUG_ASSERT( __kmp_acquire_nested_user_lock_with_checks_ != NULL );
828 ( *__kmp_acquire_nested_user_lock_with_checks_ )( lck, gtid );
832 extern int ( *__kmp_test_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
834 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
836 __kmp_test_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
838 if ( __kmp_user_lock_kind == lk_tas ) {
840 if ( __kmp_env_consistency_check ) {
841 char const *
const func =
"omp_test_nest_lock";
842 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_NEST_LOCK_T_SIZE )
843 && lck->tas.lk.depth_locked == -1 ) {
844 KMP_FATAL( LockSimpleUsedAsNestable, func );
847 KMP_DEBUG_ASSERT( gtid >= 0 );
848 if ( lck->tas.lk.poll - 1 == gtid ) {
849 return ++lck->tas.lk.depth_locked;
851 retval = ( ( lck->tas.lk.poll == 0 ) &&
852 KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) );
855 lck->tas.lk.depth_locked = 1;
859 KMP_DEBUG_ASSERT( __kmp_test_nested_user_lock_with_checks_ != NULL );
860 return ( *__kmp_test_nested_user_lock_with_checks_ )( lck, gtid );
865 __kmp_test_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
867 KMP_DEBUG_ASSERT( __kmp_test_nested_user_lock_with_checks_ != NULL );
868 return ( *__kmp_test_nested_user_lock_with_checks_ )( lck, gtid );
872 extern int ( *__kmp_release_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
875 __kmp_release_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
877 KMP_DEBUG_ASSERT( __kmp_release_nested_user_lock_with_checks_ != NULL );
878 return ( *__kmp_release_nested_user_lock_with_checks_ )( lck, gtid );
881 extern void ( *__kmp_init_nested_user_lock_with_checks_ )( kmp_user_lock_p lck );
883 static inline void __kmp_init_nested_user_lock_with_checks( kmp_user_lock_p lck )
885 KMP_DEBUG_ASSERT( __kmp_init_nested_user_lock_with_checks_ != NULL );
886 ( *__kmp_init_nested_user_lock_with_checks_ )( lck );
889 extern void ( *__kmp_destroy_nested_user_lock_with_checks_ )( kmp_user_lock_p lck );
892 __kmp_destroy_nested_user_lock_with_checks( kmp_user_lock_p lck )
894 KMP_DEBUG_ASSERT( __kmp_destroy_nested_user_lock_with_checks_ != NULL );
895 ( *__kmp_destroy_nested_user_lock_with_checks_ )( lck );
913 extern int ( *__kmp_is_user_lock_initialized_ )( kmp_user_lock_p lck );
917 extern const ident_t * ( *__kmp_get_user_lock_location_ )( kmp_user_lock_p lck );
920 __kmp_get_user_lock_location( kmp_user_lock_p lck )
922 if ( __kmp_get_user_lock_location_ != NULL ) {
923 return ( *__kmp_get_user_lock_location_ )( lck );
930 extern void ( *__kmp_set_user_lock_location_ )( kmp_user_lock_p lck,
const ident_t *loc );
933 __kmp_set_user_lock_location( kmp_user_lock_p lck,
const ident_t *loc )
935 if ( __kmp_set_user_lock_location_ != NULL ) {
936 ( *__kmp_set_user_lock_location_ )( lck, loc );
940 extern kmp_lock_flags_t ( *__kmp_get_user_lock_flags_ )( kmp_user_lock_p lck );
942 extern void ( *__kmp_set_user_lock_flags_ )( kmp_user_lock_p lck, kmp_lock_flags_t flags );
945 __kmp_set_user_lock_flags( kmp_user_lock_p lck, kmp_lock_flags_t flags )
947 if ( __kmp_set_user_lock_flags_ != NULL ) {
948 ( *__kmp_set_user_lock_flags_ )( lck, flags );
955 extern void __kmp_set_user_lock_vptrs( kmp_lock_kind_t user_lock_kind );
960 #define KMP_BIND_USER_LOCK_TEMPLATE(nest, kind, suffix) { \
961 __kmp_acquire##nest##user_lock_with_checks_ = ( void (*)( kmp_user_lock_p, kmp_int32 ) ) \
962 __kmp_acquire##nest##kind##_##suffix; \
963 __kmp_release##nest##user_lock_with_checks_ = ( int (*)( kmp_user_lock_p, kmp_int32 ) ) \
964 __kmp_release##nest##kind##_##suffix; \
965 __kmp_test##nest##user_lock_with_checks_ = ( int (*)( kmp_user_lock_p, kmp_int32 ) ) \
966 __kmp_test##nest##kind##_##suffix; \
967 __kmp_init##nest##user_lock_with_checks_ = ( void (*)( kmp_user_lock_p ) ) \
968 __kmp_init##nest##kind##_##suffix; \
969 __kmp_destroy##nest##user_lock_with_checks_ = ( void (*)( kmp_user_lock_p ) ) \
970 __kmp_destroy##nest##kind##_##suffix; \
973 #define KMP_BIND_USER_LOCK(kind) KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock)
974 #define KMP_BIND_USER_LOCK_WITH_CHECKS(kind) KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock_with_checks)
975 #define KMP_BIND_NESTED_USER_LOCK(kind) KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock)
976 #define KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(kind) KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock_with_checks)
1004 struct kmp_lock_table {
1005 kmp_lock_index_t used;
1006 kmp_lock_index_t allocated;
1007 kmp_user_lock_p * table;
1010 typedef struct kmp_lock_table kmp_lock_table_t;
1012 extern kmp_lock_table_t __kmp_user_lock_table;
1013 extern kmp_user_lock_p __kmp_lock_pool;
1015 struct kmp_block_of_locks {
1016 struct kmp_block_of_locks * next_block;
1020 typedef struct kmp_block_of_locks kmp_block_of_locks_t;
1022 extern kmp_block_of_locks_t *__kmp_lock_blocks;
1023 extern int __kmp_num_locks_in_block;
1025 extern kmp_user_lock_p __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid, kmp_lock_flags_t flags );
1026 extern void __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid, kmp_user_lock_p lck );
1027 extern kmp_user_lock_p __kmp_lookup_user_lock(
void **user_lock,
char const *func );
1028 extern void __kmp_cleanup_user_locks();
1030 #define KMP_CHECK_USER_LOCK_INIT() \
1032 if ( ! TCR_4( __kmp_init_user_locks ) ) { \
1033 __kmp_acquire_bootstrap_lock( &__kmp_initz_lock ); \
1034 if ( ! TCR_4( __kmp_init_user_locks ) ) { \
1035 TCW_4( __kmp_init_user_locks, TRUE ); \
1037 __kmp_release_bootstrap_lock( &__kmp_initz_lock ); \
1041 #endif // KMP_USE_DYNAMIC_LOCK
1046 #if KMP_USE_DYNAMIC_LOCK
1048 #define DYNA_HAS_FUTEX (KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM))
1049 #define DYNA_HAS_HLE (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC)
1050 #define DYNA_USE_FAST_FUTEX 0 && DYNA_HAS_FUTEX
1051 #define DYNA_USE_FAST_TAS 1 && DYNA_HAS_FUTEX
1058 # define FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) m(hle, a)
1059 # define DYNA_LAST_D_LOCK_SEQ lockseq_hle
1061 # define FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a)
1062 # define DYNA_LAST_D_LOCK_SEQ lockseq_futex
1063 # endif // DYNA_HAS_HLE
1064 # if KMP_USE_ADAPTIVE_LOCKS
1065 # define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) \
1066 m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \
1067 m(nested_queuing, a) m(nested_drdpa, a)
1069 # define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(drdpa, a) \
1070 m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \
1071 m(nested_queuing, a) m(nested_drdpa, a)
1072 # endif // KMP_USE_ADAPTIVE_LOCKS
1075 # define FOREACH_D_LOCK(m, a) m(tas, a) m(hle, a)
1076 # define DYNA_LAST_D_LOCK_SEQ lockseq_hle
1078 # define FOREACH_D_LOCK(m, a) m(tas, a)
1079 # define DYNA_LAST_D_LOCK_SEQ lockseq_tas
1080 # endif // DYNA_HAS_HLE
1081 # if KMP_USE_ADAPTIVE_LOCKS
1082 # define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) \
1083 m(nested_tas, a) m(nested_ticket, a) \
1084 m(nested_queuing, a) m(nested_drdpa, a)
1086 # define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(drdpa, a) \
1087 m(nested_tas, a) m(nested_ticket, a) \
1088 m(nested_queuing, a) m(nested_drdpa, a)
1089 # endif // KMP_USE_ADAPTIVE_LOCKS
1090 #endif // DYNA_HAS_FUTEX
1093 #define DYNA_LOCK_VALUE_SHIFT 8
1094 #define DYNA_LOCK_TYPE_MASK ((1<<DYNA_LOCK_VALUE_SHIFT)-1)
1095 #define DYNA_NUM_D_LOCKS DYNA_LAST_D_LOCK_SEQ
1096 #define DYNA_NUM_I_LOCKS (locktag_nested_drdpa+1)
1099 typedef kmp_uint32 kmp_dyna_lock_t;
1104 lockseq_indirect = 0,
1105 #define expand_seq(l,a) lockseq_##l,
1106 FOREACH_D_LOCK(expand_seq, 0)
1107 FOREACH_I_LOCK(expand_seq, 0)
1109 } kmp_dyna_lockseq_t;
1113 #define expand_tag(l,a) locktag_##l,
1114 FOREACH_I_LOCK(expand_tag, 0)
1116 } kmp_indirect_locktag_t;
1119 #define DYNA_IS_D_LOCK(seq) (seq >= lockseq_tas && seq <= DYNA_LAST_D_LOCK_SEQ)
1120 #define DYNA_IS_I_LOCK(seq) (seq >= lockseq_ticket && seq <= lockseq_nested_drdpa)
1121 #define DYNA_GET_I_TAG(seq) (kmp_indirect_locktag_t)(seq - lockseq_ticket)
1122 #define DYNA_GET_D_TAG(seq) (seq<<1 | 1)
1126 #define expand_tag(l,a) locktag_##l = DYNA_GET_D_TAG(lockseq_##l),
1127 FOREACH_D_LOCK(expand_tag, 0)
1129 } kmp_direct_locktag_t;
1133 kmp_user_lock_p lock;
1134 kmp_indirect_locktag_t type;
1135 } kmp_indirect_lock_t;
1138 extern void (*__kmp_direct_init_ops[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t);
1139 extern void (*__kmp_direct_destroy_ops[])(kmp_dyna_lock_t *);
1140 extern void (*(*__kmp_direct_set_ops))(kmp_dyna_lock_t *, kmp_int32);
1141 extern void (*(*__kmp_direct_unset_ops))(kmp_dyna_lock_t *, kmp_int32);
1142 extern int (*(*__kmp_direct_test_ops))(kmp_dyna_lock_t *, kmp_int32);
1145 extern void (*__kmp_indirect_init_ops[])(kmp_user_lock_p);
1146 extern void (*__kmp_indirect_destroy_ops[])(kmp_user_lock_p);
1147 extern void (*(*__kmp_indirect_set_ops))(kmp_user_lock_p, kmp_int32);
1148 extern void (*(*__kmp_indirect_unset_ops))(kmp_user_lock_p, kmp_int32);
1149 extern int (*(*__kmp_indirect_test_ops))(kmp_user_lock_p, kmp_int32);
1152 #define DYNA_EXTRACT_D_TAG(l) (*((kmp_dyna_lock_t *)(l)) & DYNA_LOCK_TYPE_MASK & -(*((kmp_dyna_lock_t *)(l)) & 1))
1155 #define DYNA_EXTRACT_I_INDEX(l) (*(kmp_lock_index_t *)(l) >> 1)
1158 #define DYNA_D_LOCK_FUNC(l, op) __kmp_direct_##op##_ops[DYNA_EXTRACT_D_TAG(l)]
1161 #define DYNA_I_LOCK_FUNC(l, op) __kmp_indirect_##op##_ops[((kmp_indirect_lock_t *)(l))->type]
1164 #define DYNA_INIT_D_LOCK(l, seq) __kmp_direct_init_ops[DYNA_GET_D_TAG(seq)]((kmp_dyna_lock_t *)l, seq)
1167 #define DYNA_INIT_I_LOCK(l, seq) __kmp_direct_init_ops[0]((kmp_dyna_lock_t *)(l), seq)
1170 #define DYNA_LOCK_FREE(type) (locktag_##type)
1173 #define DYNA_LOCK_BUSY(v, type) ((v)<<DYNA_LOCK_VALUE_SHIFT | locktag_##type)
1176 #define DYNA_LOCK_STRIP(v) ((v)>>DYNA_LOCK_VALUE_SHIFT)
1179 #define DYNA_STORE_LOCK_SEQ(type) (__kmp_user_lock_seq = lockseq_##type)
1182 extern void __kmp_init_lock_hinted(
void **,
int);
1183 extern void __kmp_init_nest_lock_hinted(
void **,
int);
1186 extern void __kmp_init_dynamic_user_locks();
1189 extern kmp_indirect_lock_t * __kmp_allocate_indirect_lock(
void **, kmp_int32, kmp_indirect_locktag_t);
1192 extern void __kmp_cleanup_indirect_user_locks();
1195 extern kmp_dyna_lockseq_t __kmp_user_lock_seq;
1198 extern void (*__kmp_indirect_set_location[DYNA_NUM_I_LOCKS])(kmp_user_lock_p,
const ident_t *);
1199 #define DYNA_SET_I_LOCK_LOCATION(lck, loc) { \
1200 if (__kmp_indirect_set_location[(lck)->type] != NULL) \
1201 __kmp_indirect_set_location[(lck)->type]((lck)->lock, loc); \
1205 extern void (*__kmp_indirect_set_flags[DYNA_NUM_I_LOCKS])(kmp_user_lock_p, kmp_lock_flags_t);
1206 #define DYNA_SET_I_LOCK_FLAGS(lck, flag) { \
1207 if (__kmp_indirect_set_flags[(lck)->type] != NULL) \
1208 __kmp_indirect_set_flags[(lck)->type]((lck)->lock, flag); \
1212 extern const ident_t * (*__kmp_indirect_get_location[DYNA_NUM_I_LOCKS])(kmp_user_lock_p);
1213 #define DYNA_GET_I_LOCK_LOCATION(lck) ( __kmp_indirect_get_location[(lck)->type] != NULL \
1214 ? __kmp_indirect_get_location[(lck)->type]((lck)->lock) \
1218 extern kmp_lock_flags_t (*__kmp_indirect_get_flags[DYNA_NUM_I_LOCKS])(kmp_user_lock_p);
1219 #define DYNA_GET_I_LOCK_FLAGS(lck) ( __kmp_indirect_get_flags[(lck)->type] != NULL \
1220 ? __kmp_indirect_get_flags[(lck)->type]((lck)->lock) \
1227 extern kmp_indirect_lock_t **__kmp_indirect_lock_table;
1229 extern kmp_lock_index_t __kmp_indirect_lock_table_size;
1231 extern kmp_lock_index_t __kmp_indirect_lock_table_next;
1235 extern int __kmp_num_locks_in_block;
1238 #define DYNA_LOOKUP_I_LOCK(l) ( (OMP_LOCK_T_SIZE < sizeof(void *)) \
1239 ? __kmp_indirect_lock_table[DYNA_EXTRACT_I_INDEX(l)] \
1240 : *((kmp_indirect_lock_t **)l) )
1244 __kmp_get_user_lock_owner(kmp_user_lock_p, kmp_uint32);
1246 #else // KMP_USE_DYNAMIC_LOCK
1248 # define DYNA_LOCK_BUSY(v, type) (v)
1249 # define DYNA_LOCK_FREE(type) 0
1250 # define DYNA_LOCK_STRIP(v) (v)
1251 # define DYNA_STORE_LOCK_SEQ(seq)
1253 #endif // KMP_USE_DYNAMIC_LOCK
1257 #endif // __cplusplus