35 #include "kmp_atomic.h"
38 typedef unsigned char uchar;
39 typedef unsigned short ushort;
562 #ifndef KMP_GOMP_COMPAT
563 int __kmp_atomic_mode = 1;
565 int __kmp_atomic_mode = 2;
570 kmp_atomic_lock_t __kmp_atomic_lock;
571 kmp_atomic_lock_t __kmp_atomic_lock_1i;
572 kmp_atomic_lock_t __kmp_atomic_lock_2i;
573 kmp_atomic_lock_t __kmp_atomic_lock_4i;
574 kmp_atomic_lock_t __kmp_atomic_lock_4r;
575 kmp_atomic_lock_t __kmp_atomic_lock_8i;
576 kmp_atomic_lock_t __kmp_atomic_lock_8r;
577 kmp_atomic_lock_t __kmp_atomic_lock_8c;
578 kmp_atomic_lock_t __kmp_atomic_lock_10r;
579 kmp_atomic_lock_t __kmp_atomic_lock_16r;
580 kmp_atomic_lock_t __kmp_atomic_lock_16c;
581 kmp_atomic_lock_t __kmp_atomic_lock_20c;
582 kmp_atomic_lock_t __kmp_atomic_lock_32c;
592 #define KMP_ATOMIC_VOLATILE volatile
594 #if ( KMP_ARCH_X86 ) && KMP_HAVE_QUAD
596 static inline void operator +=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q += rhs.q; };
597 static inline void operator -=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q -= rhs.q; };
598 static inline void operator *=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q *= rhs.q; };
599 static inline void operator /=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q /= rhs.q; };
600 static inline bool operator < ( Quad_a4_t & lhs, Quad_a4_t & rhs ) {
return lhs.q < rhs.q; }
601 static inline bool operator > ( Quad_a4_t & lhs, Quad_a4_t & rhs ) {
return lhs.q > rhs.q; }
603 static inline void operator +=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q += rhs.q; };
604 static inline void operator -=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q -= rhs.q; };
605 static inline void operator *=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q *= rhs.q; };
606 static inline void operator /=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q /= rhs.q; };
607 static inline bool operator < ( Quad_a16_t & lhs, Quad_a16_t & rhs ) {
return lhs.q < rhs.q; }
608 static inline bool operator > ( Quad_a16_t & lhs, Quad_a16_t & rhs ) {
return lhs.q > rhs.q; }
610 static inline void operator +=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q += rhs.q; };
611 static inline void operator -=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q -= rhs.q; };
612 static inline void operator *=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q *= rhs.q; };
613 static inline void operator /=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q /= rhs.q; };
615 static inline void operator +=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q += rhs.q; };
616 static inline void operator -=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q -= rhs.q; };
617 static inline void operator *=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q *= rhs.q; };
618 static inline void operator /=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q /= rhs.q; };
631 #define KMP_CHECK_GTID \
632 if ( gtid == KMP_GTID_UNKNOWN ) { \
633 gtid = __kmp_entry_gtid(); \
634 } // check and get gtid when needed
640 #define ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE, RET_TYPE) \
641 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \
643 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
644 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
648 #define ATOMIC_LOCK0 __kmp_atomic_lock // all types, for Gnu compat
649 #define ATOMIC_LOCK1i __kmp_atomic_lock_1i // char
650 #define ATOMIC_LOCK2i __kmp_atomic_lock_2i // short
651 #define ATOMIC_LOCK4i __kmp_atomic_lock_4i // long int
652 #define ATOMIC_LOCK4r __kmp_atomic_lock_4r // float
653 #define ATOMIC_LOCK8i __kmp_atomic_lock_8i // long long int
654 #define ATOMIC_LOCK8r __kmp_atomic_lock_8r // double
655 #define ATOMIC_LOCK8c __kmp_atomic_lock_8c // float complex
656 #define ATOMIC_LOCK10r __kmp_atomic_lock_10r // long double
657 #define ATOMIC_LOCK16r __kmp_atomic_lock_16r // _Quad
658 #define ATOMIC_LOCK16c __kmp_atomic_lock_16c // double complex
659 #define ATOMIC_LOCK20c __kmp_atomic_lock_20c // long double complex
660 #define ATOMIC_LOCK32c __kmp_atomic_lock_32c // _Quad complex
668 #define OP_CRITICAL(OP,LCK_ID) \
669 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
673 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
698 #ifdef KMP_GOMP_COMPAT
699 # define OP_GOMP_CRITICAL(OP,FLAG) \
700 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
702 OP_CRITICAL( OP, 0 ); \
706 # define OP_GOMP_CRITICAL(OP,FLAG)
710 # define KMP_DO_PAUSE _mm_delay_32( 1 )
712 # define KMP_DO_PAUSE KMP_CPU_PAUSE()
720 #define OP_CMPXCHG(TYPE,BITS,OP) \
722 TYPE old_value, new_value; \
723 old_value = *(TYPE volatile *)lhs; \
724 new_value = old_value OP rhs; \
725 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
726 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
727 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
731 old_value = *(TYPE volatile *)lhs; \
732 new_value = old_value OP rhs; \
744 #define OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \
748 kmp_int##BITS *vvv; \
750 struct _sss old_value, new_value; \
751 old_value.vvv = ( kmp_int##BITS * )&old_value.cmp; \
752 new_value.vvv = ( kmp_int##BITS * )&new_value.cmp; \
753 *old_value.vvv = * ( volatile kmp_int##BITS * ) lhs; \
754 new_value.cmp = old_value.cmp OP rhs; \
755 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
756 *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
757 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv ) ) \
761 *old_value.vvv = * ( volatile kmp_int##BITS * ) lhs; \
762 new_value.cmp = old_value.cmp OP rhs; \
766 #endif // USE_CMPXCHG_FIX
768 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
772 #define ATOMIC_FIXED_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
773 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
774 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
776 KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \
779 #define ATOMIC_CMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
780 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
781 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
782 OP_CMPXCHG(TYPE,BITS,OP) \
787 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
788 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
789 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
790 OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \
798 #define ATOMIC_FIXED_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
799 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
800 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
801 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
803 KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \
806 OP_CRITICAL(OP##=,LCK_ID) \
810 #define ATOMIC_CMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
811 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
812 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
813 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
814 OP_CMPXCHG(TYPE,BITS,OP) \
817 OP_CRITICAL(OP##=,LCK_ID) \
823 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
824 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
825 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
826 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
827 OP_CMPXCHG(TYPE,BITS,OP) \
830 OP_CRITICAL(OP##=,LCK_ID) \
834 #endif // USE_CMPXCHG_FIX
838 ATOMIC_FIXED_ADD( fixed4, add, kmp_int32, 32, +, 4i, 3, 0 )
839 ATOMIC_FIXED_ADD( fixed4, sub, kmp_int32, 32, -, 4i, 3, 0 )
841 ATOMIC_CMPXCHG( float4, add, kmp_real32, 32, +, 4r, 3, KMP_ARCH_X86 )
842 ATOMIC_CMPXCHG( float4, sub, kmp_real32, 32, -, 4r, 3, KMP_ARCH_X86 )
845 ATOMIC_FIXED_ADD( fixed8, add, kmp_int64, 64, +, 8i, 7, KMP_ARCH_X86 )
846 ATOMIC_FIXED_ADD( fixed8, sub, kmp_int64, 64, -, 8i, 7, KMP_ARCH_X86 )
848 ATOMIC_CMPXCHG( float8, add, kmp_real64, 64, +, 8r, 7, KMP_ARCH_X86 )
849 ATOMIC_CMPXCHG( float8, sub, kmp_real64, 64, -, 8r, 7, KMP_ARCH_X86 )
866 ATOMIC_CMPXCHG( fixed1, add, kmp_int8, 8, +, 1i, 0, KMP_ARCH_X86 )
867 ATOMIC_CMPXCHG( fixed1, andb, kmp_int8, 8, &, 1i, 0, 0 )
868 ATOMIC_CMPXCHG( fixed1, div, kmp_int8, 8, /, 1i, 0, KMP_ARCH_X86 )
869 ATOMIC_CMPXCHG( fixed1u, div, kmp_uint8, 8, /, 1i, 0, KMP_ARCH_X86 )
870 ATOMIC_CMPXCHG( fixed1, mul, kmp_int8, 8, *, 1i, 0, KMP_ARCH_X86 )
871 ATOMIC_CMPXCHG( fixed1, orb, kmp_int8, 8, |, 1i, 0, 0 )
872 ATOMIC_CMPXCHG( fixed1, shl, kmp_int8, 8, <<, 1i, 0, KMP_ARCH_X86 )
873 ATOMIC_CMPXCHG( fixed1, shr, kmp_int8, 8, >>, 1i, 0, KMP_ARCH_X86 )
874 ATOMIC_CMPXCHG( fixed1u, shr, kmp_uint8, 8, >>, 1i, 0, KMP_ARCH_X86 )
875 ATOMIC_CMPXCHG( fixed1, sub, kmp_int8, 8, -, 1i, 0, KMP_ARCH_X86 )
876 ATOMIC_CMPXCHG( fixed1, xor, kmp_int8, 8, ^, 1i, 0, 0 )
877 ATOMIC_CMPXCHG( fixed2, add, kmp_int16, 16, +, 2i, 1, KMP_ARCH_X86 )
878 ATOMIC_CMPXCHG( fixed2, andb, kmp_int16, 16, &, 2i, 1, 0 )
879 ATOMIC_CMPXCHG( fixed2, div, kmp_int16, 16, /, 2i, 1, KMP_ARCH_X86 )
880 ATOMIC_CMPXCHG( fixed2u, div, kmp_uint16, 16, /, 2i, 1, KMP_ARCH_X86 )
881 ATOMIC_CMPXCHG( fixed2, mul, kmp_int16, 16, *, 2i, 1, KMP_ARCH_X86 )
882 ATOMIC_CMPXCHG( fixed2, orb, kmp_int16, 16, |, 2i, 1, 0 )
883 ATOMIC_CMPXCHG( fixed2, shl, kmp_int16, 16, <<, 2i, 1, KMP_ARCH_X86 )
884 ATOMIC_CMPXCHG( fixed2, shr, kmp_int16, 16, >>, 2i, 1, KMP_ARCH_X86 )
885 ATOMIC_CMPXCHG( fixed2u, shr, kmp_uint16, 16, >>, 2i, 1, KMP_ARCH_X86 )
886 ATOMIC_CMPXCHG( fixed2, sub, kmp_int16, 16, -, 2i, 1, KMP_ARCH_X86 )
887 ATOMIC_CMPXCHG( fixed2, xor, kmp_int16, 16, ^, 2i, 1, 0 )
888 ATOMIC_CMPXCHG( fixed4, andb, kmp_int32, 32, &, 4i, 3, 0 )
889 ATOMIC_CMPXCHG( fixed4, div, kmp_int32, 32, /, 4i, 3, KMP_ARCH_X86 )
890 ATOMIC_CMPXCHG( fixed4u, div, kmp_uint32, 32, /, 4i, 3, KMP_ARCH_X86 )
891 ATOMIC_CMPXCHG( fixed4, mul, kmp_int32, 32, *, 4i, 3, KMP_ARCH_X86 )
892 ATOMIC_CMPXCHG( fixed4, orb, kmp_int32, 32, |, 4i, 3, 0 )
893 ATOMIC_CMPXCHG( fixed4, shl, kmp_int32, 32, <<, 4i, 3, KMP_ARCH_X86 )
894 ATOMIC_CMPXCHG( fixed4, shr, kmp_int32, 32, >>, 4i, 3, KMP_ARCH_X86 )
895 ATOMIC_CMPXCHG( fixed4u, shr, kmp_uint32, 32, >>, 4i, 3, KMP_ARCH_X86 )
896 ATOMIC_CMPXCHG( fixed4, xor, kmp_int32, 32, ^, 4i, 3, 0 )
897 ATOMIC_CMPXCHG( fixed8, andb, kmp_int64, 64, &, 8i, 7, KMP_ARCH_X86 )
898 ATOMIC_CMPXCHG( fixed8, div, kmp_int64, 64, /, 8i, 7, KMP_ARCH_X86 )
899 ATOMIC_CMPXCHG( fixed8u, div, kmp_uint64, 64, /, 8i, 7, KMP_ARCH_X86 )
900 ATOMIC_CMPXCHG( fixed8, mul, kmp_int64, 64, *, 8i, 7, KMP_ARCH_X86 )
901 ATOMIC_CMPXCHG( fixed8, orb, kmp_int64, 64, |, 8i, 7, KMP_ARCH_X86 )
902 ATOMIC_CMPXCHG( fixed8, shl, kmp_int64, 64, <<, 8i, 7, KMP_ARCH_X86 )
903 ATOMIC_CMPXCHG( fixed8, shr, kmp_int64, 64, >>, 8i, 7, KMP_ARCH_X86 )
904 ATOMIC_CMPXCHG( fixed8u, shr, kmp_uint64, 64, >>, 8i, 7, KMP_ARCH_X86 )
905 ATOMIC_CMPXCHG( fixed8, xor, kmp_int64, 64, ^, 8i, 7, KMP_ARCH_X86 )
906 ATOMIC_CMPXCHG( float4, div, kmp_real32, 32, /, 4r, 3, KMP_ARCH_X86 )
907 ATOMIC_CMPXCHG( float4, mul, kmp_real32, 32, *, 4r, 3, KMP_ARCH_X86 )
908 ATOMIC_CMPXCHG( float8, div, kmp_real64, 64, /, 8r, 7, KMP_ARCH_X86 )
909 ATOMIC_CMPXCHG( float8, mul, kmp_real64, 64, *, 8r, 7, KMP_ARCH_X86 )
920 #define ATOMIC_CRIT_L(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
921 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
922 OP_GOMP_CRITICAL( = *lhs OP, GOMP_FLAG ) \
923 OP_CRITICAL( = *lhs OP, LCK_ID ) \
926 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
930 #define ATOMIC_CMPX_L(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
931 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
932 OP_GOMP_CRITICAL( = *lhs OP, GOMP_FLAG ) \
933 OP_CMPXCHG(TYPE,BITS,OP) \
939 #define ATOMIC_CMPX_L(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
940 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
941 OP_GOMP_CRITICAL(= *lhs OP,GOMP_FLAG) \
942 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
943 OP_CMPXCHG(TYPE,BITS,OP) \
946 OP_CRITICAL(= *lhs OP,LCK_ID) \
951 ATOMIC_CMPX_L( fixed1, andl,
char, 8, &&, 1i, 0, KMP_ARCH_X86 )
952 ATOMIC_CMPX_L( fixed1, orl,
char, 8, ||, 1i, 0, KMP_ARCH_X86 )
953 ATOMIC_CMPX_L( fixed2, andl,
short, 16, &&, 2i, 1, KMP_ARCH_X86 )
954 ATOMIC_CMPX_L( fixed2, orl,
short, 16, ||, 2i, 1, KMP_ARCH_X86 )
955 ATOMIC_CMPX_L( fixed4, andl, kmp_int32, 32, &&, 4i, 3, 0 )
956 ATOMIC_CMPX_L( fixed4, orl, kmp_int32, 32, ||, 4i, 3, 0 )
957 ATOMIC_CMPX_L( fixed8, andl, kmp_int64, 64, &&, 8i, 7, KMP_ARCH_X86 )
958 ATOMIC_CMPX_L( fixed8, orl, kmp_int64, 64, ||, 8i, 7, KMP_ARCH_X86 )
971 #define MIN_MAX_CRITSECT(OP,LCK_ID) \
972 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
974 if ( *lhs OP rhs ) { \
977 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
980 #ifdef KMP_GOMP_COMPAT
981 #define GOMP_MIN_MAX_CRITSECT(OP,FLAG) \
982 if (( FLAG ) && ( __kmp_atomic_mode == 2 )) { \
984 MIN_MAX_CRITSECT( OP, 0 ); \
988 #define GOMP_MIN_MAX_CRITSECT(OP,FLAG)
992 #define MIN_MAX_CMPXCHG(TYPE,BITS,OP) \
994 TYPE KMP_ATOMIC_VOLATILE temp_val; \
997 old_value = temp_val; \
998 while ( old_value OP rhs && \
999 ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1000 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1001 *VOLATILE_CAST(kmp_int##BITS *) &rhs ) ) \
1005 old_value = temp_val; \
1011 #define MIN_MAX_CRITICAL(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1012 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1013 if ( *lhs OP rhs ) { \
1014 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \
1015 MIN_MAX_CRITSECT(OP,LCK_ID) \
1019 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1023 #define MIN_MAX_COMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1024 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1025 if ( *lhs OP rhs ) { \
1026 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \
1027 MIN_MAX_CMPXCHG(TYPE,BITS,OP) \
1034 #define MIN_MAX_COMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1035 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1036 if ( *lhs OP rhs ) { \
1037 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \
1038 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1039 MIN_MAX_CMPXCHG(TYPE,BITS,OP) \
1042 MIN_MAX_CRITSECT(OP,LCK_ID) \
1048 MIN_MAX_COMPXCHG( fixed1, max,
char, 8, <, 1i, 0, KMP_ARCH_X86 )
1049 MIN_MAX_COMPXCHG( fixed1, min,
char, 8, >, 1i, 0, KMP_ARCH_X86 )
1050 MIN_MAX_COMPXCHG( fixed2, max,
short, 16, <, 2i, 1, KMP_ARCH_X86 )
1051 MIN_MAX_COMPXCHG( fixed2, min,
short, 16, >, 2i, 1, KMP_ARCH_X86 )
1052 MIN_MAX_COMPXCHG( fixed4, max, kmp_int32, 32, <, 4i, 3, 0 )
1053 MIN_MAX_COMPXCHG( fixed4, min, kmp_int32, 32, >, 4i, 3, 0 )
1054 MIN_MAX_COMPXCHG( fixed8, max, kmp_int64, 64, <, 8i, 7, KMP_ARCH_X86 )
1055 MIN_MAX_COMPXCHG( fixed8, min, kmp_int64, 64, >, 8i, 7, KMP_ARCH_X86 )
1056 MIN_MAX_COMPXCHG( float4, max, kmp_real32, 32, <, 4r, 3, KMP_ARCH_X86 )
1057 MIN_MAX_COMPXCHG( float4, min, kmp_real32, 32, >, 4r, 3, KMP_ARCH_X86 )
1058 MIN_MAX_COMPXCHG( float8, max, kmp_real64, 64, <, 8r, 7, KMP_ARCH_X86 )
1059 MIN_MAX_COMPXCHG( float8, min, kmp_real64, 64, >, 8r, 7, KMP_ARCH_X86 )
1061 MIN_MAX_CRITICAL( float16, max, QUAD_LEGACY, <, 16r, 1 )
1062 MIN_MAX_CRITICAL( float16, min, QUAD_LEGACY, >, 16r, 1 )
1063 #if ( KMP_ARCH_X86 )
1064 MIN_MAX_CRITICAL( float16, max_a16, Quad_a16_t, <, 16r, 1 )
1065 MIN_MAX_CRITICAL( float16, min_a16, Quad_a16_t, >, 16r, 1 )
1071 #define ATOMIC_CRIT_EQV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1072 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1073 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \
1074 OP_CRITICAL(^=~,LCK_ID) \
1078 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1081 #define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1082 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1083 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \
1084 OP_CMPXCHG(TYPE,BITS,OP) \
1090 #define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1091 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1092 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \
1093 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1094 OP_CMPXCHG(TYPE,BITS,OP) \
1097 OP_CRITICAL(^=~,LCK_ID) \
1102 ATOMIC_CMPXCHG( fixed1, neqv, kmp_int8, 8, ^, 1i, 0, KMP_ARCH_X86 )
1103 ATOMIC_CMPXCHG( fixed2, neqv, kmp_int16, 16, ^, 2i, 1, KMP_ARCH_X86 )
1104 ATOMIC_CMPXCHG( fixed4, neqv, kmp_int32, 32, ^, 4i, 3, KMP_ARCH_X86 )
1105 ATOMIC_CMPXCHG( fixed8, neqv, kmp_int64, 64, ^, 8i, 7, KMP_ARCH_X86 )
1106 ATOMIC_CMPX_EQV( fixed1, eqv, kmp_int8, 8, ^~, 1i, 0, KMP_ARCH_X86 )
1107 ATOMIC_CMPX_EQV( fixed2, eqv, kmp_int16, 16, ^~, 2i, 1, KMP_ARCH_X86 )
1108 ATOMIC_CMPX_EQV( fixed4, eqv, kmp_int32, 32, ^~, 4i, 3, KMP_ARCH_X86 )
1109 ATOMIC_CMPX_EQV( fixed8, eqv, kmp_int64, 64, ^~, 8i, 7, KMP_ARCH_X86 )
1117 #define ATOMIC_CRITICAL(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1118 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1119 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1120 OP_CRITICAL(OP##=,LCK_ID) \
1125 ATOMIC_CRITICAL( float10, add,
long double, +, 10r, 1 )
1126 ATOMIC_CRITICAL( float10, sub,
long double, -, 10r, 1 )
1127 ATOMIC_CRITICAL( float10, mul,
long double, *, 10r, 1 )
1128 ATOMIC_CRITICAL( float10, div,
long double, /, 10r, 1 )
1131 ATOMIC_CRITICAL( float16, add, QUAD_LEGACY, +, 16r, 1 )
1132 ATOMIC_CRITICAL( float16, sub, QUAD_LEGACY, -, 16r, 1 )
1133 ATOMIC_CRITICAL( float16, mul, QUAD_LEGACY, *, 16r, 1 )
1134 ATOMIC_CRITICAL( float16, div, QUAD_LEGACY, /, 16r, 1 )
1135 #if ( KMP_ARCH_X86 )
1136 ATOMIC_CRITICAL( float16, add_a16, Quad_a16_t, +, 16r, 1 )
1137 ATOMIC_CRITICAL( float16, sub_a16, Quad_a16_t, -, 16r, 1 )
1138 ATOMIC_CRITICAL( float16, mul_a16, Quad_a16_t, *, 16r, 1 )
1139 ATOMIC_CRITICAL( float16, div_a16, Quad_a16_t, /, 16r, 1 )
1146 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, add, kmp_cmplx32, 64, +, 8c, 7, 1 )
1147 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7, 1 )
1148 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7, 1 )
1149 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, div, kmp_cmplx32, 64, /, 8c, 7, 1 )
1152 ATOMIC_CRITICAL( cmplx4, add, kmp_cmplx32, +, 8c, 1 )
1153 ATOMIC_CRITICAL( cmplx4, sub, kmp_cmplx32, -, 8c, 1 )
1154 ATOMIC_CRITICAL( cmplx4, mul, kmp_cmplx32, *, 8c, 1 )
1155 ATOMIC_CRITICAL( cmplx4, div, kmp_cmplx32, /, 8c, 1 )
1156 #endif // USE_CMPXCHG_FIX
1158 ATOMIC_CRITICAL( cmplx8, add, kmp_cmplx64, +, 16c, 1 )
1159 ATOMIC_CRITICAL( cmplx8, sub, kmp_cmplx64, -, 16c, 1 )
1160 ATOMIC_CRITICAL( cmplx8, mul, kmp_cmplx64, *, 16c, 1 )
1161 ATOMIC_CRITICAL( cmplx8, div, kmp_cmplx64, /, 16c, 1 )
1162 ATOMIC_CRITICAL( cmplx10, add, kmp_cmplx80, +, 20c, 1 )
1163 ATOMIC_CRITICAL( cmplx10, sub, kmp_cmplx80, -, 20c, 1 )
1164 ATOMIC_CRITICAL( cmplx10, mul, kmp_cmplx80, *, 20c, 1 )
1165 ATOMIC_CRITICAL( cmplx10, div, kmp_cmplx80, /, 20c, 1 )
1167 ATOMIC_CRITICAL( cmplx16, add, CPLX128_LEG, +, 32c, 1 )
1168 ATOMIC_CRITICAL( cmplx16, sub, CPLX128_LEG, -, 32c, 1 )
1169 ATOMIC_CRITICAL( cmplx16, mul, CPLX128_LEG, *, 32c, 1 )
1170 ATOMIC_CRITICAL( cmplx16, div, CPLX128_LEG, /, 32c, 1 )
1171 #if ( KMP_ARCH_X86 )
1172 ATOMIC_CRITICAL( cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c, 1 )
1173 ATOMIC_CRITICAL( cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c, 1 )
1174 ATOMIC_CRITICAL( cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c, 1 )
1175 ATOMIC_CRITICAL( cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c, 1 )
1183 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1191 #define OP_CRITICAL_REV(OP,LCK_ID) \
1192 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1194 (*lhs) = (rhs) OP (*lhs); \
1196 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1198 #ifdef KMP_GOMP_COMPAT
1199 #define OP_GOMP_CRITICAL_REV(OP,FLAG) \
1200 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1202 OP_CRITICAL_REV( OP, 0 ); \
1206 #define OP_GOMP_CRITICAL_REV(OP,FLAG)
1214 #define ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE, RET_TYPE) \
1215 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \
1217 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1218 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid ));
1227 #define OP_CMPXCHG_REV(TYPE,BITS,OP) \
1229 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1230 TYPE old_value, new_value; \
1232 old_value = temp_val; \
1233 new_value = rhs OP old_value; \
1234 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1235 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1236 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
1241 old_value = temp_val; \
1242 new_value = rhs OP old_value; \
1247 #define ATOMIC_CMPXCHG_REV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,GOMP_FLAG) \
1248 ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE,void) \
1249 OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \
1250 OP_CMPXCHG_REV(TYPE,BITS,OP) \
1267 ATOMIC_CMPXCHG_REV( fixed1, div, kmp_int8, 8, /, 1i, KMP_ARCH_X86 )
1268 ATOMIC_CMPXCHG_REV( fixed1u, div, kmp_uint8, 8, /, 1i, KMP_ARCH_X86 )
1269 ATOMIC_CMPXCHG_REV( fixed1, shl, kmp_int8, 8, <<, 1i, KMP_ARCH_X86 )
1270 ATOMIC_CMPXCHG_REV( fixed1, shr, kmp_int8, 8, >>, 1i, KMP_ARCH_X86 )
1271 ATOMIC_CMPXCHG_REV( fixed1u, shr, kmp_uint8, 8, >>, 1i, KMP_ARCH_X86 )
1272 ATOMIC_CMPXCHG_REV( fixed1, sub, kmp_int8, 8, -, 1i, KMP_ARCH_X86 )
1274 ATOMIC_CMPXCHG_REV( fixed2, div, kmp_int16, 16, /, 2i, KMP_ARCH_X86 )
1275 ATOMIC_CMPXCHG_REV( fixed2u, div, kmp_uint16, 16, /, 2i, KMP_ARCH_X86 )
1276 ATOMIC_CMPXCHG_REV( fixed2, shl, kmp_int16, 16, <<, 2i, KMP_ARCH_X86 )
1277 ATOMIC_CMPXCHG_REV( fixed2, shr, kmp_int16, 16, >>, 2i, KMP_ARCH_X86 )
1278 ATOMIC_CMPXCHG_REV( fixed2u, shr, kmp_uint16, 16, >>, 2i, KMP_ARCH_X86 )
1279 ATOMIC_CMPXCHG_REV( fixed2, sub, kmp_int16, 16, -, 2i, KMP_ARCH_X86 )
1281 ATOMIC_CMPXCHG_REV( fixed4, div, kmp_int32, 32, /, 4i, KMP_ARCH_X86 )
1282 ATOMIC_CMPXCHG_REV( fixed4u, div, kmp_uint32, 32, /, 4i, KMP_ARCH_X86 )
1283 ATOMIC_CMPXCHG_REV( fixed4, shl, kmp_int32, 32, <<, 4i, KMP_ARCH_X86 )
1284 ATOMIC_CMPXCHG_REV( fixed4, shr, kmp_int32, 32, >>, 4i, KMP_ARCH_X86 )
1285 ATOMIC_CMPXCHG_REV( fixed4u, shr, kmp_uint32, 32, >>, 4i, KMP_ARCH_X86 )
1286 ATOMIC_CMPXCHG_REV( fixed4, sub, kmp_int32, 32, -, 4i, KMP_ARCH_X86 )
1288 ATOMIC_CMPXCHG_REV( fixed8, div, kmp_int64, 64, /, 8i, KMP_ARCH_X86 )
1289 ATOMIC_CMPXCHG_REV( fixed8u, div, kmp_uint64, 64, /, 8i, KMP_ARCH_X86 )
1290 ATOMIC_CMPXCHG_REV( fixed8, shl, kmp_int64, 64, <<, 8i, KMP_ARCH_X86 )
1291 ATOMIC_CMPXCHG_REV( fixed8, shr, kmp_int64, 64, >>, 8i, KMP_ARCH_X86 )
1292 ATOMIC_CMPXCHG_REV( fixed8u, shr, kmp_uint64, 64, >>, 8i, KMP_ARCH_X86 )
1293 ATOMIC_CMPXCHG_REV( fixed8, sub, kmp_int64, 64, -, 8i, KMP_ARCH_X86 )
1295 ATOMIC_CMPXCHG_REV( float4, div, kmp_real32, 32, /, 4r, KMP_ARCH_X86 )
1296 ATOMIC_CMPXCHG_REV( float4, sub, kmp_real32, 32, -, 4r, KMP_ARCH_X86 )
1298 ATOMIC_CMPXCHG_REV( float8, div, kmp_real64, 64, /, 8r, KMP_ARCH_X86 )
1299 ATOMIC_CMPXCHG_REV( float8, sub, kmp_real64, 64, -, 8r, KMP_ARCH_X86 )
1307 #define ATOMIC_CRITICAL_REV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1308 ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE,void) \
1309 OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \
1310 OP_CRITICAL_REV(OP,LCK_ID) \
1315 ATOMIC_CRITICAL_REV( float10, sub,
long double, -, 10r, 1 )
1316 ATOMIC_CRITICAL_REV( float10, div,
long double, /, 10r, 1 )
1319 ATOMIC_CRITICAL_REV( float16, sub, QUAD_LEGACY, -, 16r, 1 )
1320 ATOMIC_CRITICAL_REV( float16, div, QUAD_LEGACY, /, 16r, 1 )
1321 #if ( KMP_ARCH_X86 )
1322 ATOMIC_CRITICAL_REV( float16, sub_a16, Quad_a16_t, -, 16r, 1 )
1323 ATOMIC_CRITICAL_REV( float16, div_a16, Quad_a16_t, /, 16r, 1 )
1328 ATOMIC_CRITICAL_REV( cmplx4, sub, kmp_cmplx32, -, 8c, 1 )
1329 ATOMIC_CRITICAL_REV( cmplx4, div, kmp_cmplx32, /, 8c, 1 )
1330 ATOMIC_CRITICAL_REV( cmplx8, sub, kmp_cmplx64, -, 16c, 1 )
1331 ATOMIC_CRITICAL_REV( cmplx8, div, kmp_cmplx64, /, 16c, 1 )
1332 ATOMIC_CRITICAL_REV( cmplx10, sub, kmp_cmplx80, -, 20c, 1 )
1333 ATOMIC_CRITICAL_REV( cmplx10, div, kmp_cmplx80, /, 20c, 1 )
1335 ATOMIC_CRITICAL_REV( cmplx16, sub, CPLX128_LEG, -, 32c, 1 )
1336 ATOMIC_CRITICAL_REV( cmplx16, div, CPLX128_LEG, /, 32c, 1 )
1337 #if ( KMP_ARCH_X86 )
1338 ATOMIC_CRITICAL_REV( cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c, 1 )
1339 ATOMIC_CRITICAL_REV( cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c, 1 )
1344 #endif //KMP_ARCH_X86 || KMP_ARCH_X86_64
1347 #endif //OMP_40_ENABLED
1361 #define ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1362 void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( ident_t *id_ref, int gtid, TYPE * lhs, RTYPE rhs ) \
1364 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1365 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", gtid ));
1368 #define ATOMIC_CRITICAL_FP(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG) \
1369 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1370 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1371 OP_CRITICAL(OP##=,LCK_ID) \
1375 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1378 #define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1379 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1380 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1381 OP_CMPXCHG(TYPE,BITS,OP) \
1387 #define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1388 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1389 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1390 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1391 OP_CMPXCHG(TYPE,BITS,OP) \
1394 OP_CRITICAL(OP##=,LCK_ID) \
1400 ATOMIC_CMPXCHG_MIX( fixed1,
char, mul, 8, *, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 )
1401 ATOMIC_CMPXCHG_MIX( fixed1,
char, div, 8, /, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 )
1402 ATOMIC_CMPXCHG_MIX( fixed2,
short, mul, 16, *, float8, kmp_real64, 2i, 1, KMP_ARCH_X86 )
1403 ATOMIC_CMPXCHG_MIX( fixed2,
short, div, 16, /, float8, kmp_real64, 2i, 1, KMP_ARCH_X86 )
1404 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3, 0 )
1405 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3, 0 )
1406 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7, KMP_ARCH_X86 )
1407 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7, KMP_ARCH_X86 )
1408 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1409 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1410 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1411 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1415 ATOMIC_CMPXCHG_MIX( fixed1,
char, add, 8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1416 ATOMIC_CMPXCHG_MIX( fixed1,
char, sub, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1417 ATOMIC_CMPXCHG_MIX( fixed1,
char, mul, 8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1418 ATOMIC_CMPXCHG_MIX( fixed1,
char, div, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1419 ATOMIC_CMPXCHG_MIX( fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1421 ATOMIC_CMPXCHG_MIX( fixed2,
short, add, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1422 ATOMIC_CMPXCHG_MIX( fixed2,
short, sub, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1423 ATOMIC_CMPXCHG_MIX( fixed2,
short, mul, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1424 ATOMIC_CMPXCHG_MIX( fixed2,
short, div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1425 ATOMIC_CMPXCHG_MIX( fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1427 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3, 0 )
1428 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3, 0 )
1429 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3, 0 )
1430 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3, 0 )
1431 ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3, 0 )
1433 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1434 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1435 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1436 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1437 ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1439 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1440 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1441 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1442 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1444 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1445 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1446 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1447 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1449 ATOMIC_CRITICAL_FP( float10,
long double, add, +, fp, _Quad, 10r, 1 )
1450 ATOMIC_CRITICAL_FP( float10,
long double, sub, -, fp, _Quad, 10r, 1 )
1451 ATOMIC_CRITICAL_FP( float10,
long double, mul, *, fp, _Quad, 10r, 1 )
1452 ATOMIC_CRITICAL_FP( float10,
long double, div, /, fp, _Quad, 10r, 1 )
1455 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1460 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1461 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1462 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1463 OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \
1467 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1468 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1469 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1470 OP_CMPXCHG(TYPE,BITS,OP) \
1472 #endif // USE_CMPXCHG_FIX
1476 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1477 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1478 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1479 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1480 OP_CMPXCHG(TYPE,BITS,OP) \
1483 OP_CRITICAL(OP##=,LCK_ID) \
1488 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1489 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1490 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1491 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1494 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1506 #define ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE, RET_TYPE) \
1507 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * loc ) \
1509 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1510 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
1522 #define OP_CMPXCHG_READ(TYPE,BITS,OP) \
1524 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1527 kmp_int##BITS i_val; \
1529 union f_i_union old_value; \
1531 old_value.f_val = temp_val; \
1532 old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( (kmp_int##BITS *) loc, \
1533 *VOLATILE_CAST(kmp_int##BITS *) &old_value.i_val, \
1534 *VOLATILE_CAST(kmp_int##BITS *) &old_value.i_val ); \
1535 new_value = old_value.f_val; \
1545 #define OP_CRITICAL_READ(OP,LCK_ID) \
1546 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1548 new_value = (*loc); \
1550 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1553 #ifdef KMP_GOMP_COMPAT
1554 #define OP_GOMP_CRITICAL_READ(OP,FLAG) \
1555 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1557 OP_CRITICAL_READ( OP, 0 ); \
1561 #define OP_GOMP_CRITICAL_READ(OP,FLAG)
1565 #define ATOMIC_FIXED_READ(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1566 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \
1568 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \
1569 new_value = KMP_TEST_THEN_ADD##BITS( loc, OP 0 ); \
1573 #define ATOMIC_CMPXCHG_READ(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1574 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \
1576 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \
1577 OP_CMPXCHG_READ(TYPE,BITS,OP) \
1584 #define ATOMIC_CRITICAL_READ(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1585 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \
1587 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \
1588 OP_CRITICAL_READ(OP,LCK_ID) \
1596 #if ( KMP_OS_WINDOWS )
1598 #define OP_CRITICAL_READ_WRK(OP,LCK_ID) \
1599 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1603 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1605 #ifdef KMP_GOMP_COMPAT
1606 #define OP_GOMP_CRITICAL_READ_WRK(OP,FLAG) \
1607 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1609 OP_CRITICAL_READ_WRK( OP, 0 ); \
1612 #define OP_GOMP_CRITICAL_READ_WRK(OP,FLAG)
1615 #define ATOMIC_BEGIN_READ_WRK(TYPE_ID,OP_ID,TYPE) \
1616 void __kmpc_atomic_##TYPE_ID##_##OP_ID( TYPE * out, ident_t *id_ref, int gtid, TYPE * loc ) \
1618 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1619 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
1622 #define ATOMIC_CRITICAL_READ_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1623 ATOMIC_BEGIN_READ_WRK(TYPE_ID,OP_ID,TYPE) \
1624 OP_GOMP_CRITICAL_READ_WRK(OP##=,GOMP_FLAG) \
1625 OP_CRITICAL_READ_WRK(OP,LCK_ID) \
1628 #endif // KMP_OS_WINDOWS
1632 ATOMIC_FIXED_READ( fixed4, rd, kmp_int32, 32, +, 0 )
1633 ATOMIC_FIXED_READ( fixed8, rd, kmp_int64, 64, +, KMP_ARCH_X86 )
1634 ATOMIC_CMPXCHG_READ( float4, rd, kmp_real32, 32, +, KMP_ARCH_X86 )
1635 ATOMIC_CMPXCHG_READ( float8, rd, kmp_real64, 64, +, KMP_ARCH_X86 )
1638 ATOMIC_CMPXCHG_READ( fixed1, rd, kmp_int8, 8, +, KMP_ARCH_X86 )
1639 ATOMIC_CMPXCHG_READ( fixed2, rd, kmp_int16, 16, +, KMP_ARCH_X86 )
1641 ATOMIC_CRITICAL_READ( float10, rd,
long double, +, 10r, 1 )
1643 ATOMIC_CRITICAL_READ( float16, rd, QUAD_LEGACY, +, 16r, 1 )
1644 #endif // KMP_HAVE_QUAD
1647 #if ( KMP_OS_WINDOWS )
1648 ATOMIC_CRITICAL_READ_WRK( cmplx4, rd, kmp_cmplx32, +, 8c, 1 )
1650 ATOMIC_CRITICAL_READ( cmplx4, rd, kmp_cmplx32, +, 8c, 1 )
1652 ATOMIC_CRITICAL_READ( cmplx8, rd, kmp_cmplx64, +, 16c, 1 )
1653 ATOMIC_CRITICAL_READ( cmplx10, rd, kmp_cmplx80, +, 20c, 1 )
1655 ATOMIC_CRITICAL_READ( cmplx16, rd, CPLX128_LEG, +, 32c, 1 )
1656 #if ( KMP_ARCH_X86 )
1657 ATOMIC_CRITICAL_READ( float16, a16_rd, Quad_a16_t, +, 16r, 1 )
1658 ATOMIC_CRITICAL_READ( cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c, 1 )
1667 #define ATOMIC_XCHG_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1668 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1669 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1670 KMP_XCHG_FIXED##BITS( lhs, rhs ); \
1673 #define ATOMIC_XCHG_FLOAT_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1674 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1675 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1676 KMP_XCHG_REAL##BITS( lhs, rhs ); \
1687 #define OP_CMPXCHG_WR(TYPE,BITS,OP) \
1689 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1690 TYPE old_value, new_value; \
1692 old_value = temp_val; \
1694 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1695 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1696 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
1701 old_value = temp_val; \
1707 #define ATOMIC_CMPXCHG_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1708 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1709 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1710 OP_CMPXCHG_WR(TYPE,BITS,OP) \
1718 #define ATOMIC_CRITICAL_WR(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1719 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1720 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1721 OP_CRITICAL(OP,LCK_ID) \
1725 ATOMIC_XCHG_WR( fixed1, wr, kmp_int8, 8, =, KMP_ARCH_X86 )
1726 ATOMIC_XCHG_WR( fixed2, wr, kmp_int16, 16, =, KMP_ARCH_X86 )
1727 ATOMIC_XCHG_WR( fixed4, wr, kmp_int32, 32, =, KMP_ARCH_X86 )
1728 #if ( KMP_ARCH_X86 )
1729 ATOMIC_CMPXCHG_WR( fixed8, wr, kmp_int64, 64, =, KMP_ARCH_X86 )
1731 ATOMIC_XCHG_WR( fixed8, wr, kmp_int64, 64, =, KMP_ARCH_X86 )
1734 ATOMIC_XCHG_FLOAT_WR( float4, wr, kmp_real32, 32, =, KMP_ARCH_X86 )
1735 #if ( KMP_ARCH_X86 )
1736 ATOMIC_CMPXCHG_WR( float8, wr, kmp_real64, 64, =, KMP_ARCH_X86 )
1738 ATOMIC_XCHG_FLOAT_WR( float8, wr, kmp_real64, 64, =, KMP_ARCH_X86 )
1741 ATOMIC_CRITICAL_WR( float10, wr,
long double, =, 10r, 1 )
1743 ATOMIC_CRITICAL_WR( float16, wr, QUAD_LEGACY, =, 16r, 1 )
1745 ATOMIC_CRITICAL_WR( cmplx4, wr, kmp_cmplx32, =, 8c, 1 )
1746 ATOMIC_CRITICAL_WR( cmplx8, wr, kmp_cmplx64, =, 16c, 1 )
1747 ATOMIC_CRITICAL_WR( cmplx10, wr, kmp_cmplx80, =, 20c, 1 )
1749 ATOMIC_CRITICAL_WR( cmplx16, wr, CPLX128_LEG, =, 32c, 1 )
1750 #if ( KMP_ARCH_X86 )
1751 ATOMIC_CRITICAL_WR( float16, a16_wr, Quad_a16_t, =, 16r, 1 )
1752 ATOMIC_CRITICAL_WR( cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c, 1 )
1765 #define ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,RET_TYPE) \
1766 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, int flag ) \
1768 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1769 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
1777 #define OP_CRITICAL_CPT(OP,LCK_ID) \
1778 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1782 new_value = (*lhs); \
1784 new_value = (*lhs); \
1788 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1792 #ifdef KMP_GOMP_COMPAT
1793 #define OP_GOMP_CRITICAL_CPT(OP,FLAG) \
1794 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1796 OP_CRITICAL_CPT( OP##=, 0 ); \
1799 #define OP_GOMP_CRITICAL_CPT(OP,FLAG)
1809 #define OP_CMPXCHG_CPT(TYPE,BITS,OP) \
1811 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1812 TYPE old_value, new_value; \
1814 old_value = temp_val; \
1815 new_value = old_value OP rhs; \
1816 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1817 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1818 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
1823 old_value = temp_val; \
1824 new_value = old_value OP rhs; \
1833 #define ATOMIC_CMPXCHG_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1834 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
1836 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
1837 OP_CMPXCHG_CPT(TYPE,BITS,OP) \
1841 #define ATOMIC_FIXED_ADD_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1842 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
1843 TYPE old_value, new_value; \
1844 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
1846 old_value = KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \
1848 return old_value OP rhs; \
1854 ATOMIC_FIXED_ADD_CPT( fixed4, add_cpt, kmp_int32, 32, +, 0 )
1855 ATOMIC_FIXED_ADD_CPT( fixed4, sub_cpt, kmp_int32, 32, -, 0 )
1856 ATOMIC_FIXED_ADD_CPT( fixed8, add_cpt, kmp_int64, 64, +, KMP_ARCH_X86 )
1857 ATOMIC_FIXED_ADD_CPT( fixed8, sub_cpt, kmp_int64, 64, -, KMP_ARCH_X86 )
1859 ATOMIC_CMPXCHG_CPT( float4, add_cpt, kmp_real32, 32, +, KMP_ARCH_X86 )
1860 ATOMIC_CMPXCHG_CPT( float4, sub_cpt, kmp_real32, 32, -, KMP_ARCH_X86 )
1861 ATOMIC_CMPXCHG_CPT( float8, add_cpt, kmp_real64, 64, +, KMP_ARCH_X86 )
1862 ATOMIC_CMPXCHG_CPT( float8, sub_cpt, kmp_real64, 64, -, KMP_ARCH_X86 )
1876 ATOMIC_CMPXCHG_CPT( fixed1, add_cpt, kmp_int8, 8, +, KMP_ARCH_X86 )
1877 ATOMIC_CMPXCHG_CPT( fixed1, andb_cpt, kmp_int8, 8, &, 0 )
1878 ATOMIC_CMPXCHG_CPT( fixed1, div_cpt, kmp_int8, 8, /, KMP_ARCH_X86 )
1879 ATOMIC_CMPXCHG_CPT( fixed1u, div_cpt, kmp_uint8, 8, /, KMP_ARCH_X86 )
1880 ATOMIC_CMPXCHG_CPT( fixed1, mul_cpt, kmp_int8, 8, *, KMP_ARCH_X86 )
1881 ATOMIC_CMPXCHG_CPT( fixed1, orb_cpt, kmp_int8, 8, |, 0 )
1882 ATOMIC_CMPXCHG_CPT( fixed1, shl_cpt, kmp_int8, 8, <<, KMP_ARCH_X86 )
1883 ATOMIC_CMPXCHG_CPT( fixed1, shr_cpt, kmp_int8, 8, >>, KMP_ARCH_X86 )
1884 ATOMIC_CMPXCHG_CPT( fixed1u, shr_cpt, kmp_uint8, 8, >>, KMP_ARCH_X86 )
1885 ATOMIC_CMPXCHG_CPT( fixed1, sub_cpt, kmp_int8, 8, -, KMP_ARCH_X86 )
1886 ATOMIC_CMPXCHG_CPT( fixed1, xor_cpt, kmp_int8, 8, ^, 0 )
1887 ATOMIC_CMPXCHG_CPT( fixed2, add_cpt, kmp_int16, 16, +, KMP_ARCH_X86 )
1888 ATOMIC_CMPXCHG_CPT( fixed2, andb_cpt, kmp_int16, 16, &, 0 )
1889 ATOMIC_CMPXCHG_CPT( fixed2, div_cpt, kmp_int16, 16, /, KMP_ARCH_X86 )
1890 ATOMIC_CMPXCHG_CPT( fixed2u, div_cpt, kmp_uint16, 16, /, KMP_ARCH_X86 )
1891 ATOMIC_CMPXCHG_CPT( fixed2, mul_cpt, kmp_int16, 16, *, KMP_ARCH_X86 )
1892 ATOMIC_CMPXCHG_CPT( fixed2, orb_cpt, kmp_int16, 16, |, 0 )
1893 ATOMIC_CMPXCHG_CPT( fixed2, shl_cpt, kmp_int16, 16, <<, KMP_ARCH_X86 )
1894 ATOMIC_CMPXCHG_CPT( fixed2, shr_cpt, kmp_int16, 16, >>, KMP_ARCH_X86 )
1895 ATOMIC_CMPXCHG_CPT( fixed2u, shr_cpt, kmp_uint16, 16, >>, KMP_ARCH_X86 )
1896 ATOMIC_CMPXCHG_CPT( fixed2, sub_cpt, kmp_int16, 16, -, KMP_ARCH_X86 )
1897 ATOMIC_CMPXCHG_CPT( fixed2, xor_cpt, kmp_int16, 16, ^, 0 )
1898 ATOMIC_CMPXCHG_CPT( fixed4, andb_cpt, kmp_int32, 32, &, 0 )
1899 ATOMIC_CMPXCHG_CPT( fixed4, div_cpt, kmp_int32, 32, /, KMP_ARCH_X86 )
1900 ATOMIC_CMPXCHG_CPT( fixed4u, div_cpt, kmp_uint32, 32, /, KMP_ARCH_X86 )
1901 ATOMIC_CMPXCHG_CPT( fixed4, mul_cpt, kmp_int32, 32, *, KMP_ARCH_X86 )
1902 ATOMIC_CMPXCHG_CPT( fixed4, orb_cpt, kmp_int32, 32, |, 0 )
1903 ATOMIC_CMPXCHG_CPT( fixed4, shl_cpt, kmp_int32, 32, <<, KMP_ARCH_X86 )
1904 ATOMIC_CMPXCHG_CPT( fixed4, shr_cpt, kmp_int32, 32, >>, KMP_ARCH_X86 )
1905 ATOMIC_CMPXCHG_CPT( fixed4u, shr_cpt, kmp_uint32, 32, >>, KMP_ARCH_X86 )
1906 ATOMIC_CMPXCHG_CPT( fixed4, xor_cpt, kmp_int32, 32, ^, 0 )
1907 ATOMIC_CMPXCHG_CPT( fixed8, andb_cpt, kmp_int64, 64, &, KMP_ARCH_X86 )
1908 ATOMIC_CMPXCHG_CPT( fixed8, div_cpt, kmp_int64, 64, /, KMP_ARCH_X86 )
1909 ATOMIC_CMPXCHG_CPT( fixed8u, div_cpt, kmp_uint64, 64, /, KMP_ARCH_X86 )
1910 ATOMIC_CMPXCHG_CPT( fixed8, mul_cpt, kmp_int64, 64, *, KMP_ARCH_X86 )
1911 ATOMIC_CMPXCHG_CPT( fixed8, orb_cpt, kmp_int64, 64, |, KMP_ARCH_X86 )
1912 ATOMIC_CMPXCHG_CPT( fixed8, shl_cpt, kmp_int64, 64, <<, KMP_ARCH_X86 )
1913 ATOMIC_CMPXCHG_CPT( fixed8, shr_cpt, kmp_int64, 64, >>, KMP_ARCH_X86 )
1914 ATOMIC_CMPXCHG_CPT( fixed8u, shr_cpt, kmp_uint64, 64, >>, KMP_ARCH_X86 )
1915 ATOMIC_CMPXCHG_CPT( fixed8, xor_cpt, kmp_int64, 64, ^, KMP_ARCH_X86 )
1916 ATOMIC_CMPXCHG_CPT( float4, div_cpt, kmp_real32, 32, /, KMP_ARCH_X86 )
1917 ATOMIC_CMPXCHG_CPT( float4, mul_cpt, kmp_real32, 32, *, KMP_ARCH_X86 )
1918 ATOMIC_CMPXCHG_CPT( float8, div_cpt, kmp_real64, 64, /, KMP_ARCH_X86 )
1919 ATOMIC_CMPXCHG_CPT( float8, mul_cpt, kmp_real64, 64, *, KMP_ARCH_X86 )
1932 #define OP_CRITICAL_L_CPT(OP,LCK_ID) \
1933 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1938 new_value = (*lhs); \
1940 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1943 #ifdef KMP_GOMP_COMPAT
1944 #define OP_GOMP_CRITICAL_L_CPT(OP,FLAG) \
1945 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1947 OP_CRITICAL_L_CPT( OP, 0 ); \
1951 #define OP_GOMP_CRITICAL_L_CPT(OP,FLAG)
1956 #define ATOMIC_CMPX_L_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1957 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
1959 OP_GOMP_CRITICAL_L_CPT( = *lhs OP, GOMP_FLAG ) \
1960 OP_CMPXCHG_CPT(TYPE,BITS,OP) \
1963 ATOMIC_CMPX_L_CPT( fixed1, andl_cpt,
char, 8, &&, KMP_ARCH_X86 )
1964 ATOMIC_CMPX_L_CPT( fixed1, orl_cpt,
char, 8, ||, KMP_ARCH_X86 )
1965 ATOMIC_CMPX_L_CPT( fixed2, andl_cpt,
short, 16, &&, KMP_ARCH_X86 )
1966 ATOMIC_CMPX_L_CPT( fixed2, orl_cpt,
short, 16, ||, KMP_ARCH_X86 )
1967 ATOMIC_CMPX_L_CPT( fixed4, andl_cpt, kmp_int32, 32, &&, 0 )
1968 ATOMIC_CMPX_L_CPT( fixed4, orl_cpt, kmp_int32, 32, ||, 0 )
1969 ATOMIC_CMPX_L_CPT( fixed8, andl_cpt, kmp_int64, 64, &&, KMP_ARCH_X86 )
1970 ATOMIC_CMPX_L_CPT( fixed8, orl_cpt, kmp_int64, 64, ||, KMP_ARCH_X86 )
1983 #define MIN_MAX_CRITSECT_CPT(OP,LCK_ID) \
1984 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1986 if ( *lhs OP rhs ) { \
1992 new_value = old_value; \
1994 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1998 #ifdef KMP_GOMP_COMPAT
1999 #define GOMP_MIN_MAX_CRITSECT_CPT(OP,FLAG) \
2000 if (( FLAG ) && ( __kmp_atomic_mode == 2 )) { \
2002 MIN_MAX_CRITSECT_CPT( OP, 0 ); \
2005 #define GOMP_MIN_MAX_CRITSECT_CPT(OP,FLAG)
2009 #define MIN_MAX_CMPXCHG_CPT(TYPE,BITS,OP) \
2011 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2014 old_value = temp_val; \
2015 while ( old_value OP rhs && \
2016 ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
2017 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
2018 *VOLATILE_CAST(kmp_int##BITS *) &rhs ) ) \
2022 old_value = temp_val; \
2032 #define MIN_MAX_CRITICAL_CPT(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2033 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2034 TYPE new_value, old_value; \
2035 if ( *lhs OP rhs ) { \
2036 GOMP_MIN_MAX_CRITSECT_CPT(OP,GOMP_FLAG) \
2037 MIN_MAX_CRITSECT_CPT(OP,LCK_ID) \
2042 #define MIN_MAX_COMPXCHG_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
2043 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2044 TYPE new_value, old_value; \
2045 if ( *lhs OP rhs ) { \
2046 GOMP_MIN_MAX_CRITSECT_CPT(OP,GOMP_FLAG) \
2047 MIN_MAX_CMPXCHG_CPT(TYPE,BITS,OP) \
2053 MIN_MAX_COMPXCHG_CPT( fixed1, max_cpt,
char, 8, <, KMP_ARCH_X86 )
2054 MIN_MAX_COMPXCHG_CPT( fixed1, min_cpt,
char, 8, >, KMP_ARCH_X86 )
2055 MIN_MAX_COMPXCHG_CPT( fixed2, max_cpt,
short, 16, <, KMP_ARCH_X86 )
2056 MIN_MAX_COMPXCHG_CPT( fixed2, min_cpt,
short, 16, >, KMP_ARCH_X86 )
2057 MIN_MAX_COMPXCHG_CPT( fixed4, max_cpt, kmp_int32, 32, <, 0 )
2058 MIN_MAX_COMPXCHG_CPT( fixed4, min_cpt, kmp_int32, 32, >, 0 )
2059 MIN_MAX_COMPXCHG_CPT( fixed8, max_cpt, kmp_int64, 64, <, KMP_ARCH_X86 )
2060 MIN_MAX_COMPXCHG_CPT( fixed8, min_cpt, kmp_int64, 64, >, KMP_ARCH_X86 )
2061 MIN_MAX_COMPXCHG_CPT( float4, max_cpt, kmp_real32, 32, <, KMP_ARCH_X86 )
2062 MIN_MAX_COMPXCHG_CPT( float4, min_cpt, kmp_real32, 32, >, KMP_ARCH_X86 )
2063 MIN_MAX_COMPXCHG_CPT( float8, max_cpt, kmp_real64, 64, <, KMP_ARCH_X86 )
2064 MIN_MAX_COMPXCHG_CPT( float8, min_cpt, kmp_real64, 64, >, KMP_ARCH_X86 )
2066 MIN_MAX_CRITICAL_CPT( float16, max_cpt, QUAD_LEGACY, <, 16r, 1 )
2067 MIN_MAX_CRITICAL_CPT( float16, min_cpt, QUAD_LEGACY, >, 16r, 1 )
2068 #if ( KMP_ARCH_X86 )
2069 MIN_MAX_CRITICAL_CPT( float16, max_a16_cpt, Quad_a16_t, <, 16r, 1 )
2070 MIN_MAX_CRITICAL_CPT( float16, min_a16_cpt, Quad_a16_t, >, 16r, 1 )
2075 #ifdef KMP_GOMP_COMPAT
2076 #define OP_GOMP_CRITICAL_EQV_CPT(OP,FLAG) \
2077 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2079 OP_CRITICAL_CPT( OP, 0 ); \
2082 #define OP_GOMP_CRITICAL_EQV_CPT(OP,FLAG)
2085 #define ATOMIC_CMPX_EQV_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
2086 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2088 OP_GOMP_CRITICAL_EQV_CPT(^=~,GOMP_FLAG) \
2089 OP_CMPXCHG_CPT(TYPE,BITS,OP) \
2094 ATOMIC_CMPXCHG_CPT( fixed1, neqv_cpt, kmp_int8, 8, ^, KMP_ARCH_X86 )
2095 ATOMIC_CMPXCHG_CPT( fixed2, neqv_cpt, kmp_int16, 16, ^, KMP_ARCH_X86 )
2096 ATOMIC_CMPXCHG_CPT( fixed4, neqv_cpt, kmp_int32, 32, ^, KMP_ARCH_X86 )
2097 ATOMIC_CMPXCHG_CPT( fixed8, neqv_cpt, kmp_int64, 64, ^, KMP_ARCH_X86 )
2098 ATOMIC_CMPX_EQV_CPT( fixed1, eqv_cpt, kmp_int8, 8, ^~, KMP_ARCH_X86 )
2099 ATOMIC_CMPX_EQV_CPT( fixed2, eqv_cpt, kmp_int16, 16, ^~, KMP_ARCH_X86 )
2100 ATOMIC_CMPX_EQV_CPT( fixed4, eqv_cpt, kmp_int32, 32, ^~, KMP_ARCH_X86 )
2101 ATOMIC_CMPX_EQV_CPT( fixed8, eqv_cpt, kmp_int64, 64, ^~, KMP_ARCH_X86 )
2108 #define ATOMIC_CRITICAL_CPT(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2109 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2111 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
2112 OP_CRITICAL_CPT(OP##=,LCK_ID) \
2119 #define OP_CRITICAL_CPT_WRK(OP,LCK_ID) \
2120 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2130 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2134 #ifdef KMP_GOMP_COMPAT
2135 #define OP_GOMP_CRITICAL_CPT_WRK(OP,FLAG) \
2136 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2138 OP_CRITICAL_CPT_WRK( OP##=, 0 ); \
2141 #define OP_GOMP_CRITICAL_CPT_WRK(OP,FLAG)
2145 #define ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \
2146 void __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, TYPE * out, int flag ) \
2148 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
2149 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
2152 #define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2153 ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \
2154 OP_GOMP_CRITICAL_CPT_WRK(OP,GOMP_FLAG) \
2155 OP_CRITICAL_CPT_WRK(OP##=,LCK_ID) \
2161 ATOMIC_CRITICAL_CPT( float10, add_cpt,
long double, +, 10r, 1 )
2162 ATOMIC_CRITICAL_CPT( float10, sub_cpt,
long double, -, 10r, 1 )
2163 ATOMIC_CRITICAL_CPT( float10, mul_cpt,
long double, *, 10r, 1 )
2164 ATOMIC_CRITICAL_CPT( float10, div_cpt,
long double, /, 10r, 1 )
2167 ATOMIC_CRITICAL_CPT( float16, add_cpt, QUAD_LEGACY, +, 16r, 1 )
2168 ATOMIC_CRITICAL_CPT( float16, sub_cpt, QUAD_LEGACY, -, 16r, 1 )
2169 ATOMIC_CRITICAL_CPT( float16, mul_cpt, QUAD_LEGACY, *, 16r, 1 )
2170 ATOMIC_CRITICAL_CPT( float16, div_cpt, QUAD_LEGACY, /, 16r, 1 )
2171 #if ( KMP_ARCH_X86 )
2172 ATOMIC_CRITICAL_CPT( float16, add_a16_cpt, Quad_a16_t, +, 16r, 1 )
2173 ATOMIC_CRITICAL_CPT( float16, sub_a16_cpt, Quad_a16_t, -, 16r, 1 )
2174 ATOMIC_CRITICAL_CPT( float16, mul_a16_cpt, Quad_a16_t, *, 16r, 1 )
2175 ATOMIC_CRITICAL_CPT( float16, div_a16_cpt, Quad_a16_t, /, 16r, 1 )
2182 ATOMIC_CRITICAL_CPT_WRK( cmplx4, add_cpt, kmp_cmplx32, +, 8c, 1 )
2183 ATOMIC_CRITICAL_CPT_WRK( cmplx4, sub_cpt, kmp_cmplx32, -, 8c, 1 )
2184 ATOMIC_CRITICAL_CPT_WRK( cmplx4, mul_cpt, kmp_cmplx32, *, 8c, 1 )
2185 ATOMIC_CRITICAL_CPT_WRK( cmplx4, div_cpt, kmp_cmplx32, /, 8c, 1 )
2187 ATOMIC_CRITICAL_CPT( cmplx8, add_cpt, kmp_cmplx64, +, 16c, 1 )
2188 ATOMIC_CRITICAL_CPT( cmplx8, sub_cpt, kmp_cmplx64, -, 16c, 1 )
2189 ATOMIC_CRITICAL_CPT( cmplx8, mul_cpt, kmp_cmplx64, *, 16c, 1 )
2190 ATOMIC_CRITICAL_CPT( cmplx8, div_cpt, kmp_cmplx64, /, 16c, 1 )
2191 ATOMIC_CRITICAL_CPT( cmplx10, add_cpt, kmp_cmplx80, +, 20c, 1 )
2192 ATOMIC_CRITICAL_CPT( cmplx10, sub_cpt, kmp_cmplx80, -, 20c, 1 )
2193 ATOMIC_CRITICAL_CPT( cmplx10, mul_cpt, kmp_cmplx80, *, 20c, 1 )
2194 ATOMIC_CRITICAL_CPT( cmplx10, div_cpt, kmp_cmplx80, /, 20c, 1 )
2196 ATOMIC_CRITICAL_CPT( cmplx16, add_cpt, CPLX128_LEG, +, 32c, 1 )
2197 ATOMIC_CRITICAL_CPT( cmplx16, sub_cpt, CPLX128_LEG, -, 32c, 1 )
2198 ATOMIC_CRITICAL_CPT( cmplx16, mul_cpt, CPLX128_LEG, *, 32c, 1 )
2199 ATOMIC_CRITICAL_CPT( cmplx16, div_cpt, CPLX128_LEG, /, 32c, 1 )
2200 #if ( KMP_ARCH_X86 )
2201 ATOMIC_CRITICAL_CPT( cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c, 1 )
2202 ATOMIC_CRITICAL_CPT( cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c, 1 )
2203 ATOMIC_CRITICAL_CPT( cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c, 1 )
2204 ATOMIC_CRITICAL_CPT( cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c, 1 )
2219 #define OP_CRITICAL_CPT_REV(OP,LCK_ID) \
2220 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2224 (*lhs) = (rhs) OP (*lhs); \
2225 new_value = (*lhs); \
2227 new_value = (*lhs);\
2228 (*lhs) = (rhs) OP (*lhs); \
2230 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2234 #ifdef KMP_GOMP_COMPAT
2235 #define OP_GOMP_CRITICAL_CPT_REV(OP,FLAG) \
2236 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2238 OP_CRITICAL_CPT_REV( OP, 0 ); \
2241 #define OP_GOMP_CRITICAL_CPT_REV(OP,FLAG)
2251 #define OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \
2253 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2254 TYPE old_value, new_value; \
2256 old_value = temp_val; \
2257 new_value = rhs OP old_value; \
2258 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
2259 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
2260 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
2265 old_value = temp_val; \
2266 new_value = rhs OP old_value; \
2275 #define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
2276 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2278 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2279 OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \
2280 OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \
2284 ATOMIC_CMPXCHG_CPT_REV( fixed1, div_cpt_rev, kmp_int8, 8, /, KMP_ARCH_X86 )
2285 ATOMIC_CMPXCHG_CPT_REV( fixed1u, div_cpt_rev, kmp_uint8, 8, /, KMP_ARCH_X86 )
2286 ATOMIC_CMPXCHG_CPT_REV( fixed1, shl_cpt_rev, kmp_int8, 8, <<, KMP_ARCH_X86 )
2287 ATOMIC_CMPXCHG_CPT_REV( fixed1, shr_cpt_rev, kmp_int8, 8, >>, KMP_ARCH_X86 )
2288 ATOMIC_CMPXCHG_CPT_REV( fixed1u, shr_cpt_rev, kmp_uint8, 8, >>, KMP_ARCH_X86 )
2289 ATOMIC_CMPXCHG_CPT_REV( fixed1, sub_cpt_rev, kmp_int8, 8, -, KMP_ARCH_X86 )
2290 ATOMIC_CMPXCHG_CPT_REV( fixed2, div_cpt_rev, kmp_int16, 16, /, KMP_ARCH_X86 )
2291 ATOMIC_CMPXCHG_CPT_REV( fixed2u, div_cpt_rev, kmp_uint16, 16, /, KMP_ARCH_X86 )
2292 ATOMIC_CMPXCHG_CPT_REV( fixed2, shl_cpt_rev, kmp_int16, 16, <<, KMP_ARCH_X86 )
2293 ATOMIC_CMPXCHG_CPT_REV( fixed2, shr_cpt_rev, kmp_int16, 16, >>, KMP_ARCH_X86 )
2294 ATOMIC_CMPXCHG_CPT_REV( fixed2u, shr_cpt_rev, kmp_uint16, 16, >>, KMP_ARCH_X86 )
2295 ATOMIC_CMPXCHG_CPT_REV( fixed2, sub_cpt_rev, kmp_int16, 16, -, KMP_ARCH_X86 )
2296 ATOMIC_CMPXCHG_CPT_REV( fixed4, div_cpt_rev, kmp_int32, 32, /, KMP_ARCH_X86 )
2297 ATOMIC_CMPXCHG_CPT_REV( fixed4u, div_cpt_rev, kmp_uint32, 32, /, KMP_ARCH_X86 )
2298 ATOMIC_CMPXCHG_CPT_REV( fixed4, shl_cpt_rev, kmp_int32, 32, <<, KMP_ARCH_X86 )
2299 ATOMIC_CMPXCHG_CPT_REV( fixed4, shr_cpt_rev, kmp_int32, 32, >>, KMP_ARCH_X86 )
2300 ATOMIC_CMPXCHG_CPT_REV( fixed4u, shr_cpt_rev, kmp_uint32, 32, >>, KMP_ARCH_X86 )
2301 ATOMIC_CMPXCHG_CPT_REV( fixed4, sub_cpt_rev, kmp_int32, 32, -, KMP_ARCH_X86 )
2302 ATOMIC_CMPXCHG_CPT_REV( fixed8, div_cpt_rev, kmp_int64, 64, /, KMP_ARCH_X86 )
2303 ATOMIC_CMPXCHG_CPT_REV( fixed8u, div_cpt_rev, kmp_uint64, 64, /, KMP_ARCH_X86 )
2304 ATOMIC_CMPXCHG_CPT_REV( fixed8, shl_cpt_rev, kmp_int64, 64, <<, KMP_ARCH_X86 )
2305 ATOMIC_CMPXCHG_CPT_REV( fixed8, shr_cpt_rev, kmp_int64, 64, >>, KMP_ARCH_X86 )
2306 ATOMIC_CMPXCHG_CPT_REV( fixed8u, shr_cpt_rev, kmp_uint64, 64, >>, KMP_ARCH_X86 )
2307 ATOMIC_CMPXCHG_CPT_REV( fixed8, sub_cpt_rev, kmp_int64, 64, -, KMP_ARCH_X86 )
2308 ATOMIC_CMPXCHG_CPT_REV( float4, div_cpt_rev, kmp_real32, 32, /, KMP_ARCH_X86 )
2309 ATOMIC_CMPXCHG_CPT_REV( float4, sub_cpt_rev, kmp_real32, 32, -, KMP_ARCH_X86 )
2310 ATOMIC_CMPXCHG_CPT_REV( float8, div_cpt_rev, kmp_real64, 64, /, KMP_ARCH_X86 )
2311 ATOMIC_CMPXCHG_CPT_REV( float8, sub_cpt_rev, kmp_real64, 64, -, KMP_ARCH_X86 )
2320 #define ATOMIC_CRITICAL_CPT_REV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2321 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2323 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2325 OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \
2326 OP_CRITICAL_CPT_REV(OP,LCK_ID) \
2332 ATOMIC_CRITICAL_CPT_REV( float10, sub_cpt_rev,
long double, -, 10r, 1 )
2333 ATOMIC_CRITICAL_CPT_REV( float10, div_cpt_rev,
long double, /, 10r, 1 )
2336 ATOMIC_CRITICAL_CPT_REV( float16, sub_cpt_rev, QUAD_LEGACY, -, 16r, 1 )
2337 ATOMIC_CRITICAL_CPT_REV( float16, div_cpt_rev, QUAD_LEGACY, /, 16r, 1 )
2338 #if ( KMP_ARCH_X86 )
2339 ATOMIC_CRITICAL_CPT_REV( float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r, 1 )
2340 ATOMIC_CRITICAL_CPT_REV( float16, div_a16_cpt_rev, Quad_a16_t, /, 16r, 1 )
2350 #define OP_CRITICAL_CPT_REV_WRK(OP,LCK_ID) \
2351 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2354 (*lhs) = (rhs) OP (*lhs); \
2358 (*lhs) = (rhs) OP (*lhs); \
2361 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2365 #ifdef KMP_GOMP_COMPAT
2366 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP,FLAG) \
2367 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2369 OP_CRITICAL_CPT_REV_WRK( OP, 0 ); \
2372 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP,FLAG)
2376 #define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2377 ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \
2378 OP_GOMP_CRITICAL_CPT_REV_WRK(OP,GOMP_FLAG) \
2379 OP_CRITICAL_CPT_REV_WRK(OP,LCK_ID) \
2386 ATOMIC_CRITICAL_CPT_REV_WRK( cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c, 1 )
2387 ATOMIC_CRITICAL_CPT_REV_WRK( cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c, 1 )
2389 ATOMIC_CRITICAL_CPT_REV( cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c, 1 )
2390 ATOMIC_CRITICAL_CPT_REV( cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c, 1 )
2391 ATOMIC_CRITICAL_CPT_REV( cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c, 1 )
2392 ATOMIC_CRITICAL_CPT_REV( cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c, 1 )
2394 ATOMIC_CRITICAL_CPT_REV( cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c, 1 )
2395 ATOMIC_CRITICAL_CPT_REV( cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c, 1 )
2396 #if ( KMP_ARCH_X86 )
2397 ATOMIC_CRITICAL_CPT_REV( cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c, 1 )
2398 ATOMIC_CRITICAL_CPT_REV( cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c, 1 )
2404 #define ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2405 TYPE __kmpc_atomic_##TYPE_ID##_swp( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \
2407 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
2408 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid ));
2410 #define CRITICAL_SWP(LCK_ID) \
2411 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2413 old_value = (*lhs); \
2416 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2420 #ifdef KMP_GOMP_COMPAT
2421 #define GOMP_CRITICAL_SWP(FLAG) \
2422 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2424 CRITICAL_SWP( 0 ); \
2427 #define GOMP_CRITICAL_SWP(FLAG)
2431 #define ATOMIC_XCHG_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \
2432 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2434 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2435 old_value = KMP_XCHG_FIXED##BITS( lhs, rhs ); \
2439 #define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \
2440 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2442 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2443 old_value = KMP_XCHG_REAL##BITS( lhs, rhs ); \
2448 #define CMPXCHG_SWP(TYPE,BITS) \
2450 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2451 TYPE old_value, new_value; \
2453 old_value = temp_val; \
2455 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
2456 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
2457 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
2462 old_value = temp_val; \
2469 #define ATOMIC_CMPXCHG_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \
2470 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2472 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2473 CMPXCHG_SWP(TYPE,BITS) \
2476 ATOMIC_XCHG_SWP( fixed1, kmp_int8, 8, KMP_ARCH_X86 )
2477 ATOMIC_XCHG_SWP( fixed2, kmp_int16, 16, KMP_ARCH_X86 )
2478 ATOMIC_XCHG_SWP( fixed4, kmp_int32, 32, KMP_ARCH_X86 )
2480 ATOMIC_XCHG_FLOAT_SWP( float4, kmp_real32, 32, KMP_ARCH_X86 )
2482 #if ( KMP_ARCH_X86 )
2483 ATOMIC_CMPXCHG_SWP( fixed8, kmp_int64, 64, KMP_ARCH_X86 )
2484 ATOMIC_CMPXCHG_SWP( float8, kmp_real64, 64, KMP_ARCH_X86 )
2486 ATOMIC_XCHG_SWP( fixed8, kmp_int64, 64, KMP_ARCH_X86 )
2487 ATOMIC_XCHG_FLOAT_SWP( float8, kmp_real64, 64, KMP_ARCH_X86 )
2492 #define ATOMIC_CRITICAL_SWP(TYPE_ID,TYPE,LCK_ID,GOMP_FLAG) \
2493 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2495 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2496 CRITICAL_SWP(LCK_ID) \
2505 #define ATOMIC_BEGIN_SWP_WRK(TYPE_ID,TYPE) \
2506 void __kmpc_atomic_##TYPE_ID##_swp( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, TYPE * out ) \
2508 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
2509 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid ));
2512 #define CRITICAL_SWP_WRK(LCK_ID) \
2513 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2518 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2523 #ifdef KMP_GOMP_COMPAT
2524 #define GOMP_CRITICAL_SWP_WRK(FLAG) \
2525 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2527 CRITICAL_SWP_WRK( 0 ); \
2530 #define GOMP_CRITICAL_SWP_WRK(FLAG)
2534 #define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE,LCK_ID,GOMP_FLAG) \
2535 ATOMIC_BEGIN_SWP_WRK(TYPE_ID,TYPE) \
2537 GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \
2538 CRITICAL_SWP_WRK(LCK_ID) \
2543 ATOMIC_CRITICAL_SWP( float10,
long double, 10r, 1 )
2545 ATOMIC_CRITICAL_SWP( float16, QUAD_LEGACY, 16r, 1 )
2548 ATOMIC_CRITICAL_SWP_WRK( cmplx4, kmp_cmplx32, 8c, 1 )
2553 ATOMIC_CRITICAL_SWP( cmplx8, kmp_cmplx64, 16c, 1 )
2554 ATOMIC_CRITICAL_SWP( cmplx10, kmp_cmplx80, 20c, 1 )
2556 ATOMIC_CRITICAL_SWP( cmplx16, CPLX128_LEG, 32c, 1 )
2557 #if ( KMP_ARCH_X86 )
2558 ATOMIC_CRITICAL_SWP( float16_a16, Quad_a16_t, 16r, 1 )
2559 ATOMIC_CRITICAL_SWP( cmplx16_a16, kmp_cmplx128_a16_t, 32c, 1 )
2566 #endif //OMP_40_ENABLED
2568 #endif //KMP_ARCH_X86 || KMP_ARCH_X86_64
2578 __kmpc_atomic_1(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2580 KMP_DEBUG_ASSERT( __kmp_init_serial );
2583 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2590 kmp_int8 old_value, new_value;
2592 old_value = *(kmp_int8 *) lhs;
2593 (*f)( &new_value, &old_value, rhs );
2596 while ( ! KMP_COMPARE_AND_STORE_ACQ8 ( (kmp_int8 *) lhs,
2597 *(kmp_int8 *) &old_value, *(kmp_int8 *) &new_value ) )
2601 old_value = *(kmp_int8 *) lhs;
2602 (*f)( &new_value, &old_value, rhs );
2612 #ifdef KMP_GOMP_COMPAT
2613 if ( __kmp_atomic_mode == 2 ) {
2614 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2618 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_1i, gtid );
2620 (*f)( lhs, lhs, rhs );
2622 #ifdef KMP_GOMP_COMPAT
2623 if ( __kmp_atomic_mode == 2 ) {
2624 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2628 __kmp_release_atomic_lock( & __kmp_atomic_lock_1i, gtid );
2633 __kmpc_atomic_2(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2636 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2638 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
2641 ! ( (kmp_uintptr_t) lhs & 0x1)
2645 kmp_int16 old_value, new_value;
2647 old_value = *(kmp_int16 *) lhs;
2648 (*f)( &new_value, &old_value, rhs );
2651 while ( ! KMP_COMPARE_AND_STORE_ACQ16 ( (kmp_int16 *) lhs,
2652 *(kmp_int16 *) &old_value, *(kmp_int16 *) &new_value ) )
2656 old_value = *(kmp_int16 *) lhs;
2657 (*f)( &new_value, &old_value, rhs );
2667 #ifdef KMP_GOMP_COMPAT
2668 if ( __kmp_atomic_mode == 2 ) {
2669 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2673 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_2i, gtid );
2675 (*f)( lhs, lhs, rhs );
2677 #ifdef KMP_GOMP_COMPAT
2678 if ( __kmp_atomic_mode == 2 ) {
2679 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2683 __kmp_release_atomic_lock( & __kmp_atomic_lock_2i, gtid );
2688 __kmpc_atomic_4(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2690 KMP_DEBUG_ASSERT( __kmp_init_serial );
2697 #
if KMP_ARCH_X86 || KMP_ARCH_X86_64
2700 ! ( (kmp_uintptr_t) lhs & 0x3)
2704 kmp_int32 old_value, new_value;
2706 old_value = *(kmp_int32 *) lhs;
2707 (*f)( &new_value, &old_value, rhs );
2710 while ( ! KMP_COMPARE_AND_STORE_ACQ32 ( (kmp_int32 *) lhs,
2711 *(kmp_int32 *) &old_value, *(kmp_int32 *) &new_value ) )
2715 old_value = *(kmp_int32 *) lhs;
2716 (*f)( &new_value, &old_value, rhs );
2727 #ifdef KMP_GOMP_COMPAT
2728 if ( __kmp_atomic_mode == 2 ) {
2729 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2733 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_4i, gtid );
2735 (*f)( lhs, lhs, rhs );
2737 #ifdef KMP_GOMP_COMPAT
2738 if ( __kmp_atomic_mode == 2 ) {
2739 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2743 __kmp_release_atomic_lock( & __kmp_atomic_lock_4i, gtid );
2748 __kmpc_atomic_8(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2750 KMP_DEBUG_ASSERT( __kmp_init_serial );
2753 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2755 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
2758 ! ( (kmp_uintptr_t) lhs & 0x7)
2762 kmp_int64 old_value, new_value;
2764 old_value = *(kmp_int64 *) lhs;
2765 (*f)( &new_value, &old_value, rhs );
2767 while ( ! KMP_COMPARE_AND_STORE_ACQ64 ( (kmp_int64 *) lhs,
2768 *(kmp_int64 *) &old_value,
2769 *(kmp_int64 *) &new_value ) )
2773 old_value = *(kmp_int64 *) lhs;
2774 (*f)( &new_value, &old_value, rhs );
2784 #ifdef KMP_GOMP_COMPAT
2785 if ( __kmp_atomic_mode == 2 ) {
2786 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2790 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_8i, gtid );
2792 (*f)( lhs, lhs, rhs );
2794 #ifdef KMP_GOMP_COMPAT
2795 if ( __kmp_atomic_mode == 2 ) {
2796 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2800 __kmp_release_atomic_lock( & __kmp_atomic_lock_8i, gtid );
2805 __kmpc_atomic_10(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2807 KMP_DEBUG_ASSERT( __kmp_init_serial );
2809 #ifdef KMP_GOMP_COMPAT
2810 if ( __kmp_atomic_mode == 2 ) {
2811 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2815 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_10r, gtid );
2817 (*f)( lhs, lhs, rhs );
2819 #ifdef KMP_GOMP_COMPAT
2820 if ( __kmp_atomic_mode == 2 ) {
2821 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2825 __kmp_release_atomic_lock( & __kmp_atomic_lock_10r, gtid );
2829 __kmpc_atomic_16(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2831 KMP_DEBUG_ASSERT( __kmp_init_serial );
2833 #ifdef KMP_GOMP_COMPAT
2834 if ( __kmp_atomic_mode == 2 ) {
2835 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2839 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_16c, gtid );
2841 (*f)( lhs, lhs, rhs );
2843 #ifdef KMP_GOMP_COMPAT
2844 if ( __kmp_atomic_mode == 2 ) {
2845 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2849 __kmp_release_atomic_lock( & __kmp_atomic_lock_16c, gtid );
2853 __kmpc_atomic_20(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2855 KMP_DEBUG_ASSERT( __kmp_init_serial );
2857 #ifdef KMP_GOMP_COMPAT
2858 if ( __kmp_atomic_mode == 2 ) {
2859 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2863 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_20c, gtid );
2865 (*f)( lhs, lhs, rhs );
2867 #ifdef KMP_GOMP_COMPAT
2868 if ( __kmp_atomic_mode == 2 ) {
2869 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2873 __kmp_release_atomic_lock( & __kmp_atomic_lock_20c, gtid );
2877 __kmpc_atomic_32(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2879 KMP_DEBUG_ASSERT( __kmp_init_serial );
2881 #ifdef KMP_GOMP_COMPAT
2882 if ( __kmp_atomic_mode == 2 ) {
2883 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2887 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_32c, gtid );
2889 (*f)( lhs, lhs, rhs );
2891 #ifdef KMP_GOMP_COMPAT
2892 if ( __kmp_atomic_mode == 2 ) {
2893 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2897 __kmp_release_atomic_lock( & __kmp_atomic_lock_32c, gtid );
2904 __kmpc_atomic_start(
void)
2906 int gtid = __kmp_entry_gtid();
2907 KA_TRACE(20, (
"__kmpc_atomic_start: T#%d\n", gtid));
2908 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
2913 __kmpc_atomic_end(
void)
2915 int gtid = __kmp_get_gtid();
2916 KA_TRACE(20, (
"__kmpc_atomic_end: T#%d\n", gtid));
2917 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);