Intel® OpenMP* Runtime Library
 All Classes Functions Variables Typedefs Enumerations Enumerator Groups Pages
z_Linux_util.c
1 /*
2  * z_Linux_util.c -- platform specific routines.
3  */
4 
5 /* <copyright>
6  Copyright (c) 1997-2015 Intel Corporation. All Rights Reserved.
7 
8  Redistribution and use in source and binary forms, with or without
9  modification, are permitted provided that the following conditions
10  are met:
11 
12  * Redistributions of source code must retain the above copyright
13  notice, this list of conditions and the following disclaimer.
14  * Redistributions in binary form must reproduce the above copyright
15  notice, this list of conditions and the following disclaimer in the
16  documentation and/or other materials provided with the distribution.
17  * Neither the name of Intel Corporation nor the names of its
18  contributors may be used to endorse or promote products derived
19  from this software without specific prior written permission.
20 
21  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 
33 </copyright> */
34 
35 #include "kmp.h"
36 #include "kmp_wrapper_getpid.h"
37 #include "kmp_itt.h"
38 #include "kmp_str.h"
39 #include "kmp_i18n.h"
40 #include "kmp_io.h"
41 #include "kmp_stats.h"
42 #include "kmp_wait_release.h"
43 
44 #if !KMP_OS_FREEBSD
45 # include <alloca.h>
46 #endif
47 #include <unistd.h>
48 #include <math.h> // HUGE_VAL.
49 #include <sys/time.h>
50 #include <sys/times.h>
51 #include <sys/resource.h>
52 #include <sys/syscall.h>
53 
54 #if KMP_OS_LINUX && !KMP_OS_CNK
55 # include <sys/sysinfo.h>
56 # if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
57 // We should really include <futex.h>, but that causes compatibility problems on different
58 // Linux* OS distributions that either require that you include (or break when you try to include)
59 // <pci/types.h>.
60 // Since all we need is the two macros below (which are part of the kernel ABI, so can't change)
61 // we just define the constants here and don't include <futex.h>
62 # ifndef FUTEX_WAIT
63 # define FUTEX_WAIT 0
64 # endif
65 # ifndef FUTEX_WAKE
66 # define FUTEX_WAKE 1
67 # endif
68 # endif
69 #elif KMP_OS_DARWIN
70 # include <sys/sysctl.h>
71 # include <mach/mach.h>
72 #elif KMP_OS_FREEBSD
73 # include <sys/sysctl.h>
74 # include <pthread_np.h>
75 #endif
76 
77 
78 #include <dirent.h>
79 #include <ctype.h>
80 #include <fcntl.h>
81 
82 // For non-x86 architecture
83 #if KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64)
84 # include <stdbool.h>
85 # include <ffi.h>
86 #endif
87 
88 /* ------------------------------------------------------------------------ */
89 /* ------------------------------------------------------------------------ */
90 
91 struct kmp_sys_timer {
92  struct timespec start;
93 };
94 
95 // Convert timespec to nanoseconds.
96 #define TS2NS(timespec) (((timespec).tv_sec * 1e9) + (timespec).tv_nsec)
97 
98 static struct kmp_sys_timer __kmp_sys_timer_data;
99 
100 #if KMP_HANDLE_SIGNALS
101  typedef void (* sig_func_t )( int );
102  STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[ NSIG ];
103  static sigset_t __kmp_sigset;
104 #endif
105 
106 static int __kmp_init_runtime = FALSE;
107 
108 static int __kmp_fork_count = 0;
109 
110 static pthread_condattr_t __kmp_suspend_cond_attr;
111 static pthread_mutexattr_t __kmp_suspend_mutex_attr;
112 
113 static kmp_cond_align_t __kmp_wait_cv;
114 static kmp_mutex_align_t __kmp_wait_mx;
115 
116 /* ------------------------------------------------------------------------ */
117 /* ------------------------------------------------------------------------ */
118 
119 #ifdef DEBUG_SUSPEND
120 static void
121 __kmp_print_cond( char *buffer, kmp_cond_align_t *cond )
122 {
123  KMP_SNPRINTF( buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))",
124  cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
125  cond->c_cond.__c_waiting );
126 }
127 #endif
128 
129 /* ------------------------------------------------------------------------ */
130 /* ------------------------------------------------------------------------ */
131 
132 #if ( KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED)
133 
134 /*
135  * Affinity support
136  */
137 
138 /*
139  * On some of the older OS's that we build on, these constants aren't present
140  * in <asm/unistd.h> #included from <sys.syscall.h>. They must be the same on
141  * all systems of the same arch where they are defined, and they cannot change.
142  * stone forever.
143  */
144 
145 # if KMP_ARCH_X86 || KMP_ARCH_ARM
146 # ifndef __NR_sched_setaffinity
147 # define __NR_sched_setaffinity 241
148 # elif __NR_sched_setaffinity != 241
149 # error Wrong code for setaffinity system call.
150 # endif /* __NR_sched_setaffinity */
151 # ifndef __NR_sched_getaffinity
152 # define __NR_sched_getaffinity 242
153 # elif __NR_sched_getaffinity != 242
154 # error Wrong code for getaffinity system call.
155 # endif /* __NR_sched_getaffinity */
156 
157 # elif KMP_ARCH_AARCH64
158 # ifndef __NR_sched_setaffinity
159 # define __NR_sched_setaffinity 122
160 # elif __NR_sched_setaffinity != 122
161 # error Wrong code for setaffinity system call.
162 # endif /* __NR_sched_setaffinity */
163 # ifndef __NR_sched_getaffinity
164 # define __NR_sched_getaffinity 123
165 # elif __NR_sched_getaffinity != 123
166 # error Wrong code for getaffinity system call.
167 # endif /* __NR_sched_getaffinity */
168 
169 # elif KMP_ARCH_X86_64
170 # ifndef __NR_sched_setaffinity
171 # define __NR_sched_setaffinity 203
172 # elif __NR_sched_setaffinity != 203
173 # error Wrong code for setaffinity system call.
174 # endif /* __NR_sched_setaffinity */
175 # ifndef __NR_sched_getaffinity
176 # define __NR_sched_getaffinity 204
177 # elif __NR_sched_getaffinity != 204
178 # error Wrong code for getaffinity system call.
179 # endif /* __NR_sched_getaffinity */
180 
181 # elif KMP_ARCH_PPC64
182 # ifndef __NR_sched_setaffinity
183 # define __NR_sched_setaffinity 222
184 # elif __NR_sched_setaffinity != 222
185 # error Wrong code for setaffinity system call.
186 # endif /* __NR_sched_setaffinity */
187 # ifndef __NR_sched_getaffinity
188 # define __NR_sched_getaffinity 223
189 # elif __NR_sched_getaffinity != 223
190 # error Wrong code for getaffinity system call.
191 # endif /* __NR_sched_getaffinity */
192 
193 
194 # else
195 # error Unknown or unsupported architecture
196 
197 # endif /* KMP_ARCH_* */
198 
199 int
200 __kmp_set_system_affinity( kmp_affin_mask_t const *mask, int abort_on_error )
201 {
202  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
203  "Illegal set affinity operation when not capable");
204 
205  int retval = syscall( __NR_sched_setaffinity, 0, __kmp_affin_mask_size, mask );
206  if (retval >= 0) {
207  return 0;
208  }
209  int error = errno;
210  if (abort_on_error) {
211  __kmp_msg(
212  kmp_ms_fatal,
213  KMP_MSG( FatalSysError ),
214  KMP_ERR( error ),
215  __kmp_msg_null
216  );
217  }
218  return error;
219 }
220 
221 int
222 __kmp_get_system_affinity( kmp_affin_mask_t *mask, int abort_on_error )
223 {
224  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
225  "Illegal get affinity operation when not capable");
226 
227  int retval = syscall( __NR_sched_getaffinity, 0, __kmp_affin_mask_size, mask );
228  if (retval >= 0) {
229  return 0;
230  }
231  int error = errno;
232  if (abort_on_error) {
233  __kmp_msg(
234  kmp_ms_fatal,
235  KMP_MSG( FatalSysError ),
236  KMP_ERR( error ),
237  __kmp_msg_null
238  );
239  }
240  return error;
241 }
242 
243 void
244 __kmp_affinity_bind_thread( int which )
245 {
246  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
247  "Illegal set affinity operation when not capable");
248 
249  kmp_affin_mask_t *mask = (kmp_affin_mask_t *)KMP_ALLOCA(__kmp_affin_mask_size);
250  KMP_CPU_ZERO(mask);
251  KMP_CPU_SET(which, mask);
252  __kmp_set_system_affinity(mask, TRUE);
253 }
254 
255 /*
256  * Determine if we can access affinity functionality on this version of
257  * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set
258  * __kmp_affin_mask_size to the appropriate value (0 means not capable).
259  */
260 void
261 __kmp_affinity_determine_capable(const char *env_var)
262 {
263  //
264  // Check and see if the OS supports thread affinity.
265  //
266 
267 # define KMP_CPU_SET_SIZE_LIMIT (1024*1024)
268 
269  int gCode;
270  int sCode;
271  kmp_affin_mask_t *buf;
272  buf = ( kmp_affin_mask_t * ) KMP_INTERNAL_MALLOC( KMP_CPU_SET_SIZE_LIMIT );
273 
274  // If Linux* OS:
275  // If the syscall fails or returns a suggestion for the size,
276  // then we don't have to search for an appropriate size.
277  gCode = syscall( __NR_sched_getaffinity, 0, KMP_CPU_SET_SIZE_LIMIT, buf );
278  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
279  "initial getaffinity call returned %d errno = %d\n",
280  gCode, errno));
281 
282  //if ((gCode < 0) && (errno == ENOSYS))
283  if (gCode < 0) {
284  //
285  // System call not supported
286  //
287  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
288  && (__kmp_affinity_type != affinity_none)
289  && (__kmp_affinity_type != affinity_default)
290  && (__kmp_affinity_type != affinity_disabled))) {
291  int error = errno;
292  __kmp_msg(
293  kmp_ms_warning,
294  KMP_MSG( GetAffSysCallNotSupported, env_var ),
295  KMP_ERR( error ),
296  __kmp_msg_null
297  );
298  }
299  KMP_AFFINITY_DISABLE();
300  KMP_INTERNAL_FREE(buf);
301  return;
302  }
303  if (gCode > 0) { // Linux* OS only
304  // The optimal situation: the OS returns the size of the buffer
305  // it expects.
306  //
307  // A verification of correct behavior is that Isetaffinity on a NULL
308  // buffer with the same size fails with errno set to EFAULT.
309  sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
310  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
311  "setaffinity for mask size %d returned %d errno = %d\n",
312  gCode, sCode, errno));
313  if (sCode < 0) {
314  if (errno == ENOSYS) {
315  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
316  && (__kmp_affinity_type != affinity_none)
317  && (__kmp_affinity_type != affinity_default)
318  && (__kmp_affinity_type != affinity_disabled))) {
319  int error = errno;
320  __kmp_msg(
321  kmp_ms_warning,
322  KMP_MSG( SetAffSysCallNotSupported, env_var ),
323  KMP_ERR( error ),
324  __kmp_msg_null
325  );
326  }
327  KMP_AFFINITY_DISABLE();
328  KMP_INTERNAL_FREE(buf);
329  }
330  if (errno == EFAULT) {
331  KMP_AFFINITY_ENABLE(gCode);
332  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
333  "affinity supported (mask size %d)\n",
334  (int)__kmp_affin_mask_size));
335  KMP_INTERNAL_FREE(buf);
336  return;
337  }
338  }
339  }
340 
341  //
342  // Call the getaffinity system call repeatedly with increasing set sizes
343  // until we succeed, or reach an upper bound on the search.
344  //
345  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
346  "searching for proper set size\n"));
347  int size;
348  for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
349  gCode = syscall( __NR_sched_getaffinity, 0, size, buf );
350  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
351  "getaffinity for mask size %d returned %d errno = %d\n", size,
352  gCode, errno));
353 
354  if (gCode < 0) {
355  if ( errno == ENOSYS )
356  {
357  //
358  // We shouldn't get here
359  //
360  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
361  "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
362  size));
363  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
364  && (__kmp_affinity_type != affinity_none)
365  && (__kmp_affinity_type != affinity_default)
366  && (__kmp_affinity_type != affinity_disabled))) {
367  int error = errno;
368  __kmp_msg(
369  kmp_ms_warning,
370  KMP_MSG( GetAffSysCallNotSupported, env_var ),
371  KMP_ERR( error ),
372  __kmp_msg_null
373  );
374  }
375  KMP_AFFINITY_DISABLE();
376  KMP_INTERNAL_FREE(buf);
377  return;
378  }
379  continue;
380  }
381 
382  sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
383  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
384  "setaffinity for mask size %d returned %d errno = %d\n",
385  gCode, sCode, errno));
386  if (sCode < 0) {
387  if (errno == ENOSYS) { // Linux* OS only
388  //
389  // We shouldn't get here
390  //
391  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
392  "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
393  size));
394  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
395  && (__kmp_affinity_type != affinity_none)
396  && (__kmp_affinity_type != affinity_default)
397  && (__kmp_affinity_type != affinity_disabled))) {
398  int error = errno;
399  __kmp_msg(
400  kmp_ms_warning,
401  KMP_MSG( SetAffSysCallNotSupported, env_var ),
402  KMP_ERR( error ),
403  __kmp_msg_null
404  );
405  }
406  KMP_AFFINITY_DISABLE();
407  KMP_INTERNAL_FREE(buf);
408  return;
409  }
410  if (errno == EFAULT) {
411  KMP_AFFINITY_ENABLE(gCode);
412  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
413  "affinity supported (mask size %d)\n",
414  (int)__kmp_affin_mask_size));
415  KMP_INTERNAL_FREE(buf);
416  return;
417  }
418  }
419  }
420  //int error = errno; // save uncaught error code
421  KMP_INTERNAL_FREE(buf);
422  // errno = error; // restore uncaught error code, will be printed at the next KMP_WARNING below
423 
424  //
425  // Affinity is not supported
426  //
427  KMP_AFFINITY_DISABLE();
428  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
429  "cannot determine mask size - affinity not supported\n"));
430  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
431  && (__kmp_affinity_type != affinity_none)
432  && (__kmp_affinity_type != affinity_default)
433  && (__kmp_affinity_type != affinity_disabled))) {
434  KMP_WARNING( AffCantGetMaskSize, env_var );
435  }
436 }
437 
438 #endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
439 
440 /* ------------------------------------------------------------------------ */
441 /* ------------------------------------------------------------------------ */
442 
443 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && !KMP_OS_CNK
444 
445 int
446 __kmp_futex_determine_capable()
447 {
448  int loc = 0;
449  int rc = syscall( __NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0 );
450  int retval = ( rc == 0 ) || ( errno != ENOSYS );
451 
452  KA_TRACE(10, ( "__kmp_futex_determine_capable: rc = %d errno = %d\n", rc,
453  errno ) );
454  KA_TRACE(10, ( "__kmp_futex_determine_capable: futex syscall%s supported\n",
455  retval ? "" : " not" ) );
456 
457  return retval;
458 }
459 
460 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM) && !KMP_OS_CNK
461 
462 /* ------------------------------------------------------------------------ */
463 /* ------------------------------------------------------------------------ */
464 
465 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS)
466 /*
467  * Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
468  * use compare_and_store for these routines
469  */
470 
471 kmp_int8
472 __kmp_test_then_or8( volatile kmp_int8 *p, kmp_int8 d )
473 {
474  kmp_int8 old_value, new_value;
475 
476  old_value = TCR_1( *p );
477  new_value = old_value | d;
478 
479  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
480  {
481  KMP_CPU_PAUSE();
482  old_value = TCR_1( *p );
483  new_value = old_value | d;
484  }
485  return old_value;
486 }
487 
488 kmp_int8
489 __kmp_test_then_and8( volatile kmp_int8 *p, kmp_int8 d )
490 {
491  kmp_int8 old_value, new_value;
492 
493  old_value = TCR_1( *p );
494  new_value = old_value & d;
495 
496  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
497  {
498  KMP_CPU_PAUSE();
499  old_value = TCR_1( *p );
500  new_value = old_value & d;
501  }
502  return old_value;
503 }
504 
505 kmp_int32
506 __kmp_test_then_or32( volatile kmp_int32 *p, kmp_int32 d )
507 {
508  kmp_int32 old_value, new_value;
509 
510  old_value = TCR_4( *p );
511  new_value = old_value | d;
512 
513  while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
514  {
515  KMP_CPU_PAUSE();
516  old_value = TCR_4( *p );
517  new_value = old_value | d;
518  }
519  return old_value;
520 }
521 
522 kmp_int32
523 __kmp_test_then_and32( volatile kmp_int32 *p, kmp_int32 d )
524 {
525  kmp_int32 old_value, new_value;
526 
527  old_value = TCR_4( *p );
528  new_value = old_value & d;
529 
530  while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
531  {
532  KMP_CPU_PAUSE();
533  old_value = TCR_4( *p );
534  new_value = old_value & d;
535  }
536  return old_value;
537 }
538 
539 # if KMP_ARCH_X86 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64
540 kmp_int8
541 __kmp_test_then_add8( volatile kmp_int8 *p, kmp_int8 d )
542 {
543  kmp_int8 old_value, new_value;
544 
545  old_value = TCR_1( *p );
546  new_value = old_value + d;
547 
548  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
549  {
550  KMP_CPU_PAUSE();
551  old_value = TCR_1( *p );
552  new_value = old_value + d;
553  }
554  return old_value;
555 }
556 
557 kmp_int64
558 __kmp_test_then_add64( volatile kmp_int64 *p, kmp_int64 d )
559 {
560  kmp_int64 old_value, new_value;
561 
562  old_value = TCR_8( *p );
563  new_value = old_value + d;
564 
565  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
566  {
567  KMP_CPU_PAUSE();
568  old_value = TCR_8( *p );
569  new_value = old_value + d;
570  }
571  return old_value;
572 }
573 # endif /* KMP_ARCH_X86 */
574 
575 kmp_int64
576 __kmp_test_then_or64( volatile kmp_int64 *p, kmp_int64 d )
577 {
578  kmp_int64 old_value, new_value;
579 
580  old_value = TCR_8( *p );
581  new_value = old_value | d;
582  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
583  {
584  KMP_CPU_PAUSE();
585  old_value = TCR_8( *p );
586  new_value = old_value | d;
587  }
588  return old_value;
589 }
590 
591 kmp_int64
592 __kmp_test_then_and64( volatile kmp_int64 *p, kmp_int64 d )
593 {
594  kmp_int64 old_value, new_value;
595 
596  old_value = TCR_8( *p );
597  new_value = old_value & d;
598  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
599  {
600  KMP_CPU_PAUSE();
601  old_value = TCR_8( *p );
602  new_value = old_value & d;
603  }
604  return old_value;
605 }
606 
607 #endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */
608 
609 void
610 __kmp_terminate_thread( int gtid )
611 {
612  int status;
613  kmp_info_t *th = __kmp_threads[ gtid ];
614 
615  if ( !th ) return;
616 
617  #ifdef KMP_CANCEL_THREADS
618  KA_TRACE( 10, ("__kmp_terminate_thread: kill (%d)\n", gtid ) );
619  status = pthread_cancel( th->th.th_info.ds.ds_thread );
620  if ( status != 0 && status != ESRCH ) {
621  __kmp_msg(
622  kmp_ms_fatal,
623  KMP_MSG( CantTerminateWorkerThread ),
624  KMP_ERR( status ),
625  __kmp_msg_null
626  );
627  }; // if
628  #endif
629  __kmp_yield( TRUE );
630 } //
631 
632 /* ------------------------------------------------------------------------ */
633 /* ------------------------------------------------------------------------ */
634 
635 /* ------------------------------------------------------------------------ */
636 /* ------------------------------------------------------------------------ */
637 
638 /*
639  * Set thread stack info according to values returned by
640  * pthread_getattr_np().
641  * If values are unreasonable, assume call failed and use
642  * incremental stack refinement method instead.
643  * Returns TRUE if the stack parameters could be determined exactly,
644  * FALSE if incremental refinement is necessary.
645  */
646 static kmp_int32
647 __kmp_set_stack_info( int gtid, kmp_info_t *th )
648 {
649  int stack_data;
650 #if KMP_OS_LINUX || KMP_OS_FREEBSD
651  /* Linux* OS only -- no pthread_getattr_np support on OS X* */
652  pthread_attr_t attr;
653  int status;
654  size_t size = 0;
655  void * addr = 0;
656 
657  /* Always do incremental stack refinement for ubermaster threads since the initial
658  thread stack range can be reduced by sibling thread creation so pthread_attr_getstack
659  may cause thread gtid aliasing */
660  if ( ! KMP_UBER_GTID(gtid) ) {
661 
662  /* Fetch the real thread attributes */
663  status = pthread_attr_init( &attr );
664  KMP_CHECK_SYSFAIL( "pthread_attr_init", status );
665 #if KMP_OS_FREEBSD
666  status = pthread_attr_get_np( pthread_self(), &attr );
667  KMP_CHECK_SYSFAIL( "pthread_attr_get_np", status );
668 #else
669  status = pthread_getattr_np( pthread_self(), &attr );
670  KMP_CHECK_SYSFAIL( "pthread_getattr_np", status );
671 #endif
672  status = pthread_attr_getstack( &attr, &addr, &size );
673  KMP_CHECK_SYSFAIL( "pthread_attr_getstack", status );
674  KA_TRACE( 60, ( "__kmp_set_stack_info: T#%d pthread_attr_getstack returned size: %lu, "
675  "low addr: %p\n",
676  gtid, size, addr ));
677 
678  status = pthread_attr_destroy( &attr );
679  KMP_CHECK_SYSFAIL( "pthread_attr_destroy", status );
680  }
681 
682  if ( size != 0 && addr != 0 ) { /* was stack parameter determination successful? */
683  /* Store the correct base and size */
684  TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size));
685  TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
686  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
687  return TRUE;
688  }
689 #endif /* KMP_OS_LINUX || KMP_OS_FREEBSD */
690  /* Use incremental refinement starting from initial conservative estimate */
691  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
692  TCW_PTR(th -> th.th_info.ds.ds_stackbase, &stack_data);
693  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
694  return FALSE;
695 }
696 
697 static void*
698 __kmp_launch_worker( void *thr )
699 {
700  int status, old_type, old_state;
701 #ifdef KMP_BLOCK_SIGNALS
702  sigset_t new_set, old_set;
703 #endif /* KMP_BLOCK_SIGNALS */
704  void *exit_val;
705  void * volatile padding = 0;
706  int gtid;
707  int error;
708 
709  gtid = ((kmp_info_t*)thr) -> th.th_info.ds.ds_gtid;
710  __kmp_gtid_set_specific( gtid );
711 #ifdef KMP_TDATA_GTID
712  __kmp_gtid = gtid;
713 #endif
714 #if KMP_STATS_ENABLED
715  // set __thread local index to point to thread-specific stats
716  __kmp_stats_thread_ptr = ((kmp_info_t*)thr)->th.th_stats;
717 #endif
718 
719 #if USE_ITT_BUILD
720  __kmp_itt_thread_name( gtid );
721 #endif /* USE_ITT_BUILD */
722 
723 #if KMP_AFFINITY_SUPPORTED
724  __kmp_affinity_set_init_mask( gtid, FALSE );
725 #endif
726 
727 #ifdef KMP_CANCEL_THREADS
728  status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
729  KMP_CHECK_SYSFAIL( "pthread_setcanceltype", status );
730  /* josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? */
731  status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
732  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
733 #endif
734 
735 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
736  //
737  // Set the FP control regs to be a copy of
738  // the parallel initialization thread's.
739  //
740  __kmp_clear_x87_fpu_status_word();
741  __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
742  __kmp_load_mxcsr( &__kmp_init_mxcsr );
743 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
744 
745 #ifdef KMP_BLOCK_SIGNALS
746  status = sigfillset( & new_set );
747  KMP_CHECK_SYSFAIL_ERRNO( "sigfillset", status );
748  status = pthread_sigmask( SIG_BLOCK, & new_set, & old_set );
749  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
750 #endif /* KMP_BLOCK_SIGNALS */
751 
752 #if KMP_OS_LINUX || KMP_OS_FREEBSD
753  if ( __kmp_stkoffset > 0 && gtid > 0 ) {
754  padding = KMP_ALLOCA( gtid * __kmp_stkoffset );
755  }
756 #endif
757 
758  KMP_MB();
759  __kmp_set_stack_info( gtid, (kmp_info_t*)thr );
760 
761  __kmp_check_stack_overlap( (kmp_info_t*)thr );
762 
763  exit_val = __kmp_launch_thread( (kmp_info_t *) thr );
764 
765 #ifdef KMP_BLOCK_SIGNALS
766  status = pthread_sigmask( SIG_SETMASK, & old_set, NULL );
767  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
768 #endif /* KMP_BLOCK_SIGNALS */
769 
770  return exit_val;
771 }
772 
773 
774 /* The monitor thread controls all of the threads in the complex */
775 
776 static void*
777 __kmp_launch_monitor( void *thr )
778 {
779  int status, old_type, old_state;
780 #ifdef KMP_BLOCK_SIGNALS
781  sigset_t new_set;
782 #endif /* KMP_BLOCK_SIGNALS */
783  struct timespec interval;
784  int yield_count;
785  int yield_cycles = 0;
786  int error;
787 
788  KMP_MB(); /* Flush all pending memory write invalidates. */
789 
790  KA_TRACE( 10, ("__kmp_launch_monitor: #1 launched\n" ) );
791 
792  /* register us as the monitor thread */
793  __kmp_gtid_set_specific( KMP_GTID_MONITOR );
794 #ifdef KMP_TDATA_GTID
795  __kmp_gtid = KMP_GTID_MONITOR;
796 #endif
797 
798  KMP_MB();
799 
800 #if USE_ITT_BUILD
801  __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore monitor thread.
802 #endif /* USE_ITT_BUILD */
803 
804  __kmp_set_stack_info( ((kmp_info_t*)thr)->th.th_info.ds.ds_gtid, (kmp_info_t*)thr );
805 
806  __kmp_check_stack_overlap( (kmp_info_t*)thr );
807 
808 #ifdef KMP_CANCEL_THREADS
809  status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
810  KMP_CHECK_SYSFAIL( "pthread_setcanceltype", status );
811  /* josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? */
812  status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
813  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
814 #endif
815 
816  #if KMP_REAL_TIME_FIX
817  // This is a potential fix which allows application with real-time scheduling policy work.
818  // However, decision about the fix is not made yet, so it is disabled by default.
819  { // Are program started with real-time scheduling policy?
820  int sched = sched_getscheduler( 0 );
821  if ( sched == SCHED_FIFO || sched == SCHED_RR ) {
822  // Yes, we are a part of real-time application. Try to increase the priority of the
823  // monitor.
824  struct sched_param param;
825  int max_priority = sched_get_priority_max( sched );
826  int rc;
827  KMP_WARNING( RealTimeSchedNotSupported );
828  sched_getparam( 0, & param );
829  if ( param.sched_priority < max_priority ) {
830  param.sched_priority += 1;
831  rc = sched_setscheduler( 0, sched, & param );
832  if ( rc != 0 ) {
833  int error = errno;
834  __kmp_msg(
835  kmp_ms_warning,
836  KMP_MSG( CantChangeMonitorPriority ),
837  KMP_ERR( error ),
838  KMP_MSG( MonitorWillStarve ),
839  __kmp_msg_null
840  );
841  }; // if
842  } else {
843  // We cannot abort here, because number of CPUs may be enough for all the threads,
844  // including the monitor thread, so application could potentially work...
845  __kmp_msg(
846  kmp_ms_warning,
847  KMP_MSG( RunningAtMaxPriority ),
848  KMP_MSG( MonitorWillStarve ),
849  KMP_HNT( RunningAtMaxPriority ),
850  __kmp_msg_null
851  );
852  }; // if
853  }; // if
854  TCW_4( __kmp_global.g.g_time.dt.t_value, 0 ); // AC: free thread that waits for monitor started
855  }
856  #endif // KMP_REAL_TIME_FIX
857 
858  KMP_MB(); /* Flush all pending memory write invalidates. */
859 
860  if ( __kmp_monitor_wakeups == 1 ) {
861  interval.tv_sec = 1;
862  interval.tv_nsec = 0;
863  } else {
864  interval.tv_sec = 0;
865  interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
866  }
867 
868  KA_TRACE( 10, ("__kmp_launch_monitor: #2 monitor\n" ) );
869 
870  if (__kmp_yield_cycle) {
871  __kmp_yielding_on = 0; /* Start out with yielding shut off */
872  yield_count = __kmp_yield_off_count;
873  } else {
874  __kmp_yielding_on = 1; /* Yielding is on permanently */
875  }
876 
877  while( ! TCR_4( __kmp_global.g.g_done ) ) {
878  struct timespec now;
879  struct timeval tval;
880 
881  /* This thread monitors the state of the system */
882 
883  KA_TRACE( 15, ( "__kmp_launch_monitor: update\n" ) );
884 
885  status = gettimeofday( &tval, NULL );
886  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
887  TIMEVAL_TO_TIMESPEC( &tval, &now );
888 
889  now.tv_sec += interval.tv_sec;
890  now.tv_nsec += interval.tv_nsec;
891 
892  if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
893  now.tv_sec += 1;
894  now.tv_nsec -= KMP_NSEC_PER_SEC;
895  }
896 
897  status = pthread_mutex_lock( & __kmp_wait_mx.m_mutex );
898  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
899  // AC: the monitor should not fall asleep if g_done has been set
900  if ( !TCR_4(__kmp_global.g.g_done) ) { // check once more under mutex
901  status = pthread_cond_timedwait( &__kmp_wait_cv.c_cond, &__kmp_wait_mx.m_mutex, &now );
902  if ( status != 0 ) {
903  if ( status != ETIMEDOUT && status != EINTR ) {
904  KMP_SYSFAIL( "pthread_cond_timedwait", status );
905  };
906  };
907  };
908  status = pthread_mutex_unlock( & __kmp_wait_mx.m_mutex );
909  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
910 
911  if (__kmp_yield_cycle) {
912  yield_cycles++;
913  if ( (yield_cycles % yield_count) == 0 ) {
914  if (__kmp_yielding_on) {
915  __kmp_yielding_on = 0; /* Turn it off now */
916  yield_count = __kmp_yield_off_count;
917  } else {
918  __kmp_yielding_on = 1; /* Turn it on now */
919  yield_count = __kmp_yield_on_count;
920  }
921  yield_cycles = 0;
922  }
923  } else {
924  __kmp_yielding_on = 1;
925  }
926 
927  TCW_4( __kmp_global.g.g_time.dt.t_value,
928  TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
929 
930  KMP_MB(); /* Flush all pending memory write invalidates. */
931  }
932 
933  KA_TRACE( 10, ("__kmp_launch_monitor: #3 cleanup\n" ) );
934 
935 #ifdef KMP_BLOCK_SIGNALS
936  status = sigfillset( & new_set );
937  KMP_CHECK_SYSFAIL_ERRNO( "sigfillset", status );
938  status = pthread_sigmask( SIG_UNBLOCK, & new_set, NULL );
939  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
940 #endif /* KMP_BLOCK_SIGNALS */
941 
942  KA_TRACE( 10, ("__kmp_launch_monitor: #4 finished\n" ) );
943 
944  if( __kmp_global.g.g_abort != 0 ) {
945  /* now we need to terminate the worker threads */
946  /* the value of t_abort is the signal we caught */
947 
948  int gtid;
949 
950  KA_TRACE( 10, ("__kmp_launch_monitor: #5 terminate sig=%d\n", __kmp_global.g.g_abort ) );
951 
952  /* terminate the OpenMP worker threads */
953  /* TODO this is not valid for sibling threads!!
954  * the uber master might not be 0 anymore.. */
955  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
956  __kmp_terminate_thread( gtid );
957 
958  __kmp_cleanup();
959 
960  KA_TRACE( 10, ("__kmp_launch_monitor: #6 raise sig=%d\n", __kmp_global.g.g_abort ) );
961 
962  if (__kmp_global.g.g_abort > 0)
963  raise( __kmp_global.g.g_abort );
964 
965  }
966 
967  KA_TRACE( 10, ("__kmp_launch_monitor: #7 exit\n" ) );
968 
969  return thr;
970 }
971 
972 void
973 __kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size )
974 {
975  pthread_t handle;
976  pthread_attr_t thread_attr;
977  int status;
978 
979 
980  th->th.th_info.ds.ds_gtid = gtid;
981 
982 #if KMP_STATS_ENABLED
983  // sets up worker thread stats
984  __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
985 
986  // th->th.th_stats is used to transfer thread specific stats-pointer to __kmp_launch_worker
987  // So when thread is created (goes into __kmp_launch_worker) it will
988  // set it's __thread local pointer to th->th.th_stats
989  th->th.th_stats = __kmp_stats_list.push_back(gtid);
990  if(KMP_UBER_GTID(gtid)) {
991  __kmp_stats_start_time = tsc_tick_count::now();
992  __kmp_stats_thread_ptr = th->th.th_stats;
993  __kmp_stats_init();
994  KMP_START_EXPLICIT_TIMER(OMP_serial);
995  KMP_START_EXPLICIT_TIMER(OMP_start_end);
996  }
997  __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
998 
999 #endif // KMP_STATS_ENABLED
1000 
1001  if ( KMP_UBER_GTID(gtid) ) {
1002  KA_TRACE( 10, ("__kmp_create_worker: uber thread (%d)\n", gtid ) );
1003  th -> th.th_info.ds.ds_thread = pthread_self();
1004  __kmp_set_stack_info( gtid, th );
1005  __kmp_check_stack_overlap( th );
1006  return;
1007  }; // if
1008 
1009  KA_TRACE( 10, ("__kmp_create_worker: try to create thread (%d)\n", gtid ) );
1010 
1011  KMP_MB(); /* Flush all pending memory write invalidates. */
1012 
1013 #ifdef KMP_THREAD_ATTR
1014  {
1015  status = pthread_attr_init( &thread_attr );
1016  if ( status != 0 ) {
1017  __kmp_msg(
1018  kmp_ms_fatal,
1019  KMP_MSG( CantInitThreadAttrs ),
1020  KMP_ERR( status ),
1021  __kmp_msg_null
1022  );
1023  }; // if
1024  status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1025  if ( status != 0 ) {
1026  __kmp_msg(
1027  kmp_ms_fatal,
1028  KMP_MSG( CantSetWorkerState ),
1029  KMP_ERR( status ),
1030  __kmp_msg_null
1031  );
1032  }; // if
1033 
1034  /* Set stack size for this thread now. */
1035  // Stack compensation for the stack offset.
1036  // When the thread is created and __kmp_stkoffset != 0, then
1037  // it will alloca(), and the stacks will then
1038  // be offset from a page by different values for different threads.
1039  stack_size += gtid * __kmp_stkoffset * 2;
1040 
1041  KA_TRACE( 10, ( "__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
1042  "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
1043  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size ) );
1044 
1045 # ifdef _POSIX_THREAD_ATTR_STACKSIZE
1046  status = pthread_attr_setstacksize( & thread_attr, stack_size );
1047 # ifdef KMP_BACKUP_STKSIZE
1048  if ( status != 0 ) {
1049  if ( ! __kmp_env_stksize ) {
1050  stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
1051  __kmp_stksize = KMP_BACKUP_STKSIZE;
1052  KA_TRACE( 10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
1053  "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
1054  "bytes\n",
1055  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size )
1056  );
1057  status = pthread_attr_setstacksize( &thread_attr, stack_size );
1058  }; // if
1059  }; // if
1060 # endif /* KMP_BACKUP_STKSIZE */
1061  if ( status != 0 ) {
1062  __kmp_msg(
1063  kmp_ms_fatal,
1064  KMP_MSG( CantSetWorkerStackSize, stack_size ),
1065  KMP_ERR( status ),
1066  KMP_HNT( ChangeWorkerStackSize ),
1067  __kmp_msg_null
1068  );
1069  }; // if
1070 # endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1071  }
1072 #endif /* KMP_THREAD_ATTR */
1073 
1074  {
1075  status = pthread_create( & handle, & thread_attr, __kmp_launch_worker, (void *) th );
1076  if ( status != 0 || ! handle ) { // ??? Why do we check handle??
1077 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1078  if ( status == EINVAL ) {
1079  __kmp_msg(
1080  kmp_ms_fatal,
1081  KMP_MSG( CantSetWorkerStackSize, stack_size ),
1082  KMP_ERR( status ),
1083  KMP_HNT( IncreaseWorkerStackSize ),
1084  __kmp_msg_null
1085  );
1086  };
1087  if ( status == ENOMEM ) {
1088  __kmp_msg(
1089  kmp_ms_fatal,
1090  KMP_MSG( CantSetWorkerStackSize, stack_size ),
1091  KMP_ERR( status ),
1092  KMP_HNT( DecreaseWorkerStackSize ),
1093  __kmp_msg_null
1094  );
1095  };
1096 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1097  if ( status == EAGAIN ) {
1098  __kmp_msg(
1099  kmp_ms_fatal,
1100  KMP_MSG( NoResourcesForWorkerThread ),
1101  KMP_ERR( status ),
1102  KMP_HNT( Decrease_NUM_THREADS ),
1103  __kmp_msg_null
1104  );
1105  }; // if
1106  KMP_SYSFAIL( "pthread_create", status );
1107  }; // if
1108 
1109  th->th.th_info.ds.ds_thread = handle;
1110  }
1111 
1112 #ifdef KMP_THREAD_ATTR
1113  {
1114  status = pthread_attr_destroy( & thread_attr );
1115  if ( status ) {
1116  __kmp_msg(
1117  kmp_ms_warning,
1118  KMP_MSG( CantDestroyThreadAttrs ),
1119  KMP_ERR( status ),
1120  __kmp_msg_null
1121  );
1122  }; // if
1123  }
1124 #endif /* KMP_THREAD_ATTR */
1125 
1126  KMP_MB(); /* Flush all pending memory write invalidates. */
1127 
1128  KA_TRACE( 10, ("__kmp_create_worker: done creating thread (%d)\n", gtid ) );
1129 
1130 } // __kmp_create_worker
1131 
1132 
1133 void
1134 __kmp_create_monitor( kmp_info_t *th )
1135 {
1136  pthread_t handle;
1137  pthread_attr_t thread_attr;
1138  size_t size;
1139  int status;
1140  int caller_gtid = __kmp_get_gtid();
1141  int auto_adj_size = FALSE;
1142 
1143  KA_TRACE( 10, ("__kmp_create_monitor: try to create monitor\n" ) );
1144 
1145  KMP_MB(); /* Flush all pending memory write invalidates. */
1146 
1147  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1148  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1149  #if KMP_REAL_TIME_FIX
1150  TCW_4( __kmp_global.g.g_time.dt.t_value, -1 ); // Will use it for synchronization a bit later.
1151  #else
1152  TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
1153  #endif // KMP_REAL_TIME_FIX
1154 
1155  #ifdef KMP_THREAD_ATTR
1156  if ( __kmp_monitor_stksize == 0 ) {
1157  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1158  auto_adj_size = TRUE;
1159  }
1160  status = pthread_attr_init( &thread_attr );
1161  if ( status != 0 ) {
1162  __kmp_msg(
1163  kmp_ms_fatal,
1164  KMP_MSG( CantInitThreadAttrs ),
1165  KMP_ERR( status ),
1166  __kmp_msg_null
1167  );
1168  }; // if
1169  status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1170  if ( status != 0 ) {
1171  __kmp_msg(
1172  kmp_ms_fatal,
1173  KMP_MSG( CantSetMonitorState ),
1174  KMP_ERR( status ),
1175  __kmp_msg_null
1176  );
1177  }; // if
1178 
1179  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1180  status = pthread_attr_getstacksize( & thread_attr, & size );
1181  KMP_CHECK_SYSFAIL( "pthread_attr_getstacksize", status );
1182  #else
1183  size = __kmp_sys_min_stksize;
1184  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1185  #endif /* KMP_THREAD_ATTR */
1186 
1187  if ( __kmp_monitor_stksize == 0 ) {
1188  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1189  }
1190  if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
1191  __kmp_monitor_stksize = __kmp_sys_min_stksize;
1192  }
1193 
1194  KA_TRACE( 10, ( "__kmp_create_monitor: default stacksize = %lu bytes,"
1195  "requested stacksize = %lu bytes\n",
1196  size, __kmp_monitor_stksize ) );
1197 
1198  retry:
1199 
1200  /* Set stack size for this thread now. */
1201 
1202  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1203  KA_TRACE( 10, ( "__kmp_create_monitor: setting stacksize = %lu bytes,",
1204  __kmp_monitor_stksize ) );
1205  status = pthread_attr_setstacksize( & thread_attr, __kmp_monitor_stksize );
1206  if ( status != 0 ) {
1207  if ( auto_adj_size ) {
1208  __kmp_monitor_stksize *= 2;
1209  goto retry;
1210  }
1211  __kmp_msg(
1212  kmp_ms_warning, // should this be fatal? BB
1213  KMP_MSG( CantSetMonitorStackSize, (long int) __kmp_monitor_stksize ),
1214  KMP_ERR( status ),
1215  KMP_HNT( ChangeMonitorStackSize ),
1216  __kmp_msg_null
1217  );
1218  }; // if
1219  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1220 
1221  status = pthread_create( &handle, & thread_attr, __kmp_launch_monitor, (void *) th );
1222 
1223  if ( status != 0 ) {
1224  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1225  if ( status == EINVAL ) {
1226  if ( auto_adj_size && ( __kmp_monitor_stksize < (size_t)0x40000000 ) ) {
1227  __kmp_monitor_stksize *= 2;
1228  goto retry;
1229  }
1230  __kmp_msg(
1231  kmp_ms_fatal,
1232  KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1233  KMP_ERR( status ),
1234  KMP_HNT( IncreaseMonitorStackSize ),
1235  __kmp_msg_null
1236  );
1237  }; // if
1238  if ( status == ENOMEM ) {
1239  __kmp_msg(
1240  kmp_ms_fatal,
1241  KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1242  KMP_ERR( status ),
1243  KMP_HNT( DecreaseMonitorStackSize ),
1244  __kmp_msg_null
1245  );
1246  }; // if
1247  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1248  if ( status == EAGAIN ) {
1249  __kmp_msg(
1250  kmp_ms_fatal,
1251  KMP_MSG( NoResourcesForMonitorThread ),
1252  KMP_ERR( status ),
1253  KMP_HNT( DecreaseNumberOfThreadsInUse ),
1254  __kmp_msg_null
1255  );
1256  }; // if
1257  KMP_SYSFAIL( "pthread_create", status );
1258  }; // if
1259 
1260  th->th.th_info.ds.ds_thread = handle;
1261 
1262  #if KMP_REAL_TIME_FIX
1263  // Wait for the monitor thread is really started and set its *priority*.
1264  KMP_DEBUG_ASSERT( sizeof( kmp_uint32 ) == sizeof( __kmp_global.g.g_time.dt.t_value ) );
1265  __kmp_wait_yield_4(
1266  (kmp_uint32 volatile *) & __kmp_global.g.g_time.dt.t_value, -1, & __kmp_neq_4, NULL
1267  );
1268  #endif // KMP_REAL_TIME_FIX
1269 
1270  #ifdef KMP_THREAD_ATTR
1271  status = pthread_attr_destroy( & thread_attr );
1272  if ( status != 0 ) {
1273  __kmp_msg( //
1274  kmp_ms_warning,
1275  KMP_MSG( CantDestroyThreadAttrs ),
1276  KMP_ERR( status ),
1277  __kmp_msg_null
1278  );
1279  }; // if
1280  #endif
1281 
1282  KMP_MB(); /* Flush all pending memory write invalidates. */
1283 
1284  KA_TRACE( 10, ( "__kmp_create_monitor: monitor created %#.8lx\n", th->th.th_info.ds.ds_thread ) );
1285 
1286 } // __kmp_create_monitor
1287 
1288 void
1289 __kmp_exit_thread(
1290  int exit_status
1291 ) {
1292  pthread_exit( (void *)(intptr_t) exit_status );
1293 } // __kmp_exit_thread
1294 
1295 void __kmp_resume_monitor();
1296 
1297 void
1298 __kmp_reap_monitor( kmp_info_t *th )
1299 {
1300  int status, i;
1301  void *exit_val;
1302 
1303  KA_TRACE( 10, ("__kmp_reap_monitor: try to reap monitor thread with handle %#.8lx\n",
1304  th->th.th_info.ds.ds_thread ) );
1305 
1306  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1307  // If both tid and gtid are 0, it means the monitor did not ever start.
1308  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1309  KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
1310  if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
1311  return;
1312  }; // if
1313 
1314  KMP_MB(); /* Flush all pending memory write invalidates. */
1315 
1316 
1317  /* First, check to see whether the monitor thread exists. This could prevent a hang,
1318  but if the monitor dies after the pthread_kill call and before the pthread_join
1319  call, it will still hang. */
1320 
1321  status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
1322  if (status == ESRCH) {
1323 
1324  KA_TRACE( 10, ("__kmp_reap_monitor: monitor does not exist, returning\n") );
1325 
1326  } else
1327  {
1328  __kmp_resume_monitor(); // Wake up the monitor thread
1329  status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1330  if (exit_val != th) {
1331  __kmp_msg(
1332  kmp_ms_fatal,
1333  KMP_MSG( ReapMonitorError ),
1334  KMP_ERR( status ),
1335  __kmp_msg_null
1336  );
1337  }
1338  }
1339 
1340  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1341  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1342 
1343  KA_TRACE( 10, ("__kmp_reap_monitor: done reaping monitor thread with handle %#.8lx\n",
1344  th->th.th_info.ds.ds_thread ) );
1345 
1346  KMP_MB(); /* Flush all pending memory write invalidates. */
1347 
1348 }
1349 
1350 void
1351 __kmp_reap_worker( kmp_info_t *th )
1352 {
1353  int status;
1354  void *exit_val;
1355 
1356  KMP_MB(); /* Flush all pending memory write invalidates. */
1357 
1358  KA_TRACE( 10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid ) );
1359 
1360  /* First, check to see whether the worker thread exists. This could prevent a hang,
1361  but if the worker dies after the pthread_kill call and before the pthread_join
1362  call, it will still hang. */
1363 
1364  {
1365  status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
1366  if (status == ESRCH) {
1367  KA_TRACE( 10, ("__kmp_reap_worker: worker T#%d does not exist, returning\n",
1368  th->th.th_info.ds.ds_gtid ) );
1369  }
1370  else {
1371  KA_TRACE( 10, ("__kmp_reap_worker: try to join with worker T#%d\n",
1372  th->th.th_info.ds.ds_gtid ) );
1373 
1374  status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1375 #ifdef KMP_DEBUG
1376  /* Don't expose these to the user until we understand when they trigger */
1377  if ( status != 0 ) {
1378  __kmp_msg(
1379  kmp_ms_fatal,
1380  KMP_MSG( ReapWorkerError ),
1381  KMP_ERR( status ),
1382  __kmp_msg_null
1383  );
1384  }
1385  if ( exit_val != th ) {
1386  KA_TRACE( 10, ( "__kmp_reap_worker: worker T#%d did not reap properly, "
1387  "exit_val = %p\n",
1388  th->th.th_info.ds.ds_gtid, exit_val ) );
1389  }
1390 #endif /* KMP_DEBUG */
1391  }
1392  }
1393 
1394  KA_TRACE( 10, ("__kmp_reap_worker: done reaping T#%d\n", th->th.th_info.ds.ds_gtid ) );
1395 
1396  KMP_MB(); /* Flush all pending memory write invalidates. */
1397 }
1398 
1399 
1400 /* ------------------------------------------------------------------------ */
1401 /* ------------------------------------------------------------------------ */
1402 
1403 #if KMP_HANDLE_SIGNALS
1404 
1405 
1406 static void
1407 __kmp_null_handler( int signo )
1408 {
1409  // Do nothing, for doing SIG_IGN-type actions.
1410 } // __kmp_null_handler
1411 
1412 
1413 static void
1414 __kmp_team_handler( int signo )
1415 {
1416  if ( __kmp_global.g.g_abort == 0 ) {
1417  /* Stage 1 signal handler, let's shut down all of the threads */
1418  #ifdef KMP_DEBUG
1419  __kmp_debug_printf( "__kmp_team_handler: caught signal = %d\n", signo );
1420  #endif
1421  switch ( signo ) {
1422  case SIGHUP :
1423  case SIGINT :
1424  case SIGQUIT :
1425  case SIGILL :
1426  case SIGABRT :
1427  case SIGFPE :
1428  case SIGBUS :
1429  case SIGSEGV :
1430  #ifdef SIGSYS
1431  case SIGSYS :
1432  #endif
1433  case SIGTERM :
1434  if ( __kmp_debug_buf ) {
1435  __kmp_dump_debug_buffer( );
1436  }; // if
1437  KMP_MB(); // Flush all pending memory write invalidates.
1438  TCW_4( __kmp_global.g.g_abort, signo );
1439  KMP_MB(); // Flush all pending memory write invalidates.
1440  TCW_4( __kmp_global.g.g_done, TRUE );
1441  KMP_MB(); // Flush all pending memory write invalidates.
1442  break;
1443  default:
1444  #ifdef KMP_DEBUG
1445  __kmp_debug_printf( "__kmp_team_handler: unknown signal type" );
1446  #endif
1447  break;
1448  }; // switch
1449  }; // if
1450 } // __kmp_team_handler
1451 
1452 
1453 static
1454 void __kmp_sigaction( int signum, const struct sigaction * act, struct sigaction * oldact ) {
1455  int rc = sigaction( signum, act, oldact );
1456  KMP_CHECK_SYSFAIL_ERRNO( "sigaction", rc );
1457 }
1458 
1459 
1460 static void
1461 __kmp_install_one_handler( int sig, sig_func_t handler_func, int parallel_init )
1462 {
1463  KMP_MB(); // Flush all pending memory write invalidates.
1464  KB_TRACE( 60, ( "__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init ) );
1465  if ( parallel_init ) {
1466  struct sigaction new_action;
1467  struct sigaction old_action;
1468  new_action.sa_handler = handler_func;
1469  new_action.sa_flags = 0;
1470  sigfillset( & new_action.sa_mask );
1471  __kmp_sigaction( sig, & new_action, & old_action );
1472  if ( old_action.sa_handler == __kmp_sighldrs[ sig ].sa_handler ) {
1473  sigaddset( & __kmp_sigset, sig );
1474  } else {
1475  // Restore/keep user's handler if one previously installed.
1476  __kmp_sigaction( sig, & old_action, NULL );
1477  }; // if
1478  } else {
1479  // Save initial/system signal handlers to see if user handlers installed.
1480  __kmp_sigaction( sig, NULL, & __kmp_sighldrs[ sig ] );
1481  }; // if
1482  KMP_MB(); // Flush all pending memory write invalidates.
1483 } // __kmp_install_one_handler
1484 
1485 
1486 static void
1487 __kmp_remove_one_handler( int sig )
1488 {
1489  KB_TRACE( 60, ( "__kmp_remove_one_handler( %d )\n", sig ) );
1490  if ( sigismember( & __kmp_sigset, sig ) ) {
1491  struct sigaction old;
1492  KMP_MB(); // Flush all pending memory write invalidates.
1493  __kmp_sigaction( sig, & __kmp_sighldrs[ sig ], & old );
1494  if ( ( old.sa_handler != __kmp_team_handler ) && ( old.sa_handler != __kmp_null_handler ) ) {
1495  // Restore the users signal handler.
1496  KB_TRACE( 10, ( "__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
1497  __kmp_sigaction( sig, & old, NULL );
1498  }; // if
1499  sigdelset( & __kmp_sigset, sig );
1500  KMP_MB(); // Flush all pending memory write invalidates.
1501  }; // if
1502 } // __kmp_remove_one_handler
1503 
1504 
1505 void
1506 __kmp_install_signals( int parallel_init )
1507 {
1508  KB_TRACE( 10, ( "__kmp_install_signals( %d )\n", parallel_init ) );
1509  if ( __kmp_handle_signals || ! parallel_init ) {
1510  // If ! parallel_init, we do not install handlers, just save original handlers.
1511  // Let us do it even __handle_signals is 0.
1512  sigemptyset( & __kmp_sigset );
1513  __kmp_install_one_handler( SIGHUP, __kmp_team_handler, parallel_init );
1514  __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
1515  __kmp_install_one_handler( SIGQUIT, __kmp_team_handler, parallel_init );
1516  __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
1517  __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
1518  __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
1519  __kmp_install_one_handler( SIGBUS, __kmp_team_handler, parallel_init );
1520  __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
1521  #ifdef SIGSYS
1522  __kmp_install_one_handler( SIGSYS, __kmp_team_handler, parallel_init );
1523  #endif // SIGSYS
1524  __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
1525  #ifdef SIGPIPE
1526  __kmp_install_one_handler( SIGPIPE, __kmp_team_handler, parallel_init );
1527  #endif // SIGPIPE
1528  }; // if
1529 } // __kmp_install_signals
1530 
1531 
1532 void
1533 __kmp_remove_signals( void )
1534 {
1535  int sig;
1536  KB_TRACE( 10, ( "__kmp_remove_signals()\n" ) );
1537  for ( sig = 1; sig < NSIG; ++ sig ) {
1538  __kmp_remove_one_handler( sig );
1539  }; // for sig
1540 } // __kmp_remove_signals
1541 
1542 
1543 #endif // KMP_HANDLE_SIGNALS
1544 
1545 /* ------------------------------------------------------------------------ */
1546 /* ------------------------------------------------------------------------ */
1547 
1548 void
1549 __kmp_enable( int new_state )
1550 {
1551  #ifdef KMP_CANCEL_THREADS
1552  int status, old_state;
1553  status = pthread_setcancelstate( new_state, & old_state );
1554  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
1555  KMP_DEBUG_ASSERT( old_state == PTHREAD_CANCEL_DISABLE );
1556  #endif
1557 }
1558 
1559 void
1560 __kmp_disable( int * old_state )
1561 {
1562  #ifdef KMP_CANCEL_THREADS
1563  int status;
1564  status = pthread_setcancelstate( PTHREAD_CANCEL_DISABLE, old_state );
1565  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
1566  #endif
1567 }
1568 
1569 /* ------------------------------------------------------------------------ */
1570 /* ------------------------------------------------------------------------ */
1571 
1572 static void
1573 __kmp_atfork_prepare (void)
1574 {
1575  /* nothing to do */
1576 }
1577 
1578 static void
1579 __kmp_atfork_parent (void)
1580 {
1581  /* nothing to do */
1582 }
1583 
1584 /*
1585  Reset the library so execution in the child starts "all over again" with
1586  clean data structures in initial states. Don't worry about freeing memory
1587  allocated by parent, just abandon it to be safe.
1588 */
1589 static void
1590 __kmp_atfork_child (void)
1591 {
1592  /* TODO make sure this is done right for nested/sibling */
1593  // ATT: Memory leaks are here? TODO: Check it and fix.
1594  /* KMP_ASSERT( 0 ); */
1595 
1596  ++__kmp_fork_count;
1597 
1598  __kmp_init_runtime = FALSE;
1599  __kmp_init_monitor = 0;
1600  __kmp_init_parallel = FALSE;
1601  __kmp_init_middle = FALSE;
1602  __kmp_init_serial = FALSE;
1603  TCW_4(__kmp_init_gtid, FALSE);
1604  __kmp_init_common = FALSE;
1605 
1606  TCW_4(__kmp_init_user_locks, FALSE);
1607 #if ! KMP_USE_DYNAMIC_LOCK
1608  __kmp_user_lock_table.used = 1;
1609  __kmp_user_lock_table.allocated = 0;
1610  __kmp_user_lock_table.table = NULL;
1611  __kmp_lock_blocks = NULL;
1612 #endif
1613 
1614  __kmp_all_nth = 0;
1615  TCW_4(__kmp_nth, 0);
1616 
1617  /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate here
1618  so threadprivate doesn't use stale data */
1619  KA_TRACE( 10, ( "__kmp_atfork_child: checking cache address list %p\n",
1620  __kmp_threadpriv_cache_list ) );
1621 
1622  while ( __kmp_threadpriv_cache_list != NULL ) {
1623 
1624  if ( *__kmp_threadpriv_cache_list -> addr != NULL ) {
1625  KC_TRACE( 50, ( "__kmp_atfork_child: zeroing cache at address %p\n",
1626  &(*__kmp_threadpriv_cache_list -> addr) ) );
1627 
1628  *__kmp_threadpriv_cache_list -> addr = NULL;
1629  }
1630  __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list -> next;
1631  }
1632 
1633  __kmp_init_runtime = FALSE;
1634 
1635  /* reset statically initialized locks */
1636  __kmp_init_bootstrap_lock( &__kmp_initz_lock );
1637  __kmp_init_bootstrap_lock( &__kmp_stdio_lock );
1638  __kmp_init_bootstrap_lock( &__kmp_console_lock );
1639 
1640  /* This is necessary to make sure no stale data is left around */
1641  /* AC: customers complain that we use unsafe routines in the atfork
1642  handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen
1643  in dynamic_link when check the presence of shared tbbmalloc library.
1644  Suggestion is to make the library initialization lazier, similar
1645  to what done for __kmpc_begin(). */
1646  // TODO: synchronize all static initializations with regular library
1647  // startup; look at kmp_global.c and etc.
1648  //__kmp_internal_begin ();
1649 
1650 }
1651 
1652 void
1653 __kmp_register_atfork(void) {
1654  if ( __kmp_need_register_atfork ) {
1655  int status = pthread_atfork( __kmp_atfork_prepare, __kmp_atfork_parent, __kmp_atfork_child );
1656  KMP_CHECK_SYSFAIL( "pthread_atfork", status );
1657  __kmp_need_register_atfork = FALSE;
1658  }
1659 }
1660 
1661 void
1662 __kmp_suspend_initialize( void )
1663 {
1664  int status;
1665  status = pthread_mutexattr_init( &__kmp_suspend_mutex_attr );
1666  KMP_CHECK_SYSFAIL( "pthread_mutexattr_init", status );
1667  status = pthread_condattr_init( &__kmp_suspend_cond_attr );
1668  KMP_CHECK_SYSFAIL( "pthread_condattr_init", status );
1669 }
1670 
1671 static void
1672 __kmp_suspend_initialize_thread( kmp_info_t *th )
1673 {
1674  if ( th->th.th_suspend_init_count <= __kmp_fork_count ) {
1675  /* this means we haven't initialized the suspension pthread objects for this thread
1676  in this instance of the process */
1677  int status;
1678  status = pthread_cond_init( &th->th.th_suspend_cv.c_cond, &__kmp_suspend_cond_attr );
1679  KMP_CHECK_SYSFAIL( "pthread_cond_init", status );
1680  status = pthread_mutex_init( &th->th.th_suspend_mx.m_mutex, & __kmp_suspend_mutex_attr );
1681  KMP_CHECK_SYSFAIL( "pthread_mutex_init", status );
1682  *(volatile int*)&th->th.th_suspend_init_count = __kmp_fork_count + 1;
1683  };
1684 }
1685 
1686 void
1687 __kmp_suspend_uninitialize_thread( kmp_info_t *th )
1688 {
1689  if(th->th.th_suspend_init_count > __kmp_fork_count) {
1690  /* this means we have initialize the suspension pthread objects for this thread
1691  in this instance of the process */
1692  int status;
1693 
1694  status = pthread_cond_destroy( &th->th.th_suspend_cv.c_cond );
1695  if ( status != 0 && status != EBUSY ) {
1696  KMP_SYSFAIL( "pthread_cond_destroy", status );
1697  };
1698  status = pthread_mutex_destroy( &th->th.th_suspend_mx.m_mutex );
1699  if ( status != 0 && status != EBUSY ) {
1700  KMP_SYSFAIL( "pthread_mutex_destroy", status );
1701  };
1702  --th->th.th_suspend_init_count;
1703  KMP_DEBUG_ASSERT(th->th.th_suspend_init_count == __kmp_fork_count);
1704  }
1705 }
1706 
1707 /* This routine puts the calling thread to sleep after setting the
1708  * sleep bit for the indicated flag variable to true.
1709  */
1710 template <class C>
1711 static inline void __kmp_suspend_template( int th_gtid, C *flag )
1712 {
1713  KMP_TIME_BLOCK(USER_suspend);
1714  kmp_info_t *th = __kmp_threads[th_gtid];
1715  int status;
1716  typename C::flag_t old_spin;
1717 
1718  KF_TRACE( 30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid, flag->get() ) );
1719 
1720  __kmp_suspend_initialize_thread( th );
1721 
1722  status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1723  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1724 
1725  KF_TRACE( 10, ( "__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1726  th_gtid, flag->get() ) );
1727 
1728  /* TODO: shouldn't this use release semantics to ensure that __kmp_suspend_initialize_thread
1729  gets called first?
1730  */
1731  old_spin = flag->set_sleeping();
1732 
1733  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x, was %x\n",
1734  th_gtid, flag->get(), *(flag->get()), old_spin ) );
1735 
1736  if ( flag->done_check_val(old_spin) ) {
1737  old_spin = flag->unset_sleeping();
1738  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d false alarm, reset sleep bit for spin(%p)\n",
1739  th_gtid, flag->get()) );
1740  } else {
1741  /* Encapsulate in a loop as the documentation states that this may
1742  * "with low probability" return when the condition variable has
1743  * not been signaled or broadcast
1744  */
1745  int deactivated = FALSE;
1746  TCW_PTR(th->th.th_sleep_loc, (void *)flag);
1747  while ( flag->is_sleeping() ) {
1748 #ifdef DEBUG_SUSPEND
1749  char buffer[128];
1750  __kmp_suspend_count++;
1751  __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1752  __kmp_printf( "__kmp_suspend_template: suspending T#%d: %s\n", th_gtid, buffer );
1753 #endif
1754  // Mark the thread as no longer active (only in the first iteration of the loop).
1755  if ( ! deactivated ) {
1756  th->th.th_active = FALSE;
1757  if ( th->th.th_active_in_pool ) {
1758  th->th.th_active_in_pool = FALSE;
1759  KMP_TEST_THEN_DEC32(
1760  (kmp_int32 *) &__kmp_thread_pool_active_nth );
1761  KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
1762  }
1763  deactivated = TRUE;
1764 
1765 
1766  }
1767 
1768 #if USE_SUSPEND_TIMEOUT
1769  struct timespec now;
1770  struct timeval tval;
1771  int msecs;
1772 
1773  status = gettimeofday( &tval, NULL );
1774  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
1775  TIMEVAL_TO_TIMESPEC( &tval, &now );
1776 
1777  msecs = (4*__kmp_dflt_blocktime) + 200;
1778  now.tv_sec += msecs / 1000;
1779  now.tv_nsec += (msecs % 1000)*1000;
1780 
1781  KF_TRACE( 15, ( "__kmp_suspend_template: T#%d about to perform pthread_cond_timedwait\n",
1782  th_gtid ) );
1783  status = pthread_cond_timedwait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex, & now );
1784 #else
1785  KF_TRACE( 15, ( "__kmp_suspend_template: T#%d about to perform pthread_cond_wait\n",
1786  th_gtid ) );
1787  status = pthread_cond_wait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex );
1788 #endif
1789 
1790  if ( (status != 0) && (status != EINTR) && (status != ETIMEDOUT) ) {
1791  KMP_SYSFAIL( "pthread_cond_wait", status );
1792  }
1793 #ifdef KMP_DEBUG
1794  if (status == ETIMEDOUT) {
1795  if ( flag->is_sleeping() ) {
1796  KF_TRACE( 100, ( "__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid ) );
1797  } else {
1798  KF_TRACE( 2, ( "__kmp_suspend_template: T#%d timeout wakeup, sleep bit not set!\n",
1799  th_gtid ) );
1800  }
1801  } else if ( flag->is_sleeping() ) {
1802  KF_TRACE( 100, ( "__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid ) );
1803  }
1804 #endif
1805  } // while
1806 
1807  // Mark the thread as active again (if it was previous marked as inactive)
1808  if ( deactivated ) {
1809  th->th.th_active = TRUE;
1810  if ( TCR_4(th->th.th_in_pool) ) {
1811  KMP_TEST_THEN_INC32( (kmp_int32 *) &__kmp_thread_pool_active_nth );
1812  th->th.th_active_in_pool = TRUE;
1813  }
1814  }
1815  }
1816 
1817 #ifdef DEBUG_SUSPEND
1818  {
1819  char buffer[128];
1820  __kmp_print_cond( buffer, &th->th.th_suspend_cv);
1821  __kmp_printf( "__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid, buffer );
1822  }
1823 #endif
1824 
1825 
1826  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1827  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1828 
1829  KF_TRACE( 30, ("__kmp_suspend_template: T#%d exit\n", th_gtid ) );
1830 }
1831 
1832 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
1833  __kmp_suspend_template(th_gtid, flag);
1834 }
1835 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
1836  __kmp_suspend_template(th_gtid, flag);
1837 }
1838 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
1839  __kmp_suspend_template(th_gtid, flag);
1840 }
1841 
1842 /* This routine signals the thread specified by target_gtid to wake up
1843  * after setting the sleep bit indicated by the flag argument to FALSE.
1844  * The target thread must already have called __kmp_suspend_template()
1845  */
1846 template <class C>
1847 static inline void __kmp_resume_template( int target_gtid, C *flag )
1848 {
1849  kmp_info_t *th = __kmp_threads[target_gtid];
1850  int status;
1851 
1852 #ifdef KMP_DEBUG
1853  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1854 #endif
1855 
1856  KF_TRACE( 30, ( "__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );
1857  KMP_DEBUG_ASSERT( gtid != target_gtid );
1858 
1859  __kmp_suspend_initialize_thread( th );
1860 
1861  status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1862  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1863 
1864  if (!flag) {
1865  flag = (C *)th->th.th_sleep_loc;
1866  }
1867 
1868  if (!flag) {
1869  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p)\n",
1870  gtid, target_gtid, NULL ) );
1871  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1872  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1873  return;
1874  }
1875  else { // if multiple threads are sleeping, flag should be internally referring to a specific thread here
1876  typename C::flag_t old_spin = flag->unset_sleeping();
1877  if ( ! flag->is_sleeping_val(old_spin) ) {
1878  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p): "
1879  "%u => %u\n",
1880  gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1881 
1882  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1883  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1884  return;
1885  }
1886  KF_TRACE( 5, ( "__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p): "
1887  "%u => %u\n",
1888  gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1889  }
1890  TCW_PTR(th->th.th_sleep_loc, NULL);
1891 
1892 
1893 #ifdef DEBUG_SUSPEND
1894  {
1895  char buffer[128];
1896  __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1897  __kmp_printf( "__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid, target_gtid, buffer );
1898  }
1899 #endif
1900 
1901 
1902  status = pthread_cond_signal( &th->th.th_suspend_cv.c_cond );
1903  KMP_CHECK_SYSFAIL( "pthread_cond_signal", status );
1904  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1905  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1906  KF_TRACE( 30, ( "__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n",
1907  gtid, target_gtid ) );
1908 }
1909 
1910 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
1911  __kmp_resume_template(target_gtid, flag);
1912 }
1913 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
1914  __kmp_resume_template(target_gtid, flag);
1915 }
1916 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
1917  __kmp_resume_template(target_gtid, flag);
1918 }
1919 
1920 void
1921 __kmp_resume_monitor()
1922 {
1923  KMP_TIME_BLOCK(USER_resume);
1924  int status;
1925 #ifdef KMP_DEBUG
1926  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1927  KF_TRACE( 30, ( "__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n",
1928  gtid, KMP_GTID_MONITOR ) );
1929  KMP_DEBUG_ASSERT( gtid != KMP_GTID_MONITOR );
1930 #endif
1931  status = pthread_mutex_lock( &__kmp_wait_mx.m_mutex );
1932  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1933 #ifdef DEBUG_SUSPEND
1934  {
1935  char buffer[128];
1936  __kmp_print_cond( buffer, &__kmp_wait_cv.c_cond );
1937  __kmp_printf( "__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid, KMP_GTID_MONITOR, buffer );
1938  }
1939 #endif
1940  status = pthread_cond_signal( &__kmp_wait_cv.c_cond );
1941  KMP_CHECK_SYSFAIL( "pthread_cond_signal", status );
1942  status = pthread_mutex_unlock( &__kmp_wait_mx.m_mutex );
1943  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1944  KF_TRACE( 30, ( "__kmp_resume_monitor: T#%d exiting after signaling wake up for T#%d\n",
1945  gtid, KMP_GTID_MONITOR ) );
1946 }
1947 
1948 /* ------------------------------------------------------------------------ */
1949 /* ------------------------------------------------------------------------ */
1950 
1951 void
1952 __kmp_yield( int cond )
1953 {
1954  if (cond && __kmp_yielding_on) {
1955  sched_yield();
1956  }
1957 }
1958 
1959 /* ------------------------------------------------------------------------ */
1960 /* ------------------------------------------------------------------------ */
1961 
1962 void
1963 __kmp_gtid_set_specific( int gtid )
1964 {
1965  int status;
1966  KMP_ASSERT( __kmp_init_runtime );
1967  status = pthread_setspecific( __kmp_gtid_threadprivate_key, (void*)(intptr_t)(gtid+1) );
1968  KMP_CHECK_SYSFAIL( "pthread_setspecific", status );
1969 }
1970 
1971 int
1972 __kmp_gtid_get_specific()
1973 {
1974  int gtid;
1975  if ( !__kmp_init_runtime ) {
1976  KA_TRACE( 50, ("__kmp_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
1977  return KMP_GTID_SHUTDOWN;
1978  }
1979  gtid = (int)(size_t)pthread_getspecific( __kmp_gtid_threadprivate_key );
1980  if ( gtid == 0 ) {
1981  gtid = KMP_GTID_DNE;
1982  }
1983  else {
1984  gtid--;
1985  }
1986  KA_TRACE( 50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
1987  __kmp_gtid_threadprivate_key, gtid ));
1988  return gtid;
1989 }
1990 
1991 /* ------------------------------------------------------------------------ */
1992 /* ------------------------------------------------------------------------ */
1993 
1994 double
1995 __kmp_read_cpu_time( void )
1996 {
1997  /*clock_t t;*/
1998  struct tms buffer;
1999 
2000  /*t =*/ times( & buffer );
2001 
2002  return (buffer.tms_utime + buffer.tms_cutime) / (double) CLOCKS_PER_SEC;
2003 }
2004 
2005 int
2006 __kmp_read_system_info( struct kmp_sys_info *info )
2007 {
2008  int status;
2009  struct rusage r_usage;
2010 
2011  memset( info, 0, sizeof( *info ) );
2012 
2013  status = getrusage( RUSAGE_SELF, &r_usage);
2014  KMP_CHECK_SYSFAIL_ERRNO( "getrusage", status );
2015 
2016  info->maxrss = r_usage.ru_maxrss; /* the maximum resident set size utilized (in kilobytes) */
2017  info->minflt = r_usage.ru_minflt; /* the number of page faults serviced without any I/O */
2018  info->majflt = r_usage.ru_majflt; /* the number of page faults serviced that required I/O */
2019  info->nswap = r_usage.ru_nswap; /* the number of times a process was "swapped" out of memory */
2020  info->inblock = r_usage.ru_inblock; /* the number of times the file system had to perform input */
2021  info->oublock = r_usage.ru_oublock; /* the number of times the file system had to perform output */
2022  info->nvcsw = r_usage.ru_nvcsw; /* the number of times a context switch was voluntarily */
2023  info->nivcsw = r_usage.ru_nivcsw; /* the number of times a context switch was forced */
2024 
2025  return (status != 0);
2026 }
2027 
2028 /* ------------------------------------------------------------------------ */
2029 /* ------------------------------------------------------------------------ */
2030 
2031 
2032 void
2033 __kmp_read_system_time( double *delta )
2034 {
2035  double t_ns;
2036  struct timeval tval;
2037  struct timespec stop;
2038  int status;
2039 
2040  status = gettimeofday( &tval, NULL );
2041  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
2042  TIMEVAL_TO_TIMESPEC( &tval, &stop );
2043  t_ns = TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start);
2044  *delta = (t_ns * 1e-9);
2045 }
2046 
2047 void
2048 __kmp_clear_system_time( void )
2049 {
2050  struct timeval tval;
2051  int status;
2052  status = gettimeofday( &tval, NULL );
2053  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
2054  TIMEVAL_TO_TIMESPEC( &tval, &__kmp_sys_timer_data.start );
2055 }
2056 
2057 /* ------------------------------------------------------------------------ */
2058 /* ------------------------------------------------------------------------ */
2059 
2060 #ifdef BUILD_TV
2061 
2062 void
2063 __kmp_tv_threadprivate_store( kmp_info_t *th, void *global_addr, void *thread_addr )
2064 {
2065  struct tv_data *p;
2066 
2067  p = (struct tv_data *) __kmp_allocate( sizeof( *p ) );
2068 
2069  p->u.tp.global_addr = global_addr;
2070  p->u.tp.thread_addr = thread_addr;
2071 
2072  p->type = (void *) 1;
2073 
2074  p->next = th->th.th_local.tv_data;
2075  th->th.th_local.tv_data = p;
2076 
2077  if ( p->next == 0 ) {
2078  int rc = pthread_setspecific( __kmp_tv_key, p );
2079  KMP_CHECK_SYSFAIL( "pthread_setspecific", rc );
2080  }
2081 }
2082 
2083 #endif /* BUILD_TV */
2084 
2085 /* ------------------------------------------------------------------------ */
2086 /* ------------------------------------------------------------------------ */
2087 
2088 static int
2089 __kmp_get_xproc( void ) {
2090 
2091  int r = 0;
2092 
2093  #if KMP_OS_LINUX
2094 
2095  r = sysconf( _SC_NPROCESSORS_ONLN );
2096 
2097  #elif KMP_OS_DARWIN
2098 
2099  // Bug C77011 High "OpenMP Threads and number of active cores".
2100 
2101  // Find the number of available CPUs.
2102  kern_return_t rc;
2103  host_basic_info_data_t info;
2104  mach_msg_type_number_t num = HOST_BASIC_INFO_COUNT;
2105  rc = host_info( mach_host_self(), HOST_BASIC_INFO, (host_info_t) & info, & num );
2106  if ( rc == 0 && num == HOST_BASIC_INFO_COUNT ) {
2107  // Cannot use KA_TRACE() here because this code works before trace support is
2108  // initialized.
2109  r = info.avail_cpus;
2110  } else {
2111  KMP_WARNING( CantGetNumAvailCPU );
2112  KMP_INFORM( AssumedNumCPU );
2113  }; // if
2114 
2115  #elif KMP_OS_FREEBSD
2116 
2117  int mib[] = { CTL_HW, HW_NCPU };
2118  size_t len = sizeof( r );
2119  if ( sysctl( mib, 2, &r, &len, NULL, 0 ) < 0 ) {
2120  r = 0;
2121  KMP_WARNING( CantGetNumAvailCPU );
2122  KMP_INFORM( AssumedNumCPU );
2123  }
2124 
2125  #else
2126 
2127  #error "Unknown or unsupported OS."
2128 
2129  #endif
2130 
2131  return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */
2132 
2133 } // __kmp_get_xproc
2134 
2135 int
2136 __kmp_read_from_file( char const *path, char const *format, ... )
2137 {
2138  int result;
2139  va_list args;
2140 
2141  va_start(args, format);
2142  FILE *f = fopen(path, "rb");
2143  if ( f == NULL )
2144  return 0;
2145  result = vfscanf(f, format, args);
2146  fclose(f);
2147 
2148  return result;
2149 }
2150 
2151 void
2152 __kmp_runtime_initialize( void )
2153 {
2154  int status;
2155  pthread_mutexattr_t mutex_attr;
2156  pthread_condattr_t cond_attr;
2157 
2158  if ( __kmp_init_runtime ) {
2159  return;
2160  }; // if
2161 
2162  #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 )
2163  if ( ! __kmp_cpuinfo.initialized ) {
2164  __kmp_query_cpuid( &__kmp_cpuinfo );
2165  }; // if
2166  #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
2167 
2168  __kmp_xproc = __kmp_get_xproc();
2169 
2170  if ( sysconf( _SC_THREADS ) ) {
2171 
2172  /* Query the maximum number of threads */
2173  __kmp_sys_max_nth = sysconf( _SC_THREAD_THREADS_MAX );
2174  if ( __kmp_sys_max_nth == -1 ) {
2175  /* Unlimited threads for NPTL */
2176  __kmp_sys_max_nth = INT_MAX;
2177  }
2178  else if ( __kmp_sys_max_nth <= 1 ) {
2179  /* Can't tell, just use PTHREAD_THREADS_MAX */
2180  __kmp_sys_max_nth = KMP_MAX_NTH;
2181  }
2182 
2183  /* Query the minimum stack size */
2184  __kmp_sys_min_stksize = sysconf( _SC_THREAD_STACK_MIN );
2185  if ( __kmp_sys_min_stksize <= 1 ) {
2186  __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
2187  }
2188  }
2189 
2190  /* Set up minimum number of threads to switch to TLS gtid */
2191  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
2192 
2193 
2194  #ifdef BUILD_TV
2195  {
2196  int rc = pthread_key_create( & __kmp_tv_key, 0 );
2197  KMP_CHECK_SYSFAIL( "pthread_key_create", rc );
2198  }
2199  #endif
2200 
2201  status = pthread_key_create( &__kmp_gtid_threadprivate_key, __kmp_internal_end_dest );
2202  KMP_CHECK_SYSFAIL( "pthread_key_create", status );
2203  status = pthread_mutexattr_init( & mutex_attr );
2204  KMP_CHECK_SYSFAIL( "pthread_mutexattr_init", status );
2205  status = pthread_mutex_init( & __kmp_wait_mx.m_mutex, & mutex_attr );
2206  KMP_CHECK_SYSFAIL( "pthread_mutex_init", status );
2207  status = pthread_condattr_init( & cond_attr );
2208  KMP_CHECK_SYSFAIL( "pthread_condattr_init", status );
2209  status = pthread_cond_init( & __kmp_wait_cv.c_cond, & cond_attr );
2210  KMP_CHECK_SYSFAIL( "pthread_cond_init", status );
2211 #if USE_ITT_BUILD
2212  __kmp_itt_initialize();
2213 #endif /* USE_ITT_BUILD */
2214 
2215  __kmp_init_runtime = TRUE;
2216 }
2217 
2218 void
2219 __kmp_runtime_destroy( void )
2220 {
2221  int status;
2222 
2223  if ( ! __kmp_init_runtime ) {
2224  return; // Nothing to do.
2225  };
2226 
2227 #if USE_ITT_BUILD
2228  __kmp_itt_destroy();
2229 #endif /* USE_ITT_BUILD */
2230 
2231  status = pthread_key_delete( __kmp_gtid_threadprivate_key );
2232  KMP_CHECK_SYSFAIL( "pthread_key_delete", status );
2233  #ifdef BUILD_TV
2234  status = pthread_key_delete( __kmp_tv_key );
2235  KMP_CHECK_SYSFAIL( "pthread_key_delete", status );
2236  #endif
2237 
2238  status = pthread_mutex_destroy( & __kmp_wait_mx.m_mutex );
2239  if ( status != 0 && status != EBUSY ) {
2240  KMP_SYSFAIL( "pthread_mutex_destroy", status );
2241  }
2242  status = pthread_cond_destroy( & __kmp_wait_cv.c_cond );
2243  if ( status != 0 && status != EBUSY ) {
2244  KMP_SYSFAIL( "pthread_cond_destroy", status );
2245  }
2246  #if KMP_AFFINITY_SUPPORTED
2247  __kmp_affinity_uninitialize();
2248  #endif
2249 
2250  __kmp_init_runtime = FALSE;
2251 }
2252 
2253 
2254 /* Put the thread to sleep for a time period */
2255 /* NOTE: not currently used anywhere */
2256 void
2257 __kmp_thread_sleep( int millis )
2258 {
2259  sleep( ( millis + 500 ) / 1000 );
2260 }
2261 
2262 /* Calculate the elapsed wall clock time for the user */
2263 void
2264 __kmp_elapsed( double *t )
2265 {
2266  int status;
2267 # ifdef FIX_SGI_CLOCK
2268  struct timespec ts;
2269 
2270  status = clock_gettime( CLOCK_PROCESS_CPUTIME_ID, &ts );
2271  KMP_CHECK_SYSFAIL_ERRNO( "clock_gettime", status );
2272  *t = (double) ts.tv_nsec * (1.0 / (double) KMP_NSEC_PER_SEC) +
2273  (double) ts.tv_sec;
2274 # else
2275  struct timeval tv;
2276 
2277  status = gettimeofday( & tv, NULL );
2278  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
2279  *t = (double) tv.tv_usec * (1.0 / (double) KMP_USEC_PER_SEC) +
2280  (double) tv.tv_sec;
2281 # endif
2282 }
2283 
2284 /* Calculate the elapsed wall clock tick for the user */
2285 void
2286 __kmp_elapsed_tick( double *t )
2287 {
2288  *t = 1 / (double) CLOCKS_PER_SEC;
2289 }
2290 
2291 /*
2292  Determine whether the given address is mapped into the current address space.
2293 */
2294 
2295 int
2296 __kmp_is_address_mapped( void * addr ) {
2297 
2298  int found = 0;
2299  int rc;
2300 
2301  #if KMP_OS_LINUX
2302 
2303  /*
2304  On Linux* OS, read the /proc/<pid>/maps pseudo-file to get all the address ranges mapped
2305  into the address space.
2306  */
2307 
2308  char * name = __kmp_str_format( "/proc/%d/maps", getpid() );
2309  FILE * file = NULL;
2310 
2311  file = fopen( name, "r" );
2312  KMP_ASSERT( file != NULL );
2313 
2314  for ( ; ; ) {
2315 
2316  void * beginning = NULL;
2317  void * ending = NULL;
2318  char perms[ 5 ];
2319 
2320  rc = fscanf( file, "%p-%p %4s %*[^\n]\n", & beginning, & ending, perms );
2321  if ( rc == EOF ) {
2322  break;
2323  }; // if
2324  KMP_ASSERT( rc == 3 && KMP_STRLEN( perms ) == 4 ); // Make sure all fields are read.
2325 
2326  // Ending address is not included in the region, but beginning is.
2327  if ( ( addr >= beginning ) && ( addr < ending ) ) {
2328  perms[ 2 ] = 0; // 3th and 4th character does not matter.
2329  if ( strcmp( perms, "rw" ) == 0 ) {
2330  // Memory we are looking for should be readable and writable.
2331  found = 1;
2332  }; // if
2333  break;
2334  }; // if
2335 
2336  }; // forever
2337 
2338  // Free resources.
2339  fclose( file );
2340  KMP_INTERNAL_FREE( name );
2341 
2342  #elif KMP_OS_DARWIN
2343 
2344  /*
2345  On OS X*, /proc pseudo filesystem is not available. Try to read memory using vm
2346  interface.
2347  */
2348 
2349  int buffer;
2350  vm_size_t count;
2351  rc =
2352  vm_read_overwrite(
2353  mach_task_self(), // Task to read memory of.
2354  (vm_address_t)( addr ), // Address to read from.
2355  1, // Number of bytes to be read.
2356  (vm_address_t)( & buffer ), // Address of buffer to save read bytes in.
2357  & count // Address of var to save number of read bytes in.
2358  );
2359  if ( rc == 0 ) {
2360  // Memory successfully read.
2361  found = 1;
2362  }; // if
2363 
2364  #elif KMP_OS_FREEBSD
2365 
2366  // FIXME(FreeBSD*): Implement this
2367  found = 1;
2368 
2369  #else
2370 
2371  #error "Unknown or unsupported OS"
2372 
2373  #endif
2374 
2375  return found;
2376 
2377 } // __kmp_is_address_mapped
2378 
2379 #ifdef USE_LOAD_BALANCE
2380 
2381 
2382 # if KMP_OS_DARWIN
2383 
2384 // The function returns the rounded value of the system load average
2385 // during given time interval which depends on the value of
2386 // __kmp_load_balance_interval variable (default is 60 sec, other values
2387 // may be 300 sec or 900 sec).
2388 // It returns -1 in case of error.
2389 int
2390 __kmp_get_load_balance( int max )
2391 {
2392  double averages[3];
2393  int ret_avg = 0;
2394 
2395  int res = getloadavg( averages, 3 );
2396 
2397  //Check __kmp_load_balance_interval to determine which of averages to use.
2398  // getloadavg() may return the number of samples less than requested that is
2399  // less than 3.
2400  if ( __kmp_load_balance_interval < 180 && ( res >= 1 ) ) {
2401  ret_avg = averages[0];// 1 min
2402  } else if ( ( __kmp_load_balance_interval >= 180
2403  && __kmp_load_balance_interval < 600 ) && ( res >= 2 ) ) {
2404  ret_avg = averages[1];// 5 min
2405  } else if ( ( __kmp_load_balance_interval >= 600 ) && ( res == 3 ) ) {
2406  ret_avg = averages[2];// 15 min
2407  } else {// Error occurred
2408  return -1;
2409  }
2410 
2411  return ret_avg;
2412 }
2413 
2414 # else // Linux* OS
2415 
2416 // The fuction returns number of running (not sleeping) threads, or -1 in case of error.
2417 // Error could be reported if Linux* OS kernel too old (without "/proc" support).
2418 // Counting running threads stops if max running threads encountered.
2419 int
2420 __kmp_get_load_balance( int max )
2421 {
2422  static int permanent_error = 0;
2423 
2424  static int glb_running_threads = 0; /* Saved count of the running threads for the thread balance algortihm */
2425  static double glb_call_time = 0; /* Thread balance algorithm call time */
2426 
2427  int running_threads = 0; // Number of running threads in the system.
2428 
2429  DIR * proc_dir = NULL; // Handle of "/proc/" directory.
2430  struct dirent * proc_entry = NULL;
2431 
2432  kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path.
2433  DIR * task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory.
2434  struct dirent * task_entry = NULL;
2435  int task_path_fixed_len;
2436 
2437  kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path.
2438  int stat_file = -1;
2439  int stat_path_fixed_len;
2440 
2441  int total_processes = 0; // Total number of processes in system.
2442  int total_threads = 0; // Total number of threads in system.
2443 
2444  double call_time = 0.0;
2445 
2446  __kmp_str_buf_init( & task_path );
2447  __kmp_str_buf_init( & stat_path );
2448 
2449  __kmp_elapsed( & call_time );
2450 
2451  if ( glb_call_time &&
2452  ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
2453  running_threads = glb_running_threads;
2454  goto finish;
2455  }
2456 
2457  glb_call_time = call_time;
2458 
2459  // Do not spend time on scanning "/proc/" if we have a permanent error.
2460  if ( permanent_error ) {
2461  running_threads = -1;
2462  goto finish;
2463  }; // if
2464 
2465  if ( max <= 0 ) {
2466  max = INT_MAX;
2467  }; // if
2468 
2469  // Open "/proc/" directory.
2470  proc_dir = opendir( "/proc" );
2471  if ( proc_dir == NULL ) {
2472  // Cannot open "/prroc/". Probably the kernel does not support it. Return an error now and
2473  // in subsequent calls.
2474  running_threads = -1;
2475  permanent_error = 1;
2476  goto finish;
2477  }; // if
2478 
2479  // Initialize fixed part of task_path. This part will not change.
2480  __kmp_str_buf_cat( & task_path, "/proc/", 6 );
2481  task_path_fixed_len = task_path.used; // Remember number of used characters.
2482 
2483  proc_entry = readdir( proc_dir );
2484  while ( proc_entry != NULL ) {
2485  // Proc entry is a directory and name starts with a digit. Assume it is a process'
2486  // directory.
2487  if ( proc_entry->d_type == DT_DIR && isdigit( proc_entry->d_name[ 0 ] ) ) {
2488 
2489  ++ total_processes;
2490  // Make sure init process is the very first in "/proc", so we can replace
2491  // strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes == 1.
2492  // We are going to check that total_processes == 1 => d_name == "1" is true (where
2493  // "=>" is implication). Since C++ does not have => operator, let us replace it with its
2494  // equivalent: a => b == ! a || b.
2495  KMP_DEBUG_ASSERT( total_processes != 1 || strcmp( proc_entry->d_name, "1" ) == 0 );
2496 
2497  // Construct task_path.
2498  task_path.used = task_path_fixed_len; // Reset task_path to "/proc/".
2499  __kmp_str_buf_cat( & task_path, proc_entry->d_name, KMP_STRLEN( proc_entry->d_name ) );
2500  __kmp_str_buf_cat( & task_path, "/task", 5 );
2501 
2502  task_dir = opendir( task_path.str );
2503  if ( task_dir == NULL ) {
2504  // Process can finish between reading "/proc/" directory entry and opening process'
2505  // "task/" directory. So, in general case we should not complain, but have to skip
2506  // this process and read the next one.
2507  // But on systems with no "task/" support we will spend lot of time to scan "/proc/"
2508  // tree again and again without any benefit. "init" process (its pid is 1) should
2509  // exist always, so, if we cannot open "/proc/1/task/" directory, it means "task/"
2510  // is not supported by kernel. Report an error now and in the future.
2511  if ( strcmp( proc_entry->d_name, "1" ) == 0 ) {
2512  running_threads = -1;
2513  permanent_error = 1;
2514  goto finish;
2515  }; // if
2516  } else {
2517  // Construct fixed part of stat file path.
2518  __kmp_str_buf_clear( & stat_path );
2519  __kmp_str_buf_cat( & stat_path, task_path.str, task_path.used );
2520  __kmp_str_buf_cat( & stat_path, "/", 1 );
2521  stat_path_fixed_len = stat_path.used;
2522 
2523  task_entry = readdir( task_dir );
2524  while ( task_entry != NULL ) {
2525  // It is a directory and name starts with a digit.
2526  if ( proc_entry->d_type == DT_DIR && isdigit( task_entry->d_name[ 0 ] ) ) {
2527 
2528  ++ total_threads;
2529 
2530  // Consruct complete stat file path. Easiest way would be:
2531  // __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str, task_entry->d_name );
2532  // but seriae of __kmp_str_buf_cat works a bit faster.
2533  stat_path.used = stat_path_fixed_len; // Reset stat path to its fixed part.
2534  __kmp_str_buf_cat( & stat_path, task_entry->d_name, KMP_STRLEN( task_entry->d_name ) );
2535  __kmp_str_buf_cat( & stat_path, "/stat", 5 );
2536 
2537  // Note: Low-level API (open/read/close) is used. High-level API
2538  // (fopen/fclose) works ~ 30 % slower.
2539  stat_file = open( stat_path.str, O_RDONLY );
2540  if ( stat_file == -1 ) {
2541  // We cannot report an error because task (thread) can terminate just
2542  // before reading this file.
2543  } else {
2544  /*
2545  Content of "stat" file looks like:
2546 
2547  24285 (program) S ...
2548 
2549  It is a single line (if program name does not include fanny
2550  symbols). First number is a thread id, then name of executable file
2551  name in paretheses, then state of the thread. We need just thread
2552  state.
2553 
2554  Good news: Length of program name is 15 characters max. Longer
2555  names are truncated.
2556 
2557  Thus, we need rather short buffer: 15 chars for program name +
2558  2 parenthesis, + 3 spaces + ~7 digits of pid = 37.
2559 
2560  Bad news: Program name may contain special symbols like space,
2561  closing parenthesis, or even new line. This makes parsing "stat"
2562  file not 100 % reliable. In case of fanny program names parsing
2563  may fail (report incorrect thread state).
2564 
2565  Parsing "status" file looks more promissing (due to different
2566  file structure and escaping special symbols) but reading and
2567  parsing of "status" file works slower.
2568 
2569  -- ln
2570  */
2571  char buffer[ 65 ];
2572  int len;
2573  len = read( stat_file, buffer, sizeof( buffer ) - 1 );
2574  if ( len >= 0 ) {
2575  buffer[ len ] = 0;
2576  // Using scanf:
2577  // sscanf( buffer, "%*d (%*s) %c ", & state );
2578  // looks very nice, but searching for a closing parenthesis works a
2579  // bit faster.
2580  char * close_parent = strstr( buffer, ") " );
2581  if ( close_parent != NULL ) {
2582  char state = * ( close_parent + 2 );
2583  if ( state == 'R' ) {
2584  ++ running_threads;
2585  if ( running_threads >= max ) {
2586  goto finish;
2587  }; // if
2588  }; // if
2589  }; // if
2590  }; // if
2591  close( stat_file );
2592  stat_file = -1;
2593  }; // if
2594  }; // if
2595  task_entry = readdir( task_dir );
2596  }; // while
2597  closedir( task_dir );
2598  task_dir = NULL;
2599  }; // if
2600  }; // if
2601  proc_entry = readdir( proc_dir );
2602  }; // while
2603 
2604  //
2605  // There _might_ be a timing hole where the thread executing this
2606  // code get skipped in the load balance, and running_threads is 0.
2607  // Assert in the debug builds only!!!
2608  //
2609  KMP_DEBUG_ASSERT( running_threads > 0 );
2610  if ( running_threads <= 0 ) {
2611  running_threads = 1;
2612  }
2613 
2614  finish: // Clean up and exit.
2615  if ( proc_dir != NULL ) {
2616  closedir( proc_dir );
2617  }; // if
2618  __kmp_str_buf_free( & task_path );
2619  if ( task_dir != NULL ) {
2620  closedir( task_dir );
2621  }; // if
2622  __kmp_str_buf_free( & stat_path );
2623  if ( stat_file != -1 ) {
2624  close( stat_file );
2625  }; // if
2626 
2627  glb_running_threads = running_threads;
2628 
2629  return running_threads;
2630 
2631 } // __kmp_get_load_balance
2632 
2633 # endif // KMP_OS_DARWIN
2634 
2635 #endif // USE_LOAD_BALANCE
2636 
2637 
2638 #if KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64)
2639 
2640 int __kmp_invoke_microtask( microtask_t pkfn, int gtid, int tid, int argc,
2641  void *p_argv[] )
2642 {
2643  int argc_full = argc + 2;
2644  int i;
2645  ffi_cif cif;
2646  ffi_type *types[argc_full];
2647  void *args[argc_full];
2648  void *idp[2];
2649 
2650  /* We're only passing pointers to the target. */
2651  for (i = 0; i < argc_full; i++)
2652  types[i] = &ffi_type_pointer;
2653 
2654  /* Ugly double-indirection, but that's how it goes... */
2655  idp[0] = &gtid;
2656  idp[1] = &tid;
2657  args[0] = &idp[0];
2658  args[1] = &idp[1];
2659 
2660  for (i = 0; i < argc; i++)
2661  args[2 + i] = &p_argv[i];
2662 
2663  if (ffi_prep_cif(&cif, FFI_DEFAULT_ABI, argc_full,
2664  &ffi_type_void, types) != FFI_OK)
2665  abort();
2666 
2667  ffi_call(&cif, (void (*)(void))pkfn, NULL, args);
2668 
2669  return 1;
2670 }
2671 
2672 #endif // KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64)
2673 
2674 #if KMP_ARCH_PPC64 || KMP_ARCH_AARCH64
2675 
2676 // we really only need the case with 1 argument, because CLANG always build
2677 // a struct of pointers to shared variables referenced in the outlined function
2678 int
2679 __kmp_invoke_microtask( microtask_t pkfn,
2680  int gtid, int tid,
2681  int argc, void *p_argv[] ) {
2682  switch (argc) {
2683  default:
2684  fprintf(stderr, "Too many args to microtask: %d!\n", argc);
2685  fflush(stderr);
2686  exit(-1);
2687  case 0:
2688  (*pkfn)(&gtid, &tid);
2689  break;
2690  case 1:
2691  (*pkfn)(&gtid, &tid, p_argv[0]);
2692  break;
2693  case 2:
2694  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
2695  break;
2696  case 3:
2697  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
2698  break;
2699  case 4:
2700  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3]);
2701  break;
2702  case 5:
2703  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4]);
2704  break;
2705  case 6:
2706  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2707  p_argv[5]);
2708  break;
2709  case 7:
2710  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2711  p_argv[5], p_argv[6]);
2712  break;
2713  case 8:
2714  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2715  p_argv[5], p_argv[6], p_argv[7]);
2716  break;
2717  case 9:
2718  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2719  p_argv[5], p_argv[6], p_argv[7], p_argv[8]);
2720  break;
2721  case 10:
2722  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2723  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9]);
2724  break;
2725  case 11:
2726  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2727  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2728  break;
2729  case 12:
2730  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2731  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2732  p_argv[11]);
2733  break;
2734  case 13:
2735  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2736  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2737  p_argv[11], p_argv[12]);
2738  break;
2739  case 14:
2740  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2741  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2742  p_argv[11], p_argv[12], p_argv[13]);
2743  break;
2744  case 15:
2745  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2746  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2747  p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2748  break;
2749  }
2750 
2751  return 1;
2752 }
2753 
2754 #endif
2755 
2756 // end of file //
2757 
#define KMP_START_EXPLICIT_TIMER(name)
"Starts" an explicit timer which will need a corresponding KMP_STOP_EXPLICIT_TIMER() macro...
Definition: kmp_stats.h:668
#define KMP_TIME_BLOCK(name)
Uses specified timer (name) to time code block.
Definition: kmp_stats.h:629