54 enum SYSTEM_INFORMATION_CLASS {
55 SystemProcessInformation = 5
75 SIZE_T PeakVirtualSize;
78 SIZE_T PeakWorkingSetSize;
79 SIZE_T WorkingSetSize;
80 SIZE_T QuotaPeakPagedPoolUsage;
81 SIZE_T QuotaPagedPoolUsage;
82 SIZE_T QuotaPeakNonPagedPoolUsage;
83 SIZE_T QuotaNonPagedPoolUsage;
85 SIZE_T PeakPagefileUsage;
86 SIZE_T PrivatePageCount;
89 struct SYSTEM_THREAD {
90 LARGE_INTEGER KernelTime;
91 LARGE_INTEGER UserTime;
92 LARGE_INTEGER CreateTime;
98 ULONG ContextSwitchCount;
103 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, KernelTime ) == 0 );
105 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 28 );
106 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 52 );
108 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 32 );
109 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 68 );
112 struct SYSTEM_PROCESS_INFORMATION {
113 ULONG NextEntryOffset;
114 ULONG NumberOfThreads;
115 LARGE_INTEGER Reserved[ 3 ];
116 LARGE_INTEGER CreateTime;
117 LARGE_INTEGER UserTime;
118 LARGE_INTEGER KernelTime;
119 UNICODE_STRING ImageName;
122 HANDLE ParentProcessId;
124 ULONG Reserved2[ 2 ];
125 VM_COUNTERS VMCounters;
126 IO_COUNTERS IOCounters;
127 SYSTEM_THREAD Threads[ 1 ];
129 typedef SYSTEM_PROCESS_INFORMATION * PSYSTEM_PROCESS_INFORMATION;
131 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, NextEntryOffset ) == 0 );
132 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, CreateTime ) == 32 );
133 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ImageName ) == 56 );
135 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 68 );
136 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 76 );
137 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 88 );
138 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 136 );
139 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 184 );
141 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 80 );
142 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 96 );
143 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 112 );
144 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 208 );
145 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 256 );
148 typedef NTSTATUS (NTAPI *NtQuerySystemInformation_t)( SYSTEM_INFORMATION_CLASS, PVOID, ULONG, PULONG );
149 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
151 HMODULE ntdll = NULL;
156 static HMODULE kernel32 = NULL;
167 __kmp_static_delay(
int arg ) {
169 #if KMP_ARCH_X86_64 && KMP_OS_LINUX
170 KMP_ASSERT( arg != 0 );
172 KMP_ASSERT( arg >= 0 );
178 #define __kmp_static_delay( arg )
183 __kmp_static_yield(
int arg )
188 #if KMP_HANDLE_SIGNALS
189 typedef void (* sig_func_t )( int );
190 static sig_func_t __kmp_sighldrs[ NSIG ];
191 static int __kmp_siginstalled[ NSIG ];
194 static HANDLE __kmp_monitor_ev;
195 static kmp_int64 __kmp_win32_time;
196 double __kmp_win32_tick;
198 int __kmp_init_runtime = FALSE;
199 CRITICAL_SECTION __kmp_win32_section;
202 __kmp_win32_mutex_init( kmp_win32_mutex_t *mx )
204 InitializeCriticalSection( & mx->cs );
206 __kmp_itt_system_object_created( & mx->cs,
"Critical Section" );
211 __kmp_win32_mutex_destroy( kmp_win32_mutex_t *mx )
213 DeleteCriticalSection( & mx->cs );
217 __kmp_win32_mutex_lock( kmp_win32_mutex_t *mx )
219 EnterCriticalSection( & mx->cs );
223 __kmp_win32_mutex_unlock( kmp_win32_mutex_t *mx )
225 LeaveCriticalSection( & mx->cs );
229 __kmp_win32_cond_init( kmp_win32_cond_t *cv )
231 cv->waiters_count_ = 0;
232 cv->wait_generation_count_ = 0;
233 cv->release_count_ = 0;
236 __kmp_win32_mutex_init( & cv->waiters_count_lock_ );
239 cv->event_ = CreateEvent( NULL,
244 __kmp_itt_system_object_created( cv->event_,
"Event" );
249 __kmp_win32_cond_destroy( kmp_win32_cond_t *cv )
251 __kmp_win32_mutex_destroy( & cv->waiters_count_lock_ );
252 __kmp_free_handle( cv->event_ );
253 memset( cv,
'\0',
sizeof( *cv ) );
260 __kmp_win32_cond_wait( kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx, kmp_info_t *th,
int need_decrease_load )
266 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
269 cv->waiters_count_++;
272 my_generation = cv->wait_generation_count_;
274 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
275 __kmp_win32_mutex_unlock( mx );
282 WaitForSingleObject( cv->event_, INFINITE );
284 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
289 wait_done = ( cv->release_count_ > 0 ) &&
290 ( cv->wait_generation_count_ != my_generation );
292 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_);
300 __kmp_win32_mutex_lock( mx );
301 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
303 cv->waiters_count_--;
304 cv->release_count_--;
306 last_waiter = ( cv->release_count_ == 0 );
308 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
312 ResetEvent( cv->event_ );
317 __kmp_win32_cond_broadcast( kmp_win32_cond_t *cv )
319 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
321 if( cv->waiters_count_ > 0 ) {
322 SetEvent( cv->event_ );
325 cv->release_count_ = cv->waiters_count_;
328 cv->wait_generation_count_++;
331 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
335 __kmp_win32_cond_signal( kmp_win32_cond_t *cv )
337 __kmp_win32_cond_broadcast( cv );
344 __kmp_enable(
int new_state )
346 if (__kmp_init_runtime)
347 LeaveCriticalSection( & __kmp_win32_section );
351 __kmp_disable(
int *old_state )
355 if (__kmp_init_runtime)
356 EnterCriticalSection( & __kmp_win32_section );
360 __kmp_suspend_initialize(
void )
366 __kmp_suspend_initialize_thread( kmp_info_t *th )
368 if ( ! TCR_4( th->th.th_suspend_init ) ) {
371 __kmp_win32_cond_init( &th->th.th_suspend_cv );
372 __kmp_win32_mutex_init( &th->th.th_suspend_mx );
373 TCW_4( th->th.th_suspend_init, TRUE );
378 __kmp_suspend_uninitialize_thread( kmp_info_t *th )
380 if ( TCR_4( th->th.th_suspend_init ) ) {
383 __kmp_win32_cond_destroy( & th->th.th_suspend_cv );
384 __kmp_win32_mutex_destroy( & th->th.th_suspend_mx );
385 TCW_4( th->th.th_suspend_init, FALSE );
395 __kmp_suspend(
int th_gtid,
volatile kmp_uint *spinner, kmp_uint checker )
397 kmp_info_t *th = __kmp_threads[th_gtid];
401 KF_TRACE( 30, (
"__kmp_suspend: T#%d enter for spin = %p\n", th_gtid, spinner ) );
403 __kmp_suspend_initialize_thread( th );
405 __kmp_win32_mutex_lock( &th->th.th_suspend_mx );
407 KF_TRACE( 10, (
"__kmp_suspend: T#%d setting sleep bit for spin(%p)\n",
408 th_gtid, spinner ) );
413 old_spin = KMP_TEST_THEN_OR32( (
volatile kmp_int32 *) spinner,
414 KMP_BARRIER_SLEEP_STATE );
416 KF_TRACE( 5, (
"__kmp_suspend: T#%d set sleep bit for spin(%p)==%d\n",
417 th_gtid, spinner, *spinner ) );
419 if ( old_spin == checker ) {
420 KMP_TEST_THEN_AND32( (
volatile kmp_int32 *) spinner, ~(KMP_BARRIER_SLEEP_STATE) );
422 KF_TRACE( 5, (
"__kmp_suspend: T#%d false alarm, reset sleep bit for spin(%p)\n",
426 __kmp_suspend_count++;
433 int deactivated = FALSE;
434 TCW_PTR(th->th.th_sleep_loc, spinner);
435 while ( TCR_4( *spinner ) & KMP_BARRIER_SLEEP_STATE ) {
437 KF_TRACE( 15, (
"__kmp_suspend: T#%d about to perform kmp_win32_cond_wait()\n",
445 if ( ! deactivated ) {
446 th->th.th_active = FALSE;
447 if ( th->th.th_active_in_pool ) {
448 th->th.th_active_in_pool = FALSE;
450 (kmp_int32 *) &__kmp_thread_pool_active_nth );
451 KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
456 __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
459 __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
463 if( (*spinner) & KMP_BARRIER_SLEEP_STATE ) {
464 KF_TRACE( 100, (
"__kmp_suspend: T#%d spurious wakeup\n", th_gtid ));
475 th->th.th_active = TRUE;
476 if ( TCR_4(th->th.th_in_pool) ) {
478 (kmp_int32 *) &__kmp_thread_pool_active_nth );
479 th->th.th_active_in_pool = TRUE;
485 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
487 KF_TRACE( 30, (
"__kmp_suspend: T#%d exit\n", th_gtid ) );
494 __kmp_resume(
int target_gtid,
volatile kmp_uint *spin )
496 kmp_info_t *th = __kmp_threads[target_gtid];
501 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
504 KF_TRACE( 30, (
"__kmp_resume: T#%d wants to wakeup T#%d enter\n",
505 gtid, target_gtid ) );
507 __kmp_suspend_initialize_thread( th );
509 __kmp_win32_mutex_lock( &th->th.th_suspend_mx );
511 if ( spin == NULL ) {
512 spin = (
volatile kmp_uint *)TCR_PTR(th->th.th_sleep_loc);
513 if ( spin == NULL ) {
514 KF_TRACE( 5, (
"__kmp_resume: T#%d exiting, thread T#%d already awake - spin(%p)\n",
515 gtid, target_gtid, spin ) );
517 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
522 TCW_PTR(th->th.th_sleep_loc, NULL);
523 old_spin = KMP_TEST_THEN_AND32( (kmp_int32
volatile *) spin, ~( KMP_BARRIER_SLEEP_STATE ) );
525 if ( ( old_spin & KMP_BARRIER_SLEEP_STATE ) == 0 ) {
526 KF_TRACE( 5, (
"__kmp_resume: T#%d exiting, thread T#%d already awake - spin(%p): "
528 gtid, target_gtid, spin, old_spin, *spin ) );
530 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
533 TCW_PTR(th->th.th_sleep_loc, NULL);
535 KF_TRACE( 5, (
"__kmp_resume: T#%d about to wakeup T#%d, reset sleep bit for spin(%p)\n",
536 gtid, target_gtid, spin) );
539 __kmp_win32_cond_signal( &th->th.th_suspend_cv );
541 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
543 KF_TRACE( 30, (
"__kmp_resume: T#%d exiting after signaling wake up for T#%d\n",
544 gtid, target_gtid ) );
551 __kmp_yield(
int cond )
561 __kmp_gtid_set_specific(
int gtid )
563 KA_TRACE( 50, (
"__kmp_gtid_set_specific: T#%d key:%d\n",
564 gtid, __kmp_gtid_threadprivate_key ));
565 KMP_ASSERT( __kmp_init_runtime );
566 if( ! TlsSetValue( __kmp_gtid_threadprivate_key, (LPVOID)(gtid+1)) )
567 KMP_FATAL( TLSSetValueFailed );
571 __kmp_gtid_get_specific()
574 if( !__kmp_init_runtime ) {
575 KA_TRACE( 50, (
"__kmp_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
576 return KMP_GTID_SHUTDOWN;
578 gtid = (int)(kmp_intptr_t)TlsGetValue( __kmp_gtid_threadprivate_key );
585 KA_TRACE( 50, (
"__kmp_gtid_get_specific: key:%d gtid:%d\n",
586 __kmp_gtid_threadprivate_key, gtid ));
600 __kmp_get_proc_group( kmp_affin_mask_t
const *mask )
604 struct GROUP_AFFINITY new_ga, prev_ga;
605 for (i = 0; i < __kmp_num_proc_groups; i++) {
620 __kmp_set_system_affinity( kmp_affin_mask_t
const *mask,
int abort_on_error )
625 if (__kmp_num_proc_groups > 1) {
629 struct GROUP_AFFINITY ga;
630 int group = __kmp_get_proc_group( mask );
632 if (abort_on_error) {
633 KMP_FATAL(AffinityInvalidMask,
"kmp_set_affinity");
643 ga.mask = mask[group];
644 ga.reserved[0] = ga.reserved[1] = ga.reserved[2] = 0;
646 KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
647 if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
648 DWORD error = GetLastError();
649 if (abort_on_error) {
652 KMP_MSG( CantSetThreadAffMask ),
665 if (!SetThreadAffinityMask( GetCurrentThread(), *mask )) {
666 DWORD error = GetLastError();
667 if (abort_on_error) {
670 KMP_MSG( CantSetThreadAffMask ),
682 __kmp_get_system_affinity( kmp_affin_mask_t *mask,
int abort_on_error )
687 if (__kmp_num_proc_groups > 1) {
689 struct GROUP_AFFINITY ga;
690 KMP_DEBUG_ASSERT(__kmp_GetThreadGroupAffinity != NULL);
692 if (__kmp_GetThreadGroupAffinity(GetCurrentThread(), &ga) == 0) {
693 DWORD error = GetLastError();
694 if (abort_on_error) {
697 KMP_MSG(FunctionError,
"GetThreadGroupAffinity()"),
705 if ((ga.group < 0) || (ga.group > __kmp_num_proc_groups)
710 mask[ga.group] = ga.mask;
717 kmp_affin_mask_t newMask, sysMask, retval;
719 if (!GetProcessAffinityMask(GetCurrentProcess(), &newMask, &sysMask)) {
720 DWORD error = GetLastError();
721 if (abort_on_error) {
724 KMP_MSG(FunctionError,
"GetProcessAffinityMask()"),
731 retval = SetThreadAffinityMask(GetCurrentThread(), newMask);
733 DWORD error = GetLastError();
734 if (abort_on_error) {
737 KMP_MSG(FunctionError,
"SetThreadAffinityMask()"),
744 newMask = SetThreadAffinityMask(GetCurrentThread(), retval);
746 DWORD error = GetLastError();
747 if (abort_on_error) {
750 KMP_MSG(FunctionError,
"SetThreadAffinityMask()"),
762 __kmp_affinity_bind_thread(
int proc )
767 if (__kmp_num_proc_groups > 1) {
772 struct GROUP_AFFINITY ga;
773 KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups
774 * CHAR_BIT *
sizeof(DWORD_PTR))));
775 ga.group = proc / (CHAR_BIT *
sizeof(DWORD_PTR));
776 ga.mask = 1 << (proc % (CHAR_BIT *
sizeof(DWORD_PTR)));
777 ga.reserved[0] = ga.reserved[1] = ga.reserved[2] = 0;
779 KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
780 if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
781 DWORD error = GetLastError();
782 if (__kmp_affinity_verbose) {
785 KMP_MSG( CantSetThreadAffMask ),
797 kmp_affin_mask_t mask;
799 KMP_CPU_SET(proc, &mask);
800 __kmp_set_system_affinity(&mask, TRUE);
805 __kmp_affinity_determine_capable(
const char *env_var )
812 __kmp_affin_mask_size = __kmp_num_proc_groups *
sizeof(kmp_affin_mask_t);
814 __kmp_affin_mask_size =
sizeof(kmp_affin_mask_t);
818 "__kmp_affinity_determine_capable: "
819 "Windows* OS affinity interface functional (mask size = %" KMP_SIZE_T_SPEC
").\n",
820 __kmp_affin_mask_size
825 __kmp_read_cpu_time(
void )
827 FILETIME CreationTime, ExitTime, KernelTime, UserTime;
833 status = GetProcessTimes( GetCurrentProcess(), &CreationTime,
834 &ExitTime, &KernelTime, &UserTime );
839 sec += KernelTime.dwHighDateTime;
840 sec += UserTime.dwHighDateTime;
843 sec *= (double) (1 << 16) * (double) (1 << 16);
845 sec += KernelTime.dwLowDateTime;
846 sec += UserTime.dwLowDateTime;
848 cpu_time += (sec * 100.0) / NSEC_PER_SEC;
855 __kmp_read_system_info(
struct kmp_sys_info *info )
874 __kmp_runtime_initialize(
void )
880 if ( __kmp_init_runtime ) {
884 InitializeCriticalSection( & __kmp_win32_section );
886 __kmp_itt_system_object_created( & __kmp_win32_section,
"Critical Section" );
888 __kmp_initialize_system_tick();
890 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
891 if ( ! __kmp_cpuinfo.initialized ) {
892 __kmp_query_cpuid( & __kmp_cpuinfo );
897 #if KMP_OS_WINDOWS && ! defined GUIDEDLL_EXPORTS
912 __kmp_tls_gtid_min = 0;
914 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
918 if ( !__kmp_gtid_threadprivate_key ) {
919 __kmp_gtid_threadprivate_key = TlsAlloc();
920 if( __kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES ) {
921 KMP_FATAL( TLSOutOfIndexes );
936 __kmp_str_buf_init( & path );
937 path_size = GetSystemDirectory( path.str, path.size );
938 KMP_DEBUG_ASSERT( path_size > 0 );
939 if ( path_size >= path.size ) {
943 __kmp_str_buf_reserve( & path, path_size );
944 path_size = GetSystemDirectory( path.str, path.size );
945 KMP_DEBUG_ASSERT( path_size > 0 );
947 if ( path_size > 0 && path_size < path.size ) {
952 path.used = path_size;
953 __kmp_str_buf_print( & path,
"\\%s",
"ntdll.dll" );
958 ntdll = GetModuleHandle( path.str );
961 KMP_DEBUG_ASSERT( ntdll != NULL );
962 if ( ntdll != NULL ) {
963 NtQuerySystemInformation = (NtQuerySystemInformation_t) GetProcAddress( ntdll,
"NtQuerySystemInformation" );
965 KMP_DEBUG_ASSERT( NtQuerySystemInformation != NULL );
972 if ( path_size > 0 && path_size < path.size ) {
977 path.used = path_size;
978 __kmp_str_buf_print( & path,
"\\%s",
"kernel32.dll" );
983 kernel32 = GetModuleHandle( path.str );
989 if ( kernel32 != NULL ) {
990 __kmp_GetActiveProcessorCount = (kmp_GetActiveProcessorCount_t) GetProcAddress( kernel32,
"GetActiveProcessorCount" );
991 __kmp_GetActiveProcessorGroupCount = (kmp_GetActiveProcessorGroupCount_t) GetProcAddress( kernel32,
"GetActiveProcessorGroupCount" );
992 __kmp_GetThreadGroupAffinity = (kmp_GetThreadGroupAffinity_t) GetProcAddress( kernel32,
"GetThreadGroupAffinity" );
993 __kmp_SetThreadGroupAffinity = (kmp_SetThreadGroupAffinity_t) GetProcAddress( kernel32,
"SetThreadGroupAffinity" );
1002 if ( ( __kmp_GetActiveProcessorCount != NULL )
1003 && ( __kmp_GetActiveProcessorGroupCount != NULL )
1004 && ( __kmp_GetThreadGroupAffinity != NULL )
1005 && ( __kmp_SetThreadGroupAffinity != NULL )
1006 && ( ( __kmp_num_proc_groups
1007 = __kmp_GetActiveProcessorGroupCount() ) > 1 ) ) {
1013 KA_TRACE( 10, (
"__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) );
1017 for ( i = 0; i < __kmp_num_proc_groups; i++ ) {
1018 DWORD size = __kmp_GetActiveProcessorCount( i );
1019 __kmp_xproc += size;
1020 KA_TRACE( 20, (
"__kmp_runtime_initialize: proc group %d size = %d\n", i, size ) );
1025 if ( __kmp_num_proc_groups <= 1 ) {
1026 GetSystemInfo( & info );
1027 __kmp_xproc = info.dwNumberOfProcessors;
1030 GetSystemInfo( & info );
1031 __kmp_xproc = info.dwNumberOfProcessors;
1032 #endif // KMP_ARCH_X86_64
1038 if ( __kmp_xproc <= 0 ) {
1042 KA_TRACE( 5, (
"__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc) );
1044 __kmp_str_buf_free( & path );
1047 __kmp_itt_initialize();
1050 __kmp_init_runtime = TRUE;
1054 __kmp_runtime_destroy(
void )
1056 if ( ! __kmp_init_runtime ) {
1061 __kmp_itt_destroy();
1066 KA_TRACE( 40, (
"__kmp_runtime_destroy\n" ));
1068 if( __kmp_gtid_threadprivate_key ) {
1069 TlsFree( __kmp_gtid_threadprivate_key );
1070 __kmp_gtid_threadprivate_key = 0;
1073 __kmp_affinity_uninitialize();
1074 DeleteCriticalSection( & __kmp_win32_section );
1077 NtQuerySystemInformation = NULL;
1081 __kmp_GetActiveProcessorCount = NULL;
1082 __kmp_GetActiveProcessorGroupCount = NULL;
1083 __kmp_GetThreadGroupAffinity = NULL;
1084 __kmp_SetThreadGroupAffinity = NULL;
1085 #endif // KMP_ARCH_X86_64
1087 __kmp_init_runtime = FALSE;
1092 __kmp_terminate_thread(
int gtid )
1094 kmp_info_t *th = __kmp_threads[ gtid ];
1098 KA_TRACE( 10, (
"__kmp_terminate_thread: kill (%d)\n", gtid ) );
1100 if (TerminateThread( th->th.th_info.ds.ds_thread, (DWORD) -1) == FALSE) {
1103 __kmp_free_handle( th->th.th_info.ds.ds_thread );
1110 __kmp_clear_system_time(
void )
1114 status = QueryPerformanceCounter( & time );
1115 __kmp_win32_time = (kmp_int64) time.QuadPart;
1119 __kmp_initialize_system_tick(
void )
1125 status = QueryPerformanceFrequency( & freq );
1127 DWORD error = GetLastError();
1130 KMP_MSG( FunctionError,
"QueryPerformanceFrequency()" ),
1137 __kmp_win32_tick = ((double) 1.0) / (double) freq.QuadPart;
1145 __kmp_elapsed(
double *t )
1149 status = QueryPerformanceCounter( & now );
1150 *t = ((double) now.QuadPart) * __kmp_win32_tick;
1156 __kmp_elapsed_tick(
double *t )
1158 *t = __kmp_win32_tick;
1162 __kmp_read_system_time(
double *delta )
1165 if (delta != NULL) {
1169 status = QueryPerformanceCounter( & now );
1171 *delta = ((double) (((kmp_int64) now.QuadPart) - __kmp_win32_time))
1186 __kmp_change_thread_affinity_mask(
int gtid, kmp_affin_mask_t *new_mask,
1187 kmp_affin_mask_t *old_mask )
1189 kmp_info_t *th = __kmp_threads[ gtid ];
1191 KMP_DEBUG_ASSERT( *new_mask != 0 );
1193 if ( old_mask != NULL ) {
1194 *old_mask = SetThreadAffinityMask( th -> th.th_info.ds.ds_thread, *new_mask );
1197 DWORD error = GetLastError();
1200 KMP_MSG( CantSetThreadAffMask ),
1206 if (__kmp_affinity_verbose)
1207 KMP_INFORM( ChangeAffMask,
"KMP_AFFINITY (Bind)", gtid, *old_mask, *new_mask );
1210 KMP_DEBUG_ASSERT( old_mask != NULL && *old_mask == *(th -> th.th_affin_mask ));
1212 KMP_CPU_COPY(th -> th.th_affin_mask, new_mask);
1220 __kmp_launch_worker(
void *arg )
1222 volatile void *stack_data;
1225 kmp_info_t *this_thr = (kmp_info_t *) arg;
1228 gtid = this_thr->th.th_info.ds.ds_gtid;
1229 __kmp_gtid_set_specific( gtid );
1230 #ifdef KMP_TDATA_GTID
1231 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
1232 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
1233 "reference: http://support.microsoft.com/kb/118816"
1238 __kmp_itt_thread_name( gtid );
1241 __kmp_affinity_set_init_mask( gtid, FALSE );
1243 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1248 __kmp_clear_x87_fpu_status_word();
1249 __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
1250 __kmp_load_mxcsr( &__kmp_init_mxcsr );
1253 if ( __kmp_stkoffset > 0 && gtid > 0 ) {
1254 padding = _alloca( gtid * __kmp_stkoffset );
1257 KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
1258 this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1259 TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );
1261 if ( TCR_4(__kmp_gtid_mode) < 2 ) {
1262 TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
1263 KMP_ASSERT( this_thr -> th.th_info.ds.ds_stackgrow == FALSE );
1264 __kmp_check_stack_overlap( this_thr );
1267 exit_val = __kmp_launch_thread( this_thr );
1268 KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
1269 TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );
1278 __kmp_launch_monitor(
void *arg )
1281 kmp_thread_t monitor;
1284 kmp_info_t *this_thr = (kmp_info_t *) arg;
1286 KMP_DEBUG_ASSERT(__kmp_init_monitor);
1287 TCW_4( __kmp_init_monitor, 2 );
1289 this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1290 TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );
1293 KA_TRACE( 10, (
"__kmp_launch_monitor: launched\n" ) );
1295 monitor = GetCurrentThread();
1298 status = SetThreadPriority( monitor, THREAD_PRIORITY_HIGHEST );
1300 DWORD error = GetLastError();
1303 KMP_MSG( CantSetThreadPriority ),
1310 __kmp_gtid_set_specific( KMP_GTID_MONITOR );
1311 #ifdef KMP_TDATA_GTID
1312 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
1313 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
1314 "reference: http://support.microsoft.com/kb/118816"
1319 __kmp_itt_thread_ignore();
1324 interval = ( 1000 / __kmp_monitor_wakeups );
1326 while (! TCR_4(__kmp_global.g.g_done)) {
1329 KA_TRACE( 15, (
"__kmp_launch_monitor: update\n" ) );
1331 wait_status = WaitForSingleObject( __kmp_monitor_ev, interval );
1333 if (wait_status == WAIT_TIMEOUT) {
1334 TCW_4( __kmp_global.g.g_time.dt.t_value,
1335 TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
1341 KA_TRACE( 10, (
"__kmp_launch_monitor: finished\n" ) );
1343 status = SetThreadPriority( monitor, THREAD_PRIORITY_NORMAL );
1345 DWORD error = GetLastError();
1348 KMP_MSG( CantSetThreadPriority ),
1354 if (__kmp_global.g.g_abort != 0) {
1360 KA_TRACE( 10, (
"__kmp_launch_monitor: terminate sig=%d\n", (__kmp_global.g.g_abort) ) );
1365 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1366 __kmp_terminate_thread( gtid );
1372 KA_TRACE( 10, (
"__kmp_launch_monitor: raise sig=%d\n", (__kmp_global.g.g_abort) ) );
1374 if (__kmp_global.g.g_abort > 0) {
1375 raise( __kmp_global.g.g_abort );
1379 TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );
1386 __kmp_create_worker(
int gtid, kmp_info_t *th,
size_t stack_size )
1388 kmp_thread_t handle;
1391 KA_TRACE( 10, (
"__kmp_create_worker: try to create thread (%d)\n", gtid ) );
1393 th->th.th_info.ds.ds_gtid = gtid;
1395 if ( KMP_UBER_GTID(gtid) ) {
1404 rc = DuplicateHandle(
1405 GetCurrentProcess(),
1407 GetCurrentProcess(),
1408 &th->th.th_info.ds.ds_thread,
1411 DUPLICATE_SAME_ACCESS
1414 KA_TRACE( 10, (
" __kmp_create_worker: ROOT Handle duplicated, th = %p, handle = %" KMP_UINTPTR_SPEC
"\n",
1416 th->th.th_info.ds.ds_thread ) );
1417 th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1419 if ( TCR_4(__kmp_gtid_mode) < 2 ) {
1421 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1422 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1423 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1424 __kmp_check_stack_overlap( th );
1431 KA_TRACE( 10, (
"__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC
1432 " bytes\n", stack_size ) );
1434 stack_size += gtid * __kmp_stkoffset;
1436 TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1437 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1439 KA_TRACE( 10, (
"__kmp_create_worker: (before) stack_size = %"
1441 " bytes, &__kmp_launch_worker = %p, th = %p, "
1443 (SIZE_T) stack_size,
1444 (LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
1445 (LPVOID) th, &idThread ) );
1448 handle = CreateThread( NULL, (SIZE_T) stack_size,
1449 (LPTHREAD_START_ROUTINE) __kmp_launch_worker,
1450 (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );
1453 KA_TRACE( 10, (
"__kmp_create_worker: (after) stack_size = %"
1455 " bytes, &__kmp_launch_worker = %p, th = %p, "
1456 "idThread = %u, handle = %" KMP_UINTPTR_SPEC
"\n",
1457 (SIZE_T) stack_size,
1458 (LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
1459 (LPVOID) th, idThread, handle ) );
1462 if ( handle == 0 ) {
1463 DWORD error = GetLastError();
1466 KMP_MSG( CantCreateThread ),
1471 th->th.th_info.ds.ds_thread = handle;
1477 KA_TRACE( 10, (
"__kmp_create_worker: done creating thread (%d)\n", gtid ) );
1481 __kmp_still_running(kmp_info_t *th) {
1482 return (WAIT_TIMEOUT == WaitForSingleObject( th->th.th_info.ds.ds_thread, 0));
1486 __kmp_create_monitor( kmp_info_t *th )
1488 kmp_thread_t handle;
1490 int ideal, new_ideal;
1491 int caller_gtid = __kmp_get_gtid();
1493 KA_TRACE( 10, (
"__kmp_create_monitor: try to create monitor\n" ) );
1497 __kmp_monitor_ev = CreateEvent( NULL, TRUE, FALSE, NULL );
1498 if ( __kmp_monitor_ev == NULL ) {
1499 DWORD error = GetLastError();
1502 KMP_MSG( CantCreateEvent ),
1508 __kmp_itt_system_object_created( __kmp_monitor_ev,
"Event" );
1511 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1512 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1516 if ( __kmp_monitor_stksize == 0 ) {
1517 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1519 if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
1520 __kmp_monitor_stksize = __kmp_sys_min_stksize;
1523 KA_TRACE( 10, (
"__kmp_create_monitor: requested stacksize = %d bytes\n",
1524 (
int) __kmp_monitor_stksize ) );
1526 TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
1528 handle = CreateThread( NULL, (SIZE_T) __kmp_monitor_stksize,
1529 (LPTHREAD_START_ROUTINE) __kmp_launch_monitor,
1530 (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );
1532 DWORD error = GetLastError();
1535 KMP_MSG( CantCreateThread ),
1541 th->th.th_info.ds.ds_thread = handle;
1545 KA_TRACE( 10, (
"__kmp_create_monitor: monitor created %p\n",
1546 (
void *) th->th.th_info.ds.ds_thread ) );
1559 __kmp_is_thread_alive( kmp_info_t * th, DWORD *exit_val )
1562 rc = GetExitCodeThread( th->th.th_info.ds.ds_thread, exit_val );
1564 DWORD error = GetLastError();
1567 KMP_MSG( FunctionError,
"GetExitCodeThread()" ),
1572 return ( *exit_val == STILL_ACTIVE );
1580 ExitThread( exit_status );
1587 __kmp_reap_common( kmp_info_t * th )
1593 KA_TRACE( 10, (
"__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid ) );
1613 register kmp_uint32 spins;
1615 KMP_FSYNC_SPIN_INIT( obj, (
void*) & th->th.th_info.ds.ds_alive );
1617 KMP_INIT_YIELD( spins );
1620 KMP_FSYNC_SPIN_PREPARE( obj );
1622 __kmp_is_thread_alive( th, &exit_val );
1623 __kmp_static_delay( TRUE );
1624 KMP_YIELD( TCR_4(__kmp_nth) > __kmp_avail_proc );
1625 KMP_YIELD_SPIN( spins );
1626 }
while ( exit_val == STILL_ACTIVE && TCR_4( th->th.th_info.ds.ds_alive ) );
1628 if ( exit_val == STILL_ACTIVE ) {
1629 KMP_FSYNC_CANCEL( obj );
1631 KMP_FSYNC_SPIN_ACQUIRED( obj );
1636 __kmp_free_handle( th->th.th_info.ds.ds_thread );
1643 if ( exit_val == STILL_ACTIVE ) {
1644 KA_TRACE( 1, (
"__kmp_reap_common: thread still active.\n" ) );
1645 }
else if ( (
void *) exit_val != (
void *) th) {
1646 KA_TRACE( 1, (
"__kmp_reap_common: ExitProcess / TerminateThread used?\n" ) );
1651 "__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC
"\n",
1652 th->th.th_info.ds.ds_gtid,
1653 th->th.th_info.ds.ds_thread
1657 th->th.th_info.ds.ds_thread = 0;
1658 th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1659 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1660 th->th.th_info.ds.ds_thread_id = 0;
1666 __kmp_reap_monitor( kmp_info_t *th )
1670 KA_TRACE( 10, (
"__kmp_reap_monitor: try to reap %p\n",
1671 (
void *) th->th.th_info.ds.ds_thread ) );
1676 KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
1677 if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
1683 status = SetEvent( __kmp_monitor_ev );
1684 if ( status == FALSE ) {
1685 DWORD error = GetLastError();
1688 KMP_MSG( CantSetEvent ),
1693 KA_TRACE( 10, (
"__kmp_reap_monitor: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
1694 __kmp_reap_common( th );
1696 __kmp_free_handle( __kmp_monitor_ev );
1702 __kmp_reap_worker( kmp_info_t * th )
1704 KA_TRACE( 10, (
"__kmp_reap_worker: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
1705 __kmp_reap_common( th );
1711 #if KMP_HANDLE_SIGNALS
1715 __kmp_team_handler(
int signo )
1717 if ( __kmp_global.g.g_abort == 0 ) {
1719 if ( __kmp_debug_buf ) {
1720 __kmp_dump_debug_buffer();
1723 TCW_4( __kmp_global.g.g_abort, signo );
1725 TCW_4( __kmp_global.g.g_done, TRUE );
1733 sig_func_t __kmp_signal(
int signum, sig_func_t handler ) {
1734 sig_func_t old = signal( signum, handler );
1735 if ( old == SIG_ERR ) {
1737 __kmp_msg( kmp_ms_fatal, KMP_MSG( FunctionError,
"signal" ), KMP_ERR( error ), __kmp_msg_null );
1743 __kmp_install_one_handler(
1750 KB_TRACE( 60, (
"__kmp_install_one_handler: called: sig=%d\n", sig ) );
1751 if ( parallel_init ) {
1752 old = __kmp_signal( sig, handler );
1754 if ( old == __kmp_sighldrs[ sig ] ) {
1755 __kmp_siginstalled[ sig ] = 1;
1758 old = __kmp_signal( sig, old );
1764 old = __kmp_signal( sig, SIG_DFL );
1765 __kmp_sighldrs[ sig ] = old;
1766 __kmp_signal( sig, old );
1772 __kmp_remove_one_handler(
int sig ) {
1773 if ( __kmp_siginstalled[ sig ] ) {
1776 KB_TRACE( 60, (
"__kmp_remove_one_handler: called: sig=%d\n", sig ) );
1777 old = __kmp_signal( sig, __kmp_sighldrs[ sig ] );
1778 if ( old != __kmp_team_handler ) {
1779 KB_TRACE( 10, (
"__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
1780 old = __kmp_signal( sig, old );
1782 __kmp_sighldrs[ sig ] = NULL;
1783 __kmp_siginstalled[ sig ] = 0;
1790 __kmp_install_signals(
int parallel_init )
1792 KB_TRACE( 10, (
"__kmp_install_signals: called\n" ) );
1793 if ( ! __kmp_handle_signals ) {
1794 KB_TRACE( 10, (
"__kmp_install_signals: KMP_HANDLE_SIGNALS is false - handlers not installed\n" ) );
1797 __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
1798 __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
1799 __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
1800 __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
1801 __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
1802 __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
1807 __kmp_remove_signals(
void )
1810 KB_TRACE( 10, (
"__kmp_remove_signals: called\n" ) );
1811 for ( sig = 1; sig < NSIG; ++ sig ) {
1812 __kmp_remove_one_handler( sig );
1817 #endif // KMP_HANDLE_SIGNALS
1821 __kmp_thread_sleep(
int millis )
1825 status = SleepEx( (DWORD) millis, FALSE );
1827 DWORD error = GetLastError();
1830 KMP_MSG( FunctionError,
"SleepEx()" ),
1839 __kmp_is_address_mapped(
void * addr )
1842 MEMORY_BASIC_INFORMATION lpBuffer;
1845 dwLength =
sizeof(MEMORY_BASIC_INFORMATION);
1847 status = VirtualQuery( addr, &lpBuffer, dwLength );
1849 return !((( lpBuffer.State == MEM_RESERVE) || ( lpBuffer.State == MEM_FREE )) ||
1850 (( lpBuffer.Protect == PAGE_NOACCESS ) || ( lpBuffer.Protect == PAGE_EXECUTE )));
1854 __kmp_hardware_timestamp(
void)
1858 QueryPerformanceCounter((LARGE_INTEGER*) &r);
1864 __kmp_free_handle( kmp_thread_t tHandle )
1868 rc = CloseHandle( tHandle );
1870 DWORD error = GetLastError();
1873 KMP_MSG( CantCloseHandle ),
1881 __kmp_get_load_balance(
int max ) {
1883 static ULONG glb_buff_size = 100 * 1024;
1885 static int glb_running_threads = 0;
1886 static double glb_call_time = 0;
1888 int running_threads = 0;
1889 NTSTATUS status = 0;
1890 ULONG buff_size = 0;
1891 ULONG info_size = 0;
1892 void * buffer = NULL;
1893 PSYSTEM_PROCESS_INFORMATION spi = NULL;
1896 double call_time = 0.0;
1898 __kmp_elapsed( & call_time );
1900 if ( glb_call_time &&
1901 ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
1902 running_threads = glb_running_threads;
1905 glb_call_time = call_time;
1908 if ( NtQuerySystemInformation == NULL ) {
1909 running_threads = -1;
1920 buff_size = glb_buff_size;
1922 buff_size = 2 * buff_size;
1925 buffer = KMP_INTERNAL_REALLOC( buffer, buff_size );
1926 if ( buffer == NULL ) {
1927 running_threads = -1;
1930 status = NtQuerySystemInformation( SystemProcessInformation, buffer, buff_size, & info_size );
1933 }
while ( status == STATUS_INFO_LENGTH_MISMATCH );
1934 glb_buff_size = buff_size;
1936 #define CHECK( cond ) \
1938 KMP_DEBUG_ASSERT( cond ); \
1939 if ( ! ( cond ) ) { \
1940 running_threads = -1; \
1945 CHECK( buff_size >= info_size );
1946 spi = PSYSTEM_PROCESS_INFORMATION( buffer );
1948 ptrdiff_t offset = uintptr_t( spi ) - uintptr_t( buffer );
1949 CHECK( 0 <= offset && offset +
sizeof( SYSTEM_PROCESS_INFORMATION ) < info_size );
1950 HANDLE pid = spi->ProcessId;
1951 ULONG num = spi->NumberOfThreads;
1953 size_t spi_size =
sizeof( SYSTEM_PROCESS_INFORMATION ) +
sizeof( SYSTEM_THREAD ) * ( num - 1 );
1954 CHECK( offset + spi_size < info_size );
1955 if ( spi->NextEntryOffset != 0 ) {
1956 CHECK( spi_size <= spi->NextEntryOffset );
1961 for (
int i = 0; i < num; ++ i ) {
1962 THREAD_STATE state = spi->Threads[ i ].State;
1965 if ( state == StateRunning ) {
1969 if ( running_threads >= max ) {
1975 if ( spi->NextEntryOffset == 0 ) {
1978 spi = PSYSTEM_PROCESS_INFORMATION( uintptr_t( spi ) + spi->NextEntryOffset );
1985 if ( buffer != NULL ) {
1986 KMP_INTERNAL_FREE( buffer );
1989 glb_running_threads = running_threads;
1991 return running_threads;