diff options
Diffstat (limited to 'libgomp/config/linux')
25 files changed, 2116 insertions, 0 deletions
diff --git a/libgomp/config/linux/affinity.c b/libgomp/config/linux/affinity.c new file mode 100644 index 000000000..da9f3d8fd --- /dev/null +++ b/libgomp/config/linux/affinity.c @@ -0,0 +1,103 @@ +/* Copyright (C) 2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc. + Contributed by Jakub Jelinek <jakub@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* This is a Linux specific implementation of a CPU affinity setting. */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif +#include "libgomp.h" +#include <sched.h> +#include <stdlib.h> +#include <unistd.h> + +#ifdef HAVE_PTHREAD_AFFINITY_NP + +static unsigned int affinity_counter; + +void +gomp_init_affinity (void) +{ + cpu_set_t cpuset, cpusetnew; + size_t idx, widx; + unsigned long cpus = 0; + + if (pthread_getaffinity_np (pthread_self (), sizeof (cpuset), &cpuset)) + { + gomp_error ("could not get CPU affinity set"); + free (gomp_cpu_affinity); + gomp_cpu_affinity = NULL; + gomp_cpu_affinity_len = 0; + return; + } + + CPU_ZERO (&cpusetnew); + for (widx = idx = 0; idx < gomp_cpu_affinity_len; idx++) + if (gomp_cpu_affinity[idx] < CPU_SETSIZE + && CPU_ISSET (gomp_cpu_affinity[idx], &cpuset)) + { + if (! CPU_ISSET (gomp_cpu_affinity[idx], &cpusetnew)) + { + cpus++; + CPU_SET (gomp_cpu_affinity[idx], &cpusetnew); + } + gomp_cpu_affinity[widx++] = gomp_cpu_affinity[idx]; + } + + if (widx == 0) + { + gomp_error ("no CPUs left for affinity setting"); + free (gomp_cpu_affinity); + gomp_cpu_affinity = NULL; + gomp_cpu_affinity_len = 0; + return; + } + + gomp_cpu_affinity_len = widx; + if (cpus < gomp_available_cpus) + gomp_available_cpus = cpus; + CPU_ZERO (&cpuset); + CPU_SET (gomp_cpu_affinity[0], &cpuset); + pthread_setaffinity_np (pthread_self (), sizeof (cpuset), &cpuset); + affinity_counter = 1; +} + +void +gomp_init_thread_affinity (pthread_attr_t *attr) +{ + unsigned int cpu; + cpu_set_t cpuset; + + cpu = __sync_fetch_and_add (&affinity_counter, 1); + cpu %= gomp_cpu_affinity_len; + CPU_ZERO (&cpuset); + CPU_SET (gomp_cpu_affinity[cpu], &cpuset); + pthread_attr_setaffinity_np (attr, sizeof (cpu_set_t), &cpuset); +} + +#else + +#include "../posix/affinity.c" + +#endif diff --git a/libgomp/config/linux/alpha/futex.h b/libgomp/config/linux/alpha/futex.h new file mode 100644 index 000000000..a594d1663 --- /dev/null +++ b/libgomp/config/linux/alpha/futex.h @@ -0,0 +1,109 @@ +/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. + Contributed by Richard Henderson <rth@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* Provide target-specific access to the futex system call. */ + +#ifndef SYS_futex +#define SYS_futex 394 +#endif + + +static inline void +futex_wait (int *addr, int val) +{ + register long sc_0 __asm__("$0"); + register long sc_16 __asm__("$16"); + register long sc_17 __asm__("$17"); + register long sc_18 __asm__("$18"); + register long sc_19 __asm__("$19"); + + sc_0 = SYS_futex; + sc_16 = (long) addr; + sc_17 = gomp_futex_wait; + sc_18 = val; + sc_19 = 0; + __asm volatile ("callsys" + : "=r" (sc_0), "=r"(sc_19) + : "0"(sc_0), "r" (sc_16), "r"(sc_17), "r"(sc_18), "1"(sc_19) + : "$1", "$2", "$3", "$4", "$5", "$6", "$7", "$8", + "$22", "$23", "$24", "$25", "$27", "$28", "memory"); + if (__builtin_expect (sc_19, 0) && sc_0 == ENOSYS) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sc_0 = SYS_futex; + sc_17 &= ~FUTEX_PRIVATE_FLAG; + sc_19 = 0; + __asm volatile ("callsys" + : "=r" (sc_0), "=r"(sc_19) + : "0"(sc_0), "r" (sc_16), "r"(sc_17), "r"(sc_18), + "1"(sc_19) + : "$1", "$2", "$3", "$4", "$5", "$6", "$7", "$8", + "$22", "$23", "$24", "$25", "$27", "$28", "memory"); + } +} + +static inline void +futex_wake (int *addr, int count) +{ + register long sc_0 __asm__("$0"); + register long sc_16 __asm__("$16"); + register long sc_17 __asm__("$17"); + register long sc_18 __asm__("$18"); + register long sc_19 __asm__("$19"); + + sc_0 = SYS_futex; + sc_16 = (long) addr; + sc_17 = gomp_futex_wake; + sc_18 = count; + __asm volatile ("callsys" + : "=r" (sc_0), "=r"(sc_19) + : "0"(sc_0), "r" (sc_16), "r"(sc_17), "r"(sc_18) + : "$1", "$2", "$3", "$4", "$5", "$6", "$7", "$8", + "$22", "$23", "$24", "$25", "$27", "$28", "memory"); + if (__builtin_expect (sc_19, 0) && sc_0 == ENOSYS) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sc_0 = SYS_futex; + sc_17 &= ~FUTEX_PRIVATE_FLAG; + __asm volatile ("callsys" + : "=r" (sc_0), "=r"(sc_19) + : "0"(sc_0), "r" (sc_16), "r"(sc_17), "r"(sc_18) + : "$1", "$2", "$3", "$4", "$5", "$6", "$7", "$8", + "$22", "$23", "$24", "$25", "$27", "$28", "memory"); + } +} + +static inline void +cpu_relax (void) +{ + __asm volatile ("" : : : "memory"); +} + +static inline void +atomic_write_barrier (void) +{ + __asm volatile ("wmb" : : : "memory"); +} diff --git a/libgomp/config/linux/arm/mutex.h b/libgomp/config/linux/arm/mutex.h new file mode 100644 index 000000000..30021d547 --- /dev/null +++ b/libgomp/config/linux/arm/mutex.h @@ -0,0 +1,28 @@ +/* Copyright (C) 2010 Free Software Foundation, Inc. + Contributed by ARM Ltd. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* ARM needs the same correct usage of __sync_synchronize and + __sync_lock_test_and_set as ia64. So we just use its mutex.h. */ + +#include "config/linux/ia64/mutex.h" diff --git a/libgomp/config/linux/bar.c b/libgomp/config/linux/bar.c new file mode 100644 index 000000000..3e0379eb6 --- /dev/null +++ b/libgomp/config/linux/bar.c @@ -0,0 +1,122 @@ +/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. + Contributed by Richard Henderson <rth@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* This is a Linux specific implementation of a barrier synchronization + mechanism for libgomp. This type is private to the library. This + implementation uses atomic instructions and the futex syscall. */ + +#include <limits.h> +#include "wait.h" + + +void +gomp_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state) +{ + if (__builtin_expect ((state & 1) != 0, 0)) + { + /* Next time we'll be awaiting TOTAL threads again. */ + bar->awaited = bar->total; + atomic_write_barrier (); + bar->generation += 4; + futex_wake ((int *) &bar->generation, INT_MAX); + } + else + { + unsigned int generation = state; + + do + do_wait ((int *) &bar->generation, generation); + while (bar->generation == generation); + } +} + +void +gomp_barrier_wait (gomp_barrier_t *bar) +{ + gomp_barrier_wait_end (bar, gomp_barrier_wait_start (bar)); +} + +/* Like gomp_barrier_wait, except that if the encountering thread + is not the last one to hit the barrier, it returns immediately. + The intended usage is that a thread which intends to gomp_barrier_destroy + this barrier calls gomp_barrier_wait, while all other threads + call gomp_barrier_wait_last. When gomp_barrier_wait returns, + the barrier can be safely destroyed. */ + +void +gomp_barrier_wait_last (gomp_barrier_t *bar) +{ + gomp_barrier_state_t state = gomp_barrier_wait_start (bar); + if (state & 1) + gomp_barrier_wait_end (bar, state); +} + +void +gomp_team_barrier_wake (gomp_barrier_t *bar, int count) +{ + futex_wake ((int *) &bar->generation, count == 0 ? INT_MAX : count); +} + +void +gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state) +{ + unsigned int generation; + + if (__builtin_expect ((state & 1) != 0, 0)) + { + /* Next time we'll be awaiting TOTAL threads again. */ + struct gomp_thread *thr = gomp_thread (); + struct gomp_team *team = thr->ts.team; + bar->awaited = bar->total; + atomic_write_barrier (); + if (__builtin_expect (team->task_count, 0)) + { + gomp_barrier_handle_tasks (state); + state &= ~1; + } + else + { + bar->generation = state + 3; + futex_wake ((int *) &bar->generation, INT_MAX); + return; + } + } + + generation = state; + do + { + do_wait ((int *) &bar->generation, generation); + if (__builtin_expect (bar->generation & 1, 0)) + gomp_barrier_handle_tasks (state); + if ((bar->generation & 2)) + generation |= 2; + } + while (bar->generation != state + 4); +} + +void +gomp_team_barrier_wait (gomp_barrier_t *bar) +{ + gomp_team_barrier_wait_end (bar, gomp_barrier_wait_start (bar)); +} diff --git a/libgomp/config/linux/bar.h b/libgomp/config/linux/bar.h new file mode 100644 index 000000000..9c1a8a076 --- /dev/null +++ b/libgomp/config/linux/bar.h @@ -0,0 +1,118 @@ +/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. + Contributed by Richard Henderson <rth@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* This is a Linux specific implementation of a barrier synchronization + mechanism for libgomp. This type is private to the library. This + implementation uses atomic instructions and the futex syscall. */ + +#ifndef GOMP_BARRIER_H +#define GOMP_BARRIER_H 1 + +#include "mutex.h" + +typedef struct +{ + /* Make sure total/generation is in a mostly read cacheline, while + awaited in a separate cacheline. */ + unsigned total __attribute__((aligned (64))); + unsigned generation; + unsigned awaited __attribute__((aligned (64))); +} gomp_barrier_t; +typedef unsigned int gomp_barrier_state_t; + +static inline void gomp_barrier_init (gomp_barrier_t *bar, unsigned count) +{ + bar->total = count; + bar->awaited = count; + bar->generation = 0; +} + +static inline void gomp_barrier_reinit (gomp_barrier_t *bar, unsigned count) +{ + __sync_fetch_and_add (&bar->awaited, count - bar->total); + bar->total = count; +} + +static inline void gomp_barrier_destroy (gomp_barrier_t *bar) +{ +} + +extern void gomp_barrier_wait (gomp_barrier_t *); +extern void gomp_barrier_wait_last (gomp_barrier_t *); +extern void gomp_barrier_wait_end (gomp_barrier_t *, gomp_barrier_state_t); +extern void gomp_team_barrier_wait (gomp_barrier_t *); +extern void gomp_team_barrier_wait_end (gomp_barrier_t *, + gomp_barrier_state_t); +extern void gomp_team_barrier_wake (gomp_barrier_t *, int); + +static inline gomp_barrier_state_t +gomp_barrier_wait_start (gomp_barrier_t *bar) +{ + unsigned int ret = bar->generation & ~3; + /* Do we need any barrier here or is __sync_add_and_fetch acting + as the needed LoadLoad barrier already? */ + ret += __sync_add_and_fetch (&bar->awaited, -1) == 0; + return ret; +} + +static inline bool +gomp_barrier_last_thread (gomp_barrier_state_t state) +{ + return state & 1; +} + +/* All the inlines below must be called with team->task_lock + held. */ + +static inline void +gomp_team_barrier_set_task_pending (gomp_barrier_t *bar) +{ + bar->generation |= 1; +} + +static inline void +gomp_team_barrier_clear_task_pending (gomp_barrier_t *bar) +{ + bar->generation &= ~1; +} + +static inline void +gomp_team_barrier_set_waiting_for_tasks (gomp_barrier_t *bar) +{ + bar->generation |= 2; +} + +static inline bool +gomp_team_barrier_waiting_for_tasks (gomp_barrier_t *bar) +{ + return (bar->generation & 2) != 0; +} + +static inline void +gomp_team_barrier_done (gomp_barrier_t *bar, gomp_barrier_state_t state) +{ + bar->generation = (state & ~3) + 4; +} + +#endif /* GOMP_BARRIER_H */ diff --git a/libgomp/config/linux/futex.h b/libgomp/config/linux/futex.h new file mode 100644 index 000000000..bab088a55 --- /dev/null +++ b/libgomp/config/linux/futex.h @@ -0,0 +1,75 @@ +/* Copyright (C) 2010, 2011 Free Software Foundation, Inc. + Contributed by ARM Ltd. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* Provide target-specific access to the futex system call. */ + +/* The include file hierachy above us (wait.h) has pushed visibility + hidden, this will be applied to prototypes with headers we include + with the effect that we cannot link against an external function + (syscall). The solution here is to push default visibility, include + our required headers then reinstante the original visibility. */ + +#pragma GCC visibility push(default) + +#define _GNU_SOURCE +#include <unistd.h> +#include <sys/syscall.h> + +#pragma GCC visibility pop + +static inline void +futex_wait (int *addr, int val) +{ + long err = syscall (SYS_futex, addr, gomp_futex_wait, val, NULL); + if (__builtin_expect (err == -ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + syscall (SYS_futex, addr, gomp_futex_wait, val, NULL); + } +} + +static inline void +futex_wake (int *addr, int count) +{ + long err = syscall (SYS_futex, addr, gomp_futex_wake, count); + if (__builtin_expect (err == -ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + syscall (SYS_futex, addr, gomp_futex_wake, count); + } +} + +static inline void +cpu_relax (void) +{ + __asm volatile ("" : : : "memory"); +} + +static inline void +atomic_write_barrier (void) +{ + __sync_synchronize (); +} diff --git a/libgomp/config/linux/ia64/futex.h b/libgomp/config/linux/ia64/futex.h new file mode 100644 index 000000000..e5e9aac39 --- /dev/null +++ b/libgomp/config/linux/ia64/futex.h @@ -0,0 +1,94 @@ +/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. + Contributed by Richard Henderson <rth@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* Provide target-specific access to the futex system call. */ + +#include <sys/syscall.h> + + + +static inline long +sys_futex0(int *addr, long op, int val) +{ + register long out0 asm ("out0") = (long) addr; + register long out1 asm ("out1") = op; + register long out2 asm ("out2") = val; + register long out3 asm ("out3") = 0; + register long r8 asm ("r8"); + register long r10 asm ("r10"); + register long r15 asm ("r15") = SYS_futex; + + __asm __volatile ("break 0x100000" + : "=r"(r15), "=r"(out0), "=r"(out1), "=r"(out2), "=r"(out3), + "=r"(r8), "=r"(r10) + : "r"(r15), "r"(out0), "r"(out1), "r"(out2), "r"(out3) + : "memory", "out4", "out5", "out6", "out7", + /* Non-stacked integer registers, minus r8, r10, r15. */ + "r2", "r3", "r9", "r11", "r12", "r13", "r14", "r16", "r17", "r18", + "r19", "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27", + "r28", "r29", "r30", "r31", + /* Predicate registers. */ + "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", + /* Non-rotating fp registers. */ + "f6", "f7", "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", + /* Branch registers. */ + "b6"); + return r8 & r10; +} + +static inline void +futex_wait (int *addr, int val) +{ + long err = sys_futex0 (addr, gomp_futex_wait, val); + if (__builtin_expect (err == ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wait, val); + } +} + +static inline void +futex_wake (int *addr, int count) +{ + long err = sys_futex0 (addr, gomp_futex_wake, count); + if (__builtin_expect (err == ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wake, count); + } +} + +static inline void +cpu_relax (void) +{ + __asm volatile ("hint @pause" : : : "memory"); +} + +static inline void +atomic_write_barrier (void) +{ + __sync_synchronize (); +} diff --git a/libgomp/config/linux/ia64/mutex.h b/libgomp/config/linux/ia64/mutex.h new file mode 100644 index 000000000..6e294059b --- /dev/null +++ b/libgomp/config/linux/ia64/mutex.h @@ -0,0 +1,65 @@ +/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. + Contributed by Richard Henderson <rth@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* This is a Linux specific implementation of a mutex synchronization + mechanism for libgomp. This type is private to the library. This + implementation uses atomic instructions and the futex syscall. */ + +#ifndef GOMP_MUTEX_H +#define GOMP_MUTEX_H 1 + +typedef int gomp_mutex_t; + +#define GOMP_MUTEX_INIT_0 1 + +static inline void gomp_mutex_init (gomp_mutex_t *mutex) +{ + *mutex = 0; +} + +extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex); +static inline void gomp_mutex_lock (gomp_mutex_t *mutex) +{ + if (!__sync_bool_compare_and_swap (mutex, 0, 1)) + gomp_mutex_lock_slow (mutex); +} + +extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex); + +/* IA64 needs a __sync_synchronize call before __sync_lock_test_and_set + because __sync_lock_test_and_set is not a full memory fence. */ +static inline void gomp_mutex_unlock (gomp_mutex_t *mutex) +{ + int val; + __sync_synchronize (); + val = __sync_lock_test_and_set (mutex, 0); + if (__builtin_expect (val > 1, 0)) + gomp_mutex_unlock_slow (mutex); +} + +static inline void gomp_mutex_destroy (gomp_mutex_t *mutex) +{ +} + +#endif /* GOMP_MUTEX_H */ diff --git a/libgomp/config/linux/lock.c b/libgomp/config/linux/lock.c new file mode 100644 index 000000000..405460780 --- /dev/null +++ b/libgomp/config/linux/lock.c @@ -0,0 +1,256 @@ +/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. + Contributed by Richard Henderson <rth@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* This is a Linux specific implementation of the public OpenMP locking + primitives. This implementation uses atomic instructions and the futex + syscall. */ + +#include <string.h> +#include <unistd.h> +#include <sys/syscall.h> +#include "wait.h" + + +/* The internal gomp_mutex_t and the external non-recursive omp_lock_t + have the same form. Re-use it. */ + +void +gomp_init_lock_30 (omp_lock_t *lock) +{ + gomp_mutex_init (lock); +} + +void +gomp_destroy_lock_30 (omp_lock_t *lock) +{ + gomp_mutex_destroy (lock); +} + +void +gomp_set_lock_30 (omp_lock_t *lock) +{ + gomp_mutex_lock (lock); +} + +void +gomp_unset_lock_30 (omp_lock_t *lock) +{ + gomp_mutex_unlock (lock); +} + +int +gomp_test_lock_30 (omp_lock_t *lock) +{ + return __sync_bool_compare_and_swap (lock, 0, 1); +} + +void +gomp_init_nest_lock_30 (omp_nest_lock_t *lock) +{ + memset (lock, '\0', sizeof (*lock)); +} + +void +gomp_destroy_nest_lock_30 (omp_nest_lock_t *lock) +{ +} + +void +gomp_set_nest_lock_30 (omp_nest_lock_t *lock) +{ + void *me = gomp_icv (true); + + if (lock->owner != me) + { + gomp_mutex_lock (&lock->lock); + lock->owner = me; + } + + lock->count++; +} + +void +gomp_unset_nest_lock_30 (omp_nest_lock_t *lock) +{ + if (--lock->count == 0) + { + lock->owner = NULL; + gomp_mutex_unlock (&lock->lock); + } +} + +int +gomp_test_nest_lock_30 (omp_nest_lock_t *lock) +{ + void *me = gomp_icv (true); + + if (lock->owner == me) + return ++lock->count; + + if (__sync_bool_compare_and_swap (&lock->lock, 0, 1)) + { + lock->owner = me; + lock->count = 1; + return 1; + } + + return 0; +} + +#ifdef LIBGOMP_GNU_SYMBOL_VERSIONING +/* gomp_mutex_* can be safely locked in one thread and + unlocked in another thread, so the OpenMP 2.5 and OpenMP 3.0 + non-nested locks can be the same. */ +strong_alias (gomp_init_lock_30, gomp_init_lock_25) +strong_alias (gomp_destroy_lock_30, gomp_destroy_lock_25) +strong_alias (gomp_set_lock_30, gomp_set_lock_25) +strong_alias (gomp_unset_lock_30, gomp_unset_lock_25) +strong_alias (gomp_test_lock_30, gomp_test_lock_25) + +/* The external recursive omp_nest_lock_25_t form requires additional work. */ + +/* We need an integer to uniquely identify this thread. Most generally + this is the thread's TID, which ideally we'd get this straight from + the TLS block where glibc keeps it. Unfortunately, we can't get at + that directly. + + If we don't support (or have disabled) TLS, one function call is as + good (or bad) as any other. Use the syscall all the time. + + On an ILP32 system (defined here as not LP64), we can make do with + any thread-local pointer. Ideally we'd use the TLS base address, + since that requires the least amount of arithmetic, but that's not + always available directly. Make do with the gomp_thread pointer + since it's handy. */ + +# if !defined (HAVE_TLS) +static inline int gomp_tid (void) +{ + return syscall (SYS_gettid); +} +# elif !defined(__LP64__) +static inline int gomp_tid (void) +{ + return (int) gomp_thread (); +} +# else +static __thread int tid_cache; +static inline int gomp_tid (void) +{ + int tid = tid_cache; + if (__builtin_expect (tid == 0, 0)) + tid_cache = tid = syscall (SYS_gettid); + return tid; +} +# endif + + +void +gomp_init_nest_lock_25 (omp_nest_lock_25_t *lock) +{ + memset (lock, 0, sizeof (lock)); +} + +void +gomp_destroy_nest_lock_25 (omp_nest_lock_25_t *lock) +{ +} + +void +gomp_set_nest_lock_25 (omp_nest_lock_25_t *lock) +{ + int otid, tid = gomp_tid (); + + while (1) + { + otid = __sync_val_compare_and_swap (&lock->owner, 0, tid); + if (otid == 0) + { + lock->count = 1; + return; + } + if (otid == tid) + { + lock->count++; + return; + } + + do_wait (&lock->owner, otid); + } +} + +void +gomp_unset_nest_lock_25 (omp_nest_lock_25_t *lock) +{ + /* ??? Validate that we own the lock here. */ + + if (--lock->count == 0) + { + __sync_lock_release (&lock->owner); + futex_wake (&lock->owner, 1); + } +} + +int +gomp_test_nest_lock_25 (omp_nest_lock_25_t *lock) +{ + int otid, tid = gomp_tid (); + + otid = __sync_val_compare_and_swap (&lock->owner, 0, tid); + if (otid == 0) + { + lock->count = 1; + return 1; + } + if (otid == tid) + return ++lock->count; + + return 0; +} + +omp_lock_symver (omp_init_lock) +omp_lock_symver (omp_destroy_lock) +omp_lock_symver (omp_set_lock) +omp_lock_symver (omp_unset_lock) +omp_lock_symver (omp_test_lock) +omp_lock_symver (omp_init_nest_lock) +omp_lock_symver (omp_destroy_nest_lock) +omp_lock_symver (omp_set_nest_lock) +omp_lock_symver (omp_unset_nest_lock) +omp_lock_symver (omp_test_nest_lock) + +#else + +ialias (omp_init_lock) +ialias (omp_init_nest_lock) +ialias (omp_destroy_lock) +ialias (omp_destroy_nest_lock) +ialias (omp_set_lock) +ialias (omp_set_nest_lock) +ialias (omp_unset_lock) +ialias (omp_unset_nest_lock) +ialias (omp_test_lock) +ialias (omp_test_nest_lock) + +#endif diff --git a/libgomp/config/linux/mips/futex.h b/libgomp/config/linux/mips/futex.h new file mode 100644 index 000000000..576ada452 --- /dev/null +++ b/libgomp/config/linux/mips/futex.h @@ -0,0 +1,72 @@ +/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. + Contributed by Ilie Garbacea <ilie@mips.com>, Chao-ying Fu <fu@mips.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* Provide target-specific access to the futex system call. */ + +#include <sys/syscall.h> +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 + +static inline void +sys_futex0 (int *addr, int op, int val) +{ + register unsigned long __v0 asm("$2") = (unsigned long) SYS_futex; + register unsigned long __a0 asm("$4") = (unsigned long) addr; + register unsigned long __a1 asm("$5") = (unsigned long) op; + register unsigned long __a2 asm("$6") = (unsigned long) val; + register unsigned long __a3 asm("$7") = 0; + + __asm volatile ("syscall" + /* returns $a3 (errno), $v0 (return value) */ + : "=r" (__v0), "=r" (__a3) + /* arguments in v0 (syscall) a0-a3 */ + : "r" (__v0), "r" (__a0), "r" (__a1), "r" (__a2), "r" (__a3) + /* clobbers at, v1, t0-t9, memory */ + : "$1", "$3", "$8", "$9", "$10", "$11", "$12", "$13", "$14", + "$15", "$24", "$25", "memory"); +} + +static inline void +futex_wait (int *addr, int val) +{ + sys_futex0 (addr, FUTEX_WAIT, val); +} + +static inline void +futex_wake (int *addr, int count) +{ + sys_futex0 (addr, FUTEX_WAKE, count); +} + +static inline void +cpu_relax (void) +{ + __asm volatile ("" : : : "memory"); +} + +static inline void +atomic_write_barrier (void) +{ + __sync_synchronize (); +} diff --git a/libgomp/config/linux/mips/mutex.h b/libgomp/config/linux/mips/mutex.h new file mode 100644 index 000000000..668cc11b2 --- /dev/null +++ b/libgomp/config/linux/mips/mutex.h @@ -0,0 +1,27 @@ +/* Copyright (C) 2009 Free Software Foundation, Inc. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* MIPS needs the same correct usage of __sync_synchronize and + __sync_lock_test_and_set as ia64. So we just use its mutex.h. */ + +#include "config/linux/ia64/mutex.h" diff --git a/libgomp/config/linux/mutex.c b/libgomp/config/linux/mutex.c new file mode 100644 index 000000000..3ca37c19f --- /dev/null +++ b/libgomp/config/linux/mutex.c @@ -0,0 +1,50 @@ +/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. + Contributed by Richard Henderson <rth@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* This is a Linux specific implementation of a mutex synchronization + mechanism for libgomp. This type is private to the library. This + implementation uses atomic instructions and the futex syscall. */ + +#include "wait.h" + +long int gomp_futex_wake = FUTEX_WAKE | FUTEX_PRIVATE_FLAG; +long int gomp_futex_wait = FUTEX_WAIT | FUTEX_PRIVATE_FLAG; + +void +gomp_mutex_lock_slow (gomp_mutex_t *mutex) +{ + do + { + int oldval = __sync_val_compare_and_swap (mutex, 1, 2); + if (oldval != 0) + do_wait (mutex, 2); + } + while (!__sync_bool_compare_and_swap (mutex, 0, 2)); +} + +void +gomp_mutex_unlock_slow (gomp_mutex_t *mutex) +{ + futex_wake (mutex, 1); +} diff --git a/libgomp/config/linux/mutex.h b/libgomp/config/linux/mutex.h new file mode 100644 index 000000000..1905ce012 --- /dev/null +++ b/libgomp/config/linux/mutex.h @@ -0,0 +1,70 @@ +/* Copyright (C) 2005, 2009 Free Software Foundation, Inc. + Contributed by Richard Henderson <rth@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* This is a Linux specific implementation of a mutex synchronization + mechanism for libgomp. This type is private to the library. This + implementation uses atomic instructions and the futex syscall. */ + +#ifndef GOMP_MUTEX_H +#define GOMP_MUTEX_H 1 + +typedef int gomp_mutex_t; + +#define GOMP_MUTEX_INIT_0 1 + +static inline void gomp_mutex_init (gomp_mutex_t *mutex) +{ + *mutex = 0; +} + +extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex); +static inline void gomp_mutex_lock (gomp_mutex_t *mutex) +{ + if (!__sync_bool_compare_and_swap (mutex, 0, 1)) + gomp_mutex_lock_slow (mutex); +} + +extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex); +static inline void gomp_mutex_unlock (gomp_mutex_t *mutex) +{ + /* Warning: By definition __sync_lock_test_and_set() does not have + proper memory barrier semantics for a mutex unlock operation. + However, this default implementation is written assuming that it + does, which is true for some targets. + + Targets that require additional memory barriers before + __sync_lock_test_and_set to achieve the release semantics of + mutex unlock, are encouraged to include + "config/linux/ia64/mutex.h" in a target specific mutex.h instead + of using this file. */ + int val = __sync_lock_test_and_set (mutex, 0); + if (__builtin_expect (val > 1, 0)) + gomp_mutex_unlock_slow (mutex); +} + +static inline void gomp_mutex_destroy (gomp_mutex_t *mutex) +{ +} + +#endif /* GOMP_MUTEX_H */ diff --git a/libgomp/config/linux/omp-lock.h b/libgomp/config/linux/omp-lock.h new file mode 100644 index 000000000..e65aff7fc --- /dev/null +++ b/libgomp/config/linux/omp-lock.h @@ -0,0 +1,12 @@ +/* This header is used during the build process to find the size and + alignment of the public OpenMP locks, so that we can export data + structures without polluting the namespace. + + When using the Linux futex primitive, non-recursive locks require + only one int. Recursive locks require we identify the owning task + and so require one int and a pointer. */ + +typedef int omp_lock_t; +typedef struct { int lock, count; void *owner; } omp_nest_lock_t; +typedef int omp_lock_25_t; +typedef struct { int owner, count; } omp_nest_lock_25_t; diff --git a/libgomp/config/linux/powerpc/futex.h b/libgomp/config/linux/powerpc/futex.h new file mode 100644 index 000000000..efc05c416 --- /dev/null +++ b/libgomp/config/linux/powerpc/futex.h @@ -0,0 +1,92 @@ +/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. + Contributed by Richard Henderson <rth@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* Provide target-specific access to the futex system call. */ + +#include <sys/syscall.h> + +static inline long +sys_futex0 (int *addr, int op, int val) +{ + register long int r0 __asm__ ("r0"); + register long int r3 __asm__ ("r3"); + register long int r4 __asm__ ("r4"); + register long int r5 __asm__ ("r5"); + register long int r6 __asm__ ("r6"); + + r0 = SYS_futex; + r3 = (long) addr; + r4 = op; + r5 = val; + r6 = 0; + + /* ??? The powerpc64 sysdep.h file clobbers ctr; the powerpc32 sysdep.h + doesn't. It doesn't much matter for us. In the interest of unity, + go ahead and clobber it always. */ + + __asm volatile ("sc; mfcr %0" + : "=r"(r0), "=r"(r3), "=r"(r4), "=r"(r5), "=r"(r6) + : "r"(r0), "r"(r3), "r"(r4), "r"(r5), "r"(r6) + : "r7", "r8", "r9", "r10", "r11", "r12", + "cr0", "ctr", "memory"); + if (__builtin_expect (r0 & (1 << 28), 0)) + return r3; + return 0; +} + +static inline void +futex_wait (int *addr, int val) +{ + long err = sys_futex0 (addr, gomp_futex_wait, val); + if (__builtin_expect (err == ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wait, val); + } +} + +static inline void +futex_wake (int *addr, int count) +{ + long err = sys_futex0 (addr, gomp_futex_wake, count); + if (__builtin_expect (err == ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wake, count); + } +} + +static inline void +cpu_relax (void) +{ + __asm volatile ("" : : : "memory"); +} + +static inline void +atomic_write_barrier (void) +{ + __asm volatile ("eieio" : : : "memory"); +} diff --git a/libgomp/config/linux/powerpc/mutex.h b/libgomp/config/linux/powerpc/mutex.h new file mode 100644 index 000000000..e64ff077c --- /dev/null +++ b/libgomp/config/linux/powerpc/mutex.h @@ -0,0 +1,2 @@ +/* On PowerPC __sync_lock_test_and_set isn't a full barrier. */ +#include "config/linux/ia64/mutex.h" diff --git a/libgomp/config/linux/proc.c b/libgomp/config/linux/proc.c new file mode 100644 index 000000000..01f51dfa1 --- /dev/null +++ b/libgomp/config/linux/proc.c @@ -0,0 +1,164 @@ +/* Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010 + Free Software Foundation, Inc. + Contributed by Jakub Jelinek <jakub@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* This file contains system specific routines related to counting + online processors and dynamic load balancing. */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif +#include "libgomp.h" +#include <sched.h> +#include <stdlib.h> +#include <unistd.h> +#ifdef HAVE_GETLOADAVG +# ifdef HAVE_SYS_LOADAVG_H +# include <sys/loadavg.h> +# endif +#endif + +#ifdef HAVE_PTHREAD_AFFINITY_NP +static unsigned long +cpuset_popcount (cpu_set_t *cpusetp) +{ +#ifdef CPU_COUNT + /* glibc 2.6 and above provide a macro for this. */ + return CPU_COUNT (cpusetp); +#else + size_t i; + unsigned long ret = 0; + extern int check[sizeof (cpusetp->__bits[0]) == sizeof (unsigned long int)]; + + (void) check; + for (i = 0; i < sizeof (*cpusetp) / sizeof (cpusetp->__bits[0]); i++) + { + unsigned long int mask = cpusetp->__bits[i]; + if (mask == 0) + continue; + ret += __builtin_popcountl (mask); + } + return ret; +#endif +} +#endif + +/* At startup, determine the default number of threads. It would seem + this should be related to the number of cpus online. */ + +void +gomp_init_num_threads (void) +{ +#ifdef HAVE_PTHREAD_AFFINITY_NP + cpu_set_t cpuset; + + if (pthread_getaffinity_np (pthread_self (), sizeof (cpuset), &cpuset) == 0) + { + /* Count only the CPUs this process can use. */ + gomp_global_icv.nthreads_var = cpuset_popcount (&cpuset); + if (gomp_global_icv.nthreads_var == 0) + gomp_global_icv.nthreads_var = 1; + return; + } +#endif +#ifdef _SC_NPROCESSORS_ONLN + gomp_global_icv.nthreads_var = sysconf (_SC_NPROCESSORS_ONLN); +#endif +} + +static int +get_num_procs (void) +{ +#ifdef HAVE_PTHREAD_AFFINITY_NP + cpu_set_t cpuset; + + if (gomp_cpu_affinity == NULL) + { + /* Count only the CPUs this process can use. */ + if (pthread_getaffinity_np (pthread_self (), sizeof (cpuset), + &cpuset) == 0) + { + int ret = cpuset_popcount (&cpuset); + return ret != 0 ? ret : 1; + } + } + else + { + /* We can't use pthread_getaffinity_np in this case + (we have changed it ourselves, it binds to just one CPU). + Count instead the number of different CPUs we are + using. gomp_init_affinity updated gomp_available_cpus to + the number of CPUs in the GOMP_AFFINITY mask that we are + allowed to use though. */ + return gomp_available_cpus; + } +#endif +#ifdef _SC_NPROCESSORS_ONLN + return sysconf (_SC_NPROCESSORS_ONLN); +#else + return gomp_icv (false)->nthreads_var; +#endif +} + +/* When OMP_DYNAMIC is set, at thread launch determine the number of + threads we should spawn for this team. */ +/* ??? I have no idea what best practice for this is. Surely some + function of the number of processors that are *still* online and + the load average. Here I use the number of processors online + minus the 15 minute load average. */ + +unsigned +gomp_dynamic_max_threads (void) +{ + unsigned n_onln, loadavg, nthreads_var = gomp_icv (false)->nthreads_var; + + n_onln = get_num_procs (); + if (n_onln > nthreads_var) + n_onln = nthreads_var; + + loadavg = 0; +#ifdef HAVE_GETLOADAVG + { + double dloadavg[3]; + if (getloadavg (dloadavg, 3) == 3) + { + /* Add 0.1 to get a kind of biased rounding. */ + loadavg = dloadavg[2] + 0.1; + } + } +#endif + + if (loadavg >= n_onln) + return 1; + else + return n_onln - loadavg; +} + +int +omp_get_num_procs (void) +{ + return get_num_procs (); +} + +ialias (omp_get_num_procs) diff --git a/libgomp/config/linux/ptrlock.c b/libgomp/config/linux/ptrlock.c new file mode 100644 index 000000000..7c2ad6192 --- /dev/null +++ b/libgomp/config/linux/ptrlock.c @@ -0,0 +1,67 @@ +/* Copyright (C) 2008, 2009 Free Software Foundation, Inc. + Contributed by Jakub Jelinek <jakub@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* This is a Linux specific implementation of a mutex synchronization + mechanism for libgomp. This type is private to the library. This + implementation uses atomic instructions and the futex syscall. */ + +#include <endian.h> +#include <limits.h> +#include "wait.h" + +void * +gomp_ptrlock_get_slow (gomp_ptrlock_t *ptrlock) +{ + int *intptr; + __sync_bool_compare_and_swap (ptrlock, 1, 2); + + /* futex works on ints, not pointers. + But a valid work share pointer will be at least + 8 byte aligned, so it is safe to assume the low + 32-bits of the pointer won't contain values 1 or 2. */ + __asm volatile ("" : "=r" (intptr) : "0" (ptrlock)); +#if __BYTE_ORDER == __BIG_ENDIAN + if (sizeof (*ptrlock) > sizeof (int)) + intptr += (sizeof (*ptrlock) / sizeof (int)) - 1; +#endif + do + do_wait (intptr, 2); + while (*intptr == 2); + __asm volatile ("" : : : "memory"); + return *ptrlock; +} + +void +gomp_ptrlock_set_slow (gomp_ptrlock_t *ptrlock, void *ptr) +{ + int *intptr; + + *ptrlock = ptr; + __asm volatile ("" : "=r" (intptr) : "0" (ptrlock)); +#if __BYTE_ORDER == __BIG_ENDIAN + if (sizeof (*ptrlock) > sizeof (int)) + intptr += (sizeof (*ptrlock) / sizeof (int)) - 1; +#endif + futex_wake (intptr, INT_MAX); +} diff --git a/libgomp/config/linux/ptrlock.h b/libgomp/config/linux/ptrlock.h new file mode 100644 index 000000000..97a3a1ad9 --- /dev/null +++ b/libgomp/config/linux/ptrlock.h @@ -0,0 +1,62 @@ +/* Copyright (C) 2008, 2009 Free Software Foundation, Inc. + Contributed by Jakub Jelinek <jakub@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* This is a Linux specific implementation of a mutex synchronization + mechanism for libgomp. This type is private to the library. This + implementation uses atomic instructions and the futex syscall. */ + +#ifndef GOMP_PTRLOCK_H +#define GOMP_PTRLOCK_H 1 + +typedef void *gomp_ptrlock_t; + +static inline void gomp_ptrlock_init (gomp_ptrlock_t *ptrlock, void *ptr) +{ + *ptrlock = ptr; +} + +extern void *gomp_ptrlock_get_slow (gomp_ptrlock_t *ptrlock); +static inline void *gomp_ptrlock_get (gomp_ptrlock_t *ptrlock) +{ + if ((uintptr_t) *ptrlock > 2) + return *ptrlock; + + if (__sync_bool_compare_and_swap (ptrlock, NULL, (uintptr_t) 1)) + return NULL; + + return gomp_ptrlock_get_slow (ptrlock); +} + +extern void gomp_ptrlock_set_slow (gomp_ptrlock_t *ptrlock, void *ptr); +static inline void gomp_ptrlock_set (gomp_ptrlock_t *ptrlock, void *ptr) +{ + if (!__sync_bool_compare_and_swap (ptrlock, (uintptr_t) 1, ptr)) + gomp_ptrlock_set_slow (ptrlock, ptr); +} + +static inline void gomp_ptrlock_destroy (gomp_ptrlock_t *ptrlock) +{ +} + +#endif /* GOMP_PTRLOCK_H */ diff --git a/libgomp/config/linux/s390/futex.h b/libgomp/config/linux/s390/futex.h new file mode 100644 index 000000000..060032d73 --- /dev/null +++ b/libgomp/config/linux/s390/futex.h @@ -0,0 +1,84 @@ +/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. + Contributed by Jakub Jelinek <jakub@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* Provide target-specific access to the futex system call. */ + +#include <sys/syscall.h> + +static inline long +sys_futex0 (int *addr, int op, int val) +{ + register long int gpr2 __asm__ ("2"); + register long int gpr3 __asm__ ("3"); + register long int gpr4 __asm__ ("4"); + register long int gpr5 __asm__ ("5"); + + gpr2 = (long) addr; + gpr3 = op; + gpr4 = val; + gpr5 = 0; + + __asm volatile ("svc %b1" + : "=d" (gpr2) + : "i" (SYS_futex), + "0" (gpr2), "d" (gpr3), "d" (gpr4), "d" (gpr5) + : "memory"); + return gpr2; +} + +static inline void +futex_wait (int *addr, int val) +{ + long err = sys_futex0 (addr, gomp_futex_wait, val); + if (__builtin_expect (err == -ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wait, val); + } +} + +static inline void +futex_wake (int *addr, int count) +{ + long err = sys_futex0 (addr, gomp_futex_wake, count); + if (__builtin_expect (err == -ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wake, count); + } +} + +static inline void +cpu_relax (void) +{ + __asm volatile ("" : : : "memory"); +} + +static inline void +atomic_write_barrier (void) +{ + __sync_synchronize (); +} diff --git a/libgomp/config/linux/sem.c b/libgomp/config/linux/sem.c new file mode 100644 index 000000000..ea981024c --- /dev/null +++ b/libgomp/config/linux/sem.c @@ -0,0 +1,61 @@ +/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. + Contributed by Richard Henderson <rth@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* This is a Linux specific implementation of a semaphore synchronization + mechanism for libgomp. This type is private to the library. This + implementation uses atomic instructions and the futex syscall. */ + +#include "wait.h" + + +void +gomp_sem_wait_slow (gomp_sem_t *sem) +{ + while (1) + { + int val = __sync_val_compare_and_swap (sem, 0, -1); + if (val > 0) + { + if (__sync_bool_compare_and_swap (sem, val, val - 1)) + return; + } + do_wait (sem, -1); + } +} + +void +gomp_sem_post_slow (gomp_sem_t *sem) +{ + int old, tmp = *sem, wake; + + do + { + old = tmp; + wake = old > 0 ? old + 1 : 1; + tmp = __sync_val_compare_and_swap (sem, old, wake); + } + while (old != tmp); + + futex_wake (sem, wake); +} diff --git a/libgomp/config/linux/sem.h b/libgomp/config/linux/sem.h new file mode 100644 index 000000000..9b0f0f82b --- /dev/null +++ b/libgomp/config/linux/sem.h @@ -0,0 +1,57 @@ +/* Copyright (C) 2005, 2009 Free Software Foundation, Inc. + Contributed by Richard Henderson <rth@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* This is a Linux specific implementation of a semaphore synchronization + mechanism for libgomp. This type is private to the library. This + implementation uses atomic instructions and the futex syscall. */ + +#ifndef GOMP_SEM_H +#define GOMP_SEM_H 1 + +typedef int gomp_sem_t; + +static inline void gomp_sem_init (gomp_sem_t *sem, int value) +{ + *sem = value; +} + +extern void gomp_sem_wait_slow (gomp_sem_t *); +static inline void gomp_sem_wait (gomp_sem_t *sem) +{ + if (!__sync_bool_compare_and_swap (sem, 1, 0)) + gomp_sem_wait_slow (sem); +} + +extern void gomp_sem_post_slow (gomp_sem_t *); +static inline void gomp_sem_post (gomp_sem_t *sem) +{ + if (!__sync_bool_compare_and_swap (sem, 0, 1)) + gomp_sem_post_slow (sem); +} + +static inline void gomp_sem_destroy (gomp_sem_t *sem) +{ +} + +#endif /* GOMP_SEM_H */ diff --git a/libgomp/config/linux/sparc/futex.h b/libgomp/config/linux/sparc/futex.h new file mode 100644 index 000000000..214b786f7 --- /dev/null +++ b/libgomp/config/linux/sparc/futex.h @@ -0,0 +1,108 @@ +/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. + Contributed by Jakub Jelinek <jakub@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* Provide target-specific access to the futex system call. */ + +#include <sys/syscall.h> + +static inline long +sys_futex0 (int *addr, int op, int val) +{ + register long int g1 __asm__ ("g1"); + register long int o0 __asm__ ("o0"); + register long int o1 __asm__ ("o1"); + register long int o2 __asm__ ("o2"); + register long int o3 __asm__ ("o3"); + + g1 = SYS_futex; + o0 = (long) addr; + o1 = op; + o2 = val; + o3 = 0; + +#ifdef __arch64__ +# define SYSCALL_STRING "ta\t0x6d; bcs,a,pt %%xcc, 1f; sub %%g0, %%o0, %%o0; 1:" +#else +# define SYSCALL_STRING "ta\t0x10; bcs,a 1f; sub %%g0, %%o0, %%o0; 1:" +#endif + + __asm volatile (SYSCALL_STRING + : "=r" (g1), "=r" (o0) + : "0" (g1), "1" (o0), "r" (o1), "r" (o2), "r" (o3) + : "g2", "g3", "g4", "g5", "g6", + "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", + "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", + "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", + "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", +#ifdef __arch64__ + "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46", + "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62", +#endif + "cc", "memory"); + return o0; +} + +static inline void +futex_wait (int *addr, int val) +{ + long err = sys_futex0 (addr, gomp_futex_wait, val); + if (__builtin_expect (err == ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wait, val); + } +} + +static inline void +futex_wake (int *addr, int count) +{ + long err = sys_futex0 (addr, gomp_futex_wake, count); + if (__builtin_expect (err == ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wake, count); + } +} + +static inline void +cpu_relax (void) +{ +#if defined __arch64__ || defined __sparc_v9__ + __asm volatile ("membar #LoadLoad" : : : "memory"); +#else + __asm volatile ("" : : : "memory"); +#endif +} + +static inline void +atomic_write_barrier (void) +{ +#if defined __arch64__ || defined __sparc_v9__ + __asm volatile ("membar #StoreStore" : : : "memory"); +#else + __sync_synchronize (); +#endif +} diff --git a/libgomp/config/linux/wait.h b/libgomp/config/linux/wait.h new file mode 100644 index 000000000..0e8abf111 --- /dev/null +++ b/libgomp/config/linux/wait.h @@ -0,0 +1,65 @@ +/* Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc. + Contributed by Jakub Jelinek <jakub@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* This is a Linux specific implementation of a mutex synchronization + mechanism for libgomp. This type is private to the library. This + implementation uses atomic instructions and the futex syscall. */ + +#ifndef GOMP_WAIT_H +#define GOMP_WAIT_H 1 + +#include "libgomp.h" +#include <errno.h> + +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 +#define FUTEX_PRIVATE_FLAG 128L + +#ifdef HAVE_ATTRIBUTE_VISIBILITY +# pragma GCC visibility push(hidden) +#endif + +extern long int gomp_futex_wait, gomp_futex_wake; + +#include <futex.h> + +static inline void do_wait (int *addr, int val) +{ + unsigned long long i, count = gomp_spin_count_var; + + if (__builtin_expect (gomp_managed_threads > gomp_available_cpus, 0)) + count = gomp_throttled_spin_count_var; + for (i = 0; i < count; i++) + if (__builtin_expect (*addr != val, 0)) + return; + else + cpu_relax (); + futex_wait (addr, val); +} + +#ifdef HAVE_ATTRIBUTE_VISIBILITY +# pragma GCC visibility pop +#endif + +#endif /* GOMP_WAIT_H */ diff --git a/libgomp/config/linux/x86/futex.h b/libgomp/config/linux/x86/futex.h new file mode 100644 index 000000000..cb7461d89 --- /dev/null +++ b/libgomp/config/linux/x86/futex.h @@ -0,0 +1,153 @@ +/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. + Contributed by Richard Henderson <rth@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* Provide target-specific access to the futex system call. */ + +#ifdef __LP64__ +# ifndef SYS_futex +# define SYS_futex 202 +# endif + +static inline void +futex_wait (int *addr, int val) +{ + register long r10 __asm__("%r10"); + long res; + + r10 = 0; + __asm volatile ("syscall" + : "=a" (res) + : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wait), + "d" (val), "r" (r10) + : "r11", "rcx", "memory"); + if (__builtin_expect (res == -ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + r10 = 0; + __asm volatile ("syscall" + : "=a" (res) + : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wait), + "d" (val), "r" (r10) + : "r11", "rcx", "memory"); + } +} + +static inline void +futex_wake (int *addr, int count) +{ + long res; + + __asm volatile ("syscall" + : "=a" (res) + : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wake), + "d" (count) + : "r11", "rcx", "memory"); + if (__builtin_expect (res == -ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + __asm volatile ("syscall" + : "=a" (res) + : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wake), + "d" (count) + : "r11", "rcx", "memory"); + } +} +#else +# ifndef SYS_futex +# define SYS_futex 240 +# endif + +# ifdef __PIC__ + +static inline long +sys_futex0 (int *addr, int op, int val) +{ + long res; + + __asm volatile ("xchgl\t%%ebx, %2\n\t" + "int\t$0x80\n\t" + "xchgl\t%%ebx, %2" + : "=a" (res) + : "0"(SYS_futex), "r" (addr), "c"(op), + "d"(val), "S"(0) + : "memory"); + return res; +} + +# else + +static inline long +sys_futex0 (int *addr, int op, int val) +{ + long res; + + __asm volatile ("int $0x80" + : "=a" (res) + : "0"(SYS_futex), "b" (addr), "c"(op), + "d"(val), "S"(0) + : "memory"); + return res; +} + +# endif /* __PIC__ */ + +static inline void +futex_wait (int *addr, int val) +{ + long res = sys_futex0 (addr, gomp_futex_wait, val); + if (__builtin_expect (res == -ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wait, val); + } +} + +static inline void +futex_wake (int *addr, int count) +{ + long res = sys_futex0 (addr, gomp_futex_wake, count); + if (__builtin_expect (res == -ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wake, count); + } +} + +#endif /* __LP64__ */ + +static inline void +cpu_relax (void) +{ + __asm volatile ("rep; nop" : : : "memory"); +} + +static inline void +atomic_write_barrier (void) +{ + __sync_synchronize (); +} |