/*
* Copyright (C) Igor Sysoev
*/
#ifndef _NGX_ATOMIC_H_INCLUDED_
#define _NGX_ATOMIC_H_INCLUDED_
#include <ngx_config.h>
#include <ngx_core.h>
#if (NGX_HAVE_LIBATOMIC)
#define AO_REQUIRE_CAS
#include <atomic_ops.h>
#define NGX_HAVE_ATOMIC_OPS 1
typedef long ngx_atomic_int_t;
typedef AO_t ngx_atomic_uint_t;
typedef volatile ngx_atomic_uint_t ngx_atomic_t;
#if (NGX_PTR_SIZE == 8)
#define NGX_ATOMIC_T_LEN (sizeof("-9223372036854775808") - 1)
#else
#define NGX_ATOMIC_T_LEN (sizeof("-2147483648") - 1)
#endif
#define ngx_atomic_cmp_set(lock, old, new) \
AO_compare_and_swap(lock, old, new)
#define ngx_atomic_fetch_add(value, add) \
AO_fetch_and_add(value, add)
#define ngx_memory_barrier() AO_nop()
#define ngx_cpu_pause()
#elif (NGX_DARWIN_ATOMIC)
/*
* use Darwin 8 atomic(3) and barrier(3) operations
* optimized at run-time for UP and SMP
*/
#include <libkern/OSAtomic.h>
/* "bool" conflicts with perl's CORE/handy.h */
#undef bool
#define NGX_HAVE_ATOMIC_OPS 1
#if (NGX_PTR_SIZE == 8)
typedef int64_t ngx_atomic_int_t;
typedef uint64_t ngx_atomic_uint_t;
#define NGX_ATOMIC_T_LEN (sizeof("-9223372036854775808") - 1)
#define ngx_atomic_cmp_set(lock, old, new) \
OSAtomicCompareAndSwap64Barrier(old, new, (int64_t *) lock)
#define ngx_atomic_fetch_add(value, add) \
(OSAtomicAdd64(add, (int64_t *) value) - add)
#else
typedef int32_t ngx_atomic_int_t;
typedef uint32_t ngx_atomic_uint_t;
#define NGX_ATOMIC_T_LEN (sizeof("-2147483648") - 1)
#define ngx_atomic_cmp_set(lock, old, new) \
OSAtomicCompareAndSwap32Barrier(old, new, (int32_t *) lock)
#define ngx_atomic_fetch_add(value, add) \
(OSAtomicAdd32(add, (int32_t *) value) - add)
#endif
#define ngx_memory_barrier() OSMemoryBarrier()
#define ngx_cpu_pause()
typedef volatile ngx_atomic_uint_t ngx_atomic_t;
#elif (NGX_HAVE_GCC_ATOMIC)
/* GCC 4.1 builtin atomic operations */
#define NGX_HAVE_ATOMIC_OPS 1
typedef long ngx_atomic_int_t;
typedef unsigned long ngx_atomic_uint_t;
#if (NGX_PTR_SIZE == 8)
#define NGX_ATOMIC_T_LEN (sizeof("-9223372036854775808") - 1)
#else
#define NGX_ATOMIC_T_LEN (sizeof("-2147483648") - 1)
#endif
typedef volatile ngx_atomic_uint_t ngx_atomic_t;
#define ngx_atomic_cmp_set(lock, old, set) \
__sync_bool_compare_and_swap(lock, old, set)
#define ngx_atomic_fetch_add(value, add) \
__sync_fetch_and_add(value, add)
#define ngx_memory_barrier() __sync_synchronize()
#if ( __i386__ || __i386 || __amd64__ || __amd64 )
#define ngx_cpu_pause() __asm__ ("pause")
#else
#define ngx_cpu_pause()
#endif
#elif ( __i386__ || __i386 )
typedef int32_t ngx_atomic_int_t;
typedef uint32_t ngx_atomic_uint_t;
typedef volatile ngx_atomic_uint_t ngx_atomic_t;
#define NGX_ATOMIC_T_LEN (sizeof("-2147483648") - 1)
#if ( __SUNPRO_C )
#define NGX_HAVE_ATOMIC_OPS 1
ngx_atomic_uint_t
ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
ngx_atomic_uint_t set);
ngx_atomic_int_t
ngx_atomic_fetch_add(ngx_atomic_t *value, ngx_atomic_int_t add);
/*
* Sun Studio 12 exits with segmentation fault on '__asm ("pause")',
* so ngx_cpu_pause is declared in src/os/unix/ngx_sunpro_x86.il
*/
void
ngx_cpu_pause(void);
/* the code in src/os/unix/ngx_sunpro_x86.il */
#define ngx_memory_barrier() __asm (".volatile"); __asm (".nonvolatile")
#else /* ( __GNUC__ || __INTEL_COMPILER ) */
#define NGX_HAVE_ATOMIC_OPS 1
#include "ngx_gcc_atomic_x86.h"
#endif
#elif ( __amd64__ || __amd64 )
typedef int64_t ngx_atomic_int_t;
typedef uint64_t ngx_atomic_uint_t;
typedef volatile ngx_atomic_uint_t ngx_atomic_t;
#define NGX_ATOMIC_T_LEN (sizeof("-9223372036854775808") - 1)
#if ( __SUNPRO_C )
#define NGX_HAVE_ATOMIC_OPS 1
ngx_atomic_uint_t
ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
ngx_atomic_uint_t set);
ngx_atomic_int_t
ngx_atomic_fetch_add(ngx_atomic_t *value, ngx_atomic_int_t add);
/*
* Sun Studio 12 exits with segmentation fault on '__asm ("pause")',
* so ngx_cpu_pause is declared in src/os/unix/ngx_sunpro_amd64.il
*/
void
ngx_cpu_pause(void);
/* the code in src/os/unix/ngx_sunpro_amd64.il */
#define ngx_memory_barrier() __asm (".volatile"); __asm (".nonvolatile")
#else /* ( __GNUC__ || __INTEL_COMPILER ) */
#define NGX_HAVE_ATOMIC_OPS 1
#include "ngx_gcc_atomic_amd64.h"
#endif
#elif ( __sparc__ || __sparc || __sparcv9 )
#if (NGX_PTR_SIZE == 8)
typedef int64_t ngx_atomic_int_t;
typedef uint64_t ngx_atomic_uint_t;
#define NGX_ATOMIC_T_LEN (sizeof("-9223372036854775808") - 1)
#else
typedef int32_t ngx_atomic_int_t;
typedef uint32_t ngx_atomic_uint_t;
#define NGX_ATOMIC_T_LEN (sizeof("-2147483648") - 1)
#endif
typedef volatile ngx_atomic_uint_t ngx_atomic_t;
#if ( __SUNPRO_C )
#define NGX_HAVE_ATOMIC_OPS 1
#include "ngx_sunpro_atomic_sparc64.h"
#else /* ( __GNUC__ || __INTEL_COMPILER ) */
#define NGX_HAVE_ATOMIC_OPS 1
#include "ngx_gcc_atomic_sparc64.h"
#endif
#elif ( __powerpc__ || __POWERPC__ )
#define NGX_HAVE_ATOMIC_OPS 1
#if (NGX_PTR_SIZE == 8)
typedef int64_t ngx_atomic_int_t;
typedef uint64_t ngx_atomic_uint_t;
#define NGX_ATOMIC_T_LEN (sizeof("-9223372036854775808") - 1)
#else
typedef int32_t ngx_atomic_int_t;
typedef uint32_t ngx_atomic_uint_t;
#define NGX_ATOMIC_T_LEN (sizeof("-2147483648") - 1)
#endif
typedef volatile ngx_atomic_uint_t ngx_atomic_t;
#include "ngx_gcc_atomic_ppc.h"
#endif
#if !(NGX_HAVE_ATOMIC_OPS)
#define NGX_HAVE_ATOMIC_OPS 0
typedef int32_t ngx_atomic_int_t;
typedef uint32_t ngx_atomic_uint_t;
typedef volatile ngx_atomic_uint_t ngx_atomic_t;
#define NGX_ATOMIC_T_LEN (sizeof("-2147483648") - 1)
static ngx_inline ngx_atomic_uint_t
ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
ngx_atomic_uint_t set)
{
if (*lock == old) {
*lock = set;
return 1;
}
return 0;
}
static ngx_inline ngx_atomic_int_t
ngx_atomic_fetch_add(ngx_atomic_t *value, ngx_atomic_int_t add)
{
ngx_atomic_int_t old;
old = *value;
*value += add;
return old;
}
#define ngx_memory_barrier()
#define ngx_cpu_pause()
#endif
void ngx_spinlock(ngx_atomic_t *lock, ngx_atomic_int_t value, ngx_uint_t spin);
#define ngx_trylock(lock) (*(lock) == 0 && ngx_atomic_cmp_set(lock, 0, 1))
#define ngx_unlock(lock) *(lock) = 0
#endif /* _NGX_ATOMIC_H_INCLUDED_ */
/*
* Copyright (C) Igor Sysoev
*/
#if (NGX_SMP)
#define NGX_SMP_LOCK "lock;"
#else
#define NGX_SMP_LOCK
#endif
/*
* "cmpxchgl r, [m]":
*
* if (eax == [m]) {
* zf = 1;
* [m] = r;
* } else {
* zf = 0;
* eax = [m];
* }
*
*
* The "r" means the general register.
* The "=a" and "a" are the %eax register.
* Although we can return result in any register, we use "a" because it is
* used in cmpxchgl anyway. The result is actually in %al but not in %eax,
* however, as the code is inlined gcc can test %al as well as %eax,
* and icc adds "movzbl %al, %eax" by itself.
*
* The "cc" means that flags were changed.
int cmpxchgl(fjutex, old, new)
{
__asm__ __volatile__(LOCK_PREFIX "cmpxchgl %k1,%2"
: "=a"(futex)
: "q"(new), "m"(*__xg(ptr)), "0"(old)
: "memory");
}
等价为:
eax = old;
if (futex == old) {
futex = new;
return old;
} else {
return futex;
}
锁操作:
cmpxchgl(futex, 0, 1);
如果futex没有加锁,那么对futex置1,并返回0表示原来没有加锁;
如果futex已经被加锁,那么返回1表示原来已经加锁了;
libc里会判断返回值:
如果发现futex已经被加锁,则调用futex_wait;
如果发现futex没有被加锁,则直接返回;
*/
static ngx_inline ngx_atomic_uint_t
ngx_atomic_cmp_set( ngx_atomic_t *lock, ngx_atomic_uint_t old,
ngx_atomic_uint_t set )
{
u_char res;
__asm__ volatile (
NGX_SMP_LOCK
" cmpxchgl %3, %1; "
" sete %0; "
: "=a" (res) : "m" (*lock), "a" (old), "r" (set) : "cc", "memory");
return res;
}
/*
* "xaddl r, [m]":
*
* temp = [m];
* [m] += r;
* r = temp;
*
*
* The "+r" means the general register.
* The "cc" means that flags were changed.
*/
#if !(( __GNUC__ == 2 && __GNUC_MINOR__ <= 7 ) || ( __INTEL_COMPILER >= 800 ))
/*
* icc 8.1 and 9.0 compile broken code with -march=pentium4 option:
* ngx_atomic_fetch_add() always return the input "add" value,
* so we use the gcc 2.7 version.
*
* icc 8.1 and 9.0 with -march=pentiumpro option or icc 7.1 compile
* correct code.
*/
static ngx_inline ngx_atomic_int_t
ngx_atomic_fetch_add(ngx_atomic_t *value, ngx_atomic_int_t add)
{
__asm__ volatile (
NGX_SMP_LOCK
" xaddl %0, %1; "
: "+r" (add) : "m" (*value) : "cc", "memory");
return add;
}
#else
/*
* gcc 2.7 does not support "+r", so we have to use the fixed
* %eax ("=a" and "a") and this adds two superfluous instructions in the end
* of code, something like this: "mov %eax, %edx / mov %edx, %eax".
*/
static ngx_inline ngx_atomic_int_t
ngx_atomic_fetch_add(ngx_atomic_t *value, ngx_atomic_int_t add)
{
ngx_atomic_uint_t old;
__asm__ volatile (
NGX_SMP_LOCK
" xaddl %2, %1; "
: "=a" (old) : "m" (*value), "a" (add) : "cc", "memory");
return old;
}
#endif
/*
* on x86 the write operations go in a program order, so we need only
* to disable the gcc reorder optimizations
*/
#define ngx_memory_barrier() __asm__ volatile ("" ::: "memory")
/* old "as" does not support "pause" opcode */
#define ngx_cpu_pause() __asm__ (".byte 0xf3, 0x90")