Define CACHE_LINE_SIZE in ut0counter.h to 128 on POWER Index: mysql-5.6.17/storage/innobase/include/ut0counter.h =================================================================== --- mysql-5.6.17.orig/storage/innobase/include/ut0counter.h +++ mysql-5.6.17/storage/innobase/include/ut0counter.h @@ -32,7 +32,11 @@ Created 2012/04/12 by Sunny Bains #include "os0thread.h" /** CPU cache line size */ +#ifdef __powerpc__ +#define CACHE_LINE_SIZE 128 +#else #define CACHE_LINE_SIZE 64 +#endif /** Default number of slots to use in ib_counter_t */ #define IB_N_SLOTS 64 InnoDB mutex/rw_lock should be conscious about memory ordering other than Intel Patch by Yasufumi, attached to: http://bugs.mysql.com/bug.php?id=47213 === modified file 'storage/innobase/CMakeLists.txt' --- a/storage/innobase/CMakeLists.txt revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ b/storage/innobase/CMakeLists.txt 2014-05-12 02:13:32 +0000 @@ -117,6 +117,25 @@ IF(NOT CMAKE_CROSSCOMPILING) }" HAVE_IB_GCC_ATOMIC_BUILTINS_64 ) + CHECK_C_SOURCE_RUNS( + "#include + int main() + { + __sync_synchronize(); + return(0); + }" + HAVE_IB_GCC_SYNC_SYNCHRONISE + ) + CHECK_C_SOURCE_RUNS( + "#include + int main() + { + __atomic_thread_fence(__ATOMIC_ACQUIRE); + __atomic_thread_fence(__ATOMIC_RELEASE); + return(0); + }" + HAVE_IB_GCC_ATOMIC_THREAD_FENCE + ) ENDIF() IF(HAVE_IB_GCC_ATOMIC_BUILTINS) @@ -127,6 +146,14 @@ IF(HAVE_IB_GCC_ATOMIC_BUILTINS_64) ADD_DEFINITIONS(-DHAVE_IB_GCC_ATOMIC_BUILTINS_64=1) ENDIF() +IF(HAVE_IB_GCC_SYNC_SYNCHRONISE) + ADD_DEFINITIONS(-DHAVE_IB_GCC_SYNC_SYNCHRONISE=1) +ENDIF() + +IF(HAVE_IB_GCC_ATOMIC_THREAD_FENCE) + ADD_DEFINITIONS(-DHAVE_IB_GCC_ATOMIC_THREAD_FENCE=1) +ENDIF() + # either define HAVE_IB_ATOMIC_PTHREAD_T_GCC or not IF(NOT CMAKE_CROSSCOMPILING) CHECK_C_SOURCE_RUNS( @@ -208,10 +235,21 @@ IF(CMAKE_SYSTEM_NAME STREQUAL "SunOS") return(0); } " HAVE_IB_ATOMIC_PTHREAD_T_SOLARIS) + CHECK_C_SOURCE_COMPILES( + "#include + int main() { + __machine_r_barrier(); + __machine_w_barrier(); + return(0); + }" + HAVE_IB_MACHINE_BARRIER_SOLARIS) ENDIF() IF(HAVE_IB_ATOMIC_PTHREAD_T_SOLARIS) ADD_DEFINITIONS(-DHAVE_IB_ATOMIC_PTHREAD_T_SOLARIS=1) ENDIF() + IF(HAVE_IB_MACHINE_BARRIER_SOLARIS) + ADD_DEFINITIONS(-DHAVE_IB_MACHINE_BARRIER_SOLARIS=1) + ENDIF() ENDIF() === modified file 'storage/innobase/include/os0sync.h' --- a/storage/innobase/include/os0sync.h revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ b/storage/innobase/include/os0sync.h 2014-05-12 02:18:21 +0000 @@ -685,6 +685,44 @@ for synchronization */ os_decrement_counter_by_amount(mutex, counter, 1);\ } while (0); +/** barrier definitions for memory ordering */ +#ifdef HAVE_IB_GCC_ATOMIC_THREAD_FENCE +# define HAVE_MEMORY_BARRIER +# define os_rmb __atomic_thread_fence(__ATOMIC_ACQUIRE) +# define os_wmb __atomic_thread_fence(__ATOMIC_RELEASE) +# define IB_MEMORY_BARRIER_STARTUP_MSG \ + "GCC builtin __atomic_thread_fence() is used for memory barrier" + +#elif defined(HAVE_IB_GCC_SYNC_SYNCHRONISE) +# define HAVE_MEMORY_BARRIER +# define os_rmb __sync_synchronize() +# define os_wmb __sync_synchronize() +# define IB_MEMORY_BARRIER_STARTUP_MSG \ + "GCC builtin __sync_synchronize() is used for memory barrier" + +#elif defined(HAVE_IB_MACHINE_BARRIER_SOLARIS) +# define HAVE_MEMORY_BARRIER +# include +# define os_rmb __machine_r_barrier() +# define os_wmb __machine_w_barrier() +# define IB_MEMORY_BARRIER_STARTUP_MSG \ + "Soralis memory ordering functions are used for memory barrier" + +#elif defined(HAVE_WINDOWS_MM_FENCE) +# define HAVE_MEMORY_BARRIER +# include +# define os_rmb _mm_lfence() +# define os_wmb _mm_sfence() +# define IB_MEMORY_BARRIER_STARTUP_MSG \ + "_mm_lfence() and _mm_sfence() are used for memory barrier" + +#else +# define os_rmb +# define os_wmb +# define IB_MEMORY_BARRIER_STARTUP_MSG \ + "Memory barrier is not used" +#endif + #ifndef UNIV_NONINL #include "os0sync.ic" #endif === modified file 'storage/innobase/include/sync0rw.ic' --- a/storage/innobase/include/sync0rw.ic revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ b/storage/innobase/include/sync0rw.ic 2014-05-12 03:48:49 +0000 @@ -93,6 +93,7 @@ rw_lock_set_waiter_flag( (void) os_compare_and_swap_ulint(&lock->waiters, 0, 1); #else /* INNODB_RW_LOCKS_USE_ATOMICS */ lock->waiters = 1; + os_wmb; #endif /* INNODB_RW_LOCKS_USE_ATOMICS */ } @@ -110,6 +111,7 @@ rw_lock_reset_waiter_flag( (void) os_compare_and_swap_ulint(&lock->waiters, 1, 0); #else /* INNODB_RW_LOCKS_USE_ATOMICS */ lock->waiters = 0; + os_wmb; #endif /* INNODB_RW_LOCKS_USE_ATOMICS */ } @@ -199,7 +201,10 @@ rw_lock_lock_word_decr( ulint amount) /*!< in: amount to decrement */ { #ifdef INNODB_RW_LOCKS_USE_ATOMICS - lint local_lock_word = lock->lock_word; + lint local_lock_word; + + os_rmb; + local_lock_word = lock->lock_word; while (local_lock_word > 0) { if (os_compare_and_swap_lint(&lock->lock_word, local_lock_word, === modified file 'storage/innobase/include/sync0sync.ic' --- a/storage/innobase/include/sync0sync.ic revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ b/storage/innobase/include/sync0sync.ic 2014-05-12 03:21:51 +0000 @@ -92,6 +92,7 @@ ib_mutex_test_and_set( ut_a(mutex->lock_word == 0); mutex->lock_word = 1; + os_wmb; } return((byte) ret); === modified file 'storage/innobase/srv/srv0start.cc' --- a/storage/innobase/srv/srv0start.cc revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ b/storage/innobase/srv/srv0start.cc 2014-05-12 02:22:14 +0000 @@ -1647,6 +1647,18 @@ innobase_start_or_create_for_mysql(void) "" IB_ATOMICS_STARTUP_MSG ""); ib_logf(IB_LOG_LEVEL_INFO, + "" IB_MEMORY_BARRIER_STARTUP_MSG ""); + +#ifndef HAVE_MEMORY_BARRIER +#if defined __i386__ || defined __x86_64__ || defined _M_IX86 || defined _M_X64 +#else + ib_logf(IB_LOG_LEVEL_WARN, + "Built without memory barrier might cause problem" + " for this architecture."); +#endif /* IA32 or AMD64 */ +#endif /* HAVE_MEMORY_BARRIER */ + + ib_logf(IB_LOG_LEVEL_INFO, "Compressed tables use zlib " ZLIB_VERSION #ifdef UNIV_ZIP_DEBUG " with validation" === modified file 'storage/innobase/sync/sync0arr.cc' --- a/storage/innobase/sync/sync0arr.cc revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ b/storage/innobase/sync/sync0arr.cc 2014-05-12 03:25:10 +0000 @@ -740,6 +740,7 @@ sync_arr_cell_can_wake_up( mutex = static_cast(cell->wait_object); + os_rmb; if (mutex_get_lock_word(mutex) == 0) { return(TRUE); @@ -749,6 +750,7 @@ sync_arr_cell_can_wake_up( lock = static_cast(cell->wait_object); + os_rmb; if (lock->lock_word > 0) { /* Either unlocked or only read locked. */ @@ -760,6 +762,7 @@ sync_arr_cell_can_wake_up( lock = static_cast(cell->wait_object); /* lock_word == 0 means all readers have left */ + os_rmb; if (lock->lock_word == 0) { return(TRUE); @@ -768,6 +771,7 @@ sync_arr_cell_can_wake_up( lock = static_cast(cell->wait_object); /* lock_word > 0 means no writer or reserved writer */ + os_rmb; if (lock->lock_word > 0) { return(TRUE); === modified file 'storage/innobase/sync/sync0rw.cc' --- a/storage/innobase/sync/sync0rw.cc revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ b/storage/innobase/sync/sync0rw.cc 2014-05-12 04:16:49 +0000 @@ -381,15 +381,21 @@ rw_lock_s_lock_spin( lock_loop: /* Spin waiting for the writer field to become free */ + os_rmb; while (i < SYNC_SPIN_ROUNDS && lock->lock_word <= 0) { if (srv_spin_wait_delay) { ut_delay(ut_rnd_interval(0, srv_spin_wait_delay)); } +#ifdef HAVE_MEMORY_BARRIER + i += 2; + os_rmb; +#else i++; +#endif /* HAVE_MEMORY_BARRIER */ } - if (i == SYNC_SPIN_ROUNDS) { + if (i >= SYNC_SPIN_ROUNDS) { os_thread_yield(); } @@ -476,6 +482,7 @@ rw_lock_x_lock_wait( counter_index = (size_t) os_thread_get_curr_id(); + os_rmb; ut_ad(lock->lock_word <= 0); while (lock->lock_word < 0) { @@ -483,7 +490,12 @@ rw_lock_x_lock_wait( ut_delay(ut_rnd_interval(0, srv_spin_wait_delay)); } if(i < SYNC_SPIN_ROUNDS) { +#ifdef HAVE_MEMORY_BARRIER + i += 2; + os_rmb; +#else i++; +#endif /* HAVE_MEMORY_BARRIER */ continue; } @@ -560,6 +572,10 @@ rw_lock_x_lock_low( } else { os_thread_id_t thread_id = os_thread_get_curr_id(); + if (!pass) { + os_rmb; + } + /* Decrement failed: relock or failed lock */ if (!pass && lock->recursive && os_thread_eq(lock->writer_thread, thread_id)) { @@ -638,6 +654,7 @@ lock_loop: } /* Spin waiting for the lock_word to become free */ + os_rmb; while (i < SYNC_SPIN_ROUNDS && lock->lock_word <= 0) { if (srv_spin_wait_delay) { @@ -645,9 +662,14 @@ lock_loop: srv_spin_wait_delay)); } +#ifdef HAVE_MEMORY_BARRIER + i += 2; + os_rmb; +#else i++; +#endif /* HAVE_MEMORY_BARRIER */ } - if (i == SYNC_SPIN_ROUNDS) { + if (i >= SYNC_SPIN_ROUNDS) { os_thread_yield(); } else { goto lock_loop; === modified file 'storage/innobase/sync/sync0sync.cc' --- a/storage/innobase/sync/sync0sync.cc revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ b/storage/innobase/sync/sync0sync.cc 2014-05-12 03:39:01 +0000 @@ -458,6 +458,7 @@ mutex_set_waiters( *ptr = n; /* Here we assume that the write of a single word in memory is atomic */ + os_wmb; } /******************************************************************//** @@ -499,16 +500,21 @@ mutex_loop: a memory word. */ spin_loop: - + os_rmb; while (mutex_get_lock_word(mutex) != 0 && i < SYNC_SPIN_ROUNDS) { if (srv_spin_wait_delay) { ut_delay(ut_rnd_interval(0, srv_spin_wait_delay)); } +#ifdef HAVE_MEMORY_BARRIER + i += 2; + os_rmb; +#else i++; +#endif /* HAVE_MEMORY_BARRIER */ } - if (i == SYNC_SPIN_ROUNDS) { + if (i >= SYNC_SPIN_ROUNDS) { os_thread_yield(); } Fix test and set/release of InnoDB mutex lock_word likely due to missing a memory barrier, this was using test and set rather than sync lock release. Also add memory barrier in for get lock word. Index: mysql-5.6.17/storage/innobase/include/sync0sync.ic =================================================================== --- mysql-5.6.17.orig/storage/innobase/include/sync0sync.ic +++ mysql-5.6.17/storage/innobase/include/sync0sync.ic @@ -80,7 +80,7 @@ ib_mutex_test_and_set( ib_mutex_t* mutex) /*!< in: mutex */ { #if defined(HAVE_ATOMIC_BUILTINS) - return(os_atomic_test_and_set_byte(&mutex->lock_word, 1)); + return(__sync_lock_test_and_set(&mutex->lock_word, 1)); #else ibool ret; @@ -112,7 +112,7 @@ mutex_reset_lock_word( /* In theory __sync_lock_release should be used to release the lock. Unfortunately, it does not work properly alone. The workaround is that more conservative __sync_lock_test_and_set is used instead. */ - os_atomic_test_and_set_byte(&mutex->lock_word, 0); + __sync_lock_release(&mutex->lock_word); #else mutex->lock_word = 0; @@ -130,6 +130,7 @@ mutex_get_lock_word( { ut_ad(mutex); + os_rmb; return(mutex->lock_word); } memory barrier at end of setting up innodb aio we need a memory barrier here as otherwise when IO threads start they can get a stale view of the data structure. It turns out that POWER8 is fast enough with large enough caches that this can actually be a problem. You can reproduce this by attempting to do a lot of writes shortly after server startup - you'll hit a really odd assert. Index: mysql-5.6.17/storage/innobase/os/os0file.cc =================================================================== --- mysql-5.6.17.orig/storage/innobase/os/os0file.cc +++ mysql-5.6.17/storage/innobase/os/os0file.cc @@ -3782,7 +3782,7 @@ skip_native_aio: slot->ret = 0; #endif /* WIN_ASYNC_IO */ } - + __sync_synchronize(); return(array); } On POWER, you can set thread priority rather than Intel relax cpu instruction So the correct way to write a spin loop for POWER is to lower the thread priority while you spin and raise it up afterwards. We add the asm volatile("":::"memory"); bit of asm to make GCC not optimize the spin loop down to a nop. This is a GCC barrier rather than a CPU memory barrier - and is arguably needed for x86 as well Index: mysql-5.6.17/storage/innobase/sync/sync0rw.cc =================================================================== --- mysql-5.6.17.orig/storage/innobase/sync/sync0rw.cc 2014-05-26 12:16:25.622071001 +1000 +++ mysql-5.6.17/storage/innobase/sync/sync0rw.cc 2014-05-26 17:37:14.702071000 +1000 @@ -382,6 +382,7 @@ /* Spin waiting for the writer field to become free */ os_rmb; + asm volatile ("or 1,1,1"); while (i < SYNC_SPIN_ROUNDS && lock->lock_word <= 0) { if (srv_spin_wait_delay) { ut_delay(ut_rnd_interval(0, srv_spin_wait_delay)); @@ -393,7 +394,9 @@ #else i++; #endif /* HAVE_MEMORY_BARRIER */ + asm volatile ("":::"memory"); } + asm volatile ("or 2,2,2"); if (i >= SYNC_SPIN_ROUNDS) { os_thread_yield(); Index: mysql-5.6.17/storage/innobase/sync/sync0sync.cc =================================================================== --- mysql-5.6.17.orig/storage/innobase/sync/sync0sync.cc 2014-05-26 12:16:25.622071001 +1000 +++ mysql-5.6.17/storage/innobase/sync/sync0sync.cc 2014-05-26 17:37:52.152071000 +1000 @@ -501,6 +501,7 @@ spin_loop: os_rmb; + asm volatile("or 1,1,1"); while (mutex_get_lock_word(mutex) != 0 && i < SYNC_SPIN_ROUNDS) { if (srv_spin_wait_delay) { ut_delay(ut_rnd_interval(0, srv_spin_wait_delay)); @@ -512,7 +513,9 @@ #else i++; #endif /* HAVE_MEMORY_BARRIER */ + asm volatile("":::"memory"); } + asm volatile ("or 2,2,2;"); if (i >= SYNC_SPIN_ROUNDS) { os_thread_yield();