MB-29515: Replace SpinLock with std::mutex

This replaces SpinLock with std::mutex for all uses apart from
Ephemeral's SequenceList[1].

Testing of this shows a ~5% improvement in throughput under daily perf.

[1] Still need to test the effect of this change on Ephemeral, to see
if it's a suitable change to make.

Change-Id: I04d3b78bc34e2f880690b1b3e341decf8668ad09
Reviewed-on: http://review.couchbase.org/93694
Tested-by: Build Bot <[email protected]>
Reviewed-by: Trond Norbye <[email protected]>

Author: daverigby

engines/ep/src/connmap.cc

@@ -230,7 +230,7 @@ void ConnMap::addVBConnByVBId(std::shared_ptr<ConnHandler> conn, int16_t vbid) {
     }
 
     size_t lock_num = vbid % vbConnLockNum;
-    std::lock_guard<SpinLock> lh(vbConnLocks[lock_num]);
+    std::lock_guard<std::mutex> lh(vbConnLocks[lock_num]);
     vbConns[vbid].push_back(conn);
 }
 
@@ -247,6 +247,6 @@ void ConnMap::removeVBConnByVBId_UNLOCKED(const void* connCookie,
 
 void ConnMap::removeVBConnByVBId(const void* connCookie, int16_t vbid) {
     size_t lock_num = vbid % vbConnLockNum;
-    std::lock_guard<SpinLock> lh(vbConnLocks[lock_num]);
+    std::lock_guard<std::mutex> lh(vbConnLocks[lock_num]);
     removeVBConnByVBId_UNLOCKED(connCookie, vbid);
 }

engines/ep/src/connmap.h

@@ -104,7 +104,7 @@ class ConnMap {
             std::unordered_map<const void*, std::shared_ptr<ConnHandler>>;
     CookieToConnectionMap map_;
 
-    std::vector<SpinLock> vbConnLocks;
+    std::vector<std::mutex> vbConnLocks;
     std::vector<std::list<std::shared_ptr<ConnHandler>>> vbConns;
 
     /* Handle to the engine who owns us */

engines/ep/src/dcp/dcpconnmap.cc

@@ -223,7 +223,7 @@ void DcpConnMap::closeStreamsDueToRollback(uint16_t vbucket) {
 bool DcpConnMap::handleSlowStream(uint16_t vbid,
                                   const std::string &name) {
     size_t lock_num = vbid % vbConnLockNum;
-    std::lock_guard<SpinLock> lh(vbConnLocks[lock_num]);
+    std::lock_guard<std::mutex> lh(vbConnLocks[lock_num]);
     std::list<std::shared_ptr<ConnHandler>>& vb_conns = vbConns[vbid];
 
     for (auto itr = vb_conns.begin(); itr != vb_conns.end(); ++itr) {
@@ -368,7 +368,7 @@ void DcpConnMap::manageConnections() {
 void DcpConnMap::removeVBConnections(DcpProducer& prod) {
     for (const auto vbid : prod.getVBVector()) {
         size_t lock_num = vbid % vbConnLockNum;
-        std::lock_guard<SpinLock> lh(vbConnLocks[lock_num]);
+        std::lock_guard<std::mutex> lh(vbConnLocks[lock_num]);
         std::list<std::shared_ptr<ConnHandler>>& vb_conns = vbConns[vbid];
         for (auto itr = vb_conns.begin(); itr != vb_conns.end(); ++itr) {
             if (prod.getCookie() == (*itr)->getCookie()) {
@@ -381,7 +381,7 @@ void DcpConnMap::removeVBConnections(DcpProducer& prod) {
 
 void DcpConnMap::notifyVBConnections(uint16_t vbid, uint64_t bySeqno) {
     size_t lock_num = vbid % vbConnLockNum;
-    std::lock_guard<SpinLock> lh(vbConnLocks[lock_num]);
+    std::lock_guard<std::mutex> lh(vbConnLocks[lock_num]);
 
     std::list<std::shared_ptr<ConnHandler>>& conns = vbConns[vbid];
     for (auto it = conns.begin(); it != conns.end(); ++it) {

engines/ep/src/dcp/flow-control.cc

@@ -52,7 +52,7 @@ ENGINE_ERROR_CODE FlowControl::handleFlowCtl(
     if (enabled) {
         ENGINE_ERROR_CODE ret;
         uint32_t ackable_bytes = freedBytes.load();
-        std::unique_lock<SpinLock> lh(bufferSizeLock);
+        std::unique_lock<std::mutex> lh(bufferSizeLock);
         if (pendingControl) {
             pendingControl = false;
             std::string buf_size(std::to_string(bufferSize));
@@ -109,21 +109,20 @@ void FlowControl::incrFreedBytes(uint32_t bytes)
 
 uint32_t FlowControl::getFlowControlBufSize(void)
 {
-    std::lock_guard<SpinLock> lh(bufferSizeLock);
     return bufferSize;
 }
 
 void FlowControl::setFlowControlBufSize(uint32_t newSize)
 {
-    std::lock_guard<SpinLock> lh(bufferSizeLock);
+    std::lock_guard<std::mutex> lh(bufferSizeLock);
     if (newSize != bufferSize) {
         bufferSize = newSize;
         pendingControl = true;
     }
 }
 
 bool FlowControl::isBufferSufficientlyDrained() {
-    std::lock_guard<SpinLock> lh(bufferSizeLock);
+    std::lock_guard<std::mutex> lh(bufferSizeLock);
     return isBufferSufficientlyDrained_UNLOCKED(freedBytes.load());
 }
 

engines/ep/src/dcp/flow-control.h

@@ -75,7 +75,7 @@ class FlowControl {
     Couchbase::RelaxedAtomic<uint32_t> bufferSize;
 
     /* Lock while updating buffersize and pendingControl */
-    SpinLock bufferSizeLock;
+    std::mutex bufferSizeLock;
 
     /* To keep track of when last buffer ack was sent */
     rel_time_t lastBufferAck;