Skip to content

Add a metric to track "Wrong key returned" errors #108

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
smukil wants to merge 4 commits into
base: master
Choose a base branch
from
+33 −13
Open

Add a metric to track "Wrong key returned" errors #108

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bebde36
Add a metric to track "Wrong key returned" errors
smukil Apr 20, 2021
43dc45f
Print stack trace when we notice a Wrong Key error
smukil Apr 20, 2021
81ab1cb
Consolidate duplicate code under checkWrongKeyReturned()
smukil Apr 21, 2021
7706be4
Address review comments for wrong key returned metric
smukil Apr 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions evcache-core/src/main/java/com/netflix/evcache/metrics/EVCacheMetricsFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,7 @@ public String getStatusCode(StatusCode sc) {
public static final String CHUNK_DATA_SIZE = "dataSize";
public static final String NOT_AVAILABLE = "notAvailable";
public static final String NOT_ACTIVE = "notActive";
public static final String WRONG_KEY_RETURNED = "wrongKeyReturned";

public static final String INITIAL = "initial";
public static final String SECOND = "second";
Expand Down
3 changes: 3 additions & 0 deletions evcache-core/src/main/java/com/netflix/evcache/pool/EVCacheClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,9 @@ private void incrementFailure(String metric, EVCache.Call call, String host) {
counter.increment();
}

public void reportWrongKeyReturned(String hostName) {
incrementFailure(EVCacheMetricsFactory.WRONG_KEY_RETURNED, null, hostName);
}

private boolean ensureWriteQueueSize(MemcachedNode node, String key, EVCache.Call call) throws EVCacheException {
if (node instanceof EVCacheNode) {
Expand Down
43 changes: 30 additions & 13 deletions evcache-core/src/main/java/net/spy/memcached/EVCacheMemcachedClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,22 @@ public <T> GetFuture<T> asyncGet(final String key, final Transcoder<T> tc) {
throw new UnsupportedOperationException("asyncGet");
}

// Returns 'true' if keys don't match and logs & reports the error.
// Returns 'false' if keys match.
private boolean checkWrongKeyReturned(String original_key, String returned_key) {
if (!original_key.equals(returned_key)) {
// If they keys don't match, log the error along with the key owning host information.
final String host = getHostNameByKey(returned_key);
Comment thread
smukil marked this conversation as resolved.
Outdated
log.error("Wrong key returned. Key - " + original_key + "; Returned Key " + returned_key + "; Host" + host);
Comment thread
smukil marked this conversation as resolved.
Outdated
// Print stack trace for debugging purposes.
// TODO: Consider removing this code once we've fixed the Wrong key bug(s)
new Throwable().printStackTrace();
client.reportWrongKeyReturned(host);
return true;
}
return false;
}

public <T> EVCacheOperationFuture<T> asyncGet(final String key, final Transcoder<T> tc, EVCacheGetOperationListener<T> listener) {
final CountDownLatch latch = new CountDownLatch(1);
final EVCacheOperationFuture<T> rv = new EVCacheOperationFuture<T>(key, latch, new AtomicReference<T>(null), readTimeout.get().intValue(), executorService, client);
Expand All @@ -126,10 +142,8 @@ public void receivedStatus(OperationStatus status) {
@SuppressWarnings("unchecked")
public void gotData(String k, int flags, byte[] data) {

if (!key.equals(k)) {
log.error("Wrong key returned. Key - " + key + "; Returned Key " + k);
return;
}
if (checkWrongKeyReturned(key, k)) return;
Comment thread
smukil marked this conversation as resolved.
Outdated

if (log.isDebugEnabled() && client.getPool().getEVCacheClientPoolManager().shouldLog(appName)) log.debug("Read data : key " + key + "; flags : " + flags + "; data : " + data);
if (data != null) {
if (log.isDebugEnabled() && client.getPool().getEVCacheClientPoolManager().shouldLog(appName)) log.debug("Key : " + key + "; val size : " + data.length);
Expand Down Expand Up @@ -255,7 +269,9 @@ public void complete() {
}

public void gotData(String k, int flags, long cas, byte[] data) {
if (!key.equals(k)) log.warn("Wrong key returned. Key - " + key + "; Returned Key " + k);
// TODO: Should we return here if wrong key is returned? We seem to do so everywhere else except here.
Comment thread
smukil marked this conversation as resolved.
Outdated
checkWrongKeyReturned(key, k);

if (data != null) getDataSizeDistributionSummary(EVCacheMetricsFactory.GET_AND_TOUCH_OPERATION, EVCacheMetricsFactory.READ, EVCacheMetricsFactory.IPC_SIZE_INBOUND).record(data.length);
val = new CASValue<T>(cas, tc.decode(new CachedData(flags, data, tc.getMaxSize())));
}
Expand Down Expand Up @@ -602,6 +618,11 @@ public int getReadMetricMaxValue() {
return maxReadDuration.get().intValue();
}

private String getHostNameByKey(String key) {
EVCacheNode evcNode = (EVCacheNode)getEVCacheNode(key);
Comment thread
smukil marked this conversation as resolved.
Outdated
return getHostName(evcNode.getSocketAddress());
}

private String getHostName(SocketAddress sa) {
if (sa == null) return null;
if(sa instanceof InetSocketAddress) {
Expand Down Expand Up @@ -722,10 +743,8 @@ public void receivedStatus(OperationStatus status) {
@Override
public void gotMetaData(String k, char flag, String fVal) {
if (log.isDebugEnabled()) log.debug("key " + k + "; val : " + fVal + "; flag : " + flag);
if (!key.equals(k)) {
log.error("Wrong key returned. Expected Key - " + key + "; Returned Key " + k);
return;
}
if (checkWrongKeyReturned(key, k)) return;

switch (flag) {
case 's':
evItem.getItemMetaData().setSizeInBytes(Integer.parseInt(fVal));
Expand Down Expand Up @@ -765,10 +784,8 @@ public void gotMetaData(String k, char flag, String fVal) {
@Override
public void gotData(String k, int flag, byte[] data) {
if (log.isDebugEnabled() && client.getPool().getEVCacheClientPoolManager().shouldLog(appName)) log.debug("Read data : key " + k + "; flags : " + flag + "; data : " + data);
if (!key.equals(k)) {
log.error("Wrong key returned. Expected Key - " + key + "; Returned Key " + k);
return;
}
if (checkWrongKeyReturned(key, k)) return;

if (data != null) {
if (log.isDebugEnabled() && client.getPool().getEVCacheClientPoolManager().shouldLog(appName)) log.debug("Key : " + k + "; val size : " + data.length);
getDataSizeDistributionSummary(EVCacheMetricsFactory.META_GET_OPERATION, EVCacheMetricsFactory.READ, EVCacheMetricsFactory.IPC_SIZE_INBOUND).record(data.length);
Expand Down