Skip to content

Add a metric to track "Wrong key returned" errors #108

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
smukil wants to merge 4 commits into
base: master
Choose a base branch
from
+33 −13
Open

Add a metric to track "Wrong key returned" errors #108

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bebde36
Add a metric to track "Wrong key returned" errors
smukil Apr 20, 2021
43dc45f
Print stack trace when we notice a Wrong Key error
smukil Apr 20, 2021
81ab1cb
Consolidate duplicate code under checkWrongKeyReturned()
smukil Apr 21, 2021
7706be4
Address review comments for wrong key returned metric
smukil Apr 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions evcache-core/src/main/java/com/netflix/evcache/metrics/EVCacheMetricsFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,7 @@ public String getStatusCode(StatusCode sc) {
public static final String CHUNK_DATA_SIZE = "dataSize";
public static final String NOT_AVAILABLE = "notAvailable";
public static final String NOT_ACTIVE = "notActive";
public static final String WRONG_KEY_RETURNED = "wrongKeyReturned";

public static final String INITIAL = "initial";
public static final String SECOND = "second";
Expand Down
3 changes: 3 additions & 0 deletions evcache-core/src/main/java/com/netflix/evcache/pool/EVCacheClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,9 @@ private void incrementFailure(String metric, EVCache.Call call, String host) {
counter.increment();
}

public void reportWrongKeyReturned(String hostName) {
incrementFailure(EVCacheMetricsFactory.WRONG_KEY_RETURNED, null, hostName);
}

private boolean ensureWriteQueueSize(MemcachedNode node, String key, EVCache.Call call) throws EVCacheException {
if (node instanceof EVCacheNode) {
Expand Down
27 changes: 23 additions & 4 deletions evcache-core/src/main/java/net/spy/memcached/EVCacheMemcachedClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,10 @@ public void receivedStatus(OperationStatus status) {
public void gotData(String k, int flags, byte[] data) {

if (!key.equals(k)) {
log.error("Wrong key returned. Key - " + key + "; Returned Key " + k);
// If they keys don't match, log the error along with the key owning host information.
final String host = getHostNameByKey(k);
log.error("Wrong key returned. Key - " + key + "; Returned Key " + k + "; Host" + host);
client.reportWrongKeyReturned(host);
return;
}
if (log.isDebugEnabled() && client.getPool().getEVCacheClientPoolManager().shouldLog(appName)) log.debug("Read data : key " + key + "; flags : " + flags + "; data : " + data);
Expand Down Expand Up @@ -255,7 +258,12 @@ public void complete() {
}

public void gotData(String k, int flags, long cas, byte[] data) {
if (!key.equals(k)) log.warn("Wrong key returned. Key - " + key + "; Returned Key " + k);
if (!key.equals(k)) {
Comment thread
smukil marked this conversation as resolved.
Outdated
// If they keys don't match, log the error along with the key owning host information.
final String host = getHostNameByKey(k);
log.error("Wrong key returned. Key - " + key + "; Returned Key " + k + "; Host" + host);
client.reportWrongKeyReturned(host);
}
if (data != null) getDataSizeDistributionSummary(EVCacheMetricsFactory.GET_AND_TOUCH_OPERATION, EVCacheMetricsFactory.READ, EVCacheMetricsFactory.IPC_SIZE_INBOUND).record(data.length);
val = new CASValue<T>(cas, tc.decode(new CachedData(flags, data, tc.getMaxSize())));
}
Expand Down Expand Up @@ -602,6 +610,11 @@ public int getReadMetricMaxValue() {
return maxReadDuration.get().intValue();
}

private String getHostNameByKey(String key) {
EVCacheNode evcNode = (EVCacheNode)getEVCacheNode(key);
Comment thread
smukil marked this conversation as resolved.
Outdated
return getHostName(evcNode.getSocketAddress());
}

private String getHostName(SocketAddress sa) {
if (sa == null) return null;
if(sa instanceof InetSocketAddress) {
Expand Down Expand Up @@ -723,7 +736,10 @@ public void receivedStatus(OperationStatus status) {
public void gotMetaData(String k, char flag, String fVal) {
if (log.isDebugEnabled()) log.debug("key " + k + "; val : " + fVal + "; flag : " + flag);
if (!key.equals(k)) {
log.error("Wrong key returned. Expected Key - " + key + "; Returned Key " + k);
// If they keys don't match, log the error along with the key owning host information.
final String host = getHostNameByKey(k);
log.error("Wrong key returned. Key - " + key + "; Returned Key " + k + "; Host" + host);
client.reportWrongKeyReturned(host);
return;
}
switch (flag) {
Expand Down Expand Up @@ -766,7 +782,10 @@ public void gotMetaData(String k, char flag, String fVal) {
public void gotData(String k, int flag, byte[] data) {
if (log.isDebugEnabled() && client.getPool().getEVCacheClientPoolManager().shouldLog(appName)) log.debug("Read data : key " + k + "; flags : " + flag + "; data : " + data);
if (!key.equals(k)) {
log.error("Wrong key returned. Expected Key - " + key + "; Returned Key " + k);
// If they keys don't match, log the error along with the key owning host information.
final String host = getHostNameByKey(k);
log.error("Wrong key returned. Key - " + key + "; Returned Key " + k + "; Host" + host);
client.reportWrongKeyReturned(host);
return;
}
if (data != null) {
Expand Down