Retry on connection errors and during ingest upload turtle from cache.

ge-semtk · May 20, 2021 · 8f5729d · 8f5729d
1 parent 243d0b9
commit 8f5729d
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 22 deletions.
diff --git a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/DataLoader.java b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/DataLoader.java
@@ -467,9 +467,25 @@ private int uploadTempGraph() throws Exception {
 		String s = this.cacheSei.dumpToTurtle();
 		int len = s.length();
 		if (len > 0) {
-			LocalLogger.logToStdErr("Uploading " + s.length() + " chars of ttl");
-			this.endpoint.authUploadTurtle(s.getBytes());
-			LocalLogger.logToStdErr("upload complete");
+			int tryCount = 1;
+			while (true) {
+				try {
+					LocalLogger.logToStdErr("Uploading " + s.length() + " chars of ttl");
+
+					this.endpoint.authUploadTurtle(s.getBytes());
+
+					LocalLogger.logToStdErr("upload complete");
+					return len;
+				} catch (Exception e) {
+					if (tryCount < 4) {
+						this.endpoint.logFailureAndSleep(e, tryCount);
+						tryCount ++;
+					} else {
+						throw new Exception("Giving up uploading temp graph", e);
+					}
+				}
+			}
+
 		}
 
 		return len;

diff --git a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/sparqlX/SparqlEndpointInterface.java b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/sparqlX/SparqlEndpointInterface.java
@@ -575,29 +575,34 @@ public JSONObject executeQuery(String query, SparqlResultTypes resultType) throw
 					throw e;
 
 				} else {	
-					int sleepMsec = 500;
-
-					// if we're overwhelming a server, really throttle
-					if (e.getMessage() != null && e.getMessage().contains("Address already in use: connect")) {
-						sleepMsec = 5000 * tryCount;
-					} else {
-						// normally: 2 sec per try
-						sleepMsec = 2000 * tryCount;
-					}
-
-					// randomize sleepMsec from 75% to 125% in case threads are colliding at triplestore
-					sleepMsec = (int) ((sleepMsec * 0.75) + (Math.random() * sleepMsec * 0.5));
-
+
+					logFailureAndSleep(e, tryCount);
 					this.retries += 1;
-					LocalLogger.logToStdOut (String.format("SPARQL query failed.  Sleeping %d millisec and trying again...", sleepMsec));
-					LocalLogger.logToStdErr(e.getMessage());
 
-					TimeUnit.MILLISECONDS.sleep(sleepMsec);
 				}
 			}
 		}
 	}
 
+	public void logFailureAndSleep(Exception e, int tryCount) throws InterruptedException {
+		int sleepMsec = 500;
+
+		// if we're overwhelming a server, really throttle
+		if (e.getMessage() != null && e.getMessage().contains("Address already in use: connect")) {
+			sleepMsec = 5000 * tryCount;
+		} else {
+			// normally: 2 sec per try
+			sleepMsec = 2000 * tryCount;
+		}
+
+		// randomize sleepMsec from 75% to 125% in case threads are colliding at triplestore
+		sleepMsec = (int) ((sleepMsec * 0.75) + (Math.random() * sleepMsec * 0.5));
+		LocalLogger.logToStdOut (String.format("SPARQL query failed.  Sleeping %d millisec and trying again...", sleepMsec));
+		LocalLogger.logToStdErr(e.getMessage());
+
+		TimeUnit.MILLISECONDS.sleep(sleepMsec);
+	}
+
 	/**
 	 * Run a test query depending on isAuth()
 	 * See "internal use" note
@@ -985,10 +990,10 @@ protected String explainResponseTxt(String responseTxt) {
 	public boolean isExceptionRetryAble(Exception e) {
 		if (e instanceof AuthorizationException) {
 			return false;
-		} else if (e instanceof ConnectException) {
+		} //else if (e instanceof ConnectException) {
 			// if this connection timed out, caller is already in jeopardy of timing out
-			return false;
-		}
+			//return false;
+		//}
 		return true;
 	}