GRAPH-581 - Add retry for status URL check

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/jupiterone-client-nodejs",
-  "version": "2.1.0",
+  "version": "2.1.1",
   "description": "A node.js client wrapper for JupiterOne public API",
   "repository": {
     "type": "git",

src/index.ts

@@ -111,7 +111,7 @@
     {
       maxAttempts: 5,
       delay: 1000,
       handleError(err, context, options) {
         const possibleFetchError = err as Partial<FetchError>;
         const { httpStatusCode } = possibleFetchError;
         if (httpStatusCode !== undefined) {
@@ -163,7 +163,7 @@
 
 export interface JupiterOneEntity {
   entity: JupiterOneEntityMetadata;
   properties: any;
 }
 
 export interface QueryResult {
@@ -283,7 +283,7 @@
 };
 
 export class JupiterOneClient {
   graphClient: ApolloClient<any>;
   headers?: Record<string, string>;
   account: string;
   accessToken: string;
@@ -356,23 +356,24 @@
      * this limits the looping to stop after at least {stopAfter} results are found
      * @deprecated This property is no longer supported.
      */
     stopAfter = Number.MAX_SAFE_INTEGER,
     /** same as above, this gives more fine-grained control over the starting point of the query,
      * since this method controls the `SKIP` clause in the query
      * @deprecated This property is no longer supported.
      */
     startPage = 0,
   ) {
 
     let cursor: string;
     let complete = false;
     let results: any[] = [];
 
     const limitCheck = j1ql.match(/limit (\d+)/i);
 
     let progress: any;
 
     do {
+      this.logger.debug({j1ql}, "Sending query");
       const res = await this.graphClient.query({
         query: QUERY_V1,
         variables: {
@@ -389,9 +390,10 @@
         throw new Error(`JupiterOne returned error(s) for query: '${j1ql}'`);
       }
 
+      this.logger.debug(res.data, "Retrieved response");
       const deferredUrl = res.data.queryV1.url;
       let status = JobStatus.IN_PROGRESS;
       let statusFile: any;
       const startTimeInMs = Date.now();
       do {
         if (Date.now() - startTimeInMs > QUERY_RESULTS_TIMEOUT) {
@@ -400,7 +402,7 @@
             } seconds.`,
           );
         }
-        this.logger.trace('Sleeping to wait for JobCompletion');
+        this.logger.debug('Sleeping to wait for JobCompletion');
         await sleep(100);
         statusFile = await networkRequest(deferredUrl);
         status = statusFile.status;
@@ -411,6 +413,7 @@
         throw new Error(`JupiterOne returned error(s) for query: '${statusFile.error}'`);
       }
 
+      this.logger.info("Retrieving query data");
       const result = statusFile.data;
 
       if (showProgress && !limitCheck) {

src/networkRequest.ts

@@ -1,20 +1,25 @@
 // Temporary helper file because it's difficult to mock in the current architecture
 
 import fetch from 'node-fetch';
+import { retry } from "@lifeomic/attempt";
 
 export const networkRequest = async (
   url: string,
 ): Promise<Record<string, unknown>> => {
-  const result = await fetch(url);
+  const result = await retry(async () => {
+    console.log(`Fetching ${url}`);
+    const result = await fetch(url);
+    const { status } = result;
 
-  const { status, headers } = result;
+    if (status < 200 || status >= 300) {
+      const body = await result.text();
+      throw new Error(`HTTP request failed (${status}): ${body}`);
+    }
 
-  if (status < 200 || status >= 300) {
-    const body = await result.text();
-    throw new Error(`HTTP request failed (${status}): ${body}`);
-  }
+    return result;
+  });
 
-  const contentType = headers.get('content-type');
+  const contentType = result.headers.get('content-type');
   if (contentType?.includes('application/json') === false) {
     const body = await result.text();
     throw new Error(`HTTP response is not JSON but ${contentType}: ${body}`);