Add HTTP HEAD success count. CheckerNetwork/spark-checker#104 (#475)

* Add HTTP HEAD success count. CheckerNetwork/spark-checker#104

* add passing validation test

* fix lint

* fix migration conflict

Author: juliangruber

lib/preprocess.js

@@ -35,6 +35,7 @@ export class Measurement {
     this.first_byte_at = parseDateTime(m.first_byte_at)
     this.end_at = parseDateTime(m.end_at)
     this.status_code = m.status_code
+    this.head_status_code = m.head_status_code
     this.timeout = m.timeout
     this.indexerResult = pointerize(m.indexer_result)
     this.stationId = pointerize(m.station_id)
@@ -266,6 +267,8 @@ export const assertValidMeasurement = measurement => {
     assert(measurement.end_at >= measurement.start_at, 'end_at must be greater than or equal to start_at')
     assert(measurement.end_at >= measurement.first_byte_at, 'end_at must be greater than or equal to first_byte_at')
     assert(measurement.first_byte_at >= measurement.start_at, 'first_byte_at must be greater than or equal to start_at')
+
+    assert.strictEqual(typeof measurement.head_status_code, 'number', '`head_status_code` must be a number')
   }
 }
 

lib/provider-retrieval-result-stats.js

@@ -16,17 +16,22 @@ const withPgClient = fn => async ({ createPgClient, ...args }) => {
 }
 
 export const build = committees => {
-  /** @type {Map<string, {total: number, successful: number, successfulHttp:number}>} */
+  /** @type {Map<string, {total: number, successful: number, successfulHttp:number, successfulHttpHead: number}>} */
   const providerRetrievalResultStats = new Map()
   for (const c of committees) {
     // IMPORTANT: include minority results in the calculation
     for (const m of c.measurements) {
       const minerId = m.minerId
-      const retrievalStats = providerRetrievalResultStats.get(minerId) ?? { total: 0, successful: 0, successfulHttp: 0 }
+      const retrievalStats = providerRetrievalResultStats.get(minerId) ?? { total: 0, successful: 0, successfulHttp: 0, successfulHttpHead: 0 }
       retrievalStats.total++
       if (m.retrievalResult === 'OK') {
         retrievalStats.successful++
-        if (m.protocol && m.protocol === 'http') { retrievalStats.successfulHttp++ }
+        if (m.protocol && m.protocol === 'http') {
+          retrievalStats.successfulHttp++
+          if (typeof m.head_status_code === 'number' && m.head_status_code >= 200 && m.head_status_code < 300) {
+            retrievalStats.successfulHttpHead++
+          }
+        }
       }
       providerRetrievalResultStats.set(minerId, retrievalStats)
     }

lib/public-stats.js

@@ -40,23 +40,26 @@ export const updatePublicStats = async ({ createPgClient, committees, allMeasure
  * @param {number} stats.total
  * @param {number} stats.successful
  * @param {number} stats.successfulHttp
+ * @param {number} stats.successfulHttpHead
  */
-const updateRetrievalStats = async (pgClient, minerId, { total, successful, successfulHttp }) => {
-  debug('Updating public retrieval stats for miner %s: total += %s successful += %s, successful_http += %s', minerId, total, successful, successfulHttp)
+const updateRetrievalStats = async (pgClient, minerId, { total, successful, successfulHttp, successfulHttpHead }) => {
+  debug('Updating public retrieval stats for miner %s: total += %s successful += %s, successful_http += %s, successful_http_head += %s', minerId, total, successful, successfulHttp, successfulHttpHead)
   await pgClient.query(`
     INSERT INTO retrieval_stats
-      (day, miner_id, total, successful, successful_http)
+      (day, miner_id, total, successful, successful_http, successful_http_head)
     VALUES
-      (now(), $1, $2, $3, $4)
+      (now(), $1, $2, $3, $4, $5)
     ON CONFLICT(day, miner_id) DO UPDATE SET
       total = retrieval_stats.total + $2,
       successful = retrieval_stats.successful + $3,
-      successful_http = retrieval_stats.successful_http + $4
+      successful_http = retrieval_stats.successful_http + $4,
+      successful_http_head = retrieval_stats.successful_http_head + $5
   `, [
     minerId,
     total,
     successful,
-    successfulHttp
+    successfulHttp,
+    successfulHttpHead
   ])
 }
 

lib/typings.d.ts

@@ -87,6 +87,7 @@ export interface RawMeasurement {
   finished_at: string;
 
   status_code: number | undefined | null;
+  head_status_code: number | undefined | null;
   timeout: boolean;
   byte_length: number;
   car_too_large: boolean;

migrations/024.do.successful-http-head.sql

@@ -0,0 +1 @@
+ALTER TABLE retrieval_stats ADD COLUMN successful_http_head INT;

test/evaluate.js

@@ -98,7 +98,8 @@ describe('evaluate', async function () {
       total: 1,
       successful: 1,
       // None of the measurements use http
-      successful_http: 0
+      successful_http: 0,
+      successful_http_head: 0
     }])
   })
   it('handles empty rounds', async () => {

test/helpers/test-data.js

@@ -27,6 +27,7 @@ export const VALID_MEASUREMENT = {
   stationId: VALID_STATION_ID,
   inet_group: VALID_INET_GROUP,
   status_code: 200,
+  head_status_code: 200,
   // TODO: these fields are not part of the Measurement object yet
   // timeout: false,
   // car_too_large: false,

test/preprocess.js

@@ -310,6 +310,22 @@ describe('assertValidMeasurement', () => {
       /field `indexerResult` must be set/
     )
   })
+  it('rejects measurements with OK retrieval result but head_status_code is not a number', () => {
+    assert.throws(
+      () => assertValidMeasurement({
+        ...VALID_MEASUREMENT,
+        head_status_code: null
+      }),
+      /`head_status_code` must be a number/
+    )
+    assert.throws(
+      () => assertValidMeasurement({
+        ...VALID_MEASUREMENT,
+        head_status_code: /** @type {any} */ ('200')
+      }),
+      /`head_status_code` must be a number/
+    )
+  })
 
   it('should throw an error for invalid start_at', () => {
     const measurement = {

test/provider-retrieval-result-stats.test.js

@@ -38,7 +38,8 @@ describe('Provider Retrieval Result Stats', () => {
             {
               minerId: '0',
               retrievalResult: 'OK',
-              protocol: 'http'
+              protocol: 'http',
+              head_status_code: 200
             },
             {
               minerId: '1',
@@ -62,8 +63,8 @@ describe('Provider Retrieval Result Stats', () => {
         }
       ])
       assert.deepStrictEqual(stats, new Map([
-        ['0', { total: 2, successful: 2, successfulHttp: 1 }],
-        ['1', { total: 2, successful: 0, successfulHttp: 0 }]
+        ['0', { total: 2, successful: 2, successfulHttp: 1, successfulHttpHead: 1 }],
+        ['1', { total: 2, successful: 0, successfulHttp: 0, successfulHttpHead: 0 }]
       ]))
     })
   })
@@ -142,7 +143,8 @@ describe('Provider Retrieval Result Stats', () => {
               {
                 minerId: '0',
                 retrievalResult: 'OK',
-                protocol: 'http'
+                protocol: 'http',
+                head_status_code: 200
               },
               {
                 minerId: '1',
@@ -176,8 +178,8 @@ describe('Provider Retrieval Result Stats', () => {
         contract_address: ieContractAddress,
         measurement_batches: round.measurementBatches,
         provider_retrieval_result_stats: {
-          0: { successful: 2, total: 2, successfulHttp: 1 },
-          1: { successful: 0, total: 2, successfulHttp: 0 }
+          0: { successful: 2, total: 2, successfulHttp: 1, successfulHttpHead: 1 },
+          1: { successful: 0, total: 2, successfulHttp: 0, successfulHttpHead: 0 }
         },
         round_details: 'baguqeerawg5jfpiy2g5xp5d422uwa3mpyzkmiguoeecesds7q65mn2hdoa4q',
         round_index: String(round.index),

test/public-stats.test.js

@@ -90,7 +90,7 @@ describe('public-stats', () => {
     it('calculates successful http retrievals correctly', async () => {
       /** @type {Measurement[]} */
       const honestMeasurements = [
-        { ...VALID_MEASUREMENT, protocol: 'http', retrievalResult: 'OK' },
+        { ...VALID_MEASUREMENT, protocol: 'http', retrievalResult: 'OK', head_status_code: 200 },
         { ...VALID_MEASUREMENT, protocol: 'graphsync', retrievalResult: 'OK' },
         { ...VALID_MEASUREMENT, protocol: 'http', retrievalResult: 'HTTP_500' },
         { ...VALID_MEASUREMENT, protocol: 'graphsync', retrievalResult: 'LASSIE_500' }
@@ -106,14 +106,14 @@ describe('public-stats', () => {
       })
 
       const { rows: created } = await pgClient.query(
-        'SELECT day::TEXT, total, successful, successful_http FROM retrieval_stats'
+        'SELECT day::TEXT, total, successful, successful_http, successful_http_head FROM retrieval_stats'
       )
       assert.deepStrictEqual(created, [
-        { day: today, total: 4, successful: 2, successful_http: 1 }
+        { day: today, total: 4, successful: 2, successful_http: 1, successful_http_head: 1 }
       ])
 
       // Let's add another successful http retrieval to make sure the updating process works as expected
-      honestMeasurements.push({ ...VALID_MEASUREMENT, retrievalResult: 'OK', protocol: 'http' })
+      honestMeasurements.push({ ...VALID_MEASUREMENT, retrievalResult: 'OK', protocol: 'http', head_status_code: 200 })
       committees = buildEvaluatedCommitteesFromMeasurements(honestMeasurements)
       await updatePublicStats({
         createPgClient,
@@ -123,10 +123,10 @@ describe('public-stats', () => {
       })
 
       const { rows: updated } = await pgClient.query(
-        'SELECT day::TEXT, total, successful, successful_http FROM retrieval_stats'
+        'SELECT day::TEXT, total, successful, successful_http, successful_http_head FROM retrieval_stats'
       )
       assert.deepStrictEqual(updated, [
-        { day: today, total: 4 + 5, successful: 2 + 3, successful_http: 1 + 2 }
+        { day: today, total: 4 + 5, successful: 2 + 3, successful_http: 1 + 2, successful_http_head: 1 + 2 }
       ])
     })