Skip to content

Commit 0ff1622

Browse files
juliangruberjuliangruber
committed
Add HTTP HEAD success count. CheckerNetwork/spark-checker#104 
1 parent 49de2f6 commit 0ff1622

9 files changed

+38
-22
lines changed

‎lib/preprocess.js

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ export class Measurement {
3535
this.first_byte_at = parseDateTime(m.first_byte_at)
3636
this.end_at = parseDateTime(m.end_at)
3737
this.status_code = m.status_code
38+
this.head_status_code = m.head_status_code
3839
this.timeout = m.timeout
3940
this.indexerResult = pointerize(m.indexer_result)
4041
this.stationId = pointerize(m.station_id)
@@ -248,6 +249,7 @@ export const assertValidMeasurement = measurement => {
248249
'stationId must be a hex string with 88 characters'
249250
)
250251
}
252+
assert(!(measurement.head_status_code && !measurement.status_code), '`head_status_code` must have `status_code` as well')
251253
}
252254

253255
/**

‎lib/provider-retrieval-result-stats.js

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,22 @@ const withPgClient = fn => async ({ createPgClient, ...args }) => {
1616
}
1717

1818
export const build = committees => {
19-
/** @type {Map<string, {total: number, successful: number, successfulHttp:number}>} */
19+
/** @type {Map<string, {total: number, successful: number, successfulHttp:number, successfulHttpHead: number}>} */
2020
const providerRetrievalResultStats = new Map()
2121
for (const c of committees) {
2222
// IMPORTANT: include minority results in the calculation
2323
for (const m of c.measurements) {
2424
const minerId = m.minerId
25-
const retrievalStats = providerRetrievalResultStats.get(minerId) ?? { total: 0, successful: 0, successfulHttp: 0 }
25+
const retrievalStats = providerRetrievalResultStats.get(minerId) ?? { total: 0, successful: 0, successfulHttp: 0, successfulHttpHead: 0 }
2626
retrievalStats.total++
2727
if (m.retrievalResult === 'OK') {
2828
retrievalStats.successful++
29-
if (m.protocol && m.protocol === 'http') { retrievalStats.successfulHttp++ }
29+
if (m.protocol && m.protocol === 'http') {
30+
retrievalStats.successfulHttp++
31+
if (typeof m.head_status_code === 'number' && m.head_status_code >= 200 && m.head_status_code < 300) {
32+
retrievalStats.successfulHttpHead++
33+
}
34+
}
3035
}
3136
providerRetrievalResultStats.set(minerId, retrievalStats)
3237
}

‎lib/public-stats.js

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,23 +39,26 @@ export const updatePublicStats = async ({ createPgClient, committees, allMeasure
3939
* @param {number} stats.total
4040
* @param {number} stats.successful
4141
* @param {number} stats.successfulHttp
42+
* @param {number} stats.successfulHttpHead
4243
*/
43-
const updateRetrievalStats = async (pgClient, minerId, { total, successful, successfulHttp }) => {
44-
debug('Updating public retrieval stats for miner %s: total += %s successful += %s, successful_http += %s', minerId, total, successful, successfulHttp)
44+
const updateRetrievalStats = async (pgClient, minerId, { total, successful, successfulHttp, successfulHttpHead }) => {
45+
debug('Updating public retrieval stats for miner %s: total += %s successful += %s, successful_http += %s, successful_http_head += %s', minerId, total, successful, successfulHttp, successfulHttpHead)
4546
await pgClient.query(`
4647
INSERT INTO retrieval_stats
47-
(day, miner_id, total, successful, successful_http)
48+
(day, miner_id, total, successful, successful_http, successful_http_head)
4849
VALUES
49-
(now(), $1, $2, $3, $4)
50+
(now(), $1, $2, $3, $4, $5)
5051
ON CONFLICT(day, miner_id) DO UPDATE SET
5152
total = retrieval_stats.total + $2,
5253
successful = retrieval_stats.successful + $3,
53-
successful_http = retrieval_stats.successful_http + $4
54+
successful_http = retrieval_stats.successful_http + $4,
55+
successful_http_head = retrieval_stats.successful_http_head + $5
5456
`, [
5557
minerId,
5658
total,
5759
successful,
58-
successfulHttp
60+
successfulHttp,
61+
successfulHttpHead
5962
])
6063
}
6164

‎lib/typings.d.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ export interface RawMeasurement {
8787
finished_at: string;
8888

8989
status_code: number | undefined | null;
90+
head_status_code: number | undefined | null;
9091
timeout: boolean;
9192
byte_length: number;
9293
car_too_large: boolean;

‎migrations/023.do.successful-http-head.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ALTER TABLE retrieval_stats ADD COLUMN successful_http_head INT;

‎test/evaluate.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,8 @@ describe('evaluate', async function () {
9898
total: 1,
9999
successful: 1,
100100
// None of the measurements use http
101-
successful_http: 0
101+
successful_http: 0,
102+
successful_http_head: 0
102103
}])
103104
})
104105
it('handles empty rounds', async () => {

‎test/helpers/test-data.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ export const VALID_MEASUREMENT = {
2727
stationId: VALID_STATION_ID,
2828
inet_group: VALID_INET_GROUP,
2929
status_code: 200,
30+
head_status_code: 200,
3031
// TODO: these fields are not part of the Measurement object yet
3132
// timeout: false,
3233
// car_too_large: false,

‎test/provider-retrieval-result-stats.test.js

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@ describe('Provider Retrieval Result Stats', () => {
3838
{
3939
minerId: '0',
4040
retrievalResult: 'OK',
41-
protocol: 'http'
41+
protocol: 'http',
42+
head_status_code: 200
4243
},
4344
{
4445
minerId: '1',
@@ -62,8 +63,8 @@ describe('Provider Retrieval Result Stats', () => {
6263
}
6364
])
6465
assert.deepStrictEqual(stats, new Map([
65-
['0', { total: 2, successful: 2, successfulHttp: 1 }],
66-
['1', { total: 2, successful: 0, successfulHttp: 0 }]
66+
['0', { total: 2, successful: 2, successfulHttp: 1, successfulHttpHead: 1 }],
67+
['1', { total: 2, successful: 0, successfulHttp: 0, successfulHttpHead: 0 }]
6768
]))
6869
})
6970
})
@@ -142,7 +143,8 @@ describe('Provider Retrieval Result Stats', () => {
142143
{
143144
minerId: '0',
144145
retrievalResult: 'OK',
145-
protocol: 'http'
146+
protocol: 'http',
147+
head_status_code: 200
146148
},
147149
{
148150
minerId: '1',
@@ -176,8 +178,8 @@ describe('Provider Retrieval Result Stats', () => {
176178
contract_address: ieContractAddress,
177179
measurement_batches: round.measurementBatches,
178180
provider_retrieval_result_stats: {
179-
0: { successful: 2, total: 2, successfulHttp: 1 },
180-
1: { successful: 0, total: 2, successfulHttp: 0 }
181+
0: { successful: 2, total: 2, successfulHttp: 1, successfulHttpHead: 1 },
182+
1: { successful: 0, total: 2, successfulHttp: 0, successfulHttpHead: 0 }
181183
},
182184
round_details: 'baguqeerawg5jfpiy2g5xp5d422uwa3mpyzkmiguoeecesds7q65mn2hdoa4q',
183185
round_index: String(round.index),

‎test/public-stats.test.js

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ describe('public-stats', () => {
8989
it('calculates successful http retrievals correctly', async () => {
9090
/** @type {Measurement[]} */
9191
const honestMeasurements = [
92-
{ ...VALID_MEASUREMENT, protocol: 'http', retrievalResult: 'OK' },
92+
{ ...VALID_MEASUREMENT, protocol: 'http', retrievalResult: 'OK', head_status_code: 200 },
9393
{ ...VALID_MEASUREMENT, protocol: 'graphsync', retrievalResult: 'OK' },
9494
{ ...VALID_MEASUREMENT, protocol: 'http', retrievalResult: 'HTTP_500' },
9595
{ ...VALID_MEASUREMENT, protocol: 'graphsync', retrievalResult: 'LASSIE_500' }
@@ -105,14 +105,14 @@ describe('public-stats', () => {
105105
})
106106

107107
const { rows: created } = await pgClient.query(
108-
'SELECT day::TEXT, total, successful, successful_http FROM retrieval_stats'
108+
'SELECT day::TEXT, total, successful, successful_http, successful_http_head FROM retrieval_stats'
109109
)
110110
assert.deepStrictEqual(created, [
111-
{ day: today, total: 4, successful: 2, successful_http: 1 }
111+
{ day: today, total: 4, successful: 2, successful_http: 1, successful_http_head: 1 }
112112
])
113113

114114
// Let's add another successful http retrieval to make sure the updating process works as expected
115-
honestMeasurements.push({ ...VALID_MEASUREMENT, retrievalResult: 'OK', protocol: 'http' })
115+
honestMeasurements.push({ ...VALID_MEASUREMENT, retrievalResult: 'OK', protocol: 'http', head_status_code: 200 })
116116
committees = buildEvaluatedCommitteesFromMeasurements(honestMeasurements)
117117
await updatePublicStats({
118118
createPgClient,
@@ -122,10 +122,10 @@ describe('public-stats', () => {
122122
})
123123

124124
const { rows: updated } = await pgClient.query(
125-
'SELECT day::TEXT, total, successful, successful_http FROM retrieval_stats'
125+
'SELECT day::TEXT, total, successful, successful_http, successful_http_head FROM retrieval_stats'
126126
)
127127
assert.deepStrictEqual(updated, [
128-
{ day: today, total: 4 + 5, successful: 2 + 3, successful_http: 1 + 2 }
128+
{ day: today, total: 4 + 5, successful: 2 + 3, successful_http: 1 + 2, successful_http_head: 1 + 2 }
129129
])
130130
})
131131

0 commit comments

Comments
 (0)