Skip to content

Commit

Permalink
fix collecting data for url/origin after last kruk update
Browse files Browse the repository at this point in the history
  • Loading branch information
@gotomi
gotomi committed Feb 28, 2024
1 parent 6ce9a0e commit 5065823
Show file tree
Hide file tree
Showing 9 changed files with 49 additions and 64 deletions.
97 changes: 41 additions & 56 deletions getCruxData.mjs
View file
Original file line number Diff line number Diff line change
@@ -1,71 +1,56 @@

import { getReports } from 'kruk';
import { readdir, readFile, writeFile } from 'fs/promises';
import path from 'path';
const INPUT = './src/_urls';
const OUTPUT = './src/_data';

import { getReports } from "kruk";
import { readdir, readFile, writeFile } from "fs/promises";
import path from "path";
const INPUT = "./src/_urls";
const OUTPUT = "./src/_data";

const API_KEY = process.env.PSIKUS;

function prependHttp(url, { https = true } = {}) {
if (typeof url !== 'string') {
throw new TypeError(`Expected \`url\` to be of type \`string\`, got \`${typeof url}\``);
}

url = url.trim();

if (/^\.*\/|^(?!localhost)\w+?:/.test(url)) {
return url;
}
return url.replace(/^(?!(?:\w+?:)?\/\/)/, https ? 'https://' : 'http://');
if (typeof url !== "string") {
throw new TypeError(
`Expected \`url\` to be of type \`string\`, got \`${typeof url}\``
);
}

url = url.trim();

if (/^\.*\/|^(?!localhost)\w+?:/.test(url)) {
return url;
}
return url.replace(/^(?!(?:\w+?:)?\/\/)/, https ? "https://" : "http://");
}

async function getDataAndSaveToFile(file, queryParams) {
const urlOrOrigin = queryParams.checkOrigin ? 'origin' : 'url';
const urls = await readFile(path.join(INPUT, file), "utf8");
const data = await getCrux(JSON.parse(urls).map(url => prependHttp(url)), queryParams);
const fileToWrite = file.split('.json')[0] + '-' + urlOrOrigin + '.json';
writeFile(path.join(OUTPUT, fileToWrite), JSON.stringify(data));
const urlOrOrigin = queryParams.origin ? "origin" : "url";
const urls = await readFile(path.join(INPUT, file), "utf8");
const data = await getCrux(
JSON.parse(urls).map((url) => prependHttp(url)),
queryParams
);
const fileToWrite = file.split(".json")[0] + "-" + urlOrOrigin + ".json";
writeFile(path.join(OUTPUT, fileToWrite), JSON.stringify(data));
}


try {
const files = await readdir(INPUT);
for (const file of files)
if (file.includes('.json')) {
getDataAndSaveToFile(file, {
effectiveConnectionType: '',
formFactor: 'PHONE',
checkOrigin: true
})
getDataAndSaveToFile(file, {
effectiveConnectionType: '',
formFactor: 'PHONE'
})

}
const files = await readdir(INPUT);
for (const file of files)
if (file.includes(".json")) {
getDataAndSaveToFile(file, {
effectiveConnectionType: "",
formFactor: "PHONE",
origin: true,
});
getDataAndSaveToFile(file, {
effectiveConnectionType: "",
formFactor: "PHONE",
origin: false,
});
}
} catch (err) {
console.error(err);
console.error(err);
}





async function getCrux(urls, queryParams) {
return await getReports(urls, API_KEY, queryParams);
return await getReports(urls, API_KEY, queryParams);
}













2 changes: 1 addition & 1 deletion src/_data/ecommerce-eu-origin.json
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion src/_data/ecommerce-eu-url.json
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion src/_data/ecommerce-pl-origin.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"params":{"formFactor":"PHONE","url":true,"collectionPeriod":{"firstDate":{"year":2024,"month":1,"day":29},"lastDate":{"year":2024,"month":2,"day":25}},"date":"28.02.2024"},"metrics":[{"url":"https://allegro.pl/","CLS":{"histogram":[98.07,0.89,1.04],"p75":"0.00","rank":"good"},"FCP":{"histogram":[91.07,5.91,3.02],"p75":1138,"rank":"good"},"FID":{"histogram":[90.82,6.02,3.16],"p75":23,"rank":"good"},"LCP":{"histogram":[91.26,4.38,4.36],"p75":1458,"rank":"good"},"TTFB":{"histogram":[85.72,10.94,3.34],"p75":643,"rank":"good"},"INP":{"histogram":[78.62,16.48,4.9],"p75":183,"rank":"good"}},{"url":"https://m.ceneo.pl/","CLS":{"histogram":[87.8,10.23,1.97],"p75":"0.08","rank":"good"},"FCP":{"histogram":[86.98,8.11,4.91],"p75":1330,"rank":"good"},"FID":{"histogram":[78.96,15.05,5.99],"p75":80,"rank":"good"},"LCP":{"histogram":[82.97,11.28,5.75],"p75":2069,"rank":"good"},"TTFB":{"histogram":[78.41,17.26,4.33],"p75":741,"rank":"good"},"INP":{"histogram":[28.93,47.47,23.6],"p75":483,"rank":"average"}},{"url":"https://www.mediaexpert.pl/","CLS":{"histogram":[64.92,31.02,4.06],"p75":"0.10","rank":"good"},"FCP":{"histogram":[88.04,7.45,4.51],"p75":1213,"rank":"good"},"FID":{"histogram":[83.5,12.3,4.2],"p75":54,"rank":"good"},"LCP":{"histogram":[91.06,5.31,3.63],"p75":1441,"rank":"good"},"TTFB":{"histogram":[86.86,9.81,3.32],"p75":517,"rank":"good"},"INP":{"histogram":[50.59,36.98,12.43],"p75":339,"rank":"average"}},{"url":"https://www.empik.com/","CLS":{"histogram":[49.69,37.19,13.12],"p75":"0.19","rank":"average"},"FCP":{"histogram":[84.44,9.52,6.04],"p75":1372,"rank":"good"},"FID":{"histogram":[61.38,22.09,16.52],"p75":197,"rank":"average"},"LCP":{"histogram":[72.38,13.77,13.85],"p75":2715,"rank":"average"},"TTFB":{"histogram":[87.39,8.43,4.18],"p75":529,"rank":"good"},"INP":{"histogram":[21.27,35.99,42.74],"p75":756,"rank":"poor"}},{"url":"https://www.doz.pl/","CLS":{"histogram":[95.15,3.1,1.75],"p75":"0.00","rank":"good"},"FCP":{"histogram":[87.38,8.16,4.47],"p75":1290,"rank":"good"},"FID":{"histogram":[88.27,8.38,3.34],"p75":32,"rank":"good"},"LCP":{"histogram":[90.42,6.2,3.37],"p75":1620,"rank":"good"},"TTFB":{"histogram":[82.27,13.84,3.88],"p75":669,"rank":"good"},"INP":{"histogram":[62.06,30.94,7],"p75":258,"rank":"average"}},{"url":"https://www.amazon.pl/","CLS":{"histogram":[93.74,4.01,2.25],"p75":"0.01","rank":"good"},"FCP":{"histogram":[88.52,7.51,3.96],"p75":1185,"rank":"good"},"FID":{"histogram":[93.75,5.49,0.76],"p75":26,"rank":"good"},"LCP":{"histogram":[90.91,5.65,3.44],"p75":1393,"rank":"good"},"TTFB":{"histogram":[84.74,11.91,3.35],"p75":606,"rank":"good"},"INP":{"histogram":[79.04,18.72,2.25],"p75":182,"rank":"good"}},{"url":"https://www.castorama.pl/","CLS":{"histogram":[33.61,2.12,64.27],"p75":"0.59","rank":"poor"},"FCP":{"histogram":[79.32,13.95,6.73],"p75":1674,"rank":"good"},"FID":{"histogram":[59.93,13.52,26.55],"p75":343,"rank":"poor"},"LCP":{"histogram":[75.25,15.62,9.14],"p75":2497,"rank":"good"},"TTFB":{"histogram":[67.08,27.57,5.35],"p75":996,"rank":"average"},"INP":{"histogram":[6.4,26.04,67.55],"p75":1359,"rank":"poor"}},{"url":"https://allegrolokalnie.pl/","CLS":{"histogram":[92.31,7.04,0.65],"p75":"0.02","rank":"good"},"FCP":{"histogram":[93.76,3.82,2.42],"p75":948,"rank":"good"},"FID":{"histogram":[88.15,8.44,3.41],"p75":29,"rank":"good"},"LCP":{"histogram":[80.89,12.3,6.81],"p75":2201,"rank":"good"},"TTFB":{"histogram":[92.22,5.99,1.79],"p75":419,"rank":"good"},"INP":{"histogram":[77.86,18.07,4.08],"p75":186,"rank":"good"}}]}
{"params":{"formFactor":"PHONE","origin":true,"collectionPeriod":{"firstDate":{"year":2024,"month":1,"day":30},"lastDate":{"year":2024,"month":2,"day":26}},"date":"28.02.2024"},"metrics":[{"url":"https://allegro.pl","CLS":{"histogram":[96.65,1.97,1.38],"p75":"0.00","rank":"good"},"FCP":{"histogram":[91.86,6.13,2.01],"p75":1294,"rank":"good"},"FID":{"histogram":[95.56,3.44,0.99],"p75":21,"rank":"good"},"LCP":{"histogram":[93.73,4.16,2.1],"p75":1525,"rank":"good"},"TTFB":{"histogram":[74.5,23.38,2.12],"p75":807,"rank":"average"},"INP":{"histogram":[74.88,21.88,3.24],"p75":201,"rank":"average"}},{"url":"https://m.ceneo.pl","CLS":{"histogram":[95.23,2.36,2.41],"p75":"0.00","rank":"good"},"FCP":{"histogram":[84.03,11.5,4.47],"p75":1507,"rank":"good"},"FID":{"histogram":[87.6,9.09,3.31],"p75":26,"rank":"good"},"LCP":{"histogram":[87.92,7.99,4.09],"p75":1807,"rank":"good"},"TTFB":{"histogram":[65.19,28.74,6.07],"p75":971,"rank":"average"},"INP":{"histogram":[39.86,44.33,15.8],"p75":397,"rank":"average"}},{"url":"https://www.mediaexpert.pl","CLS":{"histogram":[84.5,10.77,4.73],"p75":"0.05","rank":"good"},"FCP":{"histogram":[85.51,9.35,5.14],"p75":1251,"rank":"good"},"FID":{"histogram":[90.99,5.72,3.29],"p75":19,"rank":"good"},"LCP":{"histogram":[83.65,10.18,6.17],"p75":1893,"rank":"good"},"TTFB":{"histogram":[83.31,9.99,6.69],"p75":562,"rank":"good"},"INP":{"histogram":[43.29,35.73,20.98],"p75":445,"rank":"average"}},{"url":"https://www.empik.com","CLS":{"histogram":[67.42,17.92,14.66],"p75":"0.15","rank":"average"},"FCP":{"histogram":[82.34,12.5,5.16],"p75":1559,"rank":"good"},"FID":{"histogram":[83.49,10.86,5.65],"p75":45,"rank":"good"},"LCP":{"histogram":[78.74,13.07,8.19],"p75":2284,"rank":"good"},"TTFB":{"histogram":[63.06,29.53,7.4],"p75":1034,"rank":"average"},"INP":{"histogram":[36.9,36.19,26.91],"p75":528,"rank":"poor"}},{"url":"https://www.doz.pl","CLS":{"histogram":[93.43,4.44,2.14],"p75":"0.04","rank":"good"},"FCP":{"histogram":[90.05,6.51,3.44],"p75":1189,"rank":"good"},"FID":{"histogram":[94.73,3.7,1.57],"p75":15,"rank":"good"},"LCP":{"histogram":[92.54,4.99,2.47],"p75":1527,"rank":"good"},"TTFB":{"histogram":[84.5,12.46,3.04],"p75":644,"rank":"good"},"INP":{"histogram":[77.78,17.63,4.6],"p75":186,"rank":"good"}},{"url":"https://www.amazon.pl","CLS":{"histogram":[77.66,18.45,3.89],"p75":"0.08","rank":"good"},"FCP":{"histogram":[86.44,9.67,3.89],"p75":1355,"rank":"good"},"FID":{"histogram":[94.8,4.39,0.81],"p75":22,"rank":"good"},"LCP":{"histogram":[89.64,7.64,2.72],"p75":1768,"rank":"good"},"TTFB":{"histogram":[82.57,13.96,3.47],"p75":634,"rank":"good"},"INP":{"histogram":[79.96,16.77,3.26],"p75":177,"rank":"good"}},{"url":"https://m.aliexpress.com","CLS":{"histogram":[63.18,25.9,10.92],"p75":"0.12","rank":"average"},"FCP":{"histogram":[85.82,8.83,5.35],"p75":1241,"rank":"good"},"FID":{"histogram":[96.7,2.65,0.65],"p75":13,"rank":"good"},"LCP":{"histogram":[75.47,14.17,10.36],"p75":2478,"rank":"good"},"TTFB":{"histogram":[82.43,11.95,5.62],"p75":590,"rank":"good"},"INP":{"histogram":[72.71,20.69,6.6],"p75":216,"rank":"average"}},{"url":"https://allegrolokalnie.pl","CLS":{"histogram":[89.29,8.64,2.08],"p75":"0.02","rank":"good"},"FCP":{"histogram":[93.51,4.34,2.15],"p75":1052,"rank":"good"},"FID":{"histogram":[95.7,2.99,1.31],"p75":15,"rank":"good"},"LCP":{"histogram":[84.59,10.5,4.91],"p75":1981,"rank":"good"},"TTFB":{"histogram":[89.77,8.6,1.64],"p75":575,"rank":"good"},"INP":{"histogram":[79.8,16.34,3.85],"p75":176,"rank":"good"}}]}
2 changes: 1 addition & 1 deletion src/_data/ecommerce-pl-url.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"params":{"formFactor":"PHONE","url":true,"collectionPeriod":{"firstDate":{"year":2024,"month":1,"day":29},"lastDate":{"year":2024,"month":2,"day":25}},"date":"28.02.2024"},"metrics":[{"url":"https://allegro.pl/","CLS":{"histogram":[98.07,0.89,1.04],"p75":"0.00","rank":"good"},"FCP":{"histogram":[91.07,5.91,3.02],"p75":1138,"rank":"good"},"FID":{"histogram":[90.82,6.02,3.16],"p75":23,"rank":"good"},"LCP":{"histogram":[91.26,4.38,4.36],"p75":1458,"rank":"good"},"TTFB":{"histogram":[85.72,10.94,3.34],"p75":643,"rank":"good"},"INP":{"histogram":[78.62,16.48,4.9],"p75":183,"rank":"good"}},{"url":"https://m.ceneo.pl/","CLS":{"histogram":[87.8,10.23,1.97],"p75":"0.08","rank":"good"},"FCP":{"histogram":[86.98,8.11,4.91],"p75":1330,"rank":"good"},"FID":{"histogram":[78.96,15.05,5.99],"p75":80,"rank":"good"},"LCP":{"histogram":[82.97,11.28,5.75],"p75":2069,"rank":"good"},"TTFB":{"histogram":[78.41,17.26,4.33],"p75":741,"rank":"good"},"INP":{"histogram":[28.93,47.47,23.6],"p75":483,"rank":"average"}},{"url":"https://www.mediaexpert.pl/","CLS":{"histogram":[64.92,31.02,4.06],"p75":"0.10","rank":"good"},"FCP":{"histogram":[88.04,7.45,4.51],"p75":1213,"rank":"good"},"FID":{"histogram":[83.5,12.3,4.2],"p75":54,"rank":"good"},"LCP":{"histogram":[91.06,5.31,3.63],"p75":1441,"rank":"good"},"TTFB":{"histogram":[86.86,9.81,3.32],"p75":517,"rank":"good"},"INP":{"histogram":[50.59,36.98,12.43],"p75":339,"rank":"average"}},{"url":"https://www.empik.com/","CLS":{"histogram":[49.69,37.19,13.12],"p75":"0.19","rank":"average"},"FCP":{"histogram":[84.44,9.52,6.04],"p75":1372,"rank":"good"},"FID":{"histogram":[61.38,22.09,16.52],"p75":197,"rank":"average"},"LCP":{"histogram":[72.38,13.77,13.85],"p75":2715,"rank":"average"},"TTFB":{"histogram":[87.39,8.43,4.18],"p75":529,"rank":"good"},"INP":{"histogram":[21.27,35.99,42.74],"p75":756,"rank":"poor"}},{"url":"https://www.doz.pl/","CLS":{"histogram":[95.15,3.1,1.75],"p75":"0.00","rank":"good"},"FCP":{"histogram":[87.38,8.16,4.47],"p75":1290,"rank":"good"},"FID":{"histogram":[88.27,8.38,3.34],"p75":32,"rank":"good"},"LCP":{"histogram":[90.42,6.2,3.37],"p75":1620,"rank":"good"},"TTFB":{"histogram":[82.27,13.84,3.88],"p75":669,"rank":"good"},"INP":{"histogram":[62.06,30.94,7],"p75":258,"rank":"average"}},{"url":"https://www.amazon.pl/","CLS":{"histogram":[93.74,4.01,2.25],"p75":"0.01","rank":"good"},"FCP":{"histogram":[88.52,7.51,3.96],"p75":1185,"rank":"good"},"FID":{"histogram":[93.75,5.49,0.76],"p75":26,"rank":"good"},"LCP":{"histogram":[90.91,5.65,3.44],"p75":1393,"rank":"good"},"TTFB":{"histogram":[84.74,11.91,3.35],"p75":606,"rank":"good"},"INP":{"histogram":[79.04,18.72,2.25],"p75":182,"rank":"good"}},{"url":"https://www.castorama.pl/","CLS":{"histogram":[33.61,2.12,64.27],"p75":"0.59","rank":"poor"},"FCP":{"histogram":[79.32,13.95,6.73],"p75":1674,"rank":"good"},"FID":{"histogram":[59.93,13.52,26.55],"p75":343,"rank":"poor"},"LCP":{"histogram":[75.25,15.62,9.14],"p75":2497,"rank":"good"},"TTFB":{"histogram":[67.08,27.57,5.35],"p75":996,"rank":"average"},"INP":{"histogram":[6.4,26.04,67.55],"p75":1359,"rank":"poor"}},{"url":"https://allegrolokalnie.pl/","CLS":{"histogram":[92.31,7.04,0.65],"p75":"0.02","rank":"good"},"FCP":{"histogram":[93.76,3.82,2.42],"p75":948,"rank":"good"},"FID":{"histogram":[88.15,8.44,3.41],"p75":29,"rank":"good"},"LCP":{"histogram":[80.89,12.3,6.81],"p75":2201,"rank":"good"},"TTFB":{"histogram":[92.22,5.99,1.79],"p75":419,"rank":"good"},"INP":{"histogram":[77.86,18.07,4.08],"p75":186,"rank":"good"}}]}
{"params":{"formFactor":"PHONE","url":true,"collectionPeriod":{"firstDate":{"year":2024,"month":1,"day":30},"lastDate":{"year":2024,"month":2,"day":26}},"date":"28.02.2024"},"metrics":[{"url":"https://allegro.pl/","CLS":{"histogram":[98,0.97,1.04],"p75":"0.00","rank":"good"},"FCP":{"histogram":[91.03,5.95,3.02],"p75":1138,"rank":"good"},"FID":{"histogram":[90.94,6,3.06],"p75":23,"rank":"good"},"LCP":{"histogram":[91.35,4.38,4.27],"p75":1458,"rank":"good"},"TTFB":{"histogram":[85.73,10.93,3.34],"p75":643,"rank":"good"},"INP":{"histogram":[78.75,16.54,4.7],"p75":181,"rank":"good"}},{"url":"https://m.ceneo.pl/","CLS":{"histogram":[88.26,9.75,1.99],"p75":"0.08","rank":"good"},"FCP":{"histogram":[87.17,8.07,4.76],"p75":1324,"rank":"good"},"FID":{"histogram":[79.36,14.8,5.84],"p75":79,"rank":"good"},"LCP":{"histogram":[83.45,10.97,5.58],"p75":2043,"rank":"good"},"TTFB":{"histogram":[78.53,17.19,4.28],"p75":739,"rank":"good"},"INP":{"histogram":[29.49,47.51,22.99],"p75":477,"rank":"average"}},{"url":"https://www.mediaexpert.pl/","CLS":{"histogram":[64.44,31.36,4.2],"p75":"0.10","rank":"good"},"FCP":{"histogram":[87.93,7.62,4.45],"p75":1208,"rank":"good"},"FID":{"histogram":[83.37,12.38,4.25],"p75":54,"rank":"good"},"LCP":{"histogram":[91.08,5.26,3.66],"p75":1438,"rank":"good"},"TTFB":{"histogram":[86.89,9.73,3.38],"p75":515,"rank":"good"},"INP":{"histogram":[50.45,37.02,12.53],"p75":341,"rank":"average"}},{"url":"https://www.empik.com/","CLS":{"histogram":[49.68,37.04,13.28],"p75":"0.19","rank":"average"},"FCP":{"histogram":[84.6,9.47,5.93],"p75":1370,"rank":"good"},"FID":{"histogram":[61.32,21.98,16.69],"p75":198,"rank":"average"},"LCP":{"histogram":[72.18,13.76,14.06],"p75":2712,"rank":"average"},"TTFB":{"histogram":[87.3,8.4,4.3],"p75":531,"rank":"good"},"INP":{"histogram":[21.19,35.97,42.84],"p75":756,"rank":"poor"}},{"url":"https://www.doz.pl/","CLS":{"histogram":[95.1,3.13,1.77],"p75":"0.00","rank":"good"},"FCP":{"histogram":[87.5,8.05,4.45],"p75":1287,"rank":"good"},"FID":{"histogram":[87.97,8.58,3.45],"p75":33,"rank":"good"},"LCP":{"histogram":[90.78,5.97,3.25],"p75":1618,"rank":"good"},"TTFB":{"histogram":[82.21,13.92,3.88],"p75":668,"rank":"good"},"INP":{"histogram":[62.07,30.94,6.99],"p75":259,"rank":"average"}},{"url":"https://www.amazon.pl/","CLS":{"histogram":[93.43,4.19,2.39],"p75":"0.01","rank":"good"},"FCP":{"histogram":[88.51,7.41,4.08],"p75":1185,"rank":"good"},"FID":{"histogram":[93.74,5.48,0.78],"p75":26,"rank":"good"},"LCP":{"histogram":[90.93,5.68,3.39],"p75":1390,"rank":"good"},"TTFB":{"histogram":[84.75,11.97,3.29],"p75":606,"rank":"good"},"INP":{"histogram":[78.93,18.76,2.31],"p75":182,"rank":"good"}},{"url":"https://www.castorama.pl/","CLS":{"histogram":[33.83,2.05,64.12],"p75":"0.58","rank":"poor"},"FCP":{"histogram":[79.56,13.74,6.7],"p75":1666,"rank":"good"},"FID":{"histogram":[59.95,13.46,26.59],"p75":340,"rank":"poor"},"LCP":{"histogram":[75.45,15.67,8.88],"p75":2491,"rank":"good"},"TTFB":{"histogram":[66.8,27.74,5.46],"p75":995,"rank":"average"},"INP":{"histogram":[6.5,25.97,67.53],"p75":1362,"rank":"poor"}},{"url":"https://allegrolokalnie.pl/","CLS":{"histogram":[92.54,6.82,0.65],"p75":"0.02","rank":"good"},"FCP":{"histogram":[93.64,4.03,2.34],"p75":949,"rank":"good"},"FID":{"histogram":[88.17,8.59,3.24],"p75":29,"rank":"good"},"LCP":{"histogram":[80.7,12.45,6.85],"p75":2202,"rank":"good"},"TTFB":{"histogram":[92.24,5.97,1.79],"p75":420,"rank":"good"},"INP":{"histogram":[77.84,18.13,4.03],"p75":185,"rank":"good"}}]}
Loading

0 comments on commit 5065823

Please sign in to comment.