From fdd837128aedbf00d8c6123ac539ef0f22327c9c Mon Sep 17 00:00:00 2001 From: Dominique Hazael-Massieux Date: Tue, 14 Nov 2023 18:29:38 +0100 Subject: [PATCH] Refactor prepare-data with library and test suite --- lib/compute-group-data.js | 103 +++++++++++++++++++ prepare-data.js | 119 ++-------------------- test/compute-group-data.js | 199 +++++++++++++++++++++++++++++++++++++ 3 files changed, 312 insertions(+), 109 deletions(-) create mode 100644 lib/compute-group-data.js create mode 100644 test/compute-group-data.js diff --git a/lib/compute-group-data.js b/lib/compute-group-data.js new file mode 100644 index 000000000..6372a55f7 --- /dev/null +++ b/lib/compute-group-data.js @@ -0,0 +1,103 @@ +const shortType = { + 'business group': 'bg', + 'community group': 'cg', + 'interest group': 'ig', + 'working group': 'wg' +}; + +const listMonthsSince = d => { + const now = new Date(); + let monthCur = new Date(d.valueOf()); + const months = []; + while (monthCur < now) { + monthCur.setUTCDate(1); + months.push(monthCur.toJSON().slice(0,7)); + monthCur.setMonth(monthCur.getMonth() + 1); + } + return months; +}; + +function computeGroupData(data, staff) { + const cgData = {}; + const staffids = Array.isArray(staff) ? staff.map(s => s._links.self.href) : []; + cgData.id = data[0].id; + cgData.name = data[0].name; + cgData.type = shortType[data[0].type]; + cgData.shortname = data[0].shortname; + cgData.link = data[0]._links.homepage.href; + cgData["spec-publisher"] = data[0]["spec-publisher"]; + // Approximating creation date to date of first person joining + cgData.created = new Date(data[0]["start-date"] || ((data[4][0] || {}).created + "Z")); + cgData.participants = data[4].length; + cgData.chairs = data[2].filter(x => x).map(c => c.title); + cgData.staff = data[4].filter(u => staffids.includes(u._links?.user?.href)).map(u => { const team = staff.find(s => s._links.self.href === u._links.user.href); return { name: team.name, photo: team._links.photos?.find(p => p.name === "tiny").href } ;}); + + cgData.repositories = []; + cgData.activity = {}; + + const monthsSinceCreation = listMonthsSince(cgData.created); + + // Merge data coming from validate-repos assoication of groups/repos into the other list of data fetched from services db + if (data[1] && data[1].length) { + if (!data[3]) { + data[3] = []; + } + data[1].forEach(({items}) => { + const repoUrl = items.map(i => (i.html_url || '').split('/').slice(0,5).join('/')).filter(x => x)[0]; + if (!data[3].find(({service}) => service.type === 'repository' && (service.link === repoUrl || service.link === repoUrl + '/'))) { + data[3].push({service: {type: 'repository'}, data: {items}}); + } + }); + } + + if (data[3] && data[3].length) { + // treat forums as mailing lists + data[3].forEach(({service}) => { + if (service.type === "forum") service.type = "lists"; + cgData.repositories = cgData.repositories.concat( + ...data[3].filter(({service}) => service.type === "repository") + .map(({data}) => { + if (!data.items) return []; + return data.items.map(i => (i.html_url || '').split('/').slice(0,5).join('/'))}) + ).concat(data[3].filter(({service}) => service.type === "repository").map(({service}) => service.link)); + }); + cgData.repositories = [...new Set(cgData.repositories.filter(x => x))]; + + // aggregate by service type + data[3].forEach(({service, data}) => { + let perMonthData; + if (data && data.items) { + perMonthData = monthsSinceCreation + .reduce((acc, m) => { + acc[m] = data.items.filter(i => (i.isoDate && i.isoDate.startsWith(m)) || (i.created_at && i.created_at.startsWith(m)) || (i.commit && i.commit.committer && i.commit.committer.date && i.commit.committer.date.startsWith(m)) ).length; + return acc; + }, {}); + } else if (data && typeof data === "object" && Object.keys(data).length) { + perMonthData = data; + } else { + // console.error("Missing data for " + service.type + " of " + cgData.name); + } + if (!perMonthData) return; + if (cgData.activity[service.type]) { + cgData.activity[service.type] = Object.keys(perMonthData).reduce((acc, m) => { + if (!acc[m]) { + acc[m] = 0; + } + acc[m] += perMonthData[m] || 0; + return acc; + }, cgData.activity[service.type]); + } else { + cgData.activity[service.type] = perMonthData; + } + }); + } + + cgData.activity['join'] = monthsSinceCreation + .reduce((acc, m) => { + acc[m] = data[4].filter(j => j.created.startsWith(m)).length; + return acc; + }, {}); + return cgData; +} + +module.exports.computeGroupData = computeGroupData; diff --git a/prepare-data.js b/prepare-data.js index 499ede882..b582e10b3 100644 --- a/prepare-data.js +++ b/prepare-data.js @@ -1,125 +1,26 @@ const fs = require("fs"); -const util = require("util"); +const computeGroupData = require("./lib/compute-group-data"); const target = "./report.json"; const aggregatedData = []; -const listMonthsSince = d => { - const now = new Date(); - let monthCur = new Date(d.valueOf()); - const months = []; - while (monthCur < now) { - monthCur.setUTCDate(1); - months.push(monthCur.toJSON().slice(0,7)); - monthCur.setMonth(monthCur.getMonth() + 1); - } - return months; -} - -const shortType = { - 'business group': 'bg', - 'community group': 'cg', - 'interest group': 'ig', - 'working group': 'wg' -}; - -const loadDir = async dirPath => { - const files = await util.promisify(fs.readdir)(dirPath); - return util.promisify(fs.readFile)(dirPath + '/staff.json', 'utf-8') +const loadDir = async (dirPath) => { + const files = await fs.promises.readdir(dirPath); + return fs.promises.readFile(dirPath + '/staff.json', 'utf-8') .then(JSON.parse) .then(staff => Promise.all( files.filter(path => path.match(/\.json$/)).filter(path => path !== 'staff.json') .map( - path => util.promisify(fs.readFile)(dirPath + "/" + path, 'utf-8') + path => fs.promises.readFile(dirPath + "/" + path, 'utf-8') .then(JSON.parse) .catch(err => { console.error("Failed parsing " + path + ": " + err);}) - .then(data => { - const cgData = {}; - const staffids = Array.isArray(staff) ? staff.map(s => s._links.self.href) : []; - cgData.id = data[0].id; - cgData.name = data[0].name; - cgData.type = shortType[data[0].type]; - cgData.shortname = data[0].shortname; - cgData.link = data[0]._links.homepage.href; - cgData["spec-publisher"] = data[0]["spec-publisher"]; - // Approximating creation date to date of first person joining - cgData.created = new Date(data[0]["start-date"] || ((data[4][0] || {}).created + "Z")); - cgData.participants = data[4].length; - cgData.chairs = data[2].filter(x => x).map(c => c.title); - cgData.staff = data[4].filter(u => u._links.user && staffids.includes(u._links.user.href)).map(u => { const team = staff.find(s => s._links.self.href === u._links.user.href); return { name: team.name, photo: (team._links.photos ? team._links.photos.find(p => p.name === "tiny").href : undefined) } ;}); - - cgData.repositories = []; - cgData.activity = {}; - - const monthsSinceCreation = listMonthsSince(cgData.created); - - // Merge data coming from validate-repos assoication of groups/repos into the other list of data fetched from services db - if (data[1] && data[1].length) { - if (!data[3]) { - data[3] = []; - } - data[1].forEach(({items}) => { - const repoUrl = items.map(i => (i.html_url || '').split('/').slice(0,5).join('/')).filter(x => x)[0]; - if (!data[3].find(({service}) => service.type === 'repository' && (service.link === repoUrl || service.link === repoUrl + '/'))) { - data[3].push({service: {type: 'repository'}, data: {items}}); - } - }); - } - - if (data[3] && data[3].length) { - // treat forums as mailing lists - data[3].forEach(({service}) => { - if (service.type === "forum") service.type = "lists"; - cgData.repositories = cgData.repositories.concat( - ...data[3].filter(({service}) => service.type === "repository") - .map(({data}) => { - if (!data.items) return []; - return data.items.map(i => (i.html_url || '').split('/').slice(0,5).join('/'))}) - ).concat(data[3].filter(({service}) => service.type === "repository").map(({service}) => service.link)); - }); - cgData.repositories = [...new Set(cgData.repositories.filter(x => x))]; - - // aggregate by service type - data[3].forEach(({service, data}) => { - let perMonthData; - if (data && data.items) { - perMonthData = monthsSinceCreation - .reduce((acc, m) => { - acc[m] = data.items.filter(i => (i.isoDate && i.isoDate.startsWith(m)) || (i.created_at && i.created_at.startsWith(m)) || (i.commit && i.commit.committer && i.commit.committer.date && i.commit.committer.date.startsWith(m)) ).length; - return acc; - }, {}); - } else if (data && typeof data === "object" && Object.keys(data).length) { - perMonthData = data; - } else { - // console.error("Missing data for " + service.type + " of " + cgData.name); - } - if (!perMonthData) return; - if (cgData.activity[service.type]) { - cgData.activity[service.type] = Object.keys(perMonthData).reduce((acc, m) => { - if (!acc[m]) { - acc[m] = 0; - } - acc[m] += perMonthData[m] || 0; - return acc; - }, cgData.activity[service.type]); - } else { - cgData.activity[service.type] = perMonthData; - } - }); - } - - cgData.activity['join'] = monthsSinceCreation - .reduce((acc, m) => { - acc[m] = data[4].filter(j => j.created.startsWith(m)).length; - return acc; - }, {}); - return cgData; - }).catch(e => console.error("Error while dealing with " + path + ":" + JSON.stringify(e.stack))) - )) - ); + .then(data => computeGroupData(data, staff)).catch(e => console.error("Error while dealing with " + path + ":" + JSON.stringify(e.stack))) + ) + )); }; loadDir("./data").then(data => { - fs.writeFileSync('./report.json', JSON.stringify({timestamp: new Date(), data: data.filter(x => x)}, null, 2)); + fs.writeFileSync(target, JSON.stringify({timestamp: new Date(), data: data.filter(x => x)}, null, 2)); }); + diff --git a/test/compute-group-data.js b/test/compute-group-data.js new file mode 100644 index 000000000..f27e9120e --- /dev/null +++ b/test/compute-group-data.js @@ -0,0 +1,199 @@ +/** + * Tests the Group data wrangler + */ +/* global describe, it */ + +const {computeGroupData} = require("../lib/compute-group-data"); +const assert = require("assert"); + +const baseCgData = [ + { + id: 42, + name: "test group", + type: "community group", + shortname: "test", + _links : { homepage: { href: "https://example.com" } } + }, + [], // WG repository data + [{title: "Standing Couch"}], // chairs + [], // activity data + [{ + // this will be used as the default creation date in many tests, + // so mock activity needs to be later + created: "2023-08-14T15:44:00", + _links: { user: { href: 'https://example.test/user-id' } } + }] // participants +]; + +describe('The Group data wrangler', function () { + it('observes start date of the group', () => { + const testData = structuredClone(baseCgData); + testData[0]["start-date"] = "2011-05-05"; + const cgData = computeGroupData(testData, []); + assert.equal(cgData.created, new Date("2011-05-05").toString()); + }); + + it('assumes start date from first participant join date when start-date not set', () => { + const testData = structuredClone(baseCgData); + const cgData = computeGroupData(testData, []); + assert.equal(cgData.created, new Date("2023-08-14T15:44:00Z").toString()); + }); + + it('calculates number of participants', () => { + const testData = structuredClone(baseCgData); + const cgData = computeGroupData(testData, []); + assert.equal(cgData.participants, 1); + }); + + it('lists staff in the group with their picture', () => { + const testData = structuredClone(baseCgData); + const cgData = computeGroupData(testData, [{ + name: "Stick Cane", + _links: { + self: { + href: 'https://example.test/user-id' + }, + photos: [ + { + name: "tiny", + href: "https://example.test/user.jpg" + } + ] + } + }]); + assert.equal(cgData.staff.length, 1); + assert.equal(cgData.staff[0].name, "Stick Cane"); + assert.equal(cgData.staff[0].photo, "https://example.test/user.jpg"); + }); + + it('compiles mailing list activity data', () => { + const testData = structuredClone(baseCgData); + testData[3].push( + { + "service": { + "type": "lists", + + }, + "data": { + "2023-10": 3, + "2023-11": 1 + } + } + ); + const cgData = computeGroupData(testData, []); + assert.equal(cgData.activity.lists["2023-10"], 3); + }); + + it('compiles forum activity data', () => { + const testData = structuredClone(baseCgData); + testData[3].push( + { + "service": { + "type": "forum" + }, + "data": { + "items": [ + { + "created_at": "2023-10-19T19:05:56.303Z", + "topic_title": "Test Oct 1" + }, + { + "created_at": "2023-10-20T19:05:56.303Z", + "topic_title": "Test Oct 2" + }, + { + "created_at": "2023-11-19T19:05:56.303Z", + "topic_title": "Test Nov" + } + ] + } + } + ); + const cgData = computeGroupData(testData, []); + assert.equal(cgData.activity.lists["2023-10"], 2); + assert.equal(cgData.activity.lists["2023-11"], 1); + }); + + it('compiles repo activity data (CG mode)', () => { + const testData = structuredClone(baseCgData); + testData[3].push( + { + "service": { + "type": "repository" + }, + "data": { + "items": [ + { + "created_at": "2023-10-19T19:05:56.303Z", + } + ] + } + } + ); + const cgData = computeGroupData(testData, []); + assert.equal(cgData.activity.repository["2023-10"], 1); + }); + + it('compiles repo activity data (WG mode)', () => { + const testData = structuredClone(baseCgData); + testData[1].push( + { + items: [ + { + "created_at": "2023-10-19T19:05:56.303Z", + } + ] + } + ); + const cgData = computeGroupData(testData, []); + assert.equal(cgData.activity.repository["2023-10"], 1); + }); + + + it('compiles wiki activity data', () => { + const testData = structuredClone(baseCgData); + testData[3].push( + { + "service": { + "type": "wiki" + }, + "data": { + "items": [ + { + "isoDate": "2023-10-18T10:02:10.000Z" + } + ] + } + } + ); + const cgData = computeGroupData(testData, []); + assert.equal(cgData.activity.wiki["2023-10"], 1); + }); + + it('compiles blog activity data', () => { + const testData = structuredClone(baseCgData); + testData[3].push( + { + "service": { + "type": "rss" + }, + "data": { + "items": [ + { + "isoDate": "2023-10-18T10:02:10.000Z" + } + ] + } + } + ); + const cgData = computeGroupData(testData, []); + assert.equal(cgData.activity.rss["2023-10"], 1); + }); + + it('compiles join activity data', () => { + const testData = structuredClone(baseCgData); + const cgData = computeGroupData(testData, []); + assert.equal(cgData.activity.join["2023-08"], 1); + }); + +});