diff --git a/lib/compute-group-data.js b/lib/compute-group-data.js
new file mode 100644
index 000000000..6372a55f7
--- /dev/null
+++ b/lib/compute-group-data.js
@@ -0,0 +1,103 @@
+const shortType = {
+ 'business group': 'bg',
+ 'community group': 'cg',
+ 'interest group': 'ig',
+ 'working group': 'wg'
+};
+
+const listMonthsSince = d => {
+ const now = new Date();
+ let monthCur = new Date(d.valueOf());
+ const months = [];
+ while (monthCur < now) {
+ monthCur.setUTCDate(1);
+ months.push(monthCur.toJSON().slice(0,7));
+ monthCur.setMonth(monthCur.getMonth() + 1);
+ }
+ return months;
+};
+
+function computeGroupData(data, staff) {
+ const cgData = {};
+ const staffids = Array.isArray(staff) ? staff.map(s => s._links.self.href) : [];
+ cgData.id = data[0].id;
+ cgData.name = data[0].name;
+ cgData.type = shortType[data[0].type];
+ cgData.shortname = data[0].shortname;
+ cgData.link = data[0]._links.homepage.href;
+ cgData["spec-publisher"] = data[0]["spec-publisher"];
+ // Approximating creation date to date of first person joining
+ cgData.created = new Date(data[0]["start-date"] || ((data[4][0] || {}).created + "Z"));
+ cgData.participants = data[4].length;
+ cgData.chairs = data[2].filter(x => x).map(c => c.title);
+ cgData.staff = data[4].filter(u => staffids.includes(u._links?.user?.href)).map(u => { const team = staff.find(s => s._links.self.href === u._links.user.href); return { name: team.name, photo: team._links.photos?.find(p => p.name === "tiny").href } ;});
+
+ cgData.repositories = [];
+ cgData.activity = {};
+
+ const monthsSinceCreation = listMonthsSince(cgData.created);
+
+ // Merge data coming from validate-repos assoication of groups/repos into the other list of data fetched from services db
+ if (data[1] && data[1].length) {
+ if (!data[3]) {
+ data[3] = [];
+ }
+ data[1].forEach(({items}) => {
+ const repoUrl = items.map(i => (i.html_url || '').split('/').slice(0,5).join('/')).filter(x => x)[0];
+ if (!data[3].find(({service}) => service.type === 'repository' && (service.link === repoUrl || service.link === repoUrl + '/'))) {
+ data[3].push({service: {type: 'repository'}, data: {items}});
+ }
+ });
+ }
+
+ if (data[3] && data[3].length) {
+ // treat forums as mailing lists
+ data[3].forEach(({service}) => {
+ if (service.type === "forum") service.type = "lists";
+ cgData.repositories = cgData.repositories.concat(
+ ...data[3].filter(({service}) => service.type === "repository")
+ .map(({data}) => {
+ if (!data.items) return [];
+ return data.items.map(i => (i.html_url || '').split('/').slice(0,5).join('/'))})
+ ).concat(data[3].filter(({service}) => service.type === "repository").map(({service}) => service.link));
+ });
+ cgData.repositories = [...new Set(cgData.repositories.filter(x => x))];
+
+ // aggregate by service type
+ data[3].forEach(({service, data}) => {
+ let perMonthData;
+ if (data && data.items) {
+ perMonthData = monthsSinceCreation
+ .reduce((acc, m) => {
+ acc[m] = data.items.filter(i => (i.isoDate && i.isoDate.startsWith(m)) || (i.created_at && i.created_at.startsWith(m)) || (i.commit && i.commit.committer && i.commit.committer.date && i.commit.committer.date.startsWith(m)) ).length;
+ return acc;
+ }, {});
+ } else if (data && typeof data === "object" && Object.keys(data).length) {
+ perMonthData = data;
+ } else {
+ // console.error("Missing data for " + service.type + " of " + cgData.name);
+ }
+ if (!perMonthData) return;
+ if (cgData.activity[service.type]) {
+ cgData.activity[service.type] = Object.keys(perMonthData).reduce((acc, m) => {
+ if (!acc[m]) {
+ acc[m] = 0;
+ }
+ acc[m] += perMonthData[m] || 0;
+ return acc;
+ }, cgData.activity[service.type]);
+ } else {
+ cgData.activity[service.type] = perMonthData;
+ }
+ });
+ }
+
+ cgData.activity['join'] = monthsSinceCreation
+ .reduce((acc, m) => {
+ acc[m] = data[4].filter(j => j.created.startsWith(m)).length;
+ return acc;
+ }, {});
+ return cgData;
+}
+
+module.exports.computeGroupData = computeGroupData;
diff --git a/prepare-data.js b/prepare-data.js
index 499ede882..b582e10b3 100644
--- a/prepare-data.js
+++ b/prepare-data.js
@@ -1,125 +1,26 @@
const fs = require("fs");
-const util = require("util");
+const computeGroupData = require("./lib/compute-group-data");
const target = "./report.json";
const aggregatedData = [];
-const listMonthsSince = d => {
- const now = new Date();
- let monthCur = new Date(d.valueOf());
- const months = [];
- while (monthCur < now) {
- monthCur.setUTCDate(1);
- months.push(monthCur.toJSON().slice(0,7));
- monthCur.setMonth(monthCur.getMonth() + 1);
- }
- return months;
-}
-
-const shortType = {
- 'business group': 'bg',
- 'community group': 'cg',
- 'interest group': 'ig',
- 'working group': 'wg'
-};
-
-const loadDir = async dirPath => {
- const files = await util.promisify(fs.readdir)(dirPath);
- return util.promisify(fs.readFile)(dirPath + '/staff.json', 'utf-8')
+const loadDir = async (dirPath) => {
+ const files = await fs.promises.readdir(dirPath);
+ return fs.promises.readFile(dirPath + '/staff.json', 'utf-8')
.then(JSON.parse)
.then(staff => Promise.all(
files.filter(path => path.match(/\.json$/)).filter(path => path !== 'staff.json')
.map(
- path => util.promisify(fs.readFile)(dirPath + "/" + path, 'utf-8')
+ path => fs.promises.readFile(dirPath + "/" + path, 'utf-8')
.then(JSON.parse)
.catch(err => { console.error("Failed parsing " + path + ": " + err);})
- .then(data => {
- const cgData = {};
- const staffids = Array.isArray(staff) ? staff.map(s => s._links.self.href) : [];
- cgData.id = data[0].id;
- cgData.name = data[0].name;
- cgData.type = shortType[data[0].type];
- cgData.shortname = data[0].shortname;
- cgData.link = data[0]._links.homepage.href;
- cgData["spec-publisher"] = data[0]["spec-publisher"];
- // Approximating creation date to date of first person joining
- cgData.created = new Date(data[0]["start-date"] || ((data[4][0] || {}).created + "Z"));
- cgData.participants = data[4].length;
- cgData.chairs = data[2].filter(x => x).map(c => c.title);
- cgData.staff = data[4].filter(u => u._links.user && staffids.includes(u._links.user.href)).map(u => { const team = staff.find(s => s._links.self.href === u._links.user.href); return { name: team.name, photo: (team._links.photos ? team._links.photos.find(p => p.name === "tiny").href : undefined) } ;});
-
- cgData.repositories = [];
- cgData.activity = {};
-
- const monthsSinceCreation = listMonthsSince(cgData.created);
-
- // Merge data coming from validate-repos assoication of groups/repos into the other list of data fetched from services db
- if (data[1] && data[1].length) {
- if (!data[3]) {
- data[3] = [];
- }
- data[1].forEach(({items}) => {
- const repoUrl = items.map(i => (i.html_url || '').split('/').slice(0,5).join('/')).filter(x => x)[0];
- if (!data[3].find(({service}) => service.type === 'repository' && (service.link === repoUrl || service.link === repoUrl + '/'))) {
- data[3].push({service: {type: 'repository'}, data: {items}});
- }
- });
- }
-
- if (data[3] && data[3].length) {
- // treat forums as mailing lists
- data[3].forEach(({service}) => {
- if (service.type === "forum") service.type = "lists";
- cgData.repositories = cgData.repositories.concat(
- ...data[3].filter(({service}) => service.type === "repository")
- .map(({data}) => {
- if (!data.items) return [];
- return data.items.map(i => (i.html_url || '').split('/').slice(0,5).join('/'))})
- ).concat(data[3].filter(({service}) => service.type === "repository").map(({service}) => service.link));
- });
- cgData.repositories = [...new Set(cgData.repositories.filter(x => x))];
-
- // aggregate by service type
- data[3].forEach(({service, data}) => {
- let perMonthData;
- if (data && data.items) {
- perMonthData = monthsSinceCreation
- .reduce((acc, m) => {
- acc[m] = data.items.filter(i => (i.isoDate && i.isoDate.startsWith(m)) || (i.created_at && i.created_at.startsWith(m)) || (i.commit && i.commit.committer && i.commit.committer.date && i.commit.committer.date.startsWith(m)) ).length;
- return acc;
- }, {});
- } else if (data && typeof data === "object" && Object.keys(data).length) {
- perMonthData = data;
- } else {
- // console.error("Missing data for " + service.type + " of " + cgData.name);
- }
- if (!perMonthData) return;
- if (cgData.activity[service.type]) {
- cgData.activity[service.type] = Object.keys(perMonthData).reduce((acc, m) => {
- if (!acc[m]) {
- acc[m] = 0;
- }
- acc[m] += perMonthData[m] || 0;
- return acc;
- }, cgData.activity[service.type]);
- } else {
- cgData.activity[service.type] = perMonthData;
- }
- });
- }
-
- cgData.activity['join'] = monthsSinceCreation
- .reduce((acc, m) => {
- acc[m] = data[4].filter(j => j.created.startsWith(m)).length;
- return acc;
- }, {});
- return cgData;
- }).catch(e => console.error("Error while dealing with " + path + ":" + JSON.stringify(e.stack)))
- ))
- );
+ .then(data => computeGroupData(data, staff)).catch(e => console.error("Error while dealing with " + path + ":" + JSON.stringify(e.stack)))
+ )
+ ));
};
loadDir("./data").then(data => {
- fs.writeFileSync('./report.json', JSON.stringify({timestamp: new Date(), data: data.filter(x => x)}, null, 2));
+ fs.writeFileSync(target, JSON.stringify({timestamp: new Date(), data: data.filter(x => x)}, null, 2));
});
+
diff --git a/test/compute-group-data.js b/test/compute-group-data.js
new file mode 100644
index 000000000..f27e9120e
--- /dev/null
+++ b/test/compute-group-data.js
@@ -0,0 +1,199 @@
+/**
+ * Tests the Group data wrangler
+ */
+/* global describe, it */
+
+const {computeGroupData} = require("../lib/compute-group-data");
+const assert = require("assert");
+
+const baseCgData = [
+ {
+ id: 42,
+ name: "test group",
+ type: "community group",
+ shortname: "test",
+ _links : { homepage: { href: "https://example.com" } }
+ },
+ [], // WG repository data
+ [{title: "Standing Couch"}], // chairs
+ [], // activity data
+ [{
+ // this will be used as the default creation date in many tests,
+ // so mock activity needs to be later
+ created: "2023-08-14T15:44:00",
+ _links: { user: { href: 'https://example.test/user-id' } }
+ }] // participants
+];
+
+describe('The Group data wrangler', function () {
+ it('observes start date of the group', () => {
+ const testData = structuredClone(baseCgData);
+ testData[0]["start-date"] = "2011-05-05";
+ const cgData = computeGroupData(testData, []);
+ assert.equal(cgData.created, new Date("2011-05-05").toString());
+ });
+
+ it('assumes start date from first participant join date when start-date not set', () => {
+ const testData = structuredClone(baseCgData);
+ const cgData = computeGroupData(testData, []);
+ assert.equal(cgData.created, new Date("2023-08-14T15:44:00Z").toString());
+ });
+
+ it('calculates number of participants', () => {
+ const testData = structuredClone(baseCgData);
+ const cgData = computeGroupData(testData, []);
+ assert.equal(cgData.participants, 1);
+ });
+
+ it('lists staff in the group with their picture', () => {
+ const testData = structuredClone(baseCgData);
+ const cgData = computeGroupData(testData, [{
+ name: "Stick Cane",
+ _links: {
+ self: {
+ href: 'https://example.test/user-id'
+ },
+ photos: [
+ {
+ name: "tiny",
+ href: "https://example.test/user.jpg"
+ }
+ ]
+ }
+ }]);
+ assert.equal(cgData.staff.length, 1);
+ assert.equal(cgData.staff[0].name, "Stick Cane");
+ assert.equal(cgData.staff[0].photo, "https://example.test/user.jpg");
+ });
+
+ it('compiles mailing list activity data', () => {
+ const testData = structuredClone(baseCgData);
+ testData[3].push(
+ {
+ "service": {
+ "type": "lists",
+
+ },
+ "data": {
+ "2023-10": 3,
+ "2023-11": 1
+ }
+ }
+ );
+ const cgData = computeGroupData(testData, []);
+ assert.equal(cgData.activity.lists["2023-10"], 3);
+ });
+
+ it('compiles forum activity data', () => {
+ const testData = structuredClone(baseCgData);
+ testData[3].push(
+ {
+ "service": {
+ "type": "forum"
+ },
+ "data": {
+ "items": [
+ {
+ "created_at": "2023-10-19T19:05:56.303Z",
+ "topic_title": "Test Oct 1"
+ },
+ {
+ "created_at": "2023-10-20T19:05:56.303Z",
+ "topic_title": "Test Oct 2"
+ },
+ {
+ "created_at": "2023-11-19T19:05:56.303Z",
+ "topic_title": "Test Nov"
+ }
+ ]
+ }
+ }
+ );
+ const cgData = computeGroupData(testData, []);
+ assert.equal(cgData.activity.lists["2023-10"], 2);
+ assert.equal(cgData.activity.lists["2023-11"], 1);
+ });
+
+ it('compiles repo activity data (CG mode)', () => {
+ const testData = structuredClone(baseCgData);
+ testData[3].push(
+ {
+ "service": {
+ "type": "repository"
+ },
+ "data": {
+ "items": [
+ {
+ "created_at": "2023-10-19T19:05:56.303Z",
+ }
+ ]
+ }
+ }
+ );
+ const cgData = computeGroupData(testData, []);
+ assert.equal(cgData.activity.repository["2023-10"], 1);
+ });
+
+ it('compiles repo activity data (WG mode)', () => {
+ const testData = structuredClone(baseCgData);
+ testData[1].push(
+ {
+ items: [
+ {
+ "created_at": "2023-10-19T19:05:56.303Z",
+ }
+ ]
+ }
+ );
+ const cgData = computeGroupData(testData, []);
+ assert.equal(cgData.activity.repository["2023-10"], 1);
+ });
+
+
+ it('compiles wiki activity data', () => {
+ const testData = structuredClone(baseCgData);
+ testData[3].push(
+ {
+ "service": {
+ "type": "wiki"
+ },
+ "data": {
+ "items": [
+ {
+ "isoDate": "2023-10-18T10:02:10.000Z"
+ }
+ ]
+ }
+ }
+ );
+ const cgData = computeGroupData(testData, []);
+ assert.equal(cgData.activity.wiki["2023-10"], 1);
+ });
+
+ it('compiles blog activity data', () => {
+ const testData = structuredClone(baseCgData);
+ testData[3].push(
+ {
+ "service": {
+ "type": "rss"
+ },
+ "data": {
+ "items": [
+ {
+ "isoDate": "2023-10-18T10:02:10.000Z"
+ }
+ ]
+ }
+ }
+ );
+ const cgData = computeGroupData(testData, []);
+ assert.equal(cgData.activity.rss["2023-10"], 1);
+ });
+
+ it('compiles join activity data', () => {
+ const testData = structuredClone(baseCgData);
+ const cgData = computeGroupData(testData, []);
+ assert.equal(cgData.activity.join["2023-08"], 1);
+ });
+
+});