diff --git a/package.json b/package.json index e603b85..550f247 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "license": "MIT", "dependencies": { "express": "^4.17.1", + "luxon": "^1.19.3", "nightmare": "^3.0.2" } } diff --git a/scrape.js b/scrape.js index 8aca568..2a32857 100644 --- a/scrape.js +++ b/scrape.js @@ -1,11 +1,29 @@ -const Nightmare = require('nightmare') -const nightmare = Nightmare({ show: true }) +const Nightmare = require('nightmare') +const { DateTime } = require('luxon') -const fs = require('fs') +const fs = require('fs') const secrets = JSON.parse(fs.readFileSync('./instance/secret.json')) -module.exports = () => { - const nightmare = Nightmare({ show: false }) +const convertFromEUSADate = edate => { + return DateTime + .fromFormat(edate, 'dd/MM/yyyy HH:mm') + .toISO() +} + +const parseNameString = name => { + const [ surname, forename ] = name.split(',') + + return { + original: name, + last: surname.trim(), + first: forename.trim(), + full: `${forename} ${surname}`.trim() + } +} + +module.exports = opts => { + const DEBUG = (opts && opts.debug) || false + const nightmare = Nightmare({ show: DEBUG }) return new Promise((resolve, reject) => { nightmare @@ -17,6 +35,7 @@ module.exports = () => { .click('[value=" Login now "]') .wait('.member_list_group') .evaluate(() => { + // executes in browser context let table = document.querySelector('.member_list_group > h3 > a[href="/organisation/editmembers/8868/8872/?from=members"]').parentElement.parentElement table = table.querySelector('.msl_table > tbody') @@ -29,20 +48,27 @@ module.exports = () => { name: tr.children[0].textContent, student: tr.children[1].textContent, joined: tr.children[2].textContent, - expired: tr.children[3].textContent + expires: tr.children[3].textContent }) } - return JSON.stringify({ - members: out - }) + return out }) .end() - .then(function (result) { - resolve(result) + .then(members => { + // do date conversions without having to inject into the EUSA page + // (i.e. in the node.js context) + members = members.map(member => ({ + name: parseNameString(member.name), + student: member.student, + joined: convertFromEUSADate(member.joined), + expires: convertFromEUSADate(member.expires) + })) + + resolve(members) }) - .catch(function (error) { + .catch(error => { reject(error) }) }) diff --git a/server.js b/server.js index c6c4de0..41554ae 100644 --- a/server.js +++ b/server.js @@ -1,6 +1,7 @@ const express = require('express') const scrape_members = require('./scrape.js') const fs = require('fs') +const { DateTime } = require('luxon') const config = JSON.parse(fs.readFileSync('./instance/config.json')) @@ -12,7 +13,7 @@ const writeScrape = async () => { const members = await scrape_members() const out = { - members: JSON.parse(members).members, + members: members, date: new Date().toISOString() } diff --git a/yarn.lock b/yarn.lock index 6028324..7aedc96 100644 --- a/yarn.lock +++ b/yarn.lock @@ -719,6 +719,11 @@ loud-rejection@^1.0.0: currently-unhandled "^0.4.1" signal-exit "^3.0.0" +luxon@^1.19.3: + version "1.19.3" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-1.19.3.tgz#86c04a1395529b4386ae235a8fe16d0a82a0695b" + integrity sha512-YwTDjGRQC0QC9Iya2g2eKZfgEFqRId4ZoLHORQcfTMB/5xrTx427V7ZPjQJ1vzvhA2vJfG2bh1Kv8V8IFMWCUA== + map-obj@^1.0.0, map-obj@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d"