diff --git a/package-lock.json b/package-lock.json index afd0ebd..fcf098c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -85,6 +85,14 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.1.tgz", "integrity": "sha512-zg7Hz2k5lI8kb7U32998pRRFin7zJlkfezGJjUc2heaD4Pw2wObakCDVzkKztTm/Ln7eiVvYsjqak0Ed4LkMDA==" }, + "axios": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.20.0.tgz", + "integrity": "sha512-ANA4rr2BDcmmAQLOKft2fufrtuvlqR+cXNNinUmvfeSNCOF98PZL+7M/v1zIdGo7OLjEA9J2gXJL+j4zGsl0bA==", + "requires": { + "follow-redirects": "^1.10.0" + } + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -580,18 +588,23 @@ "pinkie-promise": "^2.0.0" } }, + "follow-redirects": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz", + "integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA==" + }, "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" }, "form-data": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", - "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz", + "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==", "requires": { "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", + "combined-stream": "^1.0.8", "mime-types": "^2.1.12" } }, @@ -1374,6 +1387,16 @@ "uuid": "^3.3.2" }, "dependencies": { + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, "qs": { "version": "6.5.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", diff --git a/package.json b/package.json index 95912ad..754cbb1 100644 --- a/package.json +++ b/package.json @@ -7,9 +7,11 @@ "author": "patrick kage", "license": "MIT", "dependencies": { + "axios": "^0.20.0", "ejs": "^3.1.5", "express": "^4.17.1", "express-session": "^1.17.1", + "form-data": "^3.0.0", "luxon": "^1.19.3", "nightmare": "^3.0.2", "passport": "^0.4.1", diff --git a/sendy.js b/sendy.js new file mode 100644 index 0000000..a1baef8 --- /dev/null +++ b/sendy.js @@ -0,0 +1,44 @@ +const axios = require('axios') +const FormData = require('form-data') + +/* + * Send a single email + */ +const add_email = async (member, sendy_info) => { + const form = new FormData() + form.append('api_key', sendy_info.api_key) + form.append('name', member.name.full) + form.append('email', `s${member.student}@ed.ac.uk`) + form.append('list', sendy_info.target_list) + form.append('boolean', 'true') + + await axios.post(sendy_info.sendy_url + 'subscribe', form, { headers: form.getHeaders() }) +} + +const sendy_sync = async (members, sendy_info) => { + // Got to send these in small batches and sequentially, because sendy is kinda shit. + + const partial_add_email = m => add_email(m, sendy_info) + + // split into batches of 10 + const BATCH_SIZE = 10 + const members_batched = members + .reduce((a, c, i) => { + // if we're at BATCH_SIZE or starting off, add a sub array + if (0 == i % BATCH_SIZE) { + a.push([]) + } + + // add to the last batch + a[a.length - 1].push(c) + + return a + }, []) + + for (let batch of members_batched) { + // resolve all these guys + await Promise.all(batch.map(partial_add_email)) + } +} + +module.exports = sendy_sync diff --git a/server.js b/server.js index c63f733..77b89fd 100644 --- a/server.js +++ b/server.js @@ -1,5 +1,6 @@ const express = require('express') const scrape_members = require('./scrape.js') +const sendy_sync = require('./sendy') const fs = require('fs') const fsAsync = require('fs').promises const { DateTime } = require('luxon') @@ -81,7 +82,12 @@ api_app.get('/refresh', async (req, res) => { }) api_app.get('/sendy_sync', async (req, res) => { - + try { + const subscribed = await sendy_sync((await readScrape()).members, secrets.sendy) + res.json({ success: true }) + } catch (e) { + res.json({ success: false, status: 'An error occurred.' }) + } }) /* --- FRONTEND --- */ @@ -169,11 +175,16 @@ app.get('/dashboard', latest, csv: makecsv(latest.members), render_time: new Date().toISOString(), + sendy: { + url: secrets.sendy.sendy_url, + list: secrets.sendy.target_list + }, apikey: apikey, user: req.user }) } ) + app.get('/dashboard/rescrape', login_guard, (req, res) => { @@ -184,6 +195,16 @@ app.get('/dashboard/rescrape', } ) +app.get('/dashboard/sendy_sync', + login_guard, + (req, res) => { + res.render('sendy_sync', { + apikey, + user: req.user + }) + } +) + // mount the api application @@ -193,7 +214,6 @@ app.use('/api', api_app) console.log('getting initial scrape...') -const dummy = async () => {} -//writeScrape() -dummy() +//const dummy = async () => {} // useful for debugging +writeScrape() .then(() => app.listen(port, () => console.log(`EUSA members api listening on port ${port}!`))) diff --git a/views/dash.ejs b/views/dash.ejs index 8c3feec..a0e3aa2 100644 --- a/views/dash.ejs +++ b/views/dash.ejs @@ -26,7 +26,7 @@

Dashboard

-

General information about the API status.

+

General information about the Members API.