update staging with latest main commits

.gitignore

@@ -78,6 +78,7 @@ web_modules/
 .env.test.local
 .env.production.local
 .env.local
+.envrc
 
 # parcel-bundler cache (https://parceljs.org/)
 .cache

config/index.js

@@ -1,22 +1,25 @@
 import HmpoConfig from 'hmpo-config'
 import yaml from 'js-yaml'
 import fs from 'fs'
-import _ from 'lodash'
 
 const readConfig = (config) => {
   return yaml.load(fs.readFileSync(`./config/${config}.yaml`, 'utf8'))
 }
 
+/**
+ *
+ * @returns {{defaultConfig, environment}}
+ */
 const getConfig = () => {
   const defaultConfig = readConfig('default')
 
   const environment = process.env.NODE_ENV || process.env.ENVIRONMENT || 'production'
 
-  console.log('USING CONFIG: ' + environment)
+  if (environment !== 'test') console.info('USING CONFIG: ' + environment)
 
   const customConfig = readConfig(environment)
 
-  const combinedConfig = _.merge({}, defaultConfig, customConfig)
+  const combinedConfig = Object.assign({}, defaultConfig, customConfig)
 
   const config = new HmpoConfig()
   config.addConfig(combinedConfig)

config/production.yaml

@@ -10,4 +10,8 @@ redis: {
     host: 'production-pub-async-redis-eihmmv.serverless.euw2.cache.amazonaws.com',
     port: '6379'
 }
-url: 'https://publish.digital-land.info/'
+url: 'https://publish.digital-land.info/'
+smartlook: {
+  key: '6383ab27d6f6a02f043a518bea629cd232dc0131',
+  region: 'eu'
+}

config/test.yaml

@@ -1,6 +1,7 @@
 asyncRequestApi: {
     url: http://localhost:8001,
-    port: 8001
+    port: 8001,
+    requestsEndpoint: 'requests'
 }
 aws: {
     region: eu-west-2,

package.json

@@ -73,7 +73,6 @@
     "hmpo-form-wizard": "^13.0.0",
     "hmpo-i18n": "^6.0.1",
     "js-yaml": "^4.1.0",
-    "lodash": "^4.17.21",
     "maplibre-gl": "^4.1.0",
     "multer": "^1.4.5-lts.1",
     "notifications-node-client": "^8.2.0",
@@ -94,7 +93,8 @@
   },
   "standard": {
     "ignore": [
-      "performance-test/"
+      "performance-test/",
+      "src/views/common/smartlook.js"
     ]
   }
 }

src/assets/js/statusPage.js

@@ -1,6 +1,7 @@
 // poll the server for the status of the job
 
 import { finishedProcessingStatuses } from '../../utils/utils.js'
+import logger from '../../utils/logger.js'
 
 export default class StatusPage {
   constructor (pollingInterval, maxPollAttempts) {
@@ -36,7 +37,7 @@ export default class StatusPage {
       fetch(statusEndpoint)
         .then(res => res.json())
         .then(data => {
-          console.log('polled request and got a status of: ' + data.status)
+          logger.info('StatusPage: polled request and got a status of: ' + data.status)
           // ToDo: handle other status' here
           if (finishedProcessingStatuses.includes(data.status)) {
             this.updatePageToComplete()
@@ -46,7 +47,7 @@ export default class StatusPage {
 
       this.pollAttempts++
       if (this.pollAttempts > this.maxPollAttempts) {
-        console.log('polling timed out')
+        logger.info('StatusPage: polling timed out')
         this.updatePageForPollingTimeout()
         clearInterval(interval)
       }

src/controllers/submitUrlController.js

@@ -17,7 +17,7 @@ class SubmitUrlController extends UploadController {
       const errors = {
         url: new SubmitUrlController.Error(error.key, error, req, res)
       }
-      logger.error('local validation failed during url submission', error)
+      logger.warn('SubmitUrlController: local validation failed during url submission', error)
       return next(errors)
     }
 
@@ -91,7 +91,7 @@ class SubmitUrlController extends UploadController {
 
       return sizeInMB <= 10
     } catch (err) {
-      console.error(err)
+      console.warn(err)
       return true // for now we will allow this file as we can't be sure
     }
   }
@@ -103,7 +103,7 @@ class SubmitUrlController extends UploadController {
     try {
       return (response.status >= 200 && response.status < 300) || response.status === 400 // need to add 400 as some servers return 400 for head requests
     } catch (err) {
-      console.error(err)
+      logger.warn(err)
       return true
     }
   }
@@ -114,7 +114,7 @@ class SubmitUrlController extends UploadController {
       const acceptedTypes = Object.values(allowedFileTypes).flat()
       return acceptedTypes.includes(contentType)
     } catch (err) {
-      console.error(err)
+      logger.warn(err)
       return false
     }
   }

src/controllers/uploadFileController.js

@@ -46,7 +46,7 @@ class UploadFileController extends UploadController {
       const errors = {
         datafile: new UploadFileController.Error(error.key, error, req, res)
       }
-      logger.error('local validation failed during file upload', error)
+      logger.warn('UploadFileController: local validation failed during file upload', error)
       return next(errors)
     }
 
@@ -58,8 +58,7 @@ class UploadFileController extends UploadController {
       const id = await postFileRequest({ ...this.getBaseFormData(req), originalFilename: req.file.originalname, uploadedFilename })
       req.body.request_id = id
 
-      // log the file name, type and size as an object
-      logger.info('file submitted for processing:', { type: 'fileUploaded', name: req.file.originalname, mimetype: req.file.mimetype, size: req.file.size })
+      logger.info('UploadFileController: file submitted for processing:', { type: 'fileUploaded', name: req.file.originalname, mimetype: req.file.mimetype, size: req.file.size })
 
       super.post(req, res, next)
     } catch (error) {
@@ -100,7 +99,7 @@ class UploadFileController extends UploadController {
       await s3.upload(params).promise()
       return uuid
     } catch (error) {
-      logger.error('Error uploading file to S3: ' + error.message)
+      logger.warn('Error uploading file to S3: ' + error.message)
       throw error
     }
   }

src/filters/validationMessageLookup.js

@@ -62,7 +62,7 @@ const validationMessages = {
 
 function validationMessageLookup (field, type) {
   if (!validationMessages[field] || !validationMessages[field][type]) {
-    logger.error('No validation message found for field ' + field + ' and type ' + type)
+    logger.warn('No validation message found for field ' + field + ' and type ' + type)
     return `An error occurred of type ${type} for field ${field}`
   }
   return validationMessages[field][type]

src/models/requestData.js

@@ -29,7 +29,7 @@ export default class RequestData {
 
   getErrorSummary () {
     if (!this.response || !this.response.data || !this.response.data['error-summary']) {
-      logger.error('trying to get error summary when there is none: request id: ' + this.id)
+      logger.warn('trying to get error summary when there is none: request id: ' + this.id)
       return []
     }
     return this.response.data['error-summary']
@@ -45,7 +45,7 @@ export default class RequestData {
 
   getError () {
     if (!this.response) {
-      logger.error('trying to get error when there are none: request id: ' + this.id)
+      logger.warn('trying to get error when there are none: request id: ' + this.id)
       return { message: 'An unknown error occurred.' }
     }
 
@@ -54,11 +54,11 @@ export default class RequestData {
 
   hasErrors () {
     if (!this.response || !this.response.data) {
-      logger.error('trying to check for errors when there are none: request id: ' + this.id)
+      logger.warn('trying to check for errors when there are none: request id: ' + this.id)
       return true
     }
     if (this.response.data['error-summary'] == null) {
-      logger.error('trying to check for errors but there is no error-summary: request id: ' + this.id)
+      logger.warn('trying to check for errors but there is no error-summary: request id: ' + this.id)
       return true
     }
     return this.response.data['error-summary'].length > 0
@@ -71,7 +71,7 @@ export default class RequestData {
 
   getColumnFieldLog () {
     if (!this.response || !this.response.data || !this.response.data['column-field-log']) {
-      logger.error('trying to get column field log when there is none: request id: ' + this.id)
+      logger.warn('trying to get column field log when there is none: request id: ' + this.id)
       return []
     }
     return this.response.data['column-field-log']

src/models/responseDetails.js

@@ -11,15 +11,15 @@ export default class ResponseDetails {
 
   getRows () {
     if (!this.response) {
-      logger.error('trying to get response details when there are none: request id: ' + this.id)
+      logger.warn('trying to get response details when there are none: request id: ' + this.id)
       return []
     }
     return this.response
   }
 
   getColumnFieldLog () {
     if (!this.columnFieldLog) {
-      logger.error('trying to get column field log when there is none: request id: ' + this.id)
+      logger.warn('trying to get column field log when there is none: request id: ' + this.id)
       return []
     }
     return this.columnFieldLog
@@ -69,7 +69,7 @@ export default class ResponseDetails {
   // This function returns an array of rows with verbose columns
   getRowsWithVerboseColumns (filterNonErrors = false) {
     if (!this.response) {
-      logger.error('trying to get response details when there are none: request id: ' + this.id)
+      logger.warn('trying to get response details when there are none: request id: ' + this.id)
       return []
     }
 
@@ -105,7 +105,7 @@ export default class ResponseDetails {
 
   getGeometries () {
     if (!this.response) {
-      logger.error('trying to get response details when there are none: request id: ' + this.id)
+      logger.warn('trying to get response details when there are none: request id: ' + this.id)
       return undefined
     }
 

src/routes/cookies.js

@@ -0,0 +1,11 @@
+import express from 'express'
+import nunjucks from 'nunjucks'
+
+const router = express.Router()
+
+router.get('/', (req, res) => {
+  const cookiesPage = nunjucks.render('cookies.html', {})
+  res.send(cookiesPage)
+})
+
+export default router

src/routes/health.js

@@ -72,7 +72,7 @@ const checkRedis = async () => {
       return false
     }
   }).catch((err) => {
-    logger.error(`checkRedis/connect: ${err.message}`)
+    logger.warn(`checkRedis/connect: ${err.message}`)
     if (config.environment !== 'test') {
       console.error(err)
     }

src/routes/privacy.js

@@ -0,0 +1,11 @@
+import express from 'express'
+import nunjucks from 'nunjucks'
+
+const router = express.Router()
+
+router.get('/', (req, res) => {
+  const privacyPage = nunjucks.render('privacy-notice.html', {})
+  res.send(privacyPage)
+})
+
+export default router

src/serverSetup/nunjucks.js

@@ -25,6 +25,11 @@ export function setupNunjucks ({ app, dataSubjects }) {
     feedbackLink: config.feedbackLink
   }
 
+  if ('smartlook' in config) {
+    globalValues.smartlookKey = config.smartlook.key
+    globalValues.smartlookRegion = config.smartlook.region
+  }
+
   Object.keys(globalValues).forEach((key) => {
     nunjucksEnv.addGlobal(key, globalValues[key])
   })

src/serverSetup/routes.js

@@ -3,11 +3,15 @@ import endpointSubmissionFormFormWisard from '../routes/form-wizard/endpoint-sub
 import accessibility from '../routes/accessibility.js'
 import polling from '../routes/api.js'
 import health from '../routes/health.js'
+import privacy from '../routes/privacy.js'
+import cookies from '../routes/cookies.js'
 
 export function setupRoutes (app) {
   app.use('/', checkFormWizard)
   app.use('/submit', endpointSubmissionFormFormWisard)
   app.use('/accessibility', accessibility)
   app.use('/api', polling)
   app.use('/health', health)
+  app.use('/privacy-notice', privacy)
+  app.use('/cookies', cookies)
 }

src/utils/fetchLocalAuthorities.js

@@ -1,6 +1,5 @@
 import axios from 'axios'
 import logger from '../../src/utils/logger.js'
-import config from '../../config/index.js'
 
 /**
  * Fetches a list of local authority names from a specified dataset.
@@ -30,18 +29,15 @@ export const fetchLocalAuthorities = async () => {
     const response = await axios.get(url)
     const names = response.data.rows.map(row => {
       if (row[1] === null) {
-        console.log('Null value found in response:', row)
+        logger.debug('Null value found in response:', row)
         return null
       } else {
         return row[1]
       }
     }).filter(name => name !== null) // Filter out null values
     return names
   } catch (error) {
-    logger.error(`fetchLocalAuthorities: Error fetching local authorities data: ${error.message}`)
-    if (config.environment !== 'test') {
-      console.error(error)
-    }
+    logger.warn(`fetchLocalAuthorities: Error fetching local authorities data: ${error.message}`, error)
     throw error
   }
 }

src/utils/logger.js

@@ -1,15 +1,22 @@
 import { createLogger, format, transports } from 'winston'
+import config from '../../config/index.js'
+
+const appTransports = () => [
+  new transports.Console(),
+  new transports.File({ filename: 'combined.log' })
+]
+
+const testTransports = () => [
+  new transports.Console({ level: 'error' })
+]
 
 const logger = createLogger({
   format: format.combine(
     format.timestamp(),
     format.json()
   ),
   defaultMeta: { service: 'lpa-data-validation-frontend' },
-  transports: [
-    new transports.Console(),
-    new transports.File({ filename: 'combined.log' })
-  ]
+  transports: config.environment === 'test' ? testTransports() : appTransports()
 })
 
 export default logger

src/views/accessibility.html

@@ -8,7 +8,7 @@
 
 This accessibility statement applies to the check planning and housing data for England service (publish.development.digital-land.info).
 
-This service is run by the Department for Levelling Up, Housing and Communities (DLUHC). We want as many people as possible to be able to use this service. For example, that means you should be able to:
+This service is run by the Ministry of Housing, Communities and Local Government (MHCLG). We want as many people as possible to be able to use this service. For example, that means you should be able to:
 
 -   change colours, contrast levels and fonts using browser or device settings
 -   zoom in up to 400% without the text spilling off the screen
@@ -40,7 +40,7 @@
 
 ## Technical information about this service's accessibility
 
-The Department for Levelling Up, Housing and Communities (DLUHC) is committed to making this service accessible, in accordance with the Public Sector Bodies (Websites and Mobile Applications) (No. 2) Accessibility Regulations 2018.
+The Ministry of Housing, Communities and Local Government (MHCLG) is committed to making this service accessible, in accordance with the Public Sector Bodies (Websites and Mobile Applications) (No. 2) Accessibility Regulations 2018.
 
 ### Compliance status
 

src/views/common/smartlook.js

@@ -0,0 +1,6 @@
+window.smartlook || (function (d) {
+  var o = smartlook = function () { o.api.push(arguments) }; const h = d.getElementsByTagName('head')[0]
+  const c = d.createElement('script'); o.api = new Array(); c.async = true; c.type = 'text/javascript'
+  c.charset = 'utf-8'; c.src = 'https://web-sdk.smartlook.com/recorder.js'; h.appendChild(c)
+})(document)
+smartlook('init', '{{ smartlookKey }}', { region: '{{ smartlookRegion }}' })