comscan has detected new scanner test (lando#30)

* disable legacy scanner by default

* first pass on new url scanning

* second pass on new url scanning

* switch GA to use extended core features for faster testing

* switch GA to use extended core features for faster testing part 2

* switch GA to use extended core features for faster testing part 3

* rework restart and rebuild for new scanner tech

* restart tests

.github/workflows/pr-core-tests.yml

@@ -16,6 +16,8 @@ jobs:
           - ubuntu-22.04
         node-version:
           - '16'
+        lando-versions:
+          - dev
         leia-tests:
           - examples/badname
           - examples/base
@@ -30,65 +32,43 @@ jobs:
           - examples/services
           - examples/tooling
     steps:
-
-      # Install deps and cache
-      # Eventually it would be great if these steps could live in a separate YAML file
-      # that could be included in line to avoid code duplication
       - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Checkout @lando/cli repo
         uses: actions/checkout@v3
-        with:
-          repository: lando/cli
-          path: cli
       - name: Install node ${{ matrix.node-version }}
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.node-version }}
+          registry-url: https://registry.npmjs.org
           cache: yarn
       - name: Install Yarn dependencies
-        run: |
-          yarn install --prefer-offline --frozen-lockfile
-          cd cli
-          yarn install --prefer-offline --frozen-lockfile
+        run: yarn install --prefer-offline --frozen-lockfile
 
       # This block should eventually become use lando/actions-hyperdrive@v2
       - name: Verify Docker dependencies
         run: |
           docker --version | grep "20.10."
           docker-compose --version | grep "1.29."
+      - name: Grab Lando CLI (${{ matrix.lando-versions }})
+        run: |
+          sudo curl -fsSL -o /usr/local/bin/lando "https://files.lando.dev/cli/lando-linux-x64-${{ matrix.lando-versions }}"
+          sudo chmod +x /usr/local/bin/lando
+
       # - name: Install via hyperdrive if above is bad
       - name: Disable usage and error reporting
         run: |
-          mkdir -p ~/.lando/cache
-          echo false > ~/.lando/cache/report_errors
+          mkdir -p /home/runner/.lando/cache
+          echo false > /home/runner/.lando/cache/report_errors
           sed -i "s/report: true.*/report: false/" config.yml
-
-      # get the cli repo, copy core into it, build the cli repo and use that
-      # we'd probably figure out a better way to dog food but since runtime 3 is on the way out
-      # we are just going to do the lazy thing
-      - name: Move core over before we package
+      - name: Symlink for external core usage
         run: |
-          # touch a file so we can test the copy
-          touch COPY_TEST
-
-          # move cli to tmp dir and replace its core
-          mv ./cli /tmp/cli
-          cd /tmp/cli
-          rm -rf ./node_modules/@lando/core
-          cp -rf $GITHUB_WORKSPACE node_modules/@lando/core
-
-          # verify the copy
-          ls -lsa ./node_modules/@lando/core
-          ls -lsa ./node_modules/@lando/core | grep COPY_TEST
-      - name: Package CLI
-        run: cd /tmp/cli && yarn cli:build
-      - name: Replace source CLI with packaged one
-        run: sudo mv /tmp/cli/dist/@lando/cli /usr/local/bin/lando
-      - name: Verify we can run the packaged CLI
+          mkdir -p /home/runner/.lando/plugins/@lando
+          ln -sf /home/runner/work/core/core /home/runner/.lando/plugins/@lando/core
+          lando --clear
+      - name: Verify Lando works and we are dogfooding this plugin for tests
         run: |
           lando version
-          lando config
+          # we use the symlinked path from above here instead of the usual
+          lando config --path plugins | grep lando-core | grep /home/runner/.lando/plugins/@lando/core || (echo "::error:: Not dogfooding this plugin correctly!" && exit 1)
 
       # This block should eventually become use lando/actions-leia@v2
       - name: Run leia tests

.github/workflows/pr-db-tools-tests.yml

@@ -1,7 +1,7 @@
 name: Database Tools Tests
 
-on:
-  pull_request:
+# on:
+#   pull_request:
 
 jobs:
   leia-tests:

examples/scanner/.lando.yml

@@ -1,4 +1,7 @@
 name: lando-scanner
+proxy:
+  scanme:
+    - lando-scanner.lndo.site
 services:
   scanme:
     type: apache
@@ -9,3 +12,23 @@ services:
   dontlookatme:
     type: apache
     scanner: false
+  options:
+    type: apache
+    scanner:
+      timeout: 1000
+      retry: 10
+      path: /another.html
+      okCodes:
+        - 404
+  allowmissing:
+    type: apache
+    scanner:
+      path: /missing.html
+      okCodes:
+        - 404
+  fail:
+    type: apache
+    scanner:
+      path: /missing.html
+      retry: 4
+

examples/scanner/another.html

@@ -0,0 +1 @@
+HI

package.json

@@ -45,12 +45,14 @@
     "clean-stacktrace": "^1.1.0",
     "copy-dir": "^0.4.0",
     "dayjs": "^1.8.25",
+    "debug": "^4.3.4",
     "dockerode": "^2.4.2",
     "github": "^12.0.0",
     "glob": "^7.1.3",
     "ip": "^1.1.8",
     "js-yaml": "^3.4.6",
     "jsonfile": "^2.4.0",
+    "listr2": "^5.0.7",
     "lodash": "^4.17.21",
     "mkdirp": "^0.5.1",
     "node-cache": "^4.1.1",
@@ -84,4 +86,4 @@
     "version-bump-prompt": "^4.2.1",
     "vuepress": "2.0.0-beta.53"
   }
-}
+}

plugins/lando-core/app.js

@@ -12,10 +12,15 @@ const warnings = require('./lib/warnings');
 const getHttpPorts = data => _.get(data, 'Config.Labels["io.lando.http-ports"]', '80,443').split(',');
 const getHttpsPorts = data => _.get(data, 'Config.Labels["io.lando.https-ports"]', '443').split(',');
 
-// Helper to get scannable or not scannable services
-const getScannable = (app, scan = true) => _.filter(app.info, service => {
-  return _.get(app, `config.services.${service.service}.scanner`, true) === scan;
+// Helpers to get scannable or not scannable services
+const getScannable = app => _.filter(app.info, service => {
+  const scanner = _.get(app, `config.services.${service.service}.scanner`, true);
+  return scanner || _.isObject(scanner);
 });
+const getUnscannable = app => _.filter(app.info, service => {
+  return _.get(app, `config.services.${service.service}.scanner`, true) === false;
+});
+
 
 // Helper to set the LANDO_LOAD_KEYS var
 const getKeys = (keys = true) => {
@@ -223,24 +228,26 @@ module.exports = (app, lando) => {
     });
   });
 
-  // Scan urls
-  app.events.on('post-start', 10, () => {
-    // Message to let the user know it could take a bit
-    console.log('Scanning to determine which services are ready... Please stand by...');
-    // Filter out any services where the scanner might be disabled
-    return app.scanUrls(_.flatMap(getScannable(app), 'urls'), {max: 16}).then(urls => {
-      // Get data about our scanned urls
-      app.urls = urls;
-      // Add in unscannable ones if we have them
-      if (!_.isEmpty(getScannable(app, false))) {
-        app.urls = app.urls.concat(_.map(_.flatMap(getScannable(app, false), 'urls'), url => ({
-          url,
-          status: true,
-          color: 'yellow',
-        })));
-      }
+  // LEGACY URL Scanner urls
+  if (_.get(lando, 'config.scanner', true) === 'legacy') {
+    app.events.on('post-start', 10, () => {
+      // Message to let the user know it could take a bit
+      console.log('Scanning to determine which services are ready... Please stand by...');
+      // Filter out any services where the scanner might be disabled
+      return app.scanUrls(_.flatMap(getScannable(app), 'urls'), {max: 16}).then(urls => {
+        // Get data about our scanned urls
+        app.urls = urls;
+        // Add in unscannable ones if we have them
+        if (!_.isEmpty(getUnscannable(app))) {
+          app.urls = app.urls.concat(_.map(_.flatMap(getUnscannable(app), 'urls'), url => ({
+            url,
+            status: true,
+            color: 'yellow',
+          })));
+        }
+      });
     });
-  });
+  };
 
   // Reset app info on a stop, this helps prevent wrong/duplicate information being reported on a restart
   app.events.on('post-stop', () => lando.utils.getInfoDefaults(app));

plugins/lando-core/lib/utils.js

@@ -80,7 +80,7 @@ exports.normalizeOverrides = (overrides, base = '.', volumes = {}) => {
 /*
  * Returns a CLI table with app start metadata info
  */
-exports.startTable = app => {
+exports.startTable = (app, {legacyScanner = false} = {}) => {
   const data = {
     name: app.name,
     location: app.root,
@@ -92,19 +92,26 @@ exports.startTable = app => {
   const urls = {};
 
   // Categorize and colorize URLS if and as appropriate
-  _.forEach(app.info, info => {
-    if (_.has(info, 'urls') && !_.isEmpty(info.urls)) {
-      urls[info.service] = _.filter(app.urls, item => {
-        item.theme = chalk[item.color](item.url);
-        return _.includes(info.urls, item.url);
-      });
-    }
-  });
+  // add legacy scanner info if appropriate
+  if (legacyScanner) {
+    _.forEach(app.info, info => {
+      if (_.has(info, 'urls') && !_.isEmpty(info.urls)) {
+        urls[info.service] = _.filter(app.urls, item => {
+          item.theme = chalk[item.color](item.url);
+          return _.includes(info.urls, item.url);
+        });
+      }
+    });
 
-  // Add service URLS
-  _.forEach(urls, (items, service) => {
-    data[service + ' urls'] = _.map(items, 'theme');
-  });
+    // Add service URLS
+    _.forEach(urls, (items, service) => {
+      data[service + ' urls'] = _.map(items, 'theme');
+    });
+
+  // add placeholder URLS for non le
+  } else {
+    data.urls = '';
+  }
 
   // Return data
   return data;