diff --git a/.github/workflows/build-util-images.yml b/.github/workflows/build-util-images.yml index 4110918c0..8645876c7 100644 --- a/.github/workflows/build-util-images.yml +++ b/.github/workflows/build-util-images.yml @@ -38,6 +38,7 @@ jobs: run: echo "tag-suffix=-edge" >> $GITHUB_OUTPUT - name: Login to DockerHub uses: docker/login-action@v2 + if: ${{ github.event_name != 'pull_request' }} with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -47,7 +48,7 @@ jobs: context: ${{ matrix.context }} file: ${{ matrix.dockerfile }} platforms: linux/amd64,linux/arm64 - push: true + push: ${{ github.event_name != 'pull_request' }} tags: devwithlando/${{ matrix.image }}:${{ matrix.tag }}${{ steps.pr.outputs.tag-suffix }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/bin/lando b/bin/lando index 693186d77..3d0a30728 100755 --- a/bin/lando +++ b/bin/lando @@ -108,6 +108,10 @@ const cores = [ path.resolve(__dirname, '..'), ]; +if (typeof _.get(config, 'plugins.@lando/core') === 'string') { + cores.unshift(path.resolve(appConfig.root, config.plugins['@lando/core'])); +} + // if appConfig points to a different core lets set that here if (typeof _.get(appConfig, 'plugins.@lando/core') === 'string') { cores.unshift(path.resolve(appConfig.root, appConfig.plugins['@lando/core'])); diff --git a/builders/_lando-compose.js b/builders/_lando-compose.js new file mode 100644 index 000000000..00279d8db --- /dev/null +++ b/builders/_lando-compose.js @@ -0,0 +1,11 @@ +'use strict'; + +module.exports = { + name: '_lando-compose', + parent: '_lando', + builder: (parent, config) => class LandoComposeServiceV3 extends parent { + constructor(id, options = {}) { + super(id, options); + }; + }, +}; diff --git a/builders/_lando.js b/builders/_lando.js index 8556de70e..750839d93 100644 --- a/builders/_lando.js +++ b/builders/_lando.js @@ -48,6 +48,8 @@ module.exports = { supportedIgnore = false, root = '', webroot = '/app', + _app = null, + appMount = '/app', } = {}, ...sources ) { @@ -81,6 +83,9 @@ module.exports = { const environment = { LANDO_SERVICE_NAME: name, LANDO_SERVICE_TYPE: type, + LANDO_WEBROOT_USER: meUser, + LANDO_WEBROOT_GROUP: meUser, + LANDO_MOUNT: appMount, }; // Handle labels @@ -96,7 +101,6 @@ module.exports = { `${userConfRoot}:/lando:cached`, `${scriptsDir}:/helpers`, `${entrypointScript}:/lando-entrypoint.sh`, - `${dataHome}:/var/www`, ]; // Handle ssl @@ -139,9 +143,16 @@ module.exports = { // Add named volumes and other thingz into our primary service const namedVols = {}; - _.set(namedVols, data, {}); - _.set(namedVols, dataHome, {}); - + if (null !== data) { + _.set(namedVols, data, {}); + } + if (null !== dataHome) { + _.set(namedVols, dataHome, {}); + volumes.push(`${dataHome}:/var/www`); + } + if (null === entrypoint) { + entrypoint = undefined; + } sources.push({ services: _.set({}, name, { entrypoint, @@ -171,6 +182,7 @@ module.exports = { info.meUser = meUser; info.hasCerts = ssl; info.api = 3; + info.appMount = appMount; // Add the healthcheck if it exists if (healthcheck) info.healthcheck = healthcheck; diff --git a/hooks/app-add-v3-services.js b/hooks/app-add-v3-services.js index fe3659406..b68e21fc5 100644 --- a/hooks/app-add-v3-services.js +++ b/hooks/app-add-v3-services.js @@ -6,6 +6,13 @@ module.exports = async (app, lando) => { // add parsed services to app object so we can use them downstream app.cachedInfo = _.get(lando.cache.get(app.composeCache), 'info', []); app.parsedServices = require('../utils/parse-v3-services')(_.get(app, 'config.services', {}), app); + app.parsedServices = app.parsedServices.concat( + require('../utils/parse-compose-services')( + _.get(app, 'config.services', {}), + _.keys(_.get(app, 'composeData[0].data[0].services', {})), + app, + ), + ); app.parsedV3Services = _(app.parsedServices).filter(service => service.api === 3).value(); app.servicesList = app.parsedV3Services.map(service => service.name); diff --git a/lib/app.js b/lib/app.js index 18f3f6326..15f452af3 100644 --- a/lib/app.js +++ b/lib/app.js @@ -273,14 +273,23 @@ module.exports = class App { // We should only need to initialize once, if we have just go right to app ready if (this.initialized) return this.events.emit('ready', this); // Get compose data if we have any, otherwise set to [] - const composeFiles = require('../utils/load-compose-files')(_.get(this, 'config.compose', []), this.root); - this.composeData = [new this.ComposeService('compose', {}, ...composeFiles)]; - // Validate and set env files - this.envFiles = require('../utils/normalize-files')(_.get(this, 'config.env_file', []), this.root); - // Log some things - this.log.verbose('initiatilizing app at %s...', this.root); - this.log.silly('app has config', this.config); - + return require('../utils/load-compose-files')( + _.get(this, 'config.compose', []), + this.root, + this._dir, + (composeFiles, outputFilePath) => + this.engine.getComposeConfig({compose: composeFiles, project: this.project, outputFilePath}), + ) + .then(composeFileData => { + if (undefined !== composeFileData) { + this.composeData = [new this.ComposeService('compose', {}, composeFileData)]; + } + // Validate and set env files + this.envFiles = require('../utils/normalize-files')(_.get(this, 'config.env_file', []), this.root); + // Log some things + this.log.verbose('initiatilizing app at %s...', this.root); + this.log.silly('app has config', this.config); + }) /** * Event that allows altering of the app object right before it is * initialized. @@ -293,8 +302,9 @@ module.exports = class App { * @event pre_init * @property {App} app The app instance. */ - return loadPlugins(this, this._lando).then(() => this.events.emit('pre-init', this)) + .then(() => loadPlugins(this, this._lando)) + .then(() => this.events.emit('pre-init', this)) // Actually assemble this thing so its ready for that engine .then(() => { // Get all the services diff --git a/lib/compose.js b/lib/compose.js index 34cc67c18..080d8a738 100644 --- a/lib/compose.js +++ b/lib/compose.js @@ -20,6 +20,7 @@ const composeFlags = { rm: '--rm', timestamps: '--timestamps', volumes: '-v', + outputFilePath: '-o', }; // Default options nad things @@ -33,6 +34,7 @@ const defaultOptions = { pull: {}, rm: {force: true, volumes: true}, up: {background: true, noRecreate: true, recreate: false, removeOrphans: true}, + config: {}, }; /* @@ -155,3 +157,8 @@ exports.start = (compose, project, opts = {}) => buildShell('up', project, compo * Run docker compose stop */ exports.stop = (compose, project, opts = {}) => buildShell('stop', project, compose, opts); + +/* + * Run docker compose config + */ +exports.config = (compose, project, opts = {}) => buildShell('config', project, compose, opts); diff --git a/lib/engine.js b/lib/engine.js index ea90ec07a..70d916b6f 100644 --- a/lib/engine.js +++ b/lib/engine.js @@ -495,5 +495,25 @@ module.exports = class Engine { // stop return this.engineCmd('stop', data); }; + + /** + * Get dumped docker compose config for compose files from project + * using a `compose` object with `{compose: compose, project: project, opts: opts}` + * + * @since 3.0.0 + * @param {Object} data Config needs a service within a compose context + * @param {Array} data.compose An Array of paths to Docker compose files + * @param {String} data.project A String of the project name (Usually this is the same as the app name) + * @param {String} [data.outputFilePath='/path/to/file.yml'] String to output path + * @param {Object} [data.opts] Options + * @return {Promise} A Promise. + * @example + * return lando.engine.stop(app); + */ + getComposeConfig(data) { + data.opts = {cmd: ['-o', data.outputFilePath]}; + delete data.outputFilePath; + return this.engineCmd('config', data); + }; }; diff --git a/lib/router.js b/lib/router.js index ca844fbf9..79694c5d4 100644 --- a/lib/router.js +++ b/lib/router.js @@ -133,3 +133,5 @@ exports.start = (data, compose) => retryEach(data, datum => compose('start', dat exports.stop = (data, compose, docker) => retryEach(data, datum => { return (datum.compose) ? compose(data.kill ? 'kill' : 'stop', datum) : docker.stop(getContainerId(datum)); }); + +exports.config = (data, compose) => retryEach(data, datum => compose('config', datum)); diff --git a/scripts/user-perm-helpers.sh b/scripts/user-perm-helpers.sh index 28db38732..a31d1ddd5 100755 --- a/scripts/user-perm-helpers.sh +++ b/scripts/user-perm-helpers.sh @@ -10,31 +10,25 @@ LANDO_MODULE="userperms" add_user() { local USER=$1 local GROUP=$2 - local UID=$3 - local GID=$4 - local DISTRO=$5 - local EXTRAS="$6" - if [ "$DISTRO" = "alpine" ]; then - if ! groups | grep "$GROUP" > /dev/null 2>&1; then addgroup -g "$GID" "$GROUP" 2>/dev/null; fi - if ! id -u "$GROUP" > /dev/null 2>&1; then adduser -H -D -G "$GROUP" -u "$UID" "$USER" "$GROUP" 2>/dev/null; fi - else - if ! groups | grep "$GROUP" > /dev/null 2>&1; then groupadd --force --gid "$GID" "$GROUP" 2>/dev/null; fi - if ! id -u "$GROUP" > /dev/null 2>&1; then useradd --gid "$GID" --uid "$UID" $EXTRAS "$USER" 2>/dev/null; fi - fi; + local WEBROOT_UID=$3 + local WEBROOT_GID=$4 + if ! getent group | cut -d: -f1 | grep "$GROUP" > /dev/null 2>&1; then addgroup -g "$WEBROOT_GID" "$GROUP" 2>/dev/null; fi + if ! id -u "$USER" > /dev/null 2>&1; then adduser -H -D -G "$GROUP" -u "$WEBROOT_UID" "$USER" "$GROUP" 2>/dev/null; fi } # Verify user verify_user() { local USER=$1 local GROUP=$2 - local DISTRO=$3 id -u "$USER" > /dev/null 2>&1 - groups | grep "$GROUP" > /dev/null 2>&1 - if [ "$DISTRO" = "alpine" ]; then + groups "$USER" | grep "$GROUP" > /dev/null 2>&1 + if command -v chsh > /dev/null 2>&1 ; then + if command -v /bin/bash > /dev/null 2>&1 ; then + chsh -s /bin/bash $USER || true + fi; + else true # is there a chsh we can use? do we need to? - else - chsh -s /bin/bash $USER || true fi; } @@ -59,11 +53,10 @@ reset_user() { if [ "$(id -u $USER)" != "$HOST_UID" ]; then usermod -o -u "$HOST_UID" "$USER" 2>/dev/null fi - groupmod -g "$HOST_GID" "$GROUP" 2>/dev/null || true - if [ "$(id -u $USER)" != "$HOST_UID" ]; then + groupmod -o -g "$HOST_GID" "$GROUP" 2>/dev/null || true + if [ "$(id -g $USER)" != "$HOST_GID" ]; then usermod -g "$HOST_GID" "$USER" 2>/dev/null || true fi - usermod -a -G "$GROUP" "$USER" 2>/dev/null || true fi; # If this mapping is incorrect lets abort here if [ "$(id -u $USER)" != "$HOST_UID" ]; then @@ -97,7 +90,6 @@ perm_sweep() { nohup find /user/.ssh -not -user $USER -execdir chown $USER:$GROUP {} \+ > /tmp/perms.out 2> /tmp/perms.err & nohup find /var/www -not -user $USER -execdir chown $USER:$GROUP {} \+ > /tmp/perms.out 2> /tmp/perms.err & nohup find /usr/local/bin -not -user $USER -execdir chown $USER:$GROUP {} \+ > /tmp/perms.out 2> /tmp/perms.err & - nohup chmod -R 755 /var/www >/dev/null 2>&1 & # Lets also make some /usr/locals chowned nohup find /usr/local/lib -not -user $USER -execdir chown $USER:$GROUP {} \+ > /tmp/perms.out 2> /tmp/perms.err & diff --git a/tasks/info.js b/tasks/info.js index 0a77a5018..94d68f191 100644 --- a/tasks/info.js +++ b/tasks/info.js @@ -34,11 +34,12 @@ module.exports = lando => ({ const getData = async () => { // go deep if (options.deep) { + const separator = _.get(app, '_config.orchestratorSeparator', '_'); return await lando.engine.list({project: app.project}) .map(async container => await lando.engine.scan(container)) .filter(container => { if (!options.service) return true; - return options.service.map(service => `/${app.project}_${service}_1`).includes(container.Name); + return options.service.map(service => `/${app.project}${separator}${service}${separator}1`).includes(container.Name); }); // normal info diff --git a/test/compose.spec.js b/test/compose.spec.js index d282600b0..0b599cdc9 100644 --- a/test/compose.spec.js +++ b/test/compose.spec.js @@ -202,4 +202,16 @@ describe('compose', () => { expect(stopResult).to.be.an('object'); }); }); + + describe('#config', () => { + it('should return the correct default options when not specified'); + it('#config should return an object.', () => { + const configResult = compose.config( + ['string1', 'string2'], + 'my_project', + myOpts, + ); + expect(configResult).to.be.an('object'); + }); + }); }); diff --git a/test/get-user.spec.js b/test/get-user.spec.js index d756b0b17..abf5c4216 100644 --- a/test/get-user.spec.js +++ b/test/get-user.spec.js @@ -29,6 +29,11 @@ describe('get-user', function() { expect(getUser('test-service', info)).to.equal('www-data'); }); + it('should return specified user if service is a "no-api" docker-compose service and user is specified', function() { + const info = [{service: 'test-service', type: 'docker-compose', meUser: 'custom-user'}]; + expect(getUser('test-service', info)).to.equal('custom-user'); + }); + it('should return "www-data" if service.api is 4 but no user is specified', function() { const info = [{service: 'test-service', api: 4}]; expect(getUser('test-service', info)).to.equal('www-data'); diff --git a/utils/build-tooling-runner.js b/utils/build-tooling-runner.js index 6436548c7..b70f93577 100644 --- a/utils/build-tooling-runner.js +++ b/utils/build-tooling-runner.js @@ -4,7 +4,8 @@ const _ = require('lodash'); const path = require('path'); const getContainer = (app, service) => { - return app?.containers?.[service] ?? `${app.project}_${service}_1`; + const separator = _.get(app, '_config.orchestratorSeparator', '_'); + return app?.containers?.[service] ?? `${app.project}${separator}${service}${separator}1`; }; const getContainerPath = (appRoot, appMount = undefined) => { diff --git a/utils/build-tooling-task.js b/utils/build-tooling-task.js index bf808dcf3..c40fa1584 100644 --- a/utils/build-tooling-task.js +++ b/utils/build-tooling-task.js @@ -22,7 +22,7 @@ module.exports = (config, injected) => { // Kick off the pre event wrappers .then(() => app.events.emit(`pre-${eventName}`, config, answers)) // Get an interable of our commandz - .then(() => _.map(require('./parse-tooling-config')(cmd, service, options, answers, canExec))) + .then(() => _.map(require('./parse-tooling-config')(cmd, service, name, options, answers, canExec))) // Build run objects .map(({command, service}) => require('./build-tooling-runner')(app, command, service, user, env, dir, appMount)) // Try to run the task quickly first and then fallback to compose launch diff --git a/utils/get-app-mounts.js b/utils/get-app-mounts.js index 694ffedb3..521b4af70 100644 --- a/utils/get-app-mounts.js +++ b/utils/get-app-mounts.js @@ -6,7 +6,7 @@ module.exports = app => _(app.services) // Objectify .map(service => _.merge({name: service}, _.get(app, `config.services.${service}`, {}))) // Set the default - .map(config => _.merge({}, config, {app_mount: _.get(config, 'app_mount', 'cached')})) + .map(config => _.merge({}, config, {app_mount: _.get(config, 'app_mount', app.config.app_mount || 'cached')})) // Filter out disabled mountes .filter(config => config.app_mount !== false && config.app_mount !== 'disabled') // Combine together diff --git a/utils/get-user.js b/utils/get-user.js index f5b8b2094..ed42f7b87 100644 --- a/utils/get-user.js +++ b/utils/get-user.js @@ -7,8 +7,8 @@ module.exports = (name, info = []) => { if (!_.find(info, {service: name})) return 'www-data'; // otherwise get the service const service = _.find(info, {service: name}); - // if this is a "no-api" service eg type "docker-compose" also return www-data - if (!service.api && service.type === 'docker-compose') return 'www-data'; + // if this is a "no-api" service eg type "docker-compose" return meUser or www-data as default + if (!service.api && service.type === 'docker-compose') return service.meUser || 'www-data'; // otherwise return different things based on the api return service.api === 4 ? service.user || 'www-data' : service.meUser || 'www-data'; }; diff --git a/utils/load-compose-files.js b/utils/load-compose-files.js index f13bf4cec..e2a5c68d8 100644 --- a/utils/load-compose-files.js +++ b/utils/load-compose-files.js @@ -2,8 +2,32 @@ const _ = require('lodash'); const Yaml = require('./../lib/yaml'); +const path = require('path'); const yaml = new Yaml(); +const fs = require('fs'); +const remove = require('./remove'); -module.exports = (files, dir) => _(require('./normalize-files')(files, dir)) - .map(file => yaml.load(file)) - .value(); +// This just runs `docker compose --project-directory ${dir} config -f ${files} --output ${outputPaths}` to +// make all paths relative to the lando config root +module.exports = async (files, dir, landoComposeConfigDir = undefined, outputConfigFunction = undefined) => { + const composeFilePaths = _(require('./normalize-files')(files, dir)).value(); + if (_.isEmpty(composeFilePaths)) { + return {}; + } + + if (undefined === outputConfigFunction) { + return _(composeFilePaths) + .map(file => yaml.load(file)) + .value(); + } + + const outputFile = path.join(landoComposeConfigDir, 'resolved-compose-config.yml'); + + fs.mkdirSync(path.dirname(outputFile), {recursive: true}); + await outputConfigFunction(composeFilePaths, outputFile); + const result = yaml.load(outputFile); + fs.unlinkSync(outputFile); + remove(path.dirname(outputFile)); + + return result; +}; diff --git a/utils/parse-compose-services.js b/utils/parse-compose-services.js new file mode 100644 index 000000000..5753166a2 --- /dev/null +++ b/utils/parse-compose-services.js @@ -0,0 +1,27 @@ +'use strict'; + +// Modules +const _ = require('lodash'); + +// adds required methods to ensure the lando v3 debugger can be injected into v4 things +module.exports = (config, composeServiceIds, app) => _(config) + // Arrayify + .map((service, name) => _.merge({}, service, {name})) + // Filter out any services which are not defined in the docker compose services + .filter(service => _.includes(composeServiceIds, service.name)) + // Build the config and ensure api is set to 3 + .map(service => _.merge({}, service, { + _app: app, + app: app.name, + home: app.config.home || app._config.home, + project: app.project, + root: app.root, + type: '_lando-compose', + userConfRoot: app._config.userConfRoot, + version: 'custom', + api: 3, + entrypoint: null, // NOTE: Do not overwrite the entrypoint from docker compose. Or should we? + data: null, // NOTE: Do not create the data volume + dataHome: null, // NOTE: Do not create the dataHome volume + })) + .value(); diff --git a/utils/parse-tooling-config.js b/utils/parse-tooling-config.js index 489fcd9fa..92908ec51 100644 --- a/utils/parse-tooling-config.js +++ b/utils/parse-tooling-config.js @@ -40,9 +40,9 @@ const handleDynamic = (config, options = {}, answers = {}, execs = {}) => { * the first three assuming they are [node, lando.js, options.name]' * Check to see if we have global lando opts and remove them if we do */ -const handleOpts = (config, argopts = []) => { +const handleOpts = (config, name, argopts = []) => { // Append any user specificed opts - argopts = argopts.concat(process.argv.slice(3)); + argopts = argopts.concat(process.argv.slice(process.argv.findIndex(value => value === name.split(' ')[0]) + 1)); // If we have no args then just return right away if (_.isEmpty(argopts)) return config; // Return @@ -68,13 +68,13 @@ const parseCommand = (cmd, service, execs) => ({ }); // adds required methods to ensure the lando v3 debugger can be injected into v4 things -module.exports = (cmd, service, options = {}, answers = {}, execs = {}) => _(cmd) +module.exports = (cmd, service, name, options = {}, answers = {}, execs = {}) => _(cmd) // Put into an object so we can handle "multi-service" tooling .map(cmd => parseCommand(cmd, service, execs)) // Handle dynamic services .map(config => handleDynamic(config, options, answers, execs)) // Add in any argv extras if they've been passed in - .map(config => handleOpts(config, handlePassthruOpts(options, answers))) + .map(config => handleOpts(config, name, handlePassthruOpts(options, answers))) // Wrap the command in /bin/sh if that makes sense .map(config => _.merge({}, config, {command: require('./shell-escape')(config.command, true, config.args, config.exec)})) // eslint-disable-line max-len // Add any args to the command and compact to remove undefined diff --git a/utils/parse-v3-services.js b/utils/parse-v3-services.js index cfed7f138..5707a322e 100644 --- a/utils/parse-v3-services.js +++ b/utils/parse-v3-services.js @@ -20,7 +20,7 @@ module.exports = (config, app) => _(config) app: app.name, confDest: path.join(app._config.userConfRoot, 'config', service.type.split(':')[0]), data: `data_${service.name}`, - home: app._config.home, + home: app.config.home || app._config.home, project: app.project, root: app.root, type: service.type.split(':')[0],