From 1c0e7df34118f2076272b99d1b85061dcaf9f684 Mon Sep 17 00:00:00 2001 From: Hanna Prinz Date: Tue, 11 Feb 2025 15:00:13 +0100 Subject: [PATCH] Replace gopass with 1Password CLI (#2550) RISDEV-6520 --- Brewfile | 2 +- README.md | 97 +++++++++++-------------------------------------------- run.sh | 36 +++++++++------------ 3 files changed, 36 insertions(+), 99 deletions(-) diff --git a/Brewfile b/Brewfile index 053c18a546..336c9da133 100644 --- a/Brewfile +++ b/Brewfile @@ -3,7 +3,7 @@ brew "lefthook" brew "gh" tap "homebrew/cask" cask "docker" -brew "gopass" +brew "1password-cli" brew "direnv" brew "nodenv" diff --git a/README.md b/README.md index 55b132a717..3619e8f2a7 100644 --- a/README.md +++ b/README.md @@ -10,22 +10,22 @@ You need (or may want) the following CLI tools. For UNIX users, there is a prepared `Brewfile`, see below. -**Necessary tools:** +### Necessary tools:** - [lefthook](https://github.com/evilmartians/lefthook#install) - manages our git hooks - [Github CLI](https://cli.github.com/) - used by lefthook to check for pipeline status before push - [docker](https://docs.docker.com/get-docker/) - our container runtime (on macOS, the easiest way is to use [Docker Desktop](https://www.docker.com/products/docker-desktop/)) -- [gopass](https://www.gopass.pw/#install) - a tool to sync secrets +- [1Password CLI](https://developer.1password.com/docs/cli/get-started/) - to access secrets - [Node.js](https://nodejs.org/en/) - JavaScript runtime & dependency management - [nodenv](https://github.com/nodenv/nodenv#installation) - manages the node.js environment -**Backend only:** +### Backend only:** - [java](https://developers.redhat.com/products/openjdk/install) - we use Java 21 in the backend -**Optional, but recommended tools:** +### Optional, but recommended tools: - [jq](https://github.com/stedolan/jq) - handy JSON Processor - [yq](https://github.com/mikefarah/yq) - handy YAML Processor @@ -57,6 +57,18 @@ to `~/.zshrc`: eval "$(direnv hook zsh)" ``` +### S3 Credentials for Lookup Table Initialization + +The lookup table initialization in your local environment will be performed with data provided by a s3 bucket. Read [here](https://platform-docs.prod.ds4g.net/user-docs/how-to-guides/access-obs-via-aws-sdk/#step-2-obtain-access_key-credentials) on how to revtrieve credentials for it. + +Then, store the credentials in 1Password: + +```shell +op item create --category login --title 'NeuRIS S3' \ +'access-key-id=[your-access-key-id]' \ +'secret-access-key=[your-secret-access-key]' +``` + ## Getting started To get started with development, run: @@ -69,52 +81,9 @@ This will install a couple of Git hooks which are supposed to help you to: - commit properly formatted source code only (and not break the build otherwise) - write [conventional commit messages](https://chris.beams.io/posts/git-commit/) +- not accidentally push on a failing pipeline -### Setup local environment - -For shared secrets required for development we're using `gopass`. To set up follow these steps: - -- If not done yet: generate a gpg keypair -- Then export your public key: `gpg --armor --export --output my-name.gpg email@example.com` -- Provide some team member the public GPG key with encryption capability (that team member will add you - as a recipient). - -Then, run: - -```bash -gopass init - -gopass clone git@github.com:digitalservicebund/neuris-password-store.git neuris --sync gitcli -``` - -> **Note** -> -> If there are any issues with this command, you need to clean the store and try again until it -> works unfortunately ☹️. Be aware that this command removes ALL gopass stores from your machine, not only project -> related ones: -> -> ``` -> rm -rf ~/.local/share/gopass/stores -> ``` - -Try if you can get access: - -```bash -gopass list neuris -``` - -Synchronize the password store: - -```bash -gopass sync -``` - -Now you can generate a new `.env` file containing the secrets. When using a Yubikey you may asked multiple times for -your pin: - -```bash -./run.sh env -``` +Also, it creates a new `.env` file containing the secrets. You will be asked to authorize requests to 1Password. > **Note** > @@ -124,34 +93,6 @@ your pin: The caselaw application requires the initialization of lookup tables by the migration application image. -#### Prerequisites - -To be able to pull the `ris-data-migration` image, log in to the GitHub Package Repository using your username and a -credential token stored in 1Password (1PW): - -If you don't have a personal access token, -read [here](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#authenticating-with-a-personal-access-token-classic) -on how to create one. Then: - -```shell -export CR_PAT=$(op read op://Employee/CR_PAT/password) -echo $CR_PAT | docker login ghcr.io -u USERNAME --password-stdin # Replace USERNAME with your GitHub username -``` - -The following step requires an OTC access token, read here for -more [info](https://platform-docs.prod.ds4g.net/user-docs/how-to-guides/access-obs-via-aws-sdk/#step-2-obtain-access_key-credentials). - -To connect to your S3 bucket, ensure your AWS credentials are stored in 1Password, and then set the following -environment variables in your shell: - -```shell -op item edit 'OTC' aws_access_key_id=[your-access-key-id] -op item edit 'OTC' aws_secret_access_key=[your-access-key-id] - -``` - -#### Run Lookup Tables Initialization with Docker - The following command will migrate the minimally required data (refdata and juris tables): Make sure the latest @@ -161,7 +102,7 @@ in [compose.yaml](https://github.com/digitalservicebund/ris-backend-service/blob and then run: ```bash -./run.sh -i +./run.sh dev -i ``` > Note: If you wish to migrate documentation units, use the instructions diff --git a/run.sh b/run.sh index 4ff163bdb5..e11810bc8e 100755 --- a/run.sh +++ b/run.sh @@ -51,6 +51,7 @@ EOF _init() { _setup_git_hooks _setup_direnv + _env } _start() { @@ -61,10 +62,6 @@ _start() { } _env() { - if ! command -v gopass > /dev/null 2>&1; then - fail "Setup requires gopass, please install first" - exit 1 - fi if ! command -v op read op://Employee/AWS_ACCESS_KEY_ID/password > /dev/null 2>&1; then fail "Setup requires AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_BUCKET_NAME to be stored in 1Password. Please see Lookup Tables Initialization section in README.md " @@ -72,24 +69,24 @@ _env() { fi cat > ./.env<< EOF -GH_PACKAGES_REPOSITORY_USER=$(gopass show -o -y neuris/maven.pkg.github.com/digitalservicebund/neuris-juris-xml-export/username) -GH_PACKAGES_REPOSITORY_TOKEN=$(gopass show -o -y neuris/maven.pkg.github.com/digitalservicebund/neuris-juris-xml-export/token) -OAUTH2_CLIENT_ISSUER=$(gopass show -o -y neuris/maven.pkg.github.com/digitalservicebund/neuris-dev-oauth2-client/issuer-uri) -OAUTH2_CLIENT_ID=$(gopass show -o -y neuris/maven.pkg.github.com/digitalservicebund/neuris-dev-oauth2-client/client-id) -OAUTH2_CLIENT_SECRET=$(gopass show -o -y neuris/maven.pkg.github.com/digitalservicebund/neuris-dev-oauth2-client/client-secret) -E2E_TEST_USER=$(gopass show -o -y neuris/maven.pkg.github.com/digitalservicebund/neuris-e2e-test-user/username) -E2E_TEST_PASSWORD=$(gopass show -o -y neuris/maven.pkg.github.com/digitalservicebund/neuris-e2e-test-user/password) -E2E_TEST_USER_BGH=$(gopass show -o -y neuris/maven.pkg.github.com/digitalservicebund/neuris-e2e-test-user-bgh/username) -E2E_TEST_PASSWORD_BGH=$(gopass show -o -y neuris/maven.pkg.github.com/digitalservicebund/neuris-e2e-test-user-bgh/password) -E2E_TEST_USER_EXTERNAL=$(gopass show -o -y neuris/maven.pkg.github.com/digitalservicebund/neuris-e2e-test-user-external/username) -E2E_TEST_PASSWORD_EXTERNAL=$(gopass show -o -y neuris/maven.pkg.github.com/digitalservicebund/neuris-e2e-test-user-external/password) +GH_PACKAGES_REPOSITORY_USER=$(op read "op://Team NeuRIS/Github Package Repository/username") +GH_PACKAGES_REPOSITORY_TOKEN=$(op read "op://Team NeuRIS/Github Package Repository/credential") +OAUTH2_CLIENT_ISSUER=$( op read "op://Team NeuRIS/neuris-local oauth client/issuer_url") +OAUTH2_CLIENT_ID=$( op read "op://Team NeuRIS/neuris-local oauth client/username") +OAUTH2_CLIENT_SECRET=$( op read "op://Team NeuRIS/neuris-local oauth client/credential") +E2E_TEST_USER=$( op read "op://Team NeuRIS/staging e2e test user DS/username") +E2E_TEST_PASSWORD=$( op read "op://Team NeuRIS/staging e2e test user DS/password") +E2E_TEST_USER_BGH=$( op read "op://Team NeuRIS/e2e_test BGH neuris/username") +E2E_TEST_PASSWORD_BGH=$( op read "op://Team NeuRIS/e2e_test BGH neuris/password-new") +E2E_TEST_USER_EXTERNAL=$( op read "op://Team NeuRIS/Neuris Staging e2e Extern/username") +E2E_TEST_PASSWORD_EXTERNAL=$( op read "op://Team NeuRIS/Neuris Staging e2e Extern/password-new") MY_UID=$(id -u) MY_GID=$(id -g) DB_URL=jdbc:postgresql://localhost:5432/postgres DB_USER=test DB_PASSWORD=test -AWS_ACCESS_KEY_ID=$(op read op://Employee/AWS_ACCESS_KEY_ID/password) -AWS_SECRET_ACCESS_KEY=$(op read op://Employee/AWS_SECRET_ACCESS_KEY/password) +AWS_ACCESS_KEY_ID=$(op read "op://Employee/NeuRIS S3/access-key-id") +AWS_SECRET_ACCESS_KEY=$(op read "op://Employee/NeuRIS S3/secret-access-key") AWS_BUCKET_NAME=neuris-migration-juris-data EOF @@ -116,6 +113,7 @@ _dev() { for arg in "$@"; do case $arg in -i|--init) + echo $GH_PACKAGES_REPOSITORY_TOKEN | docker login ghcr.io -u $GH_PACKAGES_REPOSITORY_USER --password-stdin services="initialization" ;; -n|--no-backend) @@ -172,8 +170,7 @@ _help() { echo "Usage: ./run.sh [command]" echo "" echo "Available commands:" - echo "init Set up repository for development" - echo "env Provide shell env build/test tooling" + echo "init Initialize development environment (git hooks, env vars)" echo "dev Start full-stack development environment with loopup table initialization" echo " Add '-n' or '--no-backend' to start everything but backend and initialization" echo " Add '-i' or '--init' to only initialize the lookup tables (read ./migration_image.md for prerequisites)" @@ -187,7 +184,6 @@ _help() { cmd="${1:-}" case "$cmd" in "init") _init ;; - "env") _env ;; "dev") shift _dev "$@";;