diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..f3245e76 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,8 @@ +root = true + +[*] +end_of_line = lf +indent_style = space +indent_size = 2 +insert_final_newline = true +trim_trailing_whitespace = true diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..30262b05 --- /dev/null +++ b/.env.example @@ -0,0 +1,40 @@ +# Unused atm (jest sets this to `test`) +NODE_ENV=development + +# Debug node modules - https://nodejs.org/api/cli.html#node_debugmodule +# NODE_DEBUG= + +# Debug node native modules - https://nodejs.org/api/cli.html#node_debug_nativemodule +# NODE_DEBUG_NATIVE= + +# Path to PK executable to override tests/bin target +# PK_TEST_COMMAND= + +# If set, indicates that `PK_TEST_COMMAND` is targetting docker +# PK_TEST_COMMAND_DOCKER= +# Accessing AWS for testnet.polykey.io and mainnet.polykey.io deployment +AWS_DEFAULT_REGION='ap-southeast-2' +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= + +# Path to container registry authentication file used by `skopeo` +# The file has the same contents as `DOCKER_AUTH_CONFIG` +# Use this command to acquire the auth file at `./tmp/auth.json`: +# ``` +# printf 'PASSWORD' | skopeo login \ +# --username 'USERNAME' \ +# --password-stdin \ +# $CI_REGISTRY_IMAGE \ +# --authfile=./tmp/auth.json +# ``` +# REGISTRY_AUTH_FILE= + +# Authenticate to GitHub with `gh` +# GITHUB_TOKEN= + +# To allow testing different executables in the bin tests +# Both PK_TEST_COMMAND and PK_TEST_PLATFORM must be set at the same time +# PK_TEST_COMMAND= #Specify the shell command we want to test against +# PK_TEST_PLATFORM=docker #Overrides the auto set `testPlatform` variable used for enabling platform specific tests +# PK_TEST_TMPDIR= #Sets the `global.tmpDir` variable to allow overriding the temp directory used for tests + diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 00000000..72f98eb5 --- /dev/null +++ b/.eslintignore @@ -0,0 +1 @@ +src/proto/* diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 00000000..1eec3982 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,177 @@ +{ + "env": { + "browser": true, + "commonjs": true, + "es2021": true, + "node": true, + "jest": true + }, + "parser": "@typescript-eslint/parser", + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended", + "plugin:prettier/recommended" + ], + "plugins": [ + "import" + ], + "parserOptions": { + "project": "tsconfig.json", + "sourceType": "module" + }, + "rules": { + "linebreak-style": ["error", "unix"], + "no-empty": 1, + "no-useless-catch": 1, + "no-prototype-builtins": 1, + "no-constant-condition": 0, + "no-useless-escape": 0, + "no-console": "error", + "no-restricted-globals": [ + "error", + { + "name": "global", + "message": "Use `globalThis` instead" + }, + { + "name": "window", + "message": "Use `globalThis` instead" + } + ], + "require-yield": 0, + "eqeqeq": ["error", "smart"], + "spaced-comment": [ + "warn", + "always", + { + "line": { + "exceptions": ["-"] + }, + "block": { + "exceptions": ["*"] + }, + "markers": ["/"] + } + ], + "capitalized-comments": [ + "warn", + "always", + { + "ignoreInlineComments": true, + "ignoreConsecutiveComments": true + } + ], + "curly": [ + "error", + "multi-line", + "consistent" + ], + "import/order": [ + "error", + { + "groups": [ + "type", + "builtin", + "external", + "internal", + "index", + "sibling", + "parent", + "object" + ], + "pathGroups": [ + { + "pattern": "@", + "group": "internal" + }, + { + "pattern": "@/**", + "group": "internal" + } + ], + "pathGroupsExcludedImportTypes": [ + "type" + ], + "newlines-between": "never" + } + ], + "@typescript-eslint/no-namespace": 0, + "@typescript-eslint/no-explicit-any": 0, + "@typescript-eslint/explicit-module-boundary-types": 0, + "@typescript-eslint/no-unused-vars": [ + "warn", + { + "varsIgnorePattern": "^_", + "argsIgnorePattern": "^_" + } + ], + "@typescript-eslint/no-inferrable-types": 0, + "@typescript-eslint/no-non-null-assertion": 0, + "@typescript-eslint/no-this-alias": 0, + "@typescript-eslint/no-var-requires": 0, + "@typescript-eslint/no-empty-function": 0, + "@typescript-eslint/no-empty-interface": 0, + "@typescript-eslint/consistent-type-imports": ["error"], + "@typescript-eslint/consistent-type-exports": ["error"], + "no-throw-literal": "off", + "@typescript-eslint/no-throw-literal": "off", + "@typescript-eslint/no-floating-promises": ["error", { + "ignoreVoid": true, + "ignoreIIFE": true + }], + "@typescript-eslint/no-misused-promises": ["error", { + "checksVoidReturn": false + }], + "@typescript-eslint/await-thenable": ["error"], + "@typescript-eslint/naming-convention": [ + "error", + { + "selector": "default", + "format": ["camelCase"], + "leadingUnderscore": "allow", + "trailingUnderscore": "allowSingleOrDouble" + }, + { + "selector": "function", + "format": ["camelCase", "PascalCase"], + "leadingUnderscore": "allow", + "trailingUnderscore": "allowSingleOrDouble" + }, + { + "selector": "variable", + "format": ["camelCase", "UPPER_CASE", "PascalCase"], + "leadingUnderscore": "allow", + "trailingUnderscore": "allowSingleOrDouble" + }, + { + "selector": "parameter", + "format": ["camelCase"], + "leadingUnderscore": "allow", + "trailingUnderscore": "allowSingleOrDouble" + }, + { + "selector": "typeLike", + "format": ["PascalCase"], + "trailingUnderscore": "allowSingleOrDouble" + }, + { + "selector": "enumMember", + "format": ["PascalCase", "UPPER_CASE"] + }, + { + "selector": "objectLiteralProperty", + "format": null + }, + { + "selector": "typeProperty", + "format": null + } + ], + "@typescript-eslint/ban-ts-comment": [ + "error", + { + "ts-ignore": "allow-with-description" + } + ] + } +} diff --git a/.github/workflows/codesee-arch-diagram.yml b/.github/workflows/codesee-arch-diagram.yml new file mode 100644 index 00000000..80f58e63 --- /dev/null +++ b/.github/workflows/codesee-arch-diagram.yml @@ -0,0 +1,23 @@ +# This workflow was added by CodeSee. Learn more at https://codesee.io/ +# This is v2.0 of this workflow file +on: + push: + branches: + - staging + pull_request_target: + types: [opened, synchronize, reopened] + +name: CodeSee + +permissions: read-all + +jobs: + codesee: + runs-on: ubuntu-latest + continue-on-error: true + name: Analyze the repo with CodeSee + steps: + - uses: Codesee-io/codesee-action@v2 + with: + codesee-token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} + codesee-url: https://app.codesee.io diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..78a1b31b --- /dev/null +++ b/.gitignore @@ -0,0 +1,132 @@ +/tmp +/dist +.env* +!.env.example +# nix +/result* +/builds +# node-gyp +/build +# prebuildify +/prebuilds + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env +.env.test + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# editor +.vscode/ +.idea/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..f5a0d165 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,662 @@ +workflow: + rules: + # Disable merge request pipelines + - if: $CI_MERGE_REQUEST_ID + when: never + - when: always + +variables: + GIT_SUBMODULE_STRATEGY: recursive + GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}" + GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git" + # Cache .npm + npm_config_cache: "${CI_PROJECT_DIR}/tmp/npm" + # Prefer offline node module installation + npm_config_prefer_offline: "true" + # Homebrew cache only used by macos runner + HOMEBREW_CACHE: "${CI_PROJECT_DIR}/tmp/Homebrew" + +default: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + interruptible: true + before_script: + # Replace this in windows runners that use powershell + # with `mkdir -Force "$CI_PROJECT_DIR/tmp"` + - mkdir -p "$CI_PROJECT_DIR/tmp" + +# Cached directories shared between jobs & pipelines per-branch per-runner +cache: + key: $CI_COMMIT_REF_SLUG + # Preserve cache even if job fails + when: 'always' + paths: + - ./tmp/npm/ + # Homebrew cache is only used by the macos runner + - ./tmp/Homebrew + # Chocolatey cache is only used by the windows runner + - ./tmp/chocolatey/ + # `jest` cache is configured in jest.config.js + - ./tmp/jest/ + +stages: + - check # Linting, unit tests + - build # Cross-platform library compilation, unit tests + - integration # Cross-platform application bundling, integration tests, and pre-release + - release # Cross-platform distribution and deployment + +check:scratch: + stage: check + needs: [] + script: + - > + nix-shell --arg ci true --run $' + npm test -- --ci tests/scratch.test.ts; + ' + allow_failure: true + rules: + - when: manual + +check:lint: + stage: check + needs: [] + script: + - > + nix-shell --arg ci true --run $' + npm run lint; + npm run lint-shell; + ' + rules: + # Runs on feature and staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual + +check:nix-dry: + stage: check + needs: [] + script: + - nix-build -v -v --dry-run ./release.nix + rules: + # Runs on feature and staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual + +check:test-generate: + stage: check + needs: [] + script: + - > + nix-shell --arg ci true --run $' + ./scripts/check-test-generate.sh > ./tmp/check-test.yml; + ' + artifacts: + when: always + paths: + - ./tmp/check-test.yml + rules: + # Runs on feature commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and staging and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual + +check:test: + stage: check + needs: + - check:test-generate + trigger: + include: + - artifact: tmp/check-test.yml + job: check:test-generate + strategy: depend + inherit: + variables: false + variables: + PARENT_PIPELINE_ID: $CI_PIPELINE_ID + rules: + # Runs on feature commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and staging and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual + +build:merge: + stage: build + needs: [] + allow_failure: true + script: + # Required for `gh pr create` + - git remote add upstream "$GH_PROJECT_URL" + - > + nix-shell --arg ci true --run $' + gh pr create \ + --head staging \ + --base master \ + --title "ci: merge staging to master" \ + --body "This is an automatic PR generated by the pipeline CI/CD. This will be automatically fast-forward merged if successful." \ + --assignee "@me" \ + --no-maintainer-edit \ + --repo "$GH_PROJECT_PATH" || true; + printf "Pipeline Attempt on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \ + | gh pr comment staging \ + --body-file - \ + --repo "$GH_PROJECT_PATH"; + ' + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +build:dist: + stage: build + needs: [] + script: + - > + nix-shell --arg ci true --run $' + npm run build --verbose; + ' + artifacts: + when: always + paths: + - ./dist + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +build:platforms-generate: + stage: build + needs: [] + script: + - > + nix-shell --arg ci true --run $' + ./scripts/build-platforms-generate.sh > ./tmp/build-platforms.yml; + ' + artifacts: + when: always + paths: + - ./tmp/build-platforms.yml + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +build:platforms: + stage: build + needs: + - build:platforms-generate + trigger: + include: + - artifact: tmp/build-platforms.yml + job: build:platforms-generate + strategy: depend + inherit: + variables: false + variables: + PARENT_PIPELINE_ID: $CI_PIPELINE_ID + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +build:prerelease: + stage: build + needs: + - build:dist + - build:platforms + # Don't interrupt publishing job + interruptible: false + script: + - echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc + - echo 'Publishing library prerelease' + - > + nix-shell --arg ci true --run $' + npm publish --tag prerelease --access public; + ' + after_script: + - rm -f ./.npmrc + rules: + # Only runs on tag pipeline where the tag is a prerelease version + # This requires dependencies to also run on tag pipeline + # However version tag comes with a version commit + # Dependencies must not run on the version commit + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+-.*[0-9]+$/ + +integration:builds: + stage: integration + needs: + - build:dist + - build:platforms + script: + - mkdir -p ./builds + - > + build_application="$(nix-build \ + --max-jobs "$(nproc)" --cores "$(nproc)" \ + ./release.nix \ + --attr application \ + )" + - > + nix-store --export $( \ + nix-store --query --requisites "$build_application" \ + ) | gzip > ./builds/js-polykey.closure.gz + # non-nix targets + - > + builds="$(nix-build \ + --max-jobs "$(nproc)" --cores "$(nproc)" \ + ./release.nix \ + --attr docker \ + --attr package.linux.x64.elf \ + --attr package.windows.x64.exe \ + --attr package.macos.x64.macho \ + --attr package.macos.arm64.macho)" + - cp -r $builds ./builds/ + artifacts: + paths: + - ./builds/ + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +integration:deployment: + stage: integration + needs: + - integration:builds + # Don't interrupt deploying job + interruptible: false + # Requires mutual exclusion + resource_group: integration:deployment + environment: + name: 'testnet' + deployment_tier: 'staging' + url: 'https://testnet.polykey.io' + variables: + REGISTRY_AUTH_FILE: "./tmp/registry-auth-file.json" + # Override CI_REGISTRY_IMAGE to point to ECR + CI_REGISTRY_IMAGE: '015248367786.dkr.ecr.ap-southeast-2.amazonaws.com/polykey' + script: + - echo 'Deploying container image to ECR' + - > + nix-shell --arg ci true --run $' + aws ecr get-login-password \ + | skopeo login \ + --username AWS \ + --password-stdin \ + --authfile "$REGISTRY_AUTH_FILE" \ + "$CI_REGISTRY_IMAGE"; + image=(./builds/*-docker-*); + ./scripts/deploy-image.sh "${image[0]}" \'testnet\' "$CI_REGISTRY_IMAGE"; + ' + - echo 'Deploying ECS service to testnet' + - > + nix-shell --run $' + ./scripts/deploy-service.sh \'polykey-testnet\'; + ' + after_script: + - rm -f "$REGISTRY_AUTH_FILE" + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +integration:nix: + stage: integration + needs: + - integration:builds + - job: integration:deployment + optional: true + script: + - > + build_application="$( \ + gunzip -c ./builds/js-polykey.closure.gz | \ + nix-store --import | \ + tail -1 \ + )" + - $build_application/polykey + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +integration:docker: + stage: integration + needs: + - integration:builds + - job: integration:deployment + optional: true + services: + - docker:20.10.16-dind + variables: + DOCKER_TLS_CERTDIR: "/certs" + FF_NETWORK_PER_BUILD: "true" + PK_TEST_PLATFORM: "docker" + PK_TEST_TMPDIR: "${CI_PROJECT_DIR}/tmp/test" + script: + - docker info + - mkdir $PK_TEST_TMPDIR + - > + nix-shell --arg ci true --run $' + image_and_tag="$(docker load --input ./builds/*docker* | cut -d\' \' -f3)"; + PK_TEST_COMMAND="docker run \$DOCKER_OPTIONS $image_and_tag" npm run test; + ' + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +integration:linux: + stage: integration + needs: + - integration:builds + - job: integration:deployment + optional: true + image: ubuntu:latest + script: + - for f in ./builds/*-linux-*; do "$f"; done + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +.integration:windows: + inherit: + default: + - interruptible + stage: integration + needs: + - integration:builds + - job: integration:deployment + optional: true + tags: + - windows + before_script: + - mkdir -Force "$CI_PROJECT_DIR/tmp" + script: + - Get-ChildItem -File ./builds/*-win-* | ForEach {& $_.FullName} + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +.integration:macos: + stage: integration + needs: + - integration:builds + - job: integration:deployment + optional: true + tags: + - saas-macos-medium-m1 + image: macos-12-xcode-14 + script: + - for f in ./builds/*-macos-x64*; do "$f"; done + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +integration:prerelease: + stage: integration + needs: + - integration:builds + - job: build:prerelease + optional: true + - job: integration:nix + optional: true + - job: integration:docker + optional: true + - job: integration:linux + optional: true + # - job: integration:windows + # optional: true + # - job: integration:macos + # optional: true + # Don't interrupt publishing job + interruptible: false + # Requires mutual exclusion + resource_group: integration:prerelease + variables: + REGISTRY_AUTH_FILE: "./tmp/registry-auth-file.json" + script: + - echo 'Publishing application prerelease' + - > + nix-shell --arg ci true --run $' + if gh release view "$CI_COMMIT_TAG" --repo "$GH_PROJECT_PATH" >/dev/null; then \ + gh release \ + upload "$CI_COMMIT_TAG" \ + builds/*.closure.gz \ + builds/*-docker-* \ + builds/*-linux-* \ + builds/*-win-* \ + builds/*-macos-* \ + --clobber \ + --repo "$GH_PROJECT_PATH"; \ + else \ + gh release \ + create "$CI_COMMIT_TAG" \ + builds/*.closure.gz \ + builds/*-docker-* \ + builds/*-linux-* \ + builds/*-win-* \ + builds/*-macos-* \ + --title "${CI_COMMIT_TAG}-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --notes "" \ + --prerelease \ + --target staging \ + --repo "$GH_PROJECT_PATH"; \ + fi; + ' + - echo 'Prereleasing container image' + - > + nix-shell --arg ci true --run $' + skopeo login \ + --username "$CI_REGISTRY_USER" \ + --password "$CI_REGISTRY_PASSWORD" \ + --authfile "$REGISTRY_AUTH_FILE" \ + "$CI_REGISTRY_IMAGE"; + image=(./builds/*-docker-*); + ./scripts/deploy-image.sh "${image[0]}" \'testnet\' "$CI_REGISTRY_IMAGE"; + ' + after_script: + - rm -f "$REGISTRY_AUTH_FILE" + rules: + # Only runs on tag pipeline where the tag is a prerelease version + # This requires dependencies to also run on tag pipeline + # However version tag comes with a version commit + # Dependencies must not run on the version commit + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+-.*[0-9]+$/ + +integration:merge: + stage: integration + needs: + - build:merge + - job: build:platforms + optional: true + - job: integration:nix + optional: true + - job: integration:docker + optional: true + - job: integration:linux + optional: true + # - job: integration:windows + # optional: true + # - job: integration:macos + # optional: true + # Requires mutual exclusion + resource_group: integration:merge + allow_failure: true + variables: + # Ensure that CI/CD is fetching all commits + # this is necessary to checkout origin/master + # and to also merge origin/staging + GIT_DEPTH: 0 + script: + - > + nix-shell --arg ci true --run $' + printf "Pipeline Succeeded on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \ + | gh pr comment staging \ + --body-file - \ + --repo "$GH_PROJECT_PATH"; + ' + - git remote add upstream "$GH_PROJECT_URL" + - git checkout origin/master + # Merge up to the current commit (not the latest commit) + - git merge --ff-only "$CI_COMMIT_SHA" + - git push upstream HEAD:master + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +release:deployment:branch: + stage: release + # Only needs integration:builds from the staging branch pipeline + needs: + - project: $CI_PROJECT_PATH + job: integration:builds + ref: staging + artifacts: true + # Don't interrupt deploying job + interruptible: false + # Requires mutual exclusion (also with release:deployment:tag) + resource_group: release:deployment + environment: + name: 'mainnet' + deployment_tier: 'production' + url: 'https://mainnet.polykey.io' + variables: + REGISTRY_AUTH_FILE: "./tmp/registry-auth-file.json" + # Override CI_REGISTRY_IMAGE to point to ECR + CI_REGISTRY_IMAGE: '015248367786.dkr.ecr.ap-southeast-2.amazonaws.com/polykey' + script: + - echo 'Deploying container image to ECR' + - > + nix-shell --arg ci true --run $' + aws ecr get-login-password \ + | skopeo login \ + --username AWS \ + --password-stdin \ + --authfile "$REGISTRY_AUTH_FILE" \ + "$CI_REGISTRY_IMAGE"; + image=(./builds/*-docker-*); + ./scripts/deploy-image.sh "${image[0]}" \'mainnet\' "$CI_REGISTRY_IMAGE"; + echo \'Deploying ECS service to mainnet\'; + ' + after_script: + - rm -f "$REGISTRY_AUTH_FILE" + rules: + # Runs on master commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +release:deployment:tag: + stage: release + # Tag pipelines run independently + needs: + - integration:builds + - integration:merge + # Don't interrupt deploying job + interruptible: false + # Requires mutual exclusion (also with release:deployment:branch) + resource_group: release:deployment + environment: + name: 'mainnet' + deployment_tier: 'production' + url: 'https://mainnet.polykey.io' + variables: + REGISTRY_AUTH_FILE: "./tmp/registry-auth-file.json" + # Override CI_REGISTRY_IMAGE to point to ECR + CI_REGISTRY_IMAGE: '015248367786.dkr.ecr.ap-southeast-2.amazonaws.com/polykey' + script: + - echo 'Deploying container image to ECR' + - > + nix-shell --arg ci true --run $' + aws ecr get-login-password \ + | skopeo login \ + --username AWS \ + --password-stdin \ + --authfile "$REGISTRY_AUTH_FILE" \ + "$CI_REGISTRY_IMAGE"; + image=(./builds/*-docker-*); + ./scripts/deploy-image.sh "${image[0]}" \'mainnet\' "$CI_REGISTRY_IMAGE"; + echo \'Deploying ECS service to mainnet\'; + ' + after_script: + - rm -f "$REGISTRY_AUTH_FILE" + rules: + # Runs on tag pipeline where the tag is a release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ + +release:distribution: + stage: release + needs: + - build:dist + - build:platforms + - integration:builds + - integration:merge + - release:deployment:tag + # Don't interrupt publishing job + interruptible: false + # Requires mutual exclusion + resource_group: release:distribution + variables: + REGISTRY_AUTH_FILE: "./tmp/registry-auth-file.json" + script: + - echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc + - echo 'Publishing library' + - > + nix-shell --arg ci true --run $' + npm publish --access public; + ' + - echo 'Releasing application builds' + - > + nix-shell --arg ci true --run $' + gh release \ + create "$CI_COMMIT_TAG" \ + builds/*.closure.gz \ + builds/*-docker-* \ + builds/*-linux-* \ + builds/*-win-* \ + builds/*-macos-* \ + --title "${CI_COMMIT_TAG}-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --notes "" \ + --target master \ + --repo "$GH_PROJECT_PATH"; + ' + - echo 'Releasing container image' + - > + nix-shell --arg ci true --run $' + skopeo login \ + --username "$CI_REGISTRY_USER" \ + --password "$CI_REGISTRY_PASSWORD" \ + --authfile "$REGISTRY_AUTH_FILE" \ + "$CI_REGISTRY_IMAGE"; + image=(./builds/*-docker-*); + ./scripts/deploy-image.sh "${image[0]}" \'mainnet\' "$CI_REGISTRY_IMAGE"; + ' + after_script: + - rm -f ./.npmrc + - rm -f "$REGISTRY_AUTH_FILE" + rules: + # Only runs on tag pipeline where the tag is a release version + # This requires dependencies to also run on tag pipeline + # However version tag comes with a version commit + # Dependencies must not run on the version commit + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ diff --git a/.npmignore b/.npmignore new file mode 100644 index 00000000..ea6884b3 --- /dev/null +++ b/.npmignore @@ -0,0 +1,17 @@ +.* +/*.nix +/nix +/tsconfig.json +/tsconfig.build.json +/babel.config.js +/jest.config.js +/nodemon.json +/scripts +/src +/tests +/tmp +/docs +/benches +/build +/builds +/dist/tsbuildinfo diff --git a/.npmrc b/.npmrc new file mode 100644 index 00000000..7c06da2c --- /dev/null +++ b/.npmrc @@ -0,0 +1,2 @@ +# Enables npm link +prefix=~/.npm diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 00000000..fa9699b8 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,7 @@ +{ + "semi": true, + "trailingComma": "all", + "singleQuote": true, + "printWidth": 80, + "tabWidth": 2 +} diff --git a/README.md b/README.md index e55642cc..5aa341c5 100644 --- a/README.md +++ b/README.md @@ -1 +1,289 @@ # Polykey-CLI + +staging:[](https://gitlab.com/MatrixAI/open-source/Polykey-CLI/commits/staging) +master:[](https://gitlab.com/MatrixAI/open-source/Polykey-CLI/commits/master) + +Polykey is an open-source decentralized secrets management and sharing system. It is made for today's decentralized world of people, services and devices. + +* Decentralized Encrypted Storage - No storage of secrets on third parties, secrets are stored on your device and synchronised point-to-point between Polykey nodes. +* Secure Peer-to-Peer Communications - Polykey bootstraps TLS keys by federating trusted social identities (e.g. GitHub). +* Secure Computational Workflows - Share secrets (passwords, keys, tokens and certificates) with people, between teams, and across machine infrastructure. + +
+
+
Generated using TypeDoc
Note that JavaScript libraries are not packaged in Nix. Only JavaScript applications are.
+Building the package:
+nix-build -E '(import ./pkgs.nix {}).callPackage ./default.nix {}'
+
+Building the releases:
+nix-build ./release.nix --attr application
nix-build ./release.nix --attr docker
nix-build ./release.nix --attr package.linux.x64.elf
nix-build ./release.nix --attr package.windows.x64.exe
nix-build ./release.nix --attr package.macos.x64.macho
+
+Install into Nix user profile:
+nix-env -f ./release.nix --install --attr application
+
+Install into Docker:
+loaded="$(docker load --input "$(nix-build ./release.nix --attr docker)")"
image="$(cut -d' ' -f3 <<< "$loaded")"
docker run -it "$image"
+
+
+
+ Run nix-shell
, and once you're inside, you can use:
# install (or reinstall packages from package.json)
npm install
# build the dist
npm run build
# run the repl (this allows you to import from ./src)
npm run ts-node
# run the tests
npm run test
# lint the source code
npm run lint
# automatically fix the source
npm run lintfix
+
+
+
+ When calling executables in development, use this style:
+npm run typescript-demo-lib -- p1 p2 p3
+
+The --
is necessary to make npm
understand that the parameters are for your own executable, and not parameters to npm
.
$ npm run ts-node
> import fs from 'fs';
> fs
> import { Library } from '@';
> Library
> import Library as Library2 from './src/lib/Library';
+
+You can also create test files in ./src
, and run them with npm run ts-node ./src/test.ts
.
This allows you to test individual pieces of typescript code, and it makes it easier when doing large scale architecting of TypeScript code.
+ + +Due to https://github.com/microsoft/TypeScript/issues/10866, you cannot use path aliases without a bundler like Webpack to further transform the generated JavaScript code in order to resolve the path aliases. Because this is a simple library demonstration, there's no need to use a bundler. In fact, for such libraries, it is far more efficient to not bundle the code.
+However, we have left the path alias configuration in tsconfig.json
, jest.config.js
and in the tests we are making use of the @
alias.
When developing on multiple NPM packages, it can be easier to use npm link
so that changes are immediately reflected rather than repeatedly publishing packages. To do this, you need to use npm link
. After linking a local directory, you need to provide tsconfig.json
paths so TypeScript compiler can find the right files.
For example when linking @matrixai/db
located in ../js-db
:
npm link ../js-db
+
+You would need to add these paths to tsconfig.json
:
"paths": {
"@": ["index"],
"@/*": ["*"],
"@matrixai/db": ["../node_modules/@matrixai/db/src"],
"@matrixai/db/*": ["../node_modules/@matrixai/db/src/*"]
},
+
+
+
+ There are some nuances when packaging with native modules. +Included native modules are level witch include leveldown and utp-native.
+If a module is not set to public then pkg defaults to including it as bytecode.
+To avoid this breaking with the --no-bytecode
flag we need to add --public-packages "*"
To get leveldown to work with pkg we need to include the prebuilds with the executable.
+after building with pkg you need to copy from node_modules/leveldown/prebuilds
-> path_to_executable/prebuilds
+You only need to include the prebuilds for the arch you are targeting. e.g. for linux-x64 you need prebuild/linux-x64
.
The folder structure for the executable should look like this.
+Including utp-native is simpler, you just need to add it as an asset for pkg. +Add the following lines to the package.json.
+"pkg": {
"assets": "node_modules/utp-native/**/*"
}
+
+
+
+ To make sure that the worker threads work properly you need to include the compiled worker scripts as an asset.
+This can be fixed by adding the following to package.json
"pkg": {
"assets": "dist/bin/worker.js"
}
+
+If you need to include multiple assets then add them as an array.
+"pkg": {
"assets": [
"node_modules/utp-native/**/*",
"dist/bin/worker.js"
]
}
+
+
+
+ npm run docs
+
+See the docs at: https://matrixai.github.io/TypeScript-Demo-Lib/
+ + +Publishing is handled automatically by the staging pipeline.
+Prerelease:
+# npm login
npm version prepatch --preid alpha # premajor/preminor/prepatch
git push --follow-tags
+
+Release:
+# npm login
npm version patch # major/minor/patch
git push --follow-tags
+
+Manually:
+# npm login
npm version patch # major/minor/patch
npm run build
npm publish --access public
git push
git push --tags
+
+Generated using TypeDoc
Generated using TypeDoc