diff --git a/.github/workflows/codesee-arch-diagram.yml b/.github/workflows/codesee-arch-diagram.yml new file mode 100644 index 00000000..fe56ca3f --- /dev/null +++ b/.github/workflows/codesee-arch-diagram.yml @@ -0,0 +1,87 @@ +on: + push: + branches: + - master + pull_request_target: + types: [opened, synchronize, reopened] + +name: CodeSee Map + +jobs: + test_map_action: + runs-on: ubuntu-latest + continue-on-error: true + name: Run CodeSee Map Analysis + steps: + - name: checkout + id: checkout + uses: actions/checkout@v2 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + fetch-depth: 0 + + # codesee-detect-languages has an output with id languages. + - name: Detect Languages + id: detect-languages + uses: Codesee-io/codesee-detect-languages-action@latest + + - name: Configure JDK 16 + uses: actions/setup-java@v2 + if: ${{ fromJSON(steps.detect-languages.outputs.languages).java }} + with: + java-version: '16' + distribution: 'zulu' + + # CodeSee Maps Go support uses a static binary so there's no setup step required. + + - name: Configure Node.js 14 + uses: actions/setup-node@v2 + if: ${{ fromJSON(steps.detect-languages.outputs.languages).javascript }} + with: + node-version: '14' + + - name: Configure Python 3.x + uses: actions/setup-python@v2 + if: ${{ fromJSON(steps.detect-languages.outputs.languages).python }} + with: + python-version: '3.10' + architecture: 'x64' + + - name: Configure Ruby '3.x' + uses: ruby/setup-ruby@v1 + if: ${{ fromJSON(steps.detect-languages.outputs.languages).ruby }} + with: + ruby-version: '3.0' + + # We need the rust toolchain because it uses rustc and cargo to inspect the package + - name: Configure Rust 1.x stable + uses: actions-rs/toolchain@v1 + if: ${{ fromJSON(steps.detect-languages.outputs.languages).rust }} + with: + toolchain: stable + + - name: Generate Map + id: generate-map + uses: Codesee-io/codesee-map-action@latest + with: + step: map + api_token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} + github_ref: ${{ github.ref }} + languages: ${{ steps.detect-languages.outputs.languages }} + + - name: Upload Map + id: upload-map + uses: Codesee-io/codesee-map-action@latest + with: + step: mapUpload + api_token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} + github_ref: ${{ github.ref }} + + - name: Insights + id: insights + uses: Codesee-io/codesee-map-action@latest + with: + step: insights + api_token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} + github_ref: ${{ github.ref }} diff --git a/.gitignore b/.gitignore index 3b713d5a..84856b09 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,13 @@ /dist .env* !.env.example +# nix /result* +/builds +# node-gyp +/build +# prebuildify +/prebuilds # Logs logs diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6de98bff..4d06c4ec 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,28 +1,245 @@ +workflow: + rules: + # Disable merge request pipelines + - if: $CI_MERGE_REQUEST_ID + when: never + - when: always + default: interruptible: true variables: + GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}" + GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git" GIT_SUBMODULE_STRATEGY: recursive + # Cache .npm + NPM_CONFIG_CACHE: "${CI_PROJECT_DIR}/tmp/npm" + # Prefer offline node module installation + NPM_CONFIG_PREFER_OFFLINE: "true" + # `ts-node` has its own cache + TS_CACHED_TRANSPILE_CACHE: "${CI_PROJECT_DIR}/tmp/ts-node-cache" + TS_CACHED_TRANSPILE_PORTABLE: "true" + # Homebrew cache only used by macos runner + HOMEBREW_CACHE: "${CI_PROJECT_DIR}/tmp/Homebrew" + +# Cached directories shared between jobs & pipelines per-branch per-runner +cache: + key: $CI_COMMIT_REF_SLUG + paths: + - ./tmp/npm/ + - ./tmp/ts-node-cache/ + # Homebrew cache is only used by the macos runner + - ./tmp/Homebrew + # `jest` cache is configured in jest.config.js + - ./tmp/jest/ stages: - - check + - check # Linting, unit tests + - build # Cross-platform library compilation, unit tests + - integration # Cross-platform application bundling, integration tests, and pre-release + - release # Cross-platform distribution and deployment image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner -lint: +check:lint: stage: check - interruptible: true + needs: [] script: - > nix-shell --run ' npm run lint; ' + rules: + # Runs on feature and staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual -test: +check:nix-dry: stage: check - interruptible: true + needs: [] + script: + - nix-build -v -v --dry-run ./release.nix + rules: + # Runs on feature and staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual + +check:test: + stage: check + needs: [] + script: + - > + nix-shell --run ' + npm run build --verbose; + npm test -- --ci; + ' + artifacts: + when: always + reports: + junit: + - ./tmp/junit.xml + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and staging and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual + +build:merge: + stage: build + needs: [] + allow_failure: true + script: + # Required for `gh pr create` + - git remote add upstream "$GH_PROJECT_URL" + - > + nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run ' + gh pr create \ + --head staging \ + --base master \ + --title "ci: merge staging to master" \ + --body "This is an automatic PR generated by the pipeline CI/CD. This will be automatically fast-forward merged if successful." \ + --assignee "@me" \ + --no-maintainer-edit \ + --repo "$GH_PROJECT_PATH" || true; + printf "Pipeline Attempt on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \ + | gh pr comment staging \ + --body-file - \ + --repo "$GH_PROJECT_PATH"; + ' + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +build:linux: + stage: build + needs: + - job: check:lint + optional: true script: - > nix-shell --run ' - npm run test; + npm run build --verbose; + npm test -- --ci; + ' + artifacts: + when: always + reports: + junit: + - ./tmp/junit.xml + paths: + - ./prebuilds/ + # Only the build:linux preserves the dist + - ./dist + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +build:windows: + stage: build + needs: + - job: check:lint + optional: true + tags: + - windows + before_script: + - choco install nodejs --version=16.14.2 -y + - choco install python --version=3.9.12 -y + - refreshenv + script: + - npm config set msvs_version 2019 + - npm install --ignore-scripts + - $env:Path = "$(npm bin);" + $env:Path + - npm run build --verbose + - npm test -- --ci + artifacts: + when: always + reports: + junit: + - ./tmp/junit.xml + paths: + - ./prebuilds/ + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +build:macos: + stage: build + needs: + - job: check:lint + optional: true + tags: + - shared-macos-amd64 + image: macos-11-xcode-12 + variables: + HOMEBREW_NO_INSTALL_UPGRADE: "true" + HOMEBREW_NO_INSTALL_CLEANUP: "true" + before_script: + - brew install node@16 + - brew link --overwrite node@16 + - brew install python@3.9 + - brew link --overwrite python@3.9 + - hash -r + script: + - npm install --ignore-scripts + - export PATH="$(npm bin):$PATH" + - export PREBUILD_ARCH=x64+arm64 + - npm run build --verbose + - npm test -- --ci + artifacts: + when: always + reports: + junit: + - ./tmp/junit.xml + paths: + - ./prebuilds/ + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +integration:merge: + stage: integration + needs: + - build:merge + # Requires mutual exclusion + resource_group: integration:merge + allow_failure: true + variables: + # Ensure that CI/CD is fetching all commits + # this is necessary to checkout origin/master + # and to also merge origin/staging + GIT_DEPTH: 0 + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run ' + printf "Pipeline Succeeded on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \ + | gh pr comment staging \ + --body-file - \ + --repo "$GH_PROJECT_PATH"; ' + - git remote add upstream "$GH_PROJECT_URL" + - git checkout origin/master + # Merge up to the current commit (not the latest commit) + - git merge --ff-only "$CI_COMMIT_SHA" + - git push upstream HEAD:master + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ diff --git a/.npmignore b/.npmignore index 37ae4c1d..6e59877f 100644 --- a/.npmignore +++ b/.npmignore @@ -1,15 +1,15 @@ .* +/*.nix /nix -/pkgs.nix -/default.nix -/shell.nix -/release.nix /tsconfig.json /tsconfig.build.json /jest.config.js +/scripts /src /tests /tmp /docs /benches +/build +/builds /dist/tsbuildinfo diff --git a/README.md b/README.md index 13215ea3..1d7809a6 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ npm run lintfix npm run bench ``` -View benchmarks here: https://github.com/MatrixAI/js-db/blob/master/benches/results/WorkerManager.chart.html with https://raw.githack.com/ +View benchmarks here: https://github.com/MatrixAI/js-db/blob/master/benches/results with https://raw.githack.com/ ### Docs Generation diff --git a/benches/results/DB1KiB.chart.html b/benches/results/DB1KiB.chart.html index 1e9feab0..83bfe1cf 100644 --- a/benches/results/DB1KiB.chart.html +++ b/benches/results/DB1KiB.chart.html @@ -16,7 +16,7 @@
Generated using TypeDoc
- Preparing search index...
- The search index is not available
@matrixai/dbClass DB
Hierarchy
Index
Constructors
Properties
Accessors
Methods
Constructors
constructor
Parameters
__namedParameters: { crypto?: { key: Buffer; ops: Crypto }; dbPath: string; fs: FileSystem; logger: Logger }
Optional crypto?: { key: Buffer; ops: Crypto }
key: Buffer
ops: Crypto
db Path: string
fs: FileSystem
logger: Logger
Returns DB
Properties
Readonly [init Lock]
Protected _db
Protected Optional crypto
Type declaration
key: Buffer
ops: Crypto
Readonly db Path
Protected fs
Protected logger
Protected transaction Counter
Protected Optional worker Manager
Accessors
[destroyed]
Returns boolean
[running]
Returns boolean
[status]
Returns Status
db
Returns Readonly<LevelDB<string | Buffer, Buffer>>
Methods
batch
Batches operations together atomically
-Parameters
ops: readonly DBOp[]
Returns Promise<void>
Protected canary Check
Returns Promise<void>
clear
Parameters
ops: readonly DBOp[]
Returns Promise<void>
Protected canary Check
Returns Promise<void>
clear
Clear all key values for a specific level This is not atomic, it will iterate over a snapshot of the DB
-Parameters
levelPath: readonly (string | Buffer)[] = []
Returns Promise<void>
count
Parameters
levelPath: readonly (string | Buffer)[] = []
Returns Promise<number>
del
Parameters
levelPath: readonly (string | Buffer)[] = []
Returns Promise<void>
count
Parameters
levelPath: readonly (string | Buffer)[] = []
Returns Promise<number>
del
Deletes a key from the DB
-Parameters
keyPath: string | Buffer | readonly (string | Buffer)[]
Returns Promise<void>
deserialize Decrypt
Type parameters
T
Parameters
cipherTextBuf: Buffer
raw: false
Returns Promise<T>
Parameters
cipherTextBuf: Buffer
raw: true
Returns Promise<Buffer>
destroy
Returns Promise<void>
dump
Dump from root level -It is intended for diagnostics
-Parameters
Optional levelPath: readonly (string | Buffer)[]
Optional raw: false
Returns Promise<[string, any][]>
Parameters
levelPath: undefined | readonly (string | Buffer)[]
raw: true
Returns Promise<[Buffer, Buffer][]>
get
Parameters
keyPath: string | Buffer | readonly (string | Buffer)[]
Returns Promise<void>
deserialize Decrypt
Type parameters
T
Parameters
cipherTextBuf: Buffer
raw: false
Returns Promise<T>
Parameters
cipherTextBuf: Buffer
raw: true
Returns Promise<Buffer>
destroy
Returns Promise<void>
dump
Dump from DB +This will show entries from all levels +It is intended for diagnostics +Use
+console.dir
instead ofconsole.log
to debug the result +Setroot
totrue
if you want to dump from root levelsType parameters
V
Parameters
Optional levelPath: readonly (string | Buffer)[]
Optional raw: false
Optional root: boolean
Returns Promise<[string, V][]>
Parameters
levelPath: undefined | readonly (string | Buffer)[]
raw: true
Optional root: boolean
Returns Promise<[Buffer, Buffer][]>
get
Gets a value from the DB Use raw to return the raw decrypted buffer
-Type parameters
T
Parameters
keyPath: string | Buffer | readonly (string | Buffer)[]
Optional raw: false
Returns Promise<undefined | T>
Parameters
keyPath: string | Buffer | readonly (string | Buffer)[]
raw: true
Returns Promise<undefined | Buffer>
iterator
Type parameters
T
Parameters
keyPath: string | Buffer | readonly (string | Buffer)[]
Optional raw: false
Returns Promise<undefined | T>
Parameters
keyPath: string | Buffer | readonly (string | Buffer)[]
raw: true
Returns Promise<undefined | Buffer>
iterator
Public iterator that works from the data level If keys and values are both false, this iterator will not run at all You must have at least one of them being true or undefined
-Parameters
options: AbstractIteratorOptions<any> & { keys: false; values: false }
Optional levelPath: readonly (string | Buffer)[]
Returns DBIterator<undefined, undefined>
Parameters
options: AbstractIteratorOptions<any> & { keys: false }
Optional levelPath: readonly (string | Buffer)[]
Returns DBIterator<undefined, Buffer>
Parameters
options: AbstractIteratorOptions<any> & { values: false }
Optional levelPath: readonly (string | Buffer)[]
Returns DBIterator<Buffer, undefined>
Parameters
Optional options: AbstractIteratorOptions<any>
Optional levelPath: readonly (string | Buffer)[]
Returns DBIterator<Buffer, Buffer>
put
Parameters
options: DBIteratorOptions & { keys: false; values: false }
Optional levelPath: readonly (string | Buffer)[]
Returns DBIterator<undefined, undefined>
Type parameters
V
Parameters
options: DBIteratorOptions & { keys: false; valueAsBuffer: false }
Optional levelPath: readonly (string | Buffer)[]
Returns DBIterator<undefined, V>
Parameters
options: DBIteratorOptions & { keys: false }
Optional levelPath: readonly (string | Buffer)[]
Returns DBIterator<undefined, Buffer>
Parameters
options: DBIteratorOptions & { values: false }
Optional levelPath: readonly (string | Buffer)[]
Returns DBIterator<readonly (string | Buffer)[], undefined>
Type parameters
V
Parameters
options: DBIteratorOptions & { valueAsBuffer: false }
Optional levelPath: readonly (string | Buffer)[]
Returns DBIterator<readonly (string | Buffer)[], V>
Parameters
Optional options: DBIteratorOptions
Optional levelPath: readonly (string | Buffer)[]
Returns DBIterator<readonly (string | Buffer)[], Buffer>
put
Put a key and value into the DB Use raw to put raw encrypted buffer
-Parameters
keyPath: string | Buffer | readonly (string | Buffer)[]
value: any
Optional raw: false
Returns Promise<void>
Parameters
keyPath: string | Buffer | readonly (string | Buffer)[]
value: Buffer
raw: true
Returns Promise<void>
serialize Encrypt
Parameters
value: any
raw: false
Returns Promise<Buffer>
Parameters
value: Buffer
raw: true
Returns Promise<Buffer>
set Worker Manager
Parameters
workerManager: DBWorkerManagerInterface
Returns void
Protected setup Db
Parameters
dbPath: string
Returns Promise<LevelDB<string | Buffer, Buffer>>
Protected setup Root Levels
Returns Promise<void>
start
Parameters
__namedParameters: { fresh?: boolean } = {}
Optional fresh?: boolean
Returns Promise<void>
stop
Returns Promise<void>
transaction
Returns ResourceAcquire<DBTransaction>
unset Worker Manager
Returns void
with TransactionF
Type parameters
T
Parameters
f: (tran: DBTransaction) => Promise<T>
Parameters
tran: DBTransaction
Returns Promise<T>
Returns Promise<T>
with TransactionG
Type parameters
T
TReturn
TNext
Parameters
g: (tran: DBTransaction) => AsyncGenerator<T, TReturn, TNext>
Parameters
tran: DBTransaction
Returns AsyncGenerator<T, TReturn, TNext>
Returns AsyncGenerator<T, TReturn, TNext>
Static createDB
Parameters
__namedParameters: { crypto?: { canary?: Buffer; key: Buffer; ops: Crypto }; dbPath: string; fresh?: boolean; fs?: FileSystem; logger?: Logger }
Optional crypto?: { canary?: Buffer; key: Buffer; ops: Crypto }
Optional canary?: Buffer
key: Buffer
ops: Crypto
db Path: string
Optional fresh?: boolean
Optional fs?: FileSystem
Optional logger?: Logger
Returns Promise<DB>
Generated using TypeDoc