Fix Iterator KeyPath and Enable Iterator keyAsBuffer and valueAsBuffer

.github/workflows/codesee-arch-diagram.yml

@@ -0,0 +1,87 @@
+on:
+  push:
+    branches:
+      - master
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+name: CodeSee Map
+
+jobs:
+  test_map_action:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    name: Run CodeSee Map Analysis
+    steps:
+      - name: checkout
+        id: checkout
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.ref }}
+          fetch-depth: 0
+
+      # codesee-detect-languages has an output with id languages.
+      - name: Detect Languages
+        id: detect-languages
+        uses: Codesee-io/codesee-detect-languages-action@latest
+
+      - name: Configure JDK 16
+        uses: actions/setup-java@v2
+        if: ${{ fromJSON(steps.detect-languages.outputs.languages).java }}
+        with:
+          java-version: '16'
+          distribution: 'zulu'
+
+      # CodeSee Maps Go support uses a static binary so there's no setup step required.
+
+      - name: Configure Node.js 14
+        uses: actions/setup-node@v2
+        if: ${{ fromJSON(steps.detect-languages.outputs.languages).javascript }}
+        with:
+          node-version: '14'
+
+      - name: Configure Python 3.x
+        uses: actions/setup-python@v2
+        if: ${{ fromJSON(steps.detect-languages.outputs.languages).python }}
+        with:
+          python-version: '3.10'
+          architecture: 'x64'
+
+      - name: Configure Ruby '3.x'
+        uses: ruby/setup-ruby@v1
+        if: ${{ fromJSON(steps.detect-languages.outputs.languages).ruby }}
+        with:
+          ruby-version: '3.0'
+
+      # We need the rust toolchain because it uses rustc and cargo to inspect the package
+      - name: Configure Rust 1.x stable
+        uses: actions-rs/toolchain@v1
+        if: ${{ fromJSON(steps.detect-languages.outputs.languages).rust }}
+        with:
+          toolchain: stable
+
+      - name: Generate Map
+        id: generate-map
+        uses: Codesee-io/codesee-map-action@latest
+        with:
+          step: map
+          api_token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }}
+          github_ref: ${{ github.ref }}
+          languages: ${{ steps.detect-languages.outputs.languages }}
+
+      - name: Upload Map
+        id: upload-map
+        uses: Codesee-io/codesee-map-action@latest
+        with:
+          step: mapUpload
+          api_token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }}
+          github_ref: ${{ github.ref }}
+
+      - name: Insights
+        id: insights
+        uses: Codesee-io/codesee-map-action@latest
+        with:
+          step: insights
+          api_token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }}
+          github_ref: ${{ github.ref }}

.gitignore

@@ -2,7 +2,13 @@
 /dist
 .env*
 !.env.example
+# nix
 /result*
+/builds
+# node-gyp
+/build
+# prebuildify
+/prebuilds
 
 # Logs
 logs

.gitlab-ci.yml

@@ -1,28 +1,245 @@
+workflow:
+  rules:
+    # Disable merge request pipelines
+    - if: $CI_MERGE_REQUEST_ID
+      when: never
+    - when: always
+
 default:
   interruptible: true
 
 variables:
+  GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}"
+  GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git"
   GIT_SUBMODULE_STRATEGY: recursive
+  # Cache .npm
+  NPM_CONFIG_CACHE: "${CI_PROJECT_DIR}/tmp/npm"
+  # Prefer offline node module installation
+  NPM_CONFIG_PREFER_OFFLINE: "true"
+  # `ts-node` has its own cache
+  TS_CACHED_TRANSPILE_CACHE: "${CI_PROJECT_DIR}/tmp/ts-node-cache"
+  TS_CACHED_TRANSPILE_PORTABLE: "true"
+  # Homebrew cache only used by macos runner
+  HOMEBREW_CACHE: "${CI_PROJECT_DIR}/tmp/Homebrew"
+
+# Cached directories shared between jobs & pipelines per-branch per-runner
+cache:
+  key: $CI_COMMIT_REF_SLUG
+  paths:
+    - ./tmp/npm/
+    - ./tmp/ts-node-cache/
+    # Homebrew cache is only used by the macos runner
+    - ./tmp/Homebrew
+    # `jest` cache is configured in jest.config.js
+    - ./tmp/jest/
 
 stages:
-  - check
+  - check       # Linting, unit tests
+  - build       # Cross-platform library compilation, unit tests
+  - integration # Cross-platform application bundling, integration tests, and pre-release
+  - release     # Cross-platform distribution and deployment
 
 image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner
 
-lint:
+check:lint:
   stage: check
-  interruptible: true
+  needs: []
   script:
     - >
         nix-shell --run '
         npm run lint;
         '
+  rules:
+    # Runs on feature and staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Runs on tag pipeline where the tag is a prerelease or release version
+    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Manually run on commits other than master and ignore version commits
+    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+      when: manual
 
-test:
+check:nix-dry:
   stage: check
-  interruptible: true
+  needs: []
+  script:
+    - nix-build -v -v --dry-run ./release.nix
+  rules:
+    # Runs on feature and staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Runs on tag pipeline where the tag is a prerelease or release version
+    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Manually run on commits other than master and ignore version commits
+    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+      when: manual
+
+check:test:
+  stage: check
+  needs: []
+  script:
+    - >
+        nix-shell --run '
+        npm run build --verbose;
+        npm test -- --ci;
+        '
+  artifacts:
+    when: always
+    reports:
+      junit:
+        - ./tmp/junit.xml
+  rules:
+    # Runs on staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Manually run on commits other than master and staging and ignore version commits
+    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+      when: manual
+
+build:merge:
+  stage: build
+  needs: []
+  allow_failure: true
+  script:
+    # Required for `gh pr create`
+    - git remote add upstream "$GH_PROJECT_URL"
+    - >
+        nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run '
+        gh pr create \
+          --head staging \
+          --base master \
+          --title "ci: merge staging to master" \
+          --body "This is an automatic PR generated by the pipeline CI/CD. This will be automatically fast-forward merged if successful." \
+          --assignee "@me" \
+          --no-maintainer-edit \
+          --repo "$GH_PROJECT_PATH" || true;
+        printf "Pipeline Attempt on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \
+        | gh pr comment staging \
+          --body-file - \
+          --repo "$GH_PROJECT_PATH";
+        '
+  rules:
+    # Runs on staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Runs on tag pipeline where the tag is a prerelease or release version
+    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+
+build:linux:
+  stage: build
+  needs:
+    - job: check:lint
+      optional: true
   script:
     - >
         nix-shell --run '
-        npm run test;
+        npm run build --verbose;
+        npm test -- --ci;
+        '
+  artifacts:
+    when: always
+    reports:
+      junit:
+        - ./tmp/junit.xml
+    paths:
+      - ./prebuilds/
+      # Only the build:linux preserves the dist
+      - ./dist
+  rules:
+    # Runs on staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Runs on tag pipeline where the tag is a prerelease or release version
+    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+
+build:windows:
+  stage: build
+  needs:
+    - job: check:lint
+      optional: true
+  tags:
+    - windows
+  before_script:
+    - choco install nodejs --version=16.14.2 -y
+    - choco install python --version=3.9.12 -y
+    - refreshenv
+  script:
+    - npm config set msvs_version 2019
+    - npm install --ignore-scripts
+    - $env:Path = "$(npm bin);" + $env:Path
+    - npm run build --verbose
+    - npm test -- --ci
+  artifacts:
+    when: always
+    reports:
+      junit:
+        - ./tmp/junit.xml
+    paths:
+      - ./prebuilds/
+  rules:
+    # Runs on staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Runs on tag pipeline where the tag is a prerelease or release version
+    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+
+build:macos:
+  stage: build
+  needs:
+    - job: check:lint
+      optional: true
+  tags:
+    - shared-macos-amd64
+  image: macos-11-xcode-12
+  variables:
+    HOMEBREW_NO_INSTALL_UPGRADE: "true"
+    HOMEBREW_NO_INSTALL_CLEANUP: "true"
+  before_script:
+    - brew install node@16
+    - brew link --overwrite node@16
+    - brew install [email protected]
+    - brew link --overwrite [email protected]
+    - hash -r
+  script:
+    - npm install --ignore-scripts
+    - export PATH="$(npm bin):$PATH"
+    - export PREBUILD_ARCH=x64+arm64
+    - npm run build --verbose
+    - npm test -- --ci
+  artifacts:
+    when: always
+    reports:
+      junit:
+        - ./tmp/junit.xml
+    paths:
+      - ./prebuilds/
+  rules:
+    # Runs on staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Runs on tag pipeline where the tag is a prerelease or release version
+    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+
+integration:merge:
+  stage: integration
+  needs:
+    - build:merge
+  # Requires mutual exclusion
+  resource_group: integration:merge
+  allow_failure: true
+  variables:
+    # Ensure that CI/CD is fetching all commits
+    # this is necessary to checkout origin/master
+    # and to also merge origin/staging
+    GIT_DEPTH: 0
+  script:
+    - >
+        nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run '
+        printf "Pipeline Succeeded on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \
+        | gh pr comment staging \
+          --body-file - \
+          --repo "$GH_PROJECT_PATH";
         '
+    - git remote add upstream "$GH_PROJECT_URL"
+    - git checkout origin/master
+    # Merge up to the current commit (not the latest commit)
+    - git merge --ff-only "$CI_COMMIT_SHA"
+    - git push upstream HEAD:master
+  rules:
+    # Runs on staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Runs on tag pipeline where the tag is a prerelease or release version
+    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/

.npmignore

@@ -1,15 +1,15 @@
 .*
+/*.nix
 /nix
-/pkgs.nix
-/default.nix
-/shell.nix
-/release.nix
 /tsconfig.json
 /tsconfig.build.json
 /jest.config.js
+/scripts
 /src
 /tests
 /tmp
 /docs
 /benches
+/build
+/builds
 /dist/tsbuildinfo

README.md

@@ -35,7 +35,7 @@ npm run lintfix
 npm run bench
 ```
 
-View benchmarks here: https://github.com/MatrixAI/js-db/blob/master/benches/results/WorkerManager.chart.html with https://raw.githack.com/
+View benchmarks here: https://github.com/MatrixAI/js-db/blob/master/benches/results with https://raw.githack.com/
 
 ### Docs Generation