ci: CI Testing (#57)

- Setup CI for unit + e2e tests
- PR shows summary comments added for 

resolves #29

.eslintrc.json

@@ -15,5 +15,22 @@
     "no-throw-literal": "warn",
     "semi": "off"
   },
+  "overrides": [
+    {
+      "files": ["*.ts"],
+      "parserOptions": {
+        "project": [
+          "./tsconfig.json",
+          "./e2e/tsconfig.json",
+          "./tsconfig.unit.json"
+        ]
+      },
+
+      "rules": {
+        "no-return-await": "off",
+        "@typescript-eslint/return-await": "error"
+      }
+    }
+  ],
   "ignorePatterns": ["out", "dist", "**/*.d.ts"]
 }

.github/workflows/ci.yml

@@ -0,0 +1,18 @@
+# This action orchestrates CI actions that can be run in parallel. This action
+# can be listed by any actions that use `workflow_run` as a trigger that depend
+# on all orchestrated actions being completed (e.g. pr-comment.yml).
+name: 'Orchestrator'
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  call-unit:
+    uses: ./.github/workflows/unit.yml
+
+  call-e2e:
+    uses: ./.github/workflows/e2e.yml

.github/workflows/conventional-pr-check.yml

@@ -11,12 +11,14 @@ jobs:
   pr-check:
     runs-on: ubuntu-22.04
     steps:
-      - uses: amannn/action-semantic-pull-request@v5
+      # v5.5.3
+      - uses: amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017
         id: lint_pr_title
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: marocchino/sticky-pull-request-comment@v2
+      # v2.9.0
+      - uses: marocchino/sticky-pull-request-comment@331f8f5b4215f0445d3c07b4967662a32a2d3e31
         # When the previous steps fails, the workflow would stop. By adding this
         # condition you can continue the execution with the populated error message.
         if: always() && (steps.lint_pr_title.outputs.error_message != null)
@@ -33,7 +35,8 @@ jobs:
 
       # Delete a previous comment when the issue has been resolved
       - if: ${{ steps.lint_pr_title.outputs.error_message == null }}
-        uses: marocchino/sticky-pull-request-comment@v2
+        # v2.9.0
+        uses: marocchino/sticky-pull-request-comment@331f8f5b4215f0445d3c07b4967662a32a2d3e31
         with:
           header: pr-title-lint-error
           delete: true

.github/workflows/e2e.yml

@@ -0,0 +1,38 @@
+name: End-to-end Tests
+on:
+  workflow_call:
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-e2e
+      cancel-in-progress: true
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+      - run: npm ci
+      - name: Run end-to-end tests
+        run: xvfb-run npm run test:e2e
+      - name: Publish Test Summary Results
+        if: ${{ always() }}
+        run: |
+          npm run report:ctrfmerge
+          npm run report:ctrfsummary
+          sed -i 's/<h3>Test Summary<\/h3>/<h3>End-to-end Test Summary<\/h3>/' $GITHUB_STEP_SUMMARY
+          npm run report:prcomment
+      - name: Save PR Number
+        if: ${{ always() }}
+        run: echo ${{ github.event.number }} > pr-comment/PR-number.txt
+      - name: Upload Summary as Artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr-comment
+          path: pr-comment/
+          retention-days: 1

.github/workflows/unit.yml

@@ -0,0 +1,38 @@
+name: Unit Tests
+on:
+  workflow_call:
+
+jobs:
+  unit:
+    runs-on: ubuntu-latest
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-unit
+      cancel-in-progress: true
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+      - run: npm ci
+      - name: Run ts checks, linting, and unit tests
+        run: npm run test:ci
+      - name: Publish Test Summary Results
+        if: ${{ always() }}
+        run: |
+          npm run report:junit2ctrf
+          npm run report:ctrfsummary
+          sed -i 's/<h3>Test Summary<\/h3>/<h3>Unit Test Summary<\/h3>/' $GITHUB_STEP_SUMMARY
+          npm run report:prcomment
+      - name: Save PR Number
+        if: ${{ always() }}
+        run: echo ${{ github.event.number }} > pr-comment/PR-number.txt
+      - name: Upload Summary as Artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr-comment-unit
+          path: pr-comment/
+          retention-days: 1

.gitignore

@@ -3,4 +3,6 @@ dist
 node_modules
 .vscode-test/
 .DS_Store
-.wdio-vscode-service
+.wdio-vscode-service
+e2e/reports/
+test-reports/

.prettierrc

@@ -1,5 +1,13 @@
 {
   "singleQuote": true,
   "trailingComma": "es5",
-  "arrowParens": "avoid"
+  "arrowParens": "avoid",
+  "overrides": [
+    {
+      "files": ["*.jsonc", "*.code-snippets"],
+      "options": {
+        "trailingComma": "none"
+      }
+    }
+  ]
 }

.vscode/launch.json

@@ -12,17 +12,6 @@
       "args": ["--extensionDevelopmentPath=${workspaceFolder}"],
       "outFiles": ["${workspaceFolder}/out/**/*.js"],
       "preLaunchTask": "${defaultBuildTask}"
-    },
-    {
-      "name": "Extension Tests",
-      "type": "extensionHost",
-      "request": "launch",
-      "args": [
-        "--extensionDevelopmentPath=${workspaceFolder}",
-        "--extensionTestsPath=${workspaceFolder}/out/test/suite/index"
-      ],
-      "outFiles": ["${workspaceFolder}/out/test/**/*.js"],
-      "preLaunchTask": "${defaultBuildTask}"
     }
   ]
 }

.vscode/tasks.json

@@ -5,7 +5,7 @@
   "tasks": [
     {
       "type": "npm",
-      "script": "watch",
+      "script": "ts:watch",
       "problemMatcher": "$tsc-watch",
       "isBackground": true,
       "presentation": {

CONTRIBUTING.md

@@ -1,5 +1,49 @@
 # Contributing to Deephaven in VS Code
 
+## Development
+
+### Unit Testing
+
+Unit tests are configured to run via `vitest`. To run them:
+
+```sh
+npm run test
+```
+
+### End-to-end Testing
+
+End-to-end tests are configured to run via `wdio-vscode-service`. This allows
+testing workflows and has better abstractions for ui testing than `@vscode/test-electron`.
+
+See [wdio-vscode-service](https://www.npmjs.com/package/wdio-vscode-service) for more details.
+
+To run end-to-end tests:
+
+```sh
+npm run test:e2e
+```
+
+To run using `vscode` debugger:
+
+1. Set a breakpoint in a test
+2. Either
+
+   Enable auto attach with flag (`Command palette -> Debug: Toggle Auto Attach -> Only With Flag`).
+
+   > Note you may need to open a new terminal for this to take effect.
+
+   or
+
+   Open `Javascript Debug Terminal` (Click the dropdown arrow beside the + button for adding a new terminal)
+
+3. Run the script:
+
+   ```sh
+   npm run test:e2e
+   ```
+
+4. You should see the tests start and `vscode` stop at the breakpoint.
+
 ## Installation from .VSIX
 
 This extension can also be installed directly from a `.vsix`. To get a `.vsix`, you can either:

README.md

@@ -19,7 +19,7 @@ Core server `http://localhost:10000/` is configured by default and doesn't requi
 
 e.g. `.vscode/settings.json`
 
-```jsonc
+```json
 {
   // Core servers
   "vscode-deephaven.core-servers": [

e2e/specs/test.e2e.ts

@@ -0,0 +1,26 @@
+import { browser, expect } from '@wdio/globals';
+
+// There are some tests that can be used for reference in:
+// https://github.com/stateful/vscode-marquee/blob/main/test/specs
+
+describe('VS Code Extension Testing', () => {
+  it('should be able to load VSCode', async () => {
+    const workbench = await browser.getWorkbench();
+    expect(await workbench.getTitleBar().getTitle()).toContain(
+      '[Extension Development Host]'
+    );
+  });
+
+  it('should load connection status bar item', async () => {
+    const workbench = await browser.getWorkbench();
+
+    const statusBarItem = await browser.waitUntil(async () => {
+      return workbench.getStatusBar().getItem(
+        // icon name, display text, tooltip
+        'debug-disconnect  Deephaven: Disconnected, Connect to Deephaven'
+      );
+    });
+
+    expect(statusBarItem).toBeDefined();
+  });
+});

e2e/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": ".",
+    "outDir": "out",
+    "types": [
+      "node",
+      // "webdriverio/async",
+      "@wdio/mocha-framework",
+      "expect-webdriverio",
+      "wdio-vscode-service"
+    ],
+    "target": "ES2020",
+    "moduleResolution": "node",
+    "esModuleInterop": true
+  },
+  // Override ../tsconfig.json `exclude`
+  "exclude": ["node_modules"]
+}