Update connector packaging to support Inline Binary definitions (#182)

# Update connector packaging to support multi-platform CLI binaries

This PR updates our GitHub Actions workflow to support building and
packaging CLI binaries for multiple platforms while following the latest
connector packaging specification. The changes introduce a more robust
approach to creating platform-specific CLI binaries and generating the
corresponding metadata for the Hasura CLI plugin system.

## Key Changes

### Multi-Platform Binary Support
The build workflow now creates platform-specific binaries for:
- Linux (x86_64 and ARM64)
- macOS (x86_64 and ARM64) 
- Windows (x86_64)

### Enhanced Build and Packaging Process
Each binary is properly named and packaged following Hasura's
conventions, with SHA256 checksums generated for verification. The
binaries are combined into a single tarball (cli.tar.gz) that gets
attached to GitHub releases.

### Updated CLI Initialization
The CLI initialization command has been enhanced to support the new
binary CLI manifest format. This manifest contains platform-specific
information including download URLs, SHA256 checksums, and binary paths.
When initializing a new connector configuration, the CLI can now embed
this manifest into the connector metadata, enabling the Hasura CLI to
automatically download and use the correct binary for the user's
platform.

### Modernized Metadata Generation
The connector metadata generation has been updated to align with the
latest specification from the packaging RFC. Notable changes include:
- Added support for version field in metadata
- Enhanced environment variable definitions to include required flag
- Updated command definitions to support more flexible command types
(string, dockerized, shell script)
- Implemented the new BinaryInline CLI plugin definition format
- Added support for native toolchain definitions and documentation links

## Impact
These changes make our connector more portable across different
platforms while maintaining security through checksum verification. The
updated metadata format provides better integration with the Hasura
ecosystem and improved user experience through platform-specific binary
selection.

## Testing
- Verified binary builds for all supported platforms
- Tested metadata generation with the new CLI manifest
- Confirmed tarball creation and manifest combination
- Validated GitHub release artifacts

## Related
- RFC:
https://github.com/hasura/ndc-hub/blob/main/rfcs/0011-cli-and-connector-packaging.md

.github/workflows/ship.yaml

@@ -122,16 +122,23 @@ jobs:
         include:
           - runner: ubuntu-20.04
             target: x86_64-unknown-linux-gnu
+            platform: linux-amd64
           - runner: ubuntu-20.04
             target: aarch64-unknown-linux-gnu
+            platform: linux-arm64
             linux-packages: gcc-aarch64-linux-gnu
             linker: /usr/bin/aarch64-linux-gnu-gcc
           - runner: macos-latest
             target: x86_64-apple-darwin
+            platform: darwin-amd64
+            os: macOS
           - runner: macos-latest
             target: aarch64-apple-darwin
+            platform: darwin-arm64
+            os: macOS
           - runner: windows-latest
             target: x86_64-pc-windows-msvc
+            platform: windows-amd64
             extension: .exe
             extra-rust-flags: "-C target-feature=+crt-static"
     runs-on: ${{ matrix.runner }}
@@ -200,20 +207,100 @@ jobs:
           echo "Building for target: ${CARGO_BUILD_TARGET}"
           cargo build --release --bin ndc-sqlserver-cli
 
-          mkdir -p release
-          mv -v target/${{ matrix.target }}/release/ndc-sqlserver-cli release/ndc-sqlserver-cli-${{ matrix.target }}${{ matrix.extension }}
+          # Create platform-specific directory under cli/
+          mkdir -p cli/${{ matrix.platform }}
 
-      - uses: actions/upload-artifact@v4
+          # Move the binary with the correct name
+          if [[ "${{ matrix.platform }}" == "windows-amd64" ]]; then
+            mv -v target/${{ matrix.target }}/release/ndc-sqlserver-cli${{ matrix.extension }} cli/${{ matrix.platform }}/hasura-ndc-sqlserver.exe
+          else
+            mv -v target/${{ matrix.target }}/release/ndc-sqlserver-cli cli/${{ matrix.platform }}/hasura-ndc-sqlserver
+          fi
+
+      - name: Generate manifest entry
+        if: startsWith(github.ref, 'refs/tags/v')
+        shell: bash
+        run: |
+
+          # Calculate SHA256 of the binary
+          if [[ "${{ matrix.platform }}" == "windows-amd64" ]]; then
+            SHA256=$(certutil -hashfile cli/${{ matrix.platform }}/hasura-ndc-sqlserver.exe SHA256 | grep -v "hash" | awk '{print $1}')
+          elif [[ "${{ matrix.os }}" == "macOS" ]]; then
+            SHA256=$(shasum -a 256 cli/${{ matrix.platform }}/hasura-ndc-sqlserver | cut -d' ' -f1)
+          else
+            SHA256=$(sha256sum cli/${{ matrix.platform }}/hasura-ndc-sqlserver | cut -d' ' -f1)
+          fi
+
+          # Extract tag from github.ref by removing 'refs/tags/' prefix
+          TAG=${GITHUB_REF#refs/tags/}
+
+          cat << EOF > manifest-entry.yaml
+          - selector: ${{ matrix.platform }}
+            uri: "https://github.com/${{ github.repository }}/releases/download/${TAG}/cli.tar.gz"
+            sha256: "${SHA256}"
+            bin: "cli-binary-${{matrix.platform}}/hasura-ndc-sqlserver${{ matrix.extension }}"
+          EOF
+
+      - name: Upload manifest entry
+        uses: actions/upload-artifact@v4
+        with:
+          name: manifest-${{ matrix.platform }}
+          path: manifest-entry.yaml
+          retention-days: 1
+
+      - name: Upload binary
+        uses: actions/upload-artifact@v4
+        with:
+          name: cli-binary-${{ matrix.platform }}
+          path: cli/${{ matrix.platform }}
+          retention-days: 1
+
+  create-cli-package:
+    needs: build-cli-binaries
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download all binaries
+        uses: actions/download-artifact@v4
+        with:
+          pattern: cli-binary-*
+          path: cli
+
+      - name: Create tarball
+        run: |
+          tar -czf cli.tar.gz -C cli .
+          echo "Created cli.tar.gz containing:"
+          tar -tvf cli.tar.gz
+
+      - name: Download manifest entries
+        uses: actions/download-artifact@v4
+        with:
+          path: entries
+
+      - name: Combine manifest entries
+        run: |
+          # Combine all yaml entries into a single file
+          find entries -name "manifest-entry.yaml" -exec cat {} \; > cli-manifest.yaml
+
+          echo "Generated CLI Plugin Manifest:"
+          cat cli-manifest.yaml
+
+      - name: Upload CLI package and manifest
+        uses: actions/upload-artifact@v4
         with:
-          name: ndc-sqlserver-cli-${{ matrix.target }}
-          path: release
-          if-no-files-found: error
+          name: release-artifacts
+          path: |
+            cli.tar.gz
+            cli-manifest.yaml
+          retention-days: 1
+
 
   release:
     name: release to GitHub
     needs:
       - push-docker-images # not strictly necessary, but if this fails, we should abort
       - build-cli-binaries
+      - create-cli-package
     runs-on: ubuntu-latest
     # We release when a tag is pushed.
     if: startsWith(github.ref, 'refs/tags/v')
@@ -222,23 +309,23 @@ jobs:
 
       - uses: actions/download-artifact@v4
         with:
+          name: release-artifacts
           path: release/artifacts
-          merge-multiple: true
-
-      - name: generate SHA-256 checksums
-        run: |
-          cd release/artifacts
-          sha256sum * > ./sha256sum
 
       - name: generate a changelog
         run: |
           ./scripts/release-notes.py "${GITHUB_REF_NAME}" >> release/notes.md
 
       - name: generate a connector package
         run: |
-          chmod +x ./release/artifacts/ndc-sqlserver-cli-*
-          ./release/artifacts/ndc-sqlserver-cli-x86_64-unknown-linux-gnu --context=release/package initialize --with-metadata
-          tar vczf release/artifacts/package.tar.gz -C release/package .
+          tar xvf release/artifacts/cli.tar.gz -C release/artifacts
+          tree release/artifacts
+          chmod +x ./release/artifacts/cli-binary-linux-amd64/hasura-ndc-sqlserver
+          mkdir -p metadata-configuration
+          ./release/artifacts/cli-binary-linux-amd64/hasura-ndc-sqlserver --context=metadata-configuration initialize --with-metadata --binary-cli-manifest=release/artifacts/cli-manifest.yaml
+          cat metadata-configuration/.hasura-connector/connector-metadata.yaml
+          ls metadata-configuration
+          tar -vczf release/artifacts/package.tar.gz -C metadata-configuration .
 
       - name: create a draft release
         uses: ncipollo/release-action@v1

crates/cli/src/lib.rs

@@ -36,6 +36,9 @@ pub enum Command {
         #[arg(long)]
         /// Whether to create the hasura connector metadata.
         with_metadata: bool,
+        #[arg(long)]
+        /// The path to the binary CLI manifest.
+        binary_cli_manifest: PathBuf,
     },
     /// Update the configuration by introspecting the database, using the configuration options.
     Update {
@@ -54,7 +57,10 @@ pub enum Error {
 /// Run a command in a given directory.
 pub async fn run(command: Command, context: Context<impl Environment>) -> anyhow::Result<()> {
     match command {
-        Command::Initialize { with_metadata } => initialize(with_metadata, context).await?,
+        Command::Initialize {
+            with_metadata,
+            binary_cli_manifest,
+        } => initialize(with_metadata, context, binary_cli_manifest).await?,
         Command::Update { subcommand } => update(context, subcommand).await?,
     };
     Ok(())
@@ -68,11 +74,15 @@ pub async fn run(command: Command, context: Context<impl Environment>) -> anyhow
 ///
 /// Optionally, this can also create the connector metadata, which is used by the Hasura CLI to
 /// automatically work with this CLI as a plugin.
-async fn initialize(with_metadata: bool, context: Context<impl Environment>) -> anyhow::Result<()> {
+async fn initialize(
+    with_metadata: bool,
+    context: Context<impl Environment>,
+    binary_cli_manifest: PathBuf,
+) -> anyhow::Result<()> {
     let configuration_file = context
         .context_path
         .join(configuration::CONFIGURATION_FILENAME);
-    fs::create_dir_all(&context.context_path)?; // TODO(PY): .await
+    fs::create_dir_all(&context.context_path)?;
 
     // refuse to initialize the directory unless it is empty
     let mut items_in_dir = fs::read_dir(&context.context_path)?;
@@ -97,12 +107,18 @@ async fn initialize(with_metadata: bool, context: Context<impl Environment>) ->
         serde_json::to_string_pretty(&output)? + "\n",
     )?;
 
+    // Read and parse the binary CLI manifest directly into BinaryCliPluginPlatform
+    let manifest_contents = fs::read_to_string(&binary_cli_manifest)?;
+    let platforms: Vec<metadata::BinaryCliPluginPlatform> =
+        serde_yaml::from_str(&manifest_contents)?;
+
     // if requested, create the metadata
     if with_metadata {
         let metadata_dir = context.context_path.join(".hasura-connector");
         let _ = fs::create_dir(&metadata_dir);
         let metadata_file = metadata_dir.join("connector-metadata.yaml");
         let metadata = metadata::ConnectorMetadataDefinition {
+            version: Some("v1".to_string()),
             packaging_definition: metadata::PackagingDefinition::PrebuiltDockerImage(
                 metadata::PrebuiltDockerImagePackaging {
                     docker_image: format!(
@@ -115,21 +131,25 @@ async fn initialize(with_metadata: bool, context: Context<impl Environment>) ->
                 name: "CONNECTION_URI".to_string(),
                 description: "The SQL server connection URI".to_string(),
                 default_value: None,
+                required: true,
             }],
             commands: metadata::Commands {
-                update: Some("hasura-ndc-sqlserver update".to_string()),
+                update: Some(metadata::Command::String(
+                    "hasura-ndc-sqlserver update".to_string(),
+                )),
                 watch: None,
+                print_schema_and_capabilities: None,
+                upgrade_configuration: None,
             },
-            cli_plugin: Some(metadata::CliPluginDefinition {
-                name: "ndc-sqlserver".to_string(),
-                version: context.release_version.unwrap_or("latest").to_string(),
-            }),
+            cli_plugin: Some(metadata::CliPluginDefinition::BinaryInline { platforms }),
             docker_compose_watch: vec![metadata::DockerComposeWatchItem {
                 path: "./".to_string(),
                 target: Some("/etc/connector".to_string()),
                 action: metadata::DockerComposeWatchAction::SyncAndRestart,
                 ignore: vec![],
             }],
+            native_toolchain_definition: None,
+            documentation_page: None,
         };
 
         fs::write(metadata_file, serde_yaml::to_string(&metadata)?)?;