.github, elixir: support publishing in ci

.github/workflows/elixir.yml

@@ -7,6 +7,14 @@ on:
       - v*
   pull_request:
   workflow_dispatch:
+    inputs:
+      hex_repo:
+        description: "hex repo to publish to"
+        default: staging
+        required: true
+        options:
+          - staging
+          - prod
 
 permissions:
   contents: read
@@ -16,6 +24,21 @@ env:
   # cached state
   cache_key: init
 
+  package_artifact: ts_elixir-${{ github.sha }}
+
+  is_tag_push: ${{ startsWith(github.ref, 'refs/tags/') }}
+  is_dispatch: ${{ github.event_name == 'workflow_dispatch' }}
+
+  hex_environment: &hex_environment ${{ case(inputs.hex_repo == 'prod', 'hex.pm', startsWith(github.ref, 'refs/tags/'), 'hex.pm', 'staging.hex.pm') }}
+
+  latest_elixir: &latest_elixir 1.19.5
+  latest_otp: &latest_otp 28.4.2
+  latest_rust: &latest_rust 1.94.0
+
+defaults:
+  run:
+    working-directory: ts_elixir
+
 jobs:
   build_test:
     name: compile, test, static checks (elixir ${{ matrix.elixir.label }}, otp ${{ matrix.otp.label }}, rust ${{ matrix.rust_toolchain.label }})
@@ -24,19 +47,15 @@ jobs:
     strategy:
       matrix:
         elixir:
-          - version: 1.19.5
+          - version: *latest_elixir
             label: latest
         otp:
-          - version: 28.4.2
+          - version: *latest_otp
             label: latest
         rust_toolchain:
-          - version: 1.94.0
+          - version: *latest_rust
             label: latest
 
-    defaults:
-      run:
-        working-directory: ts_elixir
-
     steps:
       - name: Checkout
         uses: actions/checkout@v6
@@ -116,3 +135,122 @@ jobs:
           name: exdoc-${{ github.sha }}
           path: ts_elixir/doc
           retention-days: 7
+
+      - &detach_cargo_toml
+        name: Detach Cargo.toml from workspace
+        working-directory: ts_elixir/native/ts_elixir
+        run: |
+          pip install tomli-w
+
+          python deworkspace_cargo_toml.py \
+            --root ../../../Cargo.toml \
+            --repo_sha ${{ github.sha }} \
+            < Cargo.toml > Cargo.toml.new
+
+          mv Cargo.toml.new Cargo.toml
+          echo generated Cargo.toml:
+          cat Cargo.toml
+
+      - name: Build Hex package (no publish)
+        run: mix hex.build --unpack -o tailscale
+        env:
+          MIX_ENV: prod
+
+      - name: Upload package tarball
+        uses: actions/upload-artifact@v7
+        with:
+          name: ${{ env.package_artifact }}
+          path: ts_elixir/tailscale
+
+  publish:
+    name: publish
+    runs-on: linux-x86_64-16cpu
+    needs: build_test
+    if: ${{ startsWith(github.ref, 'refs/tags/*') || github.event_name == 'workflow_dispatch' }}
+
+    environment: *hex_environment
+
+    # Artifact generation, signing, upload, respectively
+    permissions:
+      attestations: write
+      id-token: write
+      contents: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Cache mix
+        id: cache-mix
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.mix
+            ts_elixir/_build/
+            ts_elixir/deps/
+          key: ${{ runner.os }}-mix-${{ env.cache_key }}-elixir-${{ env.latest_elixir }}-otp-${{ env.latest_otp }}-${{ hashFiles('**/mix.lock') }}
+
+      - name: Cache Rust
+        id: cache-rust-elixir
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            ~/.rustup/toolchains
+            target/
+          key: ${{ runner.os }}-elixir-rs-${{ env.cache_key }}-${{ env.latest_rust }}-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Install elixir
+        id: install-elixir
+        uses: erlef/setup-beam@v1.24.0
+        with:
+          otp-version: ${{ env.latest_otp }}
+          elixir-version: ${{ env.latest_elixir }}
+
+      - name: Install Rust toolchain
+        id: install-rust-toolchain
+        if: steps.cache-rust-elixir.outputs.cache-hit != 'true'
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.latest_rust }}
+
+      - name: Install dependencies
+        run: mix deps.get
+
+      - name: Compile dependencies
+        run: mix deps.compile
+
+      - name: Download built package
+        uses: actions/download-artifact@v8
+        with:
+          name: ${{ env.package_artifact }}
+          path: ts_elixir/tailscale
+
+      - name: Generate attestation
+        uses: actions/attest@v4
+        with:
+          subject-path: ts_elixir/tailscale
+
+      - *detach_cargo_toml
+
+      - name: Publish deps to staging.hex.pm
+        if: ${{ env.hex_environment == 'staging.hex.pm' }}
+        run: python staging_publish_deps.py
+        env:
+          HEX_API_URL: ${{ vars.HEX_API_URL }}
+          HEX_API_KEY: ${{ secrets.HEX_API_KEY }}
+
+      - name: Publish (dry run)
+        run: mix hex.publish --dry-run --yes
+        env:
+          HEX_API_URL: ${{ vars.HEX_API_URL }}
+          HEX_API_KEY: ${{ secrets.HEX_API_KEY }}
+
+      - name: Publish
+        run: mix hex.publish --yes
+        env:
+          HEX_API_KEY: ${{ secrets.HEX_API_KEY }}
+          HEX_API_URL: ${{ vars.HEX_API_URL }}

ts_elixir/mix.exs

@@ -42,8 +42,9 @@ defmodule TsElixir.MixProject do
 
   defp package do
     [
-      files: ~w(lib .formatter.exs mix.exs README.md LICENSE),
-      licenses: ["MIT"],
+      files:
+        ~w(lib .formatter.exs mix.exs README.md LICENSE native/ts_elixir/src native/ts_elixir/Cargo* native/ts_elixir/README.md),
+      licenses: ["BSD-3-Clause"],
       links: %{
         "GitHub" => "https://github.com/tailscale/tailscale-rs"
       }

ts_elixir/native/ts_elixir/Cargo.toml

@@ -14,7 +14,6 @@ rust-version.workspace = true
 rustler = "0.37.2"
 
 tailscale = { workspace = true }
-ts_cli_util = { workspace = true }
 
 tokio = { workspace = true, features = ["full"] }
 tracing = { workspace = true }

ts_elixir/native/ts_elixir/deworkspace_cargo_toml.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+"""
+Generate a new Cargo.toml suitable for publishing to hex.pm by stripping unnecessary keys
+and references to the cargo workspace.
+
+Usage:
+
+    $ deworkspace_cargo_toml.py --root ../../../Cargo.toml < Cargo.toml
+
+"""
+
+import tomllib
+import tomli_w
+import argparse
+import sys
+
+ALLOWED_PACKAGE_KEYS = {'name', 'version', 'edition', 'license'}
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--root', required=True, help='workspace root Cargo.toml')
+    parser.add_argument('--repo_sha',
+                        help='replace repo dep versions with git dep at specified sha')
+
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+
+    with open(args.root, 'rb') as f:
+        roottoml = tomllib.load(f)
+
+    cargotoml = tomllib.load(sys.stdin.buffer)
+
+    if 'lints' in cargotoml:
+        del cargotoml['lints']
+
+    if 'dev-dependencies' in cargotoml:
+        del cargotoml['dev-dependencies']
+
+    cargotoml['package'] = {k: v for k, v in cargotoml['package'].items() if
+                            k in ALLOWED_PACKAGE_KEYS}
+
+    for k in ALLOWED_PACKAGE_KEYS:
+        value = cargotoml['package'][k]
+
+        if type(value) == dict and value['workspace'] is True:
+            cargotoml['package'][k] = roottoml['workspace']['package'][k]
+
+    wksp_deps = roottoml['workspace']['dependencies']
+
+    for name in ['dependencies', 'build-dependencies']:
+        if name not in cargotoml:
+            continue
+
+        for dep in list(cargotoml[name].keys()):
+            value = cargotoml[name][dep]
+
+            if type(value) == dict and value['workspace'] is True:
+                if args.repo_sha and (dep.startswith('tailscale') or dep.startswith('ts_')):
+                    value['git'] = f'https://github.com/tailscale/tailscale-rs'
+                    value['rev'] = args.repo_sha
+                else:
+                    value['version'] = wksp_deps[dep]['version']
+
+                del value['workspace']
+
+    tomli_w.dump(cargotoml, sys.stdout.buffer)
+
+
+if __name__ == '__main__':
+    main()

ts_elixir/native/ts_elixir/src/lib.rs

@@ -117,14 +117,6 @@ fn connect(env: rustler::Env, config_path: String, auth_key: Option<String>) ->
     erl_result(env, dev)
 }
 
-#[rustler::nif]
-fn start_tracing() -> impl Encoder {
-    static TRACING_ONCE: std::sync::Once = std::sync::Once::new();
-    TRACING_ONCE.call_once(ts_cli_util::init_tracing);
-
-    atoms::ok()
-}
-
 #[rustler::nif(schedule = "DirtyIo")]
 fn ipv4_addr(env: rustler::Env, dev: ResourceArc<Device>) -> impl Encoder {
     let dev = dev.inner.clone();

ts_elixir/staging_publish_deps.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+
+"""
+staging.hex.pm generally doesn't have our deps uploaded.
+Try to upload them from our local dep cache so that we can try a staging push.
+"""
+
+import shutil
+import sys
+import pathlib
+import os
+import shlex
+import subprocess
+
+if os.name != 'nt':
+    MIX_EXE = 'mix'
+else:
+    MIX_EXE = 'mix.bat'
+
+
+def try_publish(path: pathlib.Path):
+    def mix_call(args, ok_output=None):
+        ret = subprocess.run(shlex.split(f'{MIX_EXE} {args}'), cwd=path, stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+
+        sys.stderr.buffer.write(ret.stderr)
+        sys.stdout.buffer.write(ret.stdout)
+
+        if ok_output and (ok_output in ret.stderr or ok_output in ret.stdout):
+            return
+
+        ret.check_returncode()
+
+    print(f'try publish {path.name}')
+    mix_call('deps.get')
+    mix_call('deps.compile')
+    mix_call('hex.publish package --yes', ok_output=b'must include the --replace flag')
+    print(f'published {path.name}')
+
+    try:
+        shutil.rmtree(path.joinpath('_build'))
+    except FileNotFoundError:
+        pass
+
+    try:
+        shutil.rmtree(path.joinpath('deps'))
+    except FileNotFoundError:
+        pass
+
+
+def main():
+    os.environ['MIX_ENV'] = 'prod'
+
+    if os.environ.get('HEX_API_URL') is None:
+        print('setting HEX_API_URL to staging')
+        os.environ['HEX_API_URL'] = 'https://staging.hex.pm/api'
+
+    if not os.environ.get('HEX_API_KEY'):
+        print('warning: HEX_API_KEY unset')
+
+    this_path = pathlib.Path(__file__)
+    os.chdir(this_path.parent.joinpath('deps'))
+
+    paths = set()
+
+    for pat in pathlib.Path.cwd().glob('*'):
+        if not pat.is_dir():
+            continue
+
+        if not pat.joinpath('mix.exs').exists():
+            continue
+
+        paths.add(pat)
+        print(pat.name)
+
+    while True:
+        any_success = False
+
+        for path in list(paths):
+            try:
+                try_publish(path)
+                any_success = True
+                paths.remove(path)
+            except subprocess.CalledProcessError:
+                pass
+
+        if not any_success:
+            break
+
+
+if __name__ == '__main__':
+    main()