feat: signable binary (#4479)

* adds binary build that is independent of caxa package and is signable

* add license

* rebuild windows binary to test

* testing macos m1 runner

* fix bug in windows node command

* try to run in bash

* finalized version of workflow, switches back to being run on publish only

.github/workflows/publish-binaries.yml

@@ -1,5 +1,7 @@
 name: Build @ironfish binaries
 
+# on:
+#   push
 on:
   release:
     types:
@@ -24,7 +26,7 @@ jobs:
             arch: x86_64
             system: linux
 
-          - host: [self-hosted, macOS, ARM64]
+          - host: macos-latest-large
             arch: arm64
             system: apple
 
@@ -50,36 +52,72 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: 18
+
+      - name: Use Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.20.6'
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-      - name: npm init
-        run: npm init -y
-
-      - name: install dependencies
-        run: npm install ironfish [email protected]
-
-      - name: caxa package
-        id: caxa
+      - name: Create random identifier so binary extraction will be unique
+        id: identifier
+        shell: bash
         run: |
-          npx caxa --uncompression-message "Running the CLI for the first time may take a while, please wait..." --input . --output "${{ matrix.settings.system != 'windows' && 'ironfish' || 'ironfish.exe' }}" -- "{{caxa}}/node_modules/.bin/node" "--enable-source-maps" "{{caxa}}/node_modules/ironfish/bin/run"
-          echo "RELEASE_NAME=ironfish-${{ matrix.settings.system }}-${{ matrix.settings.arch }}-${{ github.event.release.tag_name }}.zip"
+          identifier=$(awk 'BEGIN {
+            srand(); 
+            chars = "abcdefghijklmnopqrstuvwxyz0123456789";
+            for (i = 1; i <= 10; i++) {
+                printf "%s", substr(chars, int(rand() * length(chars)) + 1, 1);
+            }
+            print "";
+          }')
+          echo "identifier=${identifier}" >> $GITHUB_OUTPUT
+
+      - name: Create build.tar.gz for binary
+        id: build
+        run: |
+          mkdir build
+          cd build
+          cp $(node -e "console.log(process.execPath)") ${{ matrix.settings.system != 'windows' && 'node' || 'node.exe' }}
+          npm init -y
+          npm install ironfish
+          tar -czf ../tools/build.tar.gz -C . .
+
+      - name: Create binary
+        id: binary
+        run: |
+          go build -ldflags "-X 'main.Identifier=${{ steps.identifier.outputs.identifier }}' -X 'main.Command={{caxac}}/${{ matrix.settings.system != 'windows' && 'node' || 'node.exe' }} --enable-source-maps {{caxac}}/node_modules/ironfish/bin/run' -X 'main.UncompressionMessage=Unpackaging ironfish application, this may take a minute when run for the first time.'" -o tools/${{ matrix.settings.system != 'windows' && 'ironfish' || 'ironfish.exe' }} tools/build-binary.go 
 
-      - name: set paths
+      - name: Set paths
         id: set_paths
         shell: bash
         run: |
-          echo "zip=ironfish-${{ matrix.settings.system }}-${{ matrix.settings.arch }}-${{ github.event.release.tag_name }}.zip" >> $GITHUB_OUTPUT
+          name="ironfish-${{ matrix.settings.system }}-${{ matrix.settings.arch }}-${{ github.event.release.tag_name }}"
+          echo "name=${name}" >> $GITHUB_OUTPUT
+          echo "zip=${name}.zip" >> $GITHUB_OUTPUT
           echo "binary=${{ matrix.settings.system != 'windows' && 'ironfish' || 'ironfish.exe' }}" >> $GITHUB_OUTPUT
 
       - name: chmod binary
+        working-directory: tools
         if: matrix.settings.system != 'windows'
         run: chmod +x ${{ steps.set_paths.outputs.binary }}
 
       - name: Zip binary
         uses: thedoctor0/[email protected]
         with:
+          directory: tools
           type: 'zip'
-          filename: ${{ steps.set_paths.outputs.zip }}
+          filename: ${{ steps.set_paths.outputs.name }}
           path: ${{ steps.set_paths.outputs.binary }}
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ steps.set_paths.outputs.name }}
+          path: tools/${{ steps.set_paths.outputs.zip }}
+          if-no-files-found: error
 
       - name: Upload Release Asset
         id: upload-release-asset

tools/build-binary.go

@@ -0,0 +1,263 @@
+// MIT License
+
+// Copyright (c) 2023 Leandro Facchinetti <[email protected]> (https://leafac.com)
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// Disclaimer:
+// This code was largely adapted for an archived repo https://github.com/leafac/caxa
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"embed"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// When building this file, a build.tar.gz should be present and will be embedded in the binary
+
+//go:embed build.tar.gz
+var data embed.FS
+
+var (
+	Identifier           string
+	Command              string
+	UncompressionMessage string
+)
+
+func main() {
+
+	var applicationDirectory string
+	for extractionAttempt := 0; true; extractionAttempt++ {
+		lock := path.Join(os.TempDir(), "caxac/locks", Identifier, strconv.Itoa(extractionAttempt))
+		applicationDirectory = path.Join(os.TempDir(), "caxac/applications", Identifier, strconv.Itoa(extractionAttempt))
+		applicationDirectoryFileInfo, err := os.Stat(applicationDirectory)
+		if err != nil && !errors.Is(err, os.ErrNotExist) {
+			log.Fatalf("caxac stub: Failed to find information about the application directory: %v", err)
+		}
+		if err == nil && !applicationDirectoryFileInfo.IsDir() {
+			log.Fatalf("caxac stub: Path to application directory already exists and isn’t a directory: %v", err)
+		}
+		if err == nil && applicationDirectoryFileInfo.IsDir() {
+			lockFileInfo, err := os.Stat(lock)
+			if err != nil && !errors.Is(err, os.ErrNotExist) {
+				log.Fatalf("caxac stub: Failed to find information about the lock: %v", err)
+			}
+			if err == nil && !lockFileInfo.IsDir() {
+				log.Fatalf("caxac stub: Path to lock already exists and isn’t a directory: %v", err)
+			}
+			if err == nil && lockFileInfo.IsDir() {
+				// Application directory exists and lock exists as well, so a previous extraction wasn’t successful or an extraction is happening right now and hasn’t finished yet, in either case, start over with a fresh name.
+				continue
+			}
+			if err != nil && errors.Is(err, os.ErrNotExist) {
+				// Application directory exists and lock doesn’t exist, so a previous extraction was successful. Use the cached version of the application directory and don’t extract again.
+				break
+			}
+		}
+		if err != nil && errors.Is(err, os.ErrNotExist) {
+			ctx, cancelCtx := context.WithCancel(context.Background())
+			if UncompressionMessage != "" {
+				fmt.Fprint(os.Stderr, UncompressionMessage)
+				go func() {
+					ticker := time.NewTicker(time.Second * 5)
+					defer ticker.Stop()
+					for {
+						select {
+						case <-ticker.C:
+							fmt.Fprint(os.Stderr, ".")
+						case <-ctx.Done():
+							fmt.Fprintln(os.Stderr, "")
+							return
+						}
+					}
+				}()
+			}
+
+			if err := os.MkdirAll(lock, 0755); err != nil {
+				log.Fatalf("caxac stub: Failed to create the lock directory: %v", err)
+			}
+
+			embeddedDataReader, err := data.Open("build.tar.gz")
+			if err != nil {
+				log.Fatalf("Failed to open embedded data: %v", err)
+			}
+			defer embeddedDataReader.Close()
+
+			if err := Untar(embeddedDataReader, applicationDirectory); err != nil {
+				log.Fatalf("caxac stub: Failed to uncompress embedded data: %v", err)
+			}
+
+			os.Remove(lock)
+
+			cancelCtx()
+			break
+		}
+	}
+	splitCommand := strings.Split(Command, " ")
+	expandedCommand := make([]string, len(splitCommand))
+	applicationDirectoryPlaceholderRegexp := regexp.MustCompile(`\{\{\s*caxac\s*\}\}`)
+	for key, commandPart := range splitCommand {
+		expandedCommand[key] = applicationDirectoryPlaceholderRegexp.ReplaceAllLiteralString(commandPart, applicationDirectory)
+	}
+
+	command := exec.Command(expandedCommand[0], append(expandedCommand[1:], os.Args[1:]...)...)
+	command.Stdin = os.Stdin
+	command.Stdout = os.Stdout
+	command.Stderr = os.Stderr
+	err := command.Run()
+	var exitError *exec.ExitError
+	if errors.As(err, &exitError) {
+		os.Exit(exitError.ExitCode())
+	} else if err != nil {
+		log.Fatalf("caxac stub: Failed to run command: %v", err)
+	}
+}
+
+//
+// Adapted from https://github.com/leafac/caxa and https://github.com/golang/build/blob/db2c93053bcd6b944723c262828c90af91b0477a/internal/untar/untar.go and https://github.com/mholt/archiver/tree/v3.5.0
+
+// Untar reads the gzip-compressed tar file from r and writes it into dir.
+func Untar(r io.Reader, dir string) error {
+	return untar(r, dir)
+}
+
+func untar(r io.Reader, dir string) (err error) {
+	t0 := time.Now()
+	nFiles := 0
+	madeDir := map[string]bool{}
+	zr, err := gzip.NewReader(r)
+	if err != nil {
+		return fmt.Errorf("requires gzip-compressed body: %v", err)
+	}
+	tr := tar.NewReader(zr)
+	loggedChtimesError := false
+	for {
+		f, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("tar error: %v", err)
+		}
+		if !validRelPath(f.Name) {
+			return fmt.Errorf("tar contained invalid name error %q", f.Name)
+		}
+		rel := filepath.FromSlash(f.Name)
+		abs := filepath.Join(dir, rel)
+
+		fi := f.FileInfo()
+		mode := fi.Mode()
+		switch {
+		case mode.IsRegular():
+			// Make the directory. This is redundant because it should
+			// already be made by a directory entry in the tar
+			// beforehand. Thus, don't check for errors; the next
+			// write will fail with the same error.
+			dir := filepath.Dir(abs)
+			if !madeDir[dir] {
+				if err := os.MkdirAll(filepath.Dir(abs), 0755); err != nil {
+					return err
+				}
+				madeDir[dir] = true
+			}
+			wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm())
+			if err != nil {
+				return err
+			}
+			n, err := io.Copy(wf, tr)
+			if closeErr := wf.Close(); closeErr != nil && err == nil {
+				err = closeErr
+			}
+			if err != nil {
+				return fmt.Errorf("error writing to %s: %v", abs, err)
+			}
+			if n != f.Size {
+				return fmt.Errorf("only wrote %d bytes to %s; expected %d", n, abs, f.Size)
+			}
+			modTime := f.ModTime
+			if modTime.After(t0) {
+				// Clamp modtimes at system time. See
+				// golang.org/issue/19062 when clock on
+				// buildlet was behind the gitmirror server
+				// doing the git-archive.
+				modTime = t0
+			}
+			if !modTime.IsZero() {
+				if err := os.Chtimes(abs, modTime, modTime); err != nil && !loggedChtimesError {
+					// benign error. Gerrit doesn't even set the
+					// modtime in these, and we don't end up relying
+					// on it anywhere (the gomote push command relies
+					// on digests only), so this is a little pointless
+					// for now.
+					// log.Printf("error changing modtime: %v (further Chtimes errors suppressed)", err)
+					loggedChtimesError = true // once is enough
+				}
+			}
+			nFiles++
+		case mode.IsDir():
+			if err := os.MkdirAll(abs, 0755); err != nil {
+				return err
+			}
+			madeDir[abs] = true
+		case f.Typeflag == tar.TypeSymlink:
+			// leafac: Added by me to support symbolic links. Adapted from https://github.com/mholt/archiver/blob/v3.5.0/tar.go#L254-L276 and https://github.com/mholt/archiver/blob/v3.5.0/archiver.go#L313-L332
+			err := os.MkdirAll(filepath.Dir(abs), 0755)
+			if err != nil {
+				return fmt.Errorf("%s: making directory for file: %v", abs, err)
+			}
+			_, err = os.Lstat(abs)
+			if err == nil {
+				err = os.Remove(abs)
+				if err != nil {
+					return fmt.Errorf("%s: failed to unlink: %+v", abs, err)
+				}
+			}
+
+			err = os.Symlink(f.Linkname, abs)
+			if err != nil {
+				return fmt.Errorf("%s: making symbolic link for: %v", abs, err)
+			}
+		default:
+			return fmt.Errorf("tar file entry %s contained unsupported file type %v", f.Name, mode)
+		}
+	}
+	return nil
+}
+
+func validRelPath(p string) bool {
+	if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") {
+		return false
+	}
+	return true
+}