[2/3] Adjust Charts Process: Update CI scripts

.github/workflows/e2e/scripts/hpo-install-helm-project-operator.sh

@@ -5,9 +5,13 @@ source $(dirname $0)/entry
 source $(dirname $0)/cluster-args.sh
 
 cd $(dirname $0)/../../../..
+source "$(pwd)/scripts/util-team-charts"
 
-latest_chart=./packages/helm-project-operator/charts
+NEWEST_CHART_VERSION=$(newest-chart-version "helm-project-operator")
+fetch-team-chart "helm-project-operator" "$NEWEST_CHART_VERSION"
+LATEST_CHART_PATH="./build/charts/helm-project-operator-${NEWEST_CHART_VERSION}.tgz"
+tar -xvzf "$LATEST_CHART_PATH" -C ./build/charts/
 
-helm upgrade --install --create-namespace -n cattle-helm-system helm-project-operator --set image.registry='',image.repository=${REPO:-rancher}/helm-project-operator,image.tag=${TAG:-dev} ${cluster_args} ${RANCHER_HELM_ARGS} ${latest_chart}
+helm upgrade --install --create-namespace -n cattle-helm-system helm-project-operator --set image.registry='',image.repository=${REPO:-rancher}/helm-project-operator,image.tag=${TAG:-dev} ${cluster_args} ${RANCHER_HELM_ARGS} ./build/charts/helm-project-operator
 
 echo "PASS: Helm Project Operator has been installed"

.github/workflows/e2e/scripts/install-federator.sh

@@ -6,9 +6,13 @@ source $(dirname $0)/entry
 source $(dirname $0)/cluster-args.sh
 
 cd $(dirname $0)/../../../..
+source "$(pwd)/scripts/util-team-charts"
 
-latest_chart=$(find ./charts/prometheus-federator -type d -maxdepth 1 -mindepth 1 | tr - \~ | sort -rV | tr \~ - | head -n1)
+NEWEST_CHART_VERSION=$(newest-chart-version "prometheus-federator")
+fetch-team-chart "prometheus-federator" "$NEWEST_CHART_VERSION"
+LATEST_CHART_PATH="./build/charts/prometheus-federator-${NEWEST_CHART_VERSION}.tgz"
+tar -xvzf "$LATEST_CHART_PATH" -C ./build/charts/
 
-helm upgrade --install --create-namespace -n cattle-monitoring-system prometheus-federator --set helmProjectOperator.image.repository=${REPO:-rancher}/prometheus-federator --set helmProjectOperator.image.tag=${TAG:-dev} ${cluster_args} ${RANCHER_HELM_ARGS} ${latest_chart}
+helm upgrade --install --create-namespace -n cattle-monitoring-system prometheus-federator --set helmProjectOperator.image.repository=${REPO:-rancher}/prometheus-federator --set helmProjectOperator.image.tag=${TAG:-dev} ${cluster_args} ${RANCHER_HELM_ARGS} ./build/charts/prometheus-federator
 
 echo "PASS: Prometheus Federator has been installed"

build.yaml

@@ -0,0 +1 @@
+rancherProjectMonitoringVersion: 0.3.4

generate.go

@@ -1,3 +1,5 @@
+//go:generate go run pkg/codegen/buildconfig/writer.go pkg/codegen/buildconfig/main.go
+
 //go:generate go run pkg/helm-locker/codegen/cleanup/main.go
 //go:generate go run pkg/helm-locker/codegen/main.go
 //go:generate go run ./pkg/helm-locker/codegen crds ./crds/helm-locker/crds.yaml

go.mod

@@ -21,7 +21,10 @@ require (
 	github.com/rancher/wrangler-cli v0.0.0-20211112052728-f172e9bf59af
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.4.0
+	github.com/stretchr/testify v1.7.0
+	golang.org/x/text v0.7.0
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	helm.sh/helm/v3 v3.8.0
 	k8s.io/api v0.23.3
 	k8s.io/apiextensions-apiserver v0.23.1
@@ -60,6 +63,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.12.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
@@ -75,14 +79,12 @@ require (
 	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
 	golang.org/x/sys v0.5.0 // indirect
 	golang.org/x/term v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	golang.org/x/tools v0.1.12 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/gorp.v1 v1.7.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	k8s.io/code-generator v0.23.3 // indirect
 	k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c // indirect
 	k8s.io/klog v1.0.0 // indirect

pkg/codegen/buildconfig/main.go

@@ -0,0 +1,44 @@
+// This program generates a Go file containing a set of exported constants that represent
+// configuration variables of Rancher at build-time.
+package main
+
+import (
+	"fmt"
+	"os"
+	"text/template"
+)
+
+func main() {
+	if err := generateGoConstantsFile(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func generateGoConstantsFile() error {
+	in, err := os.OpenFile("build.yaml", os.O_RDONLY, 0644)
+	if err != nil {
+		return err
+	}
+	out, err := os.OpenFile("pkg/buildconfig/constants.go", os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	const raw = `// Code generated by pkg/codegen/config/main.go. DO NOT EDIT.
+// Package buildconfig contains a set of exported constants that represent configuration variables of Rancher at build-time.
+package buildconfig
+
+const (
+{{ . }})
+`
+	tmpl, err := template.New("").Parse(raw)
+	if err != nil {
+		return err
+	}
+	writer := GoConstantsWriter{
+		Tmpl:   tmpl,
+		Input:  in,
+		Output: out,
+	}
+	return writer.Run()
+}

pkg/codegen/buildconfig/writer.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"go/format"
+	"io"
+	"sort"
+	"strings"
+	"text/template"
+
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
+	"gopkg.in/yaml.v3"
+)
+
+type GoConstantsWriter struct {
+	Input  io.Reader
+	Output io.Writer
+	Tmpl   *template.Template
+	buf    []byte
+	cfg    map[string]string
+}
+
+// Run loads YAML data from the pre-configured Input source, processes it, and outputs a template with formatted
+// Go constants in the pre-configured Output source. This method can only be run once, since the Input source gets fully read.
+func (f *GoConstantsWriter) Run() error {
+	if err := f.load(); err != nil {
+		return err
+	}
+	if err := f.process(); err != nil {
+		return err
+	}
+	if err := f.write(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (f *GoConstantsWriter) load() error {
+	if f.Input == nil {
+		return errors.New("nil input")
+	}
+	b, err := io.ReadAll(f.Input)
+	if err != nil {
+		return fmt.Errorf("failed to read input: %w", err)
+	}
+	if len(b) == 0 {
+		return errors.New("nothing was read")
+	}
+	if err := yaml.Unmarshal(b, &f.cfg); err != nil {
+		return fmt.Errorf("failed to unmarshal raw YAML from input: %w", err)
+	}
+	return nil
+}
+
+func (f *GoConstantsWriter) process() error {
+	if f.Tmpl == nil {
+		return errors.New("nil template")
+	}
+	// This sorts the keys alphabetically to process the map in a fixed order.
+	keys := make([]string, 0, len(f.cfg))
+	for k := range f.cfg {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	capitalize := cases.Title(language.English, cases.NoLower)
+	var builder strings.Builder
+	for _, k := range keys {
+		v := f.cfg[k]
+		// Capitalize the key to make the constant exported in the generated Go file.
+		k = capitalize.String(k)
+		s := fmt.Sprintf("\t%s = %q\n", k, v)
+		builder.WriteString(s)
+	}
+
+	buf := new(bytes.Buffer)
+	if err := f.Tmpl.Execute(buf, builder.String()); err != nil {
+		return err
+	}
+	f.buf = buf.Bytes()
+	return nil
+}
+
+func (f *GoConstantsWriter) write() error {
+	if f.Output == nil {
+		return errors.New("nil output")
+	}
+	formatted, err := format.Source(f.buf)
+	if err != nil {
+		return err
+	}
+	_, err = f.Output.Write(formatted)
+	return err
+}

pkg/codegen/buildconfig/writer_test.go

@@ -0,0 +1,118 @@
+package main_test
+
+import (
+	"bytes"
+	"io"
+	"testing"
+	"text/template"
+
+	"github.com/stretchr/testify/require"
+
+	mainpkg "github.com/rancher/prometheus-federator/pkg/codegen/buildconfig"
+)
+
+func TestGoConstantsWriterRun(t *testing.T) {
+	t.Parallel()
+	const contents = `b: 3
+a: foo
+c: 3.14`
+	in := bytes.NewBufferString(contents)
+	out := new(bytes.Buffer)
+
+	const rawTemplate = `
+	package buildconfig
+
+	const (
+	{{ . }})
+	`
+	tmpl, err := template.New("").Parse(rawTemplate)
+	require.NoError(t, err)
+	w := &mainpkg.GoConstantsWriter{
+		Tmpl:   tmpl,
+		Input:  in,
+		Output: out,
+	}
+	require.NoError(t, w.Run())
+
+	want :=
+		`package buildconfig
+
+const (
+	A = "foo"
+	B = "3"
+	C = "3.14"
+)
+`
+	got := out.String()
+	require.Equal(t, want, got)
+
+	// Running a second time with the same Input source must fail.
+	require.Error(t, w.Run())
+}
+
+func TestGoConstantsWriterFailsWithBadConfiguration(t *testing.T) {
+	t.Parallel()
+	const rawTemplate = `
+	package buildconfig
+
+	const (
+	{{ . }})
+	`
+	tmpl, err := template.New("").Parse(rawTemplate)
+	require.NoError(t, err)
+	const contents = `a: foo
+b: 3
+c: 3.14`
+	output := new(bytes.Buffer)
+
+	tests := []struct {
+		name   string
+		tmpl   *template.Template
+		input  io.Reader
+		output io.Writer
+	}{
+		{
+			name:   "nil template",
+			tmpl:   nil,
+			input:  bytes.NewBufferString(contents),
+			output: output,
+		},
+		{
+			name:   "empty template",
+			tmpl:   template.New(""),
+			input:  bytes.NewBufferString(contents),
+			output: output,
+		},
+		{
+			name:   "nil input",
+			tmpl:   tmpl,
+			input:  nil,
+			output: output,
+		},
+		{
+			name:   "empty input",
+			tmpl:   tmpl,
+			input:  bytes.NewBufferString(""),
+			output: output,
+		},
+		{
+			name:   "nil output",
+			tmpl:   tmpl,
+			input:  bytes.NewBufferString(contents),
+			output: nil,
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			w := mainpkg.GoConstantsWriter{
+				Input:  test.input,
+				Output: test.output,
+				Tmpl:   test.tmpl,
+			}
+			require.Error(t, w.Run())
+		})
+	}
+}

scripts/build-chart

@@ -1,8 +1,12 @@
 #!/usr/bin/env bash
 set -e
 
+# This actually prepares a chart for compile time embedding into the resulting binary.
+# This is always required even after splitting charts and app code.
+
 source $(dirname $0)/version
 source $(dirname $0)/util-chart
+source $(dirname $0)/util-team-charts
 
 # We don't do this for helm-locker
 if [[ "${BUILD_TARGET}" == "helm-locker" ]]; then
@@ -21,10 +25,12 @@ fi
 CHART=${CHART:-${DEFAULT_CHART_TARGET}}
 
 if [ "$BUILD_TARGET" == "prometheus-federator" ]; then
-  CHART_VERSION=${EMBEDDED_CHART_VERSION:-$(find "./charts/${CHART}" -maxdepth 1 -mindepth 1 -type d | tr - \~ | sort -rV | tr \~ - | head -n1 | cut -d'/' -f4)}
+  # Fetch asset from github OB charts repo
+  fetch-team-chart "rancher-project-monitoring" "$RANCHER_PROJECT_MONITORING"
+  CHART_DESTINATION="./build/charts/${CHART}-${CHART_VERSION}.tgz"
 
   # Prepare chart for embedding location
-  base64 -i "assets/${CHART}/${CHART}-${CHART_VERSION}.tgz" > "cmd/${BUILD_TARGET}/fs/${CHART}.tgz.base64"
+  base64 -i "${CHART_DESTINATION}" > "cmd/${BUILD_TARGET}/fs/${CHART}.tgz.base64"
 elif [ "$BUILD_TARGET" == "helm-project-operator" ]; then
   # Prepare base chart for build
   clean-old-chart "$CHART"
@@ -38,4 +44,4 @@ elif [ "$BUILD_TARGET" == "helm-project-operator" ]; then
   base64 -i "./build/charts/${CHART}-${HELM_CHART_VERSION}.tgz" > "cmd/${BUILD_TARGET}/fs/${CHART}.tgz.base64"
 fi
 
-echo "Completed ${CHART} (ver: ${HELM_CHART_VERSION:-"n/a"}) build process."
+echo "Completed ${CHART} (ver: ${HELM_CHART_VERSION:-"n/a"}) chart build process."

scripts/ci

@@ -7,7 +7,4 @@ cd $(dirname $0)
 ./test
 ./validate
 ./validate-ci
-./validate-packages
-./validate-chart
-./validate-charts
 ./package