From 1bc6e4f1da13e0b1845c6285de6f39117673b5ed Mon Sep 17 00:00:00 2001 From: bwplotka Date: Wed, 18 Dec 2024 20:24:29 +0000 Subject: [PATCH] extended comment to include version and directory used. Signed-off-by: bwplotka --- .../7a_commentmonitor_configmap_noparse.yaml | 26 +++++-- prombench/manifests/prombench/README.md | 24 +++---- tools/comment-monitor/internal/command.go | 24 ++++++- .../comment-monitor/internal/command_test.go | 71 ++++++++++++++++-- .../comment-monitor/internal/testconfig.yaml | 72 ------------------- .../expectedcomment.restart-no-flags.md | 15 ++++ .../expectedcomment.restart-version.md | 17 +++++ .../expectedcomment.start-no-flags.md | 15 ++++ .../testdata/expectedcomment.start-version.md | 17 +++++ .../internal/testdata/testconfig.yaml | 38 ++++++++++ tools/comment-monitor/main.go | 34 ++------- 11 files changed, 231 insertions(+), 122 deletions(-) delete mode 100644 tools/comment-monitor/internal/testconfig.yaml create mode 100644 tools/comment-monitor/internal/testdata/expectedcomment.restart-no-flags.md create mode 100644 tools/comment-monitor/internal/testdata/expectedcomment.restart-version.md create mode 100644 tools/comment-monitor/internal/testdata/expectedcomment.start-no-flags.md create mode 100644 tools/comment-monitor/internal/testdata/expectedcomment.start-version.md create mode 100644 tools/comment-monitor/internal/testdata/testconfig.yaml diff --git a/prombench/manifests/cluster-infra/7a_commentmonitor_configmap_noparse.yaml b/prombench/manifests/cluster-infra/7a_commentmonitor_configmap_noparse.yaml index c58c532e9..0f03c1d5f 100644 --- a/prombench/manifests/cluster-infra/7a_commentmonitor_configmap_noparse.yaml +++ b/prombench/manifests/cluster-infra/7a_commentmonitor_configmap_noparse.yaml @@ -39,7 +39,16 @@ data: ⏱️ Welcome (again) to Prometheus Benchmarking Tool. ⏱️ **Compared versions:** [**`PR-{{ index . "PR_NUMBER" }}`**](http://{{ index . "DOMAIN_NAME" }}/{{ index . "PR_NUMBER" }}/prometheus-pr) and [**`{{ index . "RELEASE" }}`**](http://{{ index . "DOMAIN_NAME" }}/{{ index . "PR_NUMBER" }}/prometheus-release) - + + {{- $version := index . "BENCHMARK_VERSION" }} + {{- $directory := index . "BENCHMARK_DIRECTORY" | print "manifests/prombench" }} + {{- with $version }} + + **Custom benchmark version:** + {{- if hasPrefix $version "@" }} [**`{{ trimPrefix $version "@" }}` commit**](https://github.com/prometheus/test-infra/tree/{{ trimPrefix $version "@" }}/prombench/{{ $directory }}) + {{- else }} [**`{{ $version }}` branch**](https://github.com/prometheus/test-infra/tree/{{ $version }}/prombench/{{ $directory }}){{ end }} + {{- end }} + After successful deployment ([check status here](https://github.com/prometheus/prometheus/actions/workflows/prombench.yml)), the benchmarking results can be viewed at: - [Prometheus Meta](http://{{ index . "DOMAIN_NAME" }}/prometheus-meta/graph?g0.expr={namespace%3D"prombench-{{ index . "PR_NUMBER" }}"}&g0.tab=1) @@ -48,7 +57,7 @@ data: - [Parca profiles (e.g. in-use memory)](http://{{ index . "DOMAIN_NAME" }}/profiles?expression_a=memory%3Ainuse_space%3Abytes%3Aspace%3Abytes%7Bpr_number%3D%22{{ index . "PR_NUMBER" }}%22%7D&time_selection_a=relative:minute|15) **Available Commands:** - * To restart benchmark: `/prombench restart {{ index . "RELEASE" }}` + * To restart benchmark: `/prombench restart {{ index . "RELEASE" }}{{ if index . "BENCHMARK_VERSION" }} --bench.version={{ index . "BENCHMARK_VERSION" }}{{ end }}{{ if index . "BENCHMARK_DIRECTORY" }} --bench.directory={{ index . "BENCHMARK_DIRECTORY" }}{{ end }}` * To stop benchmark: `/prombench cancel` * To print help: `/prombench help` @@ -65,6 +74,15 @@ data: **Compared versions:** [**`PR-{{ index . "PR_NUMBER" }}`**](http://{{ index . "DOMAIN_NAME" }}/{{ index . "PR_NUMBER" }}/prometheus-pr) and [**`{{ index . "RELEASE" }}`**](http://{{ index . "DOMAIN_NAME" }}/{{ index . "PR_NUMBER" }}/prometheus-release) + {{- $version := index . "BENCHMARK_VERSION" }} + {{- $directory := index . "BENCHMARK_DIRECTORY" | print "manifests/prombench" }} + {{- with $version }} + + **Custom benchmark version:** + {{- if hasPrefix $version "@" }} [**`{{ trimPrefix $version "@" }}` commit**](https://github.com/prometheus/test-infra/tree/{{ trimPrefix $version "@" }}/prombench/{{ $directory }}) + {{- else }} [**`{{ $version }}` branch**](https://github.com/prometheus/test-infra/tree/{{ $version }}/prombench/{{ $directory }}){{ end }} + {{- end }} + After the successful deployment ([check status here](https://github.com/prometheus/prometheus/actions/workflows/prombench.yml)), the benchmarking results can be viewed at: - [Prometheus Meta](http://{{ index . "DOMAIN_NAME" }}/prometheus-meta/graph?g0.expr={namespace%3D"prombench-{{ index . "PR_NUMBER" }}"}&g0.tab=1) @@ -73,7 +91,7 @@ data: - [Parca profiles (e.g. in-use memory)](http://{{ index . "DOMAIN_NAME" }}/profiles?expression_a=memory%3Ainuse_space%3Abytes%3Aspace%3Abytes%7Bpr_number%3D%22{{ index . "PR_NUMBER" }}%22%7D&time_selection_a=relative:minute|15) **Available Commands:** - * To restart benchmark: `/prombench restart {{ index . "RELEASE" }}` + * To restart benchmark: `/prombench restart {{ index . "RELEASE" }}{{ if index . "BENCHMARK_VERSION" }} --bench.version={{ index . "BENCHMARK_VERSION" }}{{ end }}{{ if index . "BENCHMARK_DIRECTORY" }} --bench.directory={{ index . "BENCHMARK_DIRECTORY" }}{{ end }}` * To stop benchmark: `/prombench cancel` * To print help: `/prombench help` - + diff --git a/prombench/manifests/prombench/README.md b/prombench/manifests/prombench/README.md index 73857b237..e114f784e 100644 --- a/prombench/manifests/prombench/README.md +++ b/prombench/manifests/prombench/README.md @@ -5,17 +5,6 @@ via `infra` CLI using [`make deploy`](../../Makefile) and cleaned using [`make c It assumes running cluster was created via `infra` CLI using `make cluster_create` and `make cluster_delete`. -### Variables - -It expects the following templated variables: - -* `.PR_NUMBER`: The PR number from which `/prombench` was triggered. This PR number also tells what commit to use for the `prometheus-test-pr-{{ .PR_NUMBER }}` Prometheus image building (in the init container). -* `.RELEASE`: The argument provided by `/prombench` caller representing the Prometheus version (docker image tag for `quay.io/prometheus/prometheus:{{ .RELEASE }}`) to compare with, deployed as the `prometheus-test-{{ .RELEASE }}`. -* `.DOMAIN_NAME` -* `.LOADGEN_SCALE_UP_REPLICAS` -* `.GITHUB_ORG` -* `.GITHUB_REPO` - ### Customizations > NOTE: See https://github.com/prometheus/proposals/pull/41 for design. @@ -32,7 +21,7 @@ Here are an example steps: 1. Create a new branch on https://github.com/prometheus/test-infra e.g. `benchmark/scenario1`. 2. Modify this directory to your liking e.g. changing query load, metric load of advanced Prometheus configuration. It's also possible to make Prometheus deployments and versions exactly the same, but vary in a single configuration flag, for feature benchmarking. - > WARN: When customizing this directory, don't change `1a_namespace.yaml` or `1c_cluster-role-binding.yaml` filenames as they are used for cleanup routine. Or, if you change it, know what you're doing in relation to [`make clean` job](../../Makefile). + > WARN: When customizing this directory, don't change `1a_namespace.yaml` or `1c_cluster-role-binding.yaml` filenames as they are used for cleanup routine. Or, if you change it, know what you're doing in relation to [`make clean` job](../../Makefile). 3. Push changes to the new branch. 4. From the Prometheus PR comment, call prombench as `/prombench --bench.version=benchmark/scenario1` or `/prombench --bench.version=@` to use configuration files from this custom branch. @@ -43,3 +32,14 @@ Other details: * `--bench.version` is designed for a short-term or even one-off benchmark scenario configurations. It's not designed for long-term, well maintained scenarios. For the latter reason we can later e.g. maintain multiple `manifests/prombench` directories and introduce a new `--bench.directory` flag. * Non-maintainers can follow similar process, but they will need to ask maintainer for a new branch and PR review. We can consider extending `--bench.version` to support remote repositories if this becomes a problem. * Custom benchmarking logic is implemented in the [`maybe_pull_custom_version` make job](../../Makefile) and invoked by the prombench GH job on Prometheus repo on `deploy` and `clean`. + +### Variables + +It expects the following templated variables: + +* `.PR_NUMBER`: The PR number from which `/prombench` was triggered. This PR number also tells what commit to use for the `prometheus-test-pr-{{ .PR_NUMBER }}` Prometheus image building (in the init container). +* `.RELEASE`: The argument provided by `/prombench` caller representing the Prometheus version (docker image tag for `quay.io/prometheus/prometheus:{{ .RELEASE }}`) to compare with, deployed as the `prometheus-test-{{ .RELEASE }}`. +* `.DOMAIN_NAME` +* `.LOADGEN_SCALE_UP_REPLICAS` +* `.GITHUB_ORG` +* `.GITHUB_REPO` diff --git a/tools/comment-monitor/internal/command.go b/tools/comment-monitor/internal/command.go index 8ce7245f7..28430aa4a 100644 --- a/tools/comment-monitor/internal/command.go +++ b/tools/comment-monitor/internal/command.go @@ -14,11 +14,13 @@ package internal import ( + "bytes" "errors" "fmt" "os" "regexp" "strings" + "text/template" "gopkg.in/yaml.v2" ) @@ -98,7 +100,6 @@ type Command struct { Prefix string EventType string Args map[string]string - Flags map[string]string ShouldVerifyUser bool SuccessCommentTemplate string @@ -107,6 +108,27 @@ type Command struct { DebugCMDLine string } +func (c *Command) GenerateSuccessComment() (string, error) { + argsCpy := make(map[string]string, len(c.Args)) + for k, v := range c.Args { + argsCpy[k] = v + } + for _, e := range os.Environ() { + tmp := strings.Split(e, "=") + argsCpy[tmp[0]] = tmp[1] + } + + var buf bytes.Buffer + ct := template.Must(template.New("Comment").Funcs(template.FuncMap{ + "hasPrefix": strings.HasPrefix, + "trimPrefix": strings.TrimPrefix, + }).Parse(c.SuccessCommentTemplate)) + if err := ct.Execute(&buf, argsCpy); err != nil { + return "", fmt.Errorf("templating failed: %w", err) + } + return buf.String(), nil +} + type CommandParseError struct { error help string diff --git a/tools/comment-monitor/internal/command_test.go b/tools/comment-monitor/internal/command_test.go index c37455b66..899192422 100644 --- a/tools/comment-monitor/internal/command_test.go +++ b/tools/comment-monitor/internal/command_test.go @@ -14,6 +14,7 @@ package internal import ( + "fmt" "os" "strings" "testing" @@ -103,7 +104,7 @@ func testParseCommand(t *testing.T, c *Config, cases []parseCommandCase) { } func TestParseCommand(t *testing.T) { - c, err := ParseConfig("./testconfig.yaml") + c, err := ParseConfig("testdata/testconfig.yaml") if err != nil { t.Fatal(err) } @@ -193,9 +194,7 @@ func TestParseCommand(t *testing.T) { }) } -// NOTE(bwplotka): Simplified version of TestParseCommand that literally uses -// our production comment monitoring configuration in the same repo. -func TestParseCommand_ProdCommentMonitorConfig(t *testing.T) { +func parseProdCommentMonitorConfig(t *testing.T) *Config { const prodCommentMonitorConfigMap = "../../../prombench/manifests/cluster-infra/7a_commentmonitor_configmap_noparse.yaml" b, err := os.ReadFile(prodCommentMonitorConfigMap) @@ -219,6 +218,13 @@ func TestParseCommand_ProdCommentMonitorConfig(t *testing.T) { if err != nil { t.Fatal(err) } + return c +} + +// NOTE(bwplotka): Simplified version of TestParseCommand that literally uses +// our production comment monitoring configuration in the same repo. +func TestParseCommand_ProdCommentMonitorConfig(t *testing.T) { + c := parseProdCommentMonitorConfig(t) testParseCommand(t, c, []parseCommandCase{ { comment: "/prombench", @@ -262,3 +268,60 @@ func TestParseCommand_ProdCommentMonitorConfig(t *testing.T) { {comment: "How to start prombench? I think it was something like:\n\n /prombench main\n\nYolo"}, }) } + +func TestGenerateSuccessComment_ProdCommentMonitorConfig(t *testing.T) { + c := parseProdCommentMonitorConfig(t) + + for _, tcase := range []struct { + comment string + expectSuccessCommentFile string + }{ + // Test interesting comment templates from prod as they can get complex. + { + comment: "/prombench v3.0.0", + expectSuccessCommentFile: "testdata/expectedcomment.start-no-flags.md", + }, + { + comment: "/prombench v3.0.0 --bench.version=branch1", + expectSuccessCommentFile: "testdata/expectedcomment.start-version.md", + }, + { + comment: "/prombench restart v3.0.0", + expectSuccessCommentFile: "testdata/expectedcomment.restart-no-flags.md", + }, + { + comment: "/prombench restart v3.0.0 --bench.version=@aca1803ccf5d795eee4b0848707eab26d05965cc", + expectSuccessCommentFile: "testdata/expectedcomment.restart-version.md", + }, + } { + t.Run(tcase.comment, func(t *testing.T) { + cmd, found, perr := ParseCommand(c, tcase.comment) + if perr != nil { + t.Fatal(perr) + } + if !found { + t.Fatal("expected found=true") + } + + expected, err := os.ReadFile(tcase.expectSuccessCommentFile) + if err != nil { + t.Fatal(err) + } + + // We add those in the deployment env. + cmd.Args["DOMAIN_NAME"] = "prombench.example.com" + + // We add those in comment-monitor main.go flow. + cmd.Args["PR_NUMBER"] = "15487" + cmd.Args["LAST_COMMIT_SHA"] = "a854b28c2a0d920d0f313d6cb5ee79e44763df5e" + got, err := cmd.GenerateSuccessComment() + if err != nil { + t.Fatal(err) + } + fmt.Println(got) + if diff := cmp.Diff(got, string(expected)); diff != "" { + t.Fatal(diff) + } + }) + } +} diff --git a/tools/comment-monitor/internal/testconfig.yaml b/tools/comment-monitor/internal/testconfig.yaml deleted file mode 100644 index c50c8c762..000000000 --- a/tools/comment-monitor/internal/testconfig.yaml +++ /dev/null @@ -1,72 +0,0 @@ -prefixes: -- prefix: /prombench - help: | - **Available Commands:** - * To start benchmark: `/prombench ` - * To restart benchmark: `/prombench restart ` - * To stop benchmark: `/prombench cancel` - * To print help: `/prombench help` - - **Advanced Flags for `start` and `restart` Commands:**: - * `--bench.directory` (default: `manifests/prombench`) - * `--bench.version` (default: `master`) - - **Examples:** - * `/prombench v3.0.0` - * `/prombench v3.0.0 --bench.version=@aca1803ccf5d795eee4b0848707eab26d05965cc --bench.directory=manifests/prombench` - - verify_user: true - commands: - - name: cancel - event_type: prombench_stop - comment_template: | - Benchmark cancel is in progress. - - - name: restart - event_type: prombench_restart - arg_regex: (master|main|v[0-9]+\.[0-9]+\.[0-9]+\S*) - arg_name: RELEASE - flag_args: - bench.directory: BENCHMARK_DIRECTORY - bench.version: BENCHMARK_VERSION - comment_template: | - ⏱️ Welcome (again) to Prometheus Benchmarking Tool. ⏱️ - - **Compared versions:** [**`PR-{{ index . "PR_NUMBER" }}`**](http://{{ index . "DOMAIN_NAME" }}/{{ index . "PR_NUMBER" }}/prometheus-pr) and [**`{{ index . "RELEASE" }}`**](http://{{ index . "DOMAIN_NAME" }}/{{ index . "PR_NUMBER" }}/prometheus-release) - - After successful deployment, the benchmarking results can be viewed at: - - - [Prometheus Meta](http://{{ index . "DOMAIN_NAME" }}/prometheus-meta/graph?g0.expr={namespace%3D"prombench-{{ index . "PR_NUMBER" }}"}&g0.tab=1) - - [Prombench Dashboard](http://{{ index . "DOMAIN_NAME" }}/grafana/d/7gmLoNDmz/prombench?orgId=1&var-pr-number={{ index . "PR_NUMBER" }}) - - [Grafana Exlorer, Loki logs](http://{{ index . "DOMAIN_NAME" }}/grafana/explore?orgId=1&left=["now-6h","now","loki-meta",{},{"mode":"Logs"},{"ui":[true,true,true,"none"]}]) - - [Parca profiles (e.g. in-use memory)](http://{{ index . "DOMAIN_NAME" }}/profiles?expression_a=memory%3Ainuse_space%3Abytes%3Aspace%3Abytes%7Bpr_number%3D%22{{ index . "PR_NUMBER" }}%22%7D&time_selection_a=relative:minute|15) - - **Available Commands:** - * To restart benchmark: `/prombench restart {{ index . "RELEASE" }}` - * To stop benchmark: `/prombench cancel` - * To print help: `/prombench help` - - - name: "" # start is a default (empty command). - event_type: prombench_start - arg_regex: (master|main|v[0-9]+\.[0-9]+\.[0-9]+\S*) - arg_name: RELEASE - flag_args: - bench.directory: BENCHMARK_DIRECTORY - bench.version: BENCHMARK_VERSION - label: prombench - comment_template: | - ⏱️ Welcome to Prometheus Benchmarking Tool. ⏱️ - - **Compared versions:** [**`PR-{{ index . "PR_NUMBER" }}`**](http://{{ index . "DOMAIN_NAME" }}/{{ index . "PR_NUMBER" }}/prometheus-pr) and [**`{{ index . "RELEASE" }}`**](http://{{ index . "DOMAIN_NAME" }}/{{ index . "PR_NUMBER" }}/prometheus-release) - - After successful deployment, the benchmarking results can be viewed at: - - - [Prometheus Meta](http://{{ index . "DOMAIN_NAME" }}/prometheus-meta/graph?g0.expr={namespace%3D"prombench-{{ index . "PR_NUMBER" }}"}&g0.tab=1) - - [Prombench Dashboard](http://{{ index . "DOMAIN_NAME" }}/grafana/d/7gmLoNDmz/prombench?orgId=1&var-pr-number={{ index . "PR_NUMBER" }}) - - [Grafana Explorer, Loki logs](http://{{ index . "DOMAIN_NAME" }}/grafana/explore?orgId=1&left=["now-6h","now","loki-meta",{},{"mode":"Logs"},{"ui":[true,true,true,"none"]}]) - - [Parca profiles (e.g. in-use memory)](http://{{ index . "DOMAIN_NAME" }}/profiles?expression_a=memory%3Ainuse_space%3Abytes%3Aspace%3Abytes%7Bpr_number%3D%22{{ index . "PR_NUMBER" }}%22%7D&time_selection_a=relative:minute|15) - - **Available Commands:** - * To restart benchmark: `/prombench restart {{ index . "RELEASE" }}` - * To stop benchmark: `/prombench cancel` - * To print help: `/prombench help` diff --git a/tools/comment-monitor/internal/testdata/expectedcomment.restart-no-flags.md b/tools/comment-monitor/internal/testdata/expectedcomment.restart-no-flags.md new file mode 100644 index 000000000..d9862749c --- /dev/null +++ b/tools/comment-monitor/internal/testdata/expectedcomment.restart-no-flags.md @@ -0,0 +1,15 @@ +⏱️ Welcome (again) to Prometheus Benchmarking Tool. ⏱️ + +**Compared versions:** [**`PR-15487`**](http://prombench.example.com/15487/prometheus-pr) and [**`v3.0.0`**](http://prombench.example.com/15487/prometheus-release) + +After successful deployment ([check status here](https://github.com/prometheus/prometheus/actions/workflows/prombench.yml)), the benchmarking results can be viewed at: + +- [Prometheus Meta](http://prombench.example.com/prometheus-meta/graph?g0.expr={namespace%3D"prombench-15487"}&g0.tab=1) +- [Prombench Dashboard](http://prombench.example.com/grafana/d/7gmLoNDmz/prombench?orgId=1&var-pr-number=15487) +- [Grafana Exlorer, Loki logs](http://prombench.example.com/grafana/explore?orgId=1&left=["now-6h","now","loki-meta",{},{"mode":"Logs"},{"ui":[true,true,true,"none"]}]) +- [Parca profiles (e.g. in-use memory)](http://prombench.example.com/profiles?expression_a=memory%3Ainuse_space%3Abytes%3Aspace%3Abytes%7Bpr_number%3D%2215487%22%7D&time_selection_a=relative:minute|15) + +**Available Commands:** +* To restart benchmark: `/prombench restart v3.0.0` +* To stop benchmark: `/prombench cancel` +* To print help: `/prombench help` diff --git a/tools/comment-monitor/internal/testdata/expectedcomment.restart-version.md b/tools/comment-monitor/internal/testdata/expectedcomment.restart-version.md new file mode 100644 index 000000000..88686a904 --- /dev/null +++ b/tools/comment-monitor/internal/testdata/expectedcomment.restart-version.md @@ -0,0 +1,17 @@ +⏱️ Welcome (again) to Prometheus Benchmarking Tool. ⏱️ + +**Compared versions:** [**`PR-15487`**](http://prombench.example.com/15487/prometheus-pr) and [**`v3.0.0`**](http://prombench.example.com/15487/prometheus-release) + +**Custom benchmark version:** [**`aca1803ccf5d795eee4b0848707eab26d05965cc` commit**](https://github.com/prometheus/test-infra/tree/aca1803ccf5d795eee4b0848707eab26d05965cc/prombench/manifests/prombench) + +After successful deployment ([check status here](https://github.com/prometheus/prometheus/actions/workflows/prombench.yml)), the benchmarking results can be viewed at: + +- [Prometheus Meta](http://prombench.example.com/prometheus-meta/graph?g0.expr={namespace%3D"prombench-15487"}&g0.tab=1) +- [Prombench Dashboard](http://prombench.example.com/grafana/d/7gmLoNDmz/prombench?orgId=1&var-pr-number=15487) +- [Grafana Exlorer, Loki logs](http://prombench.example.com/grafana/explore?orgId=1&left=["now-6h","now","loki-meta",{},{"mode":"Logs"},{"ui":[true,true,true,"none"]}]) +- [Parca profiles (e.g. in-use memory)](http://prombench.example.com/profiles?expression_a=memory%3Ainuse_space%3Abytes%3Aspace%3Abytes%7Bpr_number%3D%2215487%22%7D&time_selection_a=relative:minute|15) + +**Available Commands:** +* To restart benchmark: `/prombench restart v3.0.0 --bench.version=@aca1803ccf5d795eee4b0848707eab26d05965cc` +* To stop benchmark: `/prombench cancel` +* To print help: `/prombench help` diff --git a/tools/comment-monitor/internal/testdata/expectedcomment.start-no-flags.md b/tools/comment-monitor/internal/testdata/expectedcomment.start-no-flags.md new file mode 100644 index 000000000..171d442d8 --- /dev/null +++ b/tools/comment-monitor/internal/testdata/expectedcomment.start-no-flags.md @@ -0,0 +1,15 @@ +⏱️ Welcome to Prometheus Benchmarking Tool. ⏱️ + +**Compared versions:** [**`PR-15487`**](http://prombench.example.com/15487/prometheus-pr) and [**`v3.0.0`**](http://prombench.example.com/15487/prometheus-release) + +After the successful deployment ([check status here](https://github.com/prometheus/prometheus/actions/workflows/prombench.yml)), the benchmarking results can be viewed at: + +- [Prometheus Meta](http://prombench.example.com/prometheus-meta/graph?g0.expr={namespace%3D"prombench-15487"}&g0.tab=1) +- [Prombench Dashboard](http://prombench.example.com/grafana/d/7gmLoNDmz/prombench?orgId=1&var-pr-number=15487) +- [Grafana Explorer, Loki logs](http://prombench.example.com/grafana/explore?orgId=1&left=["now-6h","now","loki-meta",{},{"mode":"Logs"},{"ui":[true,true,true,"none"]}]) +- [Parca profiles (e.g. in-use memory)](http://prombench.example.com/profiles?expression_a=memory%3Ainuse_space%3Abytes%3Aspace%3Abytes%7Bpr_number%3D%2215487%22%7D&time_selection_a=relative:minute|15) + +**Available Commands:** +* To restart benchmark: `/prombench restart v3.0.0` +* To stop benchmark: `/prombench cancel` +* To print help: `/prombench help` diff --git a/tools/comment-monitor/internal/testdata/expectedcomment.start-version.md b/tools/comment-monitor/internal/testdata/expectedcomment.start-version.md new file mode 100644 index 000000000..10c5f246a --- /dev/null +++ b/tools/comment-monitor/internal/testdata/expectedcomment.start-version.md @@ -0,0 +1,17 @@ +⏱️ Welcome to Prometheus Benchmarking Tool. ⏱️ + +**Compared versions:** [**`PR-15487`**](http://prombench.example.com/15487/prometheus-pr) and [**`v3.0.0`**](http://prombench.example.com/15487/prometheus-release) + +**Custom benchmark version:** [**`branch1` branch**](https://github.com/prometheus/test-infra/tree/branch1/prombench/manifests/prombench) + +After the successful deployment ([check status here](https://github.com/prometheus/prometheus/actions/workflows/prombench.yml)), the benchmarking results can be viewed at: + +- [Prometheus Meta](http://prombench.example.com/prometheus-meta/graph?g0.expr={namespace%3D"prombench-15487"}&g0.tab=1) +- [Prombench Dashboard](http://prombench.example.com/grafana/d/7gmLoNDmz/prombench?orgId=1&var-pr-number=15487) +- [Grafana Explorer, Loki logs](http://prombench.example.com/grafana/explore?orgId=1&left=["now-6h","now","loki-meta",{},{"mode":"Logs"},{"ui":[true,true,true,"none"]}]) +- [Parca profiles (e.g. in-use memory)](http://prombench.example.com/profiles?expression_a=memory%3Ainuse_space%3Abytes%3Aspace%3Abytes%7Bpr_number%3D%2215487%22%7D&time_selection_a=relative:minute|15) + +**Available Commands:** +* To restart benchmark: `/prombench restart v3.0.0 --bench.version=branch1` +* To stop benchmark: `/prombench cancel` +* To print help: `/prombench help` diff --git a/tools/comment-monitor/internal/testdata/testconfig.yaml b/tools/comment-monitor/internal/testdata/testconfig.yaml new file mode 100644 index 000000000..58841908d --- /dev/null +++ b/tools/comment-monitor/internal/testdata/testconfig.yaml @@ -0,0 +1,38 @@ +prefixes: +- prefix: /prombench + help: | + **Available Commands:** + * To start benchmark: `/prombench ` + * To restart benchmark: `/prombench restart ` + * To stop benchmark: `/prombench cancel` + * To print help: `/prombench help` + + **Advanced Flags for `start` and `restart` Commands:**: + * `--bench.directory` (default: `manifests/prombench`) + * `--bench.version` (default: `master`) + + **Examples:** + * `/prombench v3.0.0` + * `/prombench v3.0.0 --bench.version=@aca1803ccf5d795eee4b0848707eab26d05965cc --bench.directory=manifests/prombench` + + verify_user: true + commands: + - name: cancel + event_type: prombench_stop + + - name: restart + event_type: prombench_restart + arg_regex: (master|main|v[0-9]+\.[0-9]+\.[0-9]+\S*) + arg_name: RELEASE + flag_args: + bench.directory: BENCHMARK_DIRECTORY + bench.version: BENCHMARK_VERSION + + - name: "" # start is a default (empty command). + event_type: prombench_start + arg_regex: (master|main|v[0-9]+\.[0-9]+\.[0-9]+\S*) + arg_name: RELEASE + flag_args: + bench.directory: BENCHMARK_DIRECTORY + bench.version: BENCHMARK_VERSION + label: prombench diff --git a/tools/comment-monitor/main.go b/tools/comment-monitor/main.go index 47608881a..32776d051 100644 --- a/tools/comment-monitor/main.go +++ b/tools/comment-monitor/main.go @@ -14,7 +14,6 @@ package main import ( - "bytes" "context" "fmt" "io" @@ -26,7 +25,6 @@ import ( "strconv" "strings" "syscall" - "text/template" "github.com/google/go-github/v29/github" "github.com/nelkinda/health-go" @@ -238,22 +236,21 @@ func (d *dispatcher) HandleIssue(w http.ResponseWriter, r *http.Request) { } } - allArgs := cmd.Args if cmd.EventType != "" { logger = logger.With("cmdLine", cmd.DebugCMDLine) logger.Info("dispatching a new command and updating issue") // Combine all arguments for both dispatch and the comment update. - allArgs["PR_NUMBER"] = strconv.Itoa(eventDetails.PR) - allArgs["LAST_COMMIT_SHA"], err = ghClient.GetLastCommitSHA() + cmd.Args["PR_NUMBER"] = strconv.Itoa(eventDetails.PR) + cmd.Args["LAST_COMMIT_SHA"], err = ghClient.GetLastCommitSHA() if err != nil { // TODO(bwplotka) Post comment about this failure? handleErr(w, logger, "could not fetch SHA, which likely means it's an issue, not a pull request. Non-PRs are not supported.", http.StatusBadRequest, err) return } - logger = logger.With("evenType", cmd.EventType, "args", fmt.Sprintf("%v", allArgs)) - if err = ghClient.Dispatch(cmd.EventType, allArgs); err != nil { + logger = logger.With("evenType", cmd.EventType, "args", fmt.Sprintf("%v", cmd.Args)) + if err = ghClient.Dispatch(cmd.EventType, cmd.Args); err != nil { // TODO(bwplotka) Post comment about this failure? handleErr(w, logger, "could not dispatch", http.StatusInternalServerError, err) return @@ -262,7 +259,7 @@ func (d *dispatcher) HandleIssue(w http.ResponseWriter, r *http.Request) { } // Update the issue. - comment, err := executeCommentTemplate(cmd.SuccessCommentTemplate, allArgs) + comment, err := cmd.GenerateSuccessComment() if err != nil { handleErr(w, logger, "failed to execute template", http.StatusInternalServerError, err) return @@ -280,24 +277,3 @@ func (d *dispatcher) HandleIssue(w http.ResponseWriter, r *http.Request) { } } } - -func executeCommentTemplate(commentTemplate string, args map[string]string) (string, error) { - argsCpy := make(map[string]string, len(args)) // TODO(bwplotka): Looks unsafe, we might want to type the known options. - if len(args) > 0 { - for k, v := range args { - argsCpy[k] = v - } - } - for _, e := range os.Environ() { - tmp := strings.Split(e, "=") - argsCpy[tmp[0]] = tmp[1] - } - - // Generate the comment template. - var buf bytes.Buffer - ct := template.Must(template.New("Comment").Parse(commentTemplate)) - if err := ct.Execute(&buf, argsCpy); err != nil { - return "", fmt.Errorf("templating failed: %w", err) - } - return buf.String(), nil -}