From 5ed168c1da981268e348f782c34aafc1ad1310a6 Mon Sep 17 00:00:00 2001 From: Edward Muller Date: Mon, 8 Aug 2022 15:26:26 -0700 Subject: [PATCH 1/5] minimal set of changes to bring things up to date. It's no longer necessary (or even useful) to go get the files as go modules takes care of that for us. --- .github/workflows/build.yml | 2 +- Dockerfile | 2 +- README.md | 14 ++++- go.mod | 3 +- go.sum | 2 - scripts/test | 6 ++ scripts/test-golang110/docker-compose.yml | 58 ------------------- scripts/test-golang110/test.sh | 44 -------------- scripts/test-golang111/Dockerfile | 7 --- .../docker-compose.override.yml | 15 ----- .../Dockerfile | 4 +- .../docker-compose.override.yml | 0 .../docker-compose.yml | 0 .../test.sh | 1 - .../Dockerfile | 4 +- .../docker-compose.override.yml | 0 .../docker-compose.yml | 1 + .../test.sh | 4 -- scripts/test.sh | 5 +- 19 files changed, 27 insertions(+), 145 deletions(-) create mode 100755 scripts/test delete mode 100644 scripts/test-golang110/docker-compose.yml delete mode 100755 scripts/test-golang110/test.sh delete mode 100644 scripts/test-golang111/Dockerfile delete mode 100644 scripts/test-golang114/docker-compose.override.yml rename scripts/{test-golang114 => test-golang118}/Dockerfile (80%) rename scripts/{test-golang110 => test-golang118}/docker-compose.override.yml (100%) rename scripts/{test-golang114 => test-golang118}/docker-compose.yml (100%) rename scripts/{test-golang114 => test-golang118}/test.sh (84%) rename scripts/{test-golang110 => test-golang119}/Dockerfile (78%) rename scripts/{test-golang111 => test-golang119}/docker-compose.override.yml (100%) rename scripts/{test-golang111 => test-golang119}/docker-compose.yml (96%) rename scripts/{test-golang111 => test-golang119}/test.sh (80%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bfa4134..1dd885d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go: [ '1.14' ] + go: [ '1.18' ] name: Build ${{ matrix.go }} steps: - uses: aws-actions/configure-aws-credentials@v1 diff --git a/Dockerfile b/Dockerfile index d0a3117..8d15759 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14-alpine +FROM golang:1.18-alpine COPY goroot/ /go/ # this is used to lint and build tarball diff --git a/README.md b/README.md index 40a5108..8dd6209 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,19 @@ git remote set-head origin -a ``` ## Installation -`go get github.com/signalsciences/sigsci-module-golang` + +```console +cd $(go env GOPATH)/src/github.com/signalsciences +git clone https://github.com/signalsciences/sigsci-module-golang.git +``` + +## Running tests + +Must be logged into our ECR repo + +```console +aws ecr get-login-password --region us-west-2 --profile sigsci_prod | docker login --username AWS --password-stdin 803688608479.dkr.ecr.us-west-2.amazonaws.com +``` ## Example Code Snippet ```go diff --git a/go.mod b/go.mod index c009874..b6b461f 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,8 @@ module github.com/signalsciences/sigsci-module-golang -go 1.12 +go 1.18 require ( - github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/philhofer/fwd v1.0.0 // indirect github.com/signalsciences/tlstext v1.2.0 github.com/tinylib/msgp v1.1.0 diff --git a/go.sum b/go.sum index fa39b00..43d1d59 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/signalsciences/tlstext v1.2.0 h1:ps1ZCoDz93oMK0ySe7G/2J0dpTT32cN20U+/xy0S7uk= diff --git a/scripts/test b/scripts/test new file mode 100755 index 0000000..cc242cd --- /dev/null +++ b/scripts/test @@ -0,0 +1,6 @@ +#!/bin/sh + +set -ex + +(cd ./scripts/test-golang118 && ./test.sh) +(cd ./scripts/test-golang119 && ./test.sh) diff --git a/scripts/test-golang110/docker-compose.yml b/scripts/test-golang110/docker-compose.yml deleted file mode 100644 index c40f8a8..0000000 --- a/scripts/test-golang110/docker-compose.yml +++ /dev/null @@ -1,58 +0,0 @@ -version: "3" -networks: - mtest: - -services: - # this defines our webserver uses our sigsci-module - # we only define it so it is attached to our fake network - # it will be run a few times with different options manually - # - # - web: - build: - context: . - dockerfile: Dockerfile - expose: - - "8085" - networks: - - mtest - depends_on: - - agent - - # agent - agent: - image: 803688608479.dkr.ecr.us-west-2.amazonaws.com/local-dev/sigsci-agent:latest - command: [ "-debug-log-web-inputs", "2", "-rpc-address", "9090", "-debug-rpc-test-harness", "-debug-standalone", "3" ] - expose: - - "9090" - - "12345" - networks: - - mtest - - # punching bag - punchingbag: - image: 803688608479.dkr.ecr.us-west-2.amazonaws.com/local-dev/module-testing:latest - networks: - - mtest - expose: - - "8086" - command: [ "/bin/punchingbag", "-addr", ":8086" ] - - # mtest - # - mtest: - image: 803688608479.dkr.ecr.us-west-2.amazonaws.com/local-dev/module-testing:latest - networks: - - mtest - depends_on: - - web - - agent - - punchingbag - environment: - - DISABLE_HTTP_OPTIONS=1 - - DISABLE_NOCOOKIE=1 - - MTEST_BASEURL=web:8085 - - MTEST_AGENT=agent:12345 - command: [ "/bin/wait-for", "web:8085", "--", "/bin/mtest", "-test.v" ] - - diff --git a/scripts/test-golang110/test.sh b/scripts/test-golang110/test.sh deleted file mode 100755 index e37f631..0000000 --- a/scripts/test-golang110/test.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -set -ex - -DOCKERCOMPOSE="docker-compose" - -# run at end no matter what -cleanup() { - echo "shutting down" - # capture log output - $DOCKERCOMPOSE logs --no-color agent >& agent.log - $DOCKERCOMPOSE logs --no-color web >& web.log - $DOCKERCOMPOSE logs --no-color mtest >& mtest.log - $DOCKERCOMPOSE logs --no-color punchingbag >& punchingbag.log - - # delete everything - $DOCKERCOMPOSE down - - # show output of module testing - cat mtest.log -} -trap cleanup 0 1 2 3 6 - -# attempt to clean up any leftover junk -$DOCKERCOMPOSE down - -$DOCKERCOMPOSE pull --ignore-pull-failures - -# start everything, run tests -# -# --no-color --> safe for ci -# --build --> alway build test server/module container -# --abort-on-container-exit --> without this, the other servers keep the process running -# --exit-code-from mtest --> make exit code be the result of module test -# -# > /dev/null --> output of all servers is mixed together and ugly -# we get the individual logs at end -# -if [ -d "goroot" ]; then - rm -rf goroot -fi -docker run -v ${PWD}/goroot:/go/ --rm golang:1.10.6-alpine3.8 /bin/sh -c 'apk --update add git && go get github.com/signalsciences/tlstext && go get github.com/tinylib/msgp && go get github.com/alecthomas/gometalinter' - -$DOCKERCOMPOSE up --no-color --build --abort-on-container-exit --exit-code-from mtest > /dev/null - diff --git a/scripts/test-golang111/Dockerfile b/scripts/test-golang111/Dockerfile deleted file mode 100644 index cbd58ce..0000000 --- a/scripts/test-golang111/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM golang:1.11.3-alpine3.8 - -COPY goroot/ /go/ - -# we will mount the current directory here -VOLUME [ "/go/src/github.com/signalsciences/sigsci-module-golang" ] -WORKDIR /go/src/github.com/signalsciences/sigsci-module-golang diff --git a/scripts/test-golang114/docker-compose.override.yml b/scripts/test-golang114/docker-compose.override.yml deleted file mode 100644 index 7af37fd..0000000 --- a/scripts/test-golang114/docker-compose.override.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: "3" - -services: - # this defines our webserver uses our sigsci-module - # we only define it so it is attached to our fake network - # it will be run a few times with different options manually - # - # The volumes spec is a bit weird.. this script is run in scripts/test but - # needs stuff in ../../examples. Consider moving. - web: - volumes: - - ../..:/go/src/github.com/signalsciences/sigsci-module-golang - command: [ "go", "run", "/go/src/github.com/signalsciences/sigsci-module-golang/examples/mtest/main.go" ] - environment: - - DEBUG=0 diff --git a/scripts/test-golang114/Dockerfile b/scripts/test-golang118/Dockerfile similarity index 80% rename from scripts/test-golang114/Dockerfile rename to scripts/test-golang118/Dockerfile index 17a6ba0..4f44f9b 100644 --- a/scripts/test-golang114/Dockerfile +++ b/scripts/test-golang118/Dockerfile @@ -1,6 +1,4 @@ -FROM golang:1.14-alpine - -COPY goroot/ /go/ +FROM golang:1.18-alpine # we will mount the current directory here VOLUME [ "/go/src/github.com/signalsciences/sigsci-module-golang" ] diff --git a/scripts/test-golang110/docker-compose.override.yml b/scripts/test-golang118/docker-compose.override.yml similarity index 100% rename from scripts/test-golang110/docker-compose.override.yml rename to scripts/test-golang118/docker-compose.override.yml diff --git a/scripts/test-golang114/docker-compose.yml b/scripts/test-golang118/docker-compose.yml similarity index 100% rename from scripts/test-golang114/docker-compose.yml rename to scripts/test-golang118/docker-compose.yml diff --git a/scripts/test-golang114/test.sh b/scripts/test-golang118/test.sh similarity index 84% rename from scripts/test-golang114/test.sh rename to scripts/test-golang118/test.sh index c2140da..b365a19 100755 --- a/scripts/test-golang114/test.sh +++ b/scripts/test-golang118/test.sh @@ -40,6 +40,5 @@ $DOCKERCOMPOSE pull --ignore-pull-failures if [ -d "goroot" ]; then rm -rf goroot fi -docker run -v ${PWD}/goroot:/go/ --rm golang:1.14-alpine /bin/sh -c 'apk --update add git && go get github.com/signalsciences/tlstext && go get github.com/tinylib/msgp && go get github.com/alecthomas/gometalinter' $DOCKERCOMPOSE up --no-color --build --abort-on-container-exit --exit-code-from mtest > /dev/null diff --git a/scripts/test-golang110/Dockerfile b/scripts/test-golang119/Dockerfile similarity index 78% rename from scripts/test-golang110/Dockerfile rename to scripts/test-golang119/Dockerfile index b43d134..d805bb0 100644 --- a/scripts/test-golang110/Dockerfile +++ b/scripts/test-golang119/Dockerfile @@ -1,6 +1,4 @@ -FROM golang:1.10.6-alpine3.8 - -COPY goroot/ /go/ +FROM golang:1.19-alpine # we will mount the current directory here VOLUME [ "/go/src/github.com/signalsciences/sigsci-module-golang" ] diff --git a/scripts/test-golang111/docker-compose.override.yml b/scripts/test-golang119/docker-compose.override.yml similarity index 100% rename from scripts/test-golang111/docker-compose.override.yml rename to scripts/test-golang119/docker-compose.override.yml diff --git a/scripts/test-golang111/docker-compose.yml b/scripts/test-golang119/docker-compose.yml similarity index 96% rename from scripts/test-golang111/docker-compose.yml rename to scripts/test-golang119/docker-compose.yml index 25ad43b..9b8a2b0 100644 --- a/scripts/test-golang111/docker-compose.yml +++ b/scripts/test-golang119/docker-compose.yml @@ -53,5 +53,6 @@ services: - DISABLE_NOCOOKIE=1 - MTEST_BASEURL=web:8085 - MTEST_AGENT=agent:12345 + - "MTEST_RUN_TEST_BLOCK_VIA_REDIRECT=true" command: [ "/bin/wait-for", "web:8085", "--", "/bin/mtest", "-test.v" ] diff --git a/scripts/test-golang111/test.sh b/scripts/test-golang119/test.sh similarity index 80% rename from scripts/test-golang111/test.sh rename to scripts/test-golang119/test.sh index 87139d2..b703739 100755 --- a/scripts/test-golang111/test.sh +++ b/scripts/test-golang119/test.sh @@ -37,9 +37,5 @@ $DOCKERCOMPOSE pull --ignore-pull-failures # > /dev/null --> output of all servers is mixed together and ugly # we get the individual logs at end # -if [ -d "goroot" ]; then - rm -rf goroot -fi -docker run -v ${PWD}/goroot:/go/ --rm golang:1.11.3-alpine3.8 /bin/sh -c 'apk --update add git && go get github.com/signalsciences/tlstext && go get github.com/tinylib/msgp && go get github.com/alecthomas/gometalinter' $DOCKERCOMPOSE up --no-color --build --abort-on-container-exit --exit-code-from mtest > /dev/null diff --git a/scripts/test.sh b/scripts/test.sh index b653f5f..cc242cd 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -2,6 +2,5 @@ set -ex -(cd ./scripts/test-golang110 && ./test.sh) -(cd ./scripts/test-golang111 && ./test.sh) -(cd ./scripts/test-golang114 && ./test.sh) +(cd ./scripts/test-golang118 && ./test.sh) +(cd ./scripts/test-golang119 && ./test.sh) From 77ebf52bfbf098539456a7f0b162fde916bd7b0e Mon Sep 17 00:00:00 2001 From: Edward Muller Date: Mon, 8 Aug 2022 15:46:38 -0700 Subject: [PATCH 2/5] Add go 1.17 as the baseline --- go.mod | 5 +- scripts/test-golang117/Dockerfile | 5 ++ .../docker-compose.override.yml | 15 +++++ scripts/test-golang117/docker-compose.yml | 58 +++++++++++++++++++ scripts/test-golang117/test.sh | 41 +++++++++++++ scripts/test-golang118/test.sh | 3 - scripts/test.sh | 1 + 7 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 scripts/test-golang117/Dockerfile create mode 100644 scripts/test-golang117/docker-compose.override.yml create mode 100644 scripts/test-golang117/docker-compose.yml create mode 100755 scripts/test-golang117/test.sh diff --git a/go.mod b/go.mod index b6b461f..d9ebd87 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,10 @@ module github.com/signalsciences/sigsci-module-golang -go 1.18 +go 1.17 require ( - github.com/philhofer/fwd v1.0.0 // indirect github.com/signalsciences/tlstext v1.2.0 github.com/tinylib/msgp v1.1.0 ) + +require github.com/philhofer/fwd v1.0.0 // indirect diff --git a/scripts/test-golang117/Dockerfile b/scripts/test-golang117/Dockerfile new file mode 100644 index 0000000..459a2d6 --- /dev/null +++ b/scripts/test-golang117/Dockerfile @@ -0,0 +1,5 @@ +FROM golang:1.17-alpine + +# we will mount the current directory here +VOLUME [ "/go/src/github.com/signalsciences/sigsci-module-golang" ] +WORKDIR /go/src/github.com/signalsciences/sigsci-module-golang diff --git a/scripts/test-golang117/docker-compose.override.yml b/scripts/test-golang117/docker-compose.override.yml new file mode 100644 index 0000000..7af37fd --- /dev/null +++ b/scripts/test-golang117/docker-compose.override.yml @@ -0,0 +1,15 @@ +version: "3" + +services: + # this defines our webserver uses our sigsci-module + # we only define it so it is attached to our fake network + # it will be run a few times with different options manually + # + # The volumes spec is a bit weird.. this script is run in scripts/test but + # needs stuff in ../../examples. Consider moving. + web: + volumes: + - ../..:/go/src/github.com/signalsciences/sigsci-module-golang + command: [ "go", "run", "/go/src/github.com/signalsciences/sigsci-module-golang/examples/mtest/main.go" ] + environment: + - DEBUG=0 diff --git a/scripts/test-golang117/docker-compose.yml b/scripts/test-golang117/docker-compose.yml new file mode 100644 index 0000000..9b8a2b0 --- /dev/null +++ b/scripts/test-golang117/docker-compose.yml @@ -0,0 +1,58 @@ +version: "3" +networks: + mtest: + +services: + # this defines our webserver uses our sigsci-module + # we only define it so it is attached to our fake network + # it will be run a few times with different options manually + # + # + web: + build: + context: . + dockerfile: Dockerfile + expose: + - "8085" + networks: + - mtest + depends_on: + - agent + + # agent + agent: + image: 803688608479.dkr.ecr.us-west-2.amazonaws.com/local-dev/sigsci-agent:latest + command: [ "-debug-log-web-inputs", "2", "-rpc-address", "9090", "-debug-rpc-test-harness", "-debug-standalone", "3" ] + expose: + - "9090" + - "12345" + networks: + - mtest + + # punching bag + punchingbag: + image: 803688608479.dkr.ecr.us-west-2.amazonaws.com/local-dev/module-testing:latest + networks: + - mtest + expose: + - "8086" + command: [ "/bin/punchingbag", "-addr", ":8086" ] + + # mtest + # + mtest: + image: 803688608479.dkr.ecr.us-west-2.amazonaws.com/local-dev/module-testing:latest + networks: + - mtest + depends_on: + - web + - agent + - punchingbag + environment: + - DISABLE_HTTP_OPTIONS=1 + - DISABLE_NOCOOKIE=1 + - MTEST_BASEURL=web:8085 + - MTEST_AGENT=agent:12345 + - "MTEST_RUN_TEST_BLOCK_VIA_REDIRECT=true" + command: [ "/bin/wait-for", "web:8085", "--", "/bin/mtest", "-test.v" ] + diff --git a/scripts/test-golang117/test.sh b/scripts/test-golang117/test.sh new file mode 100755 index 0000000..b703739 --- /dev/null +++ b/scripts/test-golang117/test.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -e + +DOCKERCOMPOSE="docker-compose" + +# run at end no matter what +cleanup() { + echo "shutting down" + # capture log output + $DOCKERCOMPOSE logs --no-color agent >& agent.log + $DOCKERCOMPOSE logs --no-color web >& web.log + $DOCKERCOMPOSE logs --no-color mtest >& mtest.log + $DOCKERCOMPOSE logs --no-color punchingbag >& punchingbag.log + + # delete everything + $DOCKERCOMPOSE down + + # show output of module testing + cat mtest.log +} +trap cleanup 0 1 2 3 6 + +set -x + +# attempt to clean up any leftover junk +$DOCKERCOMPOSE down + +$DOCKERCOMPOSE pull --ignore-pull-failures + +# start everything, run tests +# +# --no-color --> safe for ci +# --build --> alway build test server/module container +# --abort-on-container-exit --> without this, the other servers keep the process running +# --exit-code-from mtest --> make exit code be the result of module test +# +# > /dev/null --> output of all servers is mixed together and ugly +# we get the individual logs at end +# +$DOCKERCOMPOSE up --no-color --build --abort-on-container-exit --exit-code-from mtest > /dev/null + diff --git a/scripts/test-golang118/test.sh b/scripts/test-golang118/test.sh index b365a19..b703739 100755 --- a/scripts/test-golang118/test.sh +++ b/scripts/test-golang118/test.sh @@ -37,8 +37,5 @@ $DOCKERCOMPOSE pull --ignore-pull-failures # > /dev/null --> output of all servers is mixed together and ugly # we get the individual logs at end # -if [ -d "goroot" ]; then - rm -rf goroot -fi $DOCKERCOMPOSE up --no-color --build --abort-on-container-exit --exit-code-from mtest > /dev/null diff --git a/scripts/test.sh b/scripts/test.sh index cc242cd..636ad15 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -2,5 +2,6 @@ set -ex +(cd ./scripts/test-golang117 && ./test.sh) (cd ./scripts/test-golang118 && ./test.sh) (cd ./scripts/test-golang119 && ./test.sh) From b756ab48aa1ca3e55b0571dc90a752e8aee2eaf6 Mon Sep 17 00:00:00 2001 From: Edward Muller Date: Mon, 8 Aug 2022 16:02:15 -0700 Subject: [PATCH 3/5] Remove nearly duplicte file --- scripts/test | 6 ------ 1 file changed, 6 deletions(-) delete mode 100755 scripts/test diff --git a/scripts/test b/scripts/test deleted file mode 100755 index cc242cd..0000000 --- a/scripts/test +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -set -ex - -(cd ./scripts/test-golang118 && ./test.sh) -(cd ./scripts/test-golang119 && ./test.sh) From a0c3d3e13dfc53623ef06cc134e82f1d39c44141 Mon Sep 17 00:00:00 2001 From: Edward Muller Date: Tue, 9 Aug 2022 09:08:04 -0700 Subject: [PATCH 4/5] drop down to 1.13 for now --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d9ebd87..8f60d90 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/signalsciences/sigsci-module-golang -go 1.17 +go 1.13 require ( github.com/signalsciences/tlstext v1.2.0 From 290ff97204b93c4e788db2fe9b5375155fe122d5 Mon Sep 17 00:00:00 2001 From: Edward Muller Date: Tue, 9 Aug 2022 09:29:53 -0700 Subject: [PATCH 5/5] fix install info, remove internal testing info --- README.md | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/README.md b/README.md index 8dd6209..febcabf 100644 --- a/README.md +++ b/README.md @@ -22,16 +22,7 @@ git remote set-head origin -a ## Installation ```console -cd $(go env GOPATH)/src/github.com/signalsciences -git clone https://github.com/signalsciences/sigsci-module-golang.git -``` - -## Running tests - -Must be logged into our ECR repo - -```console -aws ecr get-login-password --region us-west-2 --profile sigsci_prod | docker login --username AWS --password-stdin 803688608479.dkr.ecr.us-west-2.amazonaws.com +go get github.com/signalsciences/sigsci-module-golang@latest ``` ## Example Code Snippet