From 7653e47908d54539f12955f8511b774e4e0c690b Mon Sep 17 00:00:00 2001 From: leaf Date: Mon, 5 Aug 2019 20:09:11 -0700 Subject: [PATCH] Re-import of release 1.6.1 for public consumption. --- .gitignore | 28 + CHANGELOG.md | 82 ++ Dockerfile | 9 + Dockerfile.git | 2 + LICENSE.md | 24 + Makefile | 31 + README.md | 76 + VERSION | 1 + clientcodec.go | 170 +++ examples/helloworld/main.go | 68 + examples/mtest/main.go | 128 ++ inspector.go | 39 + make-jenkins.sh | 45 + module.go | 518 +++++++ module_test.go | 395 +++++ responsewriter.go | 98 ++ responsewriter_test.go | 114 ++ rpc.go | 49 + rpc_gen.go | 1272 +++++++++++++++++ rpcinspector.go | 125 ++ scripts/build-docker.sh | 6 + scripts/build.sh | 46 + scripts/test-golang110/Dockerfile | 7 + .../docker-compose.override.yml | 15 + scripts/test-golang110/docker-compose.yml | 57 + scripts/test-golang110/test.sh | 46 + scripts/test-golang111/Dockerfile | 7 + .../docker-compose.override.yml | 15 + scripts/test-golang111/docker-compose.yml | 57 + scripts/test-golang111/test.sh | 46 + scripts/test.sh | 6 + version.go | 3 + 32 files changed, 3585 insertions(+) create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 Dockerfile create mode 100644 Dockerfile.git create mode 100644 LICENSE.md create mode 100644 Makefile create mode 100644 README.md create mode 100644 VERSION create mode 100644 clientcodec.go create mode 100644 examples/helloworld/main.go create mode 100644 examples/mtest/main.go create mode 100644 inspector.go create mode 100755 make-jenkins.sh create mode 100644 module.go create mode 100644 module_test.go create mode 100644 responsewriter.go create mode 100644 responsewriter_test.go create mode 100644 rpc.go create mode 100644 rpc_gen.go create mode 100644 rpcinspector.go create mode 100755 scripts/build-docker.sh create mode 100755 scripts/build.sh create mode 100644 scripts/test-golang110/Dockerfile create mode 100644 scripts/test-golang110/docker-compose.override.yml create mode 100644 scripts/test-golang110/docker-compose.yml create mode 100755 scripts/test-golang110/test.sh create mode 100644 scripts/test-golang111/Dockerfile create mode 100644 scripts/test-golang111/docker-compose.override.yml create mode 100644 scripts/test-golang111/docker-compose.yml create mode 100755 scripts/test-golang111/test.sh create mode 100755 scripts/test.sh create mode 100644 version.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9081e51 --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +*~ +*.log + +artifacts +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f8f0b4a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,82 @@ +# GoLang Module Release Notes + +## Unreleased + + +## 1.6.1 2019-06-13 + +* Cleaned up internal code + +## 1.6.0 2019-05-30 + +* Updated list of inspectable XML content types +* Added `http.Flusher` interface when the underlying handler supports this interface +* Updated timeout to include time to connect to the agent +* Cleaned up docs/code/examples + +## 1.5.0 2019-01-31 + +* Switched Update / Post RPC call to async +* Internal release for agent reverse proxy + +## 1.4.3 2018-08-07 + +* Improved error and debug messages +* Exposed more functionality to allow easier extending + + +## 1.4.2 2018-06-15 +* Improved handling of the `Host` request header +* Improved debugging output + +## 1.4.1 2018-06-04 +* Improved error and debug messages + +## 1.4.0 2018-05-24 + +* Standardized release notes +* Added support for multipart/form-data post +* Extended architecture to allow more flexibility +* Updated response writer interface to allow for WebSocket use +* Removed default filters on CONNECT/OPTIONS methods - now inspected by default +* Standardized error page +* Updated to contact agent on init for faster module registration + +## 1.3.1 2017-09-25 + +* Removed unused dependency +* Removed internal testing example + +## 1.3.0 2017-09-19 + +* Improved internal testing +* Updated msgpack serialization + +## 1.2.3 2017-09-11 + +* Standardized defaults across modules and document +* Bad release + +## 1.2.2 2017-07-02 + +* Updated to use [signalsciences/tlstext](https://github.com/signalsciences/tlstext) + +## 1.2.1 2017-03-21 + +* Added ability to send XML post bodies to agent +* Improved content-type processing + +## 1.2.0 2017-03-06 + +* Improved performance +* Exposed internal datastructures and methods + to allow alternative module implementations and + performance tests + +## 1.1.0 2017-02-28 + +* Fixed TCP vs. UDS configuration + +## 0.1.0 2016-09-02 + +* Initial release diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7dfed40 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM golang:1.10.6-alpine3.8 + +COPY goroot/ /go/ +# this is used to lint and build tarball +RUN gometalinter --install --debug + +# we will mount the current directory here +VOLUME [ "/go/src/github.com/signalsciences/sigsci-module-golang" ] +WORKDIR /go/src/github.com/signalsciences/sigsci-module-golang diff --git a/Dockerfile.git b/Dockerfile.git new file mode 100644 index 0000000..89cdcee --- /dev/null +++ b/Dockerfile.git @@ -0,0 +1,2 @@ +FROM golang:1.10.6-alpine3.8 +RUN apk --update add git diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..d0cd441 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,24 @@ +# sigsci-module-golang + +The MIT License (MIT) + +Copyright (c) 2019 Signal Sciences Corp. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9142d9d --- /dev/null +++ b/Makefile @@ -0,0 +1,31 @@ + + +build: ## build and lint locally + ./scripts/build.sh + +# clean up each time to make sure nothing is cached between runs +# +test: ## build and run integration test + ./scripts/test.sh + +init: ## install gometalinter and msgp locally + go get -u github.com/alecthomas/gometalinter + gometalinter --install --debug + go get -u github.com/tinylib/msgp/msgp + go get . + + +clean: ## cleanup + find . -name 'goroot' -type d | xargs rm -rf + rm -rf artifacts + find . -name '*.log' | xargs rm -f + go clean ./... + git gc + +# https://www.client9.com/self-documenting-makefiles/ +help: + @awk -F ':|##' '/^[^\t].+?:.*?##/ {\ + printf "\033[36m%-30s\033[0m %s\n", $$1, $$NF \ + }' $(MAKEFILE_LIST) +.DEFAULT_GOAL=help +.PHONY=help diff --git a/README.md b/README.md new file mode 100644 index 0000000..478e056 --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +# sigsci-module-golang + +The Signal Sciences module in golang allows for integrating your golang +application directly with the Signal Sciences agent at the source code +level. This golang module is written as a `http.Handler` wrapper. To +integrate your application with the module, you will need to wrap your +existing handler with the module handler. + +Example Code Snippet: +```go +// Existing http.Handler +mux := http.NewServeMux() +mux.HandleFunc("/", helloworld) + +// Wrap the existing http.Handler with the SigSci module handler +wrapped, err := sigsci.NewModule( + // Existing handler to wrap + mux, + + // Any additional module options: + //sigsci.Socket("unix", "/var/run/sigsci.sock"), + //sigsci.Timeout(100 * time.Millisecond), + //sigsci.AnomalySize(512 * 1024), + //sigsci.AnomalyDuration(1 * time.Second), + //sigsci.MaxContentLength(100000), +) +if err != nil { + log.Fatal(err) +} + +// Listen and Serve as usual using the wrapped sigsci handler +s := &http.Server{ + Handler: wrapped, + Addr: "localhost:8000", +} +log.Fatal(s.ListenAndServe()) +``` + +## Dependencies + +The golang module requires two prerequisite packages to be installed: +[MessagePack Code Generator](https://github.com/tinylib/msgp/) and the +Signal Sciences custom [tlstext](https://github.com/signalsciences/tlstext) +package. + +The easiest way to install these packages is by using the `go get` +command to download and install these packages directly from their +public GitHub repositories: + +```bash +go get -u -t github.com/tinylib/msgp/msgp +go get -u -t github.com/signalsciences/tlstext +``` + +## Examples + +The [examples](examples/) directory contains complete example code. + +To run the simple [helloworld](examples/helloworld/main.go) example: + +```bash +go run examples/helloworld/main.go +``` + +Or, if your agent is running with a non-default `rpc-address`, you can +pass the sigsci-agent address as an argument such as one of the following: + +```bash +# Another UNIX Domain socket +go run examples/helloworld/main.go /tmp/sigsci.sock +# A TCP address:port +go run examples/helloworld/main.go localhost:9999 +``` + +This will run a HTTP listener on `localhost:8000`, which will send any +traffic to this listener to a running sigsci-agent for inspection. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..fdd3be6 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.6.2 diff --git a/clientcodec.go b/clientcodec.go new file mode 100644 index 0000000..d8dd5ef --- /dev/null +++ b/clientcodec.go @@ -0,0 +1,170 @@ +package sigsci + +import ( + "fmt" + "io" + "net" + "net/rpc" + + "github.com/tinylib/msgp/msgp" +) + +// Adaptors for golang's RPC mechanism using MSGPACK +// +// * http://msgpack.org +// * https://golang.org/pkg/net/rpc/ +// + +// defines the MSGPACK RPC format +type msgpClientCodec struct { + dec *msgp.Reader + enc *msgp.Writer + c io.Closer +} + +// NewMsgpClientCodec creates a new rpc.ClientCodec from an existing connection +func NewMsgpClientCodec(conn io.ReadWriteCloser) rpc.ClientCodec { + return msgpClientCodec{ + dec: msgp.NewReader(conn), + enc: msgp.NewWriter(conn), + c: conn, + } +} + +func (c msgpClientCodec) Close() error { + return c.c.Close() +} + +func (c msgpClientCodec) WriteRequest(r *rpc.Request, x interface{}) error { + if err := c.enc.WriteArrayHeader(4); err != nil { + return fmt.Errorf("WriteRequest failed in writing array header: %s", err) + } + + if err := c.enc.WriteInt(0); err != nil { + return fmt.Errorf("WriteRequest failed in requiting rpc msg type 0: %s", err) + } + + if err := c.enc.WriteUint32(uint32(r.Seq)); err != nil { + return fmt.Errorf("WriteRequest failed in writing sequence id: %s", err) + } + + if err := c.enc.WriteString(r.ServiceMethod); err != nil { + return fmt.Errorf("WriteRequest failed in writing service method: %s", err) + } + + if err := c.enc.WriteArrayHeader(1); err != nil { + return fmt.Errorf("WriteRequest failed in writing arg array header: %s", err) + } + + if err := c.enc.WriteIntf(x); err != nil { + return fmt.Errorf("WriteRequest failed in writing %T: %s", x, err) + } + + if err := c.enc.Flush(); err != nil { + return fmt.Errorf("WriteRequest failed in flushing: %s", err) + } + + return nil +} + +func (c msgpClientCodec) ReadResponseHeader(r *rpc.Response) error { + sz, err := c.dec.ReadArrayHeader() + if err != nil || sz != 4 { + if cerr := knownError(err); cerr != nil { + return cerr + } + if err == nil && sz != 4 { + err = fmt.Errorf("invalid array size %d", sz) + } + return fmt.Errorf("ReadResponseHeader failed in initial array: %s", err) + } + + msgtype, err := c.dec.ReadUint() + if err != nil || msgtype != 1 { + if cerr := knownError(err); cerr != nil { + return cerr + } + if err == nil && msgtype != 1 { + err = fmt.Errorf("invalid message type %d", msgtype) + } + return fmt.Errorf("ReadResponseHeader failed in message type: %s", err) + } + + seqID, err := c.dec.ReadUint() + if err != nil { + if cerr := knownError(err); cerr != nil { + return cerr + } + return fmt.Errorf("ReadResponseHeader failed in error type: %s", err) + } + r.Seq = uint64(seqID) + + err = c.dec.ReadNil() + if err != nil { + if cerr := knownError(err); cerr != nil { + return cerr + } + // if there is an error maybe its not nil + // try to read string. if still an error + // then assume its bad response + rawerr, err := c.dec.ReadString() + if err != nil { + if cerr := knownError(err); cerr != nil { + return cerr + } + return fmt.Errorf("ReadResponseHeader failed in error message: %s", err) + } + return fmt.Errorf("remote error: %s", rawerr) + } + return nil +} + +func (c msgpClientCodec) ReadResponseBody(x interface{}) error { + if x == nil { + return nil + } + + // if its a decode-able object, then sort it out. + if obj, ok := x.(msgp.Decodable); ok { + if err := obj.DecodeMsg(c.dec); err != nil { + if cerr := knownError(err); cerr != nil { + return cerr + } + return fmt.Errorf("ReadResponseBody failed in obj decode: %s", err) + } + return nil + } + + // we use a plain "int" for response codes just hardwired this + // case. in future use an object to simplify this. + // + if xint, ok := x.(*int); ok { + val, err := c.dec.ReadInt() + if err != nil { + if cerr := knownError(err); cerr != nil { + return cerr + } + return fmt.Errorf("ReadResponseBody failed in int decode: %s", err) + } + *xint = val + return nil + } + + return fmt.Errorf("ReadResponseBody failed: unable to decode") +} + +// knownError checks the error against known errors, does any fixups and returns an error to indicate it is a known error that should not be handled further as well as the replaced error. A nil error is returned if the original error should be handled further. +func knownError(err error) error { + if err == nil { + return nil + } + if err == io.EOF { + return err + } + if nerr, ok := err.(net.Error); ok { + if nerr.Timeout() { + return nerr + } + } + return nil +} diff --git a/examples/helloworld/main.go b/examples/helloworld/main.go new file mode 100644 index 0000000..e947bb2 --- /dev/null +++ b/examples/helloworld/main.go @@ -0,0 +1,68 @@ +package main + +import ( + "log" + "net/http" + "os" + "strings" + + // Import the module + sigsci "github.com/signalsciences/sigsci-module-golang" +) + +func main() { + // Process sigsci-agent rpc-address if passed + sigsciAgentNetwork := "unix" + sigsciAgentAddress := "/var/run/sigsci.sock" + if len(os.Args) > 1 { + sigsciAgentAddress = os.Args[1] + } + if !strings.Contains(sigsciAgentAddress, "/") { + sigsciAgentNetwork = "tcp" + } + log.Printf("Using sigsci-agent address (pass address as program argument to change): %s:%s", sigsciAgentNetwork, sigsciAgentAddress) + + // Existing handler, in this case a simple http.ServeMux, + // but could be any http.Handler in the application + mux := http.NewServeMux() + mux.HandleFunc("/", helloworld) + + // Wrap the existing http.Handler with the SigSci module handler + wrapped, err := sigsci.NewModule( + // Existing handler to wrap + mux, + + // Any additional module options: + sigsci.Socket(sigsciAgentNetwork, sigsciAgentAddress), + //sigsci.Timeout(100 * time.Millisecond), + //sigsci.AnomalySize(512 * 1024), + //sigsci.AnomalyDuration(1 * time.Second), + //sigsci.MaxContentLength(100000), + + // Turn on debug logging for this example (do not use in production) + sigsci.Debug(true), + ) + if err != nil { + log.Fatal(err) + } + + // Listen and Serve as usual using the wrapped sigsci handler + s := &http.Server{ + Handler: wrapped, + Addr: "localhost:8000", + } + log.Printf("Server URL: http://%s/", s.Addr) + log.Fatal(s.ListenAndServe()) +} + +// helloworld just displays a banner message for testing +func helloworld(w http.ResponseWriter, r *http.Request) { + status := http.StatusOK + w.WriteHeader(status) + w.Write([]byte(` + +Hello World +

Hello World!

+ +`)) +} diff --git a/examples/mtest/main.go b/examples/mtest/main.go new file mode 100644 index 0000000..a99429f --- /dev/null +++ b/examples/mtest/main.go @@ -0,0 +1,128 @@ +package main + +import ( + "bytes" + "io/ioutil" + "log" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "strconv" + "strings" + "time" + + sigsci "github.com/signalsciences/sigsci-module-golang" +) + +var ( + debug = false +) + +func helloworld(w http.ResponseWriter, r *http.Request) { + if debug { + reqbytes, _ := httputil.DumpRequest(r, true) + reqstr := string(reqbytes) + if !strings.HasSuffix(reqstr, "\n") { + reqstr += "\n" + } + log.Printf("REQUEST %s:\n%s", r.Header.Get("Content-Type"), reqstr) + } + delay := 0 + body := []byte("OK") + code := 200 + + var err error + var qs url.Values + if r.URL != nil { + qs, err = url.ParseQuery(r.URL.RawQuery) + } + if qs == nil { + qs = make(url.Values) + } + + if num, err := strconv.Atoi(qs.Get("response_time")); err == nil { + delay = num + } + if num, err := strconv.Atoi(qs.Get("response_code")); err == nil { + code = num + } + if num, err := strconv.Atoi(qs.Get("size")); err == nil { + body = bytes.Repeat([]byte{'a'}, num) + } + if len(qs.Get("echo")) > 0 { + body, err = ioutil.ReadAll(r.Body) + if err != nil { + log.Printf("ioutil.ReadAll erred: %s", err) + } + r.Body = ioutil.NopCloser(bytes.NewReader(body)) + } + + if delay > 0 { + time.Sleep(time.Millisecond * time.Duration(delay)) + } + if code >= 300 && code < 400 { + w.Header().Set("Location", "/foo") + } + w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + + // Populate varX response headers from form values for mtest form processing + if varq := r.FormValue("varq"); len(varq) > 0 { + w.Header().Add("varq", varq) + } + if varb := r.PostFormValue("varb"); len(varb) > 0 { + w.Header().Add("varb", varb) + } + if place := r.PostFormValue("place"); len(place) > 0 { + w.Header().Add("place", place) + } + + if debug { + // Record the response so it can be logged + wrec := httptest.NewRecorder() + for k := range w.Header() { + v := w.Header().Get(k) + wrec.Header().Set(k, v) + } + wrec.WriteHeader(code) + wrec.Write(body) + resp := wrec.Result() + respbytes, _ := httputil.DumpResponse(resp, true) + respstr := string(respbytes) + if !strings.HasSuffix(respstr, "\n") { + respstr += "\n" + } + log.Printf("RESPONSE:\n%s", respstr) + } + w.WriteHeader(code) + w.Write(body) +} + +func main() { + if dbg := os.Getenv("DEBUG"); len(dbg) > 0 && dbg != "0" { + debug = true + } + + mux := http.NewServeMux() + + // "/" is handle everything + mux.HandleFunc("/response", helloworld) + + h, err := sigsci.NewModule(mux, + sigsci.Socket("tcp", "agent:9090"), + sigsci.Timeout(1500*time.Millisecond), + // Match agent defaults and better deal with mtest behavior test defaults + sigsci.MaxContentLength(300*1024), + sigsci.AnomalySize(512*1024), + ) + if err != nil { + log.Fatal(err) + } + s := &http.Server{ + Handler: h, + Addr: "0.0.0.0:8085", + } + log.Fatal(s.ListenAndServe()) +} diff --git a/inspector.go b/inspector.go new file mode 100644 index 0000000..bdcfa75 --- /dev/null +++ b/inspector.go @@ -0,0 +1,39 @@ +package sigsci + +import "net/http" + +// InspectorInitFunc is called to decide if the request should be inspected +// Return true if inspection should occur for the request or false if +// inspection should be bypassed. +type InspectorInitFunc func(*http.Request) bool + +// InspectorFiniFunc is called after any inspection on the request is completed +type InspectorFiniFunc func(*http.Request) + +// Inspector is an interface to implement how the +// module communicates with the inspection engine. +type Inspector interface { + // ModuleInit can be called when the module starts up. This allows the module + // data (e.g., `ModuleVersion`, `ServerVersion`, `ServerFlavor`, etc.) to be + // sent to the collector so that the agent shows up initialized without having + // to wait for data to be sent through the inspector. This should only be called + // once when the app/module starts. + ModuleInit(*RPCMsgIn, *RPCMsgOut) error + // PreRequest is called before the request is processed by the app. The results + // should be analyzed for any anomalies or blocking conditions. In addition, any + // `RequestID` returned in the response should be recorded for future use. + PreRequest(*RPCMsgIn, *RPCMsgOut) error + // PostRequest is called after the request has been processed by the app and the + // response data (e.g., status code, headers, etc.) is available. This should be + // called if there was NOT a `RequestID` in the response to a previous `PreRequest` + // call for the same transaction (if a `RequestID` was in the response, then it + // should be used in an `UpdateRequest` call instead). + PostRequest(*RPCMsgIn, *RPCMsgOut) error + // UpdateRequest is called after the request has been processed by the app and the + // response data (e.g., status code, headers, etc.) is available. This should be used + // instead of a `PostRequest` call when a prior `PreRequest` call for the same + // transaction included a `RequestID`. In this case, this call is updating the data + // collected in the `PreRequest` with the given response data (e.g., status code, + // headers, etc.). + UpdateRequest(*RPCMsgIn2, *RPCMsgOut) error +} diff --git a/make-jenkins.sh b/make-jenkins.sh new file mode 100755 index 0000000..7a970a9 --- /dev/null +++ b/make-jenkins.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +if [ -z "${BUILD_NUMBER}" ]; then + echo "Must be run in Jenkins with BUILD_NUMBER set" + exit 2 +fi + +set -ex + +# build / lint agent in a container +find . -name "goroot" -type d | xargs rm -rf +mkdir goroot +docker build -f Dockerfile.git -t golang-git:1.10.6-alpine3.8 . +docker run --user 1015:1015 -v ${PWD}/goroot:/go/ --rm golang-git:1.10.6-alpine3.8 /bin/sh -c 'go get github.com/signalsciences/tlstext && go get github.com/tinylib/msgp && go get github.com/alecthomas/gometalinter' +./scripts/build-docker.sh + +# run module tests +./scripts/test.sh + +BASE=$PWD +## setup our package properties by distro +PKG_NAME="sigsci-module-golang" +DST_BUCKET="s3://package-build-artifacts/${PKG_NAME}/${BUILD_NUMBER}" +VERSION=$(cat ./VERSION) + + +cd ${BASE} +aws s3 cp \ + --no-follow-symlinks \ + --cache-control="max-age=300" \ + ./artifacts/${PKG_NAME}.tar.gz ${DST_BUCKET}/${PKG_NAME}_${VERSION}.tar.gz + +aws s3 cp \ + --no-follow-symlinks \ + --cache-control="max-age=300" \ + --content-type="text/plain; charset=UTF-8" \ + VERSION ${DST_BUCKET}/VERSION + +aws s3 cp \ + --no-follow-symlinks \ + --cache-control="max-age=300" \ + --content-language="en-US" \ + --content-type="text/markdown; charset=UTF-8" \ + CHANGELOG.md ${DST_BUCKET}/CHANGELOG.md + diff --git a/module.go b/module.go new file mode 100644 index 0000000..201a965 --- /dev/null +++ b/module.go @@ -0,0 +1,518 @@ +package sigsci + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "log" + "net" + "net/http" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "time" + + "github.com/signalsciences/tlstext" +) + +const moduleVersion = "sigsci-module-golang " + version + +// Module is an http.Handler that wraps an existing handler with +// data collection and sends it to the Signal Sciences Agent for +// inspection. +type Module struct { + handler http.Handler + rpcNetwork string + rpcAddress string + debug bool + timeout time.Duration + anomalySize int64 + anomalyDuration time.Duration + maxContentLength int64 + moduleVersion string + serverVersion string + inspector Inspector + inspInit InspectorInitFunc + inspFini InspectorFiniFunc +} + +// ModuleConfigOption is a functional config option for configuring the module +// See: https://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis +type ModuleConfigOption func(*Module) error + +// NewModule wraps an existing http.Handler with one that extracts data and +// sends it to the Signal Sciences Agent for inspection. The module is configured +// via functional options. +func NewModule(h http.Handler, options ...ModuleConfigOption) (*Module, error) { + // The following are the defaults, overridden by passing in functional options + m := Module{ + handler: h, + rpcNetwork: "unix", + rpcAddress: "/var/run/sigsci.sock", + debug: false, + timeout: 100 * time.Millisecond, + anomalySize: 512 * 1024, + anomalyDuration: 1 * time.Second, + maxContentLength: 100000, + moduleVersion: moduleVersion, + serverVersion: runtime.Version(), + } + + // Override defaults from functional options + for _, opt := range options { + err := opt(&m) + if err != nil { + return nil, err + } + } + + // By default, use an RPC based inspector + if m.inspector == nil { + m.inspector = &RPCInspector{ + Network: m.rpcNetwork, + Address: m.rpcAddress, + Timeout: m.timeout, + Debug: m.debug, + } + } + + // Call ModuleInit to initialize the module data, so that the agent is + // registered on module creation + now := time.Now().UTC() + in := RPCMsgIn{ + ModuleVersion: m.moduleVersion, + ServerVersion: m.serverVersion, + ServerFlavor: "", + Timestamp: now.Unix(), + NowMillis: now.UnixNano() / 1e6, + } + out := RPCMsgOut{} + if err := m.inspector.ModuleInit(&in, &out); err != nil { + if m.debug { + log.Println("Error in moduleinit to inspector: ", err.Error()) + } + } + + return &m, nil +} + +// Version returns a SemVer version string +func Version() string { + return version +} + +// Debug turns on debug logging +func Debug(enable bool) ModuleConfigOption { + return func(m *Module) error { + m.debug = enable + return nil + } +} + +// Socket is a function argument to set where to send data to the +// Signal Sciences Agent. The network argument should be `unix` +// or `tcp` and the corresponding address should be either an absolute +// path or an `address:port`, respectively. +func Socket(network, address string) ModuleConfigOption { + return func(m *Module) error { + switch network { + case "unix": + if !filepath.IsAbs(address) { + return errors.New(`address must be an absolute path for network="unix"`) + } + case "tcp": + if _, _, err := net.SplitHostPort(address); err != nil { + return fmt.Errorf(`address must be in "address:port" form for network="tcp": %s`, err) + } + default: + return errors.New(`network must be "tcp" or "unix"`) + } + + m.rpcNetwork = network + m.rpcAddress = address + + return nil + } +} + +// AnomalySize is a function argument to indicate when to send data to +// the inspector if the response was abnormally large +func AnomalySize(size int64) ModuleConfigOption { + return func(m *Module) error { + m.anomalySize = size + return nil + } +} + +// AnomalyDuration is a function argument to indicate when to send data +// to the inspector if the response was abnormally slow +func AnomalyDuration(dur time.Duration) ModuleConfigOption { + return func(m *Module) error { + m.anomalyDuration = dur + return nil + } +} + +// MaxContentLength is a function argument to set the maximum post +// body length that will be processed +func MaxContentLength(size int64) ModuleConfigOption { + return func(m *Module) error { + m.maxContentLength = size + return nil + } +} + +// Timeout is a function argument that sets the maximum time to wait until +// receiving a reply from the inspector. Once this timeout is reached, the +// module will fail open. +func Timeout(t time.Duration) ModuleConfigOption { + return func(m *Module) error { + m.timeout = t + return nil + } +} + +// ModuleIdentifier is a function argument that sets the module name +// and version for custom setups. +// The version should be a sem-version (e.g., "1.2.3") +func ModuleIdentifier(name, version string) ModuleConfigOption { + return func(m *Module) error { + m.moduleVersion = name + " " + version + return nil + } +} + +// ServerIdentifier is a function argument that sets the server +// identifier for custom setups +func ServerIdentifier(id string) ModuleConfigOption { + return func(m *Module) error { + m.serverVersion = id + return nil + } +} + +// CustomInspector is a function argument that sets a custom inspector, +// an optional inspector initializer to decide if inspection should occur, and +// an optional inspector finalizer that can perform any post-inspection steps +func CustomInspector(insp Inspector, init InspectorInitFunc, fini InspectorFiniFunc) ModuleConfigOption { + return func(m *Module) error { + m.inspector = insp + m.inspInit = init + m.inspFini = fini + return nil + } +} + +// ServeHTTP satisfies the http.Handler interface +func (m *Module) ServeHTTP(w http.ResponseWriter, req *http.Request) { + start := time.Now().UTC() + finiwg := sync.WaitGroup{} + + // Use the inspector init/fini functions if available + if m.inspInit != nil && !m.inspInit(req) { + // No inspection is desired, so just defer to the underlying handler + m.handler.ServeHTTP(w, req) + return + } + if m.inspFini != nil { + defer func() { + // Delay the finalizer call until inspection (any pending Post + // or Update call) is complete + go func() { + finiwg.Wait() + m.inspFini(req) + }() + }() + } + + if m.debug { + log.Printf("DEBUG: calling 'RPC.PreRequest' for inspection: method=%s host=%s url=%s", req.Method, req.Host, req.URL) + } + inspin2, out, err := m.inspectorPreRequest(req) + if err != nil { + // Fail open + if m.debug { + log.Printf("ERROR: 'RPC.PreRequest' call failed (failing open): %s", err.Error()) + } + m.handler.ServeHTTP(w, req) + return + } + + rw := NewResponseWriter(w) + + wafresponse := out.WAFResponse + switch wafresponse { + case 406: + status := int(wafresponse) + http.Error(rw, fmt.Sprintf("%d %s\n", status, http.StatusText(status)), status) + case 200: + // continue with normal request + m.handler.ServeHTTP(rw, req) + default: + log.Printf("ERROR: Received invalid response code from inspector (failing open): %d", wafresponse) + // continue with normal request + m.handler.ServeHTTP(rw, req) + } + + duration := time.Since(start) + code := rw.StatusCode() + size := rw.BytesWritten() + + if len(inspin2.RequestID) > 0 { + // Do the UpdateRequest inspection in the background while the foreground hurries the response back to the end-user. + inspin2.ResponseCode = int32(code) + inspin2.ResponseSize = size + inspin2.ResponseMillis = int64(duration / time.Millisecond) + inspin2.HeadersOut = convertHeaders(rw.Header()) + if m.debug { + log.Printf("DEBUG: calling 'RPC.UpdateRequest' due to returned requestid=%s: method=%s host=%s url=%s code=%d size=%d duration=%s", inspin2.RequestID, req.Method, req.Host, req.URL, code, size, duration) + } + finiwg.Add(1) // Inspection finializer will wait for this goroutine + go func() { + defer finiwg.Done() + if err := m.inspectorUpdateRequest(inspin2); err != nil && m.debug { + log.Printf("ERROR: 'RPC.UpdateRequest' call failed: %s", err.Error()) + } + }() + } else if code >= 300 || size >= m.anomalySize || duration >= m.anomalyDuration { + // Do the PostRequest inspection in the background while the foreground hurries the response back to the end-user. + if m.debug { + log.Printf("DEBUG: calling 'RPC.PostRequest' due to anomaly: method=%s host=%s url=%s code=%d size=%d duration=%s", req.Method, req.Host, req.URL, code, size, duration) + } + inspin := NewRPCMsgIn(req, nil, code, size, duration, m.moduleVersion, m.serverVersion) + inspin.WAFResponse = wafresponse + inspin.HeadersOut = convertHeaders(rw.Header()) + + finiwg.Add(1) // Inspection finializer will wait for this goroutine + go func() { + defer finiwg.Done() + if err := m.inspectorPostRequest(inspin); err != nil && m.debug { + log.Printf("ERROR: 'RPC.PostRequest' call failed: %s", err.Error()) + } + }() + } +} + +// Inspector returns the configured inspector +func (m *Module) Inspector() Inspector { + return m.inspector +} + +// Version returns the module version string +func (m *Module) Version() string { + return m.moduleVersion +} + +// ServerVersion returns the server version string +func (m *Module) ServerVersion() string { + return m.serverVersion +} + +// inspectorPreRequest reads the body if required and makes a prerequest call to the inspector +func (m *Module) inspectorPreRequest(req *http.Request) (inspin2 RPCMsgIn2, out RPCMsgOut, err error) { + // Create message to the inspector from the input request + // see if we can read-in the post body + + var reqbody []byte + if shouldReadBody(req, m) { + // Read all of it and close + // if error, just keep going + // It's possible that it is an error event + // but not sure what it is. Likely + // the client disconnected. + reqbody, _ = ioutil.ReadAll(req.Body) + req.Body.Close() + + // make a new reader so the next handler + // can still read the post normally as if + // nothing happened + req.Body = ioutil.NopCloser(bytes.NewBuffer(reqbody)) + } + + inspin := NewRPCMsgIn(req, reqbody, -1, -1, -1, m.moduleVersion, m.serverVersion) + + if m.debug { + log.Printf("DEBUG: Making PreRequest call to inspector: %s %s", inspin.Method, inspin.URI) + } + + err = m.inspector.PreRequest(inspin, &out) + if err != nil { + if m.debug { + log.Printf("DEBUG: PreRequest call error (%s %s): %s", inspin.Method, inspin.URI, err) + } + return + } + + // set any request headers + if out.RequestID != "" { + req.Header.Add("X-Sigsci-Requestid", out.RequestID) + } + + wafresponse := out.WAFResponse + req.Header.Add("X-Sigsci-Agentresponse", strconv.Itoa(int(wafresponse))) + for _, kv := range out.RequestHeaders { + req.Header.Add(kv[0], kv[1]) + } + + inspin2 = RPCMsgIn2{ + RequestID: out.RequestID, + ResponseCode: -1, + ResponseMillis: -1, + ResponseSize: -1, + } + + if m.debug { + tags := req.Header.Get("X-Sigsci-Tags") + log.Printf("DEBUG: PreRequest call (%s %s): %d RequestID=%s Tags=%v", inspin.Method, inspin.URI, wafresponse, out.RequestID, tags) + } + + return +} + +// inspectorPostRequest makes a postrequest call to the inspector +func (m *Module) inspectorPostRequest(inspin *RPCMsgIn) error { + // Create message to agent from the input request + + if m.debug { + log.Printf("DEBUG: Making PostRequest call to inspector: %s %s", inspin.Method, inspin.URI) + } + + // NOTE: Currently the output argument is not used + err := m.inspector.PostRequest(inspin, &RPCMsgOut{}) + if err != nil { + if m.debug { + log.Printf("DEBUG: PostRequest call error (%s %s): %s", inspin.Method, inspin.URI, err) + } + } + + return err +} + +// inspectorUpdateRequest makes an updaterequest call to the inspector +func (m *Module) inspectorUpdateRequest(inspin RPCMsgIn2) error { + if m.debug { + log.Printf("DEBUG: Making UpdateRequest call to inspector: RequestID=%s", inspin.RequestID) + } + + // NOTE: Currently the output argument is not used + err := m.inspector.UpdateRequest(&inspin, &RPCMsgOut{}) + if err != nil { + if m.debug { + log.Printf("DEBUG: UpdateRequest call error (RequestID=%s): %s", inspin.RequestID, err) + } + } + + return err +} + +// NewRPCMsgIn creates a message from a go http.Request object +// End-users of the golang module never need to use this +// directly and it is only exposed for performance testing +func NewRPCMsgIn(r *http.Request, postbody []byte, code int, size int64, dur time.Duration, module, server string) *RPCMsgIn { + now := time.Now().UTC() + + // assemble a message to send to inspector + tlsProtocol := "" + tlsCipher := "" + scheme := "http" + if r.TLS != nil { + // convert golang/spec integers into something human readable + scheme = "https" + tlsProtocol = tlstext.Version(r.TLS.Version) + tlsCipher = tlstext.CipherSuite(r.TLS.CipherSuite) + } + + // golang removes Host header from req.Header map and + // promotes it to r.Host field. Add it back as the first header. + hin := convertHeaders(r.Header) + if len(r.Host) > 0 { + hin = append([][2]string{{"Host", r.Host}}, hin...) + } + + return &RPCMsgIn{ + ModuleVersion: module, + ServerVersion: server, + ServerName: r.Host, + Timestamp: now.Unix(), + NowMillis: now.UnixNano() / 1e6, + RemoteAddr: stripPort(r.RemoteAddr), + Method: r.Method, + Scheme: scheme, + URI: r.RequestURI, + Protocol: r.Proto, + TLSProtocol: tlsProtocol, + TLSCipher: tlsCipher, + ResponseCode: int32(code), + ResponseMillis: int64(dur / time.Millisecond), + ResponseSize: size, + PostBody: string(postbody), + HeadersIn: hin, + } +} + +// stripPort removes any port from an address (e.g., the client port from the RemoteAddr) +func stripPort(ipdots string) string { + host, _, err := net.SplitHostPort(ipdots) + if err != nil { + return ipdots + } + return host +} + +// shouldReadBody returns true if the body should be read +func shouldReadBody(req *http.Request, m *Module) bool { + // nothing to do + if req.Body == nil { + return false + } + + // skip reading if post is invalid or too long + if req.ContentLength <= 0 || req.ContentLength > m.maxContentLength { + return false + } + + // only read certain types of content + return inspectableContentType(req.Header.Get("Content-Type")) +} + +// inspectableContentType returns true for an inspectable content type +func inspectableContentType(s string) bool { + s = strings.ToLower(s) + switch { + + // Form + case strings.HasPrefix(s, "application/x-www-form-urlencoded"): + return true + case strings.HasPrefix(s, "multipart/form-data"): + return true + + // JSON + case strings.Contains(s, "json") || + strings.Contains(s, "javascript"): + return true + + // XML + case strings.HasPrefix(s, "text/xml") || + strings.HasPrefix(s, "application/xml") || + strings.Contains(s, "+xml"): + return true + } + + return false +} + +// converts a http.Header map to a [][2]string +func convertHeaders(h http.Header) [][2]string { + // get headers + out := make([][2]string, 0, len(h)+1) + + for key, values := range h { + for _, value := range values { + out = append(out, [2]string{key, value}) + } + } + return out +} diff --git a/module_test.go b/module_test.go new file mode 100644 index 0000000..99632ad --- /dev/null +++ b/module_test.go @@ -0,0 +1,395 @@ +package sigsci + +import ( + "bufio" + "bytes" + "crypto/tls" + "fmt" + "net/http" + "net/http/httptest" + "net/http/httputil" + "reflect" + "strconv" + "strings" + "testing" + "time" +) + +func TestNewRPCMsgFromRequest(t *testing.T) { + b := bytes.Buffer{} + b.WriteString("test") + r, err := http.NewRequest("GET", "http://localhost/", &b) + if err != nil { + t.Fatal(err) + } + r.RemoteAddr = "127.0.0.1" + r.Header.Add("If-None-Match", `W/"wyzzy"`) + r.RequestURI = "http://localhost/" + r.TLS = &tls.ConnectionState{} + + want := RPCMsgIn{ + ServerName: "localhost", + Method: "GET", + Scheme: "https", + URI: "http://localhost/", + Protocol: "HTTP/1.1", + RemoteAddr: "127.0.0.1", + HeadersIn: [][2]string{{"Host", "localhost"}, {"If-None-Match", `W/"wyzzy"`}}, + } + eq := func(got, want RPCMsgIn) (ne string, equal bool) { + switch { + case got.ServerName != want.ServerName: + return "ServerHostname", false + case got.Method != want.Method: + return "Method", false + case got.Scheme != want.Scheme: + return "Scheme", false + case got.URI != want.URI: + return "URI", false + case got.Protocol != want.Protocol: + return "Protocol", false + case got.RemoteAddr != want.RemoteAddr: + return "RemoteAddr", false + case !reflect.DeepEqual(got.HeadersIn, want.HeadersIn): + return "HeadersIn", false + default: + return "", true + } + } + + got := NewRPCMsgIn(r, nil, -1, -1, -1, "", "") + if ne, equal := eq(*got, want); !equal { + t.Errorf("NewRPCMsgIn: incorrect %q", ne) + } +} + +// helper functions + +func TestStripPort(t *testing.T) { + cases := []struct { + want string + content string + }{ + // Invalid, should not change + {"", ""}, + {"foo:bar:baz", "foo:bar:baz"}, + // Valid, should have port removed if exists + {"127.0.0.1", "127.0.0.1"}, + {"127.0.0.1", "127.0.0.1:8000"}, + } + for pos, tt := range cases { + got := stripPort(tt.content) + if got != tt.want { + t.Errorf("test %d: StripPort(%q) = %q, want %q", pos, tt.content, got, tt.want) + } + } +} + +func TestShouldReadBody(t *testing.T) { + m, err := NewModule( + http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + status := http.StatusOK + http.Error(w, fmt.Sprintf("%d %s\n", status, http.StatusText(status)), status) + }), + MaxContentLength(20), + ) + if err != nil { + t.Fatalf("Failed to create module: %s", err) + } + + cases := []struct { + want bool + genreq func() []byte + }{ + // No C-T + {false, func() []byte { + return genTestRequest("GET", "http://example.com/", "", "") + }}, + // Invalid C-T + {false, func() []byte { + return genTestRequest("GET", "http://example.com/", "bad/type", `{}`) + }}, + // Zero length + {false, func() []byte { + return genTestRequest("GET", "http://example.com/", "application/json", ``) + }}, + // Too long + {false, func() []byte { + return genTestRequest("GET", "http://example.com/", "application/json", `{"foo":"12345678901234567890"}`) + }}, + // Good to read + {true, func() []byte { + return genTestRequest("GET", "http://example.com/", "application/json", `{}`) + }}, + } + + for pos, tt := range cases { + req, err := requestParseRaw("127.0.0.1:59000", tt.genreq()) + if err != nil { + t.Fatalf("Failed to generate request: %s", err) + } + got := shouldReadBody(req, m) + if got != tt.want { + t.Errorf("test %d: expected %v got %v", pos, tt.want, got) + } + } +} + +func TestConvertHeaders(t *testing.T) { + cases := []struct { + want [][2]string // Only the order of like keys matters + content http.Header // Order of values matter + }{ + // Empty + { + [][2]string{}, + http.Header{}, + }, + // Single values + { + [][2]string{ + {http.CanonicalHeaderKey("a"), "val a"}, + {http.CanonicalHeaderKey("b"), "val b"}, + }, http.Header{ + http.CanonicalHeaderKey("a"): {"val a"}, + http.CanonicalHeaderKey("b"): {"val b"}, + }, + }, + // Multiple values + { + [][2]string{ + {http.CanonicalHeaderKey("a"), "val a"}, + {http.CanonicalHeaderKey("b"), "val b1"}, + {http.CanonicalHeaderKey("b"), "val b2"}, + }, http.Header{ + http.CanonicalHeaderKey("a"): {"val a"}, + http.CanonicalHeaderKey("b"): {"val b1", "val b2"}, + }, + }, + } + + for pos, tt := range cases { + got := convertHeaders(tt.content) + + // Convert result back to a http.Header for comparison + hmap := http.Header{} + for _, v := range got { + hmap.Add(v[0], v[1]) + } + if !reflect.DeepEqual(tt.content, hmap) { + t.Errorf("test %d: expected %#v, got %#v", pos, tt.content, hmap) + } + } +} + +func TestInspectableContentType(t *testing.T) { + cases := []struct { + want bool + content string + }{ + {true, "application/x-www-form-urlencoded"}, + {true, "application/x-www-form-urlencoded; charset=UTF-8"}, + {true, "multipart/form-data"}, + {true, "text/xml"}, + {true, "application/xml"}, + {true, "text/xml;charset=UTF-8"}, + {true, "application/xml; charset=iso-2022-kr"}, + {true, "application/rss+xml"}, + {true, "application/json"}, + {true, "application/x-javascript"}, + {true, "text/javascript"}, + {true, "text/x-javascript"}, + {true, "text/x-json"}, + {true, "application/javascript"}, + {false, "octet/stream"}, + {false, "junk/yard"}, + } + + for pos, tt := range cases { + got := inspectableContentType(tt.content) + if got != tt.want { + t.Errorf("test %d: expected %v got %v", pos, tt.want, got) + } + } +} + +func TestModule(t *testing.T) { + cases := []struct { + req []byte // Raw HTTP request + resp int32 // Inspection response (200 or 406) + tags string // Any tags in the PreRequest call + }{ + {genTestRequest("GET", "http://example.com/", "", ""), 200, ""}, + {genTestRequest("GET", "http://example.com/", "", ""), 406, "XSS"}, + {genTestRequest("GET", "http://example.com/", "", ""), 200, ""}, + {genTestRequest("OPTIONS", "http://example.com/", "", ""), 200, ""}, + {genTestRequest("OPTIONS", "http://example.com/", "", ""), 406, "XSS"}, + {genTestRequest("OPTIONS", "http://example.com/", "", ""), 200, ""}, + {genTestRequest("CONNECT", "http://example.com/", "", ""), 200, ""}, + {genTestRequest("CONNECT", "http://example.com/", "", ""), 406, "XSS"}, + {genTestRequest("CONNECT", "http://example.com/", "", ""), 200, ""}, + {genTestRequest("POST", "http://example.com/", "application/x-www-form-urlencoded", "a=1"), 200, ""}, + {genTestRequest("POST", "http://example.com/", "application/x-www-form-urlencoded", "a=1"), 406, "XSS"}, + {genTestRequest("POST", "http://example.com/", "application/x-www-form-urlencoded", "a=1"), 200, ""}, + {genTestRequest("PUT", "http://example.com/", "application/x-www-form-urlencoded", "a=1"), 200, ""}, + {genTestRequest("PUT", "http://example.com/", "application/x-www-form-urlencoded", "a=1"), 406, "XSS"}, + {genTestRequest("PUT", "http://example.com/", "application/x-www-form-urlencoded", "a=1"), 200, ""}, + {genTestRequest("POST", "http://example.com/", "text/xml;charset=UTF-8", `1`), 200, ""}, + {genTestRequest("POST", "http://example.com/", "text/xml;charset=UTF-8", `1`), 406, "XSS"}, + {genTestRequest("POST", "http://example.com/", "text/xml;charset=UTF-8", `1`), 200, ""}, + {genTestRequest("POST", "http://example.com/", "application/xml; charset=iso-2022-kr", `1`), 200, ""}, + {genTestRequest("POST", "http://example.com/", "application/xml; charset=iso-2022-kr", `1`), 406, "XSS"}, + {genTestRequest("POST", "http://example.com/", "application/xml; charset=iso-2022-kr", `1`), 200, ""}, + {genTestRequest("POST", "http://example.com/", "application/rss+xml", `1`), 200, ""}, + {genTestRequest("POST", "http://example.com/", "application/rss+xml", `1`), 406, "XSS"}, + {genTestRequest("POST", "http://example.com/", "application/rss+xml", `1`), 200, ""}, + } + + for pos, tt := range cases { + respstr := strconv.Itoa(int(tt.resp)) + m, err := NewModule( + http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + status := http.StatusOK + http.Error(w, fmt.Sprintf("%d %s\n", status, http.StatusText(status)), status) + }), + Timeout(500*time.Millisecond), + Debug(true), + CustomInspector(newTestInspector(tt.resp, tt.tags), nil, nil), + ) + if err != nil { + t.Fatalf("test %d: Failed to create module: %s", pos, err) + } + + req, err := requestParseRaw("127.0.0.1:12345", tt.req) + if err != nil { + t.Fatalf("test %d: Failed to parse request: %s\n%s", pos, err, tt.req) + } + + if dump, err := httputil.DumpRequest(req, true); err == nil { + t.Log("CLIENT REQUEST:\n" + string(dump)) + } + + if hv := req.Header.Get(`X-Sigsci-Agentresponse`); hv != "" { + t.Errorf("test %d: unexpected request header %s=%s", pos, `X-Sigsci-Agentresponse`, hv) + } + + w := httptest.NewRecorder() + m.ServeHTTP(w, req) + resp := w.Result() + + if dump, err := httputil.DumpRequest(req, true); err == nil { + t.Log("SERVER REQUEST:\n" + string(dump)) + } + if hv := req.Header.Get(`X-Sigsci-Agentresponse`); hv == "" || hv != respstr { + t.Errorf("test %d: unexpected request header %s=%s, expected %q", pos, `X-Sigsci-Agentresponse`, hv, respstr) + } + if len(tt.tags) > 0 { + if hv := req.Header.Get(`X-Sigsci-Requestid`); hv == "" { + t.Errorf("test %d: expected request header %s=%s", pos, `X-Sigsci-Requestid`, hv) + } + } + + if dump, err := httputil.DumpResponse(resp, true); err == nil { + t.Log("SERVER RESPONSE:\n" + string(dump)) + } + if resp.StatusCode != int(tt.resp) { + t.Errorf("test %d: unexpected status code=%d, expected=%d", pos, resp.StatusCode, tt.resp) + } + + } +} + +func genTestRequest(meth, uri, ctype, payload string) []byte { + var err error + var req *http.Request + + if len(payload) > 0 { + req, err = http.NewRequest(meth, uri, strings.NewReader(payload)) + if err != nil { + panic(err) + } + } else { + req, err = http.NewRequest(meth, uri, nil) + if err != nil { + panic(err) + } + } + + req.Header.Set(`User-Agent`, `SigSci Module Tester/0.1`) + if len(ctype) > 0 { + req.Header.Set(`Content-Type`, ctype) + } + + // This will add some extra headers typically added by the client + dump, err := httputil.DumpRequestOut(req, true) + if err != nil { + panic(err) + } + + return dump +} + +// requestParseRaw creates a request from the given raw HTTP data +func requestParseRaw(raddr string, raw []byte) (*http.Request, error) { + req, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(raw))) + if err != nil { + return nil, err + } + + // Set fields typically set by the server + req.RemoteAddr = raddr + + return req, nil +} + +// testInspector is a custom inspector that calls the simulator +// harness within the golang module +type testInspector struct { + resp int32 // Either 200 or 406 + tags string // EX: "XSS" (csv) +} + +func newTestInspector(resp int32, tags string) *testInspector { + return &testInspector{ + resp: resp, + tags: tags, + } +} + +func (insp *testInspector) ModuleInit(in *RPCMsgIn, out *RPCMsgOut) error { + out.WAFResponse = 200 + out.RequestID = "" + out.RequestHeaders = nil + return nil +} + +func (insp *testInspector) PreRequest(in *RPCMsgIn, out *RPCMsgOut) error { + out.WAFResponse = insp.resp + if len(insp.tags) > 0 { + out.RequestID = "0123456789abcdef01234567" + out.RequestHeaders = [][2]string{ + {"X-SigSci-Tags", insp.tags}, + } + } else { + out.RequestID = "" + out.RequestHeaders = nil + } + + return nil +} + +func (insp *testInspector) PostRequest(in *RPCMsgIn, out *RPCMsgOut) error { + out.WAFResponse = insp.resp + out.RequestID = "" + out.RequestHeaders = nil + + return nil +} + +func (insp *testInspector) UpdateRequest(in *RPCMsgIn2, out *RPCMsgOut) error { + out.WAFResponse = insp.resp + out.RequestID = "" + out.RequestHeaders = nil + + return nil +} diff --git a/responsewriter.go b/responsewriter.go new file mode 100644 index 0000000..dd84a8c --- /dev/null +++ b/responsewriter.go @@ -0,0 +1,98 @@ +package sigsci + +import ( + "bufio" + "fmt" + "net" + "net/http" +) + +// ResponseWriter is a http.ResponseWriter allowing extraction of data needed for inspection +type ResponseWriter interface { + http.ResponseWriter + BaseResponseWriter() http.ResponseWriter + StatusCode() int + BytesWritten() int64 +} + +// ResponseWriterFlusher is a ResponseWriter with a http.Flusher interface +type ResponseWriterFlusher interface { + ResponseWriter + http.Flusher +} + +// NewResponseWriter returns a ResponseWriter or ResponseWriterFlusher depending on the base http.ResponseWriter. +func NewResponseWriter(base http.ResponseWriter) ResponseWriter { + // NOTE: according to net/http docs, if WriteHeader is not called explicitly, + // the first call to Write will trigger an implicit WriteHeader(http.StatusOK). + // this is why the default code is 200 and it only changes if WriteHeader is called. + w := &responseRecorder{ + base: base, + code: 200, + } + if _, ok := w.base.(http.Flusher); ok { + return &responseRecorderFlusher{w} + } + return w +} + +// responseRecorder wraps a base http.ResponseWriter allowing extraction of additional inspection data +type responseRecorder struct { + base http.ResponseWriter + code int + size int64 +} + +// BaseResponseWriter returns the base http.ResponseWriter allowing access if needed +func (w *responseRecorder) BaseResponseWriter() http.ResponseWriter { + return w.base +} + +// StatusCode returns the status code that was used +func (w *responseRecorder) StatusCode() int { + return w.code +} + +// BytesWritten returns the number of bytes written +func (w *responseRecorder) BytesWritten() int64 { + return w.size +} + +// Header returns the header object +func (w *responseRecorder) Header() http.Header { + return w.base.Header() +} + +// WriteHeader writes the header, recording the status code for inspection +func (w *responseRecorder) WriteHeader(status int) { + w.code = status + w.base.WriteHeader(status) +} + +// Write writes data, tracking the length written for inspection +func (w *responseRecorder) Write(b []byte) (int, error) { + w.size += int64(len(b)) + return w.base.Write(b) +} + +// Hijack hijacks the connection from the HTTP handler so that it can be used directly (websockets, etc.) +// NOTE: This will fail if the wrapped http.responseRecorder is not a http.Hijacker. +func (w *responseRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) { + if h, ok := w.base.(http.Hijacker); ok { + return h.Hijack() + } + // Required for WebSockets to work + return nil, nil, fmt.Errorf("response writer (%T) does not implement http.Hijacker", w.base) +} + +// responseRecorderFlusher wraps a base http.ResponseWriter/http.Flusher allowing extraction of additional inspection data +type responseRecorderFlusher struct { + *responseRecorder +} + +// Flush flushes data if the underlying http.ResponseWriter is capable of flushing +func (w *responseRecorderFlusher) Flush() { + if f, ok := w.responseRecorder.base.(http.Flusher); ok { + f.Flush() + } +} diff --git a/responsewriter_test.go b/responsewriter_test.go new file mode 100644 index 0000000..97ba218 --- /dev/null +++ b/responsewriter_test.go @@ -0,0 +1,114 @@ +package sigsci + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" +) + +// testResponseRecorder is a httptest.ResponseRecorder without the Flusher interface +type testResponseRecorder struct { + Recorder *httptest.ResponseRecorder +} + +func (w *testResponseRecorder) Header() http.Header { + return w.Recorder.Header() +} + +func (w *testResponseRecorder) WriteHeader(status int) { + w.Recorder.WriteHeader(status) +} + +func (w *testResponseRecorder) Write(b []byte) (int, error) { + return w.Recorder.Write(b) +} + +// testResponseRecorderFlusher is a httptest.ResponseRecorder with the Flusher interface +type testResponseRecorderFlusher struct { + Recorder *httptest.ResponseRecorder +} + +func (w *testResponseRecorderFlusher) Header() http.Header { + return w.Recorder.Header() +} + +func (w *testResponseRecorderFlusher) WriteHeader(status int) { + w.Recorder.WriteHeader(status) +} + +func (w *testResponseRecorderFlusher) Write(b []byte) (int, error) { + return w.Recorder.Write(b) +} + +func (w *testResponseRecorderFlusher) Flush() { + w.Recorder.Flush() +} + +func testResponseWriter(t *testing.T, w ResponseWriter, flusher bool) { + status := 200 + respbody := []byte("123456") + + req, err := http.NewRequest(http.MethodGet, "http://example.com/", nil) + if err != nil { + t.Fatalf("Failed to generate request: %s", err) + } + + // Grab the recorder from the base response writer + var recorder *httptest.ResponseRecorder + switch rec := w.BaseResponseWriter().(type) { + case *testResponseRecorder: + recorder = rec.Recorder + case *testResponseRecorderFlusher: + recorder = rec.Recorder + default: + panic(fmt.Sprintf("unhandled recorder type: %T", w)) + } + + // This handler writes header/body and then flushes if the writer implements a http.Flusher + handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(status) + w.Write(respbody) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + }) + handler.ServeHTTP(w, req) + + // Verify the response + resp := recorder.Result() + if resp.StatusCode != status { + t.Errorf("Unexpected status code=%d, expected=%d", resp.StatusCode, status) + } + if w.StatusCode() != status { + t.Errorf("Unexpected recorder status code=%d, expected=%d", w.StatusCode(), status) + } + + // Verify body + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Failed to generate request: %s", err) + } + if string(body) != string(respbody) { + t.Errorf("Unexpected response body=%q, expected=%q", body, respbody) + } + if w.BytesWritten() != int64(len(respbody)) { + t.Errorf("Unexpected response size=%d, expected=%d", w.BytesWritten(), len(respbody)) + } + + // Verify expected flushed value + if recorder.Flushed != flusher { + t.Errorf("Unexpected flush=%v, expected %v w=%T recorder=%T", recorder.Flushed, flusher, w, recorder) + } +} + +// TestResponseWriter tests a non-flusher ResponseWriter +func TestResponseWriter(t *testing.T) { + testResponseWriter(t, NewResponseWriter(&testResponseRecorder{httptest.NewRecorder()}), false) +} + +// TestResponseWriterFlusher tests a flusher ResponseWriter +func TestResponseWriterFlusher(t *testing.T) { + testResponseWriter(t, NewResponseWriter(&testResponseRecorderFlusher{httptest.NewRecorder()}), true) +} diff --git a/rpc.go b/rpc.go new file mode 100644 index 0000000..0fd74db --- /dev/null +++ b/rpc.go @@ -0,0 +1,49 @@ +package sigsci + +//go:generate msgp -unexported -tests=false + +// +// This is for messages to and from the agent +// + +// RPCMsgIn is the primary message from the webserver module to the agent +type RPCMsgIn struct { + AccessKeyID string // AccessKeyID optional, what Site does this belong too (deprecated) + ModuleVersion string // The module build version + ServerVersion string // Main server identifier "apache 2.0.46..." + ServerFlavor string // Any other webserver configuration info (optional) + ServerName string // As in request website URL + Timestamp int64 // Start of request in the number of seconds elapsed since January 1, 1970 UTC. + NowMillis int64 // Current time, the number of milliseconds elapsed since January 1, 1970 UTC. + RemoteAddr string // Remote IP Address, from request socket + Method string // GET/POST, etc... + Scheme string // http/https + URI string // /path?query + Protocol string // HTTP protocol + TLSProtocol string // e.g. TLSv1.2 + TLSCipher string // e.g. ECDHE-RSA-AES128-GCM-SHA256 + WAFResponse int32 // Optional + ResponseCode int32 // HTTP Response Status Code, -1 if unknown + ResponseMillis int64 // HTTP Milliseconds - How many milliseconds did the full request take, -1 if unknown + ResponseSize int64 // HTTP Response size, -1 if unknown + HeadersIn [][2]string // HTTP Request headers (slice of name/value pairs); nil ok + HeadersOut [][2]string // HTTP Response headers (slice of name/value pairs); nil ok + PostBody string // HTTP Request body; empty string if none +} + +// RPCMsgOut is sent back to the webserver +type RPCMsgOut struct { + WAFResponse int32 + RequestID string `json:",omitempty"` // Set if the server expects an UpdateRequest with this ID (UUID) + RequestHeaders [][2]string `json:",omitempty"` // Any additional information in the form of additional request headers +} + +// RPCMsgIn2 is a follow-up message from the webserver to the Agent +// Note there is no formal response to this message +type RPCMsgIn2 struct { + RequestID string // The request id (UUID) + ResponseCode int32 // HTTP status code did the webserver send back + ResponseMillis int64 // How many milliseconds did the full request take + ResponseSize int64 // how many bytes did the webserver send back + HeadersOut [][2]string +} diff --git a/rpc_gen.go b/rpc_gen.go new file mode 100644 index 0000000..d2e95dd --- /dev/null +++ b/rpc_gen.go @@ -0,0 +1,1272 @@ +package sigsci + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *RPCMsgIn) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "AccessKeyID": + z.AccessKeyID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "AccessKeyID") + return + } + case "ModuleVersion": + z.ModuleVersion, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ModuleVersion") + return + } + case "ServerVersion": + z.ServerVersion, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ServerVersion") + return + } + case "ServerFlavor": + z.ServerFlavor, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ServerFlavor") + return + } + case "ServerName": + z.ServerName, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ServerName") + return + } + case "Timestamp": + z.Timestamp, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Timestamp") + return + } + case "NowMillis": + z.NowMillis, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "NowMillis") + return + } + case "RemoteAddr": + z.RemoteAddr, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "RemoteAddr") + return + } + case "Method": + z.Method, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Method") + return + } + case "Scheme": + z.Scheme, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Scheme") + return + } + case "URI": + z.URI, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "URI") + return + } + case "Protocol": + z.Protocol, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Protocol") + return + } + case "TLSProtocol": + z.TLSProtocol, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "TLSProtocol") + return + } + case "TLSCipher": + z.TLSCipher, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "TLSCipher") + return + } + case "WAFResponse": + z.WAFResponse, err = dc.ReadInt32() + if err != nil { + err = msgp.WrapError(err, "WAFResponse") + return + } + case "ResponseCode": + z.ResponseCode, err = dc.ReadInt32() + if err != nil { + err = msgp.WrapError(err, "ResponseCode") + return + } + case "ResponseMillis": + z.ResponseMillis, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ResponseMillis") + return + } + case "ResponseSize": + z.ResponseSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ResponseSize") + return + } + case "HeadersIn": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "HeadersIn") + return + } + if cap(z.HeadersIn) >= int(zb0002) { + z.HeadersIn = (z.HeadersIn)[:zb0002] + } else { + z.HeadersIn = make([][2]string, zb0002) + } + for za0001 := range z.HeadersIn { + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "HeadersIn", za0001) + return + } + if zb0003 != uint32(2) { + err = msgp.ArrayError{Wanted: uint32(2), Got: zb0003} + return + } + for za0002 := range z.HeadersIn[za0001] { + z.HeadersIn[za0001][za0002], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "HeadersIn", za0001, za0002) + return + } + } + } + case "HeadersOut": + var zb0004 uint32 + zb0004, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "HeadersOut") + return + } + if cap(z.HeadersOut) >= int(zb0004) { + z.HeadersOut = (z.HeadersOut)[:zb0004] + } else { + z.HeadersOut = make([][2]string, zb0004) + } + for za0003 := range z.HeadersOut { + var zb0005 uint32 + zb0005, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "HeadersOut", za0003) + return + } + if zb0005 != uint32(2) { + err = msgp.ArrayError{Wanted: uint32(2), Got: zb0005} + return + } + for za0004 := range z.HeadersOut[za0003] { + z.HeadersOut[za0003][za0004], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "HeadersOut", za0003, za0004) + return + } + } + } + case "PostBody": + z.PostBody, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "PostBody") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *RPCMsgIn) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 21 + // write "AccessKeyID" + err = en.Append(0xde, 0x0, 0x15, 0xab, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.AccessKeyID) + if err != nil { + err = msgp.WrapError(err, "AccessKeyID") + return + } + // write "ModuleVersion" + err = en.Append(0xad, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + if err != nil { + return + } + err = en.WriteString(z.ModuleVersion) + if err != nil { + err = msgp.WrapError(err, "ModuleVersion") + return + } + // write "ServerVersion" + err = en.Append(0xad, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + if err != nil { + return + } + err = en.WriteString(z.ServerVersion) + if err != nil { + err = msgp.WrapError(err, "ServerVersion") + return + } + // write "ServerFlavor" + err = en.Append(0xac, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x46, 0x6c, 0x61, 0x76, 0x6f, 0x72) + if err != nil { + return + } + err = en.WriteString(z.ServerFlavor) + if err != nil { + err = msgp.WrapError(err, "ServerFlavor") + return + } + // write "ServerName" + err = en.Append(0xaa, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.ServerName) + if err != nil { + err = msgp.WrapError(err, "ServerName") + return + } + // write "Timestamp" + err = en.Append(0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70) + if err != nil { + return + } + err = en.WriteInt64(z.Timestamp) + if err != nil { + err = msgp.WrapError(err, "Timestamp") + return + } + // write "NowMillis" + err = en.Append(0xa9, 0x4e, 0x6f, 0x77, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73) + if err != nil { + return + } + err = en.WriteInt64(z.NowMillis) + if err != nil { + err = msgp.WrapError(err, "NowMillis") + return + } + // write "RemoteAddr" + err = en.Append(0xaa, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x41, 0x64, 0x64, 0x72) + if err != nil { + return + } + err = en.WriteString(z.RemoteAddr) + if err != nil { + err = msgp.WrapError(err, "RemoteAddr") + return + } + // write "Method" + err = en.Append(0xa6, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64) + if err != nil { + return + } + err = en.WriteString(z.Method) + if err != nil { + err = msgp.WrapError(err, "Method") + return + } + // write "Scheme" + err = en.Append(0xa6, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Scheme) + if err != nil { + err = msgp.WrapError(err, "Scheme") + return + } + // write "URI" + err = en.Append(0xa3, 0x55, 0x52, 0x49) + if err != nil { + return + } + err = en.WriteString(z.URI) + if err != nil { + err = msgp.WrapError(err, "URI") + return + } + // write "Protocol" + err = en.Append(0xa8, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c) + if err != nil { + return + } + err = en.WriteString(z.Protocol) + if err != nil { + err = msgp.WrapError(err, "Protocol") + return + } + // write "TLSProtocol" + err = en.Append(0xab, 0x54, 0x4c, 0x53, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c) + if err != nil { + return + } + err = en.WriteString(z.TLSProtocol) + if err != nil { + err = msgp.WrapError(err, "TLSProtocol") + return + } + // write "TLSCipher" + err = en.Append(0xa9, 0x54, 0x4c, 0x53, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72) + if err != nil { + return + } + err = en.WriteString(z.TLSCipher) + if err != nil { + err = msgp.WrapError(err, "TLSCipher") + return + } + // write "WAFResponse" + err = en.Append(0xab, 0x57, 0x41, 0x46, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65) + if err != nil { + return + } + err = en.WriteInt32(z.WAFResponse) + if err != nil { + err = msgp.WrapError(err, "WAFResponse") + return + } + // write "ResponseCode" + err = en.Append(0xac, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x43, 0x6f, 0x64, 0x65) + if err != nil { + return + } + err = en.WriteInt32(z.ResponseCode) + if err != nil { + err = msgp.WrapError(err, "ResponseCode") + return + } + // write "ResponseMillis" + err = en.Append(0xae, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73) + if err != nil { + return + } + err = en.WriteInt64(z.ResponseMillis) + if err != nil { + err = msgp.WrapError(err, "ResponseMillis") + return + } + // write "ResponseSize" + err = en.Append(0xac, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.ResponseSize) + if err != nil { + err = msgp.WrapError(err, "ResponseSize") + return + } + // write "HeadersIn" + err = en.Append(0xa9, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x49, 0x6e) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.HeadersIn))) + if err != nil { + err = msgp.WrapError(err, "HeadersIn") + return + } + for za0001 := range z.HeadersIn { + err = en.WriteArrayHeader(uint32(2)) + if err != nil { + err = msgp.WrapError(err, "HeadersIn", za0001) + return + } + for za0002 := range z.HeadersIn[za0001] { + err = en.WriteString(z.HeadersIn[za0001][za0002]) + if err != nil { + err = msgp.WrapError(err, "HeadersIn", za0001, za0002) + return + } + } + } + // write "HeadersOut" + err = en.Append(0xaa, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x4f, 0x75, 0x74) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.HeadersOut))) + if err != nil { + err = msgp.WrapError(err, "HeadersOut") + return + } + for za0003 := range z.HeadersOut { + err = en.WriteArrayHeader(uint32(2)) + if err != nil { + err = msgp.WrapError(err, "HeadersOut", za0003) + return + } + for za0004 := range z.HeadersOut[za0003] { + err = en.WriteString(z.HeadersOut[za0003][za0004]) + if err != nil { + err = msgp.WrapError(err, "HeadersOut", za0003, za0004) + return + } + } + } + // write "PostBody" + err = en.Append(0xa8, 0x50, 0x6f, 0x73, 0x74, 0x42, 0x6f, 0x64, 0x79) + if err != nil { + return + } + err = en.WriteString(z.PostBody) + if err != nil { + err = msgp.WrapError(err, "PostBody") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *RPCMsgIn) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 21 + // string "AccessKeyID" + o = append(o, 0xde, 0x0, 0x15, 0xab, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x49, 0x44) + o = msgp.AppendString(o, z.AccessKeyID) + // string "ModuleVersion" + o = append(o, 0xad, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + o = msgp.AppendString(o, z.ModuleVersion) + // string "ServerVersion" + o = append(o, 0xad, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + o = msgp.AppendString(o, z.ServerVersion) + // string "ServerFlavor" + o = append(o, 0xac, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x46, 0x6c, 0x61, 0x76, 0x6f, 0x72) + o = msgp.AppendString(o, z.ServerFlavor) + // string "ServerName" + o = append(o, 0xaa, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.ServerName) + // string "Timestamp" + o = append(o, 0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70) + o = msgp.AppendInt64(o, z.Timestamp) + // string "NowMillis" + o = append(o, 0xa9, 0x4e, 0x6f, 0x77, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73) + o = msgp.AppendInt64(o, z.NowMillis) + // string "RemoteAddr" + o = append(o, 0xaa, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x41, 0x64, 0x64, 0x72) + o = msgp.AppendString(o, z.RemoteAddr) + // string "Method" + o = append(o, 0xa6, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64) + o = msgp.AppendString(o, z.Method) + // string "Scheme" + o = append(o, 0xa6, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x65) + o = msgp.AppendString(o, z.Scheme) + // string "URI" + o = append(o, 0xa3, 0x55, 0x52, 0x49) + o = msgp.AppendString(o, z.URI) + // string "Protocol" + o = append(o, 0xa8, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c) + o = msgp.AppendString(o, z.Protocol) + // string "TLSProtocol" + o = append(o, 0xab, 0x54, 0x4c, 0x53, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c) + o = msgp.AppendString(o, z.TLSProtocol) + // string "TLSCipher" + o = append(o, 0xa9, 0x54, 0x4c, 0x53, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72) + o = msgp.AppendString(o, z.TLSCipher) + // string "WAFResponse" + o = append(o, 0xab, 0x57, 0x41, 0x46, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65) + o = msgp.AppendInt32(o, z.WAFResponse) + // string "ResponseCode" + o = append(o, 0xac, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x43, 0x6f, 0x64, 0x65) + o = msgp.AppendInt32(o, z.ResponseCode) + // string "ResponseMillis" + o = append(o, 0xae, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73) + o = msgp.AppendInt64(o, z.ResponseMillis) + // string "ResponseSize" + o = append(o, 0xac, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.ResponseSize) + // string "HeadersIn" + o = append(o, 0xa9, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x49, 0x6e) + o = msgp.AppendArrayHeader(o, uint32(len(z.HeadersIn))) + for za0001 := range z.HeadersIn { + o = msgp.AppendArrayHeader(o, uint32(2)) + for za0002 := range z.HeadersIn[za0001] { + o = msgp.AppendString(o, z.HeadersIn[za0001][za0002]) + } + } + // string "HeadersOut" + o = append(o, 0xaa, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x4f, 0x75, 0x74) + o = msgp.AppendArrayHeader(o, uint32(len(z.HeadersOut))) + for za0003 := range z.HeadersOut { + o = msgp.AppendArrayHeader(o, uint32(2)) + for za0004 := range z.HeadersOut[za0003] { + o = msgp.AppendString(o, z.HeadersOut[za0003][za0004]) + } + } + // string "PostBody" + o = append(o, 0xa8, 0x50, 0x6f, 0x73, 0x74, 0x42, 0x6f, 0x64, 0x79) + o = msgp.AppendString(o, z.PostBody) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *RPCMsgIn) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "AccessKeyID": + z.AccessKeyID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AccessKeyID") + return + } + case "ModuleVersion": + z.ModuleVersion, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ModuleVersion") + return + } + case "ServerVersion": + z.ServerVersion, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ServerVersion") + return + } + case "ServerFlavor": + z.ServerFlavor, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ServerFlavor") + return + } + case "ServerName": + z.ServerName, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ServerName") + return + } + case "Timestamp": + z.Timestamp, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Timestamp") + return + } + case "NowMillis": + z.NowMillis, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "NowMillis") + return + } + case "RemoteAddr": + z.RemoteAddr, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RemoteAddr") + return + } + case "Method": + z.Method, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Method") + return + } + case "Scheme": + z.Scheme, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Scheme") + return + } + case "URI": + z.URI, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "URI") + return + } + case "Protocol": + z.Protocol, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Protocol") + return + } + case "TLSProtocol": + z.TLSProtocol, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TLSProtocol") + return + } + case "TLSCipher": + z.TLSCipher, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TLSCipher") + return + } + case "WAFResponse": + z.WAFResponse, bts, err = msgp.ReadInt32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "WAFResponse") + return + } + case "ResponseCode": + z.ResponseCode, bts, err = msgp.ReadInt32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResponseCode") + return + } + case "ResponseMillis": + z.ResponseMillis, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResponseMillis") + return + } + case "ResponseSize": + z.ResponseSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResponseSize") + return + } + case "HeadersIn": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "HeadersIn") + return + } + if cap(z.HeadersIn) >= int(zb0002) { + z.HeadersIn = (z.HeadersIn)[:zb0002] + } else { + z.HeadersIn = make([][2]string, zb0002) + } + for za0001 := range z.HeadersIn { + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "HeadersIn", za0001) + return + } + if zb0003 != uint32(2) { + err = msgp.ArrayError{Wanted: uint32(2), Got: zb0003} + return + } + for za0002 := range z.HeadersIn[za0001] { + z.HeadersIn[za0001][za0002], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "HeadersIn", za0001, za0002) + return + } + } + } + case "HeadersOut": + var zb0004 uint32 + zb0004, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "HeadersOut") + return + } + if cap(z.HeadersOut) >= int(zb0004) { + z.HeadersOut = (z.HeadersOut)[:zb0004] + } else { + z.HeadersOut = make([][2]string, zb0004) + } + for za0003 := range z.HeadersOut { + var zb0005 uint32 + zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "HeadersOut", za0003) + return + } + if zb0005 != uint32(2) { + err = msgp.ArrayError{Wanted: uint32(2), Got: zb0005} + return + } + for za0004 := range z.HeadersOut[za0003] { + z.HeadersOut[za0003][za0004], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "HeadersOut", za0003, za0004) + return + } + } + } + case "PostBody": + z.PostBody, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PostBody") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *RPCMsgIn) Msgsize() (s int) { + s = 3 + 12 + msgp.StringPrefixSize + len(z.AccessKeyID) + 14 + msgp.StringPrefixSize + len(z.ModuleVersion) + 14 + msgp.StringPrefixSize + len(z.ServerVersion) + 13 + msgp.StringPrefixSize + len(z.ServerFlavor) + 11 + msgp.StringPrefixSize + len(z.ServerName) + 10 + msgp.Int64Size + 10 + msgp.Int64Size + 11 + msgp.StringPrefixSize + len(z.RemoteAddr) + 7 + msgp.StringPrefixSize + len(z.Method) + 7 + msgp.StringPrefixSize + len(z.Scheme) + 4 + msgp.StringPrefixSize + len(z.URI) + 9 + msgp.StringPrefixSize + len(z.Protocol) + 12 + msgp.StringPrefixSize + len(z.TLSProtocol) + 10 + msgp.StringPrefixSize + len(z.TLSCipher) + 12 + msgp.Int32Size + 13 + msgp.Int32Size + 15 + msgp.Int64Size + 13 + msgp.Int64Size + 10 + msgp.ArrayHeaderSize + for za0001 := range z.HeadersIn { + s += msgp.ArrayHeaderSize + for za0002 := range z.HeadersIn[za0001] { + s += msgp.StringPrefixSize + len(z.HeadersIn[za0001][za0002]) + } + } + s += 11 + msgp.ArrayHeaderSize + for za0003 := range z.HeadersOut { + s += msgp.ArrayHeaderSize + for za0004 := range z.HeadersOut[za0003] { + s += msgp.StringPrefixSize + len(z.HeadersOut[za0003][za0004]) + } + } + s += 9 + msgp.StringPrefixSize + len(z.PostBody) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *RPCMsgIn2) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "RequestID": + z.RequestID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "RequestID") + return + } + case "ResponseCode": + z.ResponseCode, err = dc.ReadInt32() + if err != nil { + err = msgp.WrapError(err, "ResponseCode") + return + } + case "ResponseMillis": + z.ResponseMillis, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ResponseMillis") + return + } + case "ResponseSize": + z.ResponseSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ResponseSize") + return + } + case "HeadersOut": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "HeadersOut") + return + } + if cap(z.HeadersOut) >= int(zb0002) { + z.HeadersOut = (z.HeadersOut)[:zb0002] + } else { + z.HeadersOut = make([][2]string, zb0002) + } + for za0001 := range z.HeadersOut { + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "HeadersOut", za0001) + return + } + if zb0003 != uint32(2) { + err = msgp.ArrayError{Wanted: uint32(2), Got: zb0003} + return + } + for za0002 := range z.HeadersOut[za0001] { + z.HeadersOut[za0001][za0002], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "HeadersOut", za0001, za0002) + return + } + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *RPCMsgIn2) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 5 + // write "RequestID" + err = en.Append(0x85, 0xa9, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.RequestID) + if err != nil { + err = msgp.WrapError(err, "RequestID") + return + } + // write "ResponseCode" + err = en.Append(0xac, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x43, 0x6f, 0x64, 0x65) + if err != nil { + return + } + err = en.WriteInt32(z.ResponseCode) + if err != nil { + err = msgp.WrapError(err, "ResponseCode") + return + } + // write "ResponseMillis" + err = en.Append(0xae, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73) + if err != nil { + return + } + err = en.WriteInt64(z.ResponseMillis) + if err != nil { + err = msgp.WrapError(err, "ResponseMillis") + return + } + // write "ResponseSize" + err = en.Append(0xac, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.ResponseSize) + if err != nil { + err = msgp.WrapError(err, "ResponseSize") + return + } + // write "HeadersOut" + err = en.Append(0xaa, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x4f, 0x75, 0x74) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.HeadersOut))) + if err != nil { + err = msgp.WrapError(err, "HeadersOut") + return + } + for za0001 := range z.HeadersOut { + err = en.WriteArrayHeader(uint32(2)) + if err != nil { + err = msgp.WrapError(err, "HeadersOut", za0001) + return + } + for za0002 := range z.HeadersOut[za0001] { + err = en.WriteString(z.HeadersOut[za0001][za0002]) + if err != nil { + err = msgp.WrapError(err, "HeadersOut", za0001, za0002) + return + } + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *RPCMsgIn2) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "RequestID" + o = append(o, 0x85, 0xa9, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x44) + o = msgp.AppendString(o, z.RequestID) + // string "ResponseCode" + o = append(o, 0xac, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x43, 0x6f, 0x64, 0x65) + o = msgp.AppendInt32(o, z.ResponseCode) + // string "ResponseMillis" + o = append(o, 0xae, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73) + o = msgp.AppendInt64(o, z.ResponseMillis) + // string "ResponseSize" + o = append(o, 0xac, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.ResponseSize) + // string "HeadersOut" + o = append(o, 0xaa, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x4f, 0x75, 0x74) + o = msgp.AppendArrayHeader(o, uint32(len(z.HeadersOut))) + for za0001 := range z.HeadersOut { + o = msgp.AppendArrayHeader(o, uint32(2)) + for za0002 := range z.HeadersOut[za0001] { + o = msgp.AppendString(o, z.HeadersOut[za0001][za0002]) + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *RPCMsgIn2) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "RequestID": + z.RequestID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RequestID") + return + } + case "ResponseCode": + z.ResponseCode, bts, err = msgp.ReadInt32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResponseCode") + return + } + case "ResponseMillis": + z.ResponseMillis, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResponseMillis") + return + } + case "ResponseSize": + z.ResponseSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResponseSize") + return + } + case "HeadersOut": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "HeadersOut") + return + } + if cap(z.HeadersOut) >= int(zb0002) { + z.HeadersOut = (z.HeadersOut)[:zb0002] + } else { + z.HeadersOut = make([][2]string, zb0002) + } + for za0001 := range z.HeadersOut { + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "HeadersOut", za0001) + return + } + if zb0003 != uint32(2) { + err = msgp.ArrayError{Wanted: uint32(2), Got: zb0003} + return + } + for za0002 := range z.HeadersOut[za0001] { + z.HeadersOut[za0001][za0002], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "HeadersOut", za0001, za0002) + return + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *RPCMsgIn2) Msgsize() (s int) { + s = 1 + 10 + msgp.StringPrefixSize + len(z.RequestID) + 13 + msgp.Int32Size + 15 + msgp.Int64Size + 13 + msgp.Int64Size + 11 + msgp.ArrayHeaderSize + for za0001 := range z.HeadersOut { + s += msgp.ArrayHeaderSize + for za0002 := range z.HeadersOut[za0001] { + s += msgp.StringPrefixSize + len(z.HeadersOut[za0001][za0002]) + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *RPCMsgOut) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "WAFResponse": + z.WAFResponse, err = dc.ReadInt32() + if err != nil { + err = msgp.WrapError(err, "WAFResponse") + return + } + case "RequestID": + z.RequestID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "RequestID") + return + } + case "RequestHeaders": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "RequestHeaders") + return + } + if cap(z.RequestHeaders) >= int(zb0002) { + z.RequestHeaders = (z.RequestHeaders)[:zb0002] + } else { + z.RequestHeaders = make([][2]string, zb0002) + } + for za0001 := range z.RequestHeaders { + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "RequestHeaders", za0001) + return + } + if zb0003 != uint32(2) { + err = msgp.ArrayError{Wanted: uint32(2), Got: zb0003} + return + } + for za0002 := range z.RequestHeaders[za0001] { + z.RequestHeaders[za0001][za0002], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "RequestHeaders", za0001, za0002) + return + } + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *RPCMsgOut) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "WAFResponse" + err = en.Append(0x83, 0xab, 0x57, 0x41, 0x46, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65) + if err != nil { + return + } + err = en.WriteInt32(z.WAFResponse) + if err != nil { + err = msgp.WrapError(err, "WAFResponse") + return + } + // write "RequestID" + err = en.Append(0xa9, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.RequestID) + if err != nil { + err = msgp.WrapError(err, "RequestID") + return + } + // write "RequestHeaders" + err = en.Append(0xae, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.RequestHeaders))) + if err != nil { + err = msgp.WrapError(err, "RequestHeaders") + return + } + for za0001 := range z.RequestHeaders { + err = en.WriteArrayHeader(uint32(2)) + if err != nil { + err = msgp.WrapError(err, "RequestHeaders", za0001) + return + } + for za0002 := range z.RequestHeaders[za0001] { + err = en.WriteString(z.RequestHeaders[za0001][za0002]) + if err != nil { + err = msgp.WrapError(err, "RequestHeaders", za0001, za0002) + return + } + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *RPCMsgOut) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "WAFResponse" + o = append(o, 0x83, 0xab, 0x57, 0x41, 0x46, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65) + o = msgp.AppendInt32(o, z.WAFResponse) + // string "RequestID" + o = append(o, 0xa9, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x44) + o = msgp.AppendString(o, z.RequestID) + // string "RequestHeaders" + o = append(o, 0xae, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.RequestHeaders))) + for za0001 := range z.RequestHeaders { + o = msgp.AppendArrayHeader(o, uint32(2)) + for za0002 := range z.RequestHeaders[za0001] { + o = msgp.AppendString(o, z.RequestHeaders[za0001][za0002]) + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *RPCMsgOut) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "WAFResponse": + z.WAFResponse, bts, err = msgp.ReadInt32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "WAFResponse") + return + } + case "RequestID": + z.RequestID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RequestID") + return + } + case "RequestHeaders": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RequestHeaders") + return + } + if cap(z.RequestHeaders) >= int(zb0002) { + z.RequestHeaders = (z.RequestHeaders)[:zb0002] + } else { + z.RequestHeaders = make([][2]string, zb0002) + } + for za0001 := range z.RequestHeaders { + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RequestHeaders", za0001) + return + } + if zb0003 != uint32(2) { + err = msgp.ArrayError{Wanted: uint32(2), Got: zb0003} + return + } + for za0002 := range z.RequestHeaders[za0001] { + z.RequestHeaders[za0001][za0002], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RequestHeaders", za0001, za0002) + return + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *RPCMsgOut) Msgsize() (s int) { + s = 1 + 12 + msgp.Int32Size + 10 + msgp.StringPrefixSize + len(z.RequestID) + 15 + msgp.ArrayHeaderSize + for za0001 := range z.RequestHeaders { + s += msgp.ArrayHeaderSize + for za0002 := range z.RequestHeaders[za0001] { + s += msgp.StringPrefixSize + len(z.RequestHeaders[za0001][za0002]) + } + } + return +} diff --git a/rpcinspector.go b/rpcinspector.go new file mode 100644 index 0000000..d21b7e1 --- /dev/null +++ b/rpcinspector.go @@ -0,0 +1,125 @@ +package sigsci + +import ( + "fmt" + "net" + "net/rpc" + "time" +) + +// RPCInspector is an Inspector implemented as RPC calls to the agent +type RPCInspector struct { + Network string + Address string + Timeout time.Duration + Debug bool + InitRPCClientFunc func() (*rpc.Client, error) + FiniRPCClientFunc func(*rpc.Client, error) +} + +// ModuleInit sends a RPC.ModuleInit message to the agent +func (ri *RPCInspector) ModuleInit(in *RPCMsgIn, out *RPCMsgOut) error { + client, err := ri.GetRPCClient() + if err == nil { + err = client.Call("RPC.ModuleInit", in, out) + ri.CloseRPCClient(client, err) + } + + if err != nil { + return fmt.Errorf("RPC.ModuleInit call failed: %s", err) + } + + return nil +} + +// PreRequest sends a RPC.PreRequest message to the agent +func (ri *RPCInspector) PreRequest(in *RPCMsgIn, out *RPCMsgOut) error { + client, err := ri.GetRPCClient() + if err == nil { + err = client.Call("RPC.PreRequest", in, out) + ri.CloseRPCClient(client, err) + } + + if err != nil { + return fmt.Errorf("RPC.PreRequest call failed: %s", err) + } + + return nil +} + +// PostRequest sends a RPC.PostRequest message to the agent +func (ri *RPCInspector) PostRequest(in *RPCMsgIn, out *RPCMsgOut) error { + client, err := ri.GetRPCClient() + if err == nil { + var rpcout int + err = client.Call("RPC.PostRequest", in, &rpcout) + ri.CloseRPCClient(client, err) + + // Always success as the rpcout is not currently used + out.WAFResponse = 200 + out.RequestID = "" + out.RequestHeaders = nil + } + + if err != nil { + return fmt.Errorf("RPC.PostRequest call failed: %s", err) + } + + return nil +} + +// UpdateRequest sends a RPC.UpdateRequest message to the agent +func (ri *RPCInspector) UpdateRequest(in *RPCMsgIn2, out *RPCMsgOut) error { + client, err := ri.GetRPCClient() + if err == nil { + + var rpcout int + err = client.Call("RPC.UpdateRequest", in, &rpcout) + ri.CloseRPCClient(client, err) + + // Always success as the rpcout is not currently used + out.WAFResponse = 200 + out.RequestID = "" + out.RequestHeaders = nil + } + + return err +} + +// GetRPCClient gets a RPC client +func (ri *RPCInspector) GetRPCClient() (*rpc.Client, error) { + if ri.InitRPCClientFunc != nil { + return ri.InitRPCClientFunc() + } + + conn, err := ri.getConnection() + if err != nil { + return nil, err + } + rpcCodec := NewMsgpClientCodec(conn) + return rpc.NewClientWithCodec(rpcCodec), nil +} + +// CloseRPCClient closes a RPC client +func (ri *RPCInspector) CloseRPCClient(client *rpc.Client, err error) { + if ri.FiniRPCClientFunc != nil { + ri.FiniRPCClientFunc(client, err) + return + } + client.Close() +} + +func (ri *RPCInspector) makeConnection() (net.Conn, error) { + deadline := time.Now().Add(ri.Timeout) + conn, err := net.DialTimeout(ri.Network, ri.Address, ri.Timeout) + if err != nil { + return nil, err + } + conn.SetDeadline(deadline) + return conn, nil +} + +func (ri *RPCInspector) getConnection() (net.Conn, error) { + // here for future expansion to use pools, etc. + return ri.makeConnection() +} diff --git a/scripts/build-docker.sh b/scripts/build-docker.sh new file mode 100755 index 0000000..5fed5a3 --- /dev/null +++ b/scripts/build-docker.sh @@ -0,0 +1,6 @@ +#!/bin/sh -ex + +docker build -t foo . + +rm -rf goroot +docker run -v ${PWD}:/go/src/github.com/signalsciences/sigsci-module-golang --rm foo ./scripts/build.sh diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..d915cdc --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,46 @@ +#!/bin/sh +set -ex + +echo "package sigsci" > version.go +echo "" >> version.go +echo "const version = \"$(cat VERSION)\"" >> version.go +find . -name "goroot" -type d | xargs rm -rf +go generate ./... + +# make sure files made in docker are readable by all +chmod a+r *.go + +go build . +go test . + +# --enable=gosimple \ +# --enable=unused \ + +gometalinter \ + --vendor \ + --deadline=60s \ + --disable-all \ + --enable=vetshadow \ + --enable=ineffassign \ + --enable=deadcode \ + --enable=golint \ + --enable=gofmt \ + --enable=vet \ + --exclude=_gen.go \ + --exclude=/usr/local/go/src/net/lookup_unix.go \ + . + +rm -rf artifacts/ +mkdir -p artifacts/sigsci-module-golang +cp -rf \ + VERSION CHANGELOG.md LICENSE.md README.md \ + clientcodec.go rpc.go rpc_gen.go rpcinspector.go inspector.go responsewriter.go module.go version.go \ + responsewriter_test.go module_test.go \ + examples \ + artifacts/sigsci-module-golang/ + +# mtest is internal only +rm -fr artifacts/sigsci-module-golang/examples/mtest + +(cd artifacts; tar -czvf sigsci-module-golang.tar.gz sigsci-module-golang) +chmod a+rw artifacts diff --git a/scripts/test-golang110/Dockerfile b/scripts/test-golang110/Dockerfile new file mode 100644 index 0000000..b43d134 --- /dev/null +++ b/scripts/test-golang110/Dockerfile @@ -0,0 +1,7 @@ +FROM golang:1.10.6-alpine3.8 + +COPY goroot/ /go/ + +# we will mount the current directory here +VOLUME [ "/go/src/github.com/signalsciences/sigsci-module-golang" ] +WORKDIR /go/src/github.com/signalsciences/sigsci-module-golang diff --git a/scripts/test-golang110/docker-compose.override.yml b/scripts/test-golang110/docker-compose.override.yml new file mode 100644 index 0000000..7af37fd --- /dev/null +++ b/scripts/test-golang110/docker-compose.override.yml @@ -0,0 +1,15 @@ +version: "3" + +services: + # this defines our webserver uses our sigsci-module + # we only define it so it is attached to our fake network + # it will be run a few times with different options manually + # + # The volumes spec is a bit weird.. this script is run in scripts/test but + # needs stuff in ../../examples. Consider moving. + web: + volumes: + - ../..:/go/src/github.com/signalsciences/sigsci-module-golang + command: [ "go", "run", "/go/src/github.com/signalsciences/sigsci-module-golang/examples/mtest/main.go" ] + environment: + - DEBUG=0 diff --git a/scripts/test-golang110/docker-compose.yml b/scripts/test-golang110/docker-compose.yml new file mode 100644 index 0000000..ab36aef --- /dev/null +++ b/scripts/test-golang110/docker-compose.yml @@ -0,0 +1,57 @@ +version: "3" +networks: + mtest: + +services: + # this defines our webserver uses our sigsci-module + # we only define it so it is attached to our fake network + # it will be run a few times with different options manually + # + # + web: + build: + context: . + dockerfile: Dockerfile + expose: + - "8085" + networks: + - mtest + depends_on: + - agent + + # agent + agent: + image: local-dev/sigsci-agent:latest + command: [ "-debug-log-web-inputs", "2", "-rpc-address", "9090", "-debug-rpc-test-harness", "-debug-standalone", "3" ] + expose: + - "9090" + - "12345" + networks: + - mtest + + # punching bag + punchingbag: + image: local-dev/module-testing:latest + networks: + - mtest + expose: + - "8086" + command: [ "/bin/punchingbag", "-addr", ":8086" ] + + # mtest + # + mtest: + image: local-dev/module-testing:latest + networks: + - mtest + depends_on: + - web + - agent + - punchingbag + environment: + - DISABLE_HTTP_OPTIONS=1 + - DISABLE_NOCOOKIE=1 + - MTEST_BASEURL=web:8085 + - MTEST_AGENT=agent:12345 + command: [ "/bin/wait-for", "web:8085", "--", "/bin/mtest", "-test.v" ] + diff --git a/scripts/test-golang110/test.sh b/scripts/test-golang110/test.sh new file mode 100755 index 0000000..2fc3926 --- /dev/null +++ b/scripts/test-golang110/test.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -e + +DOCKERCOMPOSE="docker-compose" + +# run at end no matter what +cleanup() { + echo "shutting down" + # capture log output + $DOCKERCOMPOSE logs --no-color agent >& agent.log + $DOCKERCOMPOSE logs --no-color web >& web.log + $DOCKERCOMPOSE logs --no-color mtest >& mtest.log + $DOCKERCOMPOSE logs --no-color punchingbag >& punchingbag.log + + # delete everything + $DOCKERCOMPOSE down + + # show output of module testing + cat mtest.log +} +trap cleanup 0 1 2 3 6 + +set -x + +# attempt to clean up any leftover junk +$DOCKERCOMPOSE down + +# always get latest agent +$DOCKERCOMPOSE pull + +# start everything, run tests +# +# --no-color --> safe for jenkins +# --build --> alway build test server/module container +# --abort-on-container-exit --> without this, the other servers keep the process running +# --exit-code-from mtest --> make exit code be the result of module test +# +# > /dev/null --> output of all servers is mixed together and ugly +# we get the individual logs at end +# +if [ -d "goroot" ]; then + rm -rf goroot +fi +docker run -v ${PWD}/goroot:/go/ --rm golang:1.10.6-alpine3.8 /bin/sh -c 'apk --update add git && go get github.com/signalsciences/tlstext && go get github.com/tinylib/msgp && go get github.com/alecthomas/gometalinter' +$DOCKERCOMPOSE up --no-color --build --abort-on-container-exit --exit-code-from mtest > /dev/null + diff --git a/scripts/test-golang111/Dockerfile b/scripts/test-golang111/Dockerfile new file mode 100644 index 0000000..cbd58ce --- /dev/null +++ b/scripts/test-golang111/Dockerfile @@ -0,0 +1,7 @@ +FROM golang:1.11.3-alpine3.8 + +COPY goroot/ /go/ + +# we will mount the current directory here +VOLUME [ "/go/src/github.com/signalsciences/sigsci-module-golang" ] +WORKDIR /go/src/github.com/signalsciences/sigsci-module-golang diff --git a/scripts/test-golang111/docker-compose.override.yml b/scripts/test-golang111/docker-compose.override.yml new file mode 100644 index 0000000..7af37fd --- /dev/null +++ b/scripts/test-golang111/docker-compose.override.yml @@ -0,0 +1,15 @@ +version: "3" + +services: + # this defines our webserver uses our sigsci-module + # we only define it so it is attached to our fake network + # it will be run a few times with different options manually + # + # The volumes spec is a bit weird.. this script is run in scripts/test but + # needs stuff in ../../examples. Consider moving. + web: + volumes: + - ../..:/go/src/github.com/signalsciences/sigsci-module-golang + command: [ "go", "run", "/go/src/github.com/signalsciences/sigsci-module-golang/examples/mtest/main.go" ] + environment: + - DEBUG=0 diff --git a/scripts/test-golang111/docker-compose.yml b/scripts/test-golang111/docker-compose.yml new file mode 100644 index 0000000..ab36aef --- /dev/null +++ b/scripts/test-golang111/docker-compose.yml @@ -0,0 +1,57 @@ +version: "3" +networks: + mtest: + +services: + # this defines our webserver uses our sigsci-module + # we only define it so it is attached to our fake network + # it will be run a few times with different options manually + # + # + web: + build: + context: . + dockerfile: Dockerfile + expose: + - "8085" + networks: + - mtest + depends_on: + - agent + + # agent + agent: + image: local-dev/sigsci-agent:latest + command: [ "-debug-log-web-inputs", "2", "-rpc-address", "9090", "-debug-rpc-test-harness", "-debug-standalone", "3" ] + expose: + - "9090" + - "12345" + networks: + - mtest + + # punching bag + punchingbag: + image: local-dev/module-testing:latest + networks: + - mtest + expose: + - "8086" + command: [ "/bin/punchingbag", "-addr", ":8086" ] + + # mtest + # + mtest: + image: local-dev/module-testing:latest + networks: + - mtest + depends_on: + - web + - agent + - punchingbag + environment: + - DISABLE_HTTP_OPTIONS=1 + - DISABLE_NOCOOKIE=1 + - MTEST_BASEURL=web:8085 + - MTEST_AGENT=agent:12345 + command: [ "/bin/wait-for", "web:8085", "--", "/bin/mtest", "-test.v" ] + diff --git a/scripts/test-golang111/test.sh b/scripts/test-golang111/test.sh new file mode 100755 index 0000000..8e8e969 --- /dev/null +++ b/scripts/test-golang111/test.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -e + +DOCKERCOMPOSE="docker-compose" + +# run at end no matter what +cleanup() { + echo "shutting down" + # capture log output + $DOCKERCOMPOSE logs --no-color agent >& agent.log + $DOCKERCOMPOSE logs --no-color web >& web.log + $DOCKERCOMPOSE logs --no-color mtest >& mtest.log + $DOCKERCOMPOSE logs --no-color punchingbag >& punchingbag.log + + # delete everything + $DOCKERCOMPOSE down + + # show output of module testing + cat mtest.log +} +trap cleanup 0 1 2 3 6 + +set -x + +# attempt to clean up any leftover junk +$DOCKERCOMPOSE down + +# always get latest agent +$DOCKERCOMPOSE pull + +# start everything, run tests +# +# --no-color --> safe for jenkins +# --build --> alway build test server/module container +# --abort-on-container-exit --> without this, the other servers keep the process running +# --exit-code-from mtest --> make exit code be the result of module test +# +# > /dev/null --> output of all servers is mixed together and ugly +# we get the individual logs at end +# +if [ -d "goroot" ]; then + rm -rf goroot +fi +docker run -v ${PWD}/goroot:/go/ --rm golang:1.11.3-alpine3.8 /bin/sh -c 'apk --update add git && go get github.com/signalsciences/tlstext && go get github.com/tinylib/msgp && go get github.com/alecthomas/gometalinter' +$DOCKERCOMPOSE up --no-color --build --abort-on-container-exit --exit-code-from mtest > /dev/null + diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..aae4501 --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +set -ex + +(cd ./scripts/test-golang110 && ./test.sh) +(cd ./scripts/test-golang111 && ./test.sh) diff --git a/version.go b/version.go new file mode 100644 index 0000000..5f3734a --- /dev/null +++ b/version.go @@ -0,0 +1,3 @@ +package sigsci + +const version = "1.6.2"