diff --git a/contrib/internal/httptrace/httptrace.go b/contrib/internal/httptrace/httptrace.go index 0017159f62..92069e03d8 100644 --- a/contrib/internal/httptrace/httptrace.go +++ b/contrib/internal/httptrace/httptrace.go @@ -11,7 +11,6 @@ import ( "context" "fmt" "net/http" - "net/url" "strconv" "strings" @@ -28,29 +27,14 @@ var ( ) // StartRequestSpan starts an HTTP request span with the standard list of HTTP request span tags (http.method, http.url, -// http.useragent) with a http.Request object. Any further span start option can be added with opts. -func StartRequestSpan(r *http.Request, opts ...ddtrace.StartSpanOption) (tracer.Span, context.Context) { - return StartHttpSpan( - r.Context(), - r.Header, - r.Host, - r.Method, - urlFromRequest(r), - r.UserAgent(), - r.RemoteAddr, - opts..., - ) -} - -// StartHttpSpan starts an HTTP request span with the standard list of HTTP request span tags (http.method, http.url, // http.useragent). Any further span start option can be added with opts. -func StartHttpSpan(ctx context.Context, headers map[string][]string, host string, method string, url string, userAgent string, remoteAddr string, opts ...ddtrace.StartSpanOption) (tracer.Span, context.Context) { +func StartRequestSpan(r *http.Request, opts ...ddtrace.StartSpanOption) (tracer.Span, context.Context) { // Append our span options before the given ones so that the caller can "overwrite" them. // TODO(): rework span start option handling (https://github.com/DataDog/dd-trace-go/issues/1352) var ipTags map[string]string if cfg.traceClientIP { - ipTags, _ = httpsec.ClientIPTags(headers, true, remoteAddr) + ipTags, _ = httpsec.ClientIPTags(r.Header, true, r.RemoteAddr) } nopts := make([]ddtrace.StartSpanOption, 0, len(opts)+1+len(ipTags)) nopts = append(nopts, @@ -59,23 +43,22 @@ func StartHttpSpan(ctx context.Context, headers map[string][]string, host string cfg.Tags = make(map[string]interface{}) } cfg.Tags[ext.SpanType] = ext.SpanTypeWeb - cfg.Tags[ext.HTTPMethod] = method - cfg.Tags[ext.HTTPURL] = url - cfg.Tags[ext.HTTPUserAgent] = userAgent + cfg.Tags[ext.HTTPMethod] = r.Method + cfg.Tags[ext.HTTPURL] = urlFromRequest(r) + cfg.Tags[ext.HTTPUserAgent] = r.UserAgent() cfg.Tags["_dd.measured"] = 1 - if host != "" { - cfg.Tags["http.host"] = host + if r.Host != "" { + cfg.Tags["http.host"] = r.Host } - if spanctx, err := tracer.Extract(tracer.HTTPHeadersCarrier(headers)); err == nil { + if spanctx, err := tracer.Extract(tracer.HTTPHeadersCarrier(r.Header)); err == nil { cfg.Parent = spanctx } - for k, v := range ipTags { cfg.Tags[k] = v } }) nopts = append(nopts, opts...) - return tracer.StartSpanFromContext(ctx, namingschema.OpName(namingschema.HTTPServer), nopts...) + return tracer.StartSpanFromContext(r.Context(), namingschema.OpName(namingschema.HTTPServer), nopts...) } // FinishRequestSpan finishes the given HTTP request span and sets the expected response-related tags such as the status @@ -114,54 +97,27 @@ func urlFromRequest(r *http.Request) string { // "For most requests, fields other than Path and RawQuery will be // empty. (See RFC 7230, Section 5.3)" // This is why we don't rely on url.URL.String(), url.URL.Host, url.URL.Scheme, etc... + var url string + path := r.URL.EscapedPath() scheme := "http" if r.TLS != nil { scheme = "https" } - - return urlFromArgs( - r.URL.EscapedPath(), - scheme, - r.Host, - r.URL.RawQuery, - r.URL.EscapedFragment(), - ) -} - -// UrlFromUrl returns the full URL from a URL object. If query params are collected, they are obfuscated granted -// obfuscation is not disabled by the user (through DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP) -// See https://docs.datadoghq.com/tracing/configure_data_security#redacting-the-query-in-the-url for more information. -func UrlFromUrl(u *url.URL) string { - scheme := "http" - if u.Scheme != "" { - scheme = u.Scheme - } - return urlFromArgs( - u.EscapedPath(), - scheme, - u.Host, - u.RawQuery, - u.EscapedFragment(), - ) -} - -func urlFromArgs(escapedPath string, scheme string, host string, rawQuery string, escapedFragment string) string { - var url string - if host != "" { - url = strings.Join([]string{scheme, "://", host, escapedPath}, "") + if r.Host != "" { + url = strings.Join([]string{scheme, "://", r.Host, path}, "") } else { - url = escapedPath + url = path } // Collect the query string if we are allowed to report it and obfuscate it if possible/allowed - if cfg.queryString && rawQuery != "" { - query := rawQuery + if cfg.queryString && r.URL.RawQuery != "" { + query := r.URL.RawQuery if cfg.queryStringRegexp != nil { query = cfg.queryStringRegexp.ReplaceAllLiteralString(query, "") } url = strings.Join([]string{url, query}, "?") } - if escapedFragment != "" { - url = strings.Join([]string{url, escapedFragment}, "#") + if frag := r.URL.EscapedFragment(); frag != "" { + url = strings.Join([]string{url, frag}, "#") } return url } diff --git a/go.mod b/go.mod index a5b2e00318..699744fccd 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/elastic/go-elasticsearch/v8 v8.4.0 github.com/emicklei/go-restful v2.16.0+incompatible github.com/emicklei/go-restful/v3 v3.11.0 - github.com/envoyproxy/go-control-plane v0.13.0 + github.com/envoyproxy/go-control-plane v0.12.0 github.com/garyburd/redigo v1.6.4 github.com/gin-gonic/gin v1.9.1 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 @@ -104,7 +104,7 @@ require ( golang.org/x/time v0.6.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 google.golang.org/api v0.192.0 - google.golang.org/grpc v1.65.0 + google.golang.org/grpc v1.64.1 google.golang.org/protobuf v1.34.2 gopkg.in/jinzhu/gorm.v1 v1.9.2 gopkg.in/olivere/elastic.v3 v3.0.75 @@ -244,7 +244,6 @@ require ( github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect diff --git a/go.sum b/go.sum index 0e32547431..24620db23e 100644 --- a/go.sum +++ b/go.sum @@ -1125,8 +1125,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= -github.com/envoyproxy/go-control-plane v0.13.0 h1:HzkeUz1Knt+3bK+8LG1bxOO/jzWZmdxpwC51i202les= -github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8= +github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= +github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= @@ -1894,8 +1894,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -3066,8 +3064,8 @@ google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= -google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= -google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/internal/appsec/emitter/httpsec/http.go b/internal/appsec/emitter/httpsec/http.go index 4301d8a29e..07f6dd9ba8 100644 --- a/internal/appsec/emitter/httpsec/http.go +++ b/internal/appsec/emitter/httpsec/http.go @@ -12,12 +12,9 @@ package httpsec import ( "context" - "strings" - // Blank import needed to use embed for the default blocked response payloads _ "embed" "net/http" - "net/url" "sync" "sync/atomic" @@ -192,46 +189,3 @@ func WrapHandler(handler http.Handler, span ddtrace.Span, pathParams map[string] handler.ServeHTTP(tw, tr) }) } - -// MakeHandlerOperationArgs creates the HandlerOperationArgs value. -func MakeHandlerOperationArgs(headers map[string][]string, method string, host string, remoteAddr string, url *url.URL) HandlerOperationArgs { - cookies := filterCookiesFromHeaders(headers) - - args := HandlerOperationArgs{ - Method: method, - RequestURI: url.RequestURI(), - Host: host, - RemoteAddr: remoteAddr, - Headers: headers, - Cookies: cookies, - QueryParams: url.Query(), - PathParams: map[string]string{}, - } - - return args -} - -// Separate the cookies from the headers, return the parsed cookies and remove in place the cookies from the headers. -// Headers used for `server.request.headers.no_cookies` and `server.response.headers.no_cookies` addresses for the WAF -// Cookies are used for the `server.request.cookies` address -func filterCookiesFromHeaders(headers http.Header) map[string][]string { - cookieHeader, ok := headers["Cookie"] - if !ok { - return make(http.Header) - } - - delete(headers, "Cookie") - - cookies := make(map[string][]string, len(cookieHeader)) - for _, c := range cookieHeader { - parts := strings.Split(c, ";") - for _, part := range parts { - cookie := strings.Split(part, "=") - if len(cookie) == 2 { - cookies[cookie[0]] = append(cookies[cookie[0]], cookie[1]) - } - } - } - - return cookies -} diff --git a/internal/appsec/emitter/waf/actions/block.go b/internal/appsec/emitter/waf/actions/block.go index 40418f4410..ae802b60bd 100644 --- a/internal/appsec/emitter/waf/actions/block.go +++ b/internal/appsec/emitter/waf/actions/block.go @@ -69,11 +69,6 @@ type ( // BlockHTTP are actions that interact with an HTTP request flow BlockHTTP struct { http.Handler - - StatusCode int `mapstructure:"status_code"` - RedirectLocation string - // BlockingTemplate is a function that returns the headers to be added and body to be written to the response - BlockingTemplate func(headers map[string][]string) (map[string][]string, []byte) } ) @@ -130,11 +125,7 @@ func NewBlockAction(params map[string]any) []Action { } func newHTTPBlockRequestAction(status int, template string) *BlockHTTP { - return &BlockHTTP{ - Handler: newBlockHandler(status, template), - BlockingTemplate: newManualBlockHandler(template), - StatusCode: status, - } + return &BlockHTTP{Handler: newBlockHandler(status, template)} } // newBlockHandler creates, initializes and returns a new BlockRequestAction @@ -148,47 +139,16 @@ func newBlockHandler(status int, template string) http.Handler { return htmlHandler default: return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - h := findCorrectTemplate(jsonHandler, htmlHandler, r.Header.Get("Accept")) - h.(http.Handler).ServeHTTP(w, r) - }) - } -} - -func newManualBlockHandler(template string) func(headers map[string][]string) (map[string][]string, []byte) { - htmlHandler := newManualBlockDataHandler("text/html", blockedTemplateHTML) - jsonHandler := newManualBlockDataHandler("application/json", blockedTemplateJSON) - switch template { - case "json": - return jsonHandler - case "html": - return htmlHandler - default: - return func(headers map[string][]string) (map[string][]string, []byte) { - acceptHeader := "" - if hdr, ok := headers["Accept"]; ok && len(hdr) > 0 { - acceptHeader = hdr[0] + h := jsonHandler + hdr := r.Header.Get("Accept") + htmlIdx := strings.Index(hdr, "text/html") + jsonIdx := strings.Index(hdr, "application/json") + // Switch to html handler if text/html comes before application/json in the Accept header + if htmlIdx != -1 && (jsonIdx == -1 || htmlIdx < jsonIdx) { + h = htmlHandler } - h := findCorrectTemplate(jsonHandler, htmlHandler, acceptHeader) - return h.(func(headers map[string][]string) (map[string][]string, []byte))(headers) - } - } -} - -func findCorrectTemplate(jsonHandler interface{}, htmlHandler interface{}, acceptHeader string) interface{} { - h := jsonHandler - hdr := acceptHeader - htmlIdx := strings.Index(hdr, "text/html") - jsonIdx := strings.Index(hdr, "application/json") - // Switch to html handler if text/html comes before application/json in the Accept header - if htmlIdx != -1 && (jsonIdx == -1 || htmlIdx < jsonIdx) { - h = htmlHandler - } - return h -} - -func newManualBlockDataHandler(ct string, template []byte) func(headers map[string][]string) (map[string][]string, []byte) { - return func(headers map[string][]string) (map[string][]string, []byte) { - return map[string][]string{"Content-Type": {ct}}, template + h.ServeHTTP(w, r) + }) } } diff --git a/internal/appsec/emitter/waf/actions/http_redirect.go b/internal/appsec/emitter/waf/actions/http_redirect.go index 25e209d790..3cdca4c818 100644 --- a/internal/appsec/emitter/waf/actions/http_redirect.go +++ b/internal/appsec/emitter/waf/actions/http_redirect.go @@ -7,14 +7,10 @@ package actions import ( "net/http" - "path" - "strings" "github.com/mitchellh/mapstructure" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" - - urlpkg "net/url" ) // redirectActionParams are the dynamic parameters to be provided to a "redirect_request" @@ -42,17 +38,9 @@ func newRedirectRequestAction(status int, loc string) *BlockHTTP { // If location is not set we fall back on a default block action if loc == "" { - return &BlockHTTP{ - Handler: newBlockHandler(http.StatusForbidden, string(blockedTemplateJSON)), - StatusCode: status, - BlockingTemplate: newManualBlockHandler("json"), - } - } - return &BlockHTTP{ - Handler: http.RedirectHandler(loc, status), - StatusCode: status, - RedirectLocation: loc, + return &BlockHTTP{Handler: newBlockHandler(http.StatusForbidden, string(blockedTemplateJSON))} } + return &BlockHTTP{Handler: http.RedirectHandler(loc, status)} } // NewRedirectAction creates an action for the "redirect_request" action type @@ -64,75 +52,3 @@ func NewRedirectAction(params map[string]any) []Action { } return []Action{newRedirectRequestAction(p.StatusCode, p.Location)} } - -// HandleRedirectLocationString returns the headers and body to be written to the response when a redirect is needed -// Vendored from net/http/server.go -func HandleRedirectLocationString(oldpath string, url string, statusCode int, method string, h map[string][]string) (map[string][]string, []byte) { - if u, err := urlpkg.Parse(url); err == nil { - // If url was relative, make its path absolute by - // combining with request path. - // The client would probably do this for us, - // but doing it ourselves is more reliable. - // See RFC 7231, section 7.1.2 - if u.Scheme == "" && u.Host == "" { - if oldpath == "" { // should not happen, but avoid a crash if it does - oldpath = "/" - } - - // no leading http://server - if url == "" || url[0] != '/' { - // make relative path absolute - olddir, _ := path.Split(oldpath) - url = olddir + url - } - - var query string - if i := strings.Index(url, "?"); i != -1 { - url, query = url[:i], url[i:] - } - - // clean up but preserve trailing slash - trailing := strings.HasSuffix(url, "/") - url = path.Clean(url) - if trailing && !strings.HasSuffix(url, "/") { - url += "/" - } - url += query - } - } - - // RFC 7231 notes that a short HTML body is usually included in - // the response because older user agents may not understand 301/307. - // Do it only if the request didn't already have a Content-Type header. - _, hadCT := h["content-type"] - newHeaders := make(map[string][]string, 2) - - newHeaders["location"] = []string{url} - if !hadCT && (method == "GET" || method == "HEAD") { - newHeaders["content-length"] = []string{"text/html; charset=utf-8"} - } - - // Shouldn't send the body for POST or HEAD; that leaves GET. - var body []byte - if !hadCT && method == "GET" { - body = []byte("" + http.StatusText(statusCode) + ".\n") - } - - return newHeaders, body -} - -// Vendored from net/http/server.go -var htmlReplacer = strings.NewReplacer( - "&", "&", - "<", "<", - ">", ">", - // """ is shorter than """. - `"`, """, - // "'" is shorter than "'" and apos was not in HTML until HTML5. - "'", "'", -) - -// htmlEscape escapes special characters like "<" to become "<". -func htmlEscape(s string) string { - return htmlReplacer.Replace(s) -}