diff --git a/.gitignore b/.gitignore index 5a50568910..146b0a892c 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,7 @@ dist pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser -vendor \ No newline at end of file +vendor + +# Headless `screenshot` action +*.png \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 45f92d4485..9ac12d7d23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,6 @@ FROM golang:1.21-alpine AS build-env RUN apk add build-base WORKDIR /app COPY . /app -WORKDIR /app RUN go mod download RUN go build ./cmd/nuclei diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index c757321494..9ec413d3ed 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -3664,8 +3664,18 @@ Enum Values: - files + - waitdom + + - waitfcp + + - waitfmp + + - waitidle + - waitload + - waitstable + - getresource - extract diff --git a/cmd/integration-test/matcher-status.go b/cmd/integration-test/matcher-status.go index b88763720a..d89f7c46b0 100644 --- a/cmd/integration-test/matcher-status.go +++ b/cmd/integration-test/matcher-status.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "fmt" + "strings" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/testutils" @@ -26,9 +27,9 @@ func (h *httpNoAccess) Execute(filePath string) error { } event := &output.ResultEvent{} _ = json.Unmarshal([]byte(results[0]), event) - - if event.Error != "no address found for host" { - return fmt.Errorf("unexpected result: expecting \"no address found for host\" error but got none") + expectedError := "no address found for host" + if !strings.Contains(event.Error, expectedError) { + return fmt.Errorf("unexpected result: expecting \"%s\" error but got \"%s\"", expectedError, event.Error) } return nil } diff --git a/cmd/integration-test/workflow.go b/cmd/integration-test/workflow.go index e11575f90e..442e5169f6 100644 --- a/cmd/integration-test/workflow.go +++ b/cmd/integration-test/workflow.go @@ -13,6 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/templates" "github.com/projectdiscovery/nuclei/v3/pkg/templates/signer" "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + sliceutil "github.com/projectdiscovery/utils/slice" ) var workflowTestcases = []TestCaseInfo{ @@ -25,6 +26,7 @@ var workflowTestcases = []TestCaseInfo{ {Path: "workflow/dns-value-share-workflow.yaml", TestCase: &workflowDnsKeyValueShare{}}, {Path: "workflow/code-value-share-workflow.yaml", TestCase: &workflowCodeKeyValueShare{}, DisableOn: isCodeDisabled}, // isCodeDisabled declared in code.go {Path: "workflow/multiprotocol-value-share-workflow.yaml", TestCase: &workflowMultiProtocolKeyValueShare{}}, + {Path: "workflow/multimatch-value-share-workflow.yaml", TestCase: &workflowMultiMatchKeyValueShare{}}, {Path: "workflow/shared-cookie.yaml", TestCase: &workflowSharedCookies{}}, } @@ -229,6 +231,44 @@ func (h *workflowMultiProtocolKeyValueShare) Execute(filePath string) error { return expectResultsCount(results, 2) } +type workflowMultiMatchKeyValueShare struct{} + +// Execute executes a test case and returns an error if occurred +func (h *workflowMultiMatchKeyValueShare) Execute(filePath string) error { + var receivedData []string + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "This is test matcher text") + }) + router.GET("/path1", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "href=\"test-value-%s\"", r.URL.Query().Get("v")) + }) + router.GET("/path2", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + body, _ := io.ReadAll(r.Body) + receivedData = append(receivedData, string(body)) + fmt.Fprintf(w, "test-value") + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + + // Check if we received the data from both request to /path1 and it is not overwritten by the later one. + // They will appear in brackets because of another bug: https://github.com/orgs/projectdiscovery/discussions/3766 + if !sliceutil.Contains(receivedData, "[test-value-1]") || !sliceutil.Contains(receivedData, "[test-value-2]") { + return fmt.Errorf( + "incorrect data: did not receive both extracted data from the first request!\nReceived Data:\n\t%s\nResults:\n\t%s", + strings.Join(receivedData, "\n\t"), + strings.Join(results, "\n\t"), + ) + } + // The number of expected results is 3: the workflow's Matcher Name based condition check forwards both match, and the other branch with simple subtemplates goes with one + return expectResultsCount(results, 3) +} + type workflowSharedCookies struct{} // Execute executes a test case and returns an error if occurred diff --git a/cmd/nuclei/issue-tracker-config.yaml b/cmd/nuclei/issue-tracker-config.yaml index 51778eb0ad..b7e0e6dafc 100644 --- a/cmd/nuclei/issue-tracker-config.yaml +++ b/cmd/nuclei/issue-tracker-config.yaml @@ -142,4 +142,23 @@ # # Username for the elasticsearch instance # username: test # # Password is the password for elasticsearch instance -# password: test \ No newline at end of file +# password: test +#linear: +# # api-key is the API key for the linear account +# api-key: "" +# # allow-list sets a tracker level filter to only create issues for templates with +# # these severity labels or tags (does not affect exporters. set those globally) +# deny-list: +# severity: critical +# # deny-list sets a tracker level filter to never create issues for templates with +# # these severity labels or tags (does not affect exporters. set those globally) +# deny-list: +# severity: low +# # team-id is the ID of the team in Linear +# team-id: "" +# # project-id is the ID of the project in Linear +# project-id: "" +# # duplicate-issue-check flag to enable duplicate tracking issue check +# duplicate-issue-check: false +# # open-state-id is the ID of the open state in Linear +# open-state-id: "" diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index de29d6581e..8fe5b29573 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -123,6 +123,13 @@ func main() { runner.ParseOptions(options) + if options.ScanUploadFile != "" { + if err := runner.UploadResultsToCloud(options); err != nil { + gologger.Fatal().Msgf("could not upload scan results to cloud dashboard: %s\n", err) + } + return + } + nucleiRunner, err := runner.New(options) if err != nil { gologger.Fatal().Msgf("Could not create runner: %s\n", err) @@ -420,9 +427,11 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.CreateGroup("cloud", "Cloud", flagSet.DynamicVar(&pdcpauth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"), flagSet.StringVarP(&options.TeamID, "team-id", "tid", _pdcp.TeamIDEnv, "upload scan results to given team id (optional)"), - flagSet.BoolVarP(&options.EnableCloudUpload, "cloud-upload", "cup", false, "upload scan results to pdcp dashboard"), + flagSet.BoolVarP(&options.EnableCloudUpload, "cloud-upload", "cup", false, "upload scan results to pdcp dashboard [DEPRECATED use -dashboard]"), flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to existing scan id (optional)"), flagSet.StringVarP(&options.ScanName, "scan-name", "sname", "", "scan name to set (optional)"), + flagSet.BoolVarP(&options.EnableCloudUpload, "dashboard", "pd", false, "upload / view nuclei results in projectdiscovery cloud (pdcp) UI dashboard"), + flagSet.StringVarP(&options.ScanUploadFile, "dashboard-upload", "pdu", "", "upload / view nuclei results file (jsonl) in projectdiscovery cloud (pdcp) UI dashboard"), ) flagSet.CreateGroup("Authentication", "Authentication", diff --git a/go.mod b/go.mod index 1a478943b8..f2adacca21 100644 --- a/go.mod +++ b/go.mod @@ -19,13 +19,13 @@ require ( github.com/miekg/dns v1.1.59 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/clistats v0.1.0 - github.com/projectdiscovery/fastdialer v0.2.7 - github.com/projectdiscovery/hmap v0.0.56 + github.com/projectdiscovery/clistats v0.1.1 + github.com/projectdiscovery/fastdialer v0.2.9 + github.com/projectdiscovery/hmap v0.0.59 github.com/projectdiscovery/interactsh v1.2.0 - github.com/projectdiscovery/rawhttp v0.1.61 - github.com/projectdiscovery/retryabledns v1.0.73 - github.com/projectdiscovery/retryablehttp-go v1.0.76 + github.com/projectdiscovery/rawhttp v0.1.67 + github.com/projectdiscovery/retryabledns v1.0.77 + github.com/projectdiscovery/retryablehttp-go v1.0.78 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -38,9 +38,9 @@ require ( github.com/weppos/publicsuffix-go v0.30.2 github.com/xanzy/go-gitlab v0.107.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.26.0 + golang.org/x/net v0.29.0 golang.org/x/oauth2 v0.22.0 - golang.org/x/text v0.16.0 + golang.org/x/text v0.18.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -59,7 +59,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 github.com/cespare/xxhash v1.1.0 - github.com/charmbracelet/glamour v0.6.0 + github.com/charmbracelet/glamour v0.8.0 github.com/clbanning/mxj/v2 v2.7.0 github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c github.com/docker/go-units v0.5.0 @@ -72,7 +72,7 @@ require ( github.com/go-sql-driver/mysql v1.7.1 github.com/h2non/filetype v1.1.3 github.com/invopop/yaml v0.3.1 - github.com/kitabisa/go-ci v1.0.2 + github.com/kitabisa/go-ci v1.0.3 github.com/labstack/echo/v4 v4.10.2 github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa github.com/lib/pq v1.10.9 @@ -85,26 +85,27 @@ require ( github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb github.com/projectdiscovery/goflags v0.1.64 - github.com/projectdiscovery/gologger v1.1.21 + github.com/projectdiscovery/gologger v1.1.24 github.com/projectdiscovery/gostruct v0.0.2 github.com/projectdiscovery/gozero v0.0.2 github.com/projectdiscovery/httpx v1.6.8 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 - github.com/projectdiscovery/ratelimit v0.0.53 + github.com/projectdiscovery/ratelimit v0.0.56 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.7 github.com/projectdiscovery/uncover v1.0.9 - github.com/projectdiscovery/useragent v0.0.65 - github.com/projectdiscovery/utils v0.2.7 - github.com/projectdiscovery/wappalyzergo v0.1.14 + github.com/projectdiscovery/useragent v0.0.71 + github.com/projectdiscovery/utils v0.2.11 + github.com/projectdiscovery/wappalyzergo v0.1.18 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 + github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 github.com/stretchr/testify v1.9.0 github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 - golang.org/x/term v0.21.0 + golang.org/x/term v0.24.0 gopkg.in/yaml.v3 v3.0.1 moul.io/http2curl v1.0.0 ) @@ -119,6 +120,7 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/VividCortex/ewma v1.2.0 // indirect + github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 // indirect @@ -133,6 +135,8 @@ require ( github.com/bytedance/sonic v1.9.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/charmbracelet/lipgloss v0.13.0 // indirect + github.com/charmbracelet/x/ansi v0.3.2 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/cfssl v1.6.4 // indirect @@ -141,7 +145,7 @@ require ( github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/dlclark/regexp2 v1.11.0 // indirect + github.com/dlclark/regexp2 v1.11.4 // indirect github.com/docker/cli v24.0.5+incompatible // indirect github.com/docker/docker v24.0.9+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect @@ -192,10 +196,10 @@ require ( github.com/moby/term v0.5.0 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/muesli/reflow v0.3.0 // indirect - github.com/muesli/termenv v0.15.1 // indirect + github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect - github.com/opencontainers/runc v1.1.12 // indirect + github.com/opencontainers/runc v1.1.14 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect @@ -203,7 +207,7 @@ require ( github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/projectdiscovery/asnmap v1.1.1 // indirect github.com/projectdiscovery/cdncheck v1.1.0 // indirect - github.com/projectdiscovery/freeport v0.0.5 // indirect + github.com/projectdiscovery/freeport v0.0.6 // indirect github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect github.com/refraction-networking/utls v1.6.7 // indirect @@ -229,13 +233,13 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/ysmood/fetchup v0.2.3 // indirect github.com/ysmood/got v0.34.1 // indirect - github.com/yuin/goldmark v1.5.4 // indirect - github.com/yuin/goldmark-emoji v1.0.1 // indirect + github.com/yuin/goldmark v1.7.4 // indirect + github.com/yuin/goldmark-emoji v1.0.3 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect github.com/zeebo/blake3 v0.2.3 // indirect go.uber.org/goleak v1.3.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/sync v0.7.0 // indirect + golang.org/x/sync v0.8.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect mellium.im/sasl v0.3.1 // indirect ) @@ -277,16 +281,16 @@ require ( github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mholt/acmez v1.2.0 // indirect - github.com/microcosm-cc/bluemonday v1.0.26 // indirect + github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/networkpolicy v0.0.9 - github.com/rivo/uniseg v0.4.6 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect @@ -302,10 +306,10 @@ require ( go.etcd.io/bbolt v1.3.10 // indirect go.uber.org/zap v1.25.0 // indirect goftp.io/server/v2 v2.0.1 // indirect - golang.org/x/crypto v0.24.0 // indirect + golang.org/x/crypto v0.27.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/mod v0.17.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.25.0 // indirect golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d google.golang.org/protobuf v1.34.2 // indirect diff --git a/go.sum b/go.sum index 031530e8a4..da71e9b287 100644 --- a/go.sum +++ b/go.sum @@ -93,9 +93,15 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi78= github.com/akrylysov/pogreb v0.10.2/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= +github.com/alecthomas/assert/v2 v2.7.0 h1:QtqSACNS3tF7oasA8CU6A6sXZSBDqnm7RfpLl9bZqbE= +github.com/alecthomas/assert/v2 v2.7.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= +github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46aU4V9E= +github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I= github.com/alecthomas/kingpin/v2 v2.3.1/go.mod h1:oYL5vtsvEHZGHxU7DMp32Dvx+qL+ptGn6lWaot2vCNE= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -174,9 +180,10 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 h1:e5mnydVdCVWxP+5rPAGi2PYxC7u2 github.com/aws/aws-sdk-go-v2/service/sts v1.19.3/go.mod h1:yVGZA1CPkmUhBdA039jXNJJG7/6t+G+EBWmFq23xqnY= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= +github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= @@ -217,8 +224,14 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= -github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= +github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs= +github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw= +github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw= +github.com/charmbracelet/lipgloss v0.13.0/go.mod h1:nw4zy0SBX/F/eAO1cWdcvy6qnkDUxr8Lw7dvFrAIbbY= +github.com/charmbracelet/x/ansi v0.3.2 h1:wsEwgAN+C9U06l9dCVMX0/L3x7ptvY1qmjMwyfE6USY= +github.com/charmbracelet/x/ansi v0.3.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99klV19u0QnhiizODirwVksQB91TJKV/UaTnACcG30= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo= github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= @@ -271,8 +284,8 @@ github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c/go.mod h1: github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= -github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= +github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/cli v24.0.5+incompatible h1:WeBimjvS0eKdH4Ygx+ihVq1Q++xg36M/rMi4aXAvodc= github.com/docker/cli v24.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0= @@ -508,7 +521,6 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -559,6 +571,8 @@ github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6 github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A= github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDjJSQ= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -625,8 +639,8 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kitabisa/go-ci v1.0.2 h1:rqHf8KEbQOxVb998TbqGRo70Z7ol44io7/jLYJUvKp8= -github.com/kitabisa/go-ci v1.0.2/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U= +github.com/kitabisa/go-ci v1.0.3 h1:JmIUIvcercRQc/9x/v02ydCCqU4MadSHaNaOF8T2pGA= +github.com/kitabisa/go-ci v1.0.3/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= @@ -695,9 +709,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -708,9 +721,8 @@ github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1w github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= -github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= -github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58= -github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/microsoft/go-mssqldb v1.6.0 h1:mM3gYdVwEPFrlg/Dvr2DNVEgYFG7L42l+dGc67NNNpc= github.com/microsoft/go-mssqldb v1.6.0/go.mod h1:00mDtPbeQCRGC1HwOOR5K/gr30P1NcEG0vx6Kbv2aJU= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -747,9 +759,8 @@ github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1 github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= -github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc= -github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= -github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= +github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a h1:2MaM6YC3mGu54x+RKAA6JiFFHlHDY1UbkxqppT7wYOg= +github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a/go.mod h1:hxSnBBYLK21Vtq/PHd0S2FYCxBXzBua8ov5s1RobyRQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= @@ -781,8 +792,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss= -github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8= +github.com/opencontainers/runc v1.1.14 h1:rgSuzbmgz5DUJjeSnw337TxDbRuqjs6iqQck/2weR6w= +github.com/opencontainers/runc v1.1.14/go.mod h1:E4C2z+7BxR7GHXp0hAY53mek+x49X1LjPNeMTfRGvOA= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -830,28 +841,28 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/snk0cQ3kXE= github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc= -github.com/projectdiscovery/clistats v0.1.0 h1:b+LF1w0xhNd7cneKWMXb+/yUmF1n5szawxP4XGpmbxs= -github.com/projectdiscovery/clistats v0.1.0/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= +github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE= +github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= github.com/projectdiscovery/dsl v0.2.1 h1:TK3KD4jsg4YbvY7WJqnz1QyH4AOvAwezeBFOX97Evgk= github.com/projectdiscovery/dsl v0.2.1/go.mod h1:IRQXsmi5/g1dDZ79//A9t2vrRtxm4frRSd5t8CZVSbI= -github.com/projectdiscovery/fastdialer v0.2.7 h1:x7bEf/kDyVOuG9kHfJLMhWO3h7J+6fMDgLWkGYL8EFg= -github.com/projectdiscovery/fastdialer v0.2.7/go.mod h1:7/BGdaw7sCekuBh6lLZqPuoO5N0Qaizwvex/PjO+V8o= +github.com/projectdiscovery/fastdialer v0.2.9 h1:vDCqxVMCyUu3oVEizEK1K8K+CCcLkVDW3X2HfiWaVFA= +github.com/projectdiscovery/fastdialer v0.2.9/go.mod h1:mYv5QaNBDDSHlZO9DI0niRMw+G5hUzwIhs8QixSElUI= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= -github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= -github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= +github.com/projectdiscovery/freeport v0.0.6 h1:ROqzuXN8JPqkGdBueb3ah691nS2g2p7r3/3x2E33GbI= +github.com/projectdiscovery/freeport v0.0.6/go.mod h1:T2kIy+WrbyxBIhI8V3Y9aeNGnuhnM8tEUSK/cm9GjAg= github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg= github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY= github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI= github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4= -github.com/projectdiscovery/gologger v1.1.21 h1:FD9MGE9adY/y2IxDpQgPZIvZAkQIx3bvcuY+d1FLQUo= -github.com/projectdiscovery/gologger v1.1.21/go.mod h1:eyvlcHiSuzJQtwmh+fcfwZVaqVgSGtEP1oL1gJFgP3c= +github.com/projectdiscovery/gologger v1.1.24 h1:TmA4k9sA6ZvfyRnprZKQ0Uq34g//u5R9yTDPL9IzTOQ= +github.com/projectdiscovery/gologger v1.1.24/go.mod h1:JA0JMJ+ply+J2wD062TN4h85thm6/28jAlrntwccKVU= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E= github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8= -github.com/projectdiscovery/hmap v0.0.56 h1:QLIuFhCc7jrBH9WYpxNBDooSoH+/S8cCm7VO5rMt3lU= -github.com/projectdiscovery/hmap v0.0.56/go.mod h1:XyGKMO93nLOzpaTTCDBSKEGx/XWXjSpoyZJxT92Qd/Q= +github.com/projectdiscovery/hmap v0.0.59 h1:xWCr/GY2QJanFzwKydh/EkGdOKM4iAcN9hQvvCMgO6A= +github.com/projectdiscovery/hmap v0.0.59/go.mod h1:uHhhnPmvq9qXvCjBSQXCBAlmA1r8JGufP775IkBSbgs= github.com/projectdiscovery/httpx v1.6.8 h1:k0Y5g3ue/7QbDP0+LykIxp/VhPDLfau3UEUyuxtP7qE= github.com/projectdiscovery/httpx v1.6.8/go.mod h1:7BIsDxyRwkBjthqFmEajXrA5f3yb4tlVfLmpNdf0ZXA= github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8= @@ -866,16 +877,16 @@ github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc= github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk= github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= -github.com/projectdiscovery/ratelimit v0.0.53 h1:1ci2zaVCtJ9daDi9IXjNwpDxl2InYq375role/xqUR4= -github.com/projectdiscovery/ratelimit v0.0.53/go.mod h1:4kNYTL7xKLo1qMYHTREUCBqHyvykFF+XBWutTh9fkwg= -github.com/projectdiscovery/rawhttp v0.1.61 h1:EbskCj6kkDSG31sO5zEUFTqHp9ltccG1DdcI+MCdahQ= -github.com/projectdiscovery/rawhttp v0.1.61/go.mod h1:5XmDAKph9pLVnh87zjL+vXDpfG5W8Gz5N2BAtYTkuVU= +github.com/projectdiscovery/ratelimit v0.0.56 h1:WliU7NvfMb5hK/IJjOFlWIXU1G7+QRylMhSybaSCTI8= +github.com/projectdiscovery/ratelimit v0.0.56/go.mod h1:GbnAo+MbB4R/4jiOI1mH4KAJfovmrPnq4NElcI99fvs= +github.com/projectdiscovery/rawhttp v0.1.67 h1:HYzathMk3c8Y83hYjHM4GCBFbz/G+vABe0Lz6ajaowY= +github.com/projectdiscovery/rawhttp v0.1.67/go.mod h1:5viJ6odzc9ZuEFppj/E7HdX+u99FoYlvXnhHyTNc7N0= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.73 h1:CLqtgoI6+YVcUE2jnZPFVvwj59CHDE4HNImLf62sSuw= -github.com/projectdiscovery/retryabledns v1.0.73/go.mod h1:Wo5tQHKtx6XcYYhMypRg8dr0GGnqIb4CzoTSEw2lo7I= -github.com/projectdiscovery/retryablehttp-go v1.0.76 h1:2+4QpWnXUZFBkoeCEBXtUjSeBTdgKtaGwP8LRtKAC3g= -github.com/projectdiscovery/retryablehttp-go v1.0.76/go.mod h1:0qIv7Ld8WWSKWYXMsNAWRYU5gsxnxpBza3SjDgt6nVc= +github.com/projectdiscovery/retryabledns v1.0.77 h1:rCFSiTPNI7h9PP1uUvrmcq/6XVZVdxpha2H1ioArKpk= +github.com/projectdiscovery/retryabledns v1.0.77/go.mod h1:ce1JTjAOxLujqITtA5VLlbhLRVubx+GETYasivapai4= +github.com/projectdiscovery/retryablehttp-go v1.0.78 h1:If7/XjCWk893YrnTMaW69TNMsfE1Er3i1SWOkWbEk4E= +github.com/projectdiscovery/retryablehttp-go v1.0.78/go.mod h1:NqzTdnGihSRkF9c/aXo+3qTJXEeNwnOuza6GrlPo9qw= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -884,12 +895,12 @@ github.com/projectdiscovery/tlsx v1.1.7 h1:eSsl/SmTDL/z2CMeSrbssk4f/9oOotMP1SgXl github.com/projectdiscovery/tlsx v1.1.7/go.mod h1:g66QQ4/y4tLVjoGbzWIv+Q6xwFzxfJbEDx86Y1dYHDM= github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0JprejqLSopN9Y= github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U= -github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70= -github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A= -github.com/projectdiscovery/utils v0.2.7 h1:XWdz7SscL++jqsnQ9ecHzSZE0RK33tyPcnqcXw+vmKs= -github.com/projectdiscovery/utils v0.2.7/go.mod h1:N0N7tbdNFPegd9NpJ3onCPClaBrERcOIB88yww6UCF8= -github.com/projectdiscovery/wappalyzergo v0.1.14 h1:nt1IM4RUmqeymsXk4h6BsZbKDoS2hjFvPkT2GaI1rz4= -github.com/projectdiscovery/wappalyzergo v0.1.14/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= +github.com/projectdiscovery/useragent v0.0.71 h1:Q02L3LV15ztOQ6FfmVSqVmOd5QhvzI+yAgYOc/32Nvg= +github.com/projectdiscovery/useragent v0.0.71/go.mod h1:DHPruFLCvCvkd2qqPwwQZrP9sziv0lxQJ0R1rE1fa8E= +github.com/projectdiscovery/utils v0.2.11 h1:TO7fBG5QI256sn1YuTD87yn4+4OjGJ2wT1772uEnp4Q= +github.com/projectdiscovery/utils v0.2.11/go.mod h1:W0E74DWkKxlcyKS5XwcAwiob7+smoszPPi1NgX3vZyk= +github.com/projectdiscovery/wappalyzergo v0.1.18 h1:fFgETis0HcsNE7wREaUPYP45JqIyHgGorJaVp1RH7g4= +github.com/projectdiscovery/wappalyzergo v0.1.18/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -933,8 +944,8 @@ github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.6 h1:Sovz9sDSwbOz9tgUy8JpT+KgCkPYJEN/oYzlJiYTNLg= -github.com/rivo/uniseg v0.4.6/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -964,6 +975,8 @@ github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFt github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0= +github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -1104,11 +1117,11 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= -github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os= -github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ= +github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= +github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg= +github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= +github.com/yuin/goldmark-emoji v1.0.3 h1:aLRkLHOuBR2czCY4R8olwMjID+tENfhyFDMCRhbIQY4= +github.com/yuin/goldmark-emoji v1.0.3/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc= @@ -1185,8 +1198,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1270,7 +1283,6 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -1280,8 +1292,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1306,8 +1318,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1370,7 +1382,6 @@ golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1383,8 +1394,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1397,8 +1408,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1415,8 +1426,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/integration_tests/workflow/multimatch-value-share-template.yaml b/integration_tests/workflow/multimatch-value-share-template.yaml new file mode 100644 index 0000000000..fd66975e26 --- /dev/null +++ b/integration_tests/workflow/multimatch-value-share-template.yaml @@ -0,0 +1,23 @@ +id: multimatch-value-share-template + +info: + name: MultiMatch Value Share Template + author: tovask + severity: info + +http: + - path: + - "{{BaseURL}}/path1?v=1" + - "{{BaseURL}}/path1?v=2" + matchers: + - type: word + name: test-matcher + words: + - "href" + extractors: + - type: regex + part: body + name: extracted + regex: + - 'href="(.*)"' + group: 1 diff --git a/integration_tests/workflow/multimatch-value-share-workflow.yaml b/integration_tests/workflow/multimatch-value-share-workflow.yaml new file mode 100644 index 0000000000..f197f28864 --- /dev/null +++ b/integration_tests/workflow/multimatch-value-share-workflow.yaml @@ -0,0 +1,21 @@ +id: multimatch-value-share-workflow + +info: + name: MultiMatch Value Share Workflow + author: tovask + severity: info + description: Workflow to test value sharing when multiple matches occur in the extractor template + +workflows: + - template: workflow/multimatch-value-share-template.yaml + subtemplates: + - template: workflow/match-1.yaml + subtemplates: + - template: workflow/http-value-share-template-2.yaml + - template: workflow/multimatch-value-share-template.yaml + matchers: + - name: test-matcher + subtemplates: + - template: workflow/match-1.yaml + subtemplates: + - template: workflow/http-value-share-template-2.yaml diff --git a/internal/runner/lazy.go b/internal/runner/lazy.go index eb41374513..900850b673 100644 --- a/internal/runner/lazy.go +++ b/internal/runner/lazy.go @@ -51,6 +51,7 @@ func GetAuthTmplStore(opts types.Options, catalog catalog.Catalog, execOpts prot opts.ExcludeProtocols = nil opts.IncludeConditions = nil cfg := loader.NewConfig(&opts, catalog, execOpts) + cfg.StoreId = loader.AuthStoreId store, err := loader.New(cfg) if err != nil { return nil, errorutil.NewWithErr(err).Msgf("failed to initialize dynamic auth templates store") diff --git a/internal/runner/runner.go b/internal/runner/runner.go index bfb2bc64be..0b6da592d3 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -501,8 +501,17 @@ func (r *Runner) RunEnumeration() error { } if r.options.ShouldUseHostError() { - cache := hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount, r.options.TrackError) + maxHostError := r.options.MaxHostError + if r.options.TemplateThreads > maxHostError { + gologger.Print().Msgf("[%v] The concurrency value is higher than max-host-error", r.colorizer.BrightYellow("WRN")) + gologger.Info().Msgf("Adjusting max-host-error to the concurrency value: %d", r.options.TemplateThreads) + + maxHostError = r.options.TemplateThreads + } + + cache := hosterrorscache.New(maxHostError, hosterrorscache.DefaultMaxHostsCount, r.options.TrackError) cache.SetVerbose(r.options.Verbose) + r.hostErrors = cache executorOpts.HostErrorsCache = cache } @@ -784,6 +793,52 @@ func (r *Runner) SaveResumeConfig(path string) error { return os.WriteFile(path, data, permissionutil.ConfigFilePermission) } +// upload existing scan results to cloud with progress +func UploadResultsToCloud(options *types.Options) error { + h := &pdcpauth.PDCPCredHandler{} + creds, err := h.GetCreds() + if err != nil { + return errors.Wrap(err, "could not get credentials for cloud upload") + } + ctx := context.TODO() + uploadWriter, err := pdcp.NewUploadWriter(ctx, creds) + if err != nil { + return errors.Wrap(err, "could not create upload writer") + } + if options.ScanID != "" { + _ = uploadWriter.SetScanID(options.ScanID) + } + if options.ScanName != "" { + uploadWriter.SetScanName(options.ScanName) + } + if options.TeamID != "" { + uploadWriter.SetTeamID(options.TeamID) + } + + // Open file to count the number of results first + file, err := os.Open(options.ScanUploadFile) + if err != nil { + return errors.Wrap(err, "could not open scan upload file") + } + defer file.Close() + + gologger.Info().Msgf("Uploading scan results to cloud dashboard from %s", options.ScanUploadFile) + dec := json.NewDecoder(file) + for dec.More() { + var r output.ResultEvent + err := dec.Decode(&r) + if err != nil { + gologger.Warning().Msgf("Could not decode jsonl: %s\n", err) + continue + } + if err = uploadWriter.Write(&r); err != nil { + gologger.Warning().Msgf("[%s] failed to upload: %s\n", r.TemplateID, err) + } + } + uploadWriter.Close() + return nil +} + type WalkFunc func(reflect.Value, reflect.StructField) // Walk traverses a struct and executes a callback function on each value in the struct. diff --git a/lib/sdk_private.go b/lib/sdk_private.go index b65ecf2a2f..cacd9a1ca1 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -108,14 +108,6 @@ func (e *NucleiEngine) init(ctx context.Context) error { return err } - if e.opts.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" { - httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{}) - if err != nil { - return err - } - e.httpClient = httpclient - } - e.parser = templates.NewParser() if sharedInit == nil || protocolstate.ShouldInit() { @@ -126,6 +118,14 @@ func (e *NucleiEngine) init(ctx context.Context) error { _ = protocolinit.Init(e.opts) }) + if e.opts.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" { + httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{}) + if err != nil { + return err + } + e.httpClient = httpclient + } + e.applyRequiredDefaults(ctx) var err error diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 993a744502..b4ed1d2982 100644 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -247,7 +247,12 @@ "time", "select", "files", + "waitdom", + "waitfcp", + "waitfmp", + "waitidle", "waitload", + "waitstable", "getresource", "extract", "setmethod", diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go index ff54ce9bc0..bc657aaef7 100644 --- a/pkg/catalog/config/constants.go +++ b/pkg/catalog/config/constants.go @@ -31,7 +31,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v3.3.2` + Version = `v3.3.3` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github" diff --git a/pkg/catalog/disk/find.go b/pkg/catalog/disk/find.go index a5d46ff078..0e4021b87c 100644 --- a/pkg/catalog/disk/find.go +++ b/pkg/catalog/disk/find.go @@ -216,12 +216,17 @@ func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]s // is a file, it returns true otherwise false with no errors. func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) { if c.templatesFS != nil { - absPath = strings.TrimPrefix(absPath, "/") + absPath = strings.Trim(absPath, "/") } var info fs.File if c.templatesFS == nil { info, err = os.Open(absPath) } else { + // If we were given no path, then it's not a file, it's the root, and we can quietly return. + if absPath == "" { + return "", false, nil + } + info, err = c.templatesFS.Open(absPath) } if err != nil { @@ -263,6 +268,12 @@ func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string] }, ) } else { + // For the special case of the root directory, we need to pass "." to `fs.WalkDir`. + if absPath == "" { + absPath = "." + } + absPath = strings.TrimSuffix(absPath, "/") + err = fs.WalkDir( c.templatesFS, absPath, diff --git a/pkg/catalog/disk/path.go b/pkg/catalog/disk/path.go index a55faee8f6..5c4b6dbb95 100644 --- a/pkg/catalog/disk/path.go +++ b/pkg/catalog/disk/path.go @@ -2,6 +2,7 @@ package disk import ( "fmt" + "io/fs" "os" "path/filepath" "strings" @@ -21,6 +22,11 @@ func (c *DiskCatalog) ResolvePath(templateName, second string) (string, error) { if filepath.IsAbs(templateName) { return templateName, nil } + if c.templatesFS != nil { + if potentialPath, err := c.tryResolve(templateName); err != errNoValidCombination { + return potentialPath, nil + } + } if second != "" { secondBasePath := filepath.Join(filepath.Dir(second), templateName) if potentialPath, err := c.tryResolve(secondBasePath); err != errNoValidCombination { @@ -28,17 +34,19 @@ func (c *DiskCatalog) ResolvePath(templateName, second string) (string, error) { } } - curDirectory, err := os.Getwd() - if err != nil { - return "", err - } + if c.templatesFS == nil { + curDirectory, err := os.Getwd() + if err != nil { + return "", err + } - templatePath := filepath.Join(curDirectory, templateName) - if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination { - return potentialPath, nil + templatePath := filepath.Join(curDirectory, templateName) + if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination { + return potentialPath, nil + } } - templatePath = filepath.Join(config.DefaultConfig.GetTemplateDir(), templateName) + templatePath := filepath.Join(config.DefaultConfig.GetTemplateDir(), templateName) if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination { return potentialPath, nil } @@ -50,8 +58,14 @@ var errNoValidCombination = errors.New("no valid combination found") // tryResolve attempts to load locate the target by iterating across all the folders tree func (c *DiskCatalog) tryResolve(fullPath string) (string, error) { - if fileutil.FileOrFolderExists(fullPath) { - return fullPath, nil + if c.templatesFS == nil { + if fileutil.FileOrFolderExists(fullPath) { + return fullPath, nil + } + } else { + if _, err := fs.Stat(c.templatesFS, fullPath); err == nil { + return fullPath, nil + } } return "", errNoValidCombination } diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index e1d8371e35..48f14a4054 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -15,6 +15,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/catalog" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter" + "github.com/projectdiscovery/nuclei/v3/pkg/keys" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/templates" @@ -32,6 +33,7 @@ import ( const ( httpPrefix = "http://" httpsPrefix = "https://" + AuthStoreId = "auth_store" ) var ( @@ -40,6 +42,7 @@ var ( // Config contains the configuration options for the loader type Config struct { + StoreId string // used to set store id (optional) Templates []string TemplateURLs []string Workflows []string @@ -66,6 +69,7 @@ type Config struct { // Store is a storage for loaded nuclei templates type Store struct { + id string // id of the store (optional) tagFilter *templates.TagFilter pathFilter *filter.PathFilter config *Config @@ -131,6 +135,7 @@ func New(cfg *Config) (*Store, error) { // Create a tag filter based on provided configuration store := &Store{ + id: cfg.StoreId, config: cfg, tagFilter: tagFilter, pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{ @@ -229,6 +234,10 @@ func (store *Store) ReadTemplateFromURI(uri string, remote bool) ([]byte, error) } } +func (store *Store) ID() string { + return store.id +} + // Templates returns all the templates in the store func (store *Store) Templates() []*templates.Template { return store.templates @@ -434,7 +443,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ // increment signed/unsigned counters if tmpl.Verified { if tmpl.TemplateVerifier == "" { - templates.SignatureStats[templates.PDVerifier].Add(1) + templates.SignatureStats[keys.PDVerifier].Add(1) } else { templates.SignatureStats[tmpl.TemplateVerifier].Add(1) } @@ -471,7 +480,8 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ return } // DAST only templates - if store.config.ExecutorOptions.Options.DAST { + // Skip DAST filter when loading auth templates + if store.ID() != AuthStoreId && store.config.ExecutorOptions.Options.DAST { // check if the template is a DAST template if parsed.IsFuzzing() { loadTemplate(parsed) diff --git a/pkg/core/workflow_execute.go b/pkg/core/workflow_execute.go index 028df4d01f..0b5d7e8722 100644 --- a/pkg/core/workflow_execute.go +++ b/pkg/core/workflow_execute.go @@ -139,7 +139,8 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan defer swg.Done() // create a new context with the same input but with unset callbacks - subCtx := scan.NewScanContext(ctx.Context(), ctx.Input) + // clone the Input so that other parallel executions won't overwrite the shared variables when subsequent templates are running + subCtx := scan.NewScanContext(ctx.Context(), ctx.Input.Clone()) if err := e.runWorkflowStep(subtemplate, subCtx, results, swg, w); err != nil { gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err) } diff --git a/pkg/input/formats/openapi/generator.go b/pkg/input/formats/openapi/generator.go index 63fef7e290..e22c078769 100644 --- a/pkg/input/formats/openapi/generator.go +++ b/pkg/input/formats/openapi/generator.go @@ -27,7 +27,8 @@ import ( ) const ( - globalAuth = "globalAuth" + globalAuth = "globalAuth" + DEFAULT_HTTP_SCHEME_HEADER = "Authorization" ) // GenerateRequestsFromSchema generates http requests from an OpenAPI 3.0 document object @@ -426,18 +427,20 @@ func GenerateParameterFromSecurityScheme(scheme *openapi3.SecuritySchemeRef) (*o if !generic.EqualsAny(scheme.Value.Scheme, "basic", "bearer") { return nil, errorutil.NewWithTag("openapi", "unsupported security scheme (%s) found in openapi file", scheme.Value.Scheme) } - if scheme.Value.Name == "" { - return nil, errorutil.NewWithTag("openapi", "security scheme (%s) name is empty", scheme.Value.Scheme) + // HTTP authentication schemes basic or bearer use the Authorization header + headerName := scheme.Value.Name + if headerName == "" { + headerName = DEFAULT_HTTP_SCHEME_HEADER } // create parameters using the scheme switch scheme.Value.Scheme { case "basic": - h := openapi3.NewHeaderParameter(scheme.Value.Name) + h := openapi3.NewHeaderParameter(headerName) h.Required = true h.Description = globalAuth // differentiator for normal variables and global auth return h, nil case "bearer": - h := openapi3.NewHeaderParameter(scheme.Value.Name) + h := openapi3.NewHeaderParameter(headerName) h.Required = true h.Description = globalAuth // differentiator for normal variables and global auth return h, nil diff --git a/pkg/js/libs/mysql/mysql_private.go b/pkg/js/libs/mysql/mysql_private.go index aa89df2600..42e76a5579 100644 --- a/pkg/js/libs/mysql/mysql_private.go +++ b/pkg/js/libs/mysql/mysql_private.go @@ -53,7 +53,7 @@ func BuildDSN(opts MySQLOptions) (string, error) { } target := net.JoinHostPort(opts.Host, fmt.Sprintf("%d", opts.Port)) var dsn strings.Builder - dsn.WriteString(fmt.Sprintf("%v:%v", url.QueryEscape(opts.Username), url.QueryEscape(opts.Password))) + dsn.WriteString(fmt.Sprintf("%v:%v", url.QueryEscape(opts.Username), opts.Password)) dsn.WriteString("@") dsn.WriteString(fmt.Sprintf("%v(%v)", opts.Protocol, target)) if opts.DbName != "" { diff --git a/pkg/keys/key.go b/pkg/keys/key.go index bb0c900626..eac43f27ae 100644 --- a/pkg/keys/key.go +++ b/pkg/keys/key.go @@ -3,5 +3,7 @@ package keys import _ "embed" +const PDVerifier = "projectdiscovery/nuclei-templates" + //go:embed nuclei.crt var NucleiCert []byte // public key for verifying digital signature of templates diff --git a/pkg/output/output.go b/pkg/output/output.go index 4f02d71c38..fbc9f71306 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "log/slog" + "maps" "os" "path/filepath" "regexp" @@ -99,6 +100,15 @@ type InternalWrappedEvent struct { InteractshMatched atomic.Bool } +func (iwe *InternalWrappedEvent) CloneShallow() *InternalWrappedEvent { + return &InternalWrappedEvent{ + InternalEvent: maps.Clone(iwe.InternalEvent), + Results: nil, + OperatorsResult: nil, + UsesInteractsh: iwe.UsesInteractsh, + } +} + func (iwe *InternalWrappedEvent) HasOperatorResult() bool { iwe.RLock() defer iwe.RUnlock() @@ -190,6 +200,7 @@ type ResultEvent struct { FuzzingPosition string `json:"fuzzing_position,omitempty"` FileToIndexPosition map[string]int `json:"-"` + TemplateVerifier string `json:"-"` Error string `json:"error,omitempty"` } @@ -263,7 +274,7 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) { func (w *StandardWriter) Write(event *ResultEvent) error { // Enrich the result event with extra metadata on the template-path and url. if event.TemplatePath != "" { - event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath), types.ToString(event.TemplateID)) + event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath), types.ToString(event.TemplateID), event.TemplateVerifier) } if len(w.KeysToRedact) > 0 { @@ -435,7 +446,7 @@ func (w *StandardWriter) WriteFailure(wrappedEvent *InternalWrappedEvent) error // if no results were found, manually create a failure event event := wrappedEvent.InternalEvent - templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"])) + templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"]), types.ToString(event["template-verifier"])) var templateInfo model.Info if event["template-info"] != nil { templateInfo = event["template-info"].(model.Info) diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index 7736113dca..87a0128645 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -348,6 +348,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Matched: types.ToString(wrapped.InternalEvent["input"]), Host: fields.Host, diff --git a/pkg/protocols/common/generators/validate.go b/pkg/protocols/common/generators/validate.go index adb537d289..0aa0737149 100644 --- a/pkg/protocols/common/generators/validate.go +++ b/pkg/protocols/common/generators/validate.go @@ -22,6 +22,13 @@ func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePat return errors.New("invalid number of lines in payload") } + // For historical reasons, "validate" checks to see if the payload file exist. + // If we're using a custom helper function, then we need to skip any validation beyond just checking the string syntax. + // Actually attempting to load the file will determine whether or not it exists. + if g.options.LoadHelperFileFunction != nil { + return nil + } + // check if it's a file and try to load it if fileutil.FileExists(payloadType) { continue diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go index 1630e97c7d..bcfa27dbe4 100644 --- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -75,24 +75,34 @@ func (c *Cache) Close() { c.failedTargets.Purge() } -func (c *Cache) normalizeCacheValue(value string) string { - finalValue := value - if strings.HasPrefix(value, "http") { - if parsed, err := url.Parse(value); err == nil { - hostname := parsed.Host - finalPort := parsed.Port() - if finalPort == "" { - if parsed.Scheme == "https" { - finalPort = "443" - } else { - finalPort = "80" - } - hostname = net.JoinHostPort(parsed.Host, finalPort) +// NormalizeCacheValue processes the input value and returns a normalized cache +// value. +func (c *Cache) NormalizeCacheValue(value string) string { + var normalizedValue string = value + + u, err := url.ParseRequestURI(value) + if err != nil || u.Host == "" { + u, err2 := url.ParseRequestURI("https://" + value) + if err2 != nil { + return normalizedValue + } + + normalizedValue = u.Host + } else { + port := u.Port() + if port == "" { + switch u.Scheme { + case "https": + normalizedValue = net.JoinHostPort(u.Host, "443") + case "http": + normalizedValue = net.JoinHostPort(u.Host, "80") } - finalValue = hostname + } else { + normalizedValue = u.Host } } - return finalValue + + return normalizedValue } // ErrUnresponsiveHost is returned when a host is unresponsive @@ -166,7 +176,7 @@ func (c *Cache) GetKeyFromContext(ctx *contextargs.Context, err error) string { address = tmp.String() } } - finalValue := c.normalizeCacheValue(address) + finalValue := c.NormalizeCacheValue(address) return finalValue } diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go index 112690d875..3c93177674 100644 --- a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go +++ b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go @@ -109,7 +109,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) { // the cache is not atomic during items creation, so we pre-create them with counter to zero for _, test := range tests { - normalizedValue := cache.normalizeCacheValue(test.host) + normalizedValue := cache.NormalizeCacheValue(test.host) newItem := &cacheItem{errors: atomic.Int32{}} newItem.errors.Store(0) _ = cache.failedTargets.Set(normalizedValue, newItem) @@ -131,7 +131,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) { for _, test := range tests { require.True(t, cache.Check(newCtxArgs(test.host))) - normalizedCacheValue := cache.normalizeCacheValue(test.host) + normalizedCacheValue := cache.NormalizeCacheValue(test.host) failedTarget, err := cache.failedTargets.Get(normalizedCacheValue) require.Nil(t, err) require.NotNil(t, failedTarget) diff --git a/pkg/protocols/dns/operators.go b/pkg/protocols/dns/operators.go index 7b8467646b..0f18315303 100644 --- a/pkg/protocols/dns/operators.go +++ b/pkg/protocols/dns/operators.go @@ -114,6 +114,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), diff --git a/pkg/protocols/file/operators.go b/pkg/protocols/file/operators.go index ff18af0979..5dab75bb91 100644 --- a/pkg/protocols/file/operators.go +++ b/pkg/protocols/file/operators.go @@ -104,6 +104,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Path: types.ToString(wrapped.InternalEvent["path"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), diff --git a/pkg/protocols/headless/engine/action_types.go b/pkg/protocols/headless/engine/action_types.go index d61e377431..d66fe91130 100644 --- a/pkg/protocols/headless/engine/action_types.go +++ b/pkg/protocols/headless/engine/action_types.go @@ -45,9 +45,24 @@ const ( // ActionFilesInput performs an action on a file input. // name:files ActionFilesInput - // ActionWaitLoad waits for the page to stop loading. + // ActionWaitDOM waits for the HTML document has been completely loaded & parsed. + // name:waitdom + ActionWaitDOM + // ActionWaitFCP waits for the first piece of content (text, image, etc.) is painted on the screen. + // name:waitfcp + ActionWaitFCP + // ActionWaitFMP waits for page has rendered enough meaningful content to be useful to the user. + // name:waitfmp + ActionWaitFMP + // ActionWaitIdle waits for the network is completely idle (no ongoing network requests). + // name:waitidle + ActionWaitIdle + // ActionWaitLoad waits for the page and all its resources (like stylesheets and images) have finished loading. // name:waitload ActionWaitLoad + // ActionWaitStable waits until the page is stable. + // name:waitstable + ActionWaitStable // ActionGetResource performs a get resource action on an element // name:getresource ActionGetResource @@ -102,7 +117,12 @@ var ActionStringToAction = map[string]ActionType{ "time": ActionTimeInput, "select": ActionSelectInput, "files": ActionFilesInput, + "waitdom": ActionWaitDOM, + "waitfcp": ActionWaitFCP, + "waitfmp": ActionWaitFMP, + "waitidle": ActionWaitIdle, "waitload": ActionWaitLoad, + "waitstable": ActionWaitStable, "getresource": ActionGetResource, "extract": ActionExtract, "setmethod": ActionSetMethod, @@ -129,7 +149,12 @@ var ActionToActionString = map[ActionType]string{ ActionTimeInput: "time", ActionSelectInput: "select", ActionFilesInput: "files", + ActionWaitDOM: "waitdom", + ActionWaitFCP: "waitfcp", + ActionWaitFMP: "waitfmp", + ActionWaitIdle: "waitidle", ActionWaitLoad: "waitload", + ActionWaitStable: "waitstable", ActionGetResource: "getresource", ActionExtract: "extract", ActionSetMethod: "setmethod", diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index b34ed18b94..18ed515662 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -94,8 +94,28 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var err = p.TimeInputElement(act, outData) case ActionSelectInput: err = p.SelectInputElement(act, outData) + case ActionWaitDOM: + event := proto.PageLifecycleEventNameDOMContentLoaded + err = p.WaitPageLifecycleEvent(act, outData, event) + case ActionWaitFCP: + event := proto.PageLifecycleEventNameFirstContentfulPaint + err = p.WaitPageLifecycleEvent(act, outData, event) + case ActionWaitFMP: + event := proto.PageLifecycleEventNameFirstMeaningfulPaint + err = p.WaitPageLifecycleEvent(act, outData, event) + case ActionWaitIdle: + event := proto.PageLifecycleEventNameNetworkIdle + err = p.WaitPageLifecycleEvent(act, outData, event) case ActionWaitLoad: - err = p.WaitLoad(act, outData) + event := proto.PageLifecycleEventNameLoad + err = p.WaitPageLifecycleEvent(act, outData, event) + case ActionWaitStable: + err = p.WaitStable(act, outData) + // NOTE(dwisiswant0): Mapping `ActionWaitLoad` to `Page.WaitStable`, + // just in case waiting for the `proto.PageLifecycleEventNameLoad` event + // doesn't meet expectations. + // case ActionWaitLoad, ActionWaitStable: + // err = p.WaitStable(act, outData) case ActionGetResource: err = p.GetResource(act, outData) case ActionExtract: @@ -204,6 +224,17 @@ func createBackOffSleeper(pollTimeout, timeout time.Duration) utils.Sleeper { } } +func getNavigationFunc(p *Page, act *Action, event proto.PageLifecycleEventName) (func(), error) { + dur, err := getTimeout(p, act) + if err != nil { + return nil, errors.Wrap(err, "Wrong timeout given") + } + + fn := p.page.Timeout(dur).WaitNavigation(event) + + return fn, nil +} + func getTimeout(p *Page, act *Action) (time.Duration, error) { return geTimeParameter(p, act, "timeout", 3, time.Second) } @@ -518,20 +549,38 @@ func (p *Page) SelectInputElement(act *Action, out ActionData) error { return nil } -// WaitLoad waits for the page to load -func (p *Page) WaitLoad(act *Action, out ActionData) error { - p.page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)() - - // Wait for the window.onload event and also wait for the network requests - // to become idle for a maximum duration of 3 seconds. If the requests - // do not finish, - if err := p.page.WaitLoad(); err != nil { - return errors.Wrap(err, "could not wait load event") +// WaitPageLifecycleEvent waits for specified page lifecycle event name +func (p *Page) WaitPageLifecycleEvent(act *Action, out ActionData, event proto.PageLifecycleEventName) error { + fn, err := getNavigationFunc(p, act, event) + if err != nil { + return err } - _ = p.page.WaitIdle(1 * time.Second) + + fn() + return nil } +// WaitStable waits until the page is stable +func (p *Page) WaitStable(act *Action, out ActionData) error { + var dur time.Duration = time.Second // default stable page duration: 1s + + timeout, err := getTimeout(p, act) + if err != nil { + return errors.Wrap(err, "Wrong timeout given") + } + + argDur := act.Data["duration"] + if argDur != "" { + dur, err = time.ParseDuration(argDur) + if err != nil { + dur = time.Second + } + } + + return p.page.Timeout(timeout).WaitStable(dur) +} + // GetResource gets a resource from an element from page. func (p *Page) GetResource(act *Action, out ActionData) error { element, err := p.pageElementBy(act.Data) diff --git a/pkg/protocols/headless/engine/page_actions_test.go b/pkg/protocols/headless/engine/page_actions_test.go index 3620bb2bb4..bc85ea7417 100644 --- a/pkg/protocols/headless/engine/page_actions_test.go +++ b/pkg/protocols/headless/engine/page_actions_test.go @@ -201,7 +201,7 @@ func TestActionScreenshot(t *testing.T) { filePath := filepath.Join(os.TempDir(), "test.png") actions := []*Action{ {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitFMP}}, {ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": filePath}}, } @@ -229,7 +229,7 @@ func TestActionScreenshotToDir(t *testing.T) { actions := []*Action{ {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitFMP}}, {ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": filePath, "mkdir": "true"}}, } diff --git a/pkg/protocols/headless/operators.go b/pkg/protocols/headless/operators.go index d1b790f99c..3ad30e9c55 100644 --- a/pkg/protocols/headless/operators.go +++ b/pkg/protocols/headless/operators.go @@ -137,6 +137,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Host: fields.Host, Path: fields.Path, diff --git a/pkg/protocols/http/operators.go b/pkg/protocols/http/operators.go index 3de3e63e34..d630bfd8b0 100644 --- a/pkg/protocols/http/operators.go +++ b/pkg/protocols/http/operators.go @@ -170,6 +170,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Host: fields.Host, Port: fields.Port, diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 5a1219b83f..180ee512ce 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -1042,6 +1042,10 @@ func (request *Request) validateNFixEvent(input *contextargs.Context, gr *genera // addCNameIfAvailable adds the cname to the event if available func (request *Request) addCNameIfAvailable(hostname string, outputEvent map[string]interface{}) { + if protocolstate.Dialer == nil { + return + } + data, err := protocolstate.Dialer.GetDNSData(hostname) if err == nil { switch len(data.CNAME) { diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index fbcd1a6ff9..0bd26b88a5 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -766,6 +766,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Host: fields.Host, Port: fields.Port, diff --git a/pkg/protocols/network/operators.go b/pkg/protocols/network/operators.go index 4ff22130f4..2aa19e5b3b 100644 --- a/pkg/protocols/network/operators.go +++ b/pkg/protocols/network/operators.go @@ -100,13 +100,14 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent fields.Ip = types.ToString(wrapped.InternalEvent["ip"]) } data := &output.ResultEvent{ - TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), - TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), - Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: types.ToString(wrapped.InternalEvent["type"]), - Host: fields.Host, - Port: fields.Port, - URL: fields.URL, + TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), + Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, + Type: types.ToString(wrapped.InternalEvent["type"]), + Host: fields.Host, + Port: fields.Port, + URL: fields.URL, Matched: types.ToString(wrapped.InternalEvent["matched"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Metadata: wrapped.OperatorsResult.PayloadValues, diff --git a/pkg/protocols/offlinehttp/operators.go b/pkg/protocols/offlinehttp/operators.go index 0d906ff514..f69abc4412 100644 --- a/pkg/protocols/offlinehttp/operators.go +++ b/pkg/protocols/offlinehttp/operators.go @@ -142,6 +142,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Path: types.ToString(wrapped.InternalEvent["path"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 8b8854cd8b..a9d50c1481 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -61,6 +61,8 @@ type ExecutorOptions struct { TemplatePath string // TemplateInfo contains information block of the template request TemplateInfo model.Info + // TemplateVerifier is the verifier for the template + TemplateVerifier string // RawTemplate is the raw template for the request RawTemplate []byte // Output is a writer interface for writing output events from executer. diff --git a/pkg/protocols/ssl/ssl.go b/pkg/protocols/ssl/ssl.go index 50da53111b..681043d3bf 100644 --- a/pkg/protocols/ssl/ssl.go +++ b/pkg/protocols/ssl/ssl.go @@ -398,6 +398,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Host: fields.Host, Port: fields.Port, diff --git a/pkg/protocols/websocket/websocket.go b/pkg/protocols/websocket/websocket.go index a956de0e3f..aa099ef43a 100644 --- a/pkg/protocols/websocket/websocket.go +++ b/pkg/protocols/websocket/websocket.go @@ -402,6 +402,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Host: fields.Host, Port: fields.Port, diff --git a/pkg/protocols/whois/whois.go b/pkg/protocols/whois/whois.go index ce0bafe56c..9963ec19f8 100644 --- a/pkg/protocols/whois/whois.go +++ b/pkg/protocols/whois/whois.go @@ -177,6 +177,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Metadata: wrapped.OperatorsResult.PayloadValues, diff --git a/pkg/reporting/options.go b/pkg/reporting/options.go index 06a749d658..c5090de014 100644 --- a/pkg/reporting/options.go +++ b/pkg/reporting/options.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitlab" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear" "github.com/projectdiscovery/retryablehttp-go" ) @@ -29,6 +30,8 @@ type Options struct { Gitea *gitea.Options `yaml:"gitea"` // Jira contains configuration options for Jira Issue Tracker Jira *jira.Options `yaml:"jira"` + // Linear contains configuration options for Linear Issue Tracker + Linear *linear.Options `yaml:"linear"` // MarkdownExporter contains configuration options for Markdown Exporter Module MarkdownExporter *markdown.Options `yaml:"markdown"` // SarifExporter contains configuration options for Sarif Exporter Module diff --git a/pkg/reporting/reporting.go b/pkg/reporting/reporting.go index 889f92f3f7..c6a7d63e10 100644 --- a/pkg/reporting/reporting.go +++ b/pkg/reporting/reporting.go @@ -28,6 +28,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitlab" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" ) @@ -112,6 +113,15 @@ func New(options *Options, db string, doNotDedupe bool) (Client, error) { } client.trackers = append(client.trackers, tracker) } + if options.Linear != nil { + options.Linear.HttpClient = options.HttpClient + options.Linear.OmitRaw = options.OmitRaw + tracker, err := linear.New(options.Linear) + if err != nil { + return nil, errorutil.NewWithErr(err).Wrap(ErrReportingClientCreation) + } + client.trackers = append(client.trackers, tracker) + } if options.MarkdownExporter != nil { exporter, err := markdown.New(options.MarkdownExporter) if err != nil { @@ -195,6 +205,7 @@ func CreateConfigIfNotExists() error { GitLab: &gitlab.Options{}, Gitea: &gitea.Options{}, Jira: &jira.Options{}, + Linear: &linear.Options{}, MarkdownExporter: &markdown.Options{}, SarifExporter: &sarif.Options{}, ElasticsearchExporter: &es.Options{}, diff --git a/pkg/reporting/trackers/linear/jsonutil/jsonutil.go b/pkg/reporting/trackers/linear/jsonutil/jsonutil.go new file mode 100644 index 0000000000..cf66b7aa1e --- /dev/null +++ b/pkg/reporting/trackers/linear/jsonutil/jsonutil.go @@ -0,0 +1,312 @@ +// Package jsonutil provides a function for decoding JSON +// into a GraphQL query data structure. +// +// Taken from: https://github.com/shurcooL/graphql/blob/ed46e5a4646634fc16cb07c3b8db389542cc8847/internal/jsonutil/graphql.go +package jsonutil + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "reflect" + "strings" +) + +// UnmarshalGraphQL parses the JSON-encoded GraphQL response data and stores +// the result in the GraphQL query data structure pointed to by v. +// +// The implementation is created on top of the JSON tokenizer available +// in "encoding/json".Decoder. +func UnmarshalGraphQL(data []byte, v any) error { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + err := (&decoder{tokenizer: dec}).Decode(v) + if err != nil { + return err + } + tok, err := dec.Token() + switch err { + case io.EOF: + // Expect to get io.EOF. There shouldn't be any more + // tokens left after we've decoded v successfully. + return nil + case nil: + return fmt.Errorf("invalid token '%v' after top-level value", tok) + default: + return err + } +} + +// decoder is a JSON decoder that performs custom unmarshaling behavior +// for GraphQL query data structures. It's implemented on top of a JSON tokenizer. +type decoder struct { + tokenizer interface { + Token() (json.Token, error) + } + + // Stack of what part of input JSON we're in the middle of - objects, arrays. + parseState []json.Delim + + // Stacks of values where to unmarshal. + // The top of each stack is the reflect.Value where to unmarshal next JSON value. + // + // The reason there's more than one stack is because we might be unmarshaling + // a single JSON value into multiple GraphQL fragments or embedded structs, so + // we keep track of them all. + vs [][]reflect.Value +} + +// Decode decodes a single JSON value from d.tokenizer into v. +func (d *decoder) Decode(v any) error { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr { + return fmt.Errorf("cannot decode into non-pointer %T", v) + } + d.vs = [][]reflect.Value{{rv.Elem()}} + return d.decode() +} + +// decode decodes a single JSON value from d.tokenizer into d.vs. +func (d *decoder) decode() error { + // The loop invariant is that the top of each d.vs stack + // is where we try to unmarshal the next JSON value we see. + for len(d.vs) > 0 { + tok, err := d.tokenizer.Token() + if err == io.EOF { + return errors.New("unexpected end of JSON input") + } else if err != nil { + return err + } + + switch { + + // Are we inside an object and seeing next key (rather than end of object)? + case d.state() == '{' && tok != json.Delim('}'): + key, ok := tok.(string) + if !ok { + return errors.New("unexpected non-key in JSON input") + } + someFieldExist := false + for i := range d.vs { + v := d.vs[i][len(d.vs[i])-1] + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + var f reflect.Value + if v.Kind() == reflect.Struct { + f = fieldByGraphQLName(v, key) + if f.IsValid() { + someFieldExist = true + } + } + d.vs[i] = append(d.vs[i], f) + } + if !someFieldExist { + return fmt.Errorf("struct field for %q doesn't exist in any of %v places to unmarshal", key, len(d.vs)) + } + + // We've just consumed the current token, which was the key. + // Read the next token, which should be the value, and let the rest of code process it. + tok, err = d.tokenizer.Token() + if err == io.EOF { + return errors.New("unexpected end of JSON input") + } else if err != nil { + return err + } + + // Are we inside an array and seeing next value (rather than end of array)? + case d.state() == '[' && tok != json.Delim(']'): + someSliceExist := false + for i := range d.vs { + v := d.vs[i][len(d.vs[i])-1] + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + var f reflect.Value + if v.Kind() == reflect.Slice { + v.Set(reflect.Append(v, reflect.Zero(v.Type().Elem()))) // v = append(v, T). + f = v.Index(v.Len() - 1) + someSliceExist = true + } + d.vs[i] = append(d.vs[i], f) + } + if !someSliceExist { + return fmt.Errorf("slice doesn't exist in any of %v places to unmarshal", len(d.vs)) + } + } + + switch tok := tok.(type) { + case string, json.Number, bool, nil: + // Value. + + for i := range d.vs { + v := d.vs[i][len(d.vs[i])-1] + if !v.IsValid() { + continue + } + err := unmarshalValue(tok, v) + if err != nil { + return err + } + } + d.popAllVs() + + case json.Delim: + switch tok { + case '{': + // Start of object. + + d.pushState(tok) + + frontier := make([]reflect.Value, len(d.vs)) // Places to look for GraphQL fragments/embedded structs. + for i := range d.vs { + v := d.vs[i][len(d.vs[i])-1] + frontier[i] = v + // TODO: Do this recursively or not? Add a test case if needed. + if v.Kind() == reflect.Ptr && v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) // v = new(T). + } + } + // Find GraphQL fragments/embedded structs recursively, adding to frontier + // as new ones are discovered and exploring them further. + for len(frontier) > 0 { + v := frontier[0] + frontier = frontier[1:] + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + if v.Kind() != reflect.Struct { + continue + } + for i := 0; i < v.NumField(); i++ { + if isGraphQLFragment(v.Type().Field(i)) || v.Type().Field(i).Anonymous { + // Add GraphQL fragment or embedded struct. + d.vs = append(d.vs, []reflect.Value{v.Field(i)}) + frontier = append(frontier, v.Field(i)) + } + } + } + case '[': + // Start of array. + + d.pushState(tok) + + for i := range d.vs { + v := d.vs[i][len(d.vs[i])-1] + // TODO: Confirm this is needed, write a test case. + //if v.Kind() == reflect.Ptr && v.IsNil() { + // v.Set(reflect.New(v.Type().Elem())) // v = new(T). + //} + + // Reset slice to empty (in case it had non-zero initial value). + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + if v.Kind() != reflect.Slice { + continue + } + v.Set(reflect.MakeSlice(v.Type(), 0, 0)) // v = make(T, 0, 0). + } + case '}', ']': + // End of object or array. + d.popAllVs() + d.popState() + default: + return errors.New("unexpected delimiter in JSON input") + } + default: + return errors.New("unexpected token in JSON input") + } + } + return nil +} + +// pushState pushes a new parse state s onto the stack. +func (d *decoder) pushState(s json.Delim) { + d.parseState = append(d.parseState, s) +} + +// popState pops a parse state (already obtained) off the stack. +// The stack must be non-empty. +func (d *decoder) popState() { + d.parseState = d.parseState[:len(d.parseState)-1] +} + +// state reports the parse state on top of stack, or 0 if empty. +func (d *decoder) state() json.Delim { + if len(d.parseState) == 0 { + return 0 + } + return d.parseState[len(d.parseState)-1] +} + +// popAllVs pops from all d.vs stacks, keeping only non-empty ones. +func (d *decoder) popAllVs() { + var nonEmpty [][]reflect.Value + for i := range d.vs { + d.vs[i] = d.vs[i][:len(d.vs[i])-1] + if len(d.vs[i]) > 0 { + nonEmpty = append(nonEmpty, d.vs[i]) + } + } + d.vs = nonEmpty +} + +// fieldByGraphQLName returns an exported struct field of struct v +// that matches GraphQL name, or invalid reflect.Value if none found. +func fieldByGraphQLName(v reflect.Value, name string) reflect.Value { + for i := 0; i < v.NumField(); i++ { + if v.Type().Field(i).PkgPath != "" { + // Skip unexported field. + continue + } + if hasGraphQLName(v.Type().Field(i), name) { + return v.Field(i) + } + } + return reflect.Value{} +} + +// hasGraphQLName reports whether struct field f has GraphQL name. +func hasGraphQLName(f reflect.StructField, name string) bool { + value, ok := f.Tag.Lookup("graphql") + if !ok { + // TODO: caseconv package is relatively slow. Optimize it, then consider using it here. + //return caseconv.MixedCapsToLowerCamelCase(f.Name) == name + return strings.EqualFold(f.Name, name) + } + value = strings.TrimSpace(value) // TODO: Parse better. + if strings.HasPrefix(value, "...") { + // GraphQL fragment. It doesn't have a name. + return false + } + // Cut off anything that follows the field name, + // such as field arguments, aliases, directives. + if i := strings.IndexAny(value, "(:@"); i != -1 { + value = value[:i] + } + return strings.TrimSpace(value) == name +} + +// isGraphQLFragment reports whether struct field f is a GraphQL fragment. +func isGraphQLFragment(f reflect.StructField) bool { + value, ok := f.Tag.Lookup("graphql") + if !ok { + return false + } + value = strings.TrimSpace(value) // TODO: Parse better. + return strings.HasPrefix(value, "...") +} + +// unmarshalValue unmarshals JSON value into v. +// v must be addressable and not obtained by the use of unexported +// struct fields, otherwise unmarshalValue will panic. +func unmarshalValue(value json.Token, v reflect.Value) error { + b, err := json.Marshal(value) // TODO: Short-circuit (if profiling says it's worth it). + if err != nil { + return err + } + return json.Unmarshal(b, v.Addr().Interface()) +} diff --git a/pkg/reporting/trackers/linear/linear.go b/pkg/reporting/trackers/linear/linear.go new file mode 100644 index 0000000000..11712039f8 --- /dev/null +++ b/pkg/reporting/trackers/linear/linear.go @@ -0,0 +1,409 @@ +package linear + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + + "github.com/shurcooL/graphql" + + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v3/pkg/output" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/format" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear/jsonutil" + "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/retryablehttp-go" +) + +// Integration is a client for linear issue tracker integration +type Integration struct { + url string + httpclient *http.Client + options *Options +} + +// Options contains the configuration options for linear issue tracker client +type Options struct { + // APIKey is the API key for linear account. + APIKey string `yaml:"api-key" validate:"required"` + + // AllowList contains a list of allowed events for this tracker + AllowList *filters.Filter `yaml:"allow-list"` + // DenyList contains a list of denied events for this tracker + DenyList *filters.Filter `yaml:"deny-list"` + + // TeamID is the team id for the project + TeamID string `yaml:"team-id"` + // ProjectID is the project id for the project + ProjectID string `yaml:"project-id"` + // DuplicateIssueCheck is a bool to enable duplicate tracking issue check and update the newest + DuplicateIssueCheck bool `yaml:"duplicate-issue-check" default:"false"` + + // OpenStateID is the id of the open state for the project + OpenStateID string `yaml:"open-state-id"` + + HttpClient *retryablehttp.Client `yaml:"-"` + OmitRaw bool `yaml:"-"` +} + +// New creates a new issue tracker integration client based on options. +func New(options *Options) (*Integration, error) { + httpClient := &http.Client{ + Transport: &addHeaderTransport{ + T: http.DefaultTransport, + Key: options.APIKey, + }, + } + + integration := &Integration{ + url: "https://api.linear.app/graphql", + options: options, + httpclient: httpClient, + } + + return integration, nil +} + +// CreateIssue creates an issue in the tracker +func (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIssueResponse, error) { + summary := format.Summary(event) + description := format.CreateReportDescription(event, util.MarkdownFormatter{}, i.options.OmitRaw) + _ = description + + ctx := context.Background() + + var err error + var existingIssue *linearIssue + if i.options.DuplicateIssueCheck { + existingIssue, err = i.findIssueByTitle(ctx, summary) + if err != nil && !errors.Is(err, io.EOF) { + return nil, err + } + } + + if existingIssue == nil { + // Create a new issue + createdIssue, err := i.createIssueLinear(ctx, summary, description, priorityFromSeverity(event.Info.SeverityHolder.Severity)) + if err != nil { + return nil, err + } + return &filters.CreateIssueResponse{ + IssueID: types.ToString(createdIssue.ID), + IssueURL: types.ToString(createdIssue.URL), + }, nil + } else { + if existingIssue.State.Name == "Done" { + // Update the issue state to open + var issueUpdateInput struct { + StateID string `json:"stateId"` + } + issueUpdateInput.StateID = i.options.OpenStateID + variables := map[string]interface{}{ + "issueUpdateInput": issueUpdateInput, + "issueID": types.ToString(existingIssue.ID), + } + var resp struct { + IssueUpdate struct { + LastSyncID int `json:"lastSyncId"` + } + } + err := i.doGraphqlRequest(ctx, existingIssueUpdateStateMutation, &resp, variables, "IssueUpdate") + if err != nil { + return nil, fmt.Errorf("error reopening issue %s: %s", existingIssue.ID, err) + } + } + + commentInput := map[string]interface{}{ + "issueId": types.ToString(existingIssue.ID), + "body": description, + } + variables := map[string]interface{}{ + "commentCreateInput": commentInput, + } + var resp struct { + CommentCreate struct { + LastSyncID int `json:"lastSyncId"` + } + } + err := i.doGraphqlRequest(ctx, commentCreateExistingTicketMutation, &resp, variables, "CommentCreate") + if err != nil { + return nil, fmt.Errorf("error commenting on issue %s: %s", existingIssue.ID, err) + } + return &filters.CreateIssueResponse{ + IssueID: types.ToString(existingIssue.ID), + IssueURL: types.ToString(existingIssue.URL), + }, nil + } +} + +func priorityFromSeverity(sev severity.Severity) float64 { + switch sev { + case severity.Critical: + return linearPriorityCritical + case severity.High: + return linearPriorityHigh + case severity.Medium: + return linearPriorityMedium + case severity.Low: + return linearPriorityLow + default: + return linearPriorityNone + } +} + +type createIssueMutation struct { + IssueCreate struct { + Issue struct { + ID graphql.ID + Title graphql.String + Identifier graphql.String + State struct { + Name graphql.String + } + URL graphql.String + } + } +} + +const ( + createIssueGraphQLMutation = `mutation CreateIssue($input: IssueCreateInput!) { + issueCreate(input: $input) { + issue { + id + title + identifier + state { + name + } + url + } + } +}` + + searchExistingTicketQuery = `query ($teamID: ID, $projectID: ID, $title: String!) { + issues(filter: { + title: { eq: $title }, + team: { id: { eq: $teamID } } + project: { id: { eq: $projectID } } + }) { + nodes { + id + title + identifier + state { + name + } + url + } + } +} +` + + existingIssueUpdateStateMutation = `mutation IssueUpdate($issueUpdateInput: IssueUpdateInput!, $issueID: String!) { + issueUpdate(input: $issueUpdateInput, id: $issueID) { + lastSyncId + } +} +` + + commentCreateExistingTicketMutation = `mutation CommentCreate($commentCreateInput: CommentCreateInput!) { + commentCreate(input: $commentCreateInput) { + lastSyncId + } +} +` +) + +func (i *Integration) createIssueLinear(ctx context.Context, title, description string, priority float64) (*linearIssue, error) { + var mutation createIssueMutation + input := map[string]interface{}{ + "title": title, + "description": description, + "priority": priority, + } + if i.options.TeamID != "" { + input["teamId"] = graphql.ID(i.options.TeamID) + } + if i.options.ProjectID != "" { + input["projectId"] = i.options.ProjectID + } + + variables := map[string]interface{}{ + "input": input, + } + + err := i.doGraphqlRequest(ctx, createIssueGraphQLMutation, &mutation, variables, "CreateIssue") + if err != nil { + return nil, err + } + + return &linearIssue{ + ID: mutation.IssueCreate.Issue.ID, + Title: mutation.IssueCreate.Issue.Title, + Identifier: mutation.IssueCreate.Issue.Identifier, + State: struct { + Name graphql.String + }{ + Name: mutation.IssueCreate.Issue.State.Name, + }, + URL: mutation.IssueCreate.Issue.URL, + }, nil +} + +func (i *Integration) findIssueByTitle(ctx context.Context, title string) (*linearIssue, error) { + var query findExistingIssuesSearch + variables := map[string]interface{}{ + "title": graphql.String(title), + } + if i.options.TeamID != "" { + variables["teamId"] = graphql.ID(i.options.TeamID) + } + if i.options.ProjectID != "" { + variables["projectID"] = graphql.ID(i.options.ProjectID) + } + + err := i.doGraphqlRequest(ctx, searchExistingTicketQuery, &query, variables, "") + if err != nil { + return nil, err + } + + if len(query.Issues.Nodes) > 0 { + return &query.Issues.Nodes[0], nil + } + return nil, io.EOF +} + +func (i *Integration) Name() string { + return "linear" +} + +func (i *Integration) CloseIssue(event *output.ResultEvent) error { + // TODO: Unimplemented for now as not used in many places + // and overhead of maintaining our own API for this. + // This is too much code as it is :( + return nil +} + +// ShouldFilter determines if an issue should be logged to this tracker +func (i *Integration) ShouldFilter(event *output.ResultEvent) bool { + if i.options.AllowList != nil && !i.options.AllowList.GetMatch(event) { + return false + } + + if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) { + return false + } + + return true +} + +type linearIssue struct { + ID graphql.ID + Title graphql.String + Identifier graphql.String + State struct { + Name graphql.String + } + URL graphql.String +} + +type findExistingIssuesSearch struct { + Issues struct { + Nodes []linearIssue + } +} + +// Custom transport to add the API key to the header +type addHeaderTransport struct { + T http.RoundTripper + Key string +} + +func (adt *addHeaderTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Add("Authorization", adt.Key) + return adt.T.RoundTrip(req) +} + +const ( + linearPriorityNone = float64(0) + linearPriorityCritical = float64(1) + linearPriorityHigh = float64(2) + linearPriorityMedium = float64(3) + linearPriorityLow = float64(4) +) + +// errors represents the "errors" array in a response from a GraphQL server. +// If returned via error interface, the slice is expected to contain at least 1 element. +// +// Specification: https://spec.graphql.org/October2021/#sec-Errors. +type errorsGraphql []struct { + Message string + Locations []struct { + Line int + Column int + } +} + +// Error implements error interface. +func (e errorsGraphql) Error() string { + return e[0].Message +} + +// do executes a single GraphQL operation. +func (i *Integration) doGraphqlRequest(ctx context.Context, query string, v any, variables map[string]any, operationName string) error { + in := struct { + Query string `json:"query"` + Variables map[string]any `json:"variables,omitempty"` + OperationName string `json:"operationName,omitempty"` + }{ + Query: query, + Variables: variables, + OperationName: operationName, + } + + var buf bytes.Buffer + err := json.NewEncoder(&buf).Encode(in) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, i.url, &buf) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := i.httpclient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("non-200 OK status code: %v body: %q", resp.Status, body) + } + var out struct { + Data *json.RawMessage + Errors errorsGraphql + //Extensions any // Unused. + } + + err = json.NewDecoder(resp.Body).Decode(&out) + if err != nil { + return err + } + if out.Data != nil { + err := jsonutil.UnmarshalGraphQL(*out.Data, v) + if err != nil { + return err + } + } + if len(out.Errors) > 0 { + return out.Errors + } + return nil +} diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go index 45456ddcac..51b98007a6 100644 --- a/pkg/scan/scan_context.go +++ b/pkg/scan/scan_context.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/utils/errkit" ) type ScanContextOption func(*ScanContext) @@ -30,7 +31,7 @@ type ScanContext struct { OnWarning func(string) // unexported state fields - errors []error + error error warnings []string events []*output.InternalWrappedEvent results []*output.ResultEvent @@ -52,8 +53,8 @@ func (s *ScanContext) Context() context.Context { return s.ctx } -func (s *ScanContext) GenerateErrorMessage() string { - return joinErrors(s.errors) +func (s *ScanContext) GenerateErrorMessage() error { + return s.error } // GenerateResult returns final results slice from all events @@ -94,13 +95,16 @@ func (s *ScanContext) LogError(err error) { if err == nil { return } - if s.OnError != nil { s.OnError(err) } - s.errors = append(s.errors, err) + if s.error == nil { + s.error = err + } else { + s.error = errkit.Append(s.error, err) + } - errorMessage := s.GenerateErrorMessage() + errorMessage := s.GenerateErrorMessage().Error() for _, result := range s.results { result.Error = errorMessage @@ -129,14 +133,3 @@ func (s *ScanContext) LogWarning(format string, args ...any) { } } } - -// joinErrors joins multiple errors and returns a single error string -func joinErrors(errors []error) string { - var errorMessages []string - for _, e := range errors { - if e != nil { - errorMessages = append(errorMessages, e.Error()) - } - } - return strings.Join(errorMessages, "; ") -} diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go index a4a4ac4bad..8f1af96b74 100644 --- a/pkg/templates/cluster.go +++ b/pkg/templates/cluster.go @@ -251,23 +251,25 @@ func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) { event.InternalEvent = make(map[string]interface{}) } for _, operator := range e.operators { - result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) - event.InternalEvent["template-id"] = operator.templateID - event.InternalEvent["template-path"] = operator.templatePath - event.InternalEvent["template-info"] = operator.templateInfo + clonedEvent := event.CloneShallow() + + result, matched := operator.operator.Execute(clonedEvent.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) + clonedEvent.InternalEvent["template-id"] = operator.templateID + clonedEvent.InternalEvent["template-path"] = operator.templatePath + clonedEvent.InternalEvent["template-info"] = operator.templateInfo if result == nil && !matched && e.options.Options.MatcherStatus { - if err := e.options.Output.WriteFailure(event); err != nil { + if err := e.options.Output.WriteFailure(clonedEvent); err != nil { gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) } continue } if matched && result != nil { - event.OperatorsResult = result - event.Results = e.requests.MakeResultEvent(event) + clonedEvent.OperatorsResult = result + clonedEvent.Results = e.requests.MakeResultEvent(clonedEvent) results = true - _ = writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) + _ = writer.WriteResult(clonedEvent, e.options.Output, e.options.Progress, e.options.IssuesClient) } } }) @@ -290,14 +292,16 @@ func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.R } err := e.requests.ExecuteWithResults(inputItem, dynamicValues, nil, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { - result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) + clonedEvent := event.CloneShallow() + + result, matched := operator.operator.Execute(clonedEvent.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) if matched && result != nil { - event.OperatorsResult = result - event.InternalEvent["template-id"] = operator.templateID - event.InternalEvent["template-path"] = operator.templatePath - event.InternalEvent["template-info"] = operator.templateInfo - event.Results = e.requests.MakeResultEvent(event) - scanCtx.LogEvent(event) + clonedEvent.OperatorsResult = result + clonedEvent.InternalEvent["template-id"] = operator.templateID + clonedEvent.InternalEvent["template-path"] = operator.templatePath + clonedEvent.InternalEvent["template-info"] = operator.templateInfo + clonedEvent.Results = e.requests.MakeResultEvent(clonedEvent) + scanCtx.LogEvent(clonedEvent) } } }) diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index 54708e7ea1..01af3a999b 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -36,8 +36,7 @@ var ( ) const ( - Unsigned = "unsigned" - PDVerifier = "projectdiscovery/nuclei-templates" + Unsigned = "unsigned" ) func init() { @@ -420,7 +419,7 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e break } } - + options.TemplateVerifier = template.TemplateVerifier if !(template.Verified && verifier.Identifier() == "projectdiscovery/nuclei-templates") { template.Options.RawTemplate = data } diff --git a/pkg/templates/templates_doc.go b/pkg/templates/templates_doc.go index b484dad565..827e583d3b 100644 --- a/pkg/templates/templates_doc.go +++ b/pkg/templates/templates_doc.go @@ -1603,7 +1603,12 @@ func init() { "time", "select", "files", + "waitdom", + "waitfcp", + "waitfmp", + "waitidle", "waitload", + "waitstable", "getresource", "extract", "setmethod", diff --git a/pkg/templates/workflows.go b/pkg/templates/workflows.go index bafb7aaf31..50029af18a 100644 --- a/pkg/templates/workflows.go +++ b/pkg/templates/workflows.go @@ -4,6 +4,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/keys" "github.com/projectdiscovery/nuclei/v3/pkg/model" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/utils/stats" @@ -105,7 +106,7 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr // increment signed/unsigned counters if template.Verified { if template.TemplateVerifier == "" { - SignatureStats[PDVerifier].Add(1) + SignatureStats[keys.PDVerifier].Add(1) } else { SignatureStats[template.TemplateVerifier].Add(1) } diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index 930787aab9..a1ed685c87 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -167,7 +167,7 @@ func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEven // create event event := wrappedEvent.InternalEvent - templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"])) + templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"]), types.ToString(event["template-verifier"])) var templateInfo model.Info if ti, ok := event["template-info"].(model.Info); ok { templateInfo = ti diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go index 4ca9badf70..149deaa4d5 100644 --- a/pkg/tmplexec/exec.go +++ b/pkg/tmplexec/exec.go @@ -20,7 +20,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/generic" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/multiproto" - "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr" "github.com/projectdiscovery/utils/errkit" ) @@ -207,7 +206,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { ctx.LogError(errx) if lastMatcherEvent != nil { - lastMatcherEvent.InternalEvent["error"] = tryParseCause(fmt.Errorf("%s", ctx.GenerateErrorMessage())) + lastMatcherEvent.InternalEvent["error"] = getErrorCause(ctx.GenerateErrorMessage()) writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus) } @@ -222,7 +221,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { Info: e.options.TemplateInfo, Type: e.getTemplateType(), Host: ctx.Input.MetaInput.Input, - Error: tryParseCause(fmt.Errorf("%s", ctx.GenerateErrorMessage())), + Error: getErrorCause(ctx.GenerateErrorMessage()), }, }, OperatorsResult: &operators.Result{ @@ -235,31 +234,27 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { return executed.Load() || matched.Load(), errx } -// tryParseCause tries to parse the cause of given error +// getErrorCause tries to parse the cause of given error // this is legacy support due to use of errorutil in existing libraries // but this should not be required once all libraries are updated -func tryParseCause(err error) string { - errStr := "" - errX := errkit.FromError(err) - if errX != nil { - var errCause error - - if len(errX.Errors()) > 1 { - errCause = errX.Errors()[0] - } - if errCause == nil { - errCause = errX +func getErrorCause(err error) string { + if err == nil { + return "" + } + errx := errkit.FromError(err) + var cause error + for _, e := range errx.Errors() { + if e != nil && strings.Contains(e.Error(), "context deadline exceeded") { + continue } - - msg := strings.Trim(errCause.Error(), "{} ") - parts := strings.Split(msg, ":") - errCause = errkit.New("%s", parts[len(parts)-1]) - errKind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic).String() - errStr = errCause.Error() - errStr = strings.TrimSpace(strings.Replace(errStr, "errKind="+errKind, "", -1)) + cause = e + break } - - return errStr + if cause == nil { + cause = errkit.Append(errkit.New("could not get error cause"), errx) + } + // parseScanError prettifies the error message and removes everything except the cause + return parseScanError(cause.Error()) } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. diff --git a/pkg/tmplexec/interface.go b/pkg/tmplexec/interface.go index 67f9621116..36f139cc57 100644 --- a/pkg/tmplexec/interface.go +++ b/pkg/tmplexec/interface.go @@ -1,10 +1,15 @@ package tmplexec import ( + "errors" + "regexp" + "strings" + "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/generic" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/multiproto" + "github.com/projectdiscovery/utils/errkit" ) var ( @@ -30,3 +35,37 @@ type TemplateEngine interface { // Name returns name of template engine Name() string } + +var ( + // A temporary fix to remove errKind from error message + // this is because errkit is not used everywhere yet + reNoKind = regexp.MustCompile(`([\[][^][]+[\]]|errKind=[^ ]+) `) +) + +// parseScanError parses given scan error and only returning the cause +// instead of inefficient one +func parseScanError(msg string) string { + if msg == "" { + return "" + } + if strings.HasPrefix(msg, "ReadStatusLine:") { + // last index is actual error (from rawhttp) + parts := strings.Split(msg, ":") + msg = strings.TrimSpace(parts[len(parts)-1]) + } + if strings.Contains(msg, "read ") { + // same here + parts := strings.Split(msg, ":") + msg = strings.TrimSpace(parts[len(parts)-1]) + } + e := errkit.FromError(errors.New(msg)) + for _, err := range e.Errors() { + if err != nil && strings.Contains(err.Error(), "context deadline exceeded") { + continue + } + msg = reNoKind.ReplaceAllString(err.Error(), "") + return msg + } + wrapped := errkit.Append(errkit.New("failed to get error cause"), e).Error() + return reNoKind.ReplaceAllString(wrapped, "") +} diff --git a/pkg/types/types.go b/pkg/types/types.go index cab1aacf54..9cc88f49ff 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -23,6 +23,9 @@ var ( ErrNoMoreRequests = io.EOF ) +// LoadHelperFileFunction can be used to load a helper file. +type LoadHelperFileFunction func(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) + // Options contains the configuration options for nuclei scanner. type Options struct { // Tags contains a list of tags to execute templates for. Multiple paths @@ -384,6 +387,8 @@ type Options struct { ScanID string // ScanName is the name of the scan to be uploaded ScanName string + // ScanUploadFile is the jsonl file to upload scan results to cloud + ScanUploadFile string // TeamID is the team ID to use for cloud upload TeamID string // JsConcurrency is the number of concurrent js routines to run @@ -406,6 +411,9 @@ type Options struct { HttpApiEndpoint string // ListTemplateProfiles lists all available template profiles ListTemplateProfiles bool + // LoadHelperFileFunction is a function that will be used to execute LoadHelperFile. + // If none is provided, then the default implementation will be used. + LoadHelperFileFunction LoadHelperFileFunction // timeouts contains various types of timeouts used in nuclei // these timeouts are derived from dial-timeout (-timeout) with known multipliers // This is internally managed and does not need to be set by user by explicitly setting @@ -538,10 +546,21 @@ func (options *Options) ParseHeadlessOptionalArguments() map[string]string { return optionalArguments } -// LoadHelperFile loads a helper file needed for the template +// LoadHelperFile loads a helper file needed for the template. +// +// If LoadHelperFileFunction is set, then that function will be used. +// Otherwise, the default implementation will be used, which respects the sandbox rules and only loads files from allowed directories. +func (options *Options) LoadHelperFile(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) { + if options.LoadHelperFileFunction != nil { + return options.LoadHelperFileFunction(helperFile, templatePath, catalog) + } + return options.defaultLoadHelperFile(helperFile, templatePath, catalog) +} + +// defaultLoadHelperFile loads a helper file needed for the template // this respects the sandbox rules and only loads files from // allowed directories -func (options *Options) LoadHelperFile(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) { +func (options *Options) defaultLoadHelperFile(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) { if !options.AllowLocalFileAccess { // if global file access is disabled try loading with restrictions absPath, err := options.GetValidAbsPath(helperFile, templatePath) diff --git a/pkg/utils/template_path.go b/pkg/utils/template_path.go index f6ed8ae92c..6570d90f20 100644 --- a/pkg/utils/template_path.go +++ b/pkg/utils/template_path.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v3/pkg/keys" ) const ( @@ -12,16 +13,13 @@ const ( ) // TemplatePathURL returns the Path and URL for the provided template -func TemplatePathURL(fullPath, templateId string) (string, string) { - var templateDirectory string +func TemplatePathURL(fullPath, templateId, templateVerifier string) (path string, url string) { configData := config.DefaultConfig if configData.TemplatesDirectory != "" && strings.HasPrefix(fullPath, configData.TemplatesDirectory) { - templateDirectory = configData.TemplatesDirectory - } else { - return "", "" + path = strings.TrimPrefix(strings.TrimPrefix(fullPath, configData.TemplatesDirectory), "/") } - - finalPath := strings.TrimPrefix(strings.TrimPrefix(fullPath, templateDirectory), "/") - templateURL := TemplatesRepoURL + templateId - return finalPath, templateURL + if templateVerifier == keys.PDVerifier { + url = TemplatesRepoURL + templateId + } + return }