diff --git a/.github/workflows/ci-terraform-modules.yaml b/.github/workflows/ci-terraform-modules.yaml index f283968df0..4944ec0b59 100644 --- a/.github/workflows/ci-terraform-modules.yaml +++ b/.github/workflows/ci-terraform-modules.yaml @@ -13,6 +13,7 @@ jobs: name: 'validate tf modules' runs-on: ubuntu-latest strategy: + fail-fast: false matrix: terraform_version: [ # '~1.6.0', # doesn't support mock_provider @@ -23,7 +24,12 @@ jobs: # Modules that have tests - { name: 'worklytics-connector-specs', test: true } - { name: 'gcp-host', test: true } - - { name: 'aws-proxy-lambda', test: true } + - { name: 'gcp-proxy-api', test: true } + - { name: 'gcp-webhook-collector', test: true } + - { name: 'aws', test: true } + - { name: 'aws-host', test: true } + - { name: 'aws-proxy-bulk', test: true } + - { name: 'aws-proxy-lambda', test: true } steps: - name: Check out code uses: actions/checkout@v4 diff --git a/.github/workflows/publish-release-artifacts.yaml b/.github/workflows/publish-release-artifacts.yaml index 0436aa7e06..9c65f4993a 100644 --- a/.github/workflows/publish-release-artifacts.yaml +++ b/.github/workflows/publish-release-artifacts.yaml @@ -69,23 +69,26 @@ jobs: run: | # generate-sbom.sh runs mvn clean verify, which removes shaded deployment JARs. # Rebuild distribution artifacts for GitHub release upload. + # Each build.sh run cleans the full reactor, so stash JARs before the next build. + mkdir -p release-assets ./tools/build.sh -qd aws java/ + cp java/impl/aws/target/deployment/psoxy-aws-*.jar release-assets/ ./tools/build.sh -qd gcp java/ + cp java/impl/gcp/target/deployment/psoxy-gcp-*.jar release-assets/ - name: Locate Artifacts id: locate-artifacts if: startsWith(github.ref, 'refs/tags/v') run: | - # Find the generated JARs - verifying path assumptions from build scripts - AWS_JAR=$(find java/impl/aws/target/deployment -name "psoxy-aws-*.jar" | head -n 1) - GCP_JAR=$(find java/impl/gcp/target/deployment -name "psoxy-gcp-*.jar" | head -n 1) + AWS_JAR=$(find release-assets -name "psoxy-aws-*.jar" | head -n 1) + GCP_JAR=$(find release-assets -name "psoxy-gcp-*.jar" | head -n 1) if [ -z "$AWS_JAR" ] || [ ! -f "$AWS_JAR" ]; then - echo "::error::AWS deployment JAR not found under java/impl/aws/target/deployment" + echo "::error::AWS deployment JAR not found under release-assets" exit 1 fi if [ -z "$GCP_JAR" ] || [ ! -f "$GCP_JAR" ]; then - echo "::error::GCP deployment JAR not found under java/impl/gcp/target/deployment" + echo "::error::GCP deployment JAR not found under release-assets" exit 1 fi diff --git a/docs/aws/sbom.json b/docs/aws/sbom.json index 34ea018f7f..e02bbf5f5e 100644 --- a/docs/aws/sbom.json +++ b/docs/aws/sbom.json @@ -1,10 +1,10 @@ { "bomFormat" : "CycloneDX", "specVersion" : "1.5", - "serialNumber" : "urn:uuid:e20ccefe-3322-3d78-b902-c5eccda95d89", + "serialNumber" : "urn:uuid:04d17ffb-7c8c-39c0-ac9f-110da0a6fbde", "version" : 1, "metadata" : { - "timestamp" : "2026-06-13T13:48:56Z", + "timestamp" : "2026-06-18T18:28:17Z", "lifecycles" : [ { "phase" : "build" @@ -59,9 +59,9 @@ "component" : { "group" : "co.worklytics.psoxy", "name" : "psoxy-aws", - "version" : "0.6.5", + "version" : "0.6.6", "licenses" : [ ], - "purl" : "pkg:maven/co.worklytics.psoxy/psoxy-aws@0.6.5?type=jar", + "purl" : "pkg:maven/co.worklytics.psoxy/psoxy-aws@0.6.6?type=jar", "externalReferences" : [ { "type" : "distribution-intake", @@ -69,7 +69,7 @@ } ], "type" : "library", - "bom-ref" : "pkg:maven/co.worklytics.psoxy/psoxy-aws@0.6.5?type=jar" + "bom-ref" : "pkg:maven/co.worklytics.psoxy/psoxy-aws@0.6.6?type=jar" }, "properties" : [ { @@ -86,40 +86,40 @@ { "group" : "co.worklytics.psoxy", "name" : "psoxy-core", - "version" : "0.6.5", + "version" : "0.6.6", "scope" : "required", "hashes" : [ { "alg" : "MD5", - "content" : "ae890af4261157e000763ab1566b60da" + "content" : "6354851ecef5541ffee800a87e945875" }, { "alg" : "SHA-1", - "content" : "5684ec244a1fbd5c26b5e2eb072d38448b25f51c" + "content" : "bcf790c49ed33bdb1a36d4cfef49b13c61dce7e4" }, { "alg" : "SHA-256", - "content" : "7b6d46566c7d229c0dc990445e218afa8bedcd2ff9ff617b38a737d8d3492aef" + "content" : "b5cf492919eaa982d95c6e4848a3e1e7649f28f2c45869f98ac0acc93fe3814b" }, { "alg" : "SHA-512", - "content" : "2de8e2b501e8b3c74fb298860e980fb846681eef7ef8e280043ef20c7a8ad84cff8c6d8a7440373412f7097e0c57f377c2708797bd6c891b7ea317ed948b75cf" + "content" : "7d16243f1f920475c1d75b400002d868fddfbdebf9b080ea8e3f6c3c36253b279a8bea7f3ffefb188222ed626d34cbf99575970fe98ec293c720c96a4e924d89" }, { "alg" : "SHA-384", - "content" : "602879866244a8605aae744bcdebc2631f26a4f61a187637e65fa19d763569d04301c76ae790668e3f1055cee55b28de" + "content" : "78226094335202889ef2faf96c9444bb7d2c0ce417eb2500d2c730548bfcfa3a6099ca0bb9fd23f6b645a08905e8ee43" }, { "alg" : "SHA3-384", - "content" : "46d4177c423aeed1528ec29fff55c77a4c73cd30ea5bacc8b5096761f0e1eafd727963385d9576c1cb2915a566ff8632" + "content" : "480cc356ba982e35240c2c0209049fe5d3bf9e82982b7663412ef454ab1063265db8a875cfee6832b75c683ec644af00" }, { "alg" : "SHA3-256", - "content" : "8aae044c55fea3d7454de7b6059204ec157438850aa1197e71ceaa818f16c728" + "content" : "b0f31e47b7b6400e00f9fd4b827ec48ff9d26e48479a21ba90f91cd341a46996" }, { "alg" : "SHA3-512", - "content" : "9def3b3fb3f96ccf9764201f93f64afe572d1dad81d62d2580863a62404f2fd64d7ad97df207bde0434ee9c3b60661ff42cea63235e97e735a1a54faebe4ab75" + "content" : "21e9b664483a864d0015e0b09035686c003534a168320b1b0eceff20c0c77dfd0ac1479707fbe960bad8ef71d0bea2695ff5e76a4546660c4ce39328c9ba6875" } ], "licenses" : [ @@ -130,47 +130,47 @@ } } ], - "purl" : "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.5?type=jar", + "purl" : "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.6?type=jar", "type" : "library", - "bom-ref" : "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.5?type=jar" + "bom-ref" : "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.6?type=jar" }, { "group" : "com.avaulta.gateway", "name" : "gateway-core", - "version" : "0.6.5", + "version" : "0.6.6", "scope" : "required", "hashes" : [ { "alg" : "MD5", - "content" : "3aa908fe40b90c78ad561fb47e90c079" + "content" : "5761afbecd3f4a0347e52c005c2a1c53" }, { "alg" : "SHA-1", - "content" : "39f2c5da6811633d6ee7ab25d9142cc1fb2135ab" + "content" : "1430f70ebcdb96933a1b9e3d024275b2a5da2be5" }, { "alg" : "SHA-256", - "content" : "c516eac5f2e2e007d1ddd391e5c8f501c6eb0aca74e0ad793be22a2c2bba340e" + "content" : "bb1802066c21112d66b9e8339588da97aac032a1e4a9ef69303e8d08499f94f7" }, { "alg" : "SHA-512", - "content" : "93a7d17fee4bc3bb7f9c06ae0b29a1c3ba717b3ee0086064de098260e2a27460ebc83607dfc8da927a26a5ec1c406983184c4f49845e58baae9c159c67c998e3" + "content" : "3b977bbdfe053c841feb1e4170185bbd43618fdbe84ebe88bd8415e0b017f4dd0c320540eb62cc00e6b2a28788d38f5935c74e2e01663ce8c2a0ed12fcac66c6" }, { "alg" : "SHA-384", - "content" : "a5307e4e1cb2a1e6b68d39912a90602fb33ec8e5c9ad16e399a367b9624d1e2be8d353c12f4bf438be97ab2a38f50c98" + "content" : "6f8e74717126d333c18d38d698aa3f7043be87fa539428abeedc90851782848a65294b05f883111a591b855022465e3b" }, { "alg" : "SHA3-384", - "content" : "a7b36777ff6cc7488079de8e02110b3df2fd1c5e212c396c4aa9615ea6a76c65e880df1fcd64f51ab2f4f34d7fa36103" + "content" : "2ea2a14d387b89a6bd6bb1e59a2943695562ae326137a0fd9642299c0f008dc3a31ad46954ef84ba07a69afb656875a6" }, { "alg" : "SHA3-256", - "content" : "0ae3de4c87d49ef1742580b250a2983565d5c37eee5eb0eddcc2b1be503957bb" + "content" : "0ed7c4b5cb3adffc86c07b4b6c565b025e5c0d87e2f36b4a957a2f4a02a8ba26" }, { "alg" : "SHA3-512", - "content" : "84050579bff5107f70c0f29c6f2c038ca985b971936f25d8f7098ff52b4fd65ba39af7d93eed7f238c1983a2df098b4da8de1e9005d02b93469ba0de4e6f2b19" + "content" : "a3ec68bdd1937d69281f6bdb382dd46371cc49ddcc24ef900cdd3a556eb49e4a9a5c696830708b4be3b75cc69bc0d29ec5f1650a2e5985e4db14fd6dc8fd1f60" } ], "licenses" : [ @@ -181,9 +181,9 @@ } } ], - "purl" : "pkg:maven/com.avaulta.gateway/gateway-core@0.6.5?type=jar", + "purl" : "pkg:maven/com.avaulta.gateway/gateway-core@0.6.6?type=jar", "type" : "library", - "bom-ref" : "pkg:maven/com.avaulta.gateway/gateway-core@0.6.5?type=jar" + "bom-ref" : "pkg:maven/com.avaulta.gateway/gateway-core@0.6.6?type=jar" }, { "publisher" : "The Apache Software Foundation", @@ -10157,9 +10157,9 @@ ], "dependencies" : [ { - "ref" : "pkg:maven/co.worklytics.psoxy/psoxy-aws@0.6.5?type=jar", + "ref" : "pkg:maven/co.worklytics.psoxy/psoxy-aws@0.6.6?type=jar", "dependsOn" : [ - "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.5?type=jar", + "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.6?type=jar", "pkg:maven/org.projectlombok/lombok@1.18.42?type=jar", "pkg:maven/com.google.dagger/dagger-compiler@2.40.5?type=jar", "pkg:maven/com.google.dagger/dagger@2.40.5?type=jar", @@ -10179,9 +10179,9 @@ ] }, { - "ref" : "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.5?type=jar", + "ref" : "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.6?type=jar", "dependsOn" : [ - "pkg:maven/com.avaulta.gateway/gateway-core@0.6.5?type=jar", + "pkg:maven/com.avaulta.gateway/gateway-core@0.6.6?type=jar", "pkg:maven/org.apache.commons/commons-lang3@3.19.0?type=jar", "pkg:maven/org.apache.commons/commons-csv@1.14.1?type=jar", "pkg:maven/commons-io/commons-io@2.18.0?type=jar", @@ -10206,7 +10206,7 @@ ] }, { - "ref" : "pkg:maven/com.avaulta.gateway/gateway-core@0.6.5?type=jar", + "ref" : "pkg:maven/com.avaulta.gateway/gateway-core@0.6.6?type=jar", "dependsOn" : [ "pkg:maven/org.apache.commons/commons-lang3@3.19.0?type=jar", "pkg:maven/org.apache.commons/commons-csv@1.14.1?type=jar", diff --git a/docs/gcp/sbom.json b/docs/gcp/sbom.json index 93b7a2fb27..51e4112cfe 100644 --- a/docs/gcp/sbom.json +++ b/docs/gcp/sbom.json @@ -1,10 +1,10 @@ { "bomFormat" : "CycloneDX", "specVersion" : "1.5", - "serialNumber" : "urn:uuid:acdeac5e-a037-33cd-988c-fd36dcb732b5", + "serialNumber" : "urn:uuid:c05f478f-e25e-3ca5-b754-23273b8f4267", "version" : 1, "metadata" : { - "timestamp" : "2026-06-13T13:49:32Z", + "timestamp" : "2026-06-18T18:28:44Z", "lifecycles" : [ { "phase" : "build" @@ -59,9 +59,9 @@ "component" : { "group" : "co.worklytics.psoxy", "name" : "psoxy-gcp", - "version" : "0.6.5", + "version" : "0.6.6", "licenses" : [ ], - "purl" : "pkg:maven/co.worklytics.psoxy/psoxy-gcp@0.6.5?type=jar", + "purl" : "pkg:maven/co.worklytics.psoxy/psoxy-gcp@0.6.6?type=jar", "externalReferences" : [ { "type" : "distribution-intake", @@ -69,7 +69,7 @@ } ], "type" : "library", - "bom-ref" : "pkg:maven/co.worklytics.psoxy/psoxy-gcp@0.6.5?type=jar" + "bom-ref" : "pkg:maven/co.worklytics.psoxy/psoxy-gcp@0.6.6?type=jar" }, "properties" : [ { @@ -86,40 +86,40 @@ { "group" : "co.worklytics.psoxy", "name" : "psoxy-core", - "version" : "0.6.5", + "version" : "0.6.6", "scope" : "required", "hashes" : [ { "alg" : "MD5", - "content" : "2c4f5792b8cee67292b5ff3d9d68a771" + "content" : "87eddc8e5f702c1d998d1acdde344e74" }, { "alg" : "SHA-1", - "content" : "a73175cdc95cd3f6ed6e94d6a47b2dba920fb897" + "content" : "3f2032603ef48fb9120d307918bd46620530ecc2" }, { "alg" : "SHA-256", - "content" : "fbf50f5c0120bf3b156513c83eb9a09f21fd7a5a6c41541a27757a8cdaa6a651" + "content" : "c7be10b9ead4a542f4c2c326d74efb413f941c527511864ea41e868b1a9c5880" }, { "alg" : "SHA-512", - "content" : "da114b91b4004300abbe140d7bb655759c04c592ff4b1a0e2869797316910404dcf671a6a2cd68ee53d57520f9cafcc93e07bc7e768ba8bbce3e3d14a49c1db0" + "content" : "43fcdc7676cc91ae1ed38b4b88b4d82612bfa60cf1583bdbb82a1a1638fb1354a78c131386a8f240a022f8a7a9f14e34b686bea7e1462a6329c8dce741992815" }, { "alg" : "SHA-384", - "content" : "9e928d55785be0983f6d9f445aa5427e9a04ff1bcc1cce2fe78c18d424ecea4f792d364656d63c72b5db0a613cf453c2" + "content" : "4fe58a6da8ba4c00d4dd08ef302e890cedd03975a7c431175968c6812cc6e84feb32c1f88e6dc546af10a1c70213a9e6" }, { "alg" : "SHA3-384", - "content" : "bbed4b484c38cf71ac82eb9fc6bca5c6e473fd5df0f465c376ee389ea0d802334a4a19948c3154df24f52bd78f1ca25b" + "content" : "50a2178426c532494f967aba1441d7412a6e0faa5fe539fd5c577ffc114ff589373855b4db3d69f28d18a704241d8eca" }, { "alg" : "SHA3-256", - "content" : "a0dfdac6ced4c2d2ee3713abcaa627a28d29b2fa65e617accef5309517822d78" + "content" : "5a6ef60a278035c6f92212058fe8b9ee6c87b4adc27ef9739e3b5b35c399150e" }, { "alg" : "SHA3-512", - "content" : "3921a66450a104ff3ad7ade2498908caec7df30ce6b0fa08d876f1d651aae75f7f73df55f8c64697d7db8ca7ddfac9833c191d9525b0fee558165dff4a31e821" + "content" : "a9b6b25f67523199f6f0bed8931ed3b3c6ee461762d5a371ef8684ac6798a19b3877d6fe3537636cac80dfdfe0b1500fc74504d1f882685fabccb6252c197128" } ], "licenses" : [ @@ -130,47 +130,47 @@ } } ], - "purl" : "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.5?type=jar", + "purl" : "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.6?type=jar", "type" : "library", - "bom-ref" : "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.5?type=jar" + "bom-ref" : "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.6?type=jar" }, { "group" : "com.avaulta.gateway", "name" : "gateway-core", - "version" : "0.6.5", + "version" : "0.6.6", "scope" : "required", "hashes" : [ { "alg" : "MD5", - "content" : "69ba90853502e7b8794879a534403dd5" + "content" : "3695b96697effac61ec4284edd27087e" }, { "alg" : "SHA-1", - "content" : "b38e3a04ed42afbc663bc87a3925a9649bb8e92a" + "content" : "868705aed1f6306c1cb47a5845f0f744feaf053d" }, { "alg" : "SHA-256", - "content" : "15486cf9dddcba9271de2833f0f7a4c5eee136959b35906a24b9aad0a47c56d8" + "content" : "c3ca35438db7144a36ead65d6c05abf45a611bdc7a39fedc6bed4d904a3c8eba" }, { "alg" : "SHA-512", - "content" : "48178d22f53f789cefd694b9633d63589bdb24b346ae60934e3812655452cfa34cb13be1d840223f5085701ca001e7ceddc9908b478b57f62929b916e5c4067a" + "content" : "d0fc39ac5c829b9850f341f762061e56e351be3336e543542b4f00cdb2785302a37abd3d80c289d33f0c8f8ba94ea4e9d703b1d1401f11302fe95f2a72e1b827" }, { "alg" : "SHA-384", - "content" : "bfeecff46e37f3d43ca2cb5a029beb93042387d80340d165fd917a955377f3c413720d901e68636edf14ac2205d04952" + "content" : "2d5f32b9f2c8327ece89422d831d2a61adeeba825b98d8c5822c5585fcf61bd0df041f0e8298920ae50382f17a159d6f" }, { "alg" : "SHA3-384", - "content" : "0c2e004d6d4267a750db44489fd140c22f5b5d70526459bf448ddac0382fd7a1768fce1295602414fc2e589028ad6ce5" + "content" : "e32df957913151cc9afe2f9c58f0fdd16150a78f9475060bf1bb7ee2008cabb59f41069d727c5fca24b0071d524b1498" }, { "alg" : "SHA3-256", - "content" : "856978c0e2f6b0528b461c2d50aa4afb3ac7c1b7d29ba099c69d9346f78a3c3b" + "content" : "537955bb8865270fc345fe5f9e9c90c440fa04c5f9934c88cc907b8eec410312" }, { "alg" : "SHA3-512", - "content" : "4c14670d27dac92a5f207ff43497108cdb8cdf6850ccbf70f0b5fcae7c3841b71c456e09448b5a8c3312e34c7e0e1a233b66f9b0c30d75859b6fb86eab2289da" + "content" : "2a0a3b1d740ef1d0c381ce58e0b0325f7bdc0d08250075cc6a6a833389c84b5333d97af249cd763f9567c49e22b34bb9dd36a4b2b79060e7b9df0f35f7524bed" } ], "licenses" : [ @@ -181,9 +181,9 @@ } } ], - "purl" : "pkg:maven/com.avaulta.gateway/gateway-core@0.6.5?type=jar", + "purl" : "pkg:maven/com.avaulta.gateway/gateway-core@0.6.6?type=jar", "type" : "library", - "bom-ref" : "pkg:maven/com.avaulta.gateway/gateway-core@0.6.5?type=jar" + "bom-ref" : "pkg:maven/com.avaulta.gateway/gateway-core@0.6.6?type=jar" }, { "publisher" : "FasterXML", @@ -10589,9 +10589,9 @@ ], "dependencies" : [ { - "ref" : "pkg:maven/co.worklytics.psoxy/psoxy-gcp@0.6.5?type=jar", + "ref" : "pkg:maven/co.worklytics.psoxy/psoxy-gcp@0.6.6?type=jar", "dependsOn" : [ - "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.5?type=jar", + "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.6?type=jar", "pkg:maven/org.projectlombok/lombok@1.18.42?type=jar", "pkg:maven/com.google.dagger/dagger@2.40.5?type=jar", "pkg:maven/com.google.dagger/dagger-compiler@2.40.5?type=jar", @@ -10604,9 +10604,9 @@ ] }, { - "ref" : "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.5?type=jar", + "ref" : "pkg:maven/co.worklytics.psoxy/psoxy-core@0.6.6?type=jar", "dependsOn" : [ - "pkg:maven/com.avaulta.gateway/gateway-core@0.6.5?type=jar", + "pkg:maven/com.avaulta.gateway/gateway-core@0.6.6?type=jar", "pkg:maven/org.apache.commons/commons-lang3@3.19.0?type=jar", "pkg:maven/org.apache.commons/commons-csv@1.14.1?type=jar", "pkg:maven/commons-io/commons-io@2.18.0?type=jar", @@ -10631,7 +10631,7 @@ ] }, { - "ref" : "pkg:maven/com.avaulta.gateway/gateway-core@0.6.5?type=jar", + "ref" : "pkg:maven/com.avaulta.gateway/gateway-core@0.6.6?type=jar", "dependsOn" : [ "pkg:maven/org.apache.commons/commons-lang3@3.19.0?type=jar", "pkg:maven/org.apache.commons/commons-csv@1.14.1?type=jar", diff --git a/docs/gcp/troubleshooting.md b/docs/gcp/troubleshooting.md index 2759ffa81d..3b4ab01f2b 100644 --- a/docs/gcp/troubleshooting.md +++ b/docs/gcp/troubleshooting.md @@ -88,6 +88,49 @@ terraform { You will likely see MANY changes. These are caused by the provider version difference and should be benign. The vast majority are label changes; we utilize the `default_labels` functionality in google provider `5.x` to label all the infra created by this configuration; +## Error 400: `vpc-access-egress` annotation cannot be set without connector or network-interfaces + +If you remove Direct VPC egress or a Serverless VPC Access connector from your configuration (for example, by commenting out `vpc_config` in `terraform.tfvars`), `terraform apply` may fail with: + +``` +Error 400: Could not update Cloud Run service ... spec.template.metadata.annotations: The run.googleapis.com/vpc-access-egress annotation cannot be set without also setting the run.googleapis.com/vpc-access-connector annotation or the run.googleapis.com/network-interfaces annotation. +``` + +This is a GCP limitation: Cloud Functions gen2 cannot drop VPC egress settings through an in-place service update. Terraform attempts to remove `direct_vpc_network_interface` from the function, but the underlying Cloud Run service still has a stale egress annotation until the function is recreated. + +**Fix:** destroy the affected Cloud Function resources in Terraform, then apply to recreate them without VPC networking. + +1. Ensure `vpc_config` is removed or set to `null` in your configuration. +2. Find the resource addresses: + +```bash +terraform state list | grep google_cloudfunctions2_function +``` + +3. Destroy each affected function. Examples using the standard `module "psoxy"` layout from our GCP examples: + +```bash +terraform destroy -target='module.psoxy.module.api_connector["gcal"].google_cloudfunctions2_function.function' +terraform destroy -target='module.psoxy.module.api_connector["msft-teams"].google_cloudfunctions2_function.function' +terraform destroy -target='module.psoxy.module.webhook_collector["llm-portal"].google_cloudfunctions2_function.function' +``` + +Repeat for every connector that had VPC egress enabled. Adjust the module path if your root module is not named `psoxy`, and use the connector map keys from your `terraform.tfvars`. + +4. Recreate: + +```bash +terraform apply +``` + +Alternatively, a single apply can replace a function without a separate destroy step: + +```bash +terraform apply -replace='module.psoxy.module.api_connector["gcal"].google_cloudfunctions2_function.function' +``` + +See [VPC configuration](./vpc.md#removing-vpc-egress) for background on Direct VPC egress and removal. + ## Bulk processing failures If you need to re-trigger bulk processing of objects that have already been written to GCS (e.g., for webhook collectors), you can use the `replay-gcs-writes.sh` script. diff --git a/docs/gcp/vpc.md b/docs/gcp/vpc.md index b4d5ff043f..9ab894bc0c 100644 --- a/docs/gcp/vpc.md +++ b/docs/gcp/vpc.md @@ -140,3 +140,43 @@ resource "google_compute_router_nat" "nat" { This allows data sources to restrict access by IP. The NAT and router above may present scalability limits at high volume; size them for your workload. Your VPC must provide connectivity to all data sources you connect to. Google Workspace sources generally work with Private Google Access on the subnet; other SaaS APIs require the NAT path above to reach the public internet with your fixed IP. + +## Removing VPC egress + +GCP Cloud Functions (gen2) cannot clear Direct VPC egress or Serverless VPC Access connector settings via an in-place Terraform update. If you remove or comment out `vpc_config` and run `terraform apply`, you may see: + +``` +Error 400: Could not update Cloud Run service ... spec.template.metadata.annotations: The run.googleapis.com/vpc-access-egress annotation cannot be set without also setting the run.googleapis.com/vpc-access-connector annotation or the run.googleapis.com/network-interfaces annotation. +``` + +Workaround: destroy each affected Cloud Function in Terraform, then apply to recreate it without VPC networking. + +1. Remove `vpc_config` from your `terraform.tfvars` (or set it to `null`). +2. List function resources in state: + +```bash +terraform state list | grep google_cloudfunctions2_function +``` + +3. Destroy each function that previously had VPC egress (API connectors, webhook collectors, and bulk connectors if applicable). The resource address depends on your root module; with the standard `gcp-host` example it looks like: + +```bash +# API connector (replace CONNECTOR_ID with the map key from your config, e.g. gcal, msft-teams) +terraform destroy -target='module.psoxy.module.api_connector["CONNECTOR_ID"].google_cloudfunctions2_function.function' + +# Webhook collector +terraform destroy -target='module.psoxy.module.webhook_collector["WEBHOOK_ID"].google_cloudfunctions2_function.function' + +# Bulk connector (only if it used a serverless connector) +terraform destroy -target='module.psoxy.module.bulk_connector["BULK_ID"].google_cloudfunctions2_function.function' +``` + +4. Recreate the functions: + +```bash +terraform apply +``` + +Each destroyed function is recreated on apply. Expect brief downtime per connector while the function is destroyed and redeployed. IAM bindings and secrets are unchanged; only the Cloud Function resource is replaced. + +See also [GCP troubleshooting](./troubleshooting.md#error-400-vpc-access-egress-annotation-cannot-be-set-without-connector-or-network-interfaces) for the same error with additional context. diff --git a/docs/sources/chatgpt-enterprise/chatgpt-enterprise.yaml b/docs/sources/chatgpt-enterprise/chatgpt-enterprise.yaml index 6da1ae814f..e954e84122 100644 --- a/docs/sources/chatgpt-enterprise/chatgpt-enterprise.yaml +++ b/docs/sources/chatgpt-enterprise/chatgpt-enterprise.yaml @@ -283,8 +283,8 @@ endpoints: - ! jsonPaths: - "$.conversation.title" - - "$[?(@.actor.type =~ /account_user/i)].message.content.value" - - "$[?(@.actor.type =~ /account_user/i)].message.annotations[*].quote_text" + - "$[?(@.actor.type =~ /account_user/i || @.message.author.type == 'user' || @.message.author.role == 'user')].message.content.value" + - "$[?(@.actor.type =~ /account_user/i || @.message.author.type == 'user' || @.message.author.role == 'user')].message.annotations[*].quote_text" - "$.message.content.quote" - "$.event_details.prompt_text" keywords: @@ -412,6 +412,9 @@ endpoints: - "craft" - "design" - "linkedin" + - ! + jsonPaths: + - "$[?(@.message.author.type != 'user' && @.message.author.role != 'user')].message.content.value" responseSchema: type: "object" properties: @@ -531,6 +534,8 @@ definitions: properties: client_type: type: "string" + role: + type: "string" model: type: "string" skills_used: diff --git a/infra/examples-dev/aws/google-workspace.tf b/infra/examples-dev/aws/google-workspace.tf index 7fba7e0e70..dd1b8ac794 100644 --- a/infra/examples-dev/aws/google-workspace.tf +++ b/infra/examples-dev/aws/google-workspace.tf @@ -8,7 +8,7 @@ provider "google" { module "worklytics_connectors_google_workspace" { source = "../../modules/worklytics-connectors-google-workspace" - # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-google-workspace?ref=v0.6.5" + # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-google-workspace?ref=v0.6.6" google_workspace_connector_settings = var.google_workspace_connector_settings diff --git a/infra/examples-dev/aws/main.tf b/infra/examples-dev/aws/main.tf index a2b61be31b..9534f5b9ab 100644 --- a/infra/examples-dev/aws/main.tf +++ b/infra/examples-dev/aws/main.tf @@ -21,7 +21,7 @@ terraform { # general cases module "worklytics_connectors" { source = "../../modules/worklytics-connectors" - # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors?ref=v0.6.5" + # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors?ref=v0.6.6" enabled_connectors = var.enabled_connectors connector_settings = var.connector_settings @@ -121,7 +121,7 @@ locals { module "psoxy" { source = "../../modules/aws-host" - # source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-host?ref=v0.6.5" + # source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-host?ref=v0.6.6" environment_name = var.environment_name aws_account_id = var.aws_account_id @@ -200,7 +200,7 @@ module "connection_in_worklytics" { for_each = local.all_instances source = "../../modules/worklytics-proxy-connection-aws" - # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-proxy-connection-aws?ref=v0.6.5" + # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-proxy-connection-aws?ref=v0.6.6" proxy_instance_id = each.key worklytics_host = var.worklytics_host diff --git a/infra/examples-dev/aws/msft-365.tf b/infra/examples-dev/aws/msft-365.tf index 8bee6317bb..6250de2c14 100644 --- a/infra/examples-dev/aws/msft-365.tf +++ b/infra/examples-dev/aws/msft-365.tf @@ -2,7 +2,7 @@ module "worklytics_connectors_msft_365" { source = "../../modules/worklytics-connectors-msft-365" - # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-msft-365?ref=v0.6.5" + # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-msft-365?ref=v0.6.6" msft_365_connector_settings = var.msft_365_connector_settings @@ -52,7 +52,7 @@ module "cognito_identity_pool" { count = local.msft_365_enabled ? 1 : 0 # only provision identity pool if MSFT-365 connectors are enabled source = "../../modules/aws-cognito-pool" - # source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-pool?ref=v0.6.5" + # source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-pool?ref=v0.6.6" developer_provider_name = local.developer_provider_name name = "${local.env_qualifier}-azure-ad-federation" @@ -75,7 +75,7 @@ module "cognito_identity" { count = local.msft_365_enabled ? 1 : 0 # only provision identity pool if MSFT-365 connectors are enabled source = "../../modules/aws-cognito-identity-cli" - # source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-identity-cli?ref=v0.6.5" + # source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-identity-cli?ref=v0.6.6" aws_region = data.aws_region.current.region @@ -113,7 +113,7 @@ module "msft_connection_auth_federation" { for_each = local.provision_entraid_apps ? local.enabled_to_entraid_object : local.shared_to_entraid_object source = "../../modules/azuread-federated-credentials" - # source = "git::https://github.com/worklytics/psoxy//infra/modules/azuread-federated-credentials?ref=v0.6.5" + # source = "git::https://github.com/worklytics/psoxy//infra/modules/azuread-federated-credentials?ref=v0.6.6" application_id = each.value.connector_id display_name = "${local.env_qualifier}AccessFromAWS" diff --git a/infra/examples-dev/gcp/google-workspace.tf b/infra/examples-dev/gcp/google-workspace.tf index 7fba7e0e70..dd1b8ac794 100644 --- a/infra/examples-dev/gcp/google-workspace.tf +++ b/infra/examples-dev/gcp/google-workspace.tf @@ -8,7 +8,7 @@ provider "google" { module "worklytics_connectors_google_workspace" { source = "../../modules/worklytics-connectors-google-workspace" - # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-google-workspace?ref=v0.6.5" + # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-google-workspace?ref=v0.6.6" google_workspace_connector_settings = var.google_workspace_connector_settings diff --git a/infra/examples-dev/gcp/main.tf b/infra/examples-dev/gcp/main.tf index e264a1e3da..7015635a67 100644 --- a/infra/examples-dev/gcp/main.tf +++ b/infra/examples-dev/gcp/main.tf @@ -30,7 +30,7 @@ locals { # call this 'generic_source_connectors'? module "worklytics_connectors" { source = "../../modules/worklytics-connectors" - # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors?ref=v0.6.5" + # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors?ref=v0.6.6" base_dir = var.psoxy_base_dir enabled_connectors = var.enabled_connectors @@ -104,7 +104,7 @@ locals { module "psoxy" { source = "../../modules/gcp-host" - # source = "git::https://github.com/worklytics/psoxy//infra/modules/gcp-host?ref=v0.6.5" + # source = "git::https://github.com/worklytics/psoxy//infra/modules/gcp-host?ref=v0.6.6" gcp_project_id = var.gcp_project_id environment_name = var.environment_name @@ -168,7 +168,7 @@ module "connection_in_worklytics" { for_each = local.all_instances source = "../../modules/worklytics-proxy-connection-generic" - # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-proxy-connection-generic?ref=v0.6.5" + # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-proxy-connection-generic?ref=v0.6.6" host_platform_id = local.host_platform_id proxy_instance_id = each.key diff --git a/infra/examples-dev/gcp/msft-365.tf b/infra/examples-dev/gcp/msft-365.tf index 4b2131f7db..305e21915d 100644 --- a/infra/examples-dev/gcp/msft-365.tf +++ b/infra/examples-dev/gcp/msft-365.tf @@ -2,7 +2,7 @@ module "worklytics_connectors_msft_365" { source = "../../modules/worklytics-connectors-msft-365" - # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-msft-365?ref=v0.6.5" + # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-msft-365?ref=v0.6.6" msft_365_connector_settings = var.msft_365_connector_settings @@ -37,7 +37,7 @@ module "msft-connection-auth-federation" { for_each = module.worklytics_connectors_msft_365.enabled_api_connectors source = "../../modules/azuread-federated-credentials" - # source = "git::https://github.com/worklytics/psoxy//infra/modules/azuread-federated-credentials?ref=v0.6.5" + # source = "git::https://github.com/worklytics/psoxy//infra/modules/azuread-federated-credentials?ref=v0.6.6" application_id = each.value.connector.id display_name = "GcpFederation" diff --git a/infra/modules/aws-host/output.tf b/infra/modules/aws-host/output.tf index b475e9e6a3..a8f4c91ff2 100644 --- a/infra/modules/aws-host/output.tf +++ b/infra/modules/aws-host/output.tf @@ -47,6 +47,7 @@ output "lookup_output_buckets" { output "pseudonym_salt" { description = "Value used to salt pseudonyms (SHA-256) hashes. If migrate to new deployment, you should copy this value." value = module.psoxy.pseudonym_salt + sensitive = true } output "api_gateway_v2" { diff --git a/infra/modules/aws-host/provision_testing_infra.tftest.hcl b/infra/modules/aws-host/provision_testing_infra.tftest.hcl new file mode 100644 index 0000000000..3ecdb43e27 --- /dev/null +++ b/infra/modules/aws-host/provision_testing_infra.tftest.hcl @@ -0,0 +1,68 @@ +# aws-host should plan cleanly with and without provision_testing_infra. + +variables { + aws_account_id = "123456789012" + environment_name = "test" + psoxy_base_dir = "../../../" + deployment_bundle = "../aws-proxy-lambda/tests/deployment.zip" + deployment_bundle_hash = "dummy-hash-for-test" + provision_testing_infra = false + install_test_tool = false + + api_connectors = {} + bulk_connectors = { + "test-hris" = { + source_kind = "hris" + } + } + webhook_collectors = {} +} + +mock_provider "aws" { + mock_data "aws_region" { + defaults = { + name = "us-east-1" + } + } + + mock_data "aws_caller_identity" { + defaults = { + account_id = "123456789012" + arn = "arn:aws:iam::123456789012:user/terraform" + user_id = "AIDATEST" + } + } + + mock_data "aws_iam_session_context" { + defaults = { + issuer_arn = "arn:aws:iam::123456789012:user/terraform" + } + } +} + +run "plan_without_provision_testing_infra" { + command = plan + + assert { + error_message = "bulk connector should be provisioned through aws-proxy-bulk" + condition = length(module.bulk_connector) == 1 + } +} + +run "plan_with_provision_testing_infra" { + command = plan + + variables { + provision_testing_infra = true + } + + assert { + error_message = "bulk connector should be provisioned when testing infra is enabled" + condition = length(module.bulk_connector) == 1 + } + + assert { + error_message = "global secrets should be provisioned in SSM when testing infra is enabled" + condition = length(module.global_secrets_ssm[0].secret_arns) > 0 + } +} diff --git a/infra/modules/aws-proxy-bulk/main.tf b/infra/modules/aws-proxy-bulk/main.tf index 4d522e4458..e00193b9a5 100644 --- a/infra/modules/aws-proxy-bulk/main.tf +++ b/infra/modules/aws-proxy-bulk/main.tf @@ -337,13 +337,21 @@ locals { example_files_csv = join(",", [for f in local.all_example_files : "${var.psoxy_base_dir}${f}"]) - aws_principal_arn_when_testing = coalesce(var.aws_principal_arn_when_testing, var.aws_role_to_assume_when_testing) - aws_write_role_to_assume_when_testing = coalesce(var.aws_write_role_to_assume_when_testing, var.aws_upload_role_to_assume_when_testing) + aws_principal_arn_when_testing = ( + var.aws_principal_arn_when_testing != null ? var.aws_principal_arn_when_testing : + var.aws_role_to_assume_when_testing != null ? var.aws_role_to_assume_when_testing : + "" + ) + aws_write_role_to_assume_when_testing = ( + var.aws_write_role_to_assume_when_testing != null ? var.aws_write_role_to_assume_when_testing : + var.aws_upload_role_to_assume_when_testing != null ? var.aws_upload_role_to_assume_when_testing : + "" + ) cli_file_upload_role_args = compact([ - local.aws_write_role_to_assume_when_testing != null ? "--write-role-to-assume ${local.aws_write_role_to_assume_when_testing}" : null, + local.aws_write_role_to_assume_when_testing != "" ? "--write-role-to-assume ${local.aws_write_role_to_assume_when_testing}" : null, # test tool assumes roles via STS; user principals use default credentials instead - local.aws_principal_arn_when_testing != null && can(regex(":role/", local.aws_principal_arn_when_testing)) ? "-r ${local.aws_principal_arn_when_testing}" : null, + local.aws_principal_arn_when_testing != "" && can(regex(":role/", local.aws_principal_arn_when_testing)) ? "-r ${local.aws_principal_arn_when_testing}" : null, ]) cli_file_upload_role_args_todo = join("\n", [for arg in local.cli_file_upload_role_args : " ${arg} \\"]) cli_file_upload_role_args_shell = join("\n", [for arg in local.cli_file_upload_role_args : " ${arg} \\"]) diff --git a/infra/modules/aws-proxy-bulk/provision_testing_infra.tftest.hcl b/infra/modules/aws-proxy-bulk/provision_testing_infra.tftest.hcl new file mode 100644 index 0000000000..3d44417c80 --- /dev/null +++ b/infra/modules/aws-proxy-bulk/provision_testing_infra.tftest.hcl @@ -0,0 +1,56 @@ +# Plan succeeds with testing IAM/bucket policies enabled and disabled. + +variables { + environment_name = "test" + instance_id = "hris" + aws_account_id = "123456789012" + path_to_function_zip = "../aws-proxy-lambda/tests/deployment.zip" + function_zip_hash = "dummy-hash-for-test" + path_to_instance_ssm_parameters = "PSOXY_TEST_HRIS_" + provision_iam_policy_for_testing = false + aws_principal_arn_when_testing = null + aws_write_role_to_assume_when_testing = null +} + +mock_provider "aws" { + mock_data "aws_region" { + defaults = { + name = "us-east-1" + } + } +} + +run "plan_without_testing_infra" { + command = plan + + assert { + error_message = "testing input bucket policy should not be created when provision_iam_policy_for_testing is false" + condition = length(aws_s3_bucket_policy.testing_input) == 0 + } + + assert { + error_message = "testing sanitized bucket policy should not be created when provision_iam_policy_for_testing is false" + condition = length(aws_s3_bucket_policy.testing_sanitized) == 0 + } +} + +run "plan_with_testing_infra" { + command = plan + + variables { + provision_iam_policy_for_testing = true + test_aws_principal_arns = ["arn:aws:iam::123456789012:user/terraform"] + aws_principal_arn_when_testing = "arn:aws:iam::123456789012:role/testCaller" + aws_write_role_to_assume_when_testing = "arn:aws:iam::123456789012:role/terraform" + } + + assert { + error_message = "testing input bucket policy should be created when provision_iam_policy_for_testing is true" + condition = length(aws_s3_bucket_policy.testing_input) == 1 + } + + assert { + error_message = "testing sanitized bucket policy should be created when provision_iam_policy_for_testing is true" + condition = length(aws_s3_bucket_policy.testing_sanitized) == 1 + } +} diff --git a/infra/modules/aws-secretsmanager-secrets/main.tf b/infra/modules/aws-secretsmanager-secrets/main.tf index f5274d4a3a..c6d83ba865 100644 --- a/infra/modules/aws-secretsmanager-secrets/main.tf +++ b/infra/modules/aws-secretsmanager-secrets/main.tf @@ -11,18 +11,23 @@ locals { path_prefix = local.non_empty_path && local.non_fully_qualified_path ? "/${var.path}" : var.path PLACEHOLDER_VALUE = "fill me" - # externally_managed_secrets = { for k, spec in var.secrets : k => spec if !(spec.value_managed_by_tf) } - terraform_managed_secrets = { for k, spec in var.secrets : k => spec if spec.value_managed_by_tf } + terraform_managed_secrets = nonsensitive({ + for k, spec in var.secrets : k => { + description = spec.description + value_managed_by_tf = spec.value_managed_by_tf + sensitive = try(spec.sensitive, true) + } if spec.value_managed_by_tf + }) tf_management_description_appendix = "Value managed by a Terraform configuration; changes outside Terraform may be overwritten by subsequent 'terraform apply' runs" } # see: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter resource "aws_secretsmanager_secret" "secret" { - for_each = var.secrets + for_each = toset(nonsensitive(keys(var.secrets))) name = "${local.path_prefix}${each.key}" - description = "${each.value.description} ${each.value.value_managed_by_tf ? local.tf_management_description_appendix : ""}" + description = "${var.secrets[each.key].description} ${var.secrets[each.key].value_managed_by_tf ? local.tf_management_description_appendix : ""}" kms_key_id = var.kms_key_id } @@ -30,7 +35,7 @@ resource "aws_secretsmanager_secret_version" "terraform_managed" { for_each = local.terraform_managed_secrets secret_id = aws_secretsmanager_secret.secret[each.key].id - secret_string = each.value.value + secret_string = var.secrets[each.key].value } output "secret_ids" { diff --git a/infra/modules/aws-ssm-secrets/main.tf b/infra/modules/aws-ssm-secrets/main.tf index ba88db8e05..508d9e4206 100644 --- a/infra/modules/aws-ssm-secrets/main.tf +++ b/infra/modules/aws-ssm-secrets/main.tf @@ -11,8 +11,21 @@ locals { path_prefix = local.non_empty_path && local.non_fully_qualified_path ? "/${var.path}" : var.path PLACEHOLDER_VALUE = "fill me" - externally_managed_secrets = { for k, spec in var.secrets : k => spec if !(spec.value_managed_by_tf) } - terraform_managed_secrets = { for k, spec in var.secrets : k => spec if spec.value_managed_by_tf } + # for_each keys must not be derived from sensitive secret values; keep metadata only here + externally_managed_secrets = nonsensitive({ + for k, spec in var.secrets : k => { + description = spec.description + value_managed_by_tf = spec.value_managed_by_tf + sensitive = try(spec.sensitive, true) + } if !spec.value_managed_by_tf + }) + terraform_managed_secrets = nonsensitive({ + for k, spec in var.secrets : k => { + description = spec.description + value_managed_by_tf = spec.value_managed_by_tf + sensitive = try(spec.sensitive, true) + } if spec.value_managed_by_tf + }) } # see: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter @@ -24,7 +37,7 @@ resource "aws_ssm_parameter" "secret" { # all are added as secureString type = "SecureString" description = each.value.description - value = sensitive(coalesce(each.value.value, local.PLACEHOLDER_VALUE)) + value = sensitive(coalesce(var.secrets[each.key].value, local.PLACEHOLDER_VALUE)) key_id = coalesce(var.kms_key_id, "alias/aws/ssm") lifecycle { @@ -44,7 +57,7 @@ resource "aws_ssm_parameter" "secret_with_externally_managed_value" { # all are added as secureString type = "SecureString" description = each.value.description - value = sensitive(coalesce(each.value.value, local.PLACEHOLDER_VALUE)) + value = sensitive(coalesce(var.secrets[each.key].value, local.PLACEHOLDER_VALUE)) key_id = coalesce(var.kms_key_id, "alias/aws/ssm") lifecycle { diff --git a/infra/modules/aws/main.tf b/infra/modules/aws/main.tf index 69af90213f..216bb367b2 100644 --- a/infra/modules/aws/main.tf +++ b/infra/modules/aws/main.tf @@ -332,6 +332,7 @@ module "test_tool" { } output "secrets" { + sensitive = true value = { PSOXY_ENCRYPTION_KEY = { value = sensitive(random_password.encryption_key.result), diff --git a/infra/modules/gcp-host/api_rules.tftest.hcl b/infra/modules/gcp-host/api_rules.tftest.hcl index 6ffa0e3541..52f508b2fc 100644 --- a/infra/modules/gcp-host/api_rules.tftest.hcl +++ b/infra/modules/gcp-host/api_rules.tftest.hcl @@ -25,6 +25,13 @@ variables { } mock_provider "google" { + mock_data "google_project" { + defaults = { + project_id = "test-project-123456" + number = 123456789 + } + } + mock_data "google_compute_default_service_account" { defaults = { email = "123456789-compute@developer.gserviceaccount.com" diff --git a/infra/modules/gcp-host/bulk_rules.tftest.hcl b/infra/modules/gcp-host/bulk_rules.tftest.hcl index 467be279c5..565cac1863 100644 --- a/infra/modules/gcp-host/bulk_rules.tftest.hcl +++ b/infra/modules/gcp-host/bulk_rules.tftest.hcl @@ -47,6 +47,13 @@ variables { # Mock provider since we're only testing the logic, not actual GCP resources mock_provider "google" { + mock_data "google_project" { + defaults = { + project_id = "test-project-123456" + number = 123456789 + } + } + mock_data "google_compute_default_service_account" { defaults = { email = "123456789-compute@developer.gserviceaccount.com" diff --git a/infra/modules/gcp-host/concurrency.tftest.hcl b/infra/modules/gcp-host/concurrency.tftest.hcl index b80ba6c183..ecc80e12be 100644 --- a/infra/modules/gcp-host/concurrency.tftest.hcl +++ b/infra/modules/gcp-host/concurrency.tftest.hcl @@ -24,6 +24,13 @@ variables { # Mock provider since we're only testing the logic, not actual GCP resources mock_provider "google" { + mock_data "google_project" { + defaults = { + project_id = "test-project-123456" + number = 123456789 + } + } + mock_data "google_compute_default_service_account" { defaults = { email = "123456789-compute@developer.gserviceaccount.com" diff --git a/infra/modules/gcp-host/main.tf b/infra/modules/gcp-host/main.tf index d16ee80f77..701d1e37b2 100644 --- a/infra/modules/gcp-host/main.tf +++ b/infra/modules/gcp-host/main.tf @@ -231,7 +231,7 @@ module "api_connector" { source = "../../modules/gcp-proxy-api" - project_id = var.gcp_project_id + gcp_project = module.psoxy.gcp_project region = var.gcp_region source_kind = each.value.source_kind environment_id_prefix = local.environment_id_prefix @@ -332,7 +332,7 @@ module "webhook_collector" { source = "../../modules/gcp-webhook-collector" - project_id = var.gcp_project_id + gcp_project = module.psoxy.gcp_project region = var.gcp_region environment_id_prefix = local.environment_id_prefix instance_id = each.key @@ -397,7 +397,7 @@ module "bulk_connector" { source = "../../modules/gcp-proxy-bulk" - project_id = var.gcp_project_id + gcp_project = module.psoxy.gcp_project region = var.gcp_region environment_id_prefix = local.environment_id_prefix instance_id = each.key @@ -572,8 +572,8 @@ locals { instance.instance_id => merge( { endpoint_url = instance.cloud_function_url, - sanitized_bucket = try(instance.async_output_bucket_name, null), - test_examples = local.api_connector_test_examples[instance.instance_id], + sanitized_bucket = try(instance.async_output_bucket_name, null), + test_examples = local.api_connector_test_examples[instance.instance_id], }, instance, var.api_connectors[instance.instance_id] diff --git a/infra/modules/gcp-proxy-api/ip_lock_conditions.tftest.hcl b/infra/modules/gcp-proxy-api/ip_lock_conditions.tftest.hcl index 2e870b31cb..57b416529b 100644 --- a/infra/modules/gcp-proxy-api/ip_lock_conditions.tftest.hcl +++ b/infra/modules/gcp-proxy-api/ip_lock_conditions.tftest.hcl @@ -1,24 +1,22 @@ variables { - project_id = "test-project" + gcp_project = { + project_id = "test-project" + number = 123456789 + } environment_id_prefix = "dev-" instance_id = "test-instance" - service_account_email = "test@example.com" + config_parameter_prefix = "TEST_" + service_account_email = "testsa@test-project.iam.gserviceaccount.com" artifacts_bucket_name = "test-bucket" deployment_bundle_object_name = "bundle.zip" - builder_sa_id = "projects/test-project/serviceAccounts/builder@example.com" + builder_sa_id = "projects/test-project/serviceAccounts/builder@test-project.iam.gserviceaccount.com" source_kind = "test" + tf_runner_iam_principal = "user:terraform@example.com" allowed_data_access_ip_blocks = ["192.168.0.0/24"] } mock_provider "google" { - mock_data "google_project" { - defaults = { - project_id = "test-project" - number = "123456789" - } - } - mock_data "google_service_account" { defaults = { account_id = "test@example.com" diff --git a/infra/modules/gcp-proxy-api/main.tf b/infra/modules/gcp-proxy-api/main.tf index 81d7320e20..941982c9a3 100644 --- a/infra/modules/gcp-proxy-api/main.tf +++ b/infra/modules/gcp-proxy-api/main.tf @@ -1,14 +1,10 @@ # deployment for a single Psoxy instance in GCP project that has be initialized for Psoxy. # project itself may hold MULTIPLE psoxy instances -data "google_project" "project" { - project_id = var.project_id -} - resource "google_secret_manager_secret_iam_member" "grant_sa_accessor_on_secret" { for_each = var.secret_bindings - project = var.project_id + project = var.gcp_project.project_id secret_id = each.value.secret_id member = "serviceAccount:${var.service_account_email}" role = "roles/secretmanager.secretAccessor" @@ -65,7 +61,7 @@ module "async_output" { count = var.enable_async_processing ? 1 : 0 - project_id = var.project_id + project_id = var.gcp_project.project_id bucket_write_role_id = var.bucket_write_role_id function_service_account_email = var.service_account_email bucket_name_prefix = local.bucket_name_prefix @@ -83,7 +79,7 @@ module "async_output" { resource "google_pubsub_topic" "async_output_topic" { count = var.enable_async_processing ? 1 : 0 - project = var.project_id + project = var.gcp_project.project_id name = "${var.environment_id_prefix}${var.instance_id}-async-output" } @@ -91,7 +87,7 @@ resource "google_pubsub_topic" "async_output_topic" { resource "google_pubsub_subscription" "async_output_subscription" { count = var.enable_async_processing ? 1 : 0 - project = var.project_id + project = var.gcp_project.project_id name = "${var.environment_id_prefix}${var.instance_id}-async-output-subscription" topic = google_pubsub_topic.async_output_topic[0].name @@ -119,14 +115,14 @@ resource "google_pubsub_subscription" "async_output_subscription" { locals { # TODO: there's a `google_project_service_identity` resource in `google-beta` provider, which we might be able to leverage from 0.6+ - pubsub_service_identity = "service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" + pubsub_service_identity = "service-${var.gcp_project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" } # 1. Allow the Cloud Function's service account to publish to the topic resource "google_pubsub_topic_iam_member" "function_publisher" { count = var.enable_async_processing ? 1 : 0 - project = var.project_id + project = var.gcp_project.project_id topic = google_pubsub_topic.async_output_topic[0].id member = "serviceAccount:${var.service_account_email}" role = "roles/pubsub.publisher" @@ -140,7 +136,7 @@ resource "google_pubsub_topic_iam_member" "function_publisher" { resource "google_service_account_iam_member" "pubsub_oidc_minter" { count = var.enable_async_processing ? 1 : 0 - service_account_id = "projects/${var.project_id}/serviceAccounts/${var.service_account_email}" + service_account_id = "projects/${var.gcp_project.project_id}/serviceAccounts/${var.service_account_email}" member = "serviceAccount:${local.pubsub_service_identity}" role = "roles/iam.serviceAccountOpenIdTokenCreator" } @@ -150,7 +146,7 @@ module "side_output_bucket" { for_each = local.side_outputs_to_provision - project_id = var.project_id + project_id = var.gcp_project.project_id bucket_write_role_id = var.bucket_write_role_id function_service_account_email = var.service_account_email bucket_name_prefix = local.bucket_name_prefix @@ -217,7 +213,7 @@ locals { resource "google_service_account_iam_member" "tf_runner_act_as" { member = var.tf_runner_iam_principal role = "roles/iam.serviceAccountUser" - service_account_id = "projects/${var.project_id}/serviceAccounts/${var.service_account_email}" + service_account_id = "projects/${var.gcp_project.project_id}/serviceAccounts/${var.service_account_email}" } # migration: remove old resource address from state (destroyed in GCP) @@ -233,7 +229,7 @@ resource "google_cloudfunctions2_function" "function" { name = "${var.environment_id_prefix}${var.instance_id}" description = "Psoxy Connector - ${var.source_kind}" - project = var.project_id + project = var.gcp_project.project_id location = var.region build_config { @@ -296,7 +292,8 @@ resource "google_cloudfunctions2_function" "function" { content { key = secret_environment_variable.key - project_id = data.google_project.project.number + # project_id string (not number) avoids apply-time drift: number comes from data.google_project + project_id = var.gcp_project.project_id secret = secret_environment_variable.value.secret_id version = secret_environment_variable.value.version_number } @@ -317,7 +314,7 @@ module "service_url_parameter" { count = var.enable_async_processing ? 1 : 0 - secret_project = var.project_id + secret_project = var.gcp_project.project_id path_prefix = local.path_to_instance_config_parameters replica_locations = var.secret_replica_locations secrets = { @@ -343,7 +340,7 @@ locals { resource "google_secret_manager_secret_iam_member" "grant_sa_viewer_on_parameter" { for_each = local.secrets_to_grant_access_to - project = var.project_id + project = var.gcp_project.project_id secret_id = each.value.secret_id member = "serviceAccount:${var.service_account_email}" role = "roles/secretmanager.viewer" @@ -352,7 +349,7 @@ resource "google_secret_manager_secret_iam_member" "grant_sa_viewer_on_parameter resource "google_secret_manager_secret_iam_member" "grant_sa_accessor_on_parameter" { for_each = local.secrets_to_grant_access_to - project = var.project_id + project = var.gcp_project.project_id secret_id = each.value.secret_id member = "serviceAccount:${var.service_account_email}" role = "roles/secretmanager.secretAccessor" # this is ONLY accessing payload of a secret version diff --git a/infra/modules/gcp-proxy-api/variables.tf b/infra/modules/gcp-proxy-api/variables.tf index 52b050dc2d..f7cf5d4dee 100644 --- a/infra/modules/gcp-proxy-api/variables.tf +++ b/infra/modules/gcp-proxy-api/variables.tf @@ -1,6 +1,9 @@ -variable "project_id" { - type = string - description = "name of the gcp project" +variable "gcp_project" { + type = object({ + project_id = string + number = number + }) + description = "GCP project hosting this Psoxy instance. project_id and number must refer to the same project (same fields as data.google_project); pass from parent (eg module.psoxy.gcp_project) to avoid data.google_project lookups in each connector instance." } variable "tf_runner_iam_principal" { diff --git a/infra/modules/gcp-proxy-bulk/main.tf b/infra/modules/gcp-proxy-bulk/main.tf index d866acc2c1..469f21288d 100644 --- a/infra/modules/gcp-proxy-bulk/main.tf +++ b/infra/modules/gcp-proxy-bulk/main.tf @@ -18,10 +18,6 @@ locals { path_to_instance_config_parameters = "${coalesce(var.config_parameter_prefix, "")}${replace(upper(local.instance_id), "-", "_")}_" } -data "google_project" "project" { - project_id = var.project_id -} - resource "random_string" "bucket_id_part" { length = 8 special = false @@ -55,7 +51,7 @@ locals { # trivy:ignore:AVD-GCP-0078 # trivy:ignore:AVD-GCP-0077 resource "google_storage_bucket" "input_bucket" { - project = var.project_id + project = var.gcp_project.project_id name = coalesce(var.input_bucket_name, "${local.bucket_prefix}-input") location = var.region force_destroy = var.bucket_force_destroy @@ -89,7 +85,7 @@ resource "google_storage_bucket" "input_bucket" { module "output_bucket" { source = "../gcp-output-bucket" - project_id = var.project_id + project_id = var.gcp_project.project_id bucket_write_role_id = var.bucket_write_role_id function_service_account_email = google_service_account.service_account.email bucket_name_prefix = coalesce(var.sanitized_bucket_name, local.bucket_prefix) @@ -102,7 +98,7 @@ module "output_bucket" { } resource "google_service_account" "service_account" { - project = var.project_id + project = var.gcp_project.project_id account_id = local.sa_name display_name = "Psoxy Connector - ${var.source_kind}" description = "${local.function_name} runs as this service account" @@ -161,7 +157,7 @@ resource "google_storage_bucket_iam_member" "grant_testers_admin_on_processed_bu resource "google_secret_manager_secret_iam_member" "grant_sa_accessor_on_secret" { for_each = var.secret_bindings - project = var.project_id + project = var.gcp_project.project_id secret_id = each.value.secret_id member = "serviceAccount:${google_service_account.service_account.email}" role = "roles/secretmanager.secretAccessor" @@ -174,13 +170,13 @@ resource "google_secret_manager_secret_iam_member" "grant_sa_accessor_on_secret" # historically, we used the GCP Compute Engine default service account as the event trigger SA, and relied on it having 'Editor' by default on most GCP projects # but that does not seem to be universally true - so we'll stop relying on that assumption from 0.5.9 onwards resource "google_project_iam_member" "grant_sa_event_receiver" { - project = var.project_id + project = var.gcp_project.project_id member = "serviceAccount:${google_service_account.service_account.email}" role = "roles/eventarc.eventReceiver" } resource "google_cloud_run_service_iam_member" "grant_sa_invoker" { - project = var.project_id + project = var.gcp_project.project_id location = google_cloudfunctions2_function.function.location service = google_cloudfunctions2_function.function.name member = "serviceAccount:${google_service_account.service_account.email}" @@ -208,7 +204,7 @@ removed { resource "google_cloudfunctions2_function" "function" { name = local.function_name description = "Psoxy instance to process ${var.source_kind} files" - project = var.project_id + project = var.gcp_project.project_id location = var.region build_config { @@ -234,7 +230,7 @@ resource "google_cloudfunctions2_function" "function" { timeout_seconds = var.timeout_seconds ingress_settings = "ALLOW_INTERNAL_ONLY" - # bulk connectors do not dial out to SaaS APIs; only honor legacy serverless connector path + # bulk connectors do not dial out to SaaS APIs; only honor legacy serverless connector path vpc_connector = var.vpc_config != null && try(var.vpc_config.serverless_connector, null) != null ? var.vpc_config.serverless_connector : null vpc_connector_egress_settings = var.vpc_config != null && try(var.vpc_config.serverless_connector, null) != null ? "ALL_TRAFFIC" : null @@ -256,7 +252,8 @@ resource "google_cloudfunctions2_function" "function" { content { key = secret_environment_variable.key - project_id = data.google_project.project.number + # project_id string (not number) avoids apply-time drift: number comes from data.google_project + project_id = var.gcp_project.project_id secret = secret_environment_variable.value.secret_id version = secret_environment_variable.value.version_number } @@ -325,7 +322,7 @@ EOT Review the deployed Cloud function in GCP console: -[Function in GCP Console](https://console.cloud.google.com/functions/details/${var.region}/${google_cloudfunctions2_function.function.name}?project=${var.project_id}) +[Function in GCP Console](https://console.cloud.google.com/functions/details/${var.region}/${google_cloudfunctions2_function.function.name}?project=${var.gcp_project.project_id}) We provide some Node.js scripts to easily validate the deployment. To be able to run the test commands below, you need Node.js (>=20) and npm (v >=8) diff --git a/infra/modules/gcp-proxy-bulk/variables.tf b/infra/modules/gcp-proxy-bulk/variables.tf index 1767fea4ed..e9ccec2ccb 100644 --- a/infra/modules/gcp-proxy-bulk/variables.tf +++ b/infra/modules/gcp-proxy-bulk/variables.tf @@ -1,6 +1,9 @@ -variable "project_id" { - type = string - description = "id of GCP project that will host psoxy instance" +variable "gcp_project" { + type = object({ + project_id = string + number = number + }) + description = "GCP project hosting this Psoxy instance. project_id and number must refer to the same project (same fields as data.google_project); pass from parent (eg module.psoxy.gcp_project) to avoid data.google_project lookups in each connector instance." } variable "tf_runner_iam_principal" { diff --git a/infra/modules/gcp-webhook-collector/ip_lock_conditions.tftest.hcl b/infra/modules/gcp-webhook-collector/ip_lock_conditions.tftest.hcl index 2cd002d3cc..bbc3ad1fe4 100644 --- a/infra/modules/gcp-webhook-collector/ip_lock_conditions.tftest.hcl +++ b/infra/modules/gcp-webhook-collector/ip_lock_conditions.tftest.hcl @@ -1,5 +1,8 @@ variables { - project_id = "test-project" + gcp_project = { + project_id = "test-project" + number = 123456789 + } environment_id_prefix = "dev-" instance_id = "test-instance" config_parameter_prefix = "TEST_" @@ -17,20 +20,15 @@ variables { bucket_write_role_id = "roles/storage.objectCreator" webhook_batch_invoker_sa_email = "batch@test-project.iam.gserviceaccount.com" key_ring_id = "projects/test-project/locations/us-central1/keyRings/test-key-ring" - provision_auth_key = {} + provision_auth_key = { + rotation_days = 30 + } example_identity = "test-user@example.com" allowed_webhook_ip_blocks = ["10.0.0.0/16"] } mock_provider "google" { - mock_data "google_project" { - defaults = { - project_id = "test-project" - number = "123456789" - } - } - } run "validate_cloud_run_webhook_invokers_no_ip_condition" { diff --git a/infra/modules/gcp-webhook-collector/main.tf b/infra/modules/gcp-webhook-collector/main.tf index 74419aaba3..0e31555d23 100644 --- a/infra/modules/gcp-webhook-collector/main.tf +++ b/infra/modules/gcp-webhook-collector/main.tf @@ -21,16 +21,12 @@ locals { # deployment for a single Psoxy instance in GCP project that has be initialized for Psoxy. # project itself may hold MULTIPLE psoxy instances -data "google_project" "project" { - project_id = var.project_id -} - # perms for secrets # TODO: imho, would be cleaner to combine into a custom project role?? resource "google_secret_manager_secret_iam_member" "grant_sa_viewer_on_secret" { for_each = var.secret_bindings - project = var.project_id + project = var.gcp_project.project_id secret_id = each.value.secret_id member = "serviceAccount:${var.service_account.email}" role = "roles/secretmanager.viewer" @@ -39,7 +35,7 @@ resource "google_secret_manager_secret_iam_member" "grant_sa_viewer_on_secret" { resource "google_secret_manager_secret_iam_member" "grant_sa_accessor_on_secret" { for_each = var.secret_bindings - project = var.project_id + project = var.gcp_project.project_id secret_id = each.value.secret_id member = "serviceAccount:${var.service_account.email}" role = "roles/secretmanager.secretAccessor" # this is ONLY accessing payload of a secret version @@ -152,7 +148,7 @@ module "sanitized_webhook_output" { enable_versioning = var.enable_versioning bucket_access_logs_destination = var.bucket_access_logs_destination - project_id = var.project_id + project_id = var.gcp_project.project_id bucket_write_role_id = var.bucket_write_role_id function_service_account_email = var.service_account.email bucket_name_prefix = local.bucket_name_prefix @@ -172,7 +168,7 @@ module "side_output_bucket" { enable_versioning = var.enable_versioning bucket_access_logs_destination = var.bucket_access_logs_destination - project_id = var.project_id + project_id = var.gcp_project.project_id bucket_write_role_id = var.bucket_write_role_id function_service_account_email = var.service_account.email bucket_name_prefix = local.bucket_name_prefix @@ -241,7 +237,7 @@ locals { module "auth_issuer_secret" { source = "../../modules/gcp-secrets" - secret_project = var.project_id + secret_project = var.gcp_project.project_id path_prefix = local.path_to_instance_config_parameters replica_locations = var.secret_replica_locations secrets = { @@ -273,7 +269,7 @@ locals { resource "google_secret_manager_secret_iam_member" "grant_sa_viewer_on_parameter" { for_each = local.secrets_to_grant_access_to - project = var.project_id + project = var.gcp_project.project_id secret_id = each.value.secret_id member = "serviceAccount:${var.service_account.email}" role = "roles/secretmanager.viewer" @@ -282,7 +278,7 @@ resource "google_secret_manager_secret_iam_member" "grant_sa_viewer_on_parameter resource "google_secret_manager_secret_iam_member" "grant_sa_accessor_on_parameter" { for_each = local.secrets_to_grant_access_to - project = var.project_id + project = var.gcp_project.project_id secret_id = each.value.secret_id member = "serviceAccount:${var.service_account.email}" role = "roles/secretmanager.secretAccessor" # this is ONLY accessing payload of a secret version @@ -311,7 +307,7 @@ resource "google_cloudfunctions2_function" "function" { name = "${var.environment_id_prefix}${var.instance_id}" description = "Webhook Collector - ${var.source_kind}" - project = var.project_id + project = var.gcp_project.project_id location = var.region build_config { @@ -372,7 +368,8 @@ resource "google_cloudfunctions2_function" "function" { content { key = secret_environment_variable.key - project_id = data.google_project.project.number + # project_id string (not number) avoids apply-time drift: number comes from data.google_project + project_id = var.gcp_project.project_id secret = secret_environment_variable.value.secret_id version = secret_environment_variable.value.version_number } @@ -404,14 +401,14 @@ resource "google_cloud_run_service_iam_binding" "invokers" { # Pub/Sub topic for individual webhook messages resource "google_pubsub_topic" "webhook_topic" { name = "${var.environment_id_prefix}${var.instance_id}-webhooks" - project = var.project_id + project = var.gcp_project.project_id } # Pub/Sub subscription for batch processing resource "google_pubsub_subscription" "webhook_subscription" { name = "${var.environment_id_prefix}${var.instance_id}-webhook-subscription" topic = google_pubsub_topic.webhook_topic.name - project = var.project_id + project = var.gcp_project.project_id # Configure for batch processing ack_deadline_seconds = 600 # 10 minutes to process messages @@ -428,7 +425,7 @@ resource "google_pubsub_subscription" "webhook_subscription" { # IAM binding to allow the Cloud Function to publish to the topic resource "google_pubsub_topic_iam_member" "publisher" { - project = var.project_id + project = var.gcp_project.project_id topic = google_pubsub_topic.webhook_topic.name role = "roles/pubsub.publisher" member = "serviceAccount:${var.service_account.email}" @@ -436,7 +433,7 @@ resource "google_pubsub_topic_iam_member" "publisher" { # IAM binding to allow the Cloud Function to pull from the subscription resource "google_pubsub_subscription_iam_member" "subscriber" { - project = var.project_id + project = var.gcp_project.project_id subscription = google_pubsub_subscription.webhook_subscription.name role = "roles/pubsub.subscriber" member = "serviceAccount:${var.service_account.email}" @@ -444,7 +441,7 @@ resource "google_pubsub_subscription_iam_member" "subscriber" { resource "google_cloud_scheduler_job" "trigger_batch_processing" { - project = var.project_id + project = var.gcp_project.project_id region = var.region name = "${var.environment_id_prefix}${var.instance_id}-batch-processing" schedule = "*/${var.batch_processing_frequency_minutes} * * * *" diff --git a/infra/modules/gcp-webhook-collector/variables.tf b/infra/modules/gcp-webhook-collector/variables.tf index 1d4ffecef2..aaf6f05d46 100644 --- a/infra/modules/gcp-webhook-collector/variables.tf +++ b/infra/modules/gcp-webhook-collector/variables.tf @@ -1,6 +1,9 @@ -variable "project_id" { - type = string - description = "name of the gcp project" +variable "gcp_project" { + type = object({ + project_id = string + number = number + }) + description = "GCP project hosting this Psoxy instance. project_id and number must refer to the same project (same fields as data.google_project); pass from parent (eg module.psoxy.gcp_project) to avoid data.google_project lookups in each connector instance." } variable "tf_runner_iam_principal" { @@ -191,11 +194,9 @@ variable "provision_auth_key" { validation { condition = ( - var.provision_auth_key == null || - ( - try(var.provision_auth_key.rotation_days, null) == null || - try(var.provision_auth_key.rotation_days, 0) > 0 - ) + var.provision_auth_key == null ? true : + var.provision_auth_key.rotation_days == null ? true : + var.provision_auth_key.rotation_days > 0 ) error_message = "If `provision_auth_key` is provided, `rotation_days` must be a positive number or null." } diff --git a/infra/modules/gcp/main.tf b/infra/modules/gcp/main.tf index be2bfef2d5..b5b2085059 100644 --- a/infra/modules/gcp/main.tf +++ b/infra/modules/gcp/main.tf @@ -526,6 +526,14 @@ resource "google_project_iam_member" "data_sanitization_tester_grant" { +output "gcp_project" { + value = { + project_id = var.project_id + number = data.google_project.project.number + } + description = "GCP project hosting this Psoxy environment. project_id is the configured project id (known at plan time); number is from data.google_project (for IAM principal patterns that require it)." +} + output "artifacts_bucket_name" { value = local.artifact_bucket_name } diff --git a/java/core/src/main/java/co/worklytics/psoxy/gateway/ProxyConstants.java b/java/core/src/main/java/co/worklytics/psoxy/gateway/ProxyConstants.java index 90fbce599c..257b144963 100644 --- a/java/core/src/main/java/co/worklytics/psoxy/gateway/ProxyConstants.java +++ b/java/core/src/main/java/co/worklytics/psoxy/gateway/ProxyConstants.java @@ -22,7 +22,7 @@ public class ProxyConstants { /** * Version of the Java source code. Used to identify the version of the proxy. */ - public static final String JAVA_SOURCE_CODE_VERSION = "v0.6.5"; + public static final String JAVA_SOURCE_CODE_VERSION = "v0.6.6"; /** * a random UUID used to salt the hash of the salt. Purpose of this is to invalidate any non-purpose built rainbow table solution. diff --git a/java/core/src/main/java/co/worklytics/psoxy/rules/RulesUtils.java b/java/core/src/main/java/co/worklytics/psoxy/rules/RulesUtils.java index b113b7e9b5..619962aac6 100644 --- a/java/core/src/main/java/co/worklytics/psoxy/rules/RulesUtils.java +++ b/java/core/src/main/java/co/worklytics/psoxy/rules/RulesUtils.java @@ -10,7 +10,6 @@ import java.util.Base64; import java.util.List; import java.util.Optional; -import java.util.logging.Level; import java.util.zip.GZIPInputStream; import javax.inject.Inject; import javax.inject.Named; @@ -46,6 +45,9 @@ public class RulesUtils { */ public static final String RULES_RESOURCE_PATH = "rules.yaml"; + /** Max chars of decoded RULES YAML included in parse log lines. */ + private static final int RULES_LOG_SNIPPET_MAX_CHARS = 200; + @Inject @Named("ForYAML") ObjectMapper yamlMapper; @@ -158,7 +160,8 @@ public com.avaulta.gateway.rules.RuleSet parse(@NonNull String yamlString) { try { com.avaulta.gateway.rules.RuleSet rules = yamlMapper.readerFor(impl).readValue(yamlString); - log.log(Level.INFO, "RULES parsed as {0}", impl.getSimpleName()); + log.info(String.format("RULES parsed as %s (sha1=%s, snippet=%s)", + impl.getSimpleName(), sha(rules), rulesLogSnippet(yamlString))); validator.validate(rules); return rules; } catch (com.fasterxml.jackson.core.JsonParseException | com.fasterxml.jackson.databind.JsonMappingException e) { @@ -178,6 +181,14 @@ public com.avaulta.gateway.rules.RuleSet parse(@NonNull String yamlString) { throw new InvalidRulesException("Failed to parse RULES from config", ErrorCauses.RULES_INVALID); } + private static String rulesLogSnippet(String yaml) { + String singleLine = yaml.replace('\r', ' ').replace('\n', ' ').trim(); + if (singleLine.length() <= RULES_LOG_SNIPPET_MAX_CHARS) { + return singleLine; + } + return singleLine.substring(0, RULES_LOG_SNIPPET_MAX_CHARS) + "..."; + } + public List parseAdditionalTransforms(ConfigService config) { Optional additionalTransforms = config.getConfigPropertyAsOptional(BulkModeConfigProperty.ADDITIONAL_TRANSFORMS); CollectionType type = yamlMapper.getTypeFactory().constructCollectionType(ArrayList.class, StorageHandler.ObjectTransform.class); diff --git a/java/core/src/test/java/co/worklytics/psoxy/rules/chatgpt/ChatGPTComplianceTests.java b/java/core/src/test/java/co/worklytics/psoxy/rules/chatgpt/ChatGPTComplianceTests.java index 2b6dde2bc7..1f6137f9d9 100644 --- a/java/core/src/test/java/co/worklytics/psoxy/rules/chatgpt/ChatGPTComplianceTests.java +++ b/java/core/src/test/java/co/worklytics/psoxy/rules/chatgpt/ChatGPTComplianceTests.java @@ -49,4 +49,31 @@ void sanitizesRoleBasedUserMessageAuthorshipInLogs() { "Can you summarize the Q2 financial results for the EMEA region?", "Q2 financial results"); } + + @Test + void sanitizesAssistantMessageContentInLogs() { + String original = asJson("sample_log_message.json") + .replace("\"type\": \"ACCOUNT_USER\"", "\"type\": \"ASSISTANT\"") + .replace("\"type\": \"user\"", "\"type\": \"assistant\"") + .replace("\"type\": \"user_instructions\"", "\"type\": \"text\"") + .replace("Can you summarize the Q2 financial results for the EMEA region?", + "John Smith earns $150,000 annually in the EMEA region."); + + String sanitized = sanitize("https://api.chatgpt.com/v1/compliance/workspaces/some_id/logs/some_id", original); + + assertRedacted(sanitized, "John Smith", "$150,000"); + } + + @Test + void sanitizesUserMessageWhenActorTypeMissing() { + String original = asJson("sample_log_message.json") + .replace("\"type\": \"user\",\n \"client_type\": \"unknown\"", "\"role\": \"user\",\n \"client_type\": \"unknown\"") + .replace("\"actor\": {\n \"type\": \"ACCOUNT_USER\",\n \"user_id\": \"user-abc123def456\",\n \"user_email\": \"alice@example.com\"\n },", ""); + + String sanitized = sanitize("https://api.chatgpt.com/v1/compliance/workspaces/some_id/logs/some_id", original); + + assertRedacted(sanitized, + "Can you summarize the Q2 financial results for the EMEA region?", + "Q2 financial results"); + } } diff --git a/java/pom.xml b/java/pom.xml index bf1eba9614..987ed9ef38 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -10,7 +10,7 @@ pom - 0.6.5 + 0.6.6 UTF-8 1.18.42 2.40.5 diff --git a/tools/build.sh b/tools/build.sh index 29e6918f7f..73fc3696d3 100755 --- a/tools/build.sh +++ b/tools/build.sh @@ -87,6 +87,17 @@ mvn ${PSOXY_MAVEN_LOCAL_REPO:+-Dmaven.repo.local="$PSOXY_MAVEN_LOCAL_REPO"} \ ${PSOXY_SKIP_OPENNLP:+-DskipOpenNlpModelDownload=true} \ package $QUIET_OPTIONS -f "${PARENT_POM}" -pl impl/${IMPLEMENTATION} -am $DISTRIBUTION_PROFILE -DEPLOYMENT_ARTIFACT=$(ls "${JAVA_SOURCE_ROOT}impl/${IMPLEMENTATION}/target/deployment" | grep -E "^psoxy-.*\.jar$" | head -1) +DEPLOYMENT_DIR="${JAVA_SOURCE_ROOT}impl/${IMPLEMENTATION}/target/deployment" +if [ ! -d "$DEPLOYMENT_DIR" ]; then + printf "${ERR}Deployment directory not found at ${DEPLOYMENT_DIR}.${NC}\n" + exit 1 +fi + +DEPLOYMENT_ARTIFACT=$(find "$DEPLOYMENT_DIR" -maxdepth 1 -type f -name 'psoxy-*.jar' | head -1) +DEPLOYMENT_ARTIFACT="${DEPLOYMENT_ARTIFACT##*/}" +if [ -z "$DEPLOYMENT_ARTIFACT" ]; then + printf "${ERR}No deployment JAR found under ${DEPLOYMENT_DIR}.${NC}\n" + exit 1 +fi printf "${SUCCESS}Build complete.${NC} Deployment artifact: ${INFO}${DEPLOYMENT_ARTIFACT}${NC}\n" diff --git a/tools/init-tfvars.sh b/tools/init-tfvars.sh index 9b91cd6049..2d85ad7834 100755 --- a/tools/init-tfvars.sh +++ b/tools/init-tfvars.sh @@ -7,7 +7,7 @@ PSOXY_BASE_DIR=$2 DEPLOYMENT_ENV=${3:-"local"} HOST_PLATFORM=${4:-"aws"} -SCRIPT_VERSION="v0.6.5" +SCRIPT_VERSION="v0.6.6" if [ -z "$PSOXY_BASE_DIR" ]; then printf "Usage: init-tfvars.sh [DEPLOYMENT_ENV]\n" diff --git a/tools/js-reference/package-lock.json b/tools/js-reference/package-lock.json index af2b515b09..12bad40b1e 100644 --- a/tools/js-reference/package-lock.json +++ b/tools/js-reference/package-lock.json @@ -895,10 +895,20 @@ } }, "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.2.0.tgz", + "integrity": "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "MIT", "dependencies": { "argparse": "^2.0.1" diff --git a/tools/psoxy-test/package-lock.json b/tools/psoxy-test/package-lock.json index 40c8de7ef6..59044a7325 100644 --- a/tools/psoxy-test/package-lock.json +++ b/tools/psoxy-test/package-lock.json @@ -1005,12 +1005,6 @@ "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", "license": "BSD-3-Clause" }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.2.tgz", - "integrity": "sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw==", - "license": "BSD-3-Clause" - }, "node_modules/@protobufjs/path": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", @@ -2750,6 +2744,24 @@ "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", "license": "MIT" }, + "node_modules/fetch-blob": { + "version": "5.0.0", + "resolved": "git+ssh://git@github.com/node-fetch/fetch-blob.git#d6f1029ae59bde4768adcbf79dbce9eab15bbb1f", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=20" + } + }, "node_modules/figures": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/figures/-/figures-6.1.0.tgz", @@ -2806,15 +2818,15 @@ "license": "MIT" }, "node_modules/form-data": { - "version": "2.5.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.5.tgz", - "integrity": "sha512-jqdObeR2rxZZbPSGL+3VckHMYtu+f9//KXBsVny6JSX/pa38Fy+bGjuG8eW/H6USNQWhLi8Num++cU2yOCNz4A==", + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.6.tgz", + "integrity": "sha512-Ogz/E85h9tlfJzpI6TuFpGcHZFhLrb9Gw8wq9v40CxSCPnv7ahKr6Xgtkn0KYCDQJ8DNn5VoMO8EXr9V5PadyA==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", + "hasown": "^2.0.4", "mime-types": "^2.1.35", "safe-buffer": "^5.2.1" }, @@ -4098,9 +4110,9 @@ } }, "node_modules/protobufjs": { - "version": "7.6.2", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.2.tgz", - "integrity": "sha512-N9EiLovGEQOJSPF26Ij7qUGvahfEnq0eeYZ02aigIedkmz1qZSwjnP9SBITHJuF/6MYbIW4HDN8zdYjsjqJKXQ==", + "version": "7.6.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.4.tgz", + "integrity": "sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -4110,7 +4122,6 @@ "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", diff --git a/tools/release/lib/delete-gh-maven-packages.sh b/tools/release/lib/delete-gh-maven-packages.sh new file mode 100644 index 0000000000..3530780ecc --- /dev/null +++ b/tools/release/lib/delete-gh-maven-packages.sh @@ -0,0 +1,73 @@ +# Delete Maven package versions from the GitHub Packages registry. +# Sourced by tools/release/publish-mvn-artifacts.sh + +# delete_gh_maven_package_versions +# Deletes all co.worklytics.psoxy and com.avaulta.gateway packages at . +# Only -SNAPSHOT versions may be deleted. +delete_gh_maven_package_versions() { + local path_to_repo="$1" + local target_version="$2" + + if [ -z "$path_to_repo" ] || [ -z "$target_version" ]; then + printf "${ERR}delete_gh_maven_package_versions requires path-to-repo and version arguments.${NC}\n" + return 1 + fi + + if [[ "$target_version" != *-SNAPSHOT ]]; then + printf "${ERR}Refusing to delete non-SNAPSHOT version ${target_version}; only -SNAPSHOT packages may be removed.${NC}\n" + return 1 + fi + + if ! command -v jq >/dev/null 2>&1; then + printf "${WARN}Warning: 'jq' not found. Skipping deletion of GitHub Packages version ${target_version}.${NC}\n" + return 0 + fi + + if ! command -v gh >/dev/null 2>&1; then + printf "${WARN}Warning: 'gh' not found. Skipping deletion of GitHub Packages version ${target_version}.${NC}\n" + return 0 + fi + + printf "Checking for GitHub Packages artifacts at version ${INFO}${target_version}${NC}...\n" + + local repo_full_name + repo_full_name=$(git -C "$path_to_repo" config --get remote.origin.url | sed -E 's#.*github\.com[:/]+([^/]+/[^/.]+)(\.git)?$#\1#') + local org_name + org_name=$(printf '%s\n' "$repo_full_name" | cut -d'/' -f1) + + local group_ids=("co.worklytics.psoxy" "com.avaulta.gateway") + local packages_json + packages_json=$(gh api "/orgs/${org_name}/packages?package_type=maven" 2>/dev/null) + + if [ $? -ne 0 ]; then + printf "${ERR}Warning: Failed to list packages. Check 'read:packages' scope.${NC}\n" + return 1 + fi + + local group_id pkg versions_json version_id + for group_id in "${group_ids[@]}"; do + printf " Scanning for packages starting with: ${INFO}${group_id}${NC}...\n" + + while IFS= read -r pkg; do + [ -z "$pkg" ] && continue + printf " Checking package: ${INFO}${pkg}${NC} for version ${target_version}...\n" + + versions_json=$(gh api "/orgs/${org_name}/packages/maven/${pkg}/versions" 2>/dev/null) + if [ $? -ne 0 ]; then + printf " ${ERR}Failed to list versions for ${pkg}.${NC}\n" + continue + fi + + version_id=$(jq -r ".[] | select(.name == \"${target_version}\") | .id" <<< "$versions_json") + if [ -n "$version_id" ] && [ "$version_id" != "null" ]; then + printf " Found version ${target_version} (ID: ${version_id}). Deleting...\n" + if gh api -X DELETE "/orgs/${org_name}/packages/maven/${pkg}/versions/${version_id}" 2>/dev/null; then + printf " ${SUCCESS}✓ Deleted ${pkg}:${target_version}${NC}\n" + else + printf " ${ERR}✗ Failed to delete ${pkg}:${target_version}. Likely 403 Forbidden.${NC}\n" + printf " Ensure your token has ${INFO}delete:packages${NC} scope.\n" + fi + fi + done < <(jq -r ".[] | select(.name | startswith(\"${group_id}\")) | .name" <<< "$packages_json") + done +} diff --git a/tools/release/publish-mvn-artifacts.sh b/tools/release/publish-mvn-artifacts.sh index f1172b17a3..29f9b4f12b 100755 --- a/tools/release/publish-mvn-artifacts.sh +++ b/tools/release/publish-mvn-artifacts.sh @@ -7,6 +7,9 @@ else ERR='\033[0;31m'; SUCCESS='\033[0;32m'; WARN='\033[1;33m'; INFO='\033[0;34m'; CODE='\033[0;36m'; NC='\033[0m' fi +# shellcheck source=lib/delete-gh-maven-packages.sh +source "$(dirname "$0")/lib/delete-gh-maven-packages.sh" + PATH_TO_REPO="$1" if [ -z "$PATH_TO_REPO" ]; then @@ -63,63 +66,7 @@ if [ "$IS_RC" = true ]; then printf "RC build detected. Using version: ${INFO}${TARGET_VERSION}${NC}\n" # 3. Delete existing SNAPSHOT artifacts - printf "Checking for existing artifacts to clean up...\n" - - REPO_FULL_NAME=$(git -C "$PATH_TO_REPO" config --get remote.origin.url | sed -E 's/.*github.com[:\/](.*).git/\1/') - ORG_NAME=$(echo "$REPO_FULL_NAME" | cut -d'/' -f1) - - # List of relevant group IDs to check - GROUP_IDS=("co.worklytics.psoxy" "com.avaulta.gateway") - - printf " Fetching packages for org: ${INFO}${ORG_NAME}${NC}...\n" - - PACKAGES_JSON=$(gh api "/orgs/${ORG_NAME}/packages?package_type=maven" 2>/dev/null) - - if [ $? -ne 0 ]; then - printf "${ERR}Warning: Failed to list packages. Check 'read:packages' scope.${NC}\n" - PACKAGES_JSON="[]" - fi - - if command -v jq &> /dev/null; then - - for GROUP_ID in "${GROUP_IDS[@]}"; do - printf " Scanning for packages starting with: ${INFO}${GROUP_ID}${NC}...\n" - - # Filter packages by group ID - PACKAGE_NAMES=$(echo "$PACKAGES_JSON" | jq -r ".[] | select(.name | startswith(\"${GROUP_ID}\")) | .name") - - for PKG in $PACKAGE_NAMES; do - printf " Checking package: ${INFO}${PKG}${NC} for version ${TARGET_VERSION}...\n" - - VERSIONS_JSON=$(gh api "/orgs/${ORG_NAME}/packages/maven/${PKG}/versions" 2>/dev/null) - - if [ $? -ne 0 ]; then - printf " ${ERR}Failed to list versions for ${PKG}.${NC}\n" - continue - fi - - VERSION_ID=$(echo "$VERSIONS_JSON" | jq -r ".[] | select(.name == \"${TARGET_VERSION}\") | .id") - - if [ -n "$VERSION_ID" ] && [ "$VERSION_ID" != "null" ]; then - printf " Found version ${TARGET_VERSION} (ID: ${VERSION_ID}). Deleting...\n" - - if gh api -X DELETE "/orgs/${ORG_NAME}/packages/maven/${PKG}/versions/${VERSION_ID}" 2>/dev/null; then - printf " ${SUCCESS}✓ Deleted ${PKG}:${TARGET_VERSION}${NC}\n" - else - printf " ${ERR}✗ Failed to delete ${PKG}:${TARGET_VERSION}. Likely 403 Forbidden.${NC}\n" - printf " Ensure your token has ${INFO}delete:packages${NC} scope.\n" - fi - else - # Verbose but useful for debugging - # printf " Version ${TARGET_VERSION} not found.\n" - : - fi - done - done - else - printf "${ERR}Warning: 'jq' not found. Skipping automated cleanup of old artifacts.${NC}\n" - printf " Install 'jq' to enable this feature.\n" - fi + delete_gh_maven_package_versions "$PATH_TO_REPO" "$TARGET_VERSION" else TARGET_VERSION="${POM_VERSION}" @@ -136,3 +83,9 @@ else printf " See: https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-apache-maven-registry\n" exit 1 fi + +if [ "$IS_TAG" = true ]; then + SNAPSHOT_VERSION="${TARGET_VERSION}-SNAPSHOT" + printf "\nRelease published. Removing RC SNAPSHOT artifacts (${INFO}${SNAPSHOT_VERSION}${NC}) from GitHub Packages...\n" + delete_gh_maven_package_versions "$PATH_TO_REPO" "$SNAPSHOT_VERSION" +fi diff --git a/tools/schema-tool/package-lock.json b/tools/schema-tool/package-lock.json index e8c56de591..685b494747 100644 --- a/tools/schema-tool/package-lock.json +++ b/tools/schema-tool/package-lock.json @@ -344,14 +344,11 @@ } }, "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } + "license": "Python-2.0" }, "node_modules/array-find-index": { "version": "1.0.2", @@ -929,20 +926,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/estree-walker": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", @@ -1391,14 +1374,23 @@ } }, "node_modules/js-yaml": { - "version": "3.14.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", - "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.2.0.tgz", + "integrity": "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "MIT", "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" @@ -2017,13 +2009,6 @@ "url": "https://github.com/chalk/slice-ansi?sponsor=1" } }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "dev": true, - "license": "BSD-3-Clause" - }, "node_modules/stack-utils": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", @@ -2205,9 +2190,9 @@ } }, "node_modules/tar": { - "version": "7.5.15", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.15.tgz", - "integrity": "sha512-dzGK0boVlC4W5QFuQN1EFSl3bIDYsk7Tj40U6eIBnK2k/8ml7TZ5agbI5j5+qnoVcAA+rNtBml8SEiLxZpNqRQ==", + "version": "7.5.16", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.16.tgz", + "integrity": "sha512-56adEpPMouktRlBLXiaYFFzZ/3+JXa8P9n7WbR+ibIjtviN55mEaOkiysCnPnWm+7kkui1Dn8J9l+g6zV8731w==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { diff --git a/tools/schema-tool/package.json b/tools/schema-tool/package.json index 5a3c545599..6ee8dbf223 100644 --- a/tools/schema-tool/package.json +++ b/tools/schema-tool/package.json @@ -28,6 +28,7 @@ "brace-expansion": "^2.0.3", "fast-xml-parser": "^5.7.0", "ip-address": "^10.2.0", + "js-yaml": "^4.2.0", "uuid": "^11.1.1" } }