diff --git a/perf-benchmarking-for-releases/current_benchmark.log b/perf-benchmarking-for-releases/current_benchmark.log new file mode 100644 index 0000000..e69de29 diff --git a/perf-benchmarking-for-releases/data-prep/go.mod b/perf-benchmarking-for-releases/data-prep/go.mod new file mode 100644 index 0000000..328c25e --- /dev/null +++ b/perf-benchmarking-for-releases/data-prep/go.mod @@ -0,0 +1,55 @@ +module gcsfuse-tools/perf-benchmarking-for-releases/data-setup + +go 1.24.0 + +require cloud.google.com/go/storage v1.56.1 + +require ( + cel.dev/expr v0.24.0 // indirect + cloud.google.com/go v0.121.6 // indirect + cloud.google.com/go/auth v0.16.5 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.8.0 // indirect + cloud.google.com/go/iam v1.5.2 // indirect + cloud.google.com/go/monitoring v1.24.2 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.0.5 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/googleapis/gax-go/v2 v2.15.0 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect + github.com/zeebo/errs v1.4.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.36.0 // indirect + go.opentelemetry.io/otel/metric v1.36.0 // indirect + go.opentelemetry.io/otel/sdk v1.36.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect + go.opentelemetry.io/otel/trace v1.36.0 // indirect + golang.org/x/crypto v0.41.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + golang.org/x/time v0.12.0 // indirect + google.golang.org/api v0.248.0 // indirect + google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect + google.golang.org/grpc v1.74.2 // indirect + google.golang.org/protobuf v1.36.7 // indirect +) diff --git a/perf-benchmarking-for-releases/data-prep/go.sum b/perf-benchmarking-for-releases/data-prep/go.sum new file mode 100644 index 0000000..37c9292 --- /dev/null +++ b/perf-benchmarking-for-releases/data-prep/go.sum @@ -0,0 +1,125 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= +cloud.google.com/go v0.121.6/go.mod h1:coChdst4Ea5vUpiALcYKXEpR1S9ZgXbhEzzMcMR66vI= +cloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI= +cloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.8.0 h1:HxMRIbao8w17ZX6wBnjhcDkW6lTFpgcaobyVfZWqRLA= +cloud.google.com/go/compute/metadata v0.8.0/go.mod h1:sYOGTp851OV9bOFJ9CH7elVvyzopvWQFNNghtDQ/Biw= +cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= +cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= +cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= +cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= +cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= +cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= +cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= +cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= +cloud.google.com/go/storage v1.56.1 h1:n6gy+yLnHn0hTwBFzNn8zJ1kqWfR91wzdM8hjRF4wP0= +cloud.google.com/go/storage v1.56.1/go.mod h1:C9xuCZgFl3buo2HZU/1FncgvvOgTAs/rnh4gF4lMg0s= +cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= +cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= +github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= +github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= +go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= +go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= +go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= +go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= +go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= +go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= +go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= +go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +google.golang.org/api v0.248.0 h1:hUotakSkcwGdYUqzCRc5yGYsg4wXxpkKlW5ryVqvC1Y= +google.golang.org/api v0.248.0/go.mod h1:yAFUAF56Li7IuIQbTFoLwXTCI6XCFKueOlS7S9e4F9k= +google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4= +google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s= +google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c h1:AtEkQdl5b6zsybXcbz00j1LwNodDuH6hVifIaNqk7NQ= +google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c/go.mod h1:ea2MjsO70ssTfCjiwHgI0ZFqcw45Ksuk2ckf9G468GA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c h1:qXWI/sQtv5UKboZ/zUk7h+mrf/lXORyI+n9DKDAusdg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo= +google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= +google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= +google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= +google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/perf-benchmarking-for-releases/data-prep/main.go b/perf-benchmarking-for-releases/data-prep/main.go new file mode 100644 index 0000000..aaa7d53 --- /dev/null +++ b/perf-benchmarking-for-releases/data-prep/main.go @@ -0,0 +1,412 @@ +package main + +import ( + "bytes" + "context" + "errors" + "flag" + "fmt" + "io" + "log" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "cloud.google.com/go/storage" + "golang.org/x/sync/errgroup" + "google.golang.org/api/iterator" +) + +var ( + fileSize = flag.String("filesize", "1M", "Size of the file to create (e.g., 1K, 1M, 1G)") + numJobs = flag.Int("numjobs", 1, "Number of upload jobs per file group") + nrFile = flag.Int("nrfile", 1, "Number of file groups") + parallelism = flag.Int("parallelism", 80, "Number of parallel uploads") + region = flag.String("region", "", "Region") + projectID = flag.String("project", "", "Google Cloud project ID (required)") + bucket = flag.String("bucket", "", "bucket name") + opType = flag.String("op_type", "", "setup or delete") + benchType = flag.String("bench_type", "", "rand-read or seq-read") +) + +// parallelCopyObjects copies a list of objects from a source bucket to a destination bucket in parallel. +func parallelCopyObjects(projectID, copyBucket string, numJobs, nrfile, parallelization int) error { + ctx := context.Background() + + // 1. Create a single, shared GCS client. It's safe for concurrent use. + client, err := storage.NewClient(ctx) + if err != nil { + return fmt.Errorf("storage.NewClient: %w", err) + } + defer client.Close() + + totalJobs := numJobs * nrfile + if totalJobs == 0 { + log.Printf("No objects to copy.") + return nil + } + + // 2. Set up channels and a WaitGroup. + jobs := make(chan string, totalJobs) + errs := make(chan error, totalJobs) + var wg sync.WaitGroup + + // 3. Start the worker pool. + // We subtract 1 because the source object (0,0) is not copied. + log.Printf("šŸš€ Starting %d workers to copy %d objects...", parallelization, totalJobs-1) + for i := 0; i < parallelization; i++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + // Each worker pulls an object name from the jobs channel until it's closed. + for objectName := range jobs { + // The first object (0,0) is the source and already exists, so we skip copying it. + if objectName == *benchType+".0.0" { + continue + } + + src := client.Bucket(*bucket).Object(*benchType + ".0.0") + dst := client.Bucket(*bucket).Object(objectName) + + var lastErr error + for attempt := 0; attempt < 5; attempt++ { + copier := dst.CopierFrom(src) + copyCtx, cancel := context.WithTimeout(ctx, time.Second*120) + + if _, err := copier.Run(copyCtx); err != nil { + lastErr = err + cancel() // Always cancel on error + } else { + lastErr = nil + cancel() // Always cancel on success + break // Success, exit retry loop + } + } + + if lastErr != nil { + errs <- fmt.Errorf("worker %d failed to copy to %s after 5 attempts: %w", workerID, objectName, lastErr) + } + } + }(i + 1) + } + + // 4. Produce jobs. + for j := 0; j < numJobs; j++ { + for n := 0; n < nrfile; n++ { + jobs <- *benchType + "." + strconv.Itoa(j) + "." + strconv.Itoa(n) + } + } + close(jobs) // Close the channel to signal workers that no more jobs are coming. + + // 5. Wait and collect results. + wg.Wait() // Wait for all workers to finish. + close(errs) // Close the errors channel. + + // Check if any errors occurred. + var allErrors []string + for err := range errs { + allErrors = append(allErrors, err.Error()) + } + + if len(allErrors) > 0 { + return fmt.Errorf("finished with %d errors:\n- %s", len(allErrors), strings.Join(allErrors, "\n- ")) + } + log.Printf("\nšŸŽ‰ All objects copied successfully!\n") + return nil +} + +func deleteObjectsParallel(ctx context.Context, client *storage.Client, projectID, bucketName string, parallelism int) error { + // Ensure at least one worker to prevent deadlock if parallelism is 0 or negative. + if parallelism < 1 { + parallelism = 1 + } + + // Constants for retry logic + const maxAttempts = 4 // 1 initial attempt + 3 retries + const initialBackoff = 1 * time.Second // Starting backoff duration + + // Use a buffered channel for the job queue. + jobs := make(chan string, 100) + + // Use an errgroup to manage worker goroutines and capture the first error. + // gctx will be canceled if any goroutine in the group returns an error. + g, gctx := errgroup.WithContext(ctx) + bkt := client.Bucket(bucketName) + + // Start the worker pool (consumers) + for i := 0; i < parallelism; i++ { + g.Go(func() error { + // Each worker ranges over the jobs channel until it's closed. + for objectName := range jobs { + var err error + backoff := initialBackoff + + // --- Retry loop for Delete --- + for attempt := 0; attempt < maxAttempts; attempt++ { + // Before each attempt, check if the group context is canceled + select { + case <-gctx.Done(): + return gctx.Err() // Stop retrying if context is canceled + default: + // Continue + } + + obj := bkt.Object(objectName) + err = obj.Delete(gctx) + if err == nil { + break // Success, exit retry loop + } + + // Log the error (optional, but helpful for debugging) + // log.Printf("Attempt %d/%d: failed to delete %s: %v", attempt+1, maxAttempts, objectName, err) + + // If this was the last attempt, break out to return the error + if attempt == maxAttempts-1 { + break + } + + // Do not retry context cancellation errors; they are terminal. + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + + // Wait before the next retry, respecting context cancellation + select { + case <-time.After(backoff): + backoff *= 2 // Exponential backoff + case <-gctx.Done(): + return gctx.Err() // Canceled while waiting to retry + } + } + + // If err is still not nil after all attempts, return it to the errgroup + if err != nil { + return fmt.Errorf("failed to delete object %s after %d attempts: %w", objectName, maxAttempts, err) + } + // Otherwise, loop to the next objectName + } + return nil + }) + } + + // Start the lister (producer) in its own goroutine + g.Go(func() error { + defer close(jobs) // ALWAYS close jobs channel when producer exits + + it := bkt.Objects(gctx, nil) + for { + var attrs *storage.ObjectAttrs + var err error + backoff := initialBackoff + + // --- Retry loop for List (it.Next) --- + for attempt := 0; attempt < maxAttempts; attempt++ { + select { + case <-gctx.Done(): + return gctx.Err() + default: + } + + attrs, err = it.Next() + if err == nil { + break // Success + } + + // iterator.Done is not an error to retry; it's the signal we're finished. + if err == iterator.Done { + break // Exit retry loop; outer loop will handle 'Done' + } + + // Do not retry context cancellation errors + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + + // log.Printf("Attempt %d/%d: failed to list next object: %v", attempt+1, maxAttempts, err) + + if attempt == maxAttempts-1 { + break // Last attempt failed, break to return error + } + + // Wait with backoff + select { + case <-time.After(backoff): + backoff *= 2 + case <-gctx.Done(): + return gctx.Err() + } + } + + // --- Handle result of the retry loop --- + if err == iterator.Done { + break // Finished listing all objects, exit the producer's main loop + } + if err != nil { + // All attempts to list the next object failed. + return fmt.Errorf("failed to list objects after %d attempts: %w", maxAttempts, err) + } + + // If we are here, err is nil and attrs is valid. Send job to workers. + select { + case jobs <- attrs.Name: + case <-gctx.Done(): + return gctx.Err() + } + } + return nil + }) + + // Wait for all goroutines (workers and lister) to finish. + // Returns the first non-nil error from any of them. + return g.Wait() +} + +// createBucket creates a GCS bucket if it does not already exist. +func createBucket(ctx context.Context, client *storage.Client, projectID, bucketName, region string) error { + log.Printf("Attempting to create bucket gs://%s in region %s", bucketName, region) + bucket := client.Bucket(bucketName) + attrs := &storage.BucketAttrs{ + Location: region, + } + if err := bucket.Create(ctx, projectID, attrs); err != nil { + return fmt.Errorf("failed to create bucket %q: %w", bucketName, err) + } + log.Printf("Bucket gs://%s created in region %s.", bucketName, region) + return nil +} + +// deleteBucket deletes a GCS bucket if it exists. +func deleteBucket(ctx context.Context, client *storage.Client, bucketName string) error { + log.Printf("Attempting to delete bucket gs://%s...", bucketName) + bucket := client.Bucket(bucketName) + if err := bucket.Delete(ctx); err != nil { + return fmt.Errorf("failed to delete bucket %q: %w", bucketName, err) + } + return nil +} + +// parseSize converts a string like "10M" or "1G" into a number of bytes. +func parseSize(sizeStr string) (int64, error) { + re := regexp.MustCompile(`^(\d+)([KMG])`) + matches := re.FindStringSubmatch(strings.ToUpper(sizeStr)) + if len(matches) != 3 { + return 0, fmt.Errorf("invalid size format: %q. Use format like 1K, 2M, 3G", sizeStr) + } + + size, err := strconv.ParseInt(matches[1], 10, 64) + if err != nil { + return 0, err + } + + unit := matches[2] + switch unit { + case "K": + return size * 1024, nil + case "M": + return size * 1024 * 1024, nil + case "G": + return size * 1024 * 1024 * 1024, nil + default: + return 0, fmt.Errorf("unknown unit: %s", unit) + } +} + +// createObject creates a GCS object of a given name and size by writing a buffer repeatedly. +/* + * createObject creates an object of a specific size with zero data. + * + * It allocates a 32MB in-memory buffer and writes it repeatedly to the GCS + * writer until the target size is reached. This is a memory-efficient way + * to generate large files for benchmarking. + */ +func createObject(ctx context.Context, client *storage.Client, bucketName, objectName string, size int64) (err error) { + log.Printf("Attempting to create object gs://%s/%s of size %d bytes...", bucketName, objectName, size) + wc := client.Bucket(bucketName).Object(objectName).NewWriter(ctx) + + wc.ChunkSize = int(size) + + defer func() { + if closeErr := wc.Close(); closeErr != nil { + if err == nil { + err = fmt.Errorf("failed to close GCS writer: %w", closeErr) + } + } else if err == nil { + log.Printf("Successfully created object gs://%s/%s", bucketName, objectName) + } + }() + + // Create a 32MB buffer. The slice is zero-filled by default. + const bufferSize = 32 * 1024 * 1024 + buffer := make([]byte, bufferSize) + + var written int64 + for written < size { + bytesToWrite := int64(len(buffer)) + if size-written < bytesToWrite { + bytesToWrite = size - written + } + + n, writeErr := io.Copy(wc, bytes.NewReader(buffer[:bytesToWrite])) + if writeErr != nil { + err = fmt.Errorf("failed during Write to GCS after %d bytes: %w", written, writeErr) + return err + } + written += int64(n) + log.Printf("Written %d", written) + } + + return nil +} + +func main() { + startTime := time.Now() + defer func() { + fmt.Printf("Total time elapsed in secs: %v", time.Since(startTime).Seconds()) + }() + flag.Parse() + if *projectID == "" || *bucket == "" || *opType == "" || *region == "" || *benchType == "" { + log.Fatal("Error: --project, --bucket, --op_type, --bench_type, and --region are required.") + } + if *fileSize == "" || *numJobs <= 0 || *nrFile <= 0 || *parallelism <= 0 { + log.Fatal("Error: filesize, num_jobs, num_files, and parallelism must be positive.") + } + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + log.Fatalf("Failed to create storage client: %v", err) + } + defer client.Close() + + if *opType == "setup" { + if err := createBucket(ctx, client, *projectID, *bucket, *region); err != nil { + log.Fatalf("Failed to create bucket: %v", err) + } + if *benchType == "rand-read" || *benchType == "seq-read" { + size, err := parseSize(*fileSize) + if err != nil { + log.Fatalf("Failed to parse file size: %v", err) + } + err = createObject(ctx, client, *bucket, *benchType+".0.0", size) + if err != nil { + log.Fatalf("Failed to create object: %v", err) + } + err = parallelCopyObjects(*projectID, *bucket, *numJobs, *nrFile, *parallelism) + if err != nil { + log.Fatalf("Failed to copy objects: %v", err) + } + } + + } else if *opType == "delete" { + if err := deleteObjectsParallel(ctx, client, *projectID, *bucket, *parallelism); err != nil { + log.Fatalf("Failed to delete objects: %v", err) + } + if err := deleteBucket(ctx, client, *bucket); err != nil { + log.Fatalf("Failed to delete bucket: %v", err) + } + log.Printf("Bucket gs://%s deleted successfully.", *bucket) + } else { + log.Fatal("Error: --op_type must be either 'setup' or 'delete'.") + } +} diff --git a/perf-benchmarking-for-releases/fio-job-files/rand-read-workload.fio b/perf-benchmarking-for-releases/fio-job-files/rand-read-workload.fio new file mode 100644 index 0000000..c3387d4 --- /dev/null +++ b/perf-benchmarking-for-releases/fio-job-files/rand-read-workload.fio @@ -0,0 +1,21 @@ +[global] +ioengine=libaio +direct=1 +fadvise_hint=0 +iodepth=64 +invalidate=1 +thread=1 +openfiles=1 +group_reporting=1 +create_serialize=0 +allrandrepeat=0 +file_service_type=random +rw=randread +filename_format=$jobname.$jobnum.$filenum + +[rand-read] +directory=${DIR} +bs=${BS} +filesize=${FILESIZE} +numjobs=${NUM_JOB} +nrfiles=${NRFILES} diff --git a/perf-benchmarking-for-releases/fio-job-files/random-read-workload.fio b/perf-benchmarking-for-releases/fio-job-files/random-read-workload.fio deleted file mode 100644 index 97fe559..0000000 --- a/perf-benchmarking-for-releases/fio-job-files/random-read-workload.fio +++ /dev/null @@ -1,78 +0,0 @@ -[global] -ioengine=libaio -direct=1 -fadvise_hint=0 -iodepth=64 -invalidate=1 -thread=1 -openfiles=1 -group_reporting=1 -create_serialize=0 -allrandrepeat=0 -file_service_type=random -numjobs=128 -rw=randread -filename_format=$jobname.$jobnum/$filenum - -[read_filesize_128K_blocksize_128K] -stonewall -directory=${DIR} -bs=128K -filesize=128K -nrfiles=30 - -[read_filesize_256K_blocksize_128K] -stonewall -directory=${DIR} -bs=128K -filesize=256K -nrfiles=30 - -[read_filesize_1M_blocksize_1M] -stonewall -directory=${DIR} -bs=1M -filesize=1MB -nrfiles=30 - -[read_filesize_5M_blocksize_1M] -stonewall -directory=${DIR} -bs=1M -filesize=5MB -nrfiles=20 - -[read_filesize_10M_blocksize_1M] -stonewall -directory=${DIR} -bs=1M -filesize=10MB -nrfiles=20 - -[read_filesize_50M_blocksize_1M] -stonewall -directory=${DIR} -bs=1M -filesize=50MB -nrfiles=20 - -[read_filesize_100M_blocksize_1M] -stonewall -directory=${DIR} -bs=1M -filesize=100MB -nrfiles=10 - -[read_filesize_200M_blocksize_1M] -stonewall -directory=${DIR} -bs=1M -filesize=200MB -nrfiles=10 - -[read_filesize_1G_blocksize_1M] -stonewall -directory=${DIR} -bs=1M -filesize=1GB -nrfiles=2 diff --git a/perf-benchmarking-for-releases/fio-job-files/seq-read-workload.fio b/perf-benchmarking-for-releases/fio-job-files/seq-read-workload.fio new file mode 100644 index 0000000..c33418f --- /dev/null +++ b/perf-benchmarking-for-releases/fio-job-files/seq-read-workload.fio @@ -0,0 +1,21 @@ +[global] +ioengine=libaio +direct=1 +fadvise_hint=0 +iodepth=64 +invalidate=1 +thread=1 +openfiles=1 +group_reporting=1 +create_serialize=0 +allrandrepeat=0 +file_service_type=random +rw=read +filename_format=$jobname.$jobnum.$filenum + +[seq-read] +directory=${DIR} +bs=${BS} +filesize=${FILESIZE} +numjobs=${NUM_JOB} +nrfiles=${NRFILES} diff --git a/perf-benchmarking-for-releases/fio-job-files/seq-write-workload.fio b/perf-benchmarking-for-releases/fio-job-files/seq-write-workload.fio new file mode 100644 index 0000000..6c944a1 --- /dev/null +++ b/perf-benchmarking-for-releases/fio-job-files/seq-write-workload.fio @@ -0,0 +1,22 @@ +[global] +ioengine=sync +direct=1 +fadvise_hint=0 +verify=0 +invalidate=1 +file_append=0 +create_on_open=1 +end_fsync=1 +thread=1 +openfiles=1 +group_reporting=1 +allrandrepeat=1 +filename_format=$jobname.$jobnum.$filenum +rw=write + +[seq-write] +directory=${DIR} +numjobs=${NUM_JOB} +filesize=${FILESIZE} +nrfiles=${NRFILES} +bs=${BS} diff --git a/perf-benchmarking-for-releases/fio-job-files/sequential-read-workload.fio b/perf-benchmarking-for-releases/fio-job-files/sequential-read-workload.fio deleted file mode 100644 index 2baa2eb..0000000 --- a/perf-benchmarking-for-releases/fio-job-files/sequential-read-workload.fio +++ /dev/null @@ -1,86 +0,0 @@ -[global] -ioengine=libaio -direct=1 -fadvise_hint=0 -iodepth=64 -invalidate=1 -thread=1 -openfiles=1 -group_reporting=1 -create_serialize=0 -allrandrepeat=0 -file_service_type=random -numjobs=128 -filename_format=$jobname.$jobnum/$filenum - -[read_filesize_128K_blocksize_128K] -stonewall -directory=${DIR} -bs=128K -filesize=128K -rw=read -nrfiles=30 - -[read_filesize_256K_blocksize_128K] -stonewall -directory=${DIR} -bs=128K -filesize=256K -rw=read -nrfiles=30 - -[read_filesize_1M_blocksize_1M] -stonewall -directory=${DIR} -bs=1M -filesize=1MB -rw=read -nrfiles=30 - -[read_filesize_5M_blocksize_1M] -stonewall -directory=${DIR} -bs=1M -filesize=5MB -rw=read -nrfiles=20 - -[read_filesize_10M_blocksize_1M] -stonewall -directory=${DIR} -bs=1M -filesize=10MB -rw=read -nrfiles=20 - -[read_filesize_50M_blocksize_1M] -stonewall -directory=${DIR} -bs=1M -filesize=50MB -rw=read -nrfiles=20 - -[read_filesize_100M_blocksize_1M] -stonewall -directory=${DIR} -bs=1M -filesize=100MB -rw=read -nrfiles=10 - -[read_filesize_200M_blocksize_1M] -stonewall -directory=${DIR} -bs=1M -filesize=200MB -rw=read -nrfiles=10 - -[read_filesize_1G_blocksize_1M] -stonewall -directory=${DIR} -bs=1M -filesize=1GB -rw=read -nrfiles=2 diff --git a/perf-benchmarking-for-releases/fio-job-files/write-workload.fio b/perf-benchmarking-for-releases/fio-job-files/write-workload.fio deleted file mode 100644 index fbb2a7f..0000000 --- a/perf-benchmarking-for-releases/fio-job-files/write-workload.fio +++ /dev/null @@ -1,56 +0,0 @@ -[global] -ioengine=sync -direct=1 -fadvise_hint=0 -verify=0 -iodepth=64 -invalidate=1 -time_based=0 -file_append=0 -create_on_open=1 -thread=1 -openfiles=1 -group_reporting=1 -allrandrepeat=1 -nrfiles=2 -filename_format=$jobname.$jobnum.$filenum -rw=write -bs=1M -filesize=1G - -[write_256kb_seq] -stonewall -directory=${DIR} -numjobs=112 -nrfiles=30 -filesize=256K -bs=16K - - -[write_1mb_seq] -stonewall -directory=${DIR} -numjobs=112 -nrfiles=30 -filesize=1M - -[write_50mb_seq] -stonewall -directory=${DIR} -numjobs=112 -nrfiles=20 -filesize=50M - -[write_100mb_seq] -stonewall -directory=${DIR} -numjobs=112 -nrfiles=10 -filesize=100M - -[write_1gb_seq] -stonewall -directory=${DIR} -numjobs=112 -nrfiles=2 -filesize=1G diff --git a/perf-benchmarking-for-releases/nohup.out b/perf-benchmarking-for-releases/nohup.out new file mode 100644 index 0000000..161d891 --- /dev/null +++ b/perf-benchmarking-for-releases/nohup.out @@ -0,0 +1,2 @@ + +Running experiment for machine type: c4-standard-96 for BENCH seq-read with filesize: 10M, blocksize: 1M, numjobs: 128, nrfile: 20 diff --git a/perf-benchmarking-for-releases/results.txt b/perf-benchmarking-for-releases/results.txt new file mode 100644 index 0000000..e69de29 diff --git a/perf-benchmarking-for-releases/run-benchmarks.sh b/perf-benchmarking-for-releases/run-benchmarks.sh index 09dd0de..4368edd 100755 --- a/perf-benchmarking-for-releases/run-benchmarks.sh +++ b/perf-benchmarking-for-releases/run-benchmarks.sh @@ -14,24 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Fail on anything unexpected and print commands as they are executed. +set -xeuo pipefail + # Validate input arguments -if [ "$#" -ne 6 ]; then - echo "Usage: $0 " +if [ "$#" -ne 11 ]; then + echo "Usage: $0 " echo "" echo " can be a Git tag (e.g. v1.0.0), branch name (e.g. main), or a commit ID on master." echo "" echo "This script should be run from the 'perf-benchmarking-for-releases' directory." echo "" echo "Example:" - echo " bash run-benchmarks.sh v2.12.0 gcs-fuse-test us-south1 n2-standard-96 ubuntu-2004-lts ubuntu-os-cloud" + echo " bash run-benchmarks.sh v2.12.0 gcs-fuse-test us-south1 n2-standard-96 ubuntu-2204-lts ubuntu-os-cloud 3" exit 1 fi -# Print commands and their arguments as they are executed. -set -x -# Exit immediately if a command exits with a non-zero status. -set -e - echo "!!! Ensure your account has the following permissions:" echo "Read access to: gs://gcsfuse-release-benchmark-fio-data" echo "Read/Write access to: gs://gcsfuse-release-benchmarks-results" @@ -42,6 +40,12 @@ REGION=$3 MACHINE_TYPE=$4 IMAGE_FAMILY=$5 IMAGE_PROJECT=$6 +NUM_JOB=$7 +NRFILES=$8 +BS=$9 +FILESIZE=${10} +BENCH_TYPE=${11} + # Generate unique names for VM and buckets using timestamp and random number TIMESTAMP=$(date +%Y%m%d-%H%M%S) @@ -65,47 +69,12 @@ echo "Region: ${REGION}" echo "VM Zone: ${VM_ZONE}" echo "Machine Type: ${MACHINE_TYPE}" -# Array for LSSD supported machines -# Add machine types that support local SSDs (NVMe) here -LSSD_SUPPORTED_MACHINES=("n2-standard-96" "c2-standard-60" "c2d-standard-112" "c3-standard-88" "c3d-standard-180") - -# Check if the chosen machine type is directly present in the LSSD_SUPPORTED_MACHINES array -VM_LOCAL_SSD_ARGS="" -LSSD_ENABLED="false" - -if [[ " ${LSSD_SUPPORTED_MACHINES[@]} " =~ " ${MACHINE_TYPE} " ]]; then - echo "Machine type ${MACHINE_TYPE} supports LSSDs. Attaching 16 local NVMe SSDs (375GB each)." - LSSD_ENABLED="true" - # Construct the --local-ssd flags for 16 local SSDs - for i in {0..15}; do - VM_LOCAL_SSD_ARGS+=" --local-ssd=interface=NVME,size=375GB" - done -else - echo "Machine type ${MACHINE_TYPE} does not support LSSDs based on the configured list, or it's not set up for LSSD benchmarking." - echo "VM will be created without local SSDs." -fi - - # Cleanup function to be called on exit cleanup() { echo "Initiating cleanup..." - # Delete VM if it exists - if gcloud compute instances describe "${VM_NAME}" --zone="${VM_ZONE}" --project="${PROJECT_ID}" >/dev/null 2>&1; then - echo "Deleting VM: ${VM_NAME}" - gcloud compute instances delete "${VM_NAME}" --zone="${VM_ZONE}" --project="${PROJECT_ID}" --delete-disks=all -q >/dev/null - else - echo "VM '${VM_NAME}' not found; skipping deletion." - fi - - # Delete GCS bucket with test data if it exists - if gcloud storage buckets list --project="${PROJECT_ID}" --filter="name:(${GCS_BUCKET_WITH_FIO_TEST_DATA})" --format="value(name)" | grep -q "^${GCS_BUCKET_WITH_FIO_TEST_DATA}$"; then - echo "Deleting GCS bucket: ${GCS_BUCKET_WITH_FIO_TEST_DATA}" - gcloud storage rm -r "gs://${GCS_BUCKET_WITH_FIO_TEST_DATA}" -q >/dev/null - else - echo "Bucket '${GCS_BUCKET_WITH_FIO_TEST_DATA}' not found; skipping deletion." - fi - + echo "Deleting VM: ${VM_NAME}" + gcloud compute instances delete "${VM_NAME}" --zone="${VM_ZONE}" --project="${PROJECT_ID}" --delete-disks=all -q >/dev/null || true echo "Cleanup complete." } @@ -113,45 +82,23 @@ cleanup() { # Register the cleanup function to run on EXIT signal trap cleanup EXIT -# Create the GCS bucket for FIO test data in the specified REGION echo "Creating GCS test data bucket: gs://${GCS_BUCKET_WITH_FIO_TEST_DATA} in region: ${REGION}" -gcloud storage buckets create "gs://${GCS_BUCKET_WITH_FIO_TEST_DATA}" --project="${PROJECT_ID}" --location="${REGION}" -# Clear the existing GCSFUSE_VERSION directory in the results bucket -echo "Clearing previous data in gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}/..." -gcloud storage rm -r "gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}/**" --quiet || true +# Clear the existing GCSFUSE_VERSION directory in the results bucket for the machine-type +echo "Clearing previous data in gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}/${MACHINE_TYPE}..." +gcloud storage rm -r "gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}/${MACHINE_TYPE}/**" --quiet > /dev/null 2>&1 || true -# Upload FIO job files to the results bucket for the VM to download -echo "Uploading all .fio job files from local 'fio-job-files/' directory to gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}/fio-job-files/..." -gcloud storage cp fio-job-files/*.fio "gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}/fio-job-files/" +# Upload FIO job files to the results bucket for the VM to download" +echo "Uploading all .fio job files from local 'fio-job-files/' directory to gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}/${MACHINE_TYPE}/fio-job-files/..." +gcloud storage cp fio-job-files/*.fio "gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}/${MACHINE_TYPE}/fio-job-files/" +gcloud storage cp starter-script.sh "gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}/${MACHINE_TYPE}/starter-script.sh" echo "FIO job files uploaded." -# Get the project number -PROJECT_NUMBER=$(gcloud projects describe "$PROJECT_ID" --format="value(projectNumber)") - -# Construct the Storage Transfer Service account email -STS_ACCOUNT="project-${PROJECT_NUMBER}@storage-transfer-service.iam.gserviceaccount.com" - -# Grant the service account 'roles/storage.admin' permissions on the newly created bucket -# This allows the service account to manage the bucket and perform transfers -gcloud storage buckets add-iam-policy-binding "gs://${GCS_BUCKET_WITH_FIO_TEST_DATA}" \ - --member="serviceAccount:${STS_ACCOUNT}" \ - --role="roles/storage.admin" - -# Since file generation with fio is painfully slow, we will use storage transfer -# job to transfer test data from a fixed GCS bucket to the newly created bucket. -# Note : We need to copy only read data. -echo "Creating storage transfer job to copy read data to gs://${GCS_BUCKET_WITH_FIO_TEST_DATA}..." - -TRANSFER_JOB_NAME=$(gcloud transfer jobs create \ - gs://gcsfuse-release-benchmark-fio-data \ - gs://${GCS_BUCKET_WITH_FIO_TEST_DATA} \ - --include-prefixes=read \ - --project="${PROJECT_ID}" \ - --format="value(name)" \ - --no-async) - -echo "Transfer completed." +cd data-prep + SECONDS=0 +go run main.go --project="$PROJECT_ID" --region="$REGION" --parallelism=80 -nrfile="$NRFILES" --numjobs="$NUM_JOB" --filesize="$FILESIZE" --bucket="$GCS_BUCKET_WITH_FIO_TEST_DATA" --op_type="setup" --bench_type="$BENCH_TYPE" +echo "data-prep took $SECONDS seconds" +cd .. # Create the VM based on the config passed by user @@ -166,44 +113,32 @@ gcloud compute instances create "${VM_NAME}" \ --network-interface=network-tier=PREMIUM,nic-type=GVNIC \ --scopes=https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/devstorage.read_write \ --network-performance-configs=total-egress-bandwidth-tier=TIER_1 \ - --metadata GCSFUSE_VERSION="${GCSFUSE_VERSION}",GCS_BUCKET_WITH_FIO_TEST_DATA="${GCS_BUCKET_WITH_FIO_TEST_DATA}",RESULTS_BUCKET_NAME="${RESULTS_BUCKET_NAME}",LSSD_ENABLED="${LSSD_ENABLED}" \ - --metadata-from-file=startup-script=starter-script.sh \ - ${VM_LOCAL_SSD_ARGS} -echo "VM created. Benchmarks will run on the VM." - -echo "Waiting for benchmarks to complete on VM (polling for success.txt)..." - -SUCCESS_FILE_PATH="gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}/success.txt" -LOG_FILE_PATH="gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}/benchmark_run.log" -SLEEP_TIME=300 # 5 minutes -sleep "$SLEEP_TIME" -#max 18 retries amounting to ~1hr30mins time -MAX_RETRIES=18 - -for ((i=1; i<=MAX_RETRIES; i++)); do - if gcloud storage objects describe "${SUCCESS_FILE_PATH}" &> /dev/null; then - echo "Benchmarks completed. success.txt found." - echo "Results are available in gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}/" - echo "Benchmark log file: $LOG_FILE_PATH" - exit 0 - fi - - # Check for early failure indicators - if gcloud storage objects describe "gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}/details.txt" &> /dev/null || \ - gcloud storage objects describe "$LOG_FILE_PATH" &> /dev/null; then - echo "Benchmark log or details.txt found, but success.txt is missing. Possible error in benchmark execution." - echo "Check logs at: $LOG_FILE_PATH" - exit 1 - fi - - echo "Attempt $i/$MAX_RETRIES: success.txt not found. Sleeping for $((SLEEP_TIME / 60)) minutes..." - sleep "$SLEEP_TIME" -done - - -# Failure case: success.txt was not found after retries -echo "Timed out waiting for success.txt after $((MAX_RETRIES * SLEEP_TIME / 60)) minutes. Perhaps there is some error." -echo "Benchmark log file (for troubleshooting): $LOG_FILE_PATH" -exit 1 - -# The trap command will handle the cleanup on script exit. + --metadata BENCH_TYPE="$BENCH_TYPE",NUM_JOB="${NUM_JOB}",NRFILES="${NRFILES}",BS="${BS}",FILESIZE="${FILESIZE}",GCSFUSE_VERSION="${GCSFUSE_VERSION}",MACHINE_TYPE="${MACHINE_TYPE}",GCS_BUCKET_WITH_FIO_TEST_DATA="${GCS_BUCKET_WITH_FIO_TEST_DATA}",RESULTS_BUCKET_NAME="${RESULTS_BUCKET_NAME}" \ + +gcloud compute os-login ssh-keys add --key="$(ssh-add -L | grep publickey)" --project="$PROJECT_ID" + +MAX_RETRIES=30 +SLEEP_TIME=3 + +run_command_on_vm() { + for ((i=1; i<=MAX_RETRIES; i++)); do + sleep "$SLEEP_TIME" + if ! gcloud compute ssh "$VM_NAME" "--zone=${VM_ZONE}" "--project=${PROJECT_ID}" "--command=$1" -- -o "Hostname=nic0.${VM_NAME}.${VM_ZONE}.c.${PROJECT_ID}.internal.gcpnode.com"; then + echo "Retrying..." + else + return 0 + fi + done + echo "Failed after $((MAX_RETRIES * SLEEP_TIME)) seconds after $((MAX_RETRIES)) retries." + exit 1 +} + +gcloud storage cp starter-script.sh "gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}/${MACHINE_TYPE}/starter-script.sh" +run_command_on_vm "hostname" > /dev/null 2>&1 +run_command_on_vm "gcloud storage cp gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}/${MACHINE_TYPE}/starter-script.sh ~/" +SECONDS=0 +run_command_on_vm "bash ~/starter-script.sh" +echo "starter script took $SECONDS seconds" + + + diff --git a/perf-benchmarking-for-releases/runner.sh b/perf-benchmarking-for-releases/runner.sh new file mode 100755 index 0000000..de7c9b7 --- /dev/null +++ b/perf-benchmarking-for-releases/runner.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fail on anything unexpected and print commands as they are executed. +set -euo pipefail + +echo "" > nohup.out + +MACHINE_TYPE=("c4-standard-96") +GCSFUSE_VERSION="v3.3.0" +BENCH_TYPE="seq-read" # seq-write, rand-read, seq-read +project="gcs-fuse-test" +region="us-central1" +image_family="ubuntu-2204-lts" +image_project="ubuntu-os-cloud" +FILE_SIZE=("10M") +BS=("1M") +NUM_JOBS=("128") +NRFILE=("20") +for m in "${MACHINE_TYPE[@]}"; do +for f in "${FILE_SIZE[@]}"; do + for s in "${BS[@]}"; do + for t in "${NUM_JOBS[@]}"; do + for j in "${NRFILE[@]}"; do + echo "Running experiment for machine type: $m for BENCH $BENCH_TYPE with filesize: $f, blocksize: $s, numjobs: $t, nrfile: $j" + echo "" > current_benchmark.log + ./run-benchmarks.sh "$GCSFUSE_VERSION" "$project" "$region" "$m" "$image_family" "$image_project" "$t" "$j" "$s" "$f" "$BENCH_TYPE"> current_benchmark.log 2>&1 + RESULT_PATH="gs://gcsfuse-release-benchmarks-results/$GCSFUSE_VERSION/results/$m/$BENCH_TYPE/$f/$s" + RESULT_FILE="${BENCH_TYPE}-benchmark_res_${t}_${j}.json" + TMP_FILE=$(mktemp /tmp/${BENCH_TYPE}-bench-res.XXXXXX) + gcloud storage cp "$RESULT_PATH/$RESULT_FILE" "$TMP_FILE" + echo "Res file: $TMP_FILE" + BW_IN_MBPS=$(jq '(.jobs[].read.bw_bytes + .jobs[].write.bw_bytes) / (1000 * 1000)' "$TMP_FILE") + echo "$m $region $BENCH_TYPE FileSize=$f BlockSize=$s NumJobs=$t NrFile=$j MBPS=$BW_IN_MBPS" >> results.txt + echo "Completed Successfully..." + done + done + done + done +done diff --git a/perf-benchmarking-for-releases/starter-script.sh b/perf-benchmarking-for-releases/starter-script.sh index 2b1ecb5..f556e8d 100755 --- a/perf-benchmarking-for-releases/starter-script.sh +++ b/perf-benchmarking-for-releases/starter-script.sh @@ -31,14 +31,20 @@ fi # Install common dependencies before adding starterscriptuser if [[ "$OS_FAMILY" == "debian_ubuntu" ]]; then - sudo apt-get update - sudo apt-get install -y git fio libaio1 libaio-dev gcc make mdadm build-essential python3-setuptools python3-crcmod fuse + sudo apt-get update > /dev/null 2>&1 + sudo apt-get install -y git libaio1 libaio-dev gcc make mdadm build-essential python3-setuptools python3-crcmod fuse > /dev/null 2>&1 elif [[ "$OS_FAMILY" == "rhel_centos" ]]; then - sudo yum makecache - sudo yum -y install git fio fuse libaio libaio-devel gcc make mdadm redhat-rpm-config python3-devel python3-setuptools python3-pip - pip3 install crcmod + sudo yum makecache > /dev/null 2>&1 + sudo yum -y install git fuse libaio libaio-devel gcc make mdadm redhat-rpm-config python3-devel python3-setuptools python3-pip > /dev/null 2>&1 + pip3 install crcmod > /dev/null 2>&1 fi +# Install fio-3.39 +git clone -b fio-3.39 https://github.com/axboe/fio.git +cd fio +./configure && sudo make && sudo make install +cd .. + # Add starterscriptuser based on OS type if ! id "starterscriptuser" &>/dev/null; then if [[ "$OS_FAMILY" == "debian_ubuntu" ]]; then @@ -62,14 +68,6 @@ cleanup() { if mount | grep -q "$MNT"; then sudo umount "$MNT" || echo "Failed to unmount $MNT" fi - - # Upload logs and details - if [[ -f details.txt ]]; then - gcloud storage cp details.txt "$RESULT_PATH" || echo "Failed to upload details.txt" - fi - if [[ -f "$BENCHMARK_LOG_FILE" ]]; then - gcloud storage cp "$BENCHMARK_LOG_FILE" "$RESULT_PATH" || echo "Failed to upload benchmark log" - fi } trap cleanup EXIT @@ -80,13 +78,15 @@ echo "Current directory: $(pwd)" echo "User: $(whoami)" # Fetch metadata parameters +BS=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/BS" -H "Metadata-Flavor: Google") +FILESIZE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/FILESIZE" -H "Metadata-Flavor: Google") +NRFILES=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/NRFILES" -H "Metadata-Flavor: Google") +NUM_JOB=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/NUM_JOB" -H "Metadata-Flavor: Google") +BENCH_TYPE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/BENCH_TYPE" -H "Metadata-Flavor: Google") GCSFUSE_VERSION=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/GCSFUSE_VERSION" -H "Metadata-Flavor: Google") +MACHINE_TYPE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/MACHINE_TYPE" -H "Metadata-Flavor: Google") GCS_BUCKET_WITH_FIO_TEST_DATA=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/GCS_BUCKET_WITH_FIO_TEST_DATA" -H "Metadata-Flavor: Google") RESULTS_BUCKET_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/RESULTS_BUCKET_NAME" -H "Metadata-Flavor: Google") -LSSD_ENABLED=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/LSSD_ENABLED" -H "Metadata-Flavor: Google") - -# Add OS Family to details.txt -echo "OS Family: $OS_FAMILY" >> details.txt # Determine system architecture ARCHITECTURE="" @@ -125,65 +125,24 @@ FIO_JOB_DIR="/tmp/fio_jobs" # Download all FIO job spec files mkdir -p "$FIO_JOB_DIR" -gcloud storage cp "gs://$RESULTS_BUCKET_NAME/$GCSFUSE_VERSION/fio-job-files/*.fio" "$FIO_JOB_DIR/" - -RESULT_PATH="gs://$RESULTS_BUCKET_NAME/$GCSFUSE_VERSION/" - -# Capture versions -{ - echo "GCSFuse version: $GCSFUSE_VERSION" - echo "Go version : $(go version)" - echo "FIO version : $(fio --version)" -} >> details.txt - -# Create LSSD if enabled -if [[ "$LSSD_ENABLED" == "true" ]]; then - LSSD_DEVICES=() - for i in {0..15}; do - DEVICE_PATH="/dev/disk/by-id/google-local-nvme-ssd-$i" - LSSD_DEVICES+=("$DEVICE_PATH") - done - - sudo mdadm --create /dev/md0 --level=0 --raid-devices=16 "${LSSD_DEVICES[@]}" - sudo mdadm --detail --prefer=by-id /dev/md0 || true - sudo mkfs.ext4 -F /dev/md0 - sudo mkdir -p "$SSD_MOUNT_DIR" - sudo mount /dev/md0 "$SSD_MOUNT_DIR" - sudo chmod a+w "$SSD_MOUNT_DIR" - - reformat_and_remount_lssd() { - sudo umount /dev/md0 || echo "Warning: umount /dev/md0 failed (might not be mounted or busy)." - sudo mkfs.ext4 -F /dev/md0 - sudo mount /dev/md0 "$SSD_MOUNT_DIR" - sudo chmod a+w "$SSD_MOUNT_DIR" - } -fi +gcloud storage cp "gs://$RESULTS_BUCKET_NAME/$GCSFUSE_VERSION/$MACHINE_TYPE/fio-job-files/*.fio" "$FIO_JOB_DIR/" -# Mount GCS bucket using gcsfuse + +# Mount dir for gcsfuse mkdir -p "$MNT" +echo "Mounting GCSFuse now..." "$GCSFUSE_BIN" --implicit-dirs "$GCS_BUCKET_WITH_FIO_TEST_DATA" "$MNT" -for fio_job_file in "$FIO_JOB_DIR"/*.fio; do - job_name=$(basename "$fio_job_file" .fio) # e.g., random-read-workload - - [[ "$LSSD_ENABLED" == "true" ]] && reformat_and_remount_lssd - - # Drop Page Cache - sudo sh -c "echo 3 > /proc/sys/vm/drop_caches" +RESULT_FILE="${BENCH_TYPE}-benchmark_res_${NUM_JOB}_${NRFILES}.json" +DIR="$MNT" NUM_JOB="$NUM_JOB" BS="$BS" FILESIZE="$FILESIZE" NRFILES="$NRFILES" fio "${FIO_JOB_DIR}/${BENCH_TYPE}-workload.fio" --output-format=json --output="$RESULT_FILE" +RESULT_PATH="gs://$RESULTS_BUCKET_NAME/$GCSFUSE_VERSION/results/$MACHINE_TYPE/$BENCH_TYPE/$FILESIZE/$BS" +gcloud storage rm "$RESULT_PATH/$RESULT_FILE" || true +gcloud storage cp "$RESULT_FILE" "${RESULT_PATH}/" - RESULT_FILE="gcsfuse-${job_name}-benchmark-$(date +%Y%m%d%H%M%S).json" - - DIR="$MNT" fio "$fio_job_file" --output-format=json --output="$RESULT_FILE" - - gcloud storage cp "$RESULT_FILE" "$RESULT_PATH" - - rm -f "$RESULT_FILE" # Clean up local files -done - -# All tests ran successfully; create a success.txt file in GCS -touch success.txt -gcloud storage cp success.txt "$RESULT_PATH" -rm success.txt +rm -f "$RESULT_FILE" # Clean up local files +# Unmount GCSFuse +echo "Unmounting GCSFuse" +sudo umount "$MNT" EOF