diff --git a/perfkitbenchmarker/archive.py b/perfkitbenchmarker/archive.py index cf4107c3e0..1882eb8c9f 100644 --- a/perfkitbenchmarker/archive.py +++ b/perfkitbenchmarker/archive.py @@ -28,7 +28,7 @@ def ArchiveRun( run_temp_directory, target_bucket, prefix='', - gsutil_path='gsutil', + gsutil_path='gcloud', aws_path=AWS_PATH, ): """Archive a run directory to GCS or S3. @@ -52,7 +52,7 @@ def ArchiveRun( ) prefix_len = 5 - prefixes = {'s3://': [aws_path, 's3', 'cp'], 'gs://': [gsutil_path, 'cp']} + prefixes = {'s3://': [aws_path, 's3', 'cp'], 'gs://': [gsutil_path, 'storage', 'cp']} assert all(len(key) == prefix_len for key in prefixes), prefixes diff --git a/perfkitbenchmarker/data/beam/flink-presubmit.sh b/perfkitbenchmarker/data/beam/flink-presubmit.sh index 6e3d702eb9..9d40d5b220 100644 --- a/perfkitbenchmarker/data/beam/flink-presubmit.sh +++ b/perfkitbenchmarker/data/beam/flink-presubmit.sh @@ -1,5 +1,5 @@ #!/bin/bash echo 'Executing $0' echo 'Copying job jar file $1 into VM...' -gsutil cp "$1" ./job.jar +gcloud storage cp "$1" ./job.jar echo 'Finished copying job jar file.' diff --git a/perfkitbenchmarker/linux_benchmarks/gcsfuse_benchmark.py b/perfkitbenchmarker/linux_benchmarks/gcsfuse_benchmark.py index 52e2f95fd9..16c5837aa7 100644 --- a/perfkitbenchmarker/linux_benchmarks/gcsfuse_benchmark.py +++ b/perfkitbenchmarker/linux_benchmarks/gcsfuse_benchmark.py @@ -118,7 +118,7 @@ def _ReadThroughputTest(vm, mountpoint): """Read the files in the directory via tf.io.gfile or gcsfuse.""" data_dir = FLAGS.gcsfuse_data_dir options = f'--mountpoint="{mountpoint}"' - cmd = f'gsutil ls "{data_dir}" | {_DLVM_PYTHON} {_REMOTE_SCRIPT} {options}' + cmd = f'gcloud storage ls "{data_dir}" | {_DLVM_PYTHON} {_REMOTE_SCRIPT} {options}' logging.info(cmd) stdout, stderr = vm.RemoteCommand(cmd) logging.info(stdout) diff --git a/perfkitbenchmarker/linux_benchmarks/resnet_benchmark.py b/perfkitbenchmarker/linux_benchmarks/resnet_benchmark.py index 8b04494aaf..bbb48f250d 100644 --- a/perfkitbenchmarker/linux_benchmarks/resnet_benchmark.py +++ b/perfkitbenchmarker/linux_benchmarks/resnet_benchmark.py @@ -188,7 +188,7 @@ def Prepare(benchmark_spec): vm.RemoteCommand( 'sudo mkdir -p {data_path} && ' 'sudo chmod a+w {data_path} && ' - 'gsutil -m cp -r {data_dir}/* {data_path}'.format( + 'gcloud storage cp --recursive {data_dir}/* {data_path}'.format( data_dir=benchmark_spec.data_dir, data_path=local_data_path ) ) diff --git a/perfkitbenchmarker/linux_packages/tcmalloc.py b/perfkitbenchmarker/linux_packages/tcmalloc.py index fefd888419..ddd9a10928 100644 --- a/perfkitbenchmarker/linux_packages/tcmalloc.py +++ b/perfkitbenchmarker/linux_packages/tcmalloc.py @@ -86,7 +86,7 @@ def AptInstall(vm): vm.Install('google_cloud_sdk') local_path = '/tmp/libtcmalloc.so' vm.RemoteCommand( - 'gsutil cp {url} {path} ' + 'gcloud storage cp {url} {path} ' '&& echo "export LD_PRELOAD={path}" | sudo tee -a {tmp}'.format( url=FLAGS.tcmalloc_experimental_url, path=local_path, diff --git a/perfkitbenchmarker/log_collector.py b/perfkitbenchmarker/log_collector.py index 6863983a74..c513cd7cc7 100644 --- a/perfkitbenchmarker/log_collector.py +++ b/perfkitbenchmarker/log_collector.py @@ -65,13 +65,13 @@ def CollectPKBLogs(run_uri: str, log_local_path: str) -> None: # Generate the log path to the cloud bucket based on the invocation date of # this function. gcs_log_path = GetLogCloudPath(PKB_LOG_BUCKET.value, f'{run_uri}-pkb.log') + # The gsutil top-level flag '-h' is not supported by 'gcloud storage cp' or 'gcloud storage mv'. vm_util.IssueCommand( [ - 'gsutil', - '-h', - 'Content-Type:text/plain', + 'gcloud', + 'storage', _SAVE_LOG_TO_BUCKET_OPERATION.value, - '-Z', + '--gzip-local-all', log_local_path, gcs_log_path, ], @@ -94,13 +94,13 @@ def CollectVMLogs(run_uri: str, source_path: str) -> None: source_filename = source_path.split('/')[-1] gcs_directory_path = GetLogCloudPath(VM_LOG_BUCKET.value, run_uri) gcs_path = f'{gcs_directory_path}/{source_filename}' + # The gsutil top-level flag '-h' is not supported by 'gcloud storage mv'. vm_util.IssueCommand( [ - 'gsutil', - '-h', - 'Content-Type:text/plain', + 'gcloud', + 'storage', 'mv', - '-Z', + '--gzip-local-all', source_path, gcs_path, ], diff --git a/perfkitbenchmarker/object_storage_service.py b/perfkitbenchmarker/object_storage_service.py index f3ad133d38..bae00f1ae4 100644 --- a/perfkitbenchmarker/object_storage_service.py +++ b/perfkitbenchmarker/object_storage_service.py @@ -185,7 +185,7 @@ def MakeBucket(self, bucket, raise_on_failure=True, tag_bucket=True): def Copy(self, src_url, dst_url, recursive=False): """Copy files, objects and directories. - Note: Recursive copy behavior mimics gsutil cp -r where: + Note: Recursive copy behavior mimics gcloud storage cp --recursive where: Copy(/foo/bar, /baz, True) copies the directory bar into /baz/bar whereas aws s3 cp --recursive would copy the contents of bar into /baz. diff --git a/perfkitbenchmarker/providers/README.md b/perfkitbenchmarker/providers/README.md index 1a04853559..68c969794a 100644 --- a/perfkitbenchmarker/providers/README.md +++ b/perfkitbenchmarker/providers/README.md @@ -427,6 +427,7 @@ To create a new `~/.boto` file, issue the following command and follow the instructions given by this command: ```bash +# "gsutil config" has no direct "gcloud storage" equivalent. Its functionality is handled by "gcloud auth" and "gcloud config". $ gsutil config ``` diff --git a/perfkitbenchmarker/providers/gcp/gce_windows_virtual_machine.py b/perfkitbenchmarker/providers/gcp/gce_windows_virtual_machine.py index f1f4b13682..6a5b0beb92 100644 --- a/perfkitbenchmarker/providers/gcp/gce_windows_virtual_machine.py +++ b/perfkitbenchmarker/providers/gcp/gce_windows_virtual_machine.py @@ -27,7 +27,7 @@ from perfkitbenchmarker.providers.gcp import gcs from perfkitbenchmarker.providers.gcp import util -_WINDOWS_SHUTDOWN_SCRIPT_PS1 = 'Write-Host | gsutil cp - {preempt_marker}' +_WINDOWS_SHUTDOWN_SCRIPT_PS1 = 'Write-Host | gcloud storage cp - {preempt_marker}' _METADATA_PREEMPT_CMD_WIN = ( 'Invoke-RestMethod -Uri' diff --git a/perfkitbenchmarker/scripts/gcsfuse_scripts/read.py b/perfkitbenchmarker/scripts/gcsfuse_scripts/read.py index 55adc67bab..b14f4f511e 100644 --- a/perfkitbenchmarker/scripts/gcsfuse_scripts/read.py +++ b/perfkitbenchmarker/scripts/gcsfuse_scripts/read.py @@ -3,11 +3,11 @@ Example usages: (1) Specify `--mountpoint` to read from gcsfuse. -> gsutil ls gs://gcsfuse-benchmark/10M/ | python read.py --mountpoint=/gcs/ +> gcloud storage ls gs://gcsfuse-benchmark/10M/ | python read.py --mountpoint=/gcs/ (2) Omit `--mountpoint` to read from GCS using tf.io.gfile; specify `--iterations` to run it multiple times. -> gsutil ls gs://gcsfuse-benchmark/10M/ | python read.py --iterations=3 +> gcloud storage ls gs://gcsfuse-benchmark/10M/ | python read.py --iterations=3 """ import concurrent.futures