diff --git a/behat.yml b/behat.yml index 3a19a2bc6f..d0b7119773 100644 --- a/behat.yml +++ b/behat.yml @@ -33,6 +33,9 @@ default: s3: paths: [ "%paths.base%/features/s3" ] contexts: [ Aws\Test\Integ\S3Context ] + s3TransferManager: + paths: [ "%paths.base%/features/s3Transfer" ] + contexts: [ Aws\Test\Integ\S3TransferManagerContext ] s3Encryption: paths: [ "%paths.base%/features/s3Encryption" ] contexts: [ Aws\Test\Integ\S3EncryptionContext ] diff --git a/features/s3Transfer/s3TransferManager.feature b/features/s3Transfer/s3TransferManager.feature new file mode 100644 index 0000000000..0f7b239bbd --- /dev/null +++ b/features/s3Transfer/s3TransferManager.feature @@ -0,0 +1,149 @@ +@s3-transfer-manager @integ +Feature: S3 Transfer Manager + S3 Transfer Manager should successfully do: + - object uploads + - object multipart uploads + - object downloads + - object multipart downloads + - directory object uploads + - directory object downloads + + Scenario Outline: Successfully does a single file upload + Given I have a file with content + When I upload the file to a test bucket using the s3 transfer manager + Then the file should exist in the test bucket and its content should be + + Examples: + | filename | content | + | myfile-test-1-1.txt | Test content #1 | + | myfile-test-1-2.txt | Test content #2 | + | myfile-test-1-3.txt | Test content #3 | + + + Scenario Outline: Successfully does a single upload from a stream + Given I have a stream with content + When I do the upload to a test bucket with key + Then the object , once downloaded from the test bucket, should match the content + Examples: + | content | key | + | "This is a test text - 1" | myfile-test-2-1.txt | + | "This is a test text - 2" | myfile-test-2-2.txt | + | "This is a test text - 3" | myfile-test-2-3.txt | + + Scenario Outline: Successfully do multipart object upload from file + Given I have a file with name where its content's size is + When I do upload this file with name with the specified part size of + Then the object with name should have a total of parts and its size must be + Examples: + | filename | filesize | partsize | partnum | + | myfile-test-3-1.txt | 10485760 | 5242880 | 2 | + | myfile-test-3-2.txt | 24117248 | 5242880 | 5 | + | myfile-test-3-3.txt | 24117248 | 8388608 | 3 | + + Scenario Outline: Successfully do multipart object upload from streams + Given I want to upload a stream of size + When I do upload this stream with name and the specified part size of + Then the object with name should have a total of parts and its size must be + Examples: + | filename | filesize | partsize | partnum | + | myfile-test-4-1.txt | 10485760 | 5242880 | 2 | + | myfile-test-4-2.txt | 24117248 | 5242880 | 5 | + | myfile-test-4-3.txt | 24117248 | 8388608 | 3 | + + Scenario Outline: Does single object upload with custom checksum + Given I have a file with name and its content is + When I upload this file with name by providing a custom checksum algorithm + Then the checksum from the object with name should be equals to the calculation of the object content with the checksum algorithm + Examples: + | filename | content | checksum_algorithm | + | myfile-test-5-1.txt | This is a test file content #1 | crc32 | + | myfile-test-5-2.txt | This is a test file content #2 | crc32c | + | myfile-test-5-3.txt | This is a test file content #3 | sha256 | + | myfile-test-5-4.txt | This is a test file content #4 | sha1 | + + Scenario Outline: Does single object download + Given I have an object in S3 with name and its content is + When I do a download of the object with name + Then the object with name should have been downloaded and its content should be + Examples: + | filename | content | + | myfile-test-6-1.txt | This is a test file content #1 | + | myfile-test-6-2.txt | This is a test file content #2 | + | myfile-test-6-3.txt | This is a test file content #3 | + + Scenario Outline: Successfully does multipart object download + Given I have an object in S3 with name and its size is + When I download the object with name by using the multipart download type + Then the content size for the object with name should be + Examples: + | filename | filesize | download_type | + | myfile-test-7-1.txt | 20971520 | ranged | + | myfile-test-7-2.txt | 28311552 | ranged | + | myfile-test-7-3.txt | 12582912 | ranged | + | myfile-test-7-4.txt | 20971520 | part | + | myfile-test-7-5.txt | 28311552 | part | + | myfile-test-7-6.txt | 12582912 | part | + + Scenario Outline: Successfully does directory upload + Given I have a directory with files that I want to upload + When I upload this directory + Then the files from this directory where its count should be should exist in the bucket + Examples: + | directory | numfile | + | directory-test-1-1 | 10 | + | directory-test-1-2 | 3 | + | directory-test-1-3 | 25 | + | directory-test-1-4 | 1 | + + Scenario Outline: Successfully does a directory download + Given I have a total of objects in a bucket prefixed with + When I download all of them into the directory + Then the objects should exist as files within the directory + Examples: + | numfile | directory | + | 15 | directory-test-2-1 | + | 12 | directory-test-2-2 | + | 1 | directory-test-2-3 | + | 30 | directory-test-2-4 | + + Scenario Outline: Abort a multipart upload + Given I am uploading the file with size + When I upload the file using multipart upload and fails at part number + Then The multipart upload should have been aborted for file + Examples: + | file | size | partNumberFail | + | abort-file-1.txt | 20971520 | 3 | + | abort-file-2.txt | 41943040 | 5 | + | abort-file-3.txt | 10485760 | 1 | + + Scenario Outline: Multipart upload with custom checksum algorithm + Given I have a file to be uploaded of size + When I upload the file with custom checksum algorithm + Then The checksum validation with algorithm for file should succeed + Examples: + | file | size | algorithm | + | myfile-8-1 | 10485760 | sha256 | + | myfile-8-2 | 15728640 | sha256 | + | myfile-8-3 | 7340032 | sha256 | + | myfile-8-4 | 10485760 | crc32 | + | myfile-8-5 | 15728640 | crc32 | + | myfile-8-6 | 7340032 | crc32 | + | myfile-8-7 | 10485760 | sha1 | + | myfile-8-8 | 15728640 | sha1 | + | myfile-8-9 | 7340032 | sha1 | + + Scenario Outline: Multipart upload with custom checksum + Given I have a file to be uploaded of size + When I upload the file with custom checksum and algorithm + Then The checksum validation with checksum and algorithm for file should succeed + Examples: + | file | size | algorithm | checksum | + | myfile-9-1 | 10485760 | sha256 | sdbKFd7hCpt0S5H2EVeHtz+lLMh+drFj1PR+CSyYBSs= | + | myfile-9-2 | 15728640 | sha256 | DCKSUVOFEHo7XBcHLyOzG3ELCzXebwcOJMMsMpWktFE= | + | myfile-9-3 | 7340032 | sha256 | DnkSYSeq3vpPaB+oP4Fy/1ogzNvqdYAqR+0jxB5YKjU= | + | myfile-9-4 | 10485760 | crc32 | vMU7HA== | + | myfile-9-5 | 15728640 | crc32 | gjLQ1Q== | + | myfile-9-6 | 7340032 | crc32 | CKbfZQ== | + | myfile-9-7 | 10485760 | sha1 | baaAJRIP4FjliKgAhv7veG8TbaA= | + | myfile-9-8 | 15728640 | sha1 | wMitTD6HySDcZ+9ycl4MPnVt0zs= | + | myfile-9-9 | 7340032 | sha1 | mQTUWIygeeDpTwFw+QzyHCb0GnU= | diff --git a/src/S3/ApplyChecksumMiddleware.php b/src/S3/ApplyChecksumMiddleware.php index 2ed4519272..fcdd7af1cf 100644 --- a/src/S3/ApplyChecksumMiddleware.php +++ b/src/S3/ApplyChecksumMiddleware.php @@ -76,7 +76,8 @@ public function __invoke( $name = $command->getName(); $body = $request->getBody(); $operation = $this->api->getOperation($name); - $mode = $this->config['request_checksum_calculation'] + $mode = $command['@context']['request_checksum_calculation'] + ?? $this->config['request_checksum_calculation'] ?? self::DEFAULT_CALCULATION_MODE; $command->getMetricsBuilder()->identifyMetricByValueAndAppend( diff --git a/src/S3/S3Transfer/AbstractMultipartDownloader.php b/src/S3/S3Transfer/AbstractMultipartDownloader.php new file mode 100644 index 0000000000..a4cfc4b78e --- /dev/null +++ b/src/S3/S3Transfer/AbstractMultipartDownloader.php @@ -0,0 +1,445 @@ +downloadRequestArgs = $downloadRequestArgs; + $this->validateConfig($config); + $this->config = $config; + if ($downloadHandler === null) { + $downloadHandler = new StreamDownloadHandler(); + } + $this->downloadHandler = $downloadHandler; + $this->currentPartNo = $currentPartNo; + $this->objectPartsCount = $objectPartsCount; + $this->objectSizeInBytes = $objectSizeInBytes; + $this->eTag = $eTag; + $this->currentSnapshot = $currentSnapshot; + if ($listenerNotifier === null) { + $listenerNotifier = new TransferListenerNotifier(); + } + // Add download handler to the listener notifier + $listenerNotifier->addListener($downloadHandler); + $this->listenerNotifier = $listenerNotifier; + } + + /** + * Returns the next command for fetching the next object part. + * + * @return CommandInterface + */ + abstract protected function nextCommand(): CommandInterface; + + /** + * Compute the object dimensions, such as size and parts count. + * + * @param ResultInterface $result + * + * @return void + */ + abstract protected function computeObjectDimensions(ResultInterface $result): void; + + private function validateConfig(array &$config): void + { + if (!isset($config['target_part_size_bytes'])) { + $config['target_part_size_bytes'] = S3TransferManagerConfig::DEFAULT_TARGET_PART_SIZE_BYTES; + } + + if (!isset($config['response_checksum_validation'])) { + $config['response_checksum_validation'] = S3TransferManagerConfig::DEFAULT_RESPONSE_CHECKSUM_VALIDATION; + } + } + + /** + * @return array + */ + public function getConfig(): array + { + return $this->config; + } + + /** + * @return int + */ + public function getCurrentPartNo(): int + { + return $this->currentPartNo; + } + + /** + * @return int + */ + public function getObjectPartsCount(): int + { + return $this->objectPartsCount; + } + + /** + * @return int + */ + public function getObjectSizeInBytes(): int + { + return $this->objectSizeInBytes; + } + + /** + * @return TransferProgressSnapshot + */ + public function getCurrentSnapshot(): TransferProgressSnapshot + { + return $this->currentSnapshot; + } + + /** + * @return DownloadResult + */ + public function download(): DownloadResult + { + return $this->promise()->wait(); + } + + /** + * Returns that resolves a multipart download operation, + * or to a rejection in case of any failures. + * + * @return PromiseInterface + */ + public function promise(): PromiseInterface + { + return Coroutine::of(function () { + try { + $initialRequestResult = yield $this->initialRequest(); + $prevPartNo = $this->currentPartNo - 1; + while ($this->currentPartNo < $this->objectPartsCount) { + // To prevent infinite loops + if ($prevPartNo !== $this->currentPartNo - 1) { + throw new S3TransferException( + "Current part `$this->currentPartNo` MUST increment." + ); + } + + $prevPartNo = $this->currentPartNo; + + $command = $this->nextCommand(); + yield $this->s3Client->executeAsync($command) + ->then(function ($result) use ($command) { + $this->partDownloadCompleted( + $result, + $command->toArray() + ); + + return $result; + })->otherwise(function ($reason) { + $this->partDownloadFailed($reason); + + throw $reason; + }); + } + + if ($this->currentPartNo !== $this->objectPartsCount) { + throw new S3TransferException( + "Expected number of parts `$this->objectPartsCount`" + . " to have been transferred but got `$this->currentPartNo`." + ); + } + + // Transfer completed + $this->downloadComplete(); + + // Return response + $result = $initialRequestResult->toArray(); + unset($result['Body']); + + yield Create::promiseFor(new DownloadResult( + $this->downloadHandler->getHandlerResult(), + $result, + )); + } catch (\Throwable $e) { + $this->downloadFailed($e); + yield Create::rejectionFor($e); + } + }); + } + + /** + * Perform the initial download request. + * + * @return PromiseInterface + */ + protected function initialRequest(): PromiseInterface + { + $command = $this->nextCommand(); + // Notify download initiated + $this->downloadInitiated($command->toArray()); + + return $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) use ($command) { + // Compute object dimensions such as parts count and object size + $this->computeObjectDimensions($result); + + // If there are more than one part then save the ETag + if ($this->objectPartsCount > 1) { + $this->eTag = $result['ETag']; + } + + // Notify listeners + $this->partDownloadCompleted( + $result, + $command->toArray() + ); + + // Assign custom fields in the result + $result['ContentLength'] = $this->objectSizeInBytes; + + return $result; + })->otherwise(function ($reason) { + $this->partDownloadFailed($reason); + + throw $reason; + }); + } + + /** + * Calculates the object size from content range. + * + * @param string $contentRange + * @return int + */ + protected function computeObjectSizeFromContentRange( + string $contentRange + ): int + { + if (empty($contentRange)) { + return 0; + } + + // For extracting the object size from the ContentRange header value. + if (preg_match(self::OBJECT_SIZE_REGEX, $contentRange, $matches)) { + return $matches[1]; + } + + throw new S3TransferException( + "Invalid content range \"$contentRange\"" + ); + } + + /** + * Main purpose of this method is to propagate + * the download-initiated event to listeners, but + * also it does some computation regarding internal states + * that need to be maintained. + * + * @param array $commandArgs + * + * @return void + */ + private function downloadInitiated(array $commandArgs): void + { + if ($this->currentSnapshot === null) { + $this->currentSnapshot = new TransferProgressSnapshot( + $commandArgs['Key'], + 0, + $this->objectSizeInBytes + ); + } else { + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse() + ); + } + + $this->listenerNotifier?->transferInitiated([ + TransferListener::REQUEST_ARGS_KEY => $commandArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, + ]); + } + + /** + * Propagates download-failed event to listeners. + * + * @param \Throwable $reason + * + * @return void + */ + private function downloadFailed(\Throwable $reason): void + { + // Event already propagated. + if ($this->currentSnapshot->getReason() !== null) { + return; + } + + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse(), + $reason + ); + + $this->listenerNotifier?->transferFail([ + TransferListener::REQUEST_ARGS_KEY => $this->downloadRequestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, + 'reason' => $reason, + ]); + } + + /** + * Propagates part-download-completed to listeners. + * It also does some computation in order to maintain internal states. + * + * @param ResultInterface $result + * + * @return void + */ + private function partDownloadCompleted( + ResultInterface $result, + array $requestArgs + ): void + { + $partDownloadBytes = $result['ContentLength']; + if (isset($result['ETag'])) { + $this->eTag = $result['ETag']; + } + + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes() + $partDownloadBytes, + $this->objectSizeInBytes, + $result->toArray() + ); + $this->currentSnapshot = $newSnapshot; + $this->listenerNotifier?->bytesTransferred([ + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, + ]); + } + + /** + * Propagates part-download-failed event to listeners. + * + * @param \Throwable $reason + * + * @return void + */ + private function partDownloadFailed( + \Throwable $reason, + ): void + { + $this->downloadFailed($reason); + } + + /** + * Propagates object-download-completed event to listeners. + * + * @return void + */ + private function downloadComplete(): void + { + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->objectSizeInBytes, + $this->currentSnapshot->getResponse() + ); + $this->currentSnapshot = $newSnapshot; + $this->listenerNotifier?->transferComplete([ + TransferListener::REQUEST_ARGS_KEY => $this->downloadRequestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, + ]); + } + + /** + * @param mixed $multipartDownloadType + * + * @return string + */ + public static function chooseDownloaderClass( + string $multipartDownloadType + ): string + { + return match ($multipartDownloadType) { + AbstractMultipartDownloader::PART_GET_MULTIPART_DOWNLOADER => PartGetMultipartDownloader::class, + AbstractMultipartDownloader::RANGED_GET_MULTIPART_DOWNLOADER => RangeGetMultipartDownloader::class, + default => throw new \InvalidArgumentException( + "The config value for `multipart_download_type` must be one of:\n" + . "\t* " . AbstractMultipartDownloader::PART_GET_MULTIPART_DOWNLOADER + ."\n" + . "\t* " . AbstractMultipartDownloader::RANGED_GET_MULTIPART_DOWNLOADER + ) + }; + } +} diff --git a/src/S3/S3Transfer/AbstractMultipartUploader.php b/src/S3/S3Transfer/AbstractMultipartUploader.php new file mode 100644 index 0000000000..d77ac4d131 --- /dev/null +++ b/src/S3/S3Transfer/AbstractMultipartUploader.php @@ -0,0 +1,406 @@ +s3Client = $s3Client; + $this->requestArgs = $requestArgs; + $this->validateConfig($config); + $this->config = $config; + $this->uploadId = $uploadId; + $this->parts = $parts; + $this->currentSnapshot = $currentSnapshot; + $this->listenerNotifier = $listenerNotifier; + } + + /** + * @return PromiseInterface + */ + abstract protected function createMultipartOperation(): PromiseInterface; + + /** + * @return PromiseInterface + */ + abstract protected function completeMultipartOperation(): PromiseInterface; + + /** + * @return PromiseInterface + */ + abstract protected function processMultipartOperation(): PromiseInterface; + + /** + * @return int + */ + abstract protected function getTotalSize(): int; + + /** + * @param ResultInterface $result + * + * @return mixed + */ + abstract protected function createResponse(ResultInterface $result): mixed; + + /** + * @param array $config + * + * @return void + */ + protected function validateConfig(array &$config): void + { + if (!isset($config['target_part_size_bytes'])) { + $config['target_part_size_bytes'] = S3TransferManagerConfig::DEFAULT_TARGET_PART_SIZE_BYTES; + } + + if (!isset($config['concurrency'])) { + $config['concurrency'] = S3TransferManagerConfig::DEFAULT_CONCURRENCY; + } + + $partSize = $config['target_part_size_bytes']; + if ($partSize < self::PART_MIN_SIZE || $partSize > self::PART_MAX_SIZE) { + throw new \InvalidArgumentException( + "Part size config must be between " . self::PART_MIN_SIZE + ." and " . self::PART_MAX_SIZE . " bytes " + ."but it is configured to $partSize" + ); + } + } + + /** + * @return string|null + */ + public function getUploadId(): ?string + { + return $this->uploadId; + } + + /** + * @return array + */ + public function getParts(): array + { + return $this->parts; + } + + /** + * Get the current progress snapshot. + * @return TransferProgressSnapshot|null + */ + public function getCurrentSnapshot(): ?TransferProgressSnapshot + { + return $this->currentSnapshot; + } + + /** + * @return PromiseInterface + */ + public function promise(): PromiseInterface + { + return Coroutine::of(function () { + try { + yield $this->createMultipartOperation(); + yield $this->processMultipartOperation(); + yield $this->completeMultipartOperation(); + } finally { + $this->callOnCompletionCallbacks(); + } + })->then(function (ResultInterface $result) { + return $this->createResponse($result); + })->otherwise(function (Throwable $e) { + $this->operationFailed($e); + + throw $e; + }); + } + + /** + * @return PromiseInterface + */ + protected function abortMultipartOperation(): PromiseInterface + { + $abortMultipartUploadArgs = $this->requestArgs; + $abortMultipartUploadArgs['UploadId'] = $this->uploadId; + $command = $this->s3Client->getCommand( + 'AbortMultipartUpload', + $abortMultipartUploadArgs + ); + + return $this->s3Client->executeAsync($command); + } + + /** + * @return void + */ + protected function sortParts(): void + { + usort($this->parts, function ($partOne, $partTwo) { + return $partOne['PartNumber'] <=> $partTwo['PartNumber']; + }); + } + + /** + * @param ResultInterface $result + * @param CommandInterface $command + * @return void + */ + protected function collectPart( + ResultInterface $result, + CommandInterface $command + ): void + { + $checksumResult = match($command->getName()) { + 'UploadPart' => $result, + 'UploadPartCopy' => $result['CopyPartResult'], + default => $result[$command->getName() . 'Result'] + }; + + $partData = [ + 'PartNumber' => $command['PartNumber'], + 'ETag' => $checksumResult['ETag'], + ]; + + if (isset($command['ChecksumAlgorithm'])) { + $checksumMemberName = 'Checksum' . strtoupper($command['ChecksumAlgorithm']); + $partData[$checksumMemberName] = $checksumResult[$checksumMemberName] ?? null; + } + + $this->parts[] = $partData; + } + + /** + * @param \Iterator $commands + * @param callable $fulfilledCallback + * @param callable $rejectedCallback + * @return PromiseInterface + */ + protected function createCommandPool( + \Iterator $commands, + callable $fulfilledCallback, + callable $rejectedCallback + ): PromiseInterface + { + return (new CommandPool( + $this->s3Client, + $commands, + [ + 'concurrency' => $this->config['concurrency'], + 'fulfilled' => $fulfilledCallback, + 'rejected' => $rejectedCallback + ] + ))->promise(); + } + + /** + * @param array $requestArgs + * @return void + */ + protected function operationInitiated(array $requestArgs): void + { + if ($this->currentSnapshot === null) { + $this->currentSnapshot = new TransferProgressSnapshot( + $requestArgs['Key'], + 0, + $this->getTotalSize() + ); + } + + $this->listenerNotifier?->transferInitiated([ + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot + ]); + } + + /** + * @param ResultInterface $result + * @return void + */ + protected function operationCompleted(ResultInterface $result): void + { + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $result->toArray(), + $this->currentSnapshot->getReason(), + ); + + $this->currentSnapshot = $newSnapshot; + + $this->listenerNotifier?->transferComplete([ + TransferListener::REQUEST_ARGS_KEY => + $this->requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot + ]); + } + + /** + * @param Throwable $reason + * @return void + * + */ + protected function operationFailed(Throwable $reason): void + { + // Event already propagated + if ($this->currentSnapshot?->getReason() !== null) { + return; + } + + if ($this->currentSnapshot === null) { + $this->currentSnapshot = new TransferProgressSnapshot( + 'Unknown', + 0, + 0, + ); + } + + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse(), + $reason + ); + + if (!empty($this->uploadId)) { + error_log( + "Multipart Upload with id: " . $this->uploadId . " failed", + E_USER_WARNING + ); + $this->abortMultipartOperation()->wait(); + } + + $this->listenerNotifier?->transferFail([ + TransferListener::REQUEST_ARGS_KEY => + $this->requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, + 'reason' => $reason, + ]); + } + + /** + * @param int $partSize + * @param array $requestArgs + * @return void + */ + protected function partCompleted( + int $partSize, + array $requestArgs + ): void + { + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes() + $partSize, + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse(), + $this->currentSnapshot->getReason(), + ); + + $this->currentSnapshot = $newSnapshot; + + $this->listenerNotifier?->bytesTransferred([ + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot + ]); + } + + /** + * @return void + */ + protected function callOnCompletionCallbacks(): void + { + foreach ($this->onCompletionCallbacks as $fn) { + if (is_callable($fn)) { + call_user_func($fn); + } + } + + $this->onCompletionCallbacks = []; + } + + /** + * @param Throwable $reason + * @return void + */ + protected function partFailed(Throwable $reason): void + { + $this->operationFailed($reason); + } + + /** + * @return int + */ + protected function calculatePartSize(): int + { + return max( + $this->getTotalSize() / self::PART_MAX_NUM, + $this->config['target_part_size_bytes'] + ); + } +} diff --git a/src/S3/S3Transfer/Exception/FileDownloadException.php b/src/S3/S3Transfer/Exception/FileDownloadException.php new file mode 100644 index 0000000000..4f9dee8776 --- /dev/null +++ b/src/S3/S3Transfer/Exception/FileDownloadException.php @@ -0,0 +1,5 @@ + 'string', + 'filter' => 'callable', + 'download_object_request_modifier' => 'callable', + 'failure_policy' => 'callable', + 'max_concurrency' => 'int', + 'track_progress' => 'bool', + 'target_part_size_bytes' => 'int', + 'list_objects_v2_args' => 'array', + 'fails_when_destination_exists' => 'bool', + ]; + public const DEFAULT_MAX_CONCURRENCY = 100; + + /** @var string */ + private string $sourceBucket; + + /** @var string */ + private string $destinationDirectory; + + /** @var array */ + private readonly array $downloadRequestArgs; + + /** + * @param string $sourceBucket The bucket from where the files are going to be + * downloaded from. + * @param string $destinationDirectory The destination path where the downloaded + * files will be placed in. + * @param array $downloadRequestArgs + * @param array $config The config options for this download directory operation. + * - s3_prefix: (string, optional) This parameter will be considered just if + * not provided as part of the list_objects_v2_args config option. + * - filter: (Closure, optional) A callable which will receive an object key as + * parameter and should return true or false in order to determine + * whether the object should be downloaded. + * - download_object_request_modifier: (Closure, optional) A function that will + * be invoked right before the download request is performed and that will + * receive as parameter the request arguments for each request. + * - failure_policy: (Closure, optional) A function that will be invoked + * on a download failure and that will receive as parameters: + * - $requestArgs: (array) The arguments for the request that originated + * the failure. + * - $downloadDirectoryRequestArgs: (array) The arguments for the download + * directory request. + * - $reason: (Throwable) The exception that originated the request failure. + * - $downloadDirectoryResponse: (DownloadDirectoryResult) The download response + * to that point in the upload process. + * - track_progress: (bool, optional) Overrides the config option set + * in the transfer manager instantiation to decide whether transfer + * progress should be tracked. + * - target_part_size_bytes: (int, optional) The part size in bytes + * to be used in a range multipart download. + * - fails_when_destination_exists: (bool) Whether to fail when a destination + * file exists. + * - max_concurrency: (int, optional) The max number of concurrent downloads. + * - list_objects_v2_args: (array, optional) The arguments to be included + * as part of the listObjectV2 request in order to fetch the objects to + * be downloaded. The most common arguments would be: + * - MaxKeys: (int) Sets the maximum number of keys returned in the response. + * - Prefix: (string) To limit the response to keys that begin with the + * specified prefix. + * @param TransferListener[] $listeners The listeners for watching + * transfer events. Each listener will be cloned per file upload. + * @param TransferListener|null $progressTracker Ideally the progress + * tracker implementation provided here should be able to track multiple + * transfers at once. Please see MultiProgressTracker implementation. + */ + public function __construct( + string $sourceBucket, + string $destinationDirectory, + array $downloadRequestArgs = [], + array $config = [], + array $listeners = [], + ?TransferListener $progressTracker = null + ) { + parent::__construct($listeners, $progressTracker, $config); + if (ArnParser::isArn($sourceBucket)) { + $sourceBucket = ArnParser::parse($sourceBucket)->getResource(); + } + + $this->sourceBucket = $sourceBucket; + $this->destinationDirectory = $destinationDirectory; + $this->downloadRequestArgs = $downloadRequestArgs; + } + + /** + * @return string + */ + public function getSourceBucket(): string + { + return $this->sourceBucket; + } + + /** + * @return string + */ + public function getDestinationDirectory(): string + { + return $this->destinationDirectory; + } + + /** + * @return array + */ + public function getDownloadRequestArgs(): array + { + return $this->downloadRequestArgs; + } + + /** + * Helper method to validate the destination directory exists. + * + * @return void + */ + public function validateDestinationDirectory(): void + { + if (!file_exists($this->destinationDirectory)) { + mkdir($this->destinationDirectory, 0755, true); + } + + if (!is_dir($this->destinationDirectory)) { + throw new InvalidArgumentException( + "Destination directory `$this->destinationDirectory` is not a directory." + ); + } + } +} diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryResult.php b/src/S3/S3Transfer/Models/DownloadDirectoryResult.php new file mode 100644 index 0000000000..8449093a64 --- /dev/null +++ b/src/S3/S3Transfer/Models/DownloadDirectoryResult.php @@ -0,0 +1,66 @@ +objectsDownloaded = $objectsDownloaded; + $this->objectsFailed = $objectsFailed; + $this->reason = $reason; + } + + /** + * @return int + */ + public function getObjectsDownloaded(): int + { + return $this->objectsDownloaded; + } + + /** + * @return int + */ + public function getObjectsFailed(): int + { + return $this->objectsFailed; + } + + public function getReason(): ?Throwable + { + return $this->reason; + } + + /** + * @return string + */ + public function __toString(): string + { + return sprintf( + "DownloadDirectoryResult: %d objects downloaded, %d objects failed", + $this->objectsDownloaded, + $this->objectsFailed + ); + } +} diff --git a/src/S3/S3Transfer/Models/DownloadFileRequest.php b/src/S3/S3Transfer/Models/DownloadFileRequest.php new file mode 100644 index 0000000000..d71e2ead5b --- /dev/null +++ b/src/S3/S3Transfer/Models/DownloadFileRequest.php @@ -0,0 +1,67 @@ +destination = $destination; + $this->failsWhenDestinationExists = $failsWhenDestinationExists; + $this->downloadRequest = DownloadRequest::fromDownloadRequestAndDownloadHandler( + $downloadRequest, + new FileDownloadHandler( + $destination, + $failsWhenDestinationExists + ) + ); + } + + /** + * @return string + */ + public function getDestination(): string + { + return $this->destination; + } + + /** + * @return bool + */ + public function isFailsWhenDestinationExists(): bool + { + return $this->failsWhenDestinationExists; + } + + /** + * @return DownloadRequest + */ + public function getDownloadRequest(): DownloadRequest + { + return $this->downloadRequest; + } +} diff --git a/src/S3/S3Transfer/Models/DownloadRequest.php b/src/S3/S3Transfer/Models/DownloadRequest.php new file mode 100644 index 0000000000..918fe62cfe --- /dev/null +++ b/src/S3/S3Transfer/Models/DownloadRequest.php @@ -0,0 +1,150 @@ + 'string', + 'multipart_download_type' => 'string', + 'track_progress' => 'bool', + ]; + + /** @var string|array|null */ + private string|array|null $source; + + /** @var array */ + private array $downloadRequestArgs; + + /** @var DownloadHandler|null */ + private ?DownloadHandler $downloadHandler; + + /** + * @param string|array|null $source The object to be downloaded from S3. + * It can be either a string with a S3 URI or an array with a Bucket and Key + * properties set. + * @param array $downloadRequestArgs + * @param array $config The configuration to be used for this operation: + * - multipart_download_type: (string, optional) + * Overrides the resolved value from the transfer manager config. + * - response_checksum_validation: (string, optional) Overrides the resolved + * value from transfer manager config for whether checksum validation + * should be done. This option will be considered just if ChecksumMode + * is not present in the request args. + * - track_progress: (bool) Overrides the config option set in the transfer + * manager instantiation to decide whether transfer progress should be + * tracked. + * - target_part_size_bytes: (int) The part size in bytes to be used + * in a range multipart download. If this parameter is not provided + * then it fallbacks to the transfer manager `target_part_size_bytes` + * config value. + * @param DownloadHandler|null $downloadHandler + * @param TransferListener[]|null $listeners + * @param TransferListener|null $progressTracker + */ + public function __construct( + string|array|null $source, + array $downloadRequestArgs = [], + array $config = [], + ?DownloadHandler $downloadHandler = null, + array $listeners = [], + ?TransferListener $progressTracker = null + ) { + parent::__construct($listeners, $progressTracker, $config); + $this->source = $source; + $this->downloadRequestArgs = $downloadRequestArgs; + $this->config = $config; + if ($downloadHandler === null) { + $downloadHandler = new StreamDownloadHandler(); + } + $this->downloadHandler = $downloadHandler; + } + + /** + * @param DownloadRequest $downloadRequest + * @param FileDownloadHandler $downloadHandler + * + * @return self + */ + public static function fromDownloadRequestAndDownloadHandler( + DownloadRequest $downloadRequest, + FileDownloadHandler $downloadHandler + ): self + { + return new self( + $downloadRequest->getSource(), + $downloadRequest->getObjectRequestArgs(), + $downloadRequest->getConfig(), + $downloadHandler, + $downloadRequest->getListeners(), + $downloadRequest->getProgressTracker() + ); + } + + /** + * @return array|string|null + */ + public function getSource(): array|string|null + { + return $this->source; + } + + /** + * @return array + */ + public function getObjectRequestArgs(): array + { + return $this->downloadRequestArgs; + } + + /** + * @return DownloadHandler + */ + public function getDownloadHandler(): DownloadHandler + { + return $this->downloadHandler; + } + + /** + * Helper method to normalize the source as an array with: + * - Bucket + * - Key + * + * @return array + */ + public function normalizeSourceAsArray(): array + { + // If source is null then fall back to getObjectRequest. + $source = $this->getSource() ?? [ + 'Bucket' => $this->downloadRequestArgs['Bucket'] ?? null, + 'Key' => $this->downloadRequestArgs['Key'] ?? null, + ]; + if (is_string($source)) { + $sourceAsArray = S3TransferManager::s3UriAsBucketAndKey($source); + } elseif (is_array($source)) { + $sourceAsArray = $source; + } else { + throw new S3TransferException( + "Unsupported source type `" . gettype($source) . "`" + ); + } + + foreach (['Bucket', 'Key'] as $reqKey) { + if (empty($sourceAsArray[$reqKey])) { + throw new \InvalidArgumentException( + "`$reqKey` is required but not provided in " + . implode(', ', array_keys($sourceAsArray)) . "." + ); + } + } + + return $sourceAsArray; + } +} diff --git a/src/S3/S3Transfer/Models/DownloadResult.php b/src/S3/S3Transfer/Models/DownloadResult.php new file mode 100644 index 0000000000..64152faaf9 --- /dev/null +++ b/src/S3/S3Transfer/Models/DownloadResult.php @@ -0,0 +1,30 @@ +downloadDataResult = $downloadDataResult; + } + + /** + * @return mixed + */ + public function getDownloadDataResult(): mixed + { + return $this->downloadDataResult; + } +} diff --git a/src/S3/S3Transfer/Models/S3TransferManagerConfig.php b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php new file mode 100644 index 0000000000..695371cc62 --- /dev/null +++ b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php @@ -0,0 +1,187 @@ +targetPartSizeBytes = $targetPartSizeBytes; + $this->multipartUploadThresholdBytes = $multipartUploadThresholdBytes; + $this->requestChecksumCalculation = $requestChecksumCalculation; + $this->responseChecksumValidation = $responseChecksumValidation; + $this->multipartDownloadType = $multipartDownloadType; + $this->concurrency = $concurrency; + $this->trackProgress = $trackProgress; + $this->defaultRegion = $defaultRegion; + } + + /** $config: + * - target_part_size_bytes: (int, default=(8388608 `8MB`)) + * The minimum part size to be used in a multipart upload/download. + * - multipart_upload_threshold_bytes: (int, default=(16777216 `16 MB`)) + * The threshold to decided whether a multipart upload is needed. + * - request_checksum_calculation: (string, default=`when_supported`) + * To decide whether a checksum validation will be applied to the response. + * - response_checksum_validation: (string, default=`when_supported`) + * - multipart_download_type: (string, default='part') + * The download type to be used in a multipart download. + * - concurrency: (int, default=5) + * Maximum number of concurrent operations allowed during a multipart + * upload/download. + * - track_progress: (bool, default=false) + * To enable progress tracker in a multipart upload/download, and or + * a directory upload/download operation. + * - default_region: (string, default="us-east-2") + */ + public static function fromArray(array $config): self { + return new self( + $config['target_part_size_bytes'] + ?? self::DEFAULT_TARGET_PART_SIZE_BYTES, + $config['multipart_upload_threshold_bytes'] + ?? self::DEFAULT_MULTIPART_UPLOAD_THRESHOLD_BYTES, + $config['request_checksum_calculation'] + ?? self::DEFAULT_REQUEST_CHECKSUM_CALCULATION, + $config['response_checksum_validation'] + ?? self::DEFAULT_RESPONSE_CHECKSUM_VALIDATION, + $config['multipart_download_type'] + ?? self::DEFAULT_MULTIPART_DOWNLOAD_TYPE, + $config['concurrency'] + ?? self::DEFAULT_CONCURRENCY, + $config['track_progress'] ?? self::DEFAULT_TRACK_PROGRESS, + $config['default_region'] ?? self::DEFAULT_REGION + ); + } + + /** + * @return int + */ + public function getTargetPartSizeBytes(): int + { + return $this->targetPartSizeBytes; + } + + /** + * @return int + */ + public function getMultipartUploadThresholdBytes(): int + { + return $this->multipartUploadThresholdBytes; + } + + /** + * @return string + */ + public function getRequestChecksumCalculation(): string + { + return $this->requestChecksumCalculation; + } + + /** + * @return string + */ + public function getResponseChecksumValidation(): string + { + return $this->responseChecksumValidation; + } + + /** + * @return string + */ + public function getMultipartDownloadType(): string + { + return $this->multipartDownloadType; + } + + /** + * @return int + */ + public function getConcurrency(): int + { + return $this->concurrency; + } + + /** + * @return bool + */ + public function isTrackProgress(): bool + { + return $this->trackProgress; + } + + /** + * @return string + */ + public function getDefaultRegion(): string + { + return $this->defaultRegion; + } + + /** + * @return array + */ + public function toArray(): array + { + return [ + 'target_part_size_bytes' => $this->targetPartSizeBytes, + 'multipart_upload_threshold_bytes' => $this->multipartUploadThresholdBytes, + 'request_checksum_calculation' => $this->requestChecksumCalculation, + 'response_checksum_validation' => $this->responseChecksumValidation, + 'multipart_download_type' => $this->multipartDownloadType, + 'concurrency' => $this->concurrency, + 'track_progress' => $this->trackProgress, + 'default_region' => $this->defaultRegion, + ]; + } +} diff --git a/src/S3/S3Transfer/Models/TransferRequest.php b/src/S3/S3Transfer/Models/TransferRequest.php new file mode 100644 index 0000000000..b75a0cd81e --- /dev/null +++ b/src/S3/S3Transfer/Models/TransferRequest.php @@ -0,0 +1,95 @@ + 'bool', + ]; + + /** @var array */ + protected array $listeners; + + /** @var TransferListener|null */ + protected ?TransferListener $progressTracker; + + /** @var array */ + protected array $config; + + /** + * @param array $listeners + * @param TransferListener|null $progressTracker + * @param array $config + */ + public function __construct( + array $listeners, + ?TransferListener $progressTracker, + array $config + ) { + $this->listeners = $listeners; + $this->progressTracker = $progressTracker; + $this->config = $config; + } + + /** + * Get current listeners. + * + * @return array + */ + public function getListeners(): array + { + return $this->listeners; + } + + /** + * Get the progress tracker. + * + * @return TransferListener|null + */ + public function getProgressTracker(): ?TransferListener + { + return $this->progressTracker; + } + + /** + * @return array + */ + public function getConfig(): array + { + return $this->config; + } + + /** + * @param array $defaultConfig + * + * @return void + */ + public function updateConfigWithDefaults(array $defaultConfig): void + { + foreach (static::$configKeys as $key => $_) { + if (isset($defaultConfig[$key]) && empty($this->config[$key])) { + $this->config[$key] = $defaultConfig[$key]; + } + } + } + + /** + * For validating config. By default, it provides an empty + * implementation. + * @return void + */ + public function validateConfig(): void { + foreach (static::$configKeys as $key => $type) { + if (isset($this->config[$key]) + && !call_user_func('is_' . $type, $this->config[$key])) { + throw new InvalidArgumentException( + "The provided config `$key` must be $type." + ); + } + } + } +} diff --git a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php new file mode 100644 index 0000000000..4c4c8c0247 --- /dev/null +++ b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php @@ -0,0 +1,113 @@ + 'bool', + 'recursive' => 'bool', + 's3_prefix' => 'string', + 'filter' => 'callable', + 's3_delimiter' => 'string', + 'upload_object_request_modifier' => 'callable', + 'failure_policy' => 'callable', + 'max_concurrency' => 'int', + 'max_depth' => 'int', + 'track_progress' => 'bool', + ]; + public const DEFAULT_MAX_CONCURRENCY = 100; + + /** @var string */ + private string $sourceDirectory; + + /** @var string */ + private string $targetBucket; + + /** @var array */ + private readonly array $uploadRequestArgs; + + /** + * @param string $sourceDirectory The source directory to upload. + * @param string $targetBucket The name of the bucket to upload objects to. + * @param array $uploadRequestArgs The extract arguments to be passed in + * each upload request. + * @param array $config + * - follow_symbolic_links: (boolean, optional) Whether to follow symbolic links when + * traversing the file tree. + * - recursive: (boolean, optional) Whether to upload directories recursively. + * - s3_prefix: (string, optional) The S3 key prefix to use for each object. + * If not provided, files will be uploaded to the root of the bucket. + * - filter: (callable, optional) A callback to allow users to filter out unwanted files. + * It is invoked for each file. An example implementation is a predicate + * that takes a file and returns a boolean indicating whether this file + * should be uploaded. + * - s3_delimiter: The S3 delimiter. A delimiter causes a list operation + * to roll up all the keys that share a common prefix into a single summary list result. + * - upload_object_request_modifier: (callable, optional) A callback mechanism + * to allow customers to update individual putObjectRequest that the S3 Transfer Manager generates. + * - failure_policy: (callable, optional) The failure policy to handle failed requests. + * - max_concurrency: (int, optional) The max number of concurrent uploads. + * @param array $listeners For listening to transfer events such as transferInitiated. + * @param TransferListener|null $progressTracker For showing progress in transfers. + */ + public function __construct( + string $sourceDirectory, + string $targetBucket, + array $uploadRequestArgs = [], + array $config = [], + array $listeners = [], + ?TransferListener $progressTracker = null + ) { + parent::__construct($listeners, $progressTracker, $config); + $this->sourceDirectory = $sourceDirectory; + if (ArnParser::isArn($targetBucket)) { + $targetBucket = ArnParser::parse($targetBucket)->getResource(); + } + $this->targetBucket = $targetBucket; + $this->uploadRequestArgs = $uploadRequestArgs; + $this->config = $config; + } + + /** + * @return string + */ + public function getSourceDirectory(): string + { + return $this->sourceDirectory; + } + + /** + * @return string + */ + public function getTargetBucket(): string + { + return $this->targetBucket; + } + + /** + * @return array + */ + public function getUploadRequestArgs(): array + { + return $this->uploadRequestArgs; + } + + /** + * Helper method to validate source directory + * @return void + */ + public function validateSourceDirectory(): void + { + if (!is_dir($this->sourceDirectory)) { + throw new InvalidArgumentException( + "Please provide a valid directory path. " + . "Provided = " . $this->sourceDirectory + ); + } + } +} diff --git a/src/S3/S3Transfer/Models/UploadDirectoryResult.php b/src/S3/S3Transfer/Models/UploadDirectoryResult.php new file mode 100644 index 0000000000..2b4f58f801 --- /dev/null +++ b/src/S3/S3Transfer/Models/UploadDirectoryResult.php @@ -0,0 +1,69 @@ +objectsUploaded = $objectsUploaded; + $this->objectsFailed = $objectsFailed; + $this->reason = $exception; + } + + /** + * @return int + */ + public function getObjectsUploaded(): int + { + return $this->objectsUploaded; + } + + /** + * @return int + */ + public function getObjectsFailed(): int + { + return $this->objectsFailed; + } + + /** + * @return Throwable|null + */ + public function getReason(): ?Throwable + { + return $this->reason; + } + + /** + * @return string + */ + public function __toString(): string + { + return sprintf( + "UploadDirectoryResult: %d objects uploaded, %d objects failed", + $this->objectsUploaded, + $this->objectsFailed + ); + } +} diff --git a/src/S3/S3Transfer/Models/UploadRequest.php b/src/S3/S3Transfer/Models/UploadRequest.php new file mode 100644 index 0000000000..7c0b54c021 --- /dev/null +++ b/src/S3/S3Transfer/Models/UploadRequest.php @@ -0,0 +1,120 @@ + 'int', + 'target_part_size_bytes' => 'int', + 'track_progress' => 'bool', + 'concurrency' => 'int', + 'request_checksum_calculation' => 'string', + ]; + + /** @var StreamInterface|string */ + private StreamInterface|string $source; + + /** @var array */ + private array $uploadRequestArgs; + + /** + * @param string|StreamInterface $source + * @param array $uploadRequestArgs The putObject request arguments. + * Required parameters would be: + * - Bucket: (string, required) + * - Key: (string, required) + * @param array $config The config options for this upload operation. + * - multipart_upload_threshold_bytes: (int, optional) + * To override the default threshold for when to use multipart upload. + * - target_part_size_bytes: (int, optional) To override the default + * target part size in bytes. + * - track_progress: (bool, optional) To override the default option for + * enabling progress tracking. If this option is resolved as true and + * a progressTracker parameter is not provided then, a default implementation + * will be resolved. This option is intended to make the operation to use + * a default progress tracker implementation when $progressTracker is null. + * - concurrency: (int, optional) To override default value for concurrency. + * - request_checksum_calculation: (string, optional, defaulted to `when_supported`) + * @param TransferListener[]|null $listeners + * @param TransferListener|null $progressTracker + * + */ + public function __construct( + StreamInterface|string $source, + array $uploadRequestArgs, + array $config = [], + array $listeners = [], + ?TransferListener $progressTracker = null + ) { + parent::__construct($listeners, $progressTracker, $config); + $this->source = $source; + $this->uploadRequestArgs = $uploadRequestArgs; + } + + /** + * Get the source. + * + * @return StreamInterface|string + */ + public function getSource(): StreamInterface|string + { + return $this->source; + } + + /** + * Get the put object request. + * + * @return array + */ + public function getUploadRequestArgs(): array + { + return $this->uploadRequestArgs; + } + + /** + * Helper method for validating the given source. + * + * @return void + */ + public function validateSource(): void + { + if (is_string($this->getSource()) && !is_readable($this->getSource())) { + throw new InvalidArgumentException( + "Please provide a valid readable file path or a valid stream as source." + ); + } + } + + /** + * Helper method for validating required parameters. + * + * @param string|null $customMessage + * @return void + */ + public function validateRequiredParameters( + ?string $customMessage = null + ): void + { + $requiredParametersWithArgs = [ + 'Bucket' => $this->uploadRequestArgs['Bucket'] ?? null, + 'Key' => $this->uploadRequestArgs['Key'] ?? null, + ]; + foreach ($requiredParametersWithArgs as $key => $value) { + if (empty($value)) { + if ($customMessage !== null) { + throw new InvalidArgumentException($customMessage); + } + + // Fallback to default error message + throw new InvalidArgumentException( + "The `$key` parameter must be provided as part of the request arguments." + ); + } + } + } +} diff --git a/src/S3/S3Transfer/Models/UploadResult.php b/src/S3/S3Transfer/Models/UploadResult.php new file mode 100644 index 0000000000..30029082be --- /dev/null +++ b/src/S3/S3Transfer/Models/UploadResult.php @@ -0,0 +1,16 @@ +body = $this->parseBody($source); + $this->calculatedObjectSize = 0; + $this->isFullObjectChecksum = false; + $this->evaluateCustomChecksum(); + } + + /** + * @inheritDoc + * + * @return PromiseInterface + */ + protected function createMultipartOperation(): PromiseInterface + { + $createMultipartUploadArgs = $this->requestArgs; + if ($this->requestChecksum !== null) { + $createMultipartUploadArgs['ChecksumType'] = self::CHECKSUM_TYPE_FULL_OBJECT; + $createMultipartUploadArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; + $this->isFullObjectChecksum = true; + } elseif ($this->config['request_checksum_calculation'] === 'when_supported') { + $this->requestChecksumAlgorithm = $createMultipartUploadArgs['ChecksumAlgorithm'] + ?? self::DEFAULT_CHECKSUM_CALCULATION_ALGORITHM; + $createMultipartUploadArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; + } + + // Make sure algorithm with full object is a supported one + if (($createMultipartUploadArgs['ChecksumType'] ?? '') === self::CHECKSUM_TYPE_FULL_OBJECT) { + if (stripos($this->requestChecksumAlgorithm, 'crc') !== 0) { + return Create::rejectionFor( + new S3TransferException( + "Full object checksum algorithm must be `CRC` family base." + ) + ); + } + } + + $this->operationInitiated($createMultipartUploadArgs); + $command = $this->s3Client->getCommand( + 'CreateMultipartUpload', + $createMultipartUploadArgs + ); + + return $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) { + $this->uploadId = $result['UploadId']; + return $result; + }); + } + + /** + * @inheritDoc + * + * @return PromiseInterface + */ + protected function completeMultipartOperation(): PromiseInterface + { + $this->sortParts(); + $completeMultipartUploadArgs = $this->requestArgs; + $completeMultipartUploadArgs['UploadId'] = $this->uploadId; + $completeMultipartUploadArgs['MultipartUpload'] = [ + 'Parts' => $this->parts + ]; + $completeMultipartUploadArgs['MpuObjectSize'] = $this->getTotalSize(); + + if ($this->isFullObjectChecksum && $this->requestChecksum !== null) { + $completeMultipartUploadArgs['ChecksumType'] = self::CHECKSUM_TYPE_FULL_OBJECT; + $completeMultipartUploadArgs[ + 'Checksum' . strtoupper($this->requestChecksumAlgorithm) + ] = $this->requestChecksum; + } + + $command = $this->s3Client->getCommand( + 'CompleteMultipartUpload', + $completeMultipartUploadArgs + ); + + return $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) { + $this->operationCompleted($result); + return $result; + }); + } + + /** + * Sync upload method. + * + * @return UploadResult + */ + public function upload(): UploadResult + { + return $this->promise()->wait(); + } + + /** + * Parses the source into an instance of + * StreamInterface to be read. + * + * @param string|StreamInterface $source + * + * @return StreamInterface + */ + private function parseBody( + string|StreamInterface $source + ): StreamInterface + { + if (is_string($source)) { + // Make sure the files exists + if (!is_readable($source)) { + throw new \InvalidArgumentException( + "The source for this upload must be either a" + . " readable file path or a valid stream." + ); + } + $body = new LazyOpenStream($source, 'r'); + // To make sure the resource is closed. + $this->onCompletionCallbacks[] = function () use ($body) { + $body->close(); + }; + } elseif ($source instanceof StreamInterface) { + $body = $source; + } else { + throw new \InvalidArgumentException( + "The source must be a valid string file path or a StreamInterface." + ); + } + + return $body; + } + + /** + * Evaluates if custom checksum has been provided, + * and if so then, the values are placed in the + * respective properties. + * + * @return void + */ + private function evaluateCustomChecksum(): void + { + // Evaluation for custom provided checksums + $checksumName = self::filterChecksum($this->requestArgs); + if ($checksumName !== null) { + $this->requestChecksum = $this->requestArgs[$checksumName]; + $this->requestChecksumAlgorithm = str_replace( + 'Checksum', + '', + $checksumName + ); + $this->requestChecksumAlgorithm = strtolower( + $this->requestChecksumAlgorithm + ); + } else { + $this->requestChecksum = null; + $this->requestChecksumAlgorithm = null; + } + } + + /** + * Process a multipart upload operation. + * + * @return PromiseInterface + */ + protected function processMultipartOperation(): PromiseInterface + { + $uploadPartCommandArgs = $this->requestArgs; + $this->calculatedObjectSize = 0; + $partSize = $this->calculatePartSize(); + $partsCount = ceil($this->getTotalSize() / $partSize); + $uploadPartCommandArgs['UploadId'] = $this->uploadId; + // Customer provided checksum + if ($this->requestChecksum !== null) { + // To avoid default calculation for individual parts + $uploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; + unset($uploadPartCommandArgs['Checksum'. strtoupper($this->requestChecksumAlgorithm)]); + } elseif ($this->requestChecksumAlgorithm !== null) { + $uploadPartCommandArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; + } + + $promises = $this->createUploadPartPromises( + $uploadPartCommandArgs, + $partSize, + $partsCount, + ); + + return Each::ofLimitAll($promises, $this->config['concurrency']); + } + + /** + * @param array $uploadPartCommandArgs + * @param int $partSize + * @param int $partsCount + * + * @return \Generator + */ + private function createUploadPartPromises( + array $uploadPartCommandArgs, + int $partSize, + int $partsCount + ): \Generator + { + $partNo = count($this->parts); + $bytesRead = 0; + $isSeekable = $this->body->isSeekable() + && $this->body->getMetadata('wrapper_type') + === self::STREAM_WRAPPER_TYPE_PLAIN_FILE; + while (!$this->body->eof()) { + if ($isSeekable) { + $partBody = new LimitStream( + new LazyOpenStream( + $this->body->getMetadata('uri'), + 'r' + ), + $partSize, + $bytesRead, + ); + } else { + $body = new LimitStream( + $this->body, + $partSize, + $bytesRead, + ); + $body = $this->decorateWithHashes($body, $uploadPartCommandArgs); + $partBody = Utils::streamFor(); + Utils::copyToStream($body, $partBody); + } + + $bodyLength = $partBody->getSize(); + if ($bodyLength === 0) { + $partBody->close(); + break; + } + + $partNo++; + $bytesRead += $bodyLength; + + $uploadPartCommandArgs['PartNumber'] = $partNo; + $uploadPartCommandArgs['ContentLength'] = $bodyLength; + // Attach body + if ($isSeekable) { + $partBody->rewind(); + } + $uploadPartCommandArgs['Body'] = $partBody; + + $this->calculatedObjectSize += $bodyLength; + if ($partNo > self::PART_MAX_NUM) { + return Create::rejectionFor( + "The max number of parts has been exceeded. " . + "Max = " . self::PART_MAX_NUM + ); + } + + if ($isSeekable && $partNo > $partsCount) { + return Create::rejectionFor( + "The current part `$partNo` is over " + . "the expected number of parts `$partsCount`" + ); + } + + $command = $this->s3Client->getCommand( + 'UploadPart', + $uploadPartCommandArgs + ); + + // Advance if behind + if ($bytesRead < $this->body->tell()) { + $this->body->seek($bytesRead); + } + + yield $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) + use ($command, $partBody) { + $partBody->close(); + // To make sure we don't continue when a failure occurred + if ($this->currentSnapshot->getReason() !== null) { + throw $this->currentSnapshot->getReason(); + } + + $this->collectPart( + $result, + $command + ); + // Part Upload Completed Event + $this->partCompleted( + $command['ContentLength'], + $command->toArray() + ); + })->otherwise(function (Throwable $e) use ($partBody) { + $partBody->close(); + $this->partFailed($e); + + throw $e; + }); + } + } + + /** + * @return int + */ + protected function getTotalSize(): int + { + if ($this->calculatedObjectSize > 0) { + return $this->calculatedObjectSize; + } + + return $this->body->getSize(); + } + + /** + * @param ResultInterface $result + * + * @return UploadResult + */ + protected function createResponse(ResultInterface $result): UploadResult + { + return new UploadResult( + $result->toArray() + ); + } + + /** + * @param StreamInterface $stream + * @param array $data + * + * @return StreamInterface + */ + private function decorateWithHashes( + StreamInterface $stream, + array &$data + ): StreamInterface + { + // Decorate source with a hashing stream + $hash = new PhpHash('sha256'); + return new HashingStream($stream, $hash, function ($result) use (&$data) { + $data['ContentSHA256'] = bin2hex($result); + }); + } + + /** + * Filters a provided checksum if one was provided. + * + * @param array $requestArgs + * + * @return string|null + */ + private static function filterChecksum(array $requestArgs):? string + { + foreach (self::$supportedAlgorithms as $algorithm) { + if (isset($requestArgs[$algorithm])) { + return $algorithm; + } + } + + return null; + } +} diff --git a/src/S3/S3Transfer/PartGetMultipartDownloader.php b/src/S3/S3Transfer/PartGetMultipartDownloader.php new file mode 100644 index 0000000000..74e7efc08f --- /dev/null +++ b/src/S3/S3Transfer/PartGetMultipartDownloader.php @@ -0,0 +1,62 @@ +currentPartNo === 0) { + $this->currentPartNo = 1; + } else { + $this->currentPartNo++; + } + + $nextRequestArgs = $this->downloadRequestArgs; + $nextRequestArgs['PartNumber'] = $this->currentPartNo; + if ($this->config['response_checksum_validation'] === 'when_supported') { + $nextRequestArgs['ChecksumMode'] = 'ENABLED'; + } + + if (!empty($this->eTag)) { + $nextRequestArgs['IfMatch'] = $this->eTag; + } + + return $this->s3Client->getCommand( + self::GET_OBJECT_COMMAND, + $nextRequestArgs + ); + } + + /** + * @inheritDoc + * + * @param Result $result + * + * @return void + */ + protected function computeObjectDimensions(ResultInterface $result): void + { + if (!empty($result['PartsCount'])) { + $this->objectPartsCount = $result['PartsCount']; + } else { + $this->objectPartsCount = 1; + } + + $this->objectSizeInBytes = $this->computeObjectSizeFromContentRange( + $result['ContentRange'] ?? "" + ); + } +} diff --git a/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php b/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php new file mode 100644 index 0000000000..2063c444e6 --- /dev/null +++ b/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php @@ -0,0 +1,47 @@ + ColoredTransferProgressBarFormat::BLACK_COLOR_CODE, + ]; + } +} diff --git a/src/S3/S3Transfer/Progress/ConsoleProgressBar.php b/src/S3/S3Transfer/Progress/ConsoleProgressBar.php new file mode 100644 index 0000000000..4e4c869464 --- /dev/null +++ b/src/S3/S3Transfer/Progress/ConsoleProgressBar.php @@ -0,0 +1,103 @@ +progressBarChar = $progressBarChar; + $this->progressBarWidth = min( + $progressBarWidth, + self::MAX_PROGRESS_BAR_WIDTH + ); + $this->percentCompleted = $percentCompleted; + $this->progressBarFormat = $progressBarFormat; + } + + /** + * @return string + */ + public function getProgressBarChar(): string + { + return $this->progressBarChar; + } + + /** + * @return int + */ + public function getProgressBarWidth(): int + { + return $this->progressBarWidth; + } + + /** + * @return int + */ + public function getPercentCompleted(): int + { + return $this->percentCompleted; + } + + /** + * @return ProgressBarFormat + */ + public function getProgressBarFormat(): ProgressBarFormat + { + return $this->progressBarFormat; + } + + /** + * Set current progress percent. + * + * @param int $percent + * + * @return void + */ + public function setPercentCompleted(int $percent): void + { + $this->percentCompleted = max(0, min(100, $percent)); + } + + /** + * @inheritDoc + */ + public function render(): string + { + $filledWidth = (int) round(($this->progressBarWidth * $this->percentCompleted) / 100); + $progressBar = str_repeat($this->progressBarChar, $filledWidth) + . str_repeat(' ', $this->progressBarWidth - $filledWidth); + + // Common arguments + $this->progressBarFormat->setArg('progress_bar', $progressBar); + $this->progressBarFormat->setArg('percent', $this->percentCompleted); + + return $this->progressBarFormat->format(); + } +} diff --git a/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php b/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php new file mode 100644 index 0000000000..e080cee32f --- /dev/null +++ b/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php @@ -0,0 +1,40 @@ +singleProgressTrackers = $singleProgressTrackers; + $this->output = $output; + $this->transferCount = $transferCount; + $this->completed = $completed; + $this->failed = $failed; + $this->progressBarFactory = $progressBarFactory; + } + + /** + * @return array + */ + public function getSingleProgressTrackers(): array + { + return $this->singleProgressTrackers; + } + + /** + * @return mixed + */ + public function getOutput(): mixed + { + return $this->output; + } + + /** + * @return int + */ + public function getTransferCount(): int + { + return $this->transferCount; + } + + /** + * @return int + */ + public function getCompleted(): int + { + return $this->completed; + } + + /** + * @return int + */ + public function getFailed(): int + { + return $this->failed; + } + + /** + * @return ProgressBarFactoryInterface|Closure|null + */ + public function getProgressBarFactory(): ProgressBarFactoryInterface|Closure|null + { + return $this->progressBarFactory; + } + + /** + * @inheritDoc + */ + public function transferInitiated(array $context): void + { + $this->transferCount++; + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; + if (isset($this->singleProgressTrackers[$snapshot->getIdentifier()])) { + $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; + } else { + if ($this->progressBarFactory === null) { + $progressTracker = new SingleProgressTracker( + output: $this->output, + clear: false, + showProgressOnUpdate: false, + ); + } else { + $progressBarFactoryFn = $this->progressBarFactory; + $progressTracker = new SingleProgressTracker( + progressBar: $progressBarFactoryFn(), + output: $this->output, + clear: false, + showProgressOnUpdate: false, + ); + } + + $this->singleProgressTrackers[$snapshot->getIdentifier()] = $progressTracker; + } + + $progressTracker->transferInitiated($context); + $this->showProgress(); + } + + /** + * @inheritDoc + */ + public function bytesTransferred(array $context): void + { + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; + $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; + $progressTracker->bytesTransferred($context); + $this->showProgress(); + } + + /** + * @inheritDoc + */ + public function transferComplete(array $context): void + { + $this->completed++; + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; + $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; + $progressTracker->transferComplete($context); + $this->showProgress(); + } + + /** + * @inheritDoc + */ + public function transferFail(array $context): void + { + $this->failed++; + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; + $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; + $progressTracker->transferFail($context); + $this->showProgress(); + } + + /** + * @inheritDoc + */ + public function showProgress(): void + { + fwrite($this->output, self::CLEAR_ASCII_CODE); + $percentsSum = 0; + foreach ($this->singleProgressTrackers as $_ => $progressTracker) { + $progressTracker->showProgress(); + $percentsSum += $progressTracker->getProgressBar()->getPercentCompleted(); + } + + $allProgressBarWidth = ConsoleProgressBar::DEFAULT_PROGRESS_BAR_WIDTH; + if (count($this->singleProgressTrackers) !== 0) { + $firstKey = array_key_first($this->singleProgressTrackers); + $allProgressBarWidth = $this->singleProgressTrackers[$firstKey] + ->getProgressBar()->getProgressBarWidth(); + } + + $percent = (int) floor($percentsSum / $this->transferCount); + $multiProgressBarFormat = new MultiProgressBarFormat(); + $multiProgressBarFormat->setArgs([ + 'completed' => $this->completed, + 'failed' => $this->failed, + 'total' => $this->transferCount, + ]); + $allTransferProgressBar = new ConsoleProgressBar( + progressBarWidth: $allProgressBarWidth, + percentCompleted: $percent, + progressBarFormat: $multiProgressBarFormat + ); + fwrite( + $this->output, + sprintf( + "\n%s\n", + str_repeat( + '-', + $allTransferProgressBar->getProgressBarWidth() + ) + ) + ); + fwrite( + $this->output, + sprintf( + "%s\n", + $allTransferProgressBar->render(), + ) + ); + } +} diff --git a/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php b/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php new file mode 100644 index 0000000000..f75f6ca89e --- /dev/null +++ b/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php @@ -0,0 +1,28 @@ +args = $args; + } + + public function getArgs(): array + { + return $this->args; + } + + /** + * To set multiple arguments at once. + * It does not override all the values, instead + * it adds the arguments individually and if a value + * already exists then that value will be overridden. + * + * @param array $args + * + * @return void + */ + public function setArgs(array $args): void + { + foreach ($args as $key => $value) { + $this->args[$key] = $value; + } + } + + /** + * @param string $key + * @param mixed $value + * + * @return void + */ + public function setArg(string $key, mixed $value): void + { + $this->args[$key] = $value; + } + + /** + * @return string + */ + public function format(): string + { + $parameters = $this->getFormatParameters(); + $defaultParameterValues = $this->getFormatDefaultParameterValues(); + foreach ($parameters as $param) { + if (!array_key_exists($param, $this->args)) { + $this->args[$param] = $defaultParameterValues[$param] ?? ''; + } + } + + $replacements = []; + foreach ($parameters as $param) { + $replacements["|$param|"] = $this->args[$param] ?? ''; + } + + return strtr($this->getFormatTemplate(), $replacements); + } + + /** + * @return string + */ + abstract public function getFormatTemplate(): string; + + /** + * @return array + */ + abstract public function getFormatParameters(): array; + + /** + * @return array + */ + abstract protected function getFormatDefaultParameterValues(): array; +} diff --git a/src/S3/S3Transfer/Progress/ProgressBarInterface.php b/src/S3/S3Transfer/Progress/ProgressBarInterface.php new file mode 100644 index 0000000000..b64545eb59 --- /dev/null +++ b/src/S3/S3Transfer/Progress/ProgressBarInterface.php @@ -0,0 +1,31 @@ +progressBar = $progressBar; + if (get_resource_type($output) !== 'stream') { + throw new \InvalidArgumentException("The type for $output must be a stream"); + } + $this->output = $output; + $this->clear = $clear; + $this->currentSnapshot = $currentSnapshot; + $this->showProgressOnUpdate = $showProgressOnUpdate; + } + + /** + * @return ProgressBarInterface + */ + public function getProgressBar(): ProgressBarInterface + { + return $this->progressBar; + } + + /** + * @return mixed + */ + public function getOutput(): mixed + { + return $this->output; + } + + /** + * @return bool + */ + public function isClear(): bool + { + return $this->clear; + } + + /** + * @return TransferProgressSnapshot|null + */ + public function getCurrentSnapshot(): ?TransferProgressSnapshot + { + return $this->currentSnapshot; + } + + /** + * @return bool + */ + public function isShowProgressOnUpdate(): bool + { + return $this->showProgressOnUpdate; + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferInitiated(array $context): void + { + $this->currentSnapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; + $progressFormat = $this->progressBar->getProgressBarFormat(); + // Probably a common argument + $progressFormat->setArg( + 'object_name', + $this->currentSnapshot->getIdentifier() + ); + + $this->updateProgressBar(); + } + + /** + * @inheritDoc + * + * @return void + */ + public function bytesTransferred(array $context): void + { + $this->currentSnapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; + $progressFormat = $this->progressBar->getProgressBarFormat(); + if ($progressFormat instanceof ColoredTransferProgressBarFormat) { + $progressFormat->setArg( + 'color_code', + ColoredTransferProgressBarFormat::BLUE_COLOR_CODE + ); + } + + $this->updateProgressBar(); + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferComplete(array $context): void + { + $this->currentSnapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; + $progressFormat = $this->progressBar->getProgressBarFormat(); + if ($progressFormat instanceof ColoredTransferProgressBarFormat) { + $progressFormat->setArg( + 'color_code', + ColoredTransferProgressBarFormat::GREEN_COLOR_CODE + ); + } + + $this->updateProgressBar( + $this->currentSnapshot->getTotalBytes() === 0 + ); + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferFail(array $context): void + { + $this->currentSnapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; + $progressFormat = $this->progressBar->getProgressBarFormat(); + if ($progressFormat instanceof ColoredTransferProgressBarFormat) { + $progressFormat->setArg( + 'color_code', + ColoredTransferProgressBarFormat::RED_COLOR_CODE + ); + $progressFormat->setArg( + 'message', + $context[TransferListener::REASON_KEY] + ); + } + + $this->updateProgressBar(); + } + + /** + * Updates the progress bar with the transfer snapshot + * and also call showProgress. + * + * @param bool $forceCompletion To force the progress bar to be + * completed. This is useful for files where its size is zero, + * for which a ratio will return zero, and hence the percent + * will be zero. + * + * @return void + */ + private function updateProgressBar( + bool $forceCompletion = false + ): void + { + if (!$forceCompletion) { + $this->progressBar->setPercentCompleted( + ((int)floor($this->currentSnapshot->ratioTransferred() * 100)) + ); + } else { + $this->progressBar->setPercentCompleted(100); + } + + $this->progressBar->getProgressBarFormat()->setArgs([ + 'transferred' => $this->currentSnapshot->getTransferredBytes(), + 'to_be_transferred' => $this->currentSnapshot->getTotalBytes(), + 'unit' => 'B', + ]); + // Display progress + if ($this->showProgressOnUpdate) { + $this->showProgress(); + } + } + + /** + * @inheritDoc + * + * @return void + */ + public function showProgress(): void + { + if ($this->currentSnapshot === null) { + throw new ProgressTrackerException( + "There is not snapshot to show progress for." + ); + } + + if ($this->clear) { + fwrite($this->output, "\033[2J\033[H"); + } + + fwrite($this->output, sprintf( + "\r\n%s", + $this->progressBar->render() + )); + fflush($this->output); + } +} diff --git a/src/S3/S3Transfer/Progress/TransferListener.php b/src/S3/S3Transfer/Progress/TransferListener.php new file mode 100644 index 0000000000..d121a2761e --- /dev/null +++ b/src/S3/S3Transfer/Progress/TransferListener.php @@ -0,0 +1,51 @@ +listeners = $listeners; + } + + /** + * @param TransferListener $listener + * + * @return void + */ + public function addListener(TransferListener $listener): void + { + $this->listeners[] = $listener; + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferInitiated(array $context): void + { + foreach ($this->listeners as $listener) { + $listener->transferInitiated($context); + } + } + + /** + * @inheritDoc + * + * @return void + */ + public function bytesTransferred(array $context): void + { + foreach ($this->listeners as $listener) { + $listener->bytesTransferred($context); + } + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferComplete(array $context): void + { + foreach ($this->listeners as $listener) { + $listener->transferComplete($context); + } + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferFail(array $context): void + { + foreach ($this->listeners as $listener) { + $listener->transferFail($context); + } + } +} diff --git a/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php b/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php new file mode 100644 index 0000000000..9e3c3c729a --- /dev/null +++ b/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php @@ -0,0 +1,38 @@ +identifier = $identifier; + $this->transferredBytes = $transferredBytes; + $this->totalBytes = $totalBytes; + $this->response = $response; + $this->reason = $reason; + } + + public function getIdentifier(): string + { + return $this->identifier; + } + + /** + * @return int + */ + public function getTransferredBytes(): int + { + return $this->transferredBytes; + } + + /** + * @return int + */ + public function getTotalBytes(): int + { + return $this->totalBytes; + } + + /** + * @return array|null + */ + public function getResponse(): array|null + { + return $this->response; + } + + /** + * @return float + */ + public function ratioTransferred(): float + { + if ($this->totalBytes === 0) { + // Unable to calculate ratio + return 0; + } + + return $this->transferredBytes / $this->totalBytes; + } + + /** + * @return Throwable|string|null + */ + public function getReason(): Throwable|string|null + { + return $this->reason; + } +} diff --git a/src/S3/S3Transfer/RangeGetMultipartDownloader.php b/src/S3/S3Transfer/RangeGetMultipartDownloader.php new file mode 100644 index 0000000000..df9634378e --- /dev/null +++ b/src/S3/S3Transfer/RangeGetMultipartDownloader.php @@ -0,0 +1,76 @@ +currentPartNo === 0) { + $this->currentPartNo = 1; + } else { + $this->currentPartNo++; + } + + $nextRequestArgs = $this->downloadRequestArgs; + $partSize = $this->config['target_part_size_bytes']; + $from = ($this->currentPartNo - 1) * $partSize; + $to = ($this->currentPartNo * $partSize) - 1; + + if ($this->objectSizeInBytes !== 0) { + $to = min($this->objectSizeInBytes, $to); + } + + $nextRequestArgs['Range'] = "bytes=$from-$to"; + + if ($this->config['response_checksum_validation'] === 'when_supported') { + $nextRequestArgs['ChecksumMode'] = 'ENABLED'; + } + + if (!empty($this->eTag)) { + $nextRequestArgs['IfMatch'] = $this->eTag; + } + + return $this->s3Client->getCommand( + self::GET_OBJECT_COMMAND, + $nextRequestArgs + ); + } + + /** + * @inheritDoc + * + * @param Result $result + * + * @return void + */ + protected function computeObjectDimensions(ResultInterface $result): void + { + // Assign object size just if needed. + if ($this->objectSizeInBytes === 0) { + $this->objectSizeInBytes = $this->computeObjectSizeFromContentRange( + $result['ContentRange'] ?? "" + ); + } + + $partSize = $this->config['target_part_size_bytes']; + if ($this->objectSizeInBytes > $partSize) { + $this->objectPartsCount = intval( + ceil($this->objectSizeInBytes / $partSize) + ); + } else { + // Single download since partSize will be set to full object size. + $this->objectPartsCount = 1; + $this->currentPartNo = 1; + } + } +} diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php new file mode 100644 index 0000000000..b14440fe71 --- /dev/null +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -0,0 +1,807 @@ +config = S3TransferManagerConfig::fromArray($config ?? []); + } else { + $this->config = $config; + } + + if ($s3Client === null) { + $this->s3Client = $this->defaultS3Client(); + } else { + $this->s3Client = $s3Client; + } + } + + /** + * @return S3ClientInterface + */ + public function getS3Client(): S3ClientInterface + { + return $this->s3Client; + } + + /** + * @return S3TransferManagerConfig + */ + public function getConfig(): S3TransferManagerConfig + { + return $this->config; + } + + /** + * @param UploadRequest $uploadRequest + * + * @return PromiseInterface + */ + public function upload(UploadRequest $uploadRequest): PromiseInterface + { + // Make sure it is a valid in path in case of a string + $uploadRequest->validateSource(); + + // Valid required parameters + $uploadRequest->validateRequiredParameters(); + + $uploadRequest->updateConfigWithDefaults( + $this->config->toArray() + ); + + $uploadRequest->validateConfig(); + + $config = $uploadRequest->getConfig(); + + // Validate progress tracker + $progressTracker = $uploadRequest->getProgressTracker(); + if ($progressTracker === null + && ($config['track_progress'] + ?? $this->config->isTrackProgress())) { + $progressTracker = new SingleProgressTracker(); + } + + // Append progress tracker to listeners if not null + $listeners = $uploadRequest->getListeners(); + if ($progressTracker !== null) { + $listeners[] = $progressTracker; + } + + $listenerNotifier = new TransferListenerNotifier($listeners); + + // Validate multipart upload threshold + $mupThreshold = $config['multipart_upload_threshold_bytes'] + ?? $this->config->getMultipartUploadThresholdBytes(); + if ($mupThreshold < AbstractMultipartUploader::PART_MIN_SIZE) { + throw new InvalidArgumentException( + "The provided config `multipart_upload_threshold_bytes`" + ."must be greater than or equal to " . AbstractMultipartUploader::PART_MIN_SIZE + ); + } + + if ($this->requiresMultipartUpload($uploadRequest->getSource(), $mupThreshold)) { + return $this->tryMultipartUpload( + $uploadRequest, + $listenerNotifier + ); + } + + return $this->trySingleUpload( + $uploadRequest->getSource(), + $uploadRequest->getUploadRequestArgs(), + $listenerNotifier + ); + } + + /** + * @param UploadDirectoryRequest $uploadDirectoryRequest + * + * @return PromiseInterface + */ + public function uploadDirectory( + UploadDirectoryRequest $uploadDirectoryRequest, + ): PromiseInterface + { + $uploadDirectoryRequest->validateSourceDirectory(); + + $uploadDirectoryRequest->updateConfigWithDefaults( + $this->config->toArray() + ); + + $uploadDirectoryRequest->validateConfig(); + + $config = $uploadDirectoryRequest->getConfig(); + + $filter = $config['filter'] ?? null; + $uploadObjectRequestModifier = $config['upload_object_request_modifier'] + ?? null; + $failurePolicyCallback = $config['failure_policy'] ?? null; + + $sourceDirectory = $uploadDirectoryRequest->getSourceDirectory(); + $dirIterator = new RecursiveDirectoryIterator( + $sourceDirectory + ); + + $flags = FilesystemIterator::SKIP_DOTS; + if ($config['follow_symbolic_links'] ?? false) { + $flags |= FilesystemIterator::FOLLOW_SYMLINKS; + } + + $dirIterator->setFlags($flags); + + if ($config['recursive'] ?? false) { + $dirIterator = new RecursiveIteratorIterator( + $dirIterator, + RecursiveIteratorIterator::SELF_FIRST + ); + if (isset($config['max_depth'])) { + $dirIterator->setMaxDepth($config['max_depth']); + } + } + + $dirVisited = []; + $files = filter( + $dirIterator, + function ($file) use ($filter, &$dirVisited) { + if (is_dir($file)) { + // To avoid circular symbolic links traversal + $dirRealPath = realpath($file); + if ($dirRealPath !== false) { + if ($dirVisited[$dirRealPath] ?? false) { + throw new S3TransferException( + "A circular symbolic link traversal has been detected at $file -> $dirRealPath" + ); + } + + $dirVisited[$dirRealPath] = true; + } + } + + if ($filter !== null) { + return !is_dir($file) && $filter($file); + } + + return !is_dir($file); + } + ); + + $objectsUploaded = 0; + $objectsFailed = 0; + $promises = []; + $baseDir = rtrim($sourceDirectory, '/') . DIRECTORY_SEPARATOR; + $delimiter = $config['s3_delimiter'] ?? '/'; + $s3Prefix = $config['s3_prefix'] ?? ''; + if ($s3Prefix !== '' && !str_ends_with($s3Prefix, '/')) { + $s3Prefix .= '/'; + } + $targetBucket = $uploadDirectoryRequest->getTargetBucket(); + $progressTracker = $uploadDirectoryRequest->getProgressTracker(); + if ($progressTracker === null + && ($config['track_progress'] ?? $this->config->isTrackProgress())) { + $progressTracker = new MultiProgressTracker(); + } + foreach ($files as $file) { + $relativePath = substr($file, strlen($baseDir)); + if (str_contains($relativePath, $delimiter) && $delimiter !== '/') { + throw new S3TransferException( + "The filename `$relativePath` must not contain the provided delimiter `$delimiter`" + ); + } + $objectKey = $s3Prefix.$relativePath; + $objectKey = str_replace( + DIRECTORY_SEPARATOR, + $delimiter, + $objectKey + ); + $uploadRequestArgs = $uploadDirectoryRequest->getUploadRequestArgs(); + $uploadRequestArgs['Bucket'] = $targetBucket; + $uploadRequestArgs['Key'] = $objectKey; + + if ($uploadObjectRequestModifier !== null) { + $uploadObjectRequestModifier($uploadRequestArgs); + } + + $promises[] = $this->upload( + new UploadRequest( + $file, + $uploadRequestArgs, + $config, + array_map( + fn($listener) => clone $listener, + $uploadDirectoryRequest->getListeners() + ), + $progressTracker + ) + )->then(function (UploadResult $response) use (&$objectsUploaded) { + $objectsUploaded++; + + return $response; + })->otherwise(function (Throwable $reason) use ( + $targetBucket, + $sourceDirectory, + $failurePolicyCallback, + $uploadRequestArgs, + &$objectsUploaded, + &$objectsFailed + ) { + $objectsFailed++; + if($failurePolicyCallback !== null) { + call_user_func( + $failurePolicyCallback, + $uploadRequestArgs, + [ + "source_directory" => $sourceDirectory, + "bucket_to" => $targetBucket, + ], + $reason, + new UploadDirectoryResult( + $objectsUploaded, + $objectsFailed + ) + ); + + return; + } + + throw $reason; + }); + } + + $maxConcurrency = $config['max_concurrency'] + ?? UploadDirectoryRequest::DEFAULT_MAX_CONCURRENCY; + + return Each::ofLimitAll($promises, $maxConcurrency) + ->then(function () use (&$objectsUploaded, &$objectsFailed) { + return new UploadDirectoryResult($objectsUploaded, $objectsFailed); + })->otherwise(function (Throwable $reason) + use (&$objectsUploaded, &$objectsFailed) { + return new UploadDirectoryResult( + $objectsUploaded, + $objectsFailed, + $reason + ); + }); + } + + /** + * @param DownloadRequest $downloadRequest + * + * @return PromiseInterface + */ + public function download(DownloadRequest $downloadRequest): PromiseInterface + { + $sourceArgs = $downloadRequest->normalizeSourceAsArray(); + $getObjectRequestArgs = $downloadRequest->getObjectRequestArgs(); + + $downloadRequest->updateConfigWithDefaults($this->config->toArray()); + + $downloadRequest->validateConfig(); + + $config = $downloadRequest->getConfig(); + + $progressTracker = $downloadRequest->getProgressTracker(); + if ($progressTracker === null && $config['track_progress']) { + $progressTracker = new SingleProgressTracker(); + } + + $listeners = $downloadRequest->getListeners(); + if ($progressTracker !== null) { + $listeners[] = $progressTracker; + } + + // Build listener notifier for notifying listeners + $listenerNotifier = new TransferListenerNotifier($listeners); + + // Assign source + foreach ($sourceArgs as $key => $value) { + $getObjectRequestArgs[$key] = $value; + } + + return $this->tryMultipartDownload( + $getObjectRequestArgs, + $config, + $downloadRequest->getDownloadHandler(), + $listenerNotifier, + ); + } + + /** + * @param DownloadFileRequest $downloadFileRequest + * + * @return PromiseInterface + */ + public function downloadFile( + DownloadFileRequest $downloadFileRequest + ): PromiseInterface + { + return $this->download($downloadFileRequest->getDownloadRequest()); + } + + /** + * @param DownloadDirectoryRequest $downloadDirectoryRequest + * + * @return PromiseInterface + */ + public function downloadDirectory( + DownloadDirectoryRequest $downloadDirectoryRequest + ): PromiseInterface + { + $downloadDirectoryRequest->validateDestinationDirectory(); + $destinationDirectory = $downloadDirectoryRequest->getDestinationDirectory(); + $sourceBucket = $downloadDirectoryRequest->getSourceBucket(); + $progressTracker = $downloadDirectoryRequest->getProgressTracker(); + + $downloadDirectoryRequest->updateConfigWithDefaults( + $this->config->toArray() + ); + + $downloadDirectoryRequest->validateConfig(); + + $config = $downloadDirectoryRequest->getConfig(); + if ($progressTracker === null && $config['track_progress']) { + $progressTracker = new MultiProgressTracker(); + } + + $listArgs = [ + 'Bucket' => $sourceBucket, + ] + ($config['list_objects_v2_args'] ?? []); + + $s3Prefix = $config['s3_prefix'] ?? null; + if (empty($listArgs['Prefix']) && $s3Prefix !== null) { + $listArgs['Prefix'] = $s3Prefix; + } + + // MUST BE NULL + $listArgs['Delimiter'] = null; + + $objects = $this->s3Client + ->getPaginator('ListObjectsV2', $listArgs) + ->search('Contents[].Key'); + + $filter = $config['filter'] ?? null; + $objects = filter($objects, function (string $key) use ($filter) { + if ($filter !== null) { + return call_user_func($filter, $key) && !str_ends_with($key, "/"); + } + + return !str_ends_with($key, "/"); + }); + $objects = map($objects, function (string $key) use ($sourceBucket) { + return self::formatAsS3URI($sourceBucket, $key); + }); + + $downloadObjectRequestModifier = $config['download_object_request_modifier'] + ?? null; + $failurePolicyCallback = $config['failure_policy'] ?? null; + + $s3Delimiter = '/'; + $objectsDownloaded = 0; + $objectsFailed = 0; + $promises = []; + foreach ($objects as $object) { + $bucketAndKeyArray = self::s3UriAsBucketAndKey($object); + $objectKey = $bucketAndKeyArray['Key']; + if ($s3Prefix !== null && str_contains($objectKey, $s3Delimiter)) { + if (!str_ends_with($s3Prefix, $s3Delimiter)) { + $s3Prefix = $s3Prefix.$s3Delimiter; + } + + $objectKey = substr($objectKey, strlen($s3Prefix)); + } + + // CONVERT THE KEY DIR SEPARATOR TO OS BASED DIR SEPARATOR + if (DIRECTORY_SEPARATOR !== $s3Delimiter) { + $objectKey = str_replace( + $s3Delimiter, + DIRECTORY_SEPARATOR, + $objectKey + ); + } + + $destinationFile = $destinationDirectory . DIRECTORY_SEPARATOR . $objectKey; + if ($this->resolvesOutsideTargetDirectory($destinationFile, $objectKey)) { + throw new S3TransferException( + "Cannot download key $objectKey " + ."its relative path resolves outside the parent directory." + ); + } + + $requestArgs = $downloadDirectoryRequest->getDownloadRequestArgs(); + foreach ($bucketAndKeyArray as $key => $value) { + $requestArgs[$key] = $value; + } + if ($downloadObjectRequestModifier !== null) { + call_user_func($downloadObjectRequestModifier, $requestArgs); + } + + $promises[] = $this->downloadFile( + new DownloadFileRequest( + destination: $destinationFile, + failsWhenDestinationExists: $config['fails_when_destination_exists'] ?? false, + downloadRequest: new DownloadRequest( + source: null, // Source has been provided in the request args + downloadRequestArgs: $requestArgs, + config: [ + 'target_part_size_bytes' => $config['target_part_size_bytes'] ?? 0, + ], + downloadHandler: null, + listeners: array_map( + fn($listener) => clone $listener, + $downloadDirectoryRequest->getListeners() + ), + progressTracker: $progressTracker, + ) + ), + )->then(function () use ( + &$objectsDownloaded + ) { + $objectsDownloaded++; + })->otherwise(function (Throwable $reason) use ( + $sourceBucket, + $destinationDirectory, + $failurePolicyCallback, + &$objectsDownloaded, + &$objectsFailed, + $requestArgs + ) { + $objectsFailed++; + if ($failurePolicyCallback !== null) { + call_user_func( + $failurePolicyCallback, + $requestArgs, + [ + "destination_directory" => $destinationDirectory, + "bucket" => $sourceBucket, + ], + $reason, + new DownloadDirectoryResult( + $objectsDownloaded, + $objectsFailed + ) + ); + + return; + } + + throw $reason; + }); + } + + $maxConcurrency = $config['max_concurrency'] + ?? DownloadDirectoryRequest::DEFAULT_MAX_CONCURRENCY; + + return Each::ofLimitAll($promises, $maxConcurrency) + ->then(function () use (&$objectsFailed, &$objectsDownloaded) { + return new DownloadDirectoryResult( + $objectsDownloaded, + $objectsFailed + ); + })->otherwise(function (Throwable $reason) + use (&$objectsFailed, &$objectsDownloaded) { + return new DownloadDirectoryResult( + $objectsDownloaded, + $objectsFailed, + $reason + ); + }); + } + + /** + * Tries an object multipart download. + * + * @param array $getObjectRequestArgs + * @param array $config + * @param DownloadHandler $downloadHandler + * @param TransferListenerNotifier|null $listenerNotifier + * + * @return PromiseInterface + */ + private function tryMultipartDownload( + array $getObjectRequestArgs, + array $config, + DownloadHandler $downloadHandler, + ?TransferListenerNotifier $listenerNotifier = null, + ): PromiseInterface + { + $downloaderClassName = AbstractMultipartDownloader::chooseDownloaderClass( + strtolower($config['multipart_download_type']) + ); + $multipartDownloader = new $downloaderClassName( + $this->s3Client, + $getObjectRequestArgs, + $config, + $downloadHandler, + listenerNotifier: $listenerNotifier, + ); + + return $multipartDownloader->promise(); + } + + /** + * @param string|StreamInterface $source + * @param array $requestArgs + * @param TransferListenerNotifier|null $listenerNotifier + * + * @return PromiseInterface + */ + private function trySingleUpload( + string|StreamInterface $source, + array $requestArgs, + ?TransferListenerNotifier $listenerNotifier = null + ): PromiseInterface + { + if (is_string($source) && is_readable($source)) { + $requestArgs['SourceFile'] = $source; + $objectSize = filesize($source); + } elseif ($source instanceof StreamInterface && $source->isSeekable()) { + $requestArgs['Body'] = $source; + $objectSize = $source->getSize(); + } else { + throw new S3TransferException( + "Unable to process upload request due to the type of the source" + ); + } + + if (!empty($listenerNotifier)) { + $listenerNotifier->transferInitiated( + [ + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + $requestArgs['Key'], + 0, + $objectSize, + ), + ] + ); + + $command = $this->s3Client->getCommand('PutObject', $requestArgs); + return $this->s3Client->executeAsync($command)->then( + function (ResultInterface $result) + use ($objectSize, $listenerNotifier, $requestArgs) { + $listenerNotifier->bytesTransferred( + [ + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + $requestArgs['Key'], + $objectSize, + $objectSize, + ), + ] + ); + + $listenerNotifier->transferComplete( + [ + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + $requestArgs['Key'], + $objectSize, + $objectSize, + $result->toArray() + ), + ] + ); + + return new UploadResult( + $result->toArray() + ); + } + )->otherwise(function (Throwable $reason) + use ($objectSize, $requestArgs, $listenerNotifier) { + $listenerNotifier->transferFail( + [ + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + $requestArgs['Key'], + 0, + $objectSize, + ), + 'reason' => $reason, + ] + ); + + throw $reason; + }); + } + + $command = $this->s3Client->getCommand('PutObject', $requestArgs); + + return $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) { + return new UploadResult($result->toArray()); + }); + } + + /** + * @param UploadRequest $uploadRequest + * @param TransferListenerNotifier|null $listenerNotifier + * + * @return PromiseInterface + */ + private function tryMultipartUpload( + UploadRequest $uploadRequest, + ?TransferListenerNotifier $listenerNotifier = null, + ): PromiseInterface + { + return (new MultipartUploader( + $this->s3Client, + $uploadRequest->getUploadRequestArgs(), + $uploadRequest->getSource(), + $uploadRequest->getConfig(), + listenerNotifier: $listenerNotifier, + ))->promise(); + } + + /** + * @param string|StreamInterface $source + * @param int $mupThreshold + * + * @return bool + */ + private function requiresMultipartUpload( + string|StreamInterface $source, + int $mupThreshold + ): bool + { + if (is_string($source) && is_readable($source)) { + return filesize($source) >= $mupThreshold; + } elseif ($source instanceof StreamInterface) { + // When the stream's size is unknown then we could try a multipart upload. + if (empty($source->getSize())) { + return true; + } + + return $source->getSize() >= $mupThreshold; + } + + throw new S3TransferException( + "Unable to determine if a multipart is required" + ); + } + + /** + * Returns a default instance of S3Client. + * + * @return S3Client + */ + private function defaultS3Client(): S3ClientInterface + { + return new S3Client([ + 'region' => $this->config->getDefaultRegion(), + ]); + } + + /** + * Validates a string value is a valid S3 URI. + * Valid S3 URI Example: S3://mybucket.dev/myobject.txt + * + * @param string $uri + * + * @return bool + */ + public static function isValidS3URI(string $uri): bool + { + // in the expression `substr($uri, 5)))` the 5 belongs to the size of `s3://`. + return str_starts_with(strtolower($uri), 's3://') + && count(explode('/', substr($uri, 5))) > 1; + } + + /** + * Converts a S3 URI into an array with a Bucket and Key + * properties set. + * + * @param string $uri: The S3 URI. + * + * @return array + */ + public static function s3UriAsBucketAndKey(string $uri): array + { + $errorMessage = "Invalid URI: `$uri` provided. \nA valid S3 URI looks as `s3://bucket/key`"; + if (!self::isValidS3URI($uri)) { + throw new InvalidArgumentException($errorMessage); + } + + $path = substr($uri, 5); // without s3:// + $parts = explode('/', $path, 2); + + if (count($parts) < 2) { + throw new InvalidArgumentException($errorMessage); + } + + return [ + 'Bucket' => $parts[0], + 'Key' => $parts[1], + ]; + } + + /** + * @param string $bucket + * @param string $key + * + * @return string + */ + private static function formatAsS3URI(string $bucket, string $key): string + { + return "s3://$bucket/$key"; + } + + /** + * @param string $sink + * @param string $objectKey + * + * @return bool + */ + private function resolvesOutsideTargetDirectory( + string $sink, + string $objectKey + ): bool + { + $resolved = []; + $sections = explode('/', $sink); + $targetSectionsLength = count(explode('/', $objectKey)); + $targetSections = array_slice($sections, -($targetSectionsLength + 1)); + $targetDirectory = $targetSections[0]; + + foreach ($targetSections as $section) { + if ($section === '.' || $section === '') { + continue; + } + if ($section === '..') { + array_pop($resolved); + if (empty($resolved) || $resolved[0] !== $targetDirectory) { + return true; + } + } else { + $resolved []= $section; + } + } + + return false; + } +} diff --git a/src/S3/S3Transfer/Utils/DownloadHandler.php b/src/S3/S3Transfer/Utils/DownloadHandler.php new file mode 100644 index 0000000000..8dd7f00a97 --- /dev/null +++ b/src/S3/S3Transfer/Utils/DownloadHandler.php @@ -0,0 +1,18 @@ +destination = $destination; + $this->failsWhenDestinationExists = $failsWhenDestinationExists; + $this->temporaryDestination = ""; + } + + /** + * @return string + */ + public function getDestination(): string + { + return $this->destination; + } + + /** + * @return bool + */ + public function isFailsWhenDestinationExists(): bool + { + return $this->failsWhenDestinationExists; + } + + /** + * @param array $context + * + * @return void + */ + public function transferInitiated(array $context): void + { + if ($this->failsWhenDestinationExists && file_exists($this->destination)) { + throw new FileDownloadException( + "The destination '$this->destination' already exists." + ); + } elseif (is_dir($this->destination)) { + throw new FileDownloadException( + "The destination '$this->destination' can't be a directory." + ); + } + + // Create directory if necessary + $directory = dirname($this->destination); + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + + $uniqueId = self::getUniqueIdentifier(); + $temporaryName = $this->destination . self::TEMP_INFIX . $uniqueId; + while (file_exists($temporaryName)) { + $uniqueId = self::getUniqueIdentifier(); + $temporaryName = $this->destination . self::TEMP_INFIX . $uniqueId; + } + + // Create the file + file_put_contents($temporaryName, ""); + $this->temporaryDestination = $temporaryName; + } + + /** + * @param array $context + * + * @return void + */ + public function bytesTransferred(array $context): void + { + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; + $response = $snapshot->getResponse(); + file_put_contents( + $this->temporaryDestination, + $response['Body'], + FILE_APPEND + ); + } + + /** + * @param array $context + * + * @return void + */ + public function transferComplete(array $context): void + { + // Make sure the file is deleted if exists + if (file_exists($this->destination) && is_file($this->destination)) { + if ($this->failsWhenDestinationExists) { + throw new FileDownloadException( + "The destination '$this->destination' already exists." + ); + } else { + unlink($this->destination); + } + } + + if (!rename($this->temporaryDestination, $this->destination)) { + throw new FileDownloadException( + "Unable to rename the file `$this->temporaryDestination` to `$this->destination`." + ); + } + } + + /** + * @param array $context + * + * @return void + */ + public function transferFail(array $context): void + { + if (file_exists($this->temporaryDestination)) { + unlink($this->temporaryDestination); + } elseif (file_exists($this->destination) + && !str_contains( + $context[self::REASON_KEY], + "The destination '$this->destination' already exists.") + ) { + unlink($this->destination); + } + } + + /** + * @return string + */ + private static function getUniqueIdentifier(): string + { + $uniqueId = uniqid(); + if (strlen($uniqueId) > self::IDENTIFIER_LENGTH) { + $uniqueId = substr($uniqueId, 0, self::IDENTIFIER_LENGTH); + } else { + $uniqueId = str_pad($uniqueId, self::IDENTIFIER_LENGTH, "0"); + } + + return $uniqueId; + } + + /** + * @return string + */ + public function getHandlerResult(): string + { + return $this->destination; + } +} diff --git a/src/S3/S3Transfer/Utils/StreamDownloadHandler.php b/src/S3/S3Transfer/Utils/StreamDownloadHandler.php new file mode 100644 index 0000000000..01ac1b97bd --- /dev/null +++ b/src/S3/S3Transfer/Utils/StreamDownloadHandler.php @@ -0,0 +1,83 @@ +stream = $stream; + } + + /** + * @param array $context + * + * @return void + */ + public function transferInitiated(array $context): void + { + if (is_null($this->stream)) { + $this->stream = Utils::streamFor( + fopen('php://temp', 'w+') + ); + } else { + $this->stream->seek($this->stream->getSize()); + } + } + + /** + * @param array $context + * + * @return void + */ + public function bytesTransferred(array $context): void + { + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; + $response = $snapshot->getResponse(); + Utils::copyToStream( + $response['Body'], + $this->stream + ); + } + + /** + * @param array $context + * + * @return void + */ + public function transferComplete(array $context): void + { + $this->stream->rewind(); + } + + /** + * @param array $context + * + * @return void + */ + public function transferFail(array $context): void + { + $this->stream->close(); + $this->stream = null; + } + + /** + * @inheritDoc + * + * @return StreamInterface + */ + public function getHandlerResult(): StreamInterface + { + return $this->stream; + } +} diff --git a/tests/Integ/S3Context.php b/tests/Integ/S3Context.php index 2628417ccb..1a6ad99092 100644 --- a/tests/Integ/S3Context.php +++ b/tests/Integ/S3Context.php @@ -19,6 +19,7 @@ class S3Context implements Context, SnippetAcceptingContext { use IntegUtils; + use S3ContextTrait; const INTEG_LOG_BUCKET_PREFIX = 'aws-php-sdk-test-integ-logs'; @@ -38,18 +39,6 @@ class S3Context implements Context, SnippetAcceptingContext private $options; private $expires; - private static function getResourceName() - { - static $bucketName; - - if (empty($bucketName)) { - $bucketName = - self::getResourcePrefix() . 'aws-test-integ-s3-context'; - } - - return $bucketName; - } - /** * @BeforeSuite */ @@ -72,13 +61,7 @@ public static function deleteTempFile() */ public static function createTestBucket() { - $client = self::getSdk()->createS3(); - if (!$client->doesBucketExistV2(self::getResourceName())) { - $client->createBucket(['Bucket' => self::getResourceName()]); - $client->waitUntil('BucketExists', [ - 'Bucket' => self::getResourceName(), - ]); - } + self::doCreateTestBucket(); } /** @@ -86,64 +69,7 @@ public static function createTestBucket() */ public static function deleteTestBucket() { - $client = self::getSdk()->createS3(); - - $result = self::executeWithRetries( - $client, - 'listObjectsV2', - ['Bucket' => self::getResourceName()], - 10, - [404] - ); - - // Delete objects & wait until no longer available before deleting bucket - $client->deleteMatchingObjects(self::getResourceName(), '', '//'); - if (!empty($result['Contents']) && is_array($result['Contents'])) { - foreach ($result['Contents'] as $object) { - $client->waitUntil('ObjectNotExists', [ - 'Bucket' => self::getResourceName(), - 'Key' => $object['Key'], - '@waiter' => [ - 'maxAttempts' => 60, - 'delay' => 10, - ], - ]); - } - } - - // Delete bucket - $result = self::executeWithRetries( - $client, - 'deleteBucket', - ['Bucket' => self::getResourceName()], - 10, - [404] - ); - - // Use account number to generate a unique bucket name - $sts = new StsClient([ - 'version' => 'latest', - 'region' => 'us-east-1' - ]); - $identity = $sts->getCallerIdentity([]); - $logBucket = self::INTEG_LOG_BUCKET_PREFIX . "-{$identity['Account']}"; - - // Log bucket deletion result - if (!($client->doesBucketExistV2($logBucket))) { - $client->createBucket([ - 'Bucket' => $logBucket - ]); - } - $client->putObject([ - 'Bucket' => $logBucket, - 'Key' => self::getResourceName() . '-' . date('Y-M-d__H_i_s'), - 'Body' => print_r($result->toArray(), true) - ]); - - // Wait until bucket is no longer available - $client->waitUntil('BucketNotExists', [ - 'Bucket' => self::getResourceName(), - ]); + self::doDeleteTestBucket(); } /** @@ -353,40 +279,4 @@ private function preparePostData($postObject) 'filename' => 'file.ext', ]; } - - /** - * Executes S3 client method, adding retries for specified status codes. - * A practical work-around for the testing workflow, given eventual - * consistency constraints. - * - * @param S3Client $client - * @param string $command - * @param array $args - * @param int $retries - * @param array $statusCodes - * @return mixed - */ - private static function executeWithRetries( - $client, - $command, - $args, - $retries, - $statusCodes - ) { - $attempts = 0; - - while (true) { - try { - return call_user_func([$client, $command], $args); - } catch (S3Exception $e) { - if (!in_array($e->getStatusCode(), $statusCodes) - || $attempts >= $retries - ) { - throw $e; - } - $attempts++; - sleep((int) pow(1.2, $attempts)); - } - } - } } diff --git a/tests/Integ/S3ContextTrait.php b/tests/Integ/S3ContextTrait.php new file mode 100644 index 0000000000..e6defa7ba6 --- /dev/null +++ b/tests/Integ/S3ContextTrait.php @@ -0,0 +1,128 @@ +createS3(); + if (!$client->doesBucketExistV2(self::getResourceName())) { + $client->createBucket(['Bucket' => self::getResourceName()]); + $client->waitUntil('BucketExists', [ + 'Bucket' => self::getResourceName(), + ]); + } + } + + private static function doDeleteTestBucket(): void { + $client = self::getSdk()->createS3(); + $result = self::executeWithRetries( + $client, + 'listObjectsV2', + ['Bucket' => self::getResourceName()], + 10, + [404] + ); + + // Delete objects & wait until no longer available before deleting bucket + $client->deleteMatchingObjects(self::getResourceName(), '', '//'); + if (!empty($result['Contents']) && is_array($result['Contents'])) { + foreach ($result['Contents'] as $object) { + $client->waitUntil('ObjectNotExists', [ + 'Bucket' => self::getResourceName(), + 'Key' => $object['Key'], + '@waiter' => [ + 'maxAttempts' => 60, + 'delay' => 10, + ], + ]); + } + } + + // Delete bucket + $result = self::executeWithRetries( + $client, + 'deleteBucket', + ['Bucket' => self::getResourceName()], + 10, + [404] + ); + + // Use account number to generate a unique bucket name + $sts = new StsClient([ + 'version' => 'latest', + 'region' => 'us-east-1' + ]); + $identity = $sts->getCallerIdentity([]); + $logBucket = self::INTEG_LOG_BUCKET_PREFIX . "-{$identity['Account']}"; + + // Log bucket deletion result + if (!($client->doesBucketExistV2($logBucket))) { + $client->createBucket([ + 'Bucket' => $logBucket + ]); + } + $client->putObject([ + 'Bucket' => $logBucket, + 'Key' => self::getResourceName() . '-' . date('Y-M-d__H_i_s'), + 'Body' => print_r($result->toArray(), true) + ]); + + // Wait until bucket is no longer available + $client->waitUntil('BucketNotExists', [ + 'Bucket' => self::getResourceName(), + ]); + } + + /** + * Executes S3 client method, adding retries for specified status codes. + * A practical work-around for the testing workflow, given eventual + * consistency constraints. + * + * @param S3Client $client + * @param string $command + * @param array $args + * @param int $retries + * @param array $statusCodes + * @return mixed + */ + private static function executeWithRetries( + $client, + $command, + $args, + $retries, + $statusCodes + ) { + $attempts = 0; + + while (true) { + try { + return call_user_func([$client, $command], $args); + } catch (S3Exception $e) { + if (!in_array($e->getStatusCode(), $statusCodes) + || $attempts >= $retries + ) { + throw $e; + } + $attempts++; + sleep((int) pow(1.2, $attempts)); + } + } + } +} \ No newline at end of file diff --git a/tests/Integ/S3TransferManagerContext.php b/tests/Integ/S3TransferManagerContext.php new file mode 100644 index 0000000000..6bc2a9675a --- /dev/null +++ b/tests/Integ/S3TransferManagerContext.php @@ -0,0 +1,868 @@ +stream = Utils::streamFor(''); + // Create temporary directory + self::$tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . "s3-transfer-manager"; + if (is_dir(self::$tempDir)) { + TestsUtility::cleanUpDir(self::$tempDir); + } + + mkdir(self::$tempDir, 0777, true); + } + + /** + * @AfterScenario + */ + public function afterScenarioRuns(): void { + // Clean up temporary directory + TestsUtility::cleanUpDir(self::$tempDir); + + // Clean up data holders + $this->stream?->close(); + $this->stream = null; + } + + /** + * @Given /^I have a file (.*) with content (.*)$/ + */ + public function iHaveAFileWithContent($filename, $content): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + file_put_contents($fullFilePath, $content); + } + + /** + * @When /^I upload the file (.*) to a test bucket using the s3 transfer manager$/ + */ + public function iUploadTheFileToATestBucketUsingTheS3TransferManager($filename): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $s3TransferManager->upload( + new UploadRequest( + $fullFilePath, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + ] + ) + )->wait(); + } + + /** + * @Then /^the file (.*) should exist in the test bucket and its content should be (.*)$/ + */ + public function theFileShouldExistInTheTestBucketAndItsContentShouldBe( + $filename, + $content + ): void + { + $client = self::getSdk()->createS3(); + $response = $client->getObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + ]); + + Assert::assertEquals(200, $response['@metadata']['statusCode']); + Assert::assertEquals($content, $response['Body']->getContents()); + } + + /** + * @Given /^I have a stream with content (.*)$/ + */ + public function iHaveAStreamWithContent($content): void + { + $this->stream = Utils::streamFor($content); + } + + /** + * @When /^I do the upload to a test bucket with key (.*)$/ + */ + public function iDoTheUploadToATestBucketWithKey($key): void + { + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $s3TransferManager->upload( + new UploadRequest( + $this->stream, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $key, + ] + ) + )->wait(); + } + + /** + * @Then /^the object (.*), once downloaded from the test bucket, should match the content (.*)$/ + */ + public function theObjectOnceDownloadedFromTheTestBucketShouldMatchTheContent( + $key, + $content + ): void + { + $client = self::getSdk()->createS3(); + $response = $client->getObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $key, + ]); + + Assert::assertEquals(200, $response['@metadata']['statusCode']); + Assert::assertEquals($content, $response['Body']->getContents()); + } + + /** + * @Given /^I have a file with name (.*) where its content's size is (.*)$/ + */ + public function iHaveAFileWithNameWhereItsContentSSizeIs( + $filename, + $filesize + ): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + file_put_contents($fullFilePath, str_repeat('a', (int)$filesize)); + } + + /** + * @When /^I do upload this file with name (.*) with the specified part size of (.*)$/ + */ + public function iDoUploadThisFileWithNameWithTheSpecifiedPartSizeOf( + $filename, + $partsize + ): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + S3TransferManagerConfig::fromArray([ + 'multipart_upload_threshold_bytes' => (int)$partsize, + ]) + ); + $s3TransferManager->upload( + new UploadRequest( + $fullFilePath, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + ], + [ + 'target_part_size_bytes' => (int)$partsize, + ] + ) + )->wait(); + } + + /** + * @Given /^I want to upload a stream of size (.*)$/ + */ + public function iWantToUploadAStreamOfSize($filesize): void + { + $this->stream = Utils::streamFor(str_repeat('a', (int)$filesize)); + } + + /** + * @When /^I do upload this stream with name (.*) and the specified part size of (.*)$/ + */ + public function iDoUploadThisStreamWithNameAndTheSpecifiedPartSizeOf( + $filename, + $partsize + ): void + { + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + S3TransferManagerConfig::fromArray([ + 'multipart_upload_threshold_bytes' => (int)$partsize, + ]) + ); + $s3TransferManager->upload( + new UploadRequest( + $this->stream, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + ], + [ + 'target_part_size_bytes' => (int)$partsize, + ] + ) + )->wait(); + } + + /** + * @Then /^the object with name (.*) should have a total of (.*) parts and its size must be (.*)$/ + */ + public function theObjectWithNameShouldHaveATotalOfPartsAndItsSizeMustBe( + $filename, + $partnum, + $filesize + ): void + { + $partNo = 1; + $s3Client = self::getSdk()->createS3(); + $response = $s3Client->headObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + 'PartNumber' => $partNo + ]); + Assert::assertEquals(206, $response['@metadata']['statusCode']); + Assert::assertEquals((int)$partnum, $response['PartsCount']); + $contentLength = $response['@metadata']['headers']['content-length']; + $partNo++; + while ($partNo <= (int)$partnum) { + $response = $s3Client->headObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + 'PartNumber' => $partNo + ]); + $contentLength += $response['@metadata']['headers']['content-length']; + $partNo++; + } + + Assert::assertEquals((int)$filesize, $contentLength); + } + + /** + * @Given /^I have a file with name (.*) and its content is (.*)$/ + */ + public function iHaveAFileWithNameAndItsContentIs($filename, $content): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + file_put_contents($fullFilePath, $content); + } + + /** + * @When /^I upload this file with name (.*) by providing a custom checksum algorithm (.*)$/ + */ + public function iUploadThisFileWithNameByProvidingACustomChecksumAlgorithm( + $filename, + $checksumAlgorithm + ): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + ); + $s3TransferManager->upload( + new UploadRequest( + $fullFilePath, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + 'ChecksumAlgorithm' => $checksumAlgorithm, + ], + ) + )->wait(); + } + + /** + * @Then /^the checksum from the object with name (.*) should be equals to the calculation of the object content with the checksum algorithm (.*)$/ + */ + public function theChecksumFromTheObjectWithNameShouldBeEqualsToTheCalculationOfTheObjectContentWithTheChecksumAlgorithm( + $filename, + $checksumAlgorithm + ): void + { + $s3Client = self::getSdk()->createS3(); + $response = $s3Client->getObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + 'ChecksumMode' => 'ENABLED' + ]); + + Assert::assertEquals(200, $response['@metadata']['statusCode']); + Assert::assertEquals( + ApplyChecksumMiddleware::getEncodedValue( + $checksumAlgorithm, + $response['Body'] + ), + $response['Checksum' . strtoupper($checksumAlgorithm)] + ); + } + + /** + * @Given /^I have an object in S3 with name (.*) and its content is (.*)$/ + */ + public function iHaveAnObjectInS3withNameAndItsContentIs( + $filename, + $content + ): void + { + $client = self::getSdk()->createS3(); + $client->putObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + 'Body' => $content, + ]); + } + + /** + * @When /^I do a download of the object with name (.*)$/ + */ + public function iDoADownloadOfTheObjectWithName($filename): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + ); + $s3TransferManager->downloadFile(new DownloadFileRequest( + $fullFilePath, + false, + new DownloadRequest([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + ]) + ))->wait(); + } + + /** + * @Then /^the object with name (.*) should have been downloaded and its content should be (.*)$/ + */ + public function theObjectWithNameShouldHaveBeenDownloadedAndItsContentShouldBe( + $filename, + $content + ): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + Assert::assertFileExists($fullFilePath); + Assert::assertEquals($content, file_get_contents($fullFilePath)); + } + + /** + * @Given /^I have an object in S3 with name (.*) and its size is (.*)$/ + */ + public function iHaveAnObjectInS3withNameAndItsSizeIs( + $filename, + $filesize + ): void + { + $client = self::getSdk()->createS3(); + $client->putObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + 'Body' => str_repeat('*', (int)$filesize), + ]); + } + + /** + * @When /^I download the object with name (.*) by using the (.*) multipart download type$/ + */ + public function iDownloadTheObjectWithNameByUsingTheMultipartDownloadType( + $filename, + $downloadType + ): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + ); + $s3TransferManager->downloadFile( + new DownloadFileRequest( + $fullFilePath, + false, + new DownloadRequest( + [ + 'Bucket' => self::getResourceName(), + 'Key' => $filename + ], + [], + [ + 'multipart_download_type' => $downloadType, + ] + ) + ) + )->wait(); + } + + /** + * @Then /^the content size for the object with name (.*) should be (.*)$/ + */ + public function theContentSizeForTheObjectWithNameShouldBe( + $filename, + $filesize + ): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + Assert::assertFileExists($fullFilePath); + Assert::assertEquals((int)$filesize, filesize($fullFilePath)); + } + + /** + * @Given /^I have a directory (.*) with (.*) files that I want to upload$/ + */ + public function iHaveADirectoryWithFilesThatIWantToUpload( + $directory, + $numfile + ): void + { + $fullDirectoryPath = self::$tempDir . DIRECTORY_SEPARATOR . $directory; + if (!is_dir($fullDirectoryPath)) { + mkdir($fullDirectoryPath, 0777, true); + } + + $count = (int)$numfile; + for ($i = 0; $i < $count - 1; $i++) { + $fullFilePath = $fullDirectoryPath . DIRECTORY_SEPARATOR . "file" . ($i + 1) . ".txt"; + file_put_contents($fullFilePath, "This is a test file content #" . ($i + 1)); + } + + // Create one large file for multipart upload testing + if ($count > 0) { + $fullFilePath = $fullDirectoryPath . DIRECTORY_SEPARATOR . "file" . $count . ".txt"; + file_put_contents($fullFilePath, str_repeat('*', 1024 * 1024 * 15)); + } + } + + /** + * @When /^I upload this directory (.*)$/ + */ + public function iUploadThisDirectory($directory): void + { + $fullDirectoryPath = self::$tempDir . DIRECTORY_SEPARATOR . $directory; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + ); + $s3TransferManager->uploadDirectory( + new UploadDirectoryRequest( + $fullDirectoryPath, + self::getResourceName(), + ) + )->wait(); + } + + /** + * @Then /^the files from this directory (.*) where its count should be (.*) should exist in the bucket$/ + */ + public function theFilesFromThisDirectoryWhereItsCountShouldBeShouldExistInTheBucket( + $directory, + $numfile + ): void + { + $s3Client = self::getSdk()->createS3(); + $localDirectoryPath = self::$tempDir . DIRECTORY_SEPARATOR . $directory; + $localFiles = array_diff( + scandir($localDirectoryPath), + ['..', '.'] + ); + $uploadedCount = 0; + + foreach ($localFiles as $fileName) { + $localFilePath = $localDirectoryPath . DIRECTORY_SEPARATOR . $fileName; + + if (!is_file($localFilePath)) { + continue; + } + + $s3Key = $directory . DIRECTORY_SEPARATOR . $fileName; + + try { + // Verify the object exists in S3 + $response = $s3Client->getObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $s3Key, + ]); + + Assert::assertEquals(200, $response['@metadata']['statusCode']); + + $localContent = file_get_contents($localFilePath); + $s3Content = $response['Body']->getContents(); + + Assert::assertEquals( + $localContent, + $s3Content, + "Content mismatch for file: {$fileName}" + ); + + $uploadedCount++; + } catch (\Exception $e) { + Assert::fail("Failed to verify S3 object {$s3Key}: " . $e->getMessage()); + } + } + + Assert::assertEquals( + (int)$numfile, + $uploadedCount, + "Expected {$numfile} files but found {$uploadedCount} uploaded files" + ); + } + + /** + * @Given /^I have a total of (.*) objects in a bucket prefixed with (.*)$/ + */ + public function iHaveATotalOfObjectsInABucketPrefixedWith($numfile, $directory): void + { + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + ); + + $numFileInt = (int)$numfile; + for ($i = 0; $i < $numFileInt; $i++) { + $s3TransferManager->upload( + new UploadRequest( + Utils::streamFor("This is a test file content #" . ($i + 1)), + [ + 'Bucket' => self::getResourceName(), + 'Key' => $directory . DIRECTORY_SEPARATOR . "file" . ($i + 1) . ".txt", + ] + ) + )->wait(); + } + } + + /** + * @When /^I download all of them into the directory (.*)$/ + */ + public function iDownloadAllOfThemIntoTheDirectory($directory): void + { + $fullDirectoryPath = self::$tempDir . DIRECTORY_SEPARATOR . $directory; + if (!is_dir($fullDirectoryPath)) { + mkdir($fullDirectoryPath, 0777, true); + } + + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + ); + $s3TransferManager->downloadDirectory( + new DownloadDirectoryRequest( + self::getResourceName(), + $fullDirectoryPath, + [], + [ + 's3_prefix' => $directory . DIRECTORY_SEPARATOR, + ] + ) + )->wait(); + } + + /** + * @Then /^the objects (.*) should exist as files within the directory (.*)$/ + */ + public function theObjectsShouldExistsAsFilesWithinTheDirectory( + $numfile, + $directory + ): void + { + $fullDirectoryPath = self::$tempDir . DIRECTORY_SEPARATOR . $directory; + $s3Client = self::getSdk()->createS3(); + + // Get list of objects from S3 + $objects = $s3Client->getPaginator('ListObjectsV2', [ + 'Bucket' => self::getResourceName(), + 'Prefix' => $directory . DIRECTORY_SEPARATOR, + ]); + + $count = 0; + foreach ($objects as $page) { + if (isset($page['Contents'])) { + foreach ($page['Contents'] as $object) { + $key = $object['Key']; + $fileName = basename($key); + $localFilePath = $fullDirectoryPath . DIRECTORY_SEPARATOR . $fileName; + + // Verify the file was downloaded locally + Assert::assertFileExists($localFilePath); + + // Verify content matches S3 object + $s3Response = $s3Client->getObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $key, + ]); + $s3Content = $s3Response['Body']->getContents(); + $localContent = file_get_contents($localFilePath); + Assert::assertEquals($s3Content, $localContent); + + $count++; + } + } + } + + Assert::assertEquals((int)$numfile, $count); + } + + /** + * @Given /^I am uploading the file (.*) with size (.*)$/ + */ + public function iAmUploadingTheFileWithSize($file, $size): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + file_put_contents($fullFilePath, str_repeat('*', (int)$size)); + } + + /** + * @When /^I upload the file (.*) using multipart upload and fails at part number (.*)$/ + */ + public function iUploadTheFileUsingMultipartUploadAndFailsAtPartNumber( + $file, + $partNumberFail + ): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $transferListener = new class((int)$partNumberFail) extends TransferListener { + private int $partNumber; + private int $partNumberFail; + + public function __construct(int $partNumberFail) { + $this->partNumberFail = $partNumberFail; + $this->partNumber = 0; + } + + public function bytesTransferred(array $context): void + { + $this->partNumber++; + if ($this->partNumber === $this->partNumberFail) { + throw new \RuntimeException( + "Transfer failed at part number {$this->partNumber} failed" + ); + } + } + }; + + // To make sure transferFail is called + $testCase = new class extends TestCase {}; + $transferListener2 = $testCase->getMockBuilder( + TransferListener::class + )->getMock(); + $transferListener2->expects($testCase->once())->method('transferInitiated'); + $transferListener2->expects($testCase->once())->method('transferFail'); + + try { + $s3TransferManager->upload( + new UploadRequest( + $fullFilePath, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + ], + [], + [ + $transferListener, + $transferListener2 + ] + ) + )->wait(); + + // If we reach here, the test should fail because exception was expected + Assert::fail("Expected RuntimeException was not thrown"); + + } catch (\RuntimeException $exception) { + Assert::assertEquals( + "Transfer failed at part number {$partNumberFail} failed", + $exception->getMessage(), + ); + } + } + + /** + * @Then /^The multipart upload should have been aborted for file (.*)$/ + */ + public function theMultipartUploadShouldHaveBeenAbortedForFile($file): void + { + $client = self::getSdk()->createS3(); + $inProgressMultipartUploads = $client->listMultipartUploads([ + 'Bucket' => self::getResourceName(), + ]); + + // Make sure that, if there are in progress multipart uploads, + // none are for the file being uploaded in this test. + $multipartUploadCount = 0; + if (isset($inProgressMultipartUploads['Uploads'])) { + foreach ($inProgressMultipartUploads['Uploads'] as $upload) { + if ($upload['Key'] === $file) { + $multipartUploadCount++; + } + } + } + + Assert::assertEquals(0, $multipartUploadCount, + "Expected no in-progress multipart uploads for file: {$file}"); + } + + /** + * @Given /^I have a file (.*) to be uploaded of size (.*)$/ + */ + public function iHaveAFileToBeUploadedOfSize($file, $size): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + file_put_contents($fullFilePath, str_repeat('*', (int)$size)); + } + + /** + * @When /^I upload the file (.*) with custom checksum algorithm (.*)$/ + */ + public function iUploadTheFileWithCustomChecksumAlgorithm( + $file, + $algorithm + ): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $s3TransferManager->upload( + new UploadRequest( + $fullFilePath, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + 'ChecksumAlgorithm' => $algorithm, + ], + [ + 'multipart_upload_threshold_bytes' => 1024 * 1024 * 5, + ] + ) + )->wait(); + } + + /** + * @Then /^The checksum validation with algorithm (.*) for file (.*) should succeed$/ + */ + public function theChecksumValidationWithAlgorithmForFileShouldSucceed( + $algorithm, + $file + ): void + { + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $result = $s3TransferManager->download( + new DownloadRequest( + source: [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + ], + config: [ + 'response_checksum_validation' => 'when_supported' + ] + ) + )->wait(); + Assert::assertEqualsIgnoringCase( + $algorithm, + $result['ChecksumValidated'], + ); + } + + /** + * @When /^I upload the file (.*) with custom checksum (.*) and algorithm (.*)$/ + */ + public function iUploadTheFileWithCustomChecksumAndAlgorithm( + $file, + $checksum, + $algorithm + ): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $s3TransferManager->upload( + new UploadRequest( + $fullFilePath, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + 'Checksum'.strtoupper($algorithm) => $checksum, + ], + [ + 'multipart_upload_threshold_bytes' => 1024 * 1024 * 5, + ] + ) + )->wait(); + } + + /** + * @Then /^The checksum validation with checksum (.*) and algorithm (.*) for file (.*) should succeed$/ + */ + public function theChecksumValidationWithChecksumAndAlgorithmForFileShouldSucceed( + $checksum, + $algorithm, + $file + ): void + { + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $result = $s3TransferManager->download( + new DownloadRequest( + source: [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + ], + config: [ + 'response_checksum_validation' => 'when_supported' + ] + ) + )->wait(); + Assert::assertEqualsIgnoringCase( + $algorithm, + $result['ChecksumValidated'], + ); + Assert::assertEquals( + $checksum, + $result['Checksum'.strtoupper($algorithm)], + ); + } +} \ No newline at end of file diff --git a/tests/S3/ApplyChecksumMiddlewareTest.php b/tests/S3/ApplyChecksumMiddlewareTest.php index 257a2fa3c6..5abae7fb63 100644 --- a/tests/S3/ApplyChecksumMiddlewareTest.php +++ b/tests/S3/ApplyChecksumMiddlewareTest.php @@ -57,112 +57,143 @@ public function testFlexibleChecksums( public function getFlexibleChecksumUseCases() { return [ - // httpChecksum not modeled - [ - 'GetObject', - [], - [ + 'http_checksum_not_modeled' => [ + 'operation' => 'GetObject', + 'config' => [], + 'command_args' => [ 'Bucket' => 'foo', 'Key' => 'bar', 'ChecksumMode' => 'ENABLED' ], - null, - false, - '' + 'body' => null, + 'headers_added' => false, + 'header_value' => '' ], - // default: when_supported. defaults to crc32 - [ - 'PutObject', - [], - [ + 'default_when_supported_defaults_to_crc32' => [ + 'operation' => 'PutObject', + 'config' => [], + 'command_args' => [ 'Bucket' => 'foo', 'Key' => 'bar', 'Body' => 'abc' ], - 'abc', - true, - 'NSRBwg==' + 'body' => 'abc', + 'headers_added' => true, + 'header_value' => 'NSRBwg==' ], - // when_required when not required and no requested algorithm - [ - 'PutObject', - ['request_checksum_calculation' => 'when_required'], - [ - 'Bucket' => 'foo', - 'Key' => 'bar', - 'Body' => 'abc' - ], - 'abc', - false, - '' + 'when_required_when_not_required_and_no_requested_algorithm' => [ + 'operation' => 'PutObject', + 'config' => ['request_checksum_calculation' => 'when_required'], + 'command_args' => [], + 'body' => 'abc', + 'headers_added' => false, + 'header_value' => '' ], - // when_required when required and no requested algorithm - [ - 'PutObjectLockConfiguration', - ['request_checksum_calculation' => 'when_required'], - [ + 'when_required_when_required_and_no_requested_algorithm' => [ + 'operation' => 'PutObjectLockConfiguration', + 'config' => ['request_checksum_calculation' => 'when_required'], + 'command_args' => [ 'Bucket' => 'foo', 'Key' => 'bar', 'ObjectLockConfiguration' => 'blah' ], - 'blah', - true, - 'zilhXA==' + 'body' => 'blah', + 'headers_added' => true, + 'header_value' => 'zilhXA==' ], - // when_required when not required and requested algorithm - [ - 'PutObject', - ['request_checksum_calculation' => 'when_required'], - [ + 'when_required_when_not_required_and_requested_algorithm' => [ + 'operation' => 'PutObject', + 'config' => ['request_checksum_calculation' => 'when_required'], + 'command_args' => [ 'Bucket' => 'foo', 'Key' => 'bar', 'Body' => 'blah', 'ChecksumAlgorithm' => 'crc32', ], - 'blah', - true, - 'zilhXA==' + 'body' => 'blah', + 'headers_added' => true, + 'header_value' => 'zilhXA==' ], - // when_supported and requested algorithm - [ - 'PutObject', - [], - [ + 'when_supported_and_requested_algorithm_1' => [ + 'operation' => 'PutObject', + 'config' => [], + 'command_args' => [ 'Bucket' => 'foo', 'Key' => 'bar', 'ChecksumAlgorithm' => 'crc32c', 'Body' => 'abc' ], - 'abc', - true, - 'Nks/tw==' + 'body' => 'abc', + 'headers_added' => true, + 'header_value' => 'Nks/tw==' ], - // when_supported and requested algorithm - [ - 'PutObject', - [], - [ + 'when_supported_and_requested_algorithm_2' => [ + 'operation' => 'PutObject', + 'config' => [], + 'command_args' => [ 'Bucket' => 'foo', 'Key' => 'bar', 'ChecksumAlgorithm' => 'sha256' ], - '', - true, - '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=' + 'body' => '', + 'headers_added' => true, + 'header_value' => '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=' ], - // when_supported and requested algorithm - [ - 'PutObject', - [], - [ + 'when_supported_and_requested_algorithm_3' => [ + 'operation' => 'PutObject', + 'config' => [], + 'command_args' => [ 'Bucket' => 'foo', 'Key' => 'bar', 'ChecksumAlgorithm' => 'SHA1' ], - '', - true, - '2jmj7l5rSw0yVb/vlWAYkK/YBwk=' + 'body' => '', + 'headers_added' => true, + 'header_value' => '2jmj7l5rSw0yVb/vlWAYkK/YBwk=' ], + 'when_required_when_not_required_and_no_requested_algorithm_from_command_args' => [ + 'operation' => 'PutObject', + 'config' => [], + 'command_args' => [ + '@context' => [ + 'request_checksum_calculation' => 'when_required' + ] + ], + 'body' => 'abc', + 'headers_added' => false, + 'header_value' => '' + ], + 'when_required_when_required_and_no_requested_algorithm_from_command_args' => [ + 'operation' => 'PutObjectLockConfiguration', + 'config' => [], + 'command_args' => [ + 'Bucket' => 'foo', + 'Key' => 'bar', + 'ObjectLockConfiguration' => 'blah', + '@context' => [ + 'request_checksum_calculation' => 'when_required' + ] + ], + 'body' => 'blah', + 'headers_added' => true, + 'header_value' => 'zilhXA==' + ], + 'when_required_when_not_required_and_requested_algorithm_from_command_args' => [ + 'operation' => 'PutObject', + 'config' => [], + 'command_args' => [ + 'Bucket' => 'foo', + 'Key' => 'bar', + 'Body' => 'blah', + 'ChecksumAlgorithm' => 'crc32', + '@context' => [ + 'request_checksum_calculation' => 'when_required' + ] + ], + 'body' => 'blah', + 'headers_added' => true, + 'header_value' => 'zilhXA==' + ] ]; } diff --git a/tests/S3/S3Transfer/MultipartDownloaderTest.php b/tests/S3/S3Transfer/MultipartDownloaderTest.php new file mode 100644 index 0000000000..c5bb2c2f3d --- /dev/null +++ b/tests/S3/S3Transfer/MultipartDownloaderTest.php @@ -0,0 +1,287 @@ + PartGetMultipartDownloader::class, + AbstractMultipartDownloader::RANGED_GET_MULTIPART_DOWNLOADER => RangeGetMultipartDownloader::class, + ]; + foreach ($multipartDownloadTypes as $multipartDownloadType => $class) { + $resolvedClass = AbstractMultipartDownloader::chooseDownloaderClass($multipartDownloadType); + $this->assertEquals($class, $resolvedClass); + } + } + + /** + * Tests chooseDownloaderClass throws exception for invalid type. + * + * @return void + */ + public function testChooseDownloaderClassThrowsExceptionForInvalidType(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('The config value for `multipart_download_type` must be one of:'); + + AbstractMultipartDownloader::chooseDownloaderClass('invalidType'); + } + + /** + * Tests constants are properly defined. + * + * @return void + */ + public function testConstants(): void + { + $this->assertEquals('GetObject', AbstractMultipartDownloader::GET_OBJECT_COMMAND); + $this->assertEquals('part', AbstractMultipartDownloader::PART_GET_MULTIPART_DOWNLOADER); + $this->assertEquals('ranged', AbstractMultipartDownloader::RANGED_GET_MULTIPART_DOWNLOADER); + } + + /** + * @return void + */ + public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void + { + $listener1 = $this->getMockBuilder(TransferListener::class)->getMock(); + $listener2 = $this->getMockBuilder(TransferListener::class)->getMock(); + $listener3 = $this->getMockBuilder(TransferListener::class)->getMock(); + + $listener1->expects($this->once())->method('transferInitiated'); + $listener1->expects($this->atLeastOnce())->method('bytesTransferred'); + $listener1->expects($this->once())->method('transferComplete'); + + $listener2->expects($this->once())->method('transferInitiated'); + $listener2->expects($this->atLeastOnce())->method('bytesTransferred'); + $listener2->expects($this->once())->method('transferComplete'); + + $listener3->expects($this->once())->method('transferInitiated'); + $listener3->expects($this->atLeastOnce())->method('bytesTransferred'); + $listener3->expects($this->once())->method('transferComplete'); + + $listenerNotifier = new TransferListenerNotifier([$listener1, $listener2, $listener3]); + + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'GetObject') { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor('test data'), + 'ContentLength' => 9, + 'ContentRange' => 'bytes 0-8/9', + 'PartsCount' => 1, + 'ETag' => 'TestETag' + ])); + } + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $requestArgs = [ + 'Key' => 'test-key', + 'Bucket' => 'test-bucket', + ]; + + $multipartDownloader = new PartGetMultipartDownloader( + $s3Client, + $requestArgs, + [], + new StreamDownloadHandler(), + 0, + 0, + 0, + '', + null, + $listenerNotifier, + ); + + $response = $multipartDownloader->promise()->wait(); + $this->assertInstanceOf(DownloadResult::class, $response); + } + + /** + * @return void + */ + public function testTransferListenerNotifierNotifiesListenersOnFailure(): void + { + $listener1 = $this->getMockBuilder(TransferListener::class)->getMock(); + $listener2 = $this->getMockBuilder(TransferListener::class)->getMock(); + + $listener1->expects($this->once())->method('transferInitiated'); + $listener1->expects($this->once())->method('transferFail'); + + $listener2->expects($this->once())->method('transferInitiated'); + $listener2->expects($this->once())->method('transferFail'); + + $listenerNotifier = new TransferListenerNotifier([$listener1, $listener2]); + + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'GetObject') { + return Create::rejectionFor(new \Exception('Download failed')); + } + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $requestArgs = [ + 'Key' => 'test-key', + 'Bucket' => 'test-bucket', + ]; + + $multipartDownloader = new PartGetMultipartDownloader( + $s3Client, + $requestArgs, + [], + new StreamDownloadHandler(), + 0, + 0, + 0, + null, + null, + $listenerNotifier + ); + + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Download failed'); + $multipartDownloader->promise()->wait(); + } + + /** + * @return void + */ + public function testTransferListenerNotifierWithEmptyListeners(): void + { + $listenerNotifier = new TransferListenerNotifier([]); + + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'GetObject') { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor('test'), + 'ContentLength' => 4, + 'ContentRange' => 'bytes 0-3/4', + 'PartsCount' => 1, + 'ETag' => 'TestETag' + ])); + } + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $requestArgs = [ + 'Key' => 'test-key', + 'Bucket' => 'test-bucket', + ]; + + $multipartDownloader = new PartGetMultipartDownloader( + $s3Client, + $requestArgs, + [], + new StreamDownloadHandler(), + 0, + 0, + 0, + null, + null, + $listenerNotifier + ); + + $response = $multipartDownloader->promise()->wait(); + $this->assertInstanceOf(DownloadResult::class, $response); + } + + /** + * @return void + */ + public function testConfigIsSetToDefaultValues(): void { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $multipartDownloader = new PartGetMultipartDownloader( + $mockClient, + [], + [], + new StreamDownloadHandler(), + ); + $config = $multipartDownloader->getConfig(); + $this->assertEquals( + S3TransferManagerConfig::DEFAULT_TARGET_PART_SIZE_BYTES, + $config['target_part_size_bytes'] + ); + $this->assertEquals( + S3TransferManagerConfig::DEFAULT_RESPONSE_CHECKSUM_VALIDATION, + $config['response_checksum_validation'] + ); + } + + /** + * @return void + */ + public function testCustomConfigIsSet(): void { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $multipartDownloader = new PartGetMultipartDownloader( + $mockClient, + [], + [ + 'target_part_size_bytes' => 1024 * 1024 * 10, + 'response_checksum_validation' => 'when_required', + ], + new StreamDownloadHandler(), + ); + $config = $multipartDownloader->getConfig(); + $this->assertEquals( + 1024 * 1024 * 10, + $config['target_part_size_bytes'] + ); + $this->assertEquals( + 'when_required', + $config['response_checksum_validation'] + ); + } +} diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php new file mode 100644 index 0000000000..fc830eb487 --- /dev/null +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -0,0 +1,1316 @@ +getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + -> willReturnCallback(function ($command) use ($expected) + { + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result([ + 'UploadId' => 'FooUploadId' + ])); + } elseif ($command->getName() === 'UploadPart') { + return Create::promiseFor(new Result([ + 'ETag' => 'FooETag' + ])); + } + + if (isset($expected[$command->getName()])) { + $expectedOperationLevel = $expected['operations'][$command->getName()] ?? []; + foreach ($expectedOperationLevel as $key => $value) { + $this->assertArrayHasKey($key, $command); + $this->assertEquals($value, $command[$key]); + } + } + + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + -> willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + $requestArgs = [ + 'Key' => 'FooKey', + 'Bucket' => 'FooBucket', + ...$commandArgs + ]; + $tempDir = null; + if ($sourceConfig['type'] === 'stream') { + $source = Utils::streamFor( + str_repeat('*', $sourceConfig['size']) + ); + } elseif ($sourceConfig['type'] === 'no_seekable_stream') { + $source = Utils::streamFor( + str_repeat('*', $sourceConfig['size']) + ); + } elseif ($sourceConfig['type'] === 'file') { + $tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'multipart-uploader-test/'; + if (!is_dir($tempDir)) { + mkdir($tempDir, 0777, true); + } + $source = $tempDir . DIRECTORY_SEPARATOR . 'temp-file.txt'; + file_put_contents($source, str_repeat('*', $sourceConfig['size'])); + } else { + $this->fail("Unsupported Source type"); + } + + try { + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + $source, + $config, + ); + /** @var UploadResult $response */ + $response = $multipartUploader->promise()->wait(); + $snapshot = $multipartUploader->getCurrentSnapshot(); + + $this->assertInstanceOf(UploadResult::class, $response); + $this->assertCount($expected['parts'], $multipartUploader->getParts()); + $this->assertEquals($expected['bytesUploaded'], $snapshot->getTransferredBytes()); + $this->assertEquals($expected['bytesUploaded'], $snapshot->getTotalBytes()); + } finally { + if ($source instanceof StreamInterface) { + $source->close(); + } + + if (!is_null($tempDir)) { + TestsUtility::cleanUpDir($tempDir); + } + } + } + + /** + * @return array[] + */ + public function multipartUploadProvider(): array { + return [ + '5_parts_upload' => [ + 'source_config' => [ + 'type' => 'stream', + 'size' => 10240000 * 5 + ], + 'command_args' => [], + 'config' => [ + 'target_part_size_bytes' => 10240000, + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' + ], + 'expected' => [ + 'succeed' => true, + 'parts' => 5, + 'bytesUploaded' => 10240000 * 5, + ] + ], + '100_parts_upload' => [ + 'source_config' => [ + 'type' => 'stream', + 'size' => 10240000 * 100 + ], + 'command_args' => [], + 'config' => [ + 'target_part_size_bytes' => 10240000, + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' + ], + 'expected' => [ + 'succeed' => true, + 'parts' => 100, + 'bytesUploaded' => 10240000 * 100, + ] + ], + '5_parts_no_seekable_stream' => [ + 'source_config' => [ + 'type' => 'no_seekable_stream', + 'size' => 10240000 * 5 + ], + 'command_args' => [], + 'config' => [ + 'target_part_size_bytes' => 10240000, + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' + ], + 'expected' => [ + 'succeed' => true, + 'parts' => 5, + 'bytesUploaded' => 10240000 * 5, + ] + ], + '100_parts_no_seekable_stream' => [ + 'source_config' => [ + 'type' => 'no_seekable_stream', + 'size' => 10240000 * 100 + ], + 'command_args' => [], + 'config' => [ + 'target_part_size_bytes' => 10240000, + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' + ], + 'expected' => [ + 'succeed' => true, + 'parts' => 100, + 'bytesUploaded' => 10240000 * 100, + ] + ], + '100_parts_with_custom_checksum' => [ + 'source_config' => [ + 'type' => 'file', + 'size' => 10240000 * 100 + ], + 'command_args' => [ + 'ChecksumCRC32' => 'FooChecksum', + ], + 'config' => [ + 'target_part_size_bytes' => 10240000, + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' + ], + 'expected' => [ + 'succeed' => true, + 'parts' => 100, + 'bytesUploaded' => 10240000 * 100, + 'CreateMultipartUpload' => [ + 'ChecksumType' => 'FULL_OBJECT', + 'ChecksumAlgorithm' => 'CRC32' + ], + 'CompleteMultipartUpload' => [ + 'ChecksumType' => 'FULL_OBJECT', + 'ChecksumAlgorithm' => 'CRC32', + 'ChecksumCRC32' => 'FooChecksum', + ] + ] + ], + ]; + } + + /** + * @return S3ClientInterface + */ + private function getMultipartUploadS3Client(): S3ClientInterface + { + return new S3Client([ + 'region' => 'us-east-2', + 'http_handler' => function (RequestInterface $request) { + $uri = $request->getUri(); + // Create multipart upload + if ($uri->getQuery() === 'uploads') { + $body = << + + Foo + Test file + FooUploadId + +EOF; + return new Response(200, [], $body); + } + + // Parts upload + if (str_starts_with($request->getUri(), 'uploadId=') && str_contains($request->getUri(), 'partNumber=')) { + return new Response(200, ['ETag' => random_bytes(16)]); + } + + // Complete multipart upload + return new Response(200, [], null); + } + ]); + } + + + /** + * @param int $partSize + * @param bool $expectError + * + * @dataProvider validatePartSizeProvider + * + * @return void + */ + public function testValidatePartSize( + int $partSize, + bool $expectError + ): void { + if ($expectError) { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage( + "Part size config must be between " . AbstractMultipartUploader::PART_MIN_SIZE + ." and " . AbstractMultipartUploader::PART_MAX_SIZE . " bytes " + ."but it is configured to $partSize" + ); + } else { + $this->assertTrue(true); + } + + new MultipartUploader( + $this->getMultipartUploadS3Client(), + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + Utils::streamFor(), + [ + 'target_part_size_bytes' => $partSize, + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' + ], + ); + } + + /** + * @return array + */ + public function validatePartSizeProvider(): array { + return [ + 'part_size_over_max' => [ + 'part_size' => AbstractMultipartUploader::PART_MAX_SIZE + 1, + 'expectError' => true, + ], + 'part_size_under_min' => [ + 'part_size' => AbstractMultipartUploader::PART_MIN_SIZE - 1, + 'expectError' => true, + ], + 'part_size_between_valid_range_1' => [ + 'part_size' => AbstractMultipartUploader::PART_MAX_SIZE - 1, + 'expectError' => false, + ], + 'part_size_between_valid_range_2' => [ + 'part_size' => AbstractMultipartUploader::PART_MIN_SIZE + 1, + 'expectError' => false, + ] + ]; + } + + /** + * @param string|int $source + * @param bool $expectError + * + * @dataProvider invalidSourceStringProvider + * + * @return void + */ + public function testInvalidSourceStringThrowsException( + string|int $source, + bool $expectError + ): void + { + $tempDir = null; + if ($expectError) { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage( + "The source for this upload must be either a readable file path or a valid stream." + ); + } else { + $this->assertTrue(true); + $tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'multipart-upload-test'; + if (!is_dir($tempDir)) { + mkdir($tempDir, 0777, true); + } + + $source = $tempDir . DIRECTORY_SEPARATOR . $source; + file_put_contents($source, 'foo'); + } + + try { + new MultipartUploader( + $this->getMultipartUploadS3Client(), + ([ + 'Bucket' => 'test-bucket', + 'Key' => 'test-key' + ]), + $source, + [ + 'target_part_size_bytes' => 1024 * 1024 * 5, + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' + ] + ); + } finally { + if (!is_null($tempDir)) { + TestsUtility::cleanUpDir($tempDir); + } + } + } + + /** + * @return array[] + */ + public function invalidSourceStringProvider(): array { + return [ + 'invalid_source_file_path_1' => [ + 'source' => 'invalid', + 'expectError' => true, + ], + 'invalid_source_file_path_2' => [ + 'source' => 'invalid_2', + 'expectError' => true, + ], + 'invalid_source_3' => [ + 'source' => 12345, + 'expectError' => true, + ], + 'valid_source' => [ + 'source' => 'myfile.txt', + 'expectError' => false, + ], + ]; + } + + /** + * @return void + */ + public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void + { + $listener1 = $this->getMockBuilder(TransferListener::class)->getMock(); + $listener2 = $this->getMockBuilder(TransferListener::class)->getMock(); + $listener3 = $this->getMockBuilder(TransferListener::class)->getMock(); + + $listener1->expects($this->once())->method('transferInitiated'); + $listener1->expects($this->atLeastOnce())->method('bytesTransferred'); + $listener1->expects($this->once())->method('transferComplete'); + + $listener2->expects($this->once())->method('transferInitiated'); + $listener2->expects($this->atLeastOnce())->method('bytesTransferred'); + $listener2->expects($this->once())->method('transferComplete'); + + $listener3->expects($this->once())->method('transferInitiated'); + $listener3->expects($this->atLeastOnce())->method('bytesTransferred'); + $listener3->expects($this->once())->method('transferComplete'); + + $listenerNotifier = new TransferListenerNotifier([$listener1, $listener2, $listener3]); + + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result([ + 'UploadId' => 'TestUploadId' + ])); + } elseif ($command->getName() === 'UploadPart') { + return Create::promiseFor(new Result([ + 'ETag' => 'TestETag' + ])); + } + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $stream = Utils::streamFor(str_repeat('*', 10240000)); // 10MB + $requestArgs = [ + 'Key' => 'test-key', + 'Bucket' => 'test-bucket', + ]; + + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + $stream, + [ + 'target_part_size_bytes' => 5242880, // 5MB + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' + ], + null, + [], + null, + $listenerNotifier + ); + + $response = $multipartUploader->promise()->wait(); + $this->assertInstanceOf(UploadResult::class, $response); + } + + /** + * Test to make sure createMultipart, uploadPart, and completeMultipart + * operations are called. + * + * @return void + */ + public function testMultipartOperationsAreCalled(): void { + $operationsCalled = [ + 'CreateMultipartUpload' => false, + 'UploadPart' => false, + 'CompleteMultipartUpload' => false, + ]; + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) use (&$operationsCalled) { + $operationsCalled[$command->getName()] = true; + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result([ + 'UploadId' => 'TestUploadId' + ])); + } elseif ($command->getName() === 'UploadPart') { + return Create::promiseFor(new Result([ + 'ETag' => 'TestETag' + ])); + } + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $stream = Utils::streamFor(str_repeat('*', 1024 * 1024 * 5)); + $requestArgs = [ + 'Key' => 'test-key', + 'Bucket' => 'test-bucket', + ]; + + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + $stream, + [ + 'target_part_size_bytes' => 5242880, // 5MB + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' + ] + ); + + $multipartUploader->promise()->wait(); + foreach ($operationsCalled as $key => $value) { + $this->assertTrue($value, 'Operation {' . $key . '} was not called'); + } + } + + /** + * @param array $sourceConfig + * @param array $checksumConfig + * @param array $expectedOperationHeaders + * + * @dataProvider multipartUploadWithCustomChecksumProvider + * + * @return void + */ + public function testMultipartUploadWithCustomChecksum( + array $sourceConfig, + array $checksumConfig, + array $expectedOperationHeaders, + ): void { + // $operationsCalled: To make sure each expected operation is invoked. + $operationsCalled = []; + foreach (array_keys($expectedOperationHeaders) as $key) { + $operationsCalled[$key] = false; + } + + $s3Client = $this->getMultipartUploadS3Client(); + $s3Client->getHandlerList()->appendSign( + function (callable $handler) use (&$operationsCalled, $expectedOperationHeaders) { + return function ( + CommandInterface $command, + RequestInterface $request + ) use ($handler, &$operationsCalled, $expectedOperationHeaders) { + $operationsCalled[$command->getName()] = true; + $expectedHeaders = $expectedOperationHeaders[$command->getName()] ?? []; + $has = $expectedHeaders['has'] ?? []; + $hasNot = $expectedHeaders['has_not'] ?? []; + foreach ($has as $key => $value) { + $this->assertArrayHasKey($key, $request->getHeaders()); + $this->assertEquals($value, $request->getHeader($key)[0]); + } + + foreach ($hasNot as $value) { + $this->assertArrayNotHasKey($value, $request->getHeaders()); + } + + return $handler($command, $request); + }; + } + ); + $requestArgs = [ + 'Key' => 'FooKey', + 'Bucket' => 'FooBucket', + ...$checksumConfig, + ]; + $tempDir = null; + if ($sourceConfig['type'] === 'stream') { + $source = Utils::streamFor( + str_repeat($sourceConfig['char'], $sourceConfig['size']) + ); + } elseif ($sourceConfig['type'] === 'no_seekable_stream') { + $source = Utils::streamFor( + str_repeat($sourceConfig['char'], $sourceConfig['size']) + ); + } elseif ($sourceConfig['type'] === 'file') { + $tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'multipart-uploader-test/'; + if (!is_dir($tempDir)) { + mkdir($tempDir, 0777, true); + } + $source = $tempDir . DIRECTORY_SEPARATOR . 'temp-file.txt'; + file_put_contents($source, str_repeat($sourceConfig['char'], $sourceConfig['size'])); + } else { + $this->fail("Unsupported Source type"); + } + + try { + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + $source, + [ + 'target_part_size_bytes' => 5242880, // 5MB + 'concurrency' => 3, + 'request_checksum_calculation' => 'when_supported' + ] + ); + /** @var UploadResult $response */ + $response = $multipartUploader->promise()->wait(); + foreach ($operationsCalled as $key => $value) { + $this->assertTrue($value, 'Operation {' . $key . '} was not called'); + } + $this->assertInstanceOf(UploadResult::class, $response); + } finally { + if ($source instanceof StreamInterface) { + $source->close(); + } + + if (!is_null($tempDir)) { + TestsUtility::cleanUpDir($tempDir); + } + } + } + + /** + * @return array + */ + public function multipartUploadWithCustomChecksumProvider(): array { + return [ + 'custom_checksum_crc32_1' => [ + 'source_config' => [ + 'type' => 'stream', + 'size' => 1024 * 1024 * 20, + 'char' => '*' + ], + 'checksum_config' => [ + 'ChecksumCRC32' => '+IIKcQ==', + ], + 'expected_operation_headers' => [ + 'CreateMultipartUpload' => [ + 'has' => [ + 'x-amz-checksum-algorithm' => 'crc32', + 'x-amz-checksum-type' => 'FULL_OBJECT' + ] + ], + 'UploadPart' => [ + 'has_not' => [ + 'x-amz-checksum-algorithm', + 'x-amz-checksum-type', + 'x-amz-checksum-crc32' + ] + ], + 'CompleteMultipartUpload' => [ + 'has' => [ + 'x-amz-checksum-crc32' => '+IIKcQ==', + 'x-amz-checksum-type' => 'FULL_OBJECT', + ], + ] + ] + ] + ]; + } + + /** + * @return void + */ + public function testMultipartUploadAbort() { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('Upload failed'); + $abortMultipartCalled = false; + $abortMultipartCalledTimes = 0; + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) + use (&$abortMultipartCalled, &$abortMultipartCalledTimes) { + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result([ + 'UploadId' => 'TestUploadId' + ])); + } elseif ($command->getName() === 'UploadPart') { + if ($command['PartNumber'] == 3) { + return Create::rejectionFor(new S3TransferException('Upload failed')); + } + } elseif ($command->getName() === 'AbortMultipartUpload') { + $abortMultipartCalled = true; + $abortMultipartCalledTimes++; + } + + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + $requestArgs = [ + 'Bucket' => 'test-bucket', + 'Key' => 'test-key', + ]; + $source = Utils::streamFor(str_repeat('*', 1024 * 1024 * 20)); + try { + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + $source, + [ + 'target_part_size_bytes' => 5242880, // 5MB + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' + ] + ); + $multipartUploader->promise()->wait(); + } finally { + $this->assertTrue($abortMultipartCalled); + $this->assertEquals(1, $abortMultipartCalledTimes); + $source->close(); + } + } + + /** + * @return void + */ + public function testTransferListenerNotifierNotifiesListenersOnFailure(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Upload failed'); + + $listener1 = $this->getMockBuilder(TransferListener::class)->getMock(); + $listener2 = $this->getMockBuilder(TransferListener::class)->getMock(); + + $listener1->expects($this->once())->method('transferInitiated'); + $listener1->expects($this->once())->method('transferFail'); + + $listener2->expects($this->once())->method('transferInitiated'); + $listener2->expects($this->once())->method('transferFail'); + + $listenerNotifier = new TransferListenerNotifier([$listener1, $listener2]); + + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result([ + 'UploadId' => 'TestUploadId' + ])); + } elseif ($command->getName() === 'UploadPart') { + return Create::rejectionFor(new \Exception('Upload failed')); + } + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $stream = Utils::streamFor(str_repeat('*', 10240000)); + $requestArgs = [ + 'Key' => 'test-key', + 'Bucket' => 'test-bucket', + ]; + + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + $stream, + [ + 'target_part_size_bytes' => 5242880, // 5MB + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' + ], + null, + [], + null, + $listenerNotifier + ); + + $multipartUploader->promise()->wait(); + } + + /** + * @return void + */ + public function testTransferListenerNotifierWithEmptyListeners(): void + { + $listenerNotifier = new TransferListenerNotifier([]); + + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result([ + 'UploadId' => 'TestUploadId' + ])); + } elseif ($command->getName() === 'UploadPart') { + return Create::promiseFor(new Result([ + 'ETag' => 'TestETag' + ])); + } + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $stream = Utils::streamFor(str_repeat('*', 1024)); + $requestArgs = [ + 'Key' => 'test-key', + 'Bucket' => 'test-bucket', + ]; + + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + $stream, + [ + 'target_part_size_bytes' => 5242880, // 5MB + 'concurrency' => 1, + ], + null, + [], + null, + $listenerNotifier + ); + + $response = $multipartUploader->promise()->wait(); + $this->assertInstanceOf(UploadResult::class, $response); + } + + /** + * This test makes sure that when full object checksum type is resolved + * then, if a custom algorithm provide is not CRC family then it should fail. + * + * @param array $checksumConfig + * @param bool $expectsError + * + * @dataProvider fullObjectChecksumWorksJustWithCRCProvider + * + * @return void + */ + public function testFullObjectChecksumWorksJustWithCRC( + array $checksumConfig, + bool $expectsError + ): void + { + $s3Client = $this->getMultipartUploadS3Client(); + $requestArgs = [ + 'Key' => 'FooKey', + 'Bucket' => 'FooBucket', + ...$checksumConfig, + ]; + + try { + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + Utils::streamFor(''), + [ + 'target_part_size_bytes' => 5242880, // 5MB + 'concurrency' => 3, + 'request_checksum_calculation' => 'when_supported' + ] + ); + $response = $multipartUploader->promise()->wait(); + if ($expectsError) { + $this->fail("An expected exception has not been raised"); + } else { + $this->assertInstanceOf(UploadResult::class, $response); + } + } catch (S3TransferException $exception) { + if ($expectsError) { + $this->assertEquals( + "Full object checksum algorithm must be `CRC` family base.", + $exception->getMessage() + ); + } else { + $this->fail("An exception has been thrown when not expected"); + } + } + } + + /** + * @return Generator + */ + public function fullObjectChecksumWorksJustWithCRCProvider(): Generator { + yield 'sha_256_should_fail' => [ + 'checksum_config' => [ + 'ChecksumSHA256' => '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=' + ], + 'expects_error' => true, + ]; + + yield 'sha_1_should_fail' => [ + 'checksum_config' => [ + 'ChecksumSHA1' => '2jmj7l5rSw0yVb/vlWAYkK/YBwk=' + ], + 'expects_error' => true, + ]; + + yield 'crc32_should_fail' => [ + 'checksum_config' => [ + 'ChecksumCRC32' => 'AAAAAA==' + ], + 'expects_error' => false, + ]; + } + + /** + * @param array $sourceConfig + * @param array $requestArgs + * @param array $expectedInputArgs + * @param bool $expectsError + * @param int|null $errorOnPartNumber + * @return void + * @dataProvider inputArgumentsPerOperationProvider + */ + public function testInputArgumentsPerOperation( + array $sourceConfig, + array $requestArgs, + array $expectedInputArgs, + bool $expectsError, + ?int $errorOnPartNumber = null + ): void + { + try { + $calledCommands = array_map(function () { + return 1; + }, $expectedInputArgs); + $this->assertNotEmpty( + $calledCommands, + "Expected input arguments should not be empty" + ); + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method( + 'getCommand' + )->willReturnCallback( + function ($commandName, $args) + use (&$calledCommands, $expectedInputArgs) { + if (isset($expectedInputArgs[$commandName])) { + $calledCommands[$commandName] = 0; + $expected = $expectedInputArgs[$commandName]; + foreach ($expected as $key => $value) { + $this->assertArrayHasKey($key, $args); + $this->assertEquals( + $value, + $args[$key] + ); + } + } + + return new Command($commandName, $args); + }); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) + use ($errorOnPartNumber, $expectsError) { + if ($command->getName() === 'UploadPart') { + if ($expectsError && $command['PartNumber'] === $errorOnPartNumber) { + return Create::rejectionFor( + new S3TransferException('Upload failed') + ); + } + + return Create::promiseFor(new Result([])); + } + + return match ($command->getName()) { + 'CreateMultipartUpload' => Create::promiseFor(new Result([ + 'UploadId' => 'FooUploadId', + ])), + 'CompleteMultipartUpload', + 'AbortMultipartUpload', + 'PutObject' => Create::promiseFor(new Result([])), + default => null, + }; + }); + $source = Utils::streamFor( + str_repeat( + $sourceConfig['char'], + $sourceConfig['size'] + ) + ); + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + Utils::streamFor($source) + ); + $multipartUploader->upload(); + foreach ($calledCommands as $key => $value) { + $this->assertEquals( + 0, + $value, + "$key not called" + ); + } + $this->assertFalse( + $expectsError, + "Expected error while uploading" + ); + } catch (S3TransferException $exception) { + $this->assertTrue( + $expectsError, + "Unexpected error while uploading" . "\n" . $exception->getMessage() + ); + } + } + + /** + * @return Generator + */ + public function inputArgumentsPerOperationProvider(): Generator + { + yield 'test_input_fields_are_copied_without_custom_checksums' => [ + // Source config to generate a stub body + 'source_config' => [ + 'size' => 1024 * 1024 * 10, + 'char' => '#' + ], + 'request_args' => [ + "ACL" => 'private', + "Bucket" => 'test-bucket', + "BucketKeyEnabled" => 'test-bucket-key-enabled', + "CacheControl" => 'test-cache-control', + "ContentDisposition" => 'test-content-disposition', + "ContentEncoding" => 'test-content-encoding', + "ContentLanguage" => 'test-content-language', + "ContentType" => 'test-content-type', + "ExpectedBucketOwner" => 'test-bucket-owner', + "Expires" => 'test-expires', + "GrantFullControl" => 'test-grant-control', + "GrantRead" => 'test-grant-control', + "GrantReadACP" => 'test-grant-control', + "GrantWriteACP" => 'test-grant-control', + "Key" => 'test-key', + "Metadata" => [ + 'metadata-1' => 'test-metadata-1', + 'metadata-2' => 'test-metadata-2', + ], + "ObjectLockLegalHoldStatus" => 'test-object-lock-legal-hold', + "ObjectLockMode" => 'test-object-lock-mode', + "ObjectLockRetainUntilDate" => 'test-object-lock-retain-until', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + "SSEKMSEncryptionContext" => 'test-sse-kms-encryption-context', + "SSEKMSKeyId" => 'test-sse-kms-key-id', + "ServerSideEncryption" => 'test-server-side-encryption', + "StorageClass" => 'test-storage-class', + "Tagging" => 'test-tagging', + "WebsiteRedirectLocation" => 'test-website-redirect-location', + ], + 'expected_input_args' => [ + 'CreateMultipartUpload' => [ + "ACL" => 'private', + "Bucket" => 'test-bucket', + "BucketKeyEnabled" => 'test-bucket-key-enabled', + "CacheControl" => 'test-cache-control', + "ContentDisposition" => 'test-content-disposition', + "ContentEncoding" => 'test-content-encoding', + "ContentLanguage" => 'test-content-language', + "ContentType" => 'test-content-type', + "ExpectedBucketOwner" => 'test-bucket-owner', + "Expires" => 'test-expires', + "GrantFullControl" => 'test-grant-control', + "GrantRead" => 'test-grant-control', + "GrantReadACP" => 'test-grant-control', + "GrantWriteACP" => 'test-grant-control', + "Key" => 'test-key', + "Metadata" => [ + 'metadata-1' => 'test-metadata-1', + 'metadata-2' => 'test-metadata-2', + ], + "ObjectLockLegalHoldStatus" => 'test-object-lock-legal-hold', + "ObjectLockMode" => 'test-object-lock-mode', + "ObjectLockRetainUntilDate" => 'test-object-lock-retain-until', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + "SSEKMSEncryptionContext" => 'test-sse-kms-encryption-context', + "SSEKMSKeyId" => 'test-sse-kms-key-id', + "ServerSideEncryption" => 'test-server-side-encryption', + "StorageClass" => 'test-storage-class', + "Tagging" => 'test-tagging', + "WebsiteRedirectLocation" => 'test-website-redirect-location', + ], + 'UploadPart' => [ + "Bucket" => 'test-bucket', + "UploadId" => "FooUploadId", // Fixed from test + "ExpectedBucketOwner" => 'test-bucket-owner', + "Key" => 'test-key', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + ], + 'CompleteMultipartUpload' => [ + "Bucket" => 'test-bucket', + "UploadId" => "FooUploadId", // Fixed from test + "ExpectedBucketOwner" => 'test-bucket-owner', + "Key" => 'test-key', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + ], + ], + 'expects_error' => false, + ]; + + yield 'test_input_fields_are_copied_with_custom_checksum_crc32' => [ + // Source config to generate a stub body + 'source_config' => [ + 'size' => 1024 * 1024 * 10, + 'char' => '#' + ], + 'request_args' => [ + 'ChecksumCRC32' => 'tx0IFA==', + "ACL" => 'private', + "Bucket" => 'test-bucket', + "BucketKeyEnabled" => 'test-bucket-key-enabled', + "CacheControl" => 'test-cache-control', + "ContentDisposition" => 'test-content-disposition', + "ContentEncoding" => 'test-content-encoding', + "ContentLanguage" => 'test-content-language', + "ContentType" => 'test-content-type', + "ExpectedBucketOwner" => 'test-bucket-owner', + "Expires" => 'test-expires', + "GrantFullControl" => 'test-grant-control', + "GrantRead" => 'test-grant-control', + "GrantReadACP" => 'test-grant-control', + "GrantWriteACP" => 'test-grant-control', + "Key" => 'test-key', + "Metadata" => [ + 'metadata-1' => 'test-metadata-1', + 'metadata-2' => 'test-metadata-2', + ], + "ObjectLockLegalHoldStatus" => 'test-object-lock-legal-hold', + "ObjectLockMode" => 'test-object-lock-mode', + "ObjectLockRetainUntilDate" => 'test-object-lock-retain-until', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + "SSEKMSEncryptionContext" => 'test-sse-kms-encryption-context', + "SSEKMSKeyId" => 'test-sse-kms-key-id', + "ServerSideEncryption" => 'test-server-side-encryption', + "StorageClass" => 'test-storage-class', + "Tagging" => 'test-tagging', + "WebsiteRedirectLocation" => 'test-website-redirect-location', + ], + 'expected_input_args' => [ + 'CreateMultipartUpload' => [ + "ACL" => 'private', + "Bucket" => 'test-bucket', + "BucketKeyEnabled" => 'test-bucket-key-enabled', + "CacheControl" => 'test-cache-control', + "ContentDisposition" => 'test-content-disposition', + "ContentEncoding" => 'test-content-encoding', + "ContentLanguage" => 'test-content-language', + "ContentType" => 'test-content-type', + "ExpectedBucketOwner" => 'test-bucket-owner', + "Expires" => 'test-expires', + "GrantFullControl" => 'test-grant-control', + "GrantRead" => 'test-grant-control', + "GrantReadACP" => 'test-grant-control', + "GrantWriteACP" => 'test-grant-control', + "Key" => 'test-key', + "Metadata" => [ + 'metadata-1' => 'test-metadata-1', + 'metadata-2' => 'test-metadata-2', + ], + "ObjectLockLegalHoldStatus" => 'test-object-lock-legal-hold', + "ObjectLockMode" => 'test-object-lock-mode', + "ObjectLockRetainUntilDate" => 'test-object-lock-retain-until', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + "SSEKMSEncryptionContext" => 'test-sse-kms-encryption-context', + "SSEKMSKeyId" => 'test-sse-kms-key-id', + "ServerSideEncryption" => 'test-server-side-encryption', + "StorageClass" => 'test-storage-class', + "Tagging" => 'test-tagging', + "WebsiteRedirectLocation" => 'test-website-redirect-location', + 'ChecksumType' => 'FULL_OBJECT', + 'ChecksumAlgorithm' => 'crc32', + ], + 'UploadPart' => [ + "Bucket" => 'test-bucket', + "UploadId" => "FooUploadId", // Fixed from test + "ExpectedBucketOwner" => 'test-bucket-owner', + "Key" => 'test-key', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + ], + 'CompleteMultipartUpload' => [ + "Bucket" => 'test-bucket', + "UploadId" => "FooUploadId", // Fixed from test + "ExpectedBucketOwner" => 'test-bucket-owner', + "Key" => 'test-key', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + 'ChecksumType' => 'FULL_OBJECT', + 'ChecksumCRC32' => 'tx0IFA==', // From default algorithm used + ], + ], + 'expects_error' => false, + ]; + + yield 'test_input_fields_are_copied_with_error' => [ + // Source config to generate a stub body + 'source_config' => [ + 'size' => 1024 * 1024 * 10, + 'char' => '#' + ], + 'request_args' => [ + 'ChecksumCRC32' => 'tx0IFA==', + "ACL" => 'private', + "Bucket" => 'test-bucket', + "BucketKeyEnabled" => 'test-bucket-key-enabled', + "CacheControl" => 'test-cache-control', + "ContentDisposition" => 'test-content-disposition', + "ContentEncoding" => 'test-content-encoding', + "ContentLanguage" => 'test-content-language', + "ContentType" => 'test-content-type', + "ExpectedBucketOwner" => 'test-bucket-owner', + "Expires" => 'test-expires', + "GrantFullControl" => 'test-grant-control', + "GrantRead" => 'test-grant-control', + "GrantReadACP" => 'test-grant-control', + "GrantWriteACP" => 'test-grant-control', + "Key" => 'test-key', + "Metadata" => [ + 'metadata-1' => 'test-metadata-1', + 'metadata-2' => 'test-metadata-2', + ], + "ObjectLockLegalHoldStatus" => 'test-object-lock-legal-hold', + "ObjectLockMode" => 'test-object-lock-mode', + "ObjectLockRetainUntilDate" => 'test-object-lock-retain-until', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + "SSEKMSEncryptionContext" => 'test-sse-kms-encryption-context', + "SSEKMSKeyId" => 'test-sse-kms-key-id', + "ServerSideEncryption" => 'test-server-side-encryption', + "StorageClass" => 'test-storage-class', + "Tagging" => 'test-tagging', + "WebsiteRedirectLocation" => 'test-website-redirect-location', + ], + 'expected_input_args' => [ + 'CreateMultipartUpload' => [ + "ACL" => 'private', + "Bucket" => 'test-bucket', + "BucketKeyEnabled" => 'test-bucket-key-enabled', + "CacheControl" => 'test-cache-control', + "ContentDisposition" => 'test-content-disposition', + "ContentEncoding" => 'test-content-encoding', + "ContentLanguage" => 'test-content-language', + "ContentType" => 'test-content-type', + "ExpectedBucketOwner" => 'test-bucket-owner', + "Expires" => 'test-expires', + "GrantFullControl" => 'test-grant-control', + "GrantRead" => 'test-grant-control', + "GrantReadACP" => 'test-grant-control', + "GrantWriteACP" => 'test-grant-control', + "Key" => 'test-key', + "Metadata" => [ + 'metadata-1' => 'test-metadata-1', + 'metadata-2' => 'test-metadata-2', + ], + "ObjectLockLegalHoldStatus" => 'test-object-lock-legal-hold', + "ObjectLockMode" => 'test-object-lock-mode', + "ObjectLockRetainUntilDate" => 'test-object-lock-retain-until', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + "SSEKMSEncryptionContext" => 'test-sse-kms-encryption-context', + "SSEKMSKeyId" => 'test-sse-kms-key-id', + "ServerSideEncryption" => 'test-server-side-encryption', + "StorageClass" => 'test-storage-class', + "Tagging" => 'test-tagging', + "WebsiteRedirectLocation" => 'test-website-redirect-location', + 'ChecksumType' => 'FULL_OBJECT', + 'ChecksumAlgorithm' => 'crc32', + ], + 'UploadPart' => [ + "Bucket" => 'test-bucket', + "UploadId" => "FooUploadId", // Fixed from test + "ExpectedBucketOwner" => 'test-bucket-owner', + "Key" => 'test-key', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + ], + 'AbortMultipartUpload' => [ + "Bucket" => 'test-bucket', + "UploadId" => "FooUploadId", // Fixed from test + "ExpectedBucketOwner" => 'test-bucket-owner', + "Key" => 'test-key', + "RequestPayer" => 'test-request-payer', + ], + ], + 'expects_error' => true, + 'error_on_part_number' => 2 + ]; + } +} diff --git a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php new file mode 100644 index 0000000000..c1d3b4d50f --- /dev/null +++ b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php @@ -0,0 +1,311 @@ +getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $remainingToTransfer = $objectSizeInBytes; + $mockClient->method('executeAsync') + -> willReturnCallback(function ($command) + use ( + $objectSizeInBytes, + $partsCount, + $targetPartSize, + &$remainingToTransfer + ) { + $currentPartLength = min( + $targetPartSize, + $remainingToTransfer + ); + $from = $objectSizeInBytes - $remainingToTransfer; + $to = $from + $currentPartLength; + $remainingToTransfer -= $currentPartLength; + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor('Foo'), + 'PartsCount' => $partsCount, + 'PartNumber' => $command['PartNumber'], + 'ContentRange' => "bytes $from-$to/$objectSizeInBytes", + 'ContentLength' => $currentPartLength + ])); + }); + $mockClient->method('getCommand') + -> willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $downloader = new PartGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'FooBucket', + 'Key' => $objectKey, + ], + [ + 'minimum_part_size' => $targetPartSize, + ], + new StreamDownloadHandler() + ); + /** @var DownloadResult $response */ + $response = $downloader->promise()->wait(); + $snapshot = $downloader->getCurrentSnapshot(); + + $this->assertInstanceOf(DownloadResult::class, $response); + $this->assertEquals($objectKey, $snapshot->getIdentifier()); + $this->assertEquals($objectSizeInBytes, $snapshot->getTotalBytes()); + $this->assertEquals($objectSizeInBytes, $snapshot->getTransferredBytes()); + $this->assertEquals($partsCount, $downloader->getObjectPartsCount()); + $this->assertEquals($partsCount, $downloader->getCurrentPartNo()); + } + + /** + * Part get multipart downloader data provider. + * + * @return array[] + */ + public function partGetMultipartDownloaderProvider(): array { + return [ + [ + 'objectKey' => 'ObjectKey_1', + 'objectSizeInBytes' => 1024 * 10, + 'targetPartSize' => 1024 * 2, + ], + [ + 'objectKey' => 'ObjectKey_2', + 'objectSizeInBytes' => 1024 * 100, + 'targetPartSize' => 1024 * 5, + ], + [ + 'objectKey' => 'ObjectKey_3', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 512, + ], + [ + 'objectKey' => 'ObjectKey_4', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 256, + ], + [ + 'objectKey' => 'ObjectKey_5', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 458, + ] + ]; + } + + /** + * Tests nextCommand method increments part number correctly. + * + * @return void + */ + public function testNextCommandIncrementsPartNumber(): void + { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $downloader = new PartGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ], + [], + new StreamDownloadHandler() + ); + + // Use reflection to test the protected nextCommand method + $reflection = new \ReflectionClass($downloader); + $nextCommandMethod = $reflection->getMethod('nextCommand'); + + // First call should set part number to 1 + $command1 = $nextCommandMethod->invoke($downloader); + $this->assertEquals(1, $command1['PartNumber']); + $this->assertEquals(1, $downloader->getCurrentPartNo()); + + // Second call should increment to 2 + $command2 = $nextCommandMethod->invoke($downloader); + $this->assertEquals(2, $command2['PartNumber']); + $this->assertEquals(2, $downloader->getCurrentPartNo()); + } + + /** + * Tests computeObjectDimensions method correctly calculates object size. + * + * @return void + */ + public function testComputeObjectDimensions(): void + { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $downloader = new PartGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ], + [], + new StreamDownloadHandler() + ); + + // Use reflection to test the protected computeObjectDimensions method + $reflection = new \ReflectionClass($downloader); + $computeObjectDimensionsMethod = $reflection->getMethod('computeObjectDimensions'); + + $result = new Result([ + 'PartsCount' => 5, + 'ContentRange' => 'bytes 0-1023/2048' + ]); + + $computeObjectDimensionsMethod->invoke($downloader, $result); + + $this->assertEquals(5, $downloader->getObjectPartsCount()); + $this->assertEquals(2048, $downloader->getObjectSizeInBytes()); + } + + /** + * Test IfMatch is properly called in each part get operation. + * + * @param int $objectSizeInBytes + * @param int $targetPartSize + * @param string $eTag + * + * @dataProvider ifMatchIsPresentInEachPartRequestAfterFirstProvider + * + * @return void + */ + public function testIfMatchIsPresentInEachRangeRequestAfterFirst( + int $objectSizeInBytes, + int $targetPartSize, + string $eTag + ): void + { + $firstRequestCalled = false; + $ifMatchCalledTimes = 0; + $partsCount = ceil($objectSizeInBytes / $targetPartSize); + $remainingToTransfer = $objectSizeInBytes; + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) + use ($eTag, &$ifMatchCalledTimes) { + if (isset($args['IfMatch'])) { + $ifMatchCalledTimes++; + $this->assertEquals( + $eTag, + $args['IfMatch'] + ); + } + + return new Command($commandName, $args); + }); + $s3Client->method('executeAsync') + -> willReturnCallback(function ($command) + use ( + $eTag, + $objectSizeInBytes, + $partsCount, + $targetPartSize, + &$remainingToTransfer, + &$firstRequestCalled + ) { + $firstRequestCalled = true; + $currentPartLength = min( + $targetPartSize, + $remainingToTransfer + ); + $from = $objectSizeInBytes - $remainingToTransfer; + $to = $from + $currentPartLength; + $remainingToTransfer -= $currentPartLength; + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor('Foo'), + 'PartsCount' => $partsCount, + 'PartNumber' => $command['PartNumber'], + 'ContentRange' => "bytes $from-$to/$objectSizeInBytes", + 'ContentLength' => $currentPartLength, + 'ETag' => $eTag, + ])); + }); + $requestArgs = [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ]; + $partGetMultipartDownloader = new PartGetMultipartDownloader( + $s3Client, + $requestArgs, + [ + 'target_part_size_bytes' => $targetPartSize, + ] + ); + $partGetMultipartDownloader->download(); + $this->assertTrue($firstRequestCalled); + $this->assertEquals( + $partsCount - 1, + $ifMatchCalledTimes + ); + } + + /** + * @return Generator + */ + public function ifMatchIsPresentInEachPartRequestAfterFirstProvider(): Generator + { + yield 'multipart_download_with_3_parts_1' => [ + 'object_size_in_bytes' => 1024 * 1024 * 20, + 'target_part_size_bytes' => 8 * 1024 * 1024, + 'eTag' => 'ETag1234', + ]; + + yield 'multipart_download_with_2_parts_1' => [ + 'object_size_in_bytes' => 1024 * 1024 * 16, + 'target_part_size_bytes' => 8 * 1024 * 1024, + 'eTag' => 'ETag12345678', + ]; + + yield 'multipart_download_with_5_parts_1' => [ + 'object_size_in_bytes' => 1024 * 1024 * 40, + 'target_part_size_bytes' => 8 * 1024 * 1024, + 'eTag' => 'ETag12345678', + ]; + } +} diff --git a/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php new file mode 100644 index 0000000000..393bb13400 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php @@ -0,0 +1,279 @@ +assertEquals( + ConsoleProgressBar::DEFAULT_PROGRESS_BAR_WIDTH, + $progressBar->getProgressBarWidth() + ); + $this->assertEquals( + ConsoleProgressBar::DEFAULT_PROGRESS_BAR_CHAR, + $progressBar->getProgressBarChar() + ); + $this->assertEquals( + 0, + $progressBar->getPercentCompleted() + ); + $this->assertInstanceOf( + ColoredTransferProgressBarFormat::class, + $progressBar->getProgressBarFormat() + ); + } + + /** + * Tests the percent is updated properly. + * + * @return void + */ + public function testSetPercentCompleted(): void + { + $progressBar = new ConsoleProgressBar(); + $progressBar->setPercentCompleted(10); + $this->assertEquals(10, $progressBar->getPercentCompleted()); + $progressBar->setPercentCompleted(100); + $this->assertEquals(100, $progressBar->getPercentCompleted()); + } + + /** + * @return void + */ + public function testSetCustomValues(): void + { + $progressBar = new ConsoleProgressBar( + progressBarChar: '-', + progressBarWidth: 10, + percentCompleted: 25, + progressBarFormat: new PlainProgressBarFormat() + ); + $this->assertEquals('-', $progressBar->getProgressBarChar()); + $this->assertEquals(10, $progressBar->getProgressBarWidth()); + $this->assertEquals(25, $progressBar->getPercentCompleted()); + $this->assertInstanceOf( + PlainProgressBarFormat::class, + $progressBar->getProgressBarFormat() + ); + } + + /** + * To make sure the percent is not over 100. + * + * @return void + */ + public function testPercentIsNotOverOneHundred(): void + { + $progressBar = new ConsoleProgressBar(); + $progressBar->setPercentCompleted(150); + $this->assertEquals(100, $progressBar->getPercentCompleted()); + } + + /** + * @param string $progressBarChar + * @param int $progressBarWidth + * @param int $percentCompleted + * @param ProgressBarFormatTest $progressBarFormat + * @param array $progressBarFormatArgs + * @param string $expectedOutput + * + * @return void + * @dataProvider progressBarRenderingProvider + * + */ + public function testProgressBarRendering( + string $progressBarChar, + int $progressBarWidth, + int $percentCompleted, + ProgressBarFormat $progressBarFormat, + array $progressBarFormatArgs, + string $expectedOutput + ): void + { + $progressBarFormat->setArgs($progressBarFormatArgs); + $progressBar = new ConsoleProgressBar( + $progressBarChar, + $progressBarWidth, + $percentCompleted, + $progressBarFormat, + ); + + $this->assertEquals($expectedOutput, $progressBar->render()); + } + + /** + * Data provider for testing progress bar rendering. + * + * @return array + */ + public function progressBarRenderingProvider(): array + { + return [ + 'plain_progress_bar_format_1' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 50, + 'percent_completed' => 15, + 'progress_bar_format' => new PlainProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + ], + 'expected_output' => "FooObject:\n[######## ] 15%" + ], + 'plain_progress_bar_format_2' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 50, + 'percent_completed' => 45, + 'progress_bar_format' => new PlainProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + ], + 'expected_output' => "FooObject:\n[####################### ] 45%" + ], + 'plain_progress_bar_format_3' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 50, + 'percent_completed' => 100, + 'progress_bar_format' => new PlainProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + ], + 'expected_output' => "FooObject:\n[##################################################] 100%" + ], + 'plain_progress_bar_format_4' => [ + 'progress_bar_char' => '.', + 'progress_bar_width' => 50, + 'percent_completed' => 100, + 'progress_bar_format' => new PlainProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + ], + 'expected_output' => "FooObject:\n[..................................................] 100%" + ], + 'transfer_progress_bar_format_1' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 50, + 'percent_completed' => 23, + 'progress_bar_format' => new TransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + 'transferred' => 23, + 'to_be_transferred' => 100, + 'unit' => 'B' + ], + 'expected_output' => "FooObject:\n[############ ] 23% 23/100 B" + ], + 'transfer_progress_bar_format_2' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 25, + 'percent_completed' => 75, + 'progress_bar_format' => new TransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + 'transferred' => 75, + 'to_be_transferred' => 100, + 'unit' => 'B' + ], + 'expected_output' => "FooObject:\n[################### ] 75% 75/100 B" + ], + 'transfer_progress_bar_format_3' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 30, + 'percent_completed' => 100, + 'progress_bar_format' => new TransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + 'transferred' => 100, + 'to_be_transferred' => 100, + 'unit' => 'B' + ], + 'expected_output' => "FooObject:\n[##############################] 100% 100/100 B" + ], + 'transfer_progress_bar_format_4' => [ + 'progress_bar_char' => '*', + 'progress_bar_width' => 30, + 'percent_completed' => 100, + 'progress_bar_format' => new TransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + 'transferred' => 100, + 'to_be_transferred' => 100, + 'unit' => 'B' + ], + 'expected_output' => "FooObject:\n[******************************] 100% 100/100 B" + ], + 'colored_progress_bar_format_1' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 20, + 'percent_completed' => 10, + 'progress_bar_format' => new ColoredTransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'ObjectName_1', + 'transferred' => 10, + 'to_be_transferred' => 100, + 'unit' => 'B' + ], + 'expected_output' => "ObjectName_1:\n\033[30m[## ] 10% 10/100 B \033[0m" + ], + 'colored_progress_bar_format_2' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 20, + 'percent_completed' => 50, + 'progress_bar_format' => new ColoredTransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'ObjectName_2', + 'transferred' => 50, + 'to_be_transferred' => 100, + 'unit' => 'B', + 'color_code' => ColoredTransferProgressBarFormat::BLUE_COLOR_CODE + ], + 'expected_output' => "ObjectName_2:\n\033[34m[########## ] 50% 50/100 B \033[0m" + ], + 'colored_progress_bar_format_3' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 25, + 'percent_completed' => 100, + 'progress_bar_format' => new ColoredTransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'ObjectName_3', + 'transferred' => 100, + 'to_be_transferred' => 100, + 'unit' => 'B', + 'color_code' => ColoredTransferProgressBarFormat::GREEN_COLOR_CODE + ], + 'expected_output' => "ObjectName_3:\n\033[32m[#########################] 100% 100/100 B \033[0m" + ], + 'colored_progress_bar_format_4' => [ + 'progress_bar_char' => '=', + 'progress_bar_width' => 25, + 'percent_completed' => 100, + 'progress_bar_format' => new ColoredTransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'ObjectName_3', + 'transferred' => 100, + 'to_be_transferred' => 100, + 'unit' => 'B', + 'color_code' => ColoredTransferProgressBarFormat::GREEN_COLOR_CODE + ], + 'expected_output' => "ObjectName_3:\n\033[32m[=========================] 100% 100/100 B \033[0m" + ] + ]; + } +} diff --git a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php new file mode 100644 index 0000000000..debe0276c1 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php @@ -0,0 +1,733 @@ +assertEquals([], $progressTracker->getSingleProgressTrackers()); + $this->assertEquals(STDOUT, $progressTracker->getOutput()); + $this->assertEquals(0, $progressTracker->getTransferCount()); + $this->assertEquals(0, $progressTracker->getCompleted()); + $this->assertEquals(0, $progressTracker->getFailed()); + } + + /** + * @dataProvider customInitializationProvider + * + * @param array $progressTrackers + * @param mixed $output + * @param int $transferCount + * @param int $completed + * @param int $failed + * + * @return void + */ + public function testCustomInitialization( + array $progressTrackers, + mixed $output, + int $transferCount, + int $completed, + int $failed + ): void + { + $progressTracker = new MultiProgressTracker( + $progressTrackers, + $output, + $transferCount, + $completed, + $failed + ); + $this->assertSame($output, $progressTracker->getOutput()); + $this->assertSame($transferCount, $progressTracker->getTransferCount()); + $this->assertSame($completed, $progressTracker->getCompleted()); + $this->assertSame($failed, $progressTracker->getFailed()); + } + + /** + * @param ProgressBarFactoryInterface $progressBarFactory + * @param callable $eventInvoker + * @param array $expectedOutputs + * + * @return void + * @dataProvider multiProgressTrackerProvider + * + */ + public function testMultiProgressTracker( + Closure $progressBarFactory, + callable $eventInvoker, + array $expectedOutputs, + ): void + { + $output = fopen("php://temp", "w+"); + $progressTracker = new MultiProgressTracker( + output: $output, + progressBarFactory: $progressBarFactory + ); + $eventInvoker($progressTracker); + + $this->assertEquals( + $expectedOutputs['transfer_count'], + $progressTracker->getTransferCount() + ); + $this->assertEquals( + $expectedOutputs['completed'], + $progressTracker->getCompleted() + ); + $this->assertEquals( + $expectedOutputs['failed'], + $progressTracker->getFailed() + ); + $progress = $expectedOutputs['progress']; + if (is_array($progress)) { + $progress = join('', $progress); + } + rewind($output); + $this->assertEquals( + $progress, + stream_get_contents($output), + ); + } + + /** + * @return array + */ + public function customInitializationProvider(): array + { + return [ + 'custom_initialization_1' => [ + 'progress_trackers' => [ + new SingleProgressTracker(), + new SingleProgressTracker(), + ], + 'output' => STDOUT, + 'transfer_count' => 20, + 'completed' => 20, + 'failed' => 0, + ], + 'custom_initialization_2' => [ + 'progress_trackers' => [ + new SingleProgressTracker(), + ], + 'output' => STDOUT, + 'transfer_count' => 25, + 'completed' => 20, + 'failed' => 5, + ], + 'custom_initialization_3' => [ + 'progress_trackers' => [ + new SingleProgressTracker(), + new SingleProgressTracker(), + new SingleProgressTracker(), + new SingleProgressTracker(), + ], + 'output' => fopen("php://temp", "w"), + 'transfer_count' => 50, + 'completed' => 35, + 'failed' => 15, + ] + ]; + } + + /** + * @return array + */ + public function multiProgressTrackerProvider(): array + { + return [ + 'multi_progress_tracker_1_single_tracking_object' => [ + 'progress_bar_factory' => function() { + return new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat(), + ); + }, + 'event_invoker' => function (MultiProgressTracker $tracker): void + { + $tracker->transferInitiated([ + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + 'Foo', + 0, + 1024 + ) + ]); + $tracker->bytesTransferred([ + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + 'Foo', + 512, + 1024 + ) + ]); + }, + 'expected_outputs' => [ + 'transfer_count' => 1, + 'completed' => 0, + 'failed' => 0, + 'progress' => [ + "\033[2J\033[H\r\n", + "Foo:\n[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/1, Failed: 0/1\n", + "\033[2J\033[H\r\n", + "Foo:\n[########## ] 50%\n", + "--------------------\n", + "[########## ] 50% Completed: 0/1, Failed: 0/1\n" + ] + ], + ], + 'multi_progress_tracker_2' => [ + 'progress_bar_factory' => function() { + return new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat(), + ); + }, + 'event_invoker' => function (MultiProgressTracker $progressTracker): void + { + $events = [ + 'transfer_initiated' => [ + TransferListener::REQUEST_ARGS_KEY => [], + 'total_bytes' => 1024 + ], + 'transfer_progress_1' => [ + TransferListener::REQUEST_ARGS_KEY => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 342, + ], + 'transfer_progress_2' => [ + TransferListener::REQUEST_ARGS_KEY => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 684, + ], + 'transfer_progress_3' => [ + TransferListener::REQUEST_ARGS_KEY => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 1024, + ], + 'transfer_complete' => [ + TransferListener::REQUEST_ARGS_KEY => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 1024, + ] + ]; + foreach ($events as $eventName => $event) { + if ($eventName === 'transfer_initiated') { + for ($i = 0; $i < 3; $i++) { + $progressTracker->transferInitiated([ + TransferListener::REQUEST_ARGS_KEY => $event[TransferListener::REQUEST_ARGS_KEY], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + "FooObject_$i", + 0, + $event['total_bytes'], + ) + ]); + } + } elseif (str_starts_with($eventName, 'transfer_progress')) { + for ($i = 0; $i < 3; $i++) { + $progressTracker->bytesTransferred([ + TransferListener::REQUEST_ARGS_KEY => $event[TransferListener::REQUEST_ARGS_KEY], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + "FooObject_$i", + $event['bytes_transferred'], + $event['total_bytes'], + ) + ]); + } + } elseif ($eventName === 'transfer_complete') { + for ($i = 0; $i < 3; $i++) { + $progressTracker->transferComplete([ + TransferListener::REQUEST_ARGS_KEY => $event[TransferListener::REQUEST_ARGS_KEY], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + "FooObject_$i", + $event['bytes_transferred'], + $event['total_bytes'], + ) + ]); + } + } + } + }, + 'expected_outputs' => [ + 'transfer_count' => 3, + 'completed' => 3, + 'failed' => 0, + 'progress' => [ + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/1, Failed: 0/1\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/2, Failed: 0/2\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\n", + "--------------------\n", + "[## ] 11% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[ ] 0%\n", + "--------------------\n", + "[#### ] 22% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[####### ] 33%\n", + "--------------------\n", + "[####### ] 33% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[####### ] 33%\n", + "--------------------\n", + "[######### ] 44% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[####### ] 33%\n", + "--------------------\n", + "[########### ] 55% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[############# ] 66%\n", + "--------------------\n", + "[############# ] 66% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[############# ] 66%\n", + "--------------------\n", + "[############### ] 77% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[############# ] 66%\n", + "--------------------\n", + "[################## ] 88% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\n", + "--------------------\n", + "[####################] 100% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\n", + "--------------------\n", + "[####################] 100% Completed: 1/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\n", + "--------------------\n", + "[####################] 100% Completed: 2/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\n", + "--------------------\n", + "[####################] 100% Completed: 3/3, Failed: 0/3\n", + + ] + ], + ], + 'multi_progress_tracker_3' => [ + 'progress_bar_factory' => function() { + return new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat(), + ); + }, + 'event_invoker' => function (MultiProgressTracker $progressTracker): void + { + $events = [ + 'transfer_initiated' => [ + TransferListener::REQUEST_ARGS_KEY => [], + 'total_bytes' => 1024 + ], + 'transfer_progress_1' => [ + TransferListener::REQUEST_ARGS_KEY => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 342, + ], + 'transfer_progress_2' => [ + TransferListener::REQUEST_ARGS_KEY => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 684, + ], + 'transfer_progress_3' => [ + TransferListener::REQUEST_ARGS_KEY => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 1024, + ], + 'transfer_complete' => [ + TransferListener::REQUEST_ARGS_KEY => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 1024, + ], + 'transfer_fail' => [ + TransferListener::REQUEST_ARGS_KEY => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 0, + 'reason' => 'Transfer failed' + ] + ]; + foreach ($events as $eventName => $event) { + if ($eventName === 'transfer_initiated') { + for ($i = 0; $i < 5; $i++) { + $progressTracker->transferInitiated([ + TransferListener::REQUEST_ARGS_KEY => $event[TransferListener::REQUEST_ARGS_KEY], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + "FooObject_$i", + 0, + $event['total_bytes'], + ) + ]); + } + } elseif (str_starts_with($eventName, 'transfer_progress')) { + for ($i = 0; $i < 3; $i++) { + $progressTracker->bytesTransferred([ + TransferListener::REQUEST_ARGS_KEY => $event[TransferListener::REQUEST_ARGS_KEY], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + "FooObject_$i", + $event['bytes_transferred'], + $event['total_bytes'], + ) + ]); + } + } elseif ($eventName === 'transfer_complete') { + for ($i = 0; $i < 3; $i++) { + $progressTracker->transferComplete([ + TransferListener::REQUEST_ARGS_KEY => $event[TransferListener::REQUEST_ARGS_KEY], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + "FooObject_$i", + $event['bytes_transferred'], + $event['total_bytes'], + ) + ]); + } + } elseif ($eventName === 'transfer_fail') { + // Just two of them will fail + for ($i = 3; $i < 5; $i++) { + $progressTracker->transferFail([ + TransferListener::REQUEST_ARGS_KEY => $event[TransferListener::REQUEST_ARGS_KEY], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + "FooObject_$i", + 0, + $event['total_bytes'], + ), + 'reason' => $event['reason'] + ]); + } + } + } + }, + 'expected_outputs' => [ + 'transfer_count' => 5, + 'completed' => 3, + 'failed' => 2, + 'progress' => [ + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/1, Failed: 0/1\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/2, Failed: 0/2\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\r\n", + "FooObject_3:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/4, Failed: 0/4\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[# ] 6% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[ ] 0%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[### ] 13% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[####### ] 33%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[#### ] 19% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[####### ] 33%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[##### ] 26% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[####### ] 33%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[####### ] 33% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[############# ] 66%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[######## ] 39% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[############# ] 66%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[######### ] 46% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[############# ] 66%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[########### ] 53% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 1/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 2/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 3/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 3/5, Failed: 1/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 3/5, Failed: 2/5\n", + ] + ], + ] + ]; + } +} diff --git a/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php b/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php new file mode 100644 index 0000000000..e89f7e0615 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php @@ -0,0 +1,153 @@ +setArgs($args); + + $this->assertEquals($expectedFormat, $progressBarFormat->format()); + } + + /** + * @return array[] + */ + public function progressBarFormatProvider(): array + { + return [ + 'plain_progress_bar_format_1' => [ + 'implementation_class' => PlainProgressBarFormat::class, + 'args' => [ + 'object_name' => 'foo', + 'progress_bar' => '..........', + 'percent' => 100, + ], + 'expected_format' => "foo:\n[..........] 100%", + ], + 'plain_progress_bar_format_2' => [ + 'implementation_class' => PlainProgressBarFormat::class, + 'args' => [ + 'object_name' => 'foo', + 'progress_bar' => '..... ', + 'percent' => 50, + ], + 'expected_format' => "foo:\n[..... ] 50%", + ], + 'transfer_progress_bar_format_1' => [ + 'implementation_class' => TransferProgressBarFormat::class, + 'args' => [ + 'object_name' => 'foo', + 'progress_bar' => '..........', + 'percent' => 100, + 'transferred' => 100, + 'to_be_transferred' => 100, + 'unit' => 'B' + ], + 'expected_format' => "foo:\n[..........] 100% 100/100 B", + ], + 'transfer_progress_bar_format_2' => [ + 'implementation_class' => TransferProgressBarFormat::class, + 'args' => [ + 'object_name' => 'foo', + 'progress_bar' => '..... ', + 'percent' => 50, + 'transferred' => 50, + 'to_be_transferred' => 100, + 'unit' => 'B' + ], + 'expected_format' => "foo:\n[..... ] 50% 50/100 B", + ], + 'colored_transfer_progress_bar_format_1_color_code_black_defaulted' => [ + 'implementation_class' => ColoredTransferProgressBarFormat::class, + 'args' => [ + 'progress_bar' => '..... ', + 'percent' => 50, + 'transferred' => 50, + 'to_be_transferred' => 100, + 'unit' => 'B', + 'object_name' => 'FooObject' + ], + 'expected_format' => "FooObject:\n\033[30m[..... ] 50% 50/100 B \033[0m", + ], + 'colored_transfer_progress_bar_format_1' => [ + 'implementation_class' => ColoredTransferProgressBarFormat::class, + 'args' => [ + 'progress_bar' => '..... ', + 'percent' => 50, + 'transferred' => 50, + 'to_be_transferred' => 100, + 'unit' => 'B', + 'object_name' => 'FooObject', + 'color_code' => ColoredTransferProgressBarFormat::BLUE_COLOR_CODE + ], + 'expected_format' => "FooObject:\n\033[34m[..... ] 50% 50/100 B \033[0m", + ], + 'colored_transfer_progress_bar_format_2' => [ + 'implementation_class' => ColoredTransferProgressBarFormat::class, + 'args' => [ + 'progress_bar' => '..... ', + 'percent' => 50, + 'transferred' => 50, + 'to_be_transferred' => 100, + 'unit' => 'B', + 'object_name' => 'FooObject', + 'color_code' => ColoredTransferProgressBarFormat::GREEN_COLOR_CODE + ], + 'expected_format' => "FooObject:\n\033[32m[..... ] 50% 50/100 B \033[0m", + ], + 'colored_transfer_progress_bar_format_3' => [ + 'implementation_class' => ColoredTransferProgressBarFormat::class, + 'args' => [ + 'progress_bar' => '..... ', + 'percent' => 50, + 'transferred' => 50, + 'to_be_transferred' => 100, + 'unit' => 'B', + 'object_name' => 'FooObject', + 'color_code' => ColoredTransferProgressBarFormat::RED_COLOR_CODE + ], + 'expected_format' => "FooObject:\n\033[31m[..... ] 50% 50/100 B \033[0m", + ], + 'colored_transfer_progress_bar_format_4' => [ + 'implementation_class' => ColoredTransferProgressBarFormat::class, + 'args' => [ + 'progress_bar' => '..........', + 'percent' => 100, + 'transferred' => 100, + 'to_be_transferred' => 100, + 'unit' => 'B', + 'object_name' => 'FooObject', + 'color_code' => ColoredTransferProgressBarFormat::BLUE_COLOR_CODE + ], + 'expected_format' => "FooObject:\n\033[34m[..........] 100% 100/100 B \033[0m", + ], + ]; + } +} diff --git a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php new file mode 100644 index 0000000000..738ee878a8 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php @@ -0,0 +1,300 @@ +assertInstanceOf(ConsoleProgressBar::class, $progressTracker->getProgressBar()); + $this->assertEquals(STDOUT, $progressTracker->getOutput()); + $this->assertTrue($progressTracker->isClear()); + $this->assertNull($progressTracker->getCurrentSnapshot()); + } + + /** + * @param ProgressBarInterface $progressBar + * @param mixed $output + * @param bool $clear + * @param TransferProgressSnapshot $snapshot + * + * @dataProvider customInitializationProvider + * + * @return void + */ + public function testCustomInitialization( + ProgressBarInterface $progressBar, + mixed $output, + bool $clear, + TransferProgressSnapshot $snapshot + ): void + { + $progressTracker = new SingleProgressTracker( + $progressBar, + $output, + $clear, + $snapshot, + ); + $this->assertSame($progressBar, $progressTracker->getProgressBar()); + $this->assertSame($output, $progressTracker->getOutput()); + $this->assertSame($clear, $progressTracker->isClear()); + $this->assertSame($snapshot, $progressTracker->getCurrentSnapshot()); + } + + /** + * @return array[] + */ + public function customInitializationProvider(): array + { + return [ + 'initialization_1' => [ + 'progress_bar' => new ConsoleProgressBar(), + 'output' => STDOUT, + 'clear' => true, + 'snapshot' => new TransferProgressSnapshot( + 'Foo', + 0, + 10 + ), + ], + 'initialization_2' => [ + 'progress_bar' => new ConsoleProgressBar(), + 'output' => fopen('php://temp', 'w'), + 'clear' => true, + 'snapshot' => new TransferProgressSnapshot( + 'FooTest', + 50, + 500 + ), + ], + ]; + } + + /** + * @param ProgressBarInterface $progressBar + * @param callable $eventInvoker + * @param array $expectedOutputs + * + * @dataProvider singleProgressTrackingProvider + * + * @return void + */ + public function testSingleProgressTracking( + ProgressBarInterface $progressBar, + callable $eventInvoker, + array $expectedOutputs, + ): void + { + $output = fopen('php://temp', 'w'); + $progressTracker = new SingleProgressTracker( + $progressBar, + $output, + ); + $eventInvoker($progressTracker); + $this->assertEquals( + $expectedOutputs['identifier'], + $progressTracker->getCurrentSnapshot()->getIdentifier() + ); + $this->assertEquals( + $expectedOutputs['transferred_bytes'], + $progressTracker->getCurrentSnapshot()->getTransferredBytes() + ); + $this->assertEquals( + $expectedOutputs['total_bytes'], + $progressTracker->getCurrentSnapshot()->getTotalBytes() + ); + + $progress = $expectedOutputs['progress']; + if (is_array($progress)) { + $progress = join('', $expectedOutputs['progress']); + } + rewind($output); + $this->assertEquals( + $progress, + stream_get_contents($output) + ); + } + + /** + * @return array[] + */ + public function singleProgressTrackingProvider(): array + { + return [ + 'progress_rendering_1_transfer_initiated' => [ + 'progress_bar' => new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat() + ), + 'event_invoker' => function (singleProgressTracker $progressTracker): void + { + $progressTracker->transferInitiated([ + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + 'Foo', + 0, + 1024 + ) + ]); + }, + 'expected_outputs' => [ + 'identifier' => 'Foo', + 'transferred_bytes' => 0, + 'total_bytes' => 1024, + 'progress' => [ + "\033[2J\033[H\r\n", + "Foo:\n[ ] 0%" + ] + ], + ], + 'progress_rendering_2_transfer_progress' => [ + 'progress_bar' => new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat() + ), + 'event_invoker' => function (singleProgressTracker $progressTracker): void + { + $progressTracker->transferInitiated([ + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + 'Foo', + 0, + 1024 + ) + ]); + $progressTracker->bytesTransferred([ + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + 'Foo', + 256, + 1024 + ) + ]); + }, + 'expected_outputs' => [ + 'identifier' => 'Foo', + 'transferred_bytes' => 256, + 'total_bytes' => 1024, + 'progress' => [ + "\033[2J\033[H\r\n", + "Foo:\n[ ] 0%", + "\033[2J\033[H\r\n", + "Foo:\n[##### ] 25%" + ] + ], + ], + 'progress_rendering_3_transfer_force_completion_when_total_bytes_zero' => [ + 'progress_bar' => new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat() + ), + 'event_invoker' => function (singleProgressTracker $progressTracker): void + { + $progressTracker->transferInitiated([ + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + 'Foo', + 0, + 0 + ) + ]); + $progressTracker->bytesTransferred([ + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + 'Foo', + 1024, + 0 + ) + ]); + $progressTracker->transferComplete([ + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + 'Foo', + 2048, + 0 + ) + ]); + }, + 'expected_outputs' => [ + 'identifier' => 'Foo', + 'transferred_bytes' => 2048, + 'total_bytes' => 0, + 'progress' => [ + "\033[2J\033[H\r\n", + "Foo:\n[ ] 0%", + "\033[2J\033[H\r\n", + "Foo:\n[ ] 0%", + "\033[2J\033[H\r\n", + "Foo:\n[####################] 100%" + ] + ], + ], + 'progress_rendering_4_transfer_fail_with_colored_transfer_format' => [ + 'progress_bar' => new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new ColoredTransferProgressBarFormat() + ), + 'event_invoker' => function (singleProgressTracker $progressTracker): void + { + $progressTracker->transferInitiated([ + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + 'Foo', + 0, + 1024 + ) + ]); + $progressTracker->bytesTransferred([ + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + 'Foo', + 512, + 1024 + ) + ]); + $progressTracker->transferFail([ + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( + 'Foo', + 512, + 1024 + ), + 'reason' => "Error transferring!" + ]); + }, + 'expected_outputs' => [ + 'identifier' => 'Foo', + 'transferred_bytes' => 512, + 'total_bytes' => 1024, + 'progress' => [ + "\033[2J\033[H\r\n", + "Foo:\n", + "\033[30m[ ] 0% 0/1024 B ", + "\033[0m", + "\033[2J\033[H\r\n", + "Foo:\n", + "\033[34m[########## ] 50% 512/1024 B ", + "\033[0m", + "\033[2J\033[H\r\n", + "Foo:\n", + "\033[31m[########## ] 50% 512/1024 B Error transferring!", + "\033[0m" + ] + ], + ] + ]; + } +} diff --git a/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php new file mode 100644 index 0000000000..c0c90d1ea6 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php @@ -0,0 +1,39 @@ +getMockBuilder(TransferListener::class) + ->getMock(), + $this->getMockBuilder(TransferListener::class) + ->getMock(), + $this->getMockBuilder(TransferListener::class) + ->getMock(), + $this->getMockBuilder(TransferListener::class) + ->getMock(), + $this->getMockBuilder(TransferListener::class) + ->getMock(), + ]; + foreach ($listeners as $listener) { + $listener->expects($this->once())->method('transferInitiated'); + $listener->expects($this->once())->method('bytesTransferred'); + $listener->expects($this->once())->method('transferComplete'); + $listener->expects($this->once())->method('transferFail'); + } + $listenerNotifier = new TransferListenerNotifier($listeners); + $listenerNotifier->transferInitiated([]); + $listenerNotifier->bytesTransferred([]); + $listenerNotifier->transferComplete([]); + $listenerNotifier->transferFail([]); + } +} diff --git a/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php new file mode 100644 index 0000000000..5a003a8159 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php @@ -0,0 +1,84 @@ + 'Bar'] + ); + + $this->assertEquals($snapshot->getIdentifier(), 'FooObject'); + $this->assertEquals($snapshot->getTransferredBytes(), 0); + $this->assertEquals($snapshot->getTotalBytes(), 10); + $this->assertEquals($snapshot->getResponse(), ['Foo' => 'Bar']); + } + + /** + * @param int $transferredBytes + * @param int $totalBytes + * @param float $expectedRatio + * + * @return void + * @dataProvider ratioTransferredProvider + * + */ + public function testRatioTransferred( + int $transferredBytes, + int $totalBytes, + float $expectedRatio + ): void + { + $snapshot = new TransferProgressSnapshot( + 'FooObject', + $transferredBytes, + $totalBytes + ); + $this->assertEquals($expectedRatio, $snapshot->ratioTransferred()); + } + + /** + * @return array + */ + public function ratioTransferredProvider(): array + { + return [ + 'ratio_1' => [ + 'transferred_bytes' => 10, + 'total_bytes' => 100, + 'expected_ratio' => 10 / 100, + ], + 'ratio_2_transferred_bytes_zero' => [ + 'transferred_bytes' => 0, + 'total_bytes' => 100, + 'expected_ratio' => 0, + ], + 'ratio_3_unknown_total_bytes' => [ + 'transferred_bytes' => 100, + 'total_bytes' => 0, + 'expected_ratio' => 0, + ], + 'ratio_4' => [ + 'transferred_bytes' => 50, + 'total_bytes' => 256, + 'expected_ratio' => 50 / 256, + ], + 'ratio_5' => [ + 'transferred_bytes' => 250, + 'total_bytes' => 256, + 'expected_ratio' => 250 / 256, + ], + ]; + } +} diff --git a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php new file mode 100644 index 0000000000..ab6816e014 --- /dev/null +++ b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php @@ -0,0 +1,359 @@ +getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $remainingToTransfer = $objectSizeInBytes; + $mockClient->method('executeAsync') + -> willReturnCallback(function ($command) + use ( + $objectSizeInBytes, + $partsCount, + $targetPartSize, + &$remainingToTransfer + ) { + $currentPartLength = min( + $targetPartSize, + $remainingToTransfer + ); + $from = $objectSizeInBytes - $remainingToTransfer; + $to = $from + $currentPartLength; + $remainingToTransfer -= $currentPartLength; + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor('Foo'), + 'PartsCount' => $partsCount, + 'PartNumber' => $command['PartNumber'] ?? 1, + 'ContentRange' => "bytes $from-$to/$objectSizeInBytes", + 'ContentLength' => $currentPartLength + ])); + }); + $mockClient->method('getCommand') + -> willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $downloader = new RangeGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'FooBucket', + 'Key' => $objectKey, + ], + [ + 'target_part_size_bytes' => $targetPartSize, + ], + new StreamDownloadHandler() + ); + /** @var DownloadResult $response */ + $response = $downloader->promise()->wait(); + $snapshot = $downloader->getCurrentSnapshot(); + + $this->assertInstanceOf(DownloadResult::class, $response); + $this->assertEquals($objectKey, $snapshot->getIdentifier()); + $this->assertEquals($objectSizeInBytes, $snapshot->getTotalBytes()); + $this->assertEquals($objectSizeInBytes, $snapshot->getTransferredBytes()); + $this->assertEquals($partsCount, $downloader->getObjectPartsCount()); + $this->assertEquals($partsCount, $downloader->getCurrentPartNo()); + } + + /** + * Range get multipart downloader data provider. + * + * @return array[] + */ + public function rangeGetMultipartDownloaderProvider(): array { + return [ + [ + 'objectKey' => 'ObjectKey_1', + 'objectSizeInBytes' => 1024 * 10, + 'targetPartSize' => 1024 * 2, + ], + [ + 'objectKey' => 'ObjectKey_2', + 'objectSizeInBytes' => 1024 * 100, + 'targetPartSize' => 1024 * 5, + ], + [ + 'objectKey' => 'ObjectKey_3', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 512, + ], + [ + 'objectKey' => 'ObjectKey_4', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 256, + ], + [ + 'objectKey' => 'ObjectKey_5', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 458, + ] + ]; + } + + /** + * Tests nextCommand method generates correct range headers. + * + * @return void + */ + public function testNextCommandGeneratesCorrectRangeHeaders(): void + { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $partSize = 1024; + $downloader = new RangeGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ], + [ + 'target_part_size_bytes' => $partSize, + ], + new StreamDownloadHandler() + ); + + // Use reflection to test the protected nextCommand method + $reflection = new \ReflectionClass($downloader); + $nextCommandMethod = $reflection->getMethod('nextCommand'); + + // First call should create range 0-1023 + $command1 = $nextCommandMethod->invoke($downloader); + $this->assertEquals('bytes=0-1023', $command1['Range']); + $this->assertEquals(1, $downloader->getCurrentPartNo()); + + // Second call should create range 1024-2047 + $command2 = $nextCommandMethod->invoke($downloader); + $this->assertEquals('bytes=1024-2047', $command2['Range']); + $this->assertEquals(2, $downloader->getCurrentPartNo()); + } + + /** + * Tests computeObjectDimensions method for single part download. + * + * @return void + */ + public function testComputeObjectDimensionsForSinglePart(): void + { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $partSize = 2048; + $downloader = new RangeGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ], + [ + 'minimum_part_size' => $partSize, + ], + new StreamDownloadHandler(), + ); + + // Use reflection to test the protected computeObjectDimensions method + $reflection = new \ReflectionClass($downloader); + $computeObjectDimensionsMethod = $reflection->getMethod('computeObjectDimensions'); + + // Simulate object smaller than part size + $result = new Result([ + 'ContentRange' => 'bytes 0-511/512' + ]); + + $computeObjectDimensionsMethod->invoke($downloader, $result); + + // Should be single part download + $this->assertEquals(1, $downloader->getObjectPartsCount()); + $this->assertEquals(512, $downloader->getObjectSizeInBytes()); + } + + /** + * Tests nextCommand method includes IfMatch header when ETag is present. + * + * @return void + * @throws \ReflectionException + */ + public function testNextCommandIncludesIfMatchWhenETagPresent(): void + { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $eTag = '"abc123"'; + $downloader = new RangeGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ], + [ + 'minimum_part_size' => 1024, + ], + new StreamDownloadHandler(), + 0, // currentPartNo + 0, // objectPartsCount + 0, // objectSizeInBytes + $eTag // eTag + ); + + // Use reflection to test the protected nextCommand method + $reflection = new \ReflectionClass($downloader); + $nextCommandMethod = $reflection->getMethod('nextCommand'); + + $command = $nextCommandMethod->invoke($downloader); + $this->assertEquals($eTag, $command['IfMatch']); + } + + /** + * Test IfMatch is properly called in each part get operation. + * + * @param int $objectSizeInBytes + * @param int $targetPartSize + * @param string $eTag + * + * @dataProvider ifMatchIsPresentInEachRangeRequestAfterFirstProvider + * + * @return void + */ + public function testIfMatchIsPresentInEachRangeRequestAfterFirst( + int $objectSizeInBytes, + int $targetPartSize, + string $eTag + ): void + { + $firstRequestCalled = false; + $ifMatchCalledTimes = 0; + $partsCount = ceil($objectSizeInBytes / $targetPartSize); + $remainingToTransfer = $objectSizeInBytes; + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) + use ($eTag, &$ifMatchCalledTimes) { + if (isset($args['IfMatch'])) { + $ifMatchCalledTimes++; + $this->assertEquals( + $eTag, + $args['IfMatch'] + ); + } + + return new Command($commandName, $args); + }); + $s3Client->method('executeAsync') + -> willReturnCallback(function ($command) + use ( + $eTag, + $objectSizeInBytes, + $partsCount, + $targetPartSize, + &$remainingToTransfer, + &$firstRequestCalled + ) { + $firstRequestCalled = true; + $currentPartLength = min( + $targetPartSize, + $remainingToTransfer + ); + $from = $objectSizeInBytes - $remainingToTransfer; + $to = $from + $currentPartLength; + $remainingToTransfer -= $currentPartLength; + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor('Foo'), + 'ContentRange' => "bytes $from-$to/$objectSizeInBytes", + 'ContentLength' => $currentPartLength, + 'ETag' => $eTag, + ])); + }); + $requestArgs = [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ]; + $rangeGetMultipartDownloader = new RangeGetMultipartDownloader( + $s3Client, + $requestArgs, + [ + 'target_part_size_bytes' => $targetPartSize, + ] + ); + $rangeGetMultipartDownloader->download(); + $this->assertTrue($firstRequestCalled); + $this->assertEquals( + $partsCount - 1, + $ifMatchCalledTimes + ); + } + + /** + * @return Generator + */ + public function ifMatchIsPresentInEachRangeRequestAfterFirstProvider(): Generator + { + yield 'multipart_download_with_3_parts_1' => [ + 'object_size_in_bytes' => 1024 * 1024 * 20, + 'target_part_size_bytes' => 8 * 1024 * 1024, + 'eTag' => 'ETag1234', + ]; + + yield 'multipart_download_with_2_parts_1' => [ + 'object_size_in_bytes' => 1024 * 1024 * 16, + 'target_part_size_bytes' => 8 * 1024 * 1024, + 'eTag' => 'ETag12345678', + ]; + + yield 'multipart_download_with_5_parts_1' => [ + 'object_size_in_bytes' => 1024 * 1024 * 40, + 'target_part_size_bytes' => 8 * 1024 * 1024, + 'eTag' => 'ETag12345678', + ]; + } +} diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php new file mode 100644 index 0000000000..541fbe6f33 --- /dev/null +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -0,0 +1,3793 @@ + << + + {Bucket} + {Key} + {UploadId} + +EOF, + 'ListObjectsV2' => << + {Bucket} + {Prefix} + 3 + 1000 + false + {Contents} + +EOF, + 'ListObjectsV2::Contents' => << + {Key} + {Size} + 2025-05-20T14:45:08.000Z + FixedETag + CRC64NVME + FULL_OBJECT + STANDARD + +EOF + ]; + + /** + * @return void + */ + public function testDefaultConfigIsSet(): void + { + $manager = new S3TransferManager(); + $this->assertArrayHasKey( + 'target_part_size_bytes', + $manager->getConfig()->toArray() + ); + $this->assertArrayHasKey( + 'multipart_upload_threshold_bytes', + $manager->getConfig()->toArray() + ); + $this->assertArrayHasKey( + 'request_checksum_calculation', + $manager->getConfig()->toArray() + ); + $this->assertArrayHasKey( + 'response_checksum_validation', + $manager->getConfig()->toArray() + ); + $this->assertArrayHasKey( + 'multipart_download_type', + $manager->getConfig()->toArray() + ); + $this->assertArrayHasKey( + 'concurrency', + $manager->getConfig()->toArray() + ); + $this->assertArrayHasKey( + 'track_progress', + $manager->getConfig()->toArray() + ); + $this->assertArrayHasKey( + 'default_region', + $manager->getConfig()->toArray() + ); + $this->assertInstanceOf( + S3Client::class, + $manager->getS3Client() + ); + } + + /** + * @return void + */ + public function testCustomConfigIsSet(): void + { + $manager = new S3TransferManager( + null, + [ + 'target_part_size_bytes' => 1024, + 'multipart_upload_threshold_bytes' => 1024, + 'request_checksum_calculation' => 'when_required', + 'response_checksum_validation' => 'when_required', + 'checksum_algorithm' => 'sha256', + 'multipart_download_type' => 'partGet', + 'concurrency' => 20, + 'track_progress' => true, + 'default_region' => 'us-west-1', + ] + ); + $config = $manager->getConfig()->toArray(); + $this->assertEquals(1024, $config['target_part_size_bytes']); + $this->assertEquals(1024, $config['multipart_upload_threshold_bytes']); + $this->assertEquals('when_required', $config['request_checksum_calculation']); + $this->assertEquals('when_required', $config['response_checksum_validation']); + $this->assertEquals('partGet', $config['multipart_download_type']); + $this->assertEquals(20, $config['concurrency']); + $this->assertTrue($config['track_progress']); + $this->assertEquals('us-west-1', $config['default_region']); + } + + /** + * @return void + */ + public function testUploadExpectsAReadableSource(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("Please provide a valid readable file path or a valid stream as source."); + $manager = new S3TransferManager(); + $manager->upload( + new UploadRequest( + "noreadablefile", + [] + ), + )->wait(); + } + + /** + * @dataProvider uploadBucketAndKeyProvider + * + * @param array $bucketKeyArgs + * @param string $missingProperty + * + * @return void + */ + public function testUploadFailsWhenBucketAndKeyAreNotProvided( + array $bucketKeyArgs, + string $missingProperty + ): void + { + $manager = new S3TransferManager(); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The `$missingProperty` parameter must be provided as part of the request arguments."); + $manager->upload( + new UploadRequest( + Utils::streamFor(), + $bucketKeyArgs + ) + )->wait(); + } + + /** + * @return array[] + */ + public function uploadBucketAndKeyProvider(): array + { + return [ + 'bucket_missing' => [ + 'bucket_key_args' => [ + 'Key' => 'Key', + ], + 'missing_property' => 'Bucket', + ], + 'key_missing' => [ + 'bucket_key_args' => [ + 'Bucket' => 'Bucket', + ], + 'missing_property' => 'Key', + ], + ]; + } + + /** + * @return void + */ + public function testUploadFailsWhenMultipartThresholdIsLessThanMinSize(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The provided config `multipart_upload_threshold_bytes`" + . "must be greater than or equal to " . MultipartUploader::PART_MIN_SIZE); + $manager = new S3TransferManager(); + $manager->upload( + new UploadRequest( + Utils::streamFor(), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => MultipartUploader::PART_MIN_SIZE - 1 + ] + ) + )->wait(); + } + + /** + * This tests takes advantage of the transfer listeners to validate + * if a multipart upload was done. How?, it will check if bytesTransfer + * event happens more than once, which only will occur in a multipart upload. + * + * @return void + */ + public function testDoesMultipartUploadWhenApplicable(): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $transferListener = $this->createMock(TransferListener::class); + $expectedPartCount = 2; + $transferListener->expects($this->exactly($expectedPartCount)) + ->method('bytesTransferred'); + $manager->upload( + new UploadRequest( + Utils::streamFor( + str_repeat("#", MultipartUploader::PART_MIN_SIZE * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'target_part_size_bytes' => MultipartUploader::PART_MIN_SIZE, + 'multipart_upload_threshold_bytes' => MultipartUploader::PART_MIN_SIZE, + ], + [ + $transferListener, + ] + ) + )->wait(); + } + + /** + * @return void + */ + public function testDoesSingleUploadWhenApplicable(): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $transferListener = $this->createMock(TransferListener::class); + $transferListener->expects($this->once()) + ->method('bytesTransferred'); + $manager->upload( + new UploadRequest( + Utils::streamFor( + str_repeat("#", MultipartUploader::PART_MIN_SIZE - 1) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => MultipartUploader::PART_MIN_SIZE, + ], + [ + $transferListener, + ] + ) + )->wait(); + } + + /** + * @return void + */ + public function testUploadUsesTransferManagerConfigDefaultMupThreshold(): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $expectedPartCount = 2; + $transferListener = $this->createMock(TransferListener::class); + $transferListener->expects($this->exactly($expectedPartCount)) + ->method('bytesTransferred'); + $manager->upload( + new UploadRequest( + Utils::streamFor( + str_repeat("#", $manager->getConfig()->toArray()['multipart_upload_threshold_bytes']) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'target_part_size_bytes' => intval( + $manager->getConfig()->toArray()['multipart_upload_threshold_bytes'] / $expectedPartCount + ), + ], + [ + $transferListener, + ] + ) + )->wait(); + } + + /** + * + * @param int $mupThreshold + * @param int $expectedPartCount + * @param int $expectedPartSize + * @param bool $isMultipartUpload + * + * @dataProvider uploadUsesCustomMupThresholdProvider + * + * @return void + */ + public function testUploadUsesCustomMupThreshold( + int $mupThreshold, + int $expectedPartCount, + int $expectedPartSize, + bool $isMultipartUpload + ): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $transferListener = $this->createMock(TransferListener::class); + $transferListener->expects($this->exactly($expectedPartCount)) + ->method('bytesTransferred'); + $expectedIncrementalPartSize = $expectedPartSize; + $transferListener->method('bytesTransferred') + -> willReturnCallback(function ($context) use ($expectedPartSize, &$expectedIncrementalPartSize) { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; + $this->assertEquals($expectedIncrementalPartSize, $snapshot->getTransferredBytes()); + $expectedIncrementalPartSize += $expectedPartSize; + }); + $manager->upload( + new UploadRequest( + Utils::streamFor( + str_repeat("#", $expectedPartSize * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => $mupThreshold, + 'target_part_size_bytes' => $expectedPartSize, + ], + [ + $transferListener, + ] + ) + )->wait(); + if ($isMultipartUpload) { + $this->assertGreaterThan(1, $expectedPartCount); + } + } + + /** + * @return array + */ + public function uploadUsesCustomMupThresholdProvider(): array + { + return [ + 'mup_threshold_multipart_upload' => [ + 'multipart_upload_threshold_bytes' => 1024 * 1024 * 7, + 'expected_part_count' => 3, + 'expected_part_size' => 1024 * 1024 * 7, + 'is_multipart_upload' => true, + ], + 'mup_threshold_single_upload' => [ + 'multipart_upload_threshold_bytes' => 1024 * 1024 * 7, + 'expected_part_count' => 1, + 'expected_part_size' => 1024 * 1024 * 5, + 'is_multipart_upload' => false, + ] + ]; + } + + /** + * @return void + */ + public function testUploadUsesTransferManagerConfigDefaultTargetPartSize(): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $expectedPartCount = 2; + $transferListener = $this->createMock(TransferListener::class); + $transferListener->expects($this->exactly($expectedPartCount)) + ->method('bytesTransferred'); + $manager->upload( + new UploadRequest( + Utils::streamFor( + str_repeat("#", $manager->getConfig()->toArray()['target_part_size_bytes'] * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => $manager->getConfig()->toArray()['target_part_size_bytes'], + ], + [ + $transferListener, + ] + ) + )->wait(); + } + + /** + * @return void + */ + public function testUploadUsesCustomPartSize(): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $expectedPartCount = 2; + $expectedPartSize = 6 * 1024 * 1024; // 6 MBs + $transferListener = $this->getMockBuilder(TransferListener::class) + ->onlyMethods(['bytesTransferred']) + ->getMock(); + $expectedIncrementalPartSize = $expectedPartSize; + $transferListener->method('bytesTransferred') + ->willReturnCallback(function ($context) use ( + $expectedPartSize, + &$expectedIncrementalPartSize + ) { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; + $this->assertEquals($expectedIncrementalPartSize, $snapshot->getTransferredBytes()); + $expectedIncrementalPartSize += $expectedPartSize; + }); + $transferListener->expects($this->exactly($expectedPartCount)) + ->method('bytesTransferred'); + + $manager->upload( + new UploadRequest( + Utils::streamFor( + str_repeat("#", $expectedPartSize * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'target_part_size_bytes' => $expectedPartSize, + 'multipart_upload_threshold_bytes' => $expectedPartSize, + ], + [ + $transferListener, + ] + ) + )->wait(); + } + + /** + * @return void + */ + public function testUploadUsesDefaultChecksumAlgorithm(): void + { + $manager = new S3TransferManager(); + $this->testUploadResolvedChecksum( + null, // No checksum provided + MultipartUploader::DEFAULT_CHECKSUM_CALCULATION_ALGORITHM, + ); + } + + /** + * @param string $checksumAlgorithm + * + * @dataProvider uploadUsesCustomChecksumAlgorithmProvider + * + * @return void + */ + public function testUploadUsesCustomChecksumAlgorithm( + string $checksumAlgorithm, + ): void + { + $this->testUploadResolvedChecksum( + $checksumAlgorithm, + $checksumAlgorithm + ); + } + + /** + * @return array[] + */ + public function uploadUsesCustomChecksumAlgorithmProvider(): array + { + return [ + 'checksum_crc32c' => [ + 'checksum_algorithm' => 'crc32c', + ], + 'checksum_crc32' => [ + 'checksum_algorithm' => 'crc32', + ] + ]; + } + + /** + * @param string|null $checksumAlgorithm + * @param string $expectedChecksum + * + * @return void + */ + private function testUploadResolvedChecksum( + ?string $checksumAlgorithm, + string $expectedChecksum + ): void { + $client = $this->getS3ClientMock([ + 'getCommand' => function ( + string $commandName, + array $args + ) use ( + $expectedChecksum + ) { + if ($commandName !== 'CompleteMultipartUpload') { + $this->assertEquals( + strtoupper($expectedChecksum), + strtoupper($args['ChecksumAlgorithm']) + ); + } else { + $this->assertTrue(true); + } + + return new Command($commandName, $args); + }, + 'executeAsync' => function () { + return Create::promiseFor(new Result([])); + } + ]); + $putObjectRequestArgs = [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ]; + if ($checksumAlgorithm !== null) { + $putObjectRequestArgs['ChecksumAlgorithm'] = $checksumAlgorithm; + } + + $manager = new S3TransferManager( + $client, + ); + $manager->upload( + new UploadRequest( + Utils::streamFor(), + $putObjectRequestArgs, + ) + )->wait(); + } + + /** + * @param string $directory + * @param bool $isDirectoryValid + * + * @dataProvider uploadDirectoryValidatesProvidedDirectoryProvider + * + * @return void + */ + public function testUploadDirectoryValidatesProvidedDirectory( + string $directory, + bool $isDirectoryValid + ): void + { + if (!$isDirectoryValid) { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + "Please provide a valid directory path. " + . "Provided = " . $directory); + } else { + $this->assertTrue(true); + } + + try { + $manager = new S3TransferManager( + $this->getS3ClientMock(), + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + ) + )->wait(); + } finally { + // Clean up resources + if ($isDirectoryValid && is_dir($directory)) { + TestsUtility::cleanUpDir($directory); + } + } + } + + /** + * @return array[] + */ + public function uploadDirectoryValidatesProvidedDirectoryProvider(): array + { + $validDirectory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($validDirectory)) { + mkdir($validDirectory, 0777, true); + } + + $invalidDirectory = sys_get_temp_dir() . "/invalid-directory-test"; + if (is_dir($invalidDirectory)) { + rmdir($invalidDirectory); + } + + return [ + 'valid_directory' => [ + 'directory' => $validDirectory, + 'is_valid_directory' => true, + ], + 'invalid_directory' => [ + 'directory' => $invalidDirectory, + 'is_valid_directory' => false, + ] + ]; + } + + /** + * @return void + */ + public function testUploadDirectoryFailsOnInvalidFilter(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'The provided config `filter` must be callable' + ); + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'filter' => 'invalid_filter', + ] + ) + )->wait(); + } finally { + TestsUtility::cleanUpDir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFileFilter(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + + $filesCreated = []; + $validFilesCount = 0; + for ($i = 0; $i < 10; $i++) { + $fileName = "file-$i"; + if ($i % 2 === 0) { + $fileName .= "-valid"; + $validFilesCount++; + } + + $filePathName = $directory . "/" . $fileName . ".txt"; + file_put_contents($filePathName, "test"); + $filesCreated[] = $filePathName; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $calledTimes = 0; + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'filter' => function (string $objectKey) { + return str_ends_with($objectKey, "-valid.txt"); + }, + 'upload_object_request_modifier' => function ($requestArgs) use (&$calledTimes) { + $this->assertStringContainsString( + 'valid.txt', + $requestArgs["Key"] + ); + $calledTimes++; + } + ] + ) + )->wait(); + $this->assertEquals($validFilesCount, $calledTimes); + } finally { + TestsUtility::cleanUpDir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryRecursive(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + $subDirectory = $directory . "/sub-directory"; + if (!is_dir($subDirectory)) { + mkdir($subDirectory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $subDirectory . "/subdir-file-1.txt", + $subDirectory . "/subdir-file-2.txt", + ]; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + // Remove the directory from the file path to leave + // just what will be the object key + $objectKey = str_replace($directory . "/", "", $file); + $objectKeys[$objectKey] = false; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + ] + ) + )->wait(); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated); + } + } finally { + TestsUtility::cleanUpDir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryNonRecursive(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + $subDirectory = $directory . "/sub-directory"; + if (!is_dir($subDirectory)) { + mkdir($subDirectory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $subDirectory . "/subdir-file-1.txt", + $subDirectory . "/subdir-file-2.txt", + ]; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + // Remove the directory from the file path to leave + // just what will be the object key + $objectKey = str_replace($directory . "/", "", $file); + $objectKeys[$objectKey] = false; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'recursive' => false, + ] + ) + )->wait(); + $subDirPrefix = str_replace($directory . "/", "", $subDirectory); + foreach ($objectKeys as $key => $validated) { + if (str_starts_with($key, $subDirPrefix)) { + // Files in subdirectory should have been ignored + $this->assertFalse($validated, "Key {$key} should have not been considered"); + } else { + $this->assertTrue($validated, "Key {$key} should have been considered"); + } + } + } finally { + TestsUtility::cleanUpDir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFollowsSymbolicLink(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + $linkDirectory = sys_get_temp_dir() . "/link-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + if (!is_dir($linkDirectory)) { + mkdir($linkDirectory, 0777, true); + } + $symLinkDirectory = $directory . "/upload-directory-test-link"; + if (is_link($symLinkDirectory)) { + unlink($symLinkDirectory); + } + symlink($linkDirectory, $symLinkDirectory); + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $linkDirectory . "/symlink-file-1.txt", + $linkDirectory . "/symlink-file-2.txt", + ]; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + // Remove the directory from the file path to leave + // just what will be the object key + $objectKey = str_replace($directory . "/", "", $file); + $objectKey = str_replace($linkDirectory . "/", "", $objectKey); + if (str_contains($objectKey, 'symlink-file')) { + $objectKey = "upload-directory-test-link/" . $objectKey; + } + + $objectKeys[$objectKey] = false; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + // First lets make sure that when follows_symbolic_link is false + // the directory in the link will not be traversed. + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + 'follow_symbolic_links' => false, + ] + ) + )->wait(); + foreach ($objectKeys as $key => $validated) { + if (str_contains($key, "symlink")) { + // Files in subdirectory should have been ignored + $this->assertFalse($validated, "Key {$key} should have not been considered"); + } else { + $this->assertTrue($validated, "Key {$key} should have been considered"); + } + } + // Now let's enable follow_symbolic_links and all files should have + // been considered, included the ones in the symlink directory. + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + 'follow_symbolic_links' => true, + ] + ) + )->wait(); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated, "Key {$key} should have been considered"); + } + } finally { + foreach ($files as $file) { + unlink($file); + } + + unlink($symLinkDirectory); + rmdir($linkDirectory); + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFailsOnCircularSymbolicLinkTraversal() { + $parentDirectory = sys_get_temp_dir() . "/upload-directory-test"; + $linkToParent = $parentDirectory . "/link_to_parent"; + if (is_dir($parentDirectory)) { + TestsUtility::cleanUpDir($parentDirectory); + } + + mkdir($parentDirectory, 0777, true); + symlink($parentDirectory, $linkToParent); + $operationCompleted = false; + try { + $s3Client = new S3Client([ + 'region' => 'us-west-2', + ]); + $s3TransferManager = new S3TransferManager( + $s3Client, + ); + $s3TransferManager->uploadDirectory( + new UploadDirectoryRequest( + $parentDirectory, + "Bucket", + [], + [ + 'recursive' => true, + 'follow_symbolic_links' => true + ] + ) + )->wait(); + $operationCompleted = true; + $this->fail( + "Upload directory should have been failed!" + ); + } catch (RuntimeException $exception) { + if (!$operationCompleted) { + $this->assertStringContainsString( + "A circular symbolic link traversal has been detected at", + $exception->getMessage() + ); + } + } finally { + unlink($linkToParent); + rmdir($parentDirectory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryUsesProvidedPrefix(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", + $directory . "/dir-file-5.txt", + ]; + $s3Prefix = 'expenses-files/'; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + $objectKey = str_replace($directory . "/", "", $file); + $objectKeys[$s3Prefix . $objectKey] = false; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 's3_prefix' => $s3Prefix + ] + ) + )->wait(); + + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated, "Key {$key} should have been validated"); + } + } finally { + TestsUtility::cleanUpDir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryUsesProvidedDelimiter(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", + $directory . "/dir-file-5.txt", + ]; + $s3Prefix = 'expenses-files/today/records/'; + $s3Delimiter = '|'; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + $objectKey = str_replace($directory . "/", "", $file); + $objectKey = $s3Prefix . $objectKey; + $objectKey = str_replace("/", $s3Delimiter, $objectKey); + $objectKeys[$objectKey] = false; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 's3_prefix' => $s3Prefix, + 's3_delimiter' => $s3Delimiter, + ] + ) + )->wait(); + + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated, "Key {$key} should have been validated"); + } + } finally { + TestsUtility::cleanUpDir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFailsOnInvalidPutObjectRequestCallback(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The provided config `upload_object_request_modifier` must be callable."); + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'upload_object_request_modifier' => false, + ] + ) + )->wait(); + } finally { + TestsUtility::cleanUpDir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryPutObjectRequestCallbackWorks(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + ]; + foreach ($files as $file) { + file_put_contents($file, "test"); + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function ($command) { + $this->assertEquals("Test", $command['FooParameter']); + + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $called = 0; + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'upload_object_request_modifier' => function ( + &$requestArgs + ) use (&$called) { + $requestArgs["FooParameter"] = "Test"; + $called++; + }, + ] + ) + )->wait(); + $this->assertEquals(count($files), $called); + } finally { + TestsUtility::cleanUpDir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryUsesFailurePolicy(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + ]; + foreach ($files as $file) { + file_put_contents($file, "test"); + } + try { + $client = new S3Client([ + 'region' => 'us-east-2', + 'handler' => function ($command) { + if (str_contains($command['Key'], "dir-file-2.txt")) { + return Create::rejectionFor( + new Exception("Failed uploading second file") + ); + } + + return Create::promiseFor(new Result([])); + } + ]); + $manager = new S3TransferManager( + $client, + [ + 'concurrency' => 1, // To make uploads to be one after the other + ] + ); + $called = false; + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'failure_policy' => function ( + array $requestArgs, + array $uploadDirectoryRequestArgs, + \Throwable $reason, + UploadDirectoryResult $uploadDirectoryResponse + ) use ($directory, &$called) { + $called = true; + $this->assertEquals( + $directory, + $uploadDirectoryRequestArgs["source_directory"] + ); + $this->assertEquals( + "Bucket", + $uploadDirectoryRequestArgs["bucket_to"] + ); + $this->assertEquals( + "Failed uploading second file", + $reason->getMessage() + ); + $this->assertEquals( + 1, + $uploadDirectoryResponse->getObjectsUploaded() + ); + $this->assertEquals( + 1, + $uploadDirectoryResponse->getObjectsFailed() + ); + }, + ] + ) + )->wait(); + $this->assertTrue($called); + } finally { + TestsUtility::cleanUpDir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFailsOnInvalidFailurePolicy(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The provided config `failure_policy` must be callable."); + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'failure_policy' => false, + ] + ) + )->wait(); + } finally { + TestsUtility::cleanUpDir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): void + { + $s3Delimiter = "*"; + $fileNameWithDelimiter = "dir-file-$s3Delimiter.txt"; + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage( + "The filename `$fileNameWithDelimiter` must not contain the provided delimiter `$s3Delimiter`" + ); + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", + $directory . "/$fileNameWithDelimiter", + ]; + foreach ($files as $file) { + file_put_contents($file, "test"); + } + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + ['s3_delimiter' => $s3Delimiter] + ) + )->wait(); + } finally { + TestsUtility::cleanUpDir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryTracksMultipleFiles(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", + ]; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + $objectKey = str_replace($directory . "/", "", $file); + $objectKeys[$objectKey] = false; + } + + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $transferListener = $this->getMockBuilder(TransferListener::class) + ->disableOriginalConstructor() + ->getMock(); + $transferListener->expects($this->exactly(count($files))) + ->method('transferInitiated'); + $transferListener->expects($this->exactly(count($files))) + ->method('transferComplete'); + $transferListener->method('bytesTransferred') + ->willReturnCallback(function(array $context) use (&$objectKeys) { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; + $objectKeys[$snapshot->getIdentifier()] = true; + }); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [], + [ + $transferListener + ] + ) + )->wait(); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue( + $validated, + "The object key `$key` should have been validated." + ); + } + } finally { + TestsUtility::cleanUpDir($directory); + } + } + + /** + * @return void + */ + public function testDownloadFailsOnInvalidS3UriSource(): void + { + $invalidS3Uri = "invalid-s3-uri"; + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("Invalid URI: `$invalidS3Uri` provided. " + . "\nA valid S3 URI looks as `s3://bucket/key`"); + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $manager->download( + new DownloadRequest( + $invalidS3Uri + ) + ); + } + + /** + * @dataProvider downloadFailsWhenSourceAsArrayMissesBucketOrKeyPropertyProvider + * + * @param array $sourceAsArray + * @param string $expectedExceptionMessage + * + * @return void + */ + public function testDownloadFailsWhenSourceAsArrayMissesBucketOrKeyProperty( + array $sourceAsArray, + string $expectedExceptionMessage, + ): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage($expectedExceptionMessage); + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $manager->download( + new DownloadRequest($sourceAsArray) + ); + } + + /** + * @return array + */ + public function downloadFailsWhenSourceAsArrayMissesBucketOrKeyPropertyProvider(): array + { + return [ + 'missing_key' => [ + 'source' => [ + 'Bucket' => 'bucket', + ], + 'expected_exception' => "`Key` is required but not provided" + ], + 'missing_bucket' => [ + 'source' => [ + 'Key' => 'key', + ], + 'expected_exception' => "`Bucket` is required but not provided" + ] + ]; + } + + /** + * @return void + */ + public function testDownloadWorksWithS3UriAsSource(): void + { + $sourceAsArray = [ + 'Bucket' => 'bucket', + 'Key' => 'key', + ]; + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function(CommandInterface $command) use ( + $sourceAsArray, + &$called + ) { + $called = true; + $this->assertEquals($sourceAsArray['Bucket'], $command['Bucket']); + $this->assertEquals($sourceAsArray['Key'], $command['Key']); + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + 'PartsCount' => 1, + '@metadata' => [] + ])); + }, + ]); + $manager = new S3TransferManager( + $client + ); + $manager->download( + new DownloadRequest($sourceAsArray) + )->wait(); + $this->assertTrue($called); + } + + /** + * @return void + */ + public function testDownloadWorksWithBucketAndKeyAsSource(): void + { + $bucket = 'bucket'; + $key = 'key'; + $sourceAsS3Uri = "s3://$bucket/$key"; + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function(CommandInterface $command) use ( + $bucket, + $key, + &$called + ) { + $called = true; + $this->assertEquals($bucket, $command['Bucket']); + $this->assertEquals($key, $command['Key']); + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + 'PartsCount' => 1, + '@metadata' => [] + ])); + }, + ]); + $manager = new S3TransferManager( + $client + ); + $manager->download( + new DownloadRequest( + $sourceAsS3Uri + ), + )->wait(); + $this->assertTrue($called); + } + + /** + * + * @param array $transferManagerConfig + * @param array $downloadConfig + * @param array $downloadArgs + * @param bool $expectedChecksumMode + * + * @return void + * @dataProvider downloadAppliesChecksumProvider + * + */ + public function testDownloadAppliesChecksumMode( + array $transferManagerConfig, + array $downloadConfig, + array $downloadArgs, + bool $expectedChecksumMode, + ): void + { + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $expectedChecksumMode, + &$called + ) { + $called = true; + if ($expectedChecksumMode) { + $this->assertEquals( + 'ENABLED', + $command['ChecksumMode'], + ); + } else { + $this->assertArrayNotHasKey('ChecksumMode', $command); + } + + if ($command->getName() === AbstractMultipartDownloader::GET_OBJECT_COMMAND) { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + 'PartsCount' => 1, + '@metadata' => [] + ])); + } + + return Create::promiseFor(new Result([])); + } + ]); + $manager = new S3TransferManager( + $client, + $transferManagerConfig, + ); + $manager->download( + new DownloadRequest( + "s3://bucket/key", + $downloadArgs, + $downloadConfig + ) + )->wait(); + $this->assertTrue($called); + } + + /** + * @return array + */ + public function downloadAppliesChecksumProvider(): array + { + return [ + 'checksum_mode_from_default_transfer_manager_config' => [ + 'transfer_manager_config' => [], + 'download_config' => [], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => true, + ], + 'checksum_mode_enabled_by_transfer_manager_config' => [ + 'transfer_manager_config' => [ + 'response_checksum_validation' => 'when_supported' + ], + 'download_config' => [], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => true, + ], + 'checksum_mode_disabled_by_transfer_manager_config' => [ + 'transfer_manager_config' => [ + 'response_checksum_validation' => 'when_required' + ], + 'download_config' => [], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => false, + ], + 'checksum_mode_enabled_by_download_config' => [ + 'transfer_manager_config' => [], + 'download_config' => [ + 'response_checksum_validation' => 'when_supported' + ], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => true, + ], + 'checksum_mode_disabled_by_download_config' => [ + 'transfer_manager_config' => [], + 'download_config' => [ + 'response_checksum_validation' => 'when_required' + ], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => false, + ], + 'checksum_mode_download_config_overrides_transfer_manager_config' => [ + 'transfer_manager_config' => [ + 'response_checksum_validation' => 'when_required' + ], + 'download_config' => [ + 'response_checksum_validation' => 'when_supported' + ], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => true, + ] + ]; + } + + /** + * @param string $multipartDownloadType + * @param string $expectedParameter + * + * @dataProvider downloadChoosesMultipartDownloadTypeProvider + * + * @return void + */ + public function testDownloadChoosesMultipartDownloadType( + string $multipartDownloadType, + string $expectedParameter + ): void + { + $calledOnce = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + &$calledOnce, + $expectedParameter + ) { + $this->assertTrue( + isset($command[$expectedParameter]), + ); + $calledOnce = true; + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + 'PartsCount' => 1, + '@metadata' => [] + ])); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->download( + new DownloadRequest( + "s3://bucket/key", + [], + ['multipart_download_type' => $multipartDownloadType] + ) + )->wait(); + $this->assertTrue($calledOnce); + } + + /** + * @return array + */ + public function downloadChoosesMultipartDownloadTypeProvider(): array + { + return [ + 'part_get_multipart_download' => [ + 'multipart_download_type' => AbstractMultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, + 'expected_parameter' => 'PartNumber' + ], + 'range_get_multipart_download' => [ + 'multipart_download_type' => AbstractMultipartDownloader::RANGED_GET_MULTIPART_DOWNLOADER, + 'expected_parameter' => 'Range' + ] + ]; + } + + /** + * @param int $minimumPartSize + * @param int $objectSize + * @param array $expectedRangeSizes + * + * @return void + * + * @dataProvider rangeGetMultipartDownloadMinimumPartSizeProvider + * + */ + public function testRangeGetMultipartDownloadMinimumPartSize( + int $minimumPartSize, + int $objectSize, + array $expectedRangeSizes + ): void + { + $calledTimes = 0; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $objectSize, + $expectedRangeSizes, + &$calledTimes, + ) { + $this->assertTrue(isset($command['Range'])); + $range = str_replace("bytes=", "", $command['Range']); + $rangeParts = explode("-", $range); + $this->assertEquals( + (intval($rangeParts[1]) - intval($rangeParts[0])) + 1, + $expectedRangeSizes[$calledTimes] + ); + $calledTimes++; + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + 'ContentRange' => "0-$objectSize/$objectSize", + 'ETag' => 'TestEtag', + '@metadata' => [] + ])); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->download( + new DownloadRequest( + "s3://bucket/key", + [], + [ + 'multipart_download_type' => AbstractMultipartDownloader::RANGED_GET_MULTIPART_DOWNLOADER, + 'target_part_size_bytes' => $minimumPartSize, + ] + ) + )->wait(); + $this->assertEquals(count($expectedRangeSizes), $calledTimes); + } + + /** + * @return array + */ + public function rangeGetMultipartDownloadMinimumPartSizeProvider(): array + { + return [ + 'minimum_part_size_1' => [ + 'minimum_part_size' => 1024, + 'object_size' => 3072, + 'expected_range_sizes' => [ + 1024, + 1024, + 1024 + ] + ], + 'minimum_part_size_2' => [ + 'minimum_part_size' => 1024, + 'object_size' => 2000, + 'expected_range_sizes' => [ + 1024, + 977, + ] + ], + 'minimum_part_size_3' => [ + 'minimum_part_size' => 1024 * 1024 * 10, + 'object_size' => 1024 * 1024 * 25, + 'expected_range_sizes' => [ + 1024 * 1024 * 10, + 1024 * 1024 * 10, + (1024 * 1024 * 5) + 1 + ] + ], + 'minimum_part_size_4' => [ + 'minimum_part_size' => 1024 * 1024 * 25, + 'object_size' => 1024 * 1024 * 100, + 'expected_range_sizes' => [ + 1024 * 1024 * 25, + 1024 * 1024 * 25, + 1024 * 1024 * 25, + 1024 * 1024 * 25, + ] + ] + ]; + } + + /** + * @return void + */ + public function testDownloadDirectoryCreatesDestinationDirectory(): void + { + $destinationDirectory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . uniqid(); + if (is_dir($destinationDirectory)) { + rmdir($destinationDirectory); + } + + try { + $client = $this->getS3ClientMock([ + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + }, + 'executeAsync' => function (CommandInterface $command) { + return Create::promiseFor(new Result([])); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory + ) + )->wait(); + $this->assertFileExists($destinationDirectory); + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } + } + + /** + * @param array $config + * @param string $expectedS3Prefix + * + * @dataProvider downloadDirectoryAppliesS3PrefixProvider + * + * @return void + */ + public function testDownloadDirectoryAppliesS3Prefix( + array $config, + string $expectedS3Prefix + ): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $listObjectsCalled = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $expectedS3Prefix, + &$called, + &$listObjectsCalled, + ) { + $called = true; + if ($command->getName() === "ListObjectsV2") { + $listObjectsCalled = true; + $this->assertEquals( + $expectedS3Prefix, + $command['Prefix'] + ); + } + + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + $config + ) + )->wait(); + + $this->assertTrue($called); + $this->assertTrue($listObjectsCalled); + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } + } + + /** + * @return array + */ + public function downloadDirectoryAppliesS3PrefixProvider(): array + { + return [ + 's3_prefix_from_config' => [ + 'config' => [ + 's3_prefix' => 'TestPrefix', + ], + 'expected_s3_prefix' => 'TestPrefix' + ], + 's3_prefix_from_list_objects_v2_args' => [ + 'config' => [ + 'list_objects_v2_args' => [ + 'Prefix' => 'PrefixFromArgs' + ], + ], + 'expected_s3_prefix' => 'PrefixFromArgs' + ], + 's3_prefix_from_config_is_ignored_when_present_in_list_object_args' => [ + 'config' => [ + 's3_prefix' => 'TestPrefix', + 'list_objects_v2_args' => [ + 'Prefix' => 'PrefixFromArgs' + ], + ], + 'expected_s3_prefix' => 'PrefixFromArgs' + ], + ]; + } + + /** + * @return void + */ + public function testDownloadDirectoryFailsOnInvalidFilter(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The provided config `filter` must be callable."); + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + &$called, + ) { + $called = true; + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + ['filter' => false] + ) + )->wait(); + $this->assertTrue($called); + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } + } + + /** + * @return void + */ + public function testDownloadDirectoryFailsOnInvalidFailurePolicy(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The provided config `failure_policy` must be callable."); + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + &$called, + ) { + $called = true; + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + ['failure_policy' => false] + ) + )->wait(); + $this->assertTrue($called); + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } + } + + /** + * @return void + */ + public function testDownloadDirectoryUsesFailurePolicy(): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + + try { + $client = new S3Client([ + 'region' => 'us-west-2', + 'handler' => function (CommandInterface $command) { + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => [ + [ + 'Key' => 'file1.txt', + ], + [ + 'Key' => 'file2.txt', + ] + ] + ])); + } elseif ($command->getName() === 'GetObject') { + if ($command['Key'] === 'file2.txt') { + return Create::rejectionFor( + new Exception("Failed downloading file") + ); + } + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + 'PartsCount' => 1, + '@metadata' => [] + ])); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + ['failure_policy' => function ( + array $requestArgs, + array $uploadDirectoryRequestArgs, + \Throwable $reason, + DownloadDirectoryResult $downloadDirectoryResponse + ) use ($destinationDirectory, &$called) { + $called = true; + $this->assertEquals( + $destinationDirectory, + $uploadDirectoryRequestArgs['destination_directory'] + ); + $this->assertEquals( + "Failed downloading file", + $reason->getMessage() + ); + $this->assertEquals( + 1, + $downloadDirectoryResponse->getObjectsDownloaded() + ); + $this->assertEquals( + 1, + $downloadDirectoryResponse->getObjectsFailed() + ); + }] + ) + )->wait(); + $this->assertTrue($called); + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } + } + + /** + * @param Closure $filter + * @param array $objectList + * @param array $expectedObjectList + * + * @dataProvider downloadDirectoryAppliesFilter + * + * @return void + */ + public function testDownloadDirectoryAppliesFilter( + Closure $filter, + array $objectList, + array $expectedObjectList, + ): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $downloadObjectKeys = []; + foreach ($expectedObjectList as $objectKey) { + $downloadObjectKeys[$objectKey] = false; + } + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $objectList, + &$called, + &$downloadObjectKeys + ) { + $called = true; + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $objectList, + ])); + } elseif ($command->getName() === 'GetObject') { + $downloadObjectKeys[$command['Key']] = true; + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + 'PartsCount' => 1, + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + ['filter' => $filter] + ) + )->wait(); + + $this->assertTrue($called); + foreach ($downloadObjectKeys as $key => $validated) { + $this->assertTrue( + $validated, + "The key `$key` should have been validated" + ); + } + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } + } + + /** + * @return array[] + */ + public function downloadDirectoryAppliesFilter(): array + { + return [ + 'filter_1' => [ + 'filter' => function (string $objectKey) { + return str_starts_with($objectKey, "folder_2/"); + }, + 'object_list' => [ + [ + 'Key' => 'folder_1/key_1.txt', + ], + [ + 'Key' => 'folder_1/key_2.txt' + ], + [ + 'Key' => 'folder_2/key_1.txt' + ], + [ + 'Key' => 'folder_2/key_2.txt' + ] + ], + 'expected_object_list' => [ + "folder_2/key_1.txt", + "folder_2/key_2.txt", + ] + ], + 'filter_2' => [ + 'filter' => function (string $objectKey) { + return $objectKey === "folder_2/key_1.txt"; + }, + 'object_list' => [ + [ + 'Key' => 'folder_1/key_1.txt', + ], + [ + 'Key' => 'folder_1/key_2.txt' + ], + [ + 'Key' => 'folder_2/key_1.txt' + ], + [ + 'Key' => 'folder_2/key_2.txt' + ] + ], + 'expected_object_list' => [ + "folder_2/key_1.txt", + ] + ], + 'filter_3' => [ + 'filter' => function (string $objectKey) { + return $objectKey !== "folder_2/key_1.txt"; + }, + 'object_list' => [ + [ + 'Key' => 'folder_1/key_1.txt', + ], + [ + 'Key' => 'folder_1/key_2.txt' + ], + [ + 'Key' => 'folder_2/key_1.txt' + ], + [ + 'Key' => 'folder_2/key_2.txt' + ] + ], + 'expected_object_list' => [ + "folder_2/key_2.txt", + "folder_1/key_1.txt", + "folder_1/key_1.txt", + ] + ] + ]; + } + + /** + * @return void + */ + public function testDownloadDirectoryFailsOnInvalidGetObjectRequestCallback(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + "The provided config `download_object_request_modifier` must be callable." + ); + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) { + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => [], + ])); + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + ['download_object_request_modifier' => false] + ) + )->wait(); + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } + } + + /** + * @return void + */ + public function testDownloadDirectoryGetObjectRequestCallbackWorks(): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $listObjectsContent = [ + [ + 'Key' => 'folder_1/key_1.txt', + ] + ]; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ($listObjectsContent) { + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $listObjectsContent, + ])); + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + 'PartsCount' => 1, + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $getObjectRequestCallback = function($requestArgs) use (&$called) { + $called = true; + $this->assertTrue(isset($requestArgs['CustomParameter'])); + $this->assertEquals( + 'CustomParameterValue', + $requestArgs['CustomParameter'] + ); + }; + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [ + 'CustomParameter' => 'CustomParameterValue' + ], + ['download_object_request_modifier' => $getObjectRequestCallback] + ) + )->wait(); + $this->assertTrue($called); + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } + } + + /** + * @param array $listObjectsContent + * @param array $expectedFileKeys + * + * @dataProvider downloadDirectoryCreateFilesProvider + * + * @return void + */ + public function testDownloadDirectoryCreateFiles( + array $listObjectsContent, + array $expectedFileKeys, + ): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $listObjectsContent, + &$called + ) { + $called = true; + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $listObjectsContent, + ])); + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor( + "Test file " . $command['Key'] + ), + 'PartsCount' => 1, + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + ) + )->wait(); + $this->assertTrue($called); + foreach ($expectedFileKeys as $key) { + $file = $destinationDirectory . "/" . $key; + $this->assertFileExists($file); + $this->assertEquals( + "Test file " . $key, + file_get_contents($file) + ); + } + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } + } + + /** + * @return array + */ + public function downloadDirectoryCreateFilesProvider(): array + { + return [ + 'files_1' => [ + 'list_objects_content' => [ + [ + 'Key' => 'file1.txt' + ], + [ + 'Key' => 'file2.txt' + ], + [ + 'Key' => 'file3.txt' + ], + [ + 'Key' => 'file4.txt' + ], + [ + 'Key' => 'file5.txt' + ] + ], + 'expected_file_keys' => [ + 'file1.txt', + 'file2.txt', + 'file3.txt', + 'file4.txt', + 'file5.txt' + ] + ] + ]; + } + + /** + * @param string|null $prefix + * @param array $objects + * @param array $expectedOutput + * + * @return void + * @dataProvider resolvesOutsideTargetDirectoryProvider + */ + public function testResolvesOutsideTargetDirectory( + ?string $prefix, + array $objects, + array $expectedOutput + ): void + { + if ($expectedOutput['success'] === false) { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessageMatches( + '/Cannot download key [^\s]+ its relative path' + .' resolves outside the parent directory\./' + ); + } + + $bucket = "test-bucket"; + $directory = "test-directory"; + try { + $fullDirectoryPath = sys_get_temp_dir() . DIRECTORY_SEPARATOR . $directory; + if (is_dir($fullDirectoryPath)) { + TestsUtility::cleanUpDir($fullDirectoryPath); + } + mkdir($fullDirectoryPath, 0777, true); + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $objects, + &$called + ) { + $called = true; + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $objects, + ])); + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor( + "Test file " . $command['Key'] + ), + 'PartsCount' => 1, + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + $bucket, + $fullDirectoryPath, + [], + [ + 's3_prefix' => $prefix, + ] + ) + )->wait(); + $this->assertTrue($called); + // Validate the expected file output + if ($expectedOutput['success']) { + $this->assertFileExists( + $fullDirectoryPath + . DIRECTORY_SEPARATOR + . $expectedOutput['filename'] + ); + } + } finally { + TestsUtility::cleanUpDir($directory); + } + } + + /** + * @return array + */ + public function resolvesOutsideTargetDirectoryProvider(): array + { + return [ + 'download_directory_1_linux' => [ + 'prefix' => null, + 'objects' => [ + [ + 'Key' => '2023/Jan/1.png' + ], + ], + 'expected_output' => [ + 'success' => true, + 'filename' => '2023/Jan/1.png', + ] + ], + 'download_directory_2' => [ + 'prefix' => '2023/Jan/', + 'objects' => [ + [ + 'Key' => '2023/Jan/1.png' + ] + ], + 'expected_output' => [ + 'success' => true, + 'filename' => '1.png', + ] + ], + 'download_directory_3' => [ + 'prefix' => '2023/Jan', + 'objects' => [ + [ + 'Key' => '2023/Jan/1.png' + ] + ], + 'expected_output' => [ + 'success' => true, + 'filename' => '1.png', + ] + ], + 'download_directory_6' => [ + 'prefix' => '2023', + 'objects' => [ + [ + 'Key' => '2023/Jan/1.png' + ] + ], + 'expected_output' => [ + 'success' => true, + 'filename' => 'Jan/1.png', + ] + ], + 'download_directory_7_fails' => [ + 'prefix' => null, + 'objects' => [ + [ + 'Key' => '../2023/Jan/1.png' + ] + ], + 'expected_output' => [ + 'success' => false, + ] + ], + 'download_directory_9_fails' => [ + 'prefix' => null, + 'objects' => [ + [ + 'Key' => 'foo/../2023/../../Jan/1.png' + ] + ], + 'expected_output' => [ + 'success' => false, + ] + ], + 'download_directory_10_fails' => [ + 'prefix' => null, + 'objects' => [ + [ + 'Key' => '../test-2/object.dat' + ] + ], + 'expected_output' => [ + 'success' => false, + ] + ], + ]; + } + + /** + * @param string $testId + * @param array $config + * @param array $requestArgs + * @param array $expectations + * @param array $outcomes + * + * @return void + * @dataProvider modeledDownloadCasesProvider + * + */ + public function testModeledCasesForDownload( + string $testId, + array $config, + array $requestArgs, + array $expectations, + array $outcomes + ): void + { + $testsToSkip = [ + "Test download with part GET - validation failure when part count mismatch" => true, + ]; + if ($testsToSkip[$testId] ?? false) { + $this->markTestSkipped( + "The test `" . $testId . "` is not supported yet." + ); + } + + // Outcomes has only one item for now + $outcome = $outcomes[0]; + // Standardize config + $this->parseConfigFromCamelCaseToSnakeCase($config); + // Standardize request + $this->parseRequestArgsFromCamelCaseToPascalCase($requestArgs); + + if (isset($config['multipart_download_type']) && $config['multipart_download_type'] === 'RANGE') { + $config['multipart_download_type'] = 'RANGED'; + } + + // Operational values + $totalBytesReceived = 0; + $totalPartsReceived = 0; + // Mock client to validate expected requests + $s3Client = $this->getS3ClientWithSequentialResponses( + array_map(function ($expectation) { + $operation = $expectation['request']['operation']; + + return array_merge( + $expectation['response'], + ['operation' => $operation] + ); + }, $expectations), + function ( + string $operation, + array|string|null $body, + ?array &$headers + ): StreamInterface + { + $fixedBody = Utils::streamFor( + str_repeat( + '*', + $headers['Content-Length'] + ) + ); + + if (isset($headers['ChecksumAlgorithm'])) { + // Checksum injection when expected to succeed at checksum validation + // This is needed because the checksum in the test is wrong + $algorithm = strtolower($headers['ChecksumAlgorithm']); + $checksumValue = ApplyChecksumMiddleware::getEncodedValue( + $algorithm, + $fixedBody + ); + $headers['Checksum'.strtoupper($algorithm)] = $checksumValue; + $fixedBody->rewind(); + } + + // If body was provided then we override the fixed one + if ($body !== null) { + $fixedBody = Utils::streamFor($body); + } + + return $fixedBody; + }, + ); + $s3TransferManager = new S3TransferManager( + $s3Client, + ); + try { + $response = $s3TransferManager->download( + new DownloadRequest( + [ + 'Bucket' => 'test-bucket', + 'Key' => 'test-key', + ], + downloadRequestArgs: $requestArgs, + config: $config, + listeners: [ + new class($totalBytesReceived, $totalPartsReceived) + extends TransferListener { + private int $totalBytesReceived; + private int $totalPartsReceived; + + public function __construct( + int &$totalBytesReceived, + int &$totalPartsReceived + ) + { + $this->totalBytesReceived =& $totalBytesReceived; + $this->totalPartsReceived =& $totalPartsReceived; + } + + /** + * @param array $context + * + * @return void + */ + public function bytesTransferred(array $context): void { + $snapshot = $context[ + TransferListener::PROGRESS_SNAPSHOT_KEY + ]; + $this->totalBytesReceived = $snapshot->getTransferredBytes(); + $this->totalPartsReceived++; + } + } + ] + ) + )->wait(); + $this->assertEquals( + "success", + $outcome['result'], + "Operation should have failed at this point" + ); + $this->assertInstanceOf( + DownloadResult::class, + $response, + ); + if (isset($outcome['totalBytes'])) { + $this->assertEquals( + $outcome['totalBytes'], + $totalBytesReceived + ); + } + if (isset($outcome['totalParts'])) { + $this->assertEquals( + $outcome['totalParts'], + $totalPartsReceived + ); + } + if (isset($outcome['checksumValidated'])) { + $this->assertArrayHasKey( + 'ChecksumValidated', + $response + ); + $this->assertEquals( + $outcome['checksumAlgorithm'], + $response['ChecksumValidated'] + ); + } + } catch (S3TransferException | S3Exception $e) { + $this->assertEquals( + "error", + $outcome['result'], + "Operation did not expect a failure" + ); + + $this->assertStringContainsString( + $outcome['errorMessage'], + $e->getMessage() + ); + } + } + + /** + * @param string $testId + * @param array $config + * @param array $requestArgs + * @param array $expectations + * @param array $outcomes + * + * @return void + * @dataProvider modeledUploadCasesProvider + * + */ + public function testModeledCasesForUpload( + string $testId, + array $config, + array $requestArgs, + array $expectations, + array $outcomes + ): void + { + $testsToSkip = [ + "Test upload with multipart upload - validation failure when part size mismatch" => true, + "Test upload with multipart upload - validation failure when part count mismatch" => true + ]; + if ($testsToSkip[$testId] ?? false) { + $this->markTestSkipped( + "The test `" . $testId . "` is not supported yet." + ); + } + + // Outcomes has only one item for now + $outcome = $outcomes[0]; + // Standardize config + $this->parseConfigFromCamelCaseToSnakeCase($config); + // Standardize request + $this->parseRequestArgsFromCamelCaseToPascalCase($requestArgs); + + // Operational values + $contentLength = $requestArgs['ContentLength']; + $totalBytesReceived = 0; + $totalPartsReceived = 0; + // Mock client to validate expected requests + $s3Client = $this->getS3ClientWithSequentialResponses( + array_map(function ($expectation) { + $operation = $expectation['request']['operation']; + + return array_merge( + $expectation['response'], + ['operation' => $operation] + ); + }, $expectations), + function (string $operation, ?array $body): StreamInterface { + $template = self::$s3BodyTemplates[$operation] ?? ""; + if ($body === null) { + $body = []; + } + + foreach ($body as $key => $value) { + $template = str_replace("{{$key}}", $value, $template); + } + + return Utils::streamFor( + $template, + ); + } + ); + + $s3TransferManager = new S3TransferManager( + $s3Client, + ); + try { + $response = $s3TransferManager->upload( + new UploadRequest( + Utils::streamFor( + str_repeat('#', $contentLength), + ), + uploadRequestArgs: $requestArgs, + config: array_merge( + $config, + ['concurrency' => 1], + ), + listeners: [ + new class($totalBytesReceived, $totalPartsReceived) + extends TransferListener { + private int $totalBytesReceived; + private int $totalPartsReceived; + + public function __construct( + int &$totalBytesReceived, + int &$totalPartsReceived + ) + { + $this->totalBytesReceived =& $totalBytesReceived; + $this->totalPartsReceived =& $totalPartsReceived; + } + + /** + * @param array $context + * + * @return void + */ + public function bytesTransferred(array $context): void { + $snapshot = $context[ + TransferListener::PROGRESS_SNAPSHOT_KEY + ]; + $this->totalBytesReceived = $snapshot->getTransferredBytes(); + $this->totalPartsReceived++; + } + } + ] + ) + )->wait(); + $this->assertEquals( + "success", + $outcome['result'], + "Operation should have failed at this point" + ); + $this->assertInstanceOf( + UploadResult::class, + $response, + ); + if (isset($outcome['totalBytes'])) { + $this->assertEquals( + $outcome['totalBytes'], + $totalBytesReceived + ); + } + if (isset($outcome['totalParts'])) { + $this->assertEquals( + $outcome['totalParts'], + $totalPartsReceived + ); + } + } catch (S3TransferException | S3Exception $e) { + $this->assertEquals( + "error", + $outcome['result'], + "Operation did not expect a failure" + ); + + $this->assertStringContainsString( + $outcome['errorMessage'], + $e->getMessage() + ); + } + } + + /** + * @param string $testId + * @param array $config + * @param array $uploadDirectoryRequestArgs + * @param array|null $sourceStructure + * @param array $expectations + * @param array $outcomes + * + * @return void + * @dataProvider modeledUploadDirectoryCasesProvider + */ + public function testModeledCasesForUploadDirectory( + string $testId, + array $config, + array $uploadDirectoryRequestArgs, + ?array $sourceStructure, + array $expectations, + array $outcomes + ) { + $testsToSkip = [ + "Test upload directory - S3 directory bucket" => true, + ]; + if ($testsToSkip[$testId] ?? false) { + $this->markTestSkipped( + "The test `" . $testId . "` is not supported yet." + ); + } + // Parse config and request args + $this->parseConfigFromCamelCaseToSnakeCase($config); + $this->parseConfigFromCamelCaseToSnakeCase($uploadDirectoryRequestArgs); + // Extract bucket and source + $bucket = $uploadDirectoryRequestArgs['bucket']; + unset($uploadDirectoryRequestArgs['bucket']); + $source = $uploadDirectoryRequestArgs['source']; + unset($uploadDirectoryRequestArgs['source']); + // Now lets merge what is remaining in $uploadDirectoryRequestArgs into config + $config = array_merge( + $config, + $uploadDirectoryRequestArgs, + ); + // Now let`s convert filter into its proper type + if (isset($config['filter'])) { + $filterExpression = $config['filter']; + $config['filter'] = function ($file) use ($filterExpression) { + return fnmatch($filterExpression, $file) == true; + }; + } + + // Now let`s convert failure policy into is proper type + if (isset($config['failure_policy'])) { + if ($config['failure_policy'] === 'CONTINUE_ON_FAILURE') { + $config['failure_policy'] = function () { + return true; + }; + } + } + + // Prepare source directory + $sourceDirectory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . "upload-directory-test"; + $source = $sourceDirectory . DIRECTORY_SEPARATOR . $source; + if ($sourceStructure !== null) { + // Create source folder first + if (is_dir($source)) { + TestsUtility::cleanUpDir($source); + } + + mkdir($source, 0777, true); + + // Populate source folder with test files + foreach ($sourceStructure as $src) { + $sourcePath = $sourceDirectory . $src['path']; + $sourceParent = dirname($sourcePath); + if (!is_dir($sourceParent)) { + mkdir($sourceParent, 0777, true); + } + + $remaining = $src['size']; + $chunkLimit = 1024; + // Populate the source + while ($remaining > 0) { + $chunkSize = min($chunkLimit, $remaining); + file_put_contents( + $sourcePath, + str_repeat( + '#', + $chunkSize + ), + FILE_APPEND + ); + + $remaining -= $chunkSize; + } + } + } + + // Now lets orchestrate request-response + $s3Client = $this->getS3ClientWithSequentialResponses( + array_map(function ($expectation) { + $operation = $expectation['request']['operation']; + + return array_merge( + $expectation['response'], + ['operation' => $operation] + ); + }, $expectations), + function (string $operation, ?array $body): StreamInterface { + $template = self::$s3BodyTemplates[$operation] ?? ""; + if ($body === null) { + $body = []; + } + + foreach ($body as $key => $value) { + $template = str_replace("{{$key}}", $value, $template); + } + + return Utils::streamFor( + $template, + ); + } + ); + // Get outcome + $outcome = $outcomes[0]; + try { + $s3TransferManager = new S3TransferManager( + $s3Client, + ); + $result = $s3TransferManager->uploadDirectory( + new UploadDirectoryRequest( + $source, + $bucket, + [], + $config, + ) + )->wait(); + + if ($outcome['result'] === 'failure') { + $this->fail( + "A failure was expected on this test" + ); + } + // Evaluate outcome + $this->assertEquals( + $outcome['objectsUploaded'], + $result->getObjectsUploaded() + ); + $this->assertEquals( + $outcome['objectsFailed'], + $result->getObjectsFailed() + ); + } catch (Exception $exception) { + if ($outcome['result'] !== 'failure') { + $this->fail( + "A failure was not expected on this test but got: " . $exception->getMessage() + ); + } + $this->assertTrue(true); + } finally { + TestsUtility::cleanUpDir($sourceDirectory); + } + } + + /** + * @param string $testId + * @param array $config + * @param array $downloadDirectoryRequestArgs + * @param array $s3Objects + * @param array $expectations + * @param array $expectedFiles + * @param array $outcomes + * + * @return void + * @dataProvider modeledDownloadDirectoryCasesProvider + * + */ + public function testModeledCasesForDownloadDirectory( + string $testId, + array $config, + array $downloadDirectoryRequestArgs, + array $s3Objects, + array $expectations, + array $expectedFiles, + array $outcomes + ) { + $testsToSkip = [ + "Test download directory - S3 directory bucket" => true, + ]; + if ($testsToSkip[$testId] ?? false) { + $this->markTestSkipped( + "The test `" . $testId . "` is not supported yet." + ); + } + // Parse config and request args + $this->parseConfigFromCamelCaseToSnakeCase($config); + $this->parseConfigFromCamelCaseToSnakeCase($downloadDirectoryRequestArgs); + // Extract bucket and destination + $bucket = $downloadDirectoryRequestArgs['bucket']; + unset($downloadDirectoryRequestArgs['bucket']); + $destination = $downloadDirectoryRequestArgs['destination']; + unset($downloadDirectoryRequestArgs['destination']); + // Now lets merge what is remaining in $downloadDirectoryRequestArgs into config + $config = array_merge( + $config, + $downloadDirectoryRequestArgs, + ); + // Now let`s convert filter into its proper type + if (isset($config['filter'])) { + $filterExpression = $config['filter']; + $config['filter'] = function ($file) use ($filterExpression) { + return fnmatch($filterExpression, $file) == true; + }; + } + // Prepare destination directory + $baseDirectory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . "download-directory-test"; + $targetDirectory = $baseDirectory . DIRECTORY_SEPARATOR . $destination; + if (is_dir($targetDirectory)) { + TestsUtility::cleanUpDir($targetDirectory); + } + + mkdir($targetDirectory, 0777, true); + + // Get prefix so it can be used in body creation for ListObjectsV2 + $prefix = $config['prefix'] ?? ''; + // Prepare the pool of responses + $s3Client = $this->getS3ClientWithSequentialResponses( + array_map(function ($expectation) { + $operation = $expectation['request']['operation']; + + return array_merge( + $expectation['response'], + ['operation' => $operation] + ); + }, $expectations), + function ( + string $operation, + array|string|null $body, + ?array &$headers + ) use ($prefix, $bucket): StreamInterface + { + if ($operation === 'ListObjectsV2') { + $listObjectsV2Template = self::$s3BodyTemplates[$operation]; + $listObjectsV2ContentsTemplate = self::$s3BodyTemplates[ + $operation . "::Contents" + ]; + $bodyBuilder = str_replace( + "{{Bucket}}", + $bucket, + $listObjectsV2Template + ); + $bodyBuilder = str_replace( + "{{Prefix}}", + $prefix, + $bodyBuilder + ); + + + $itemBuilder = ""; + foreach ($body as $item) { + $itemBuilder = $itemBuilder . "\n$listObjectsV2ContentsTemplate"; + $itemBuilder = str_replace( + ['{Key}', '{Size}'], + [$item['key'], $item['size']], + $itemBuilder + ); + } + + $bodyBuilder = str_replace( + ['{Contents}'], + [$itemBuilder], + $bodyBuilder + ); + + $fixedBody = Utils::streamFor($bodyBuilder); + $body = null; + } else { + $fixedBody = Utils::streamFor( + str_repeat( + '*', + $headers['Content-Length'] + ) + ); + $body = null; + } + + if (isset($headers['ChecksumAlgorithm'])) { + // Checksum injection when expected to succeed at checksum validation + // This is needed because the checksum in the test is wrong + $algorithm = strtolower($headers['ChecksumAlgorithm']); + $checksumValue = ApplyChecksumMiddleware::getEncodedValue( + $algorithm, + $fixedBody + ); + $headers['Checksum'.strtoupper($algorithm)] = $checksumValue; + $fixedBody->rewind(); + } + + // If body was provided then we override the fixed one + if ($body !== null) { + $fixedBody = Utils::streamFor($body); + } + + return $fixedBody; + }, + ); + $outcome = $outcomes[0]; + try { + $s3TransferManager = new S3TransferManager( + $s3Client, + ); + $result = $s3TransferManager->downloadDirectory( + new DownloadDirectoryRequest( + $bucket, + $targetDirectory, + [], + $config, + ) + )->wait(); + if ($outcome['result'] !== 'success') { + $this->fail( + "A failure was expected on this test" + ); + } + foreach ($expectedFiles as $expectedFile) { + $filePath = $baseDirectory . DIRECTORY_SEPARATOR . $expectedFile['path']; + $this->assertFileExists( + $filePath, + ); + $this->assertEquals( + $expectedFile['size'], + filesize($filePath), + ); + } + $this->assertEquals( + $outcome['objectsDownloaded'], + $result->getObjectsDownloaded() + ); + $this->assertEquals( + $outcome['objectsFailed'], + $result->getObjectsFailed() + ); + } catch (Exception $exception) { + if ($outcome['result'] === 'success') { + $this->fail( + "A failure was not expected on this test and got: " . $exception->getMessage() + ); + } + $this->assertTrue(true); + } finally { + TestsUtility::cleanUpDir($targetDirectory); + } + } + + /** + * @param array $responses + * @param callable $bodyBuilder + * A callable to build the body of the response. It receives as + * parameter: + * - The operation that the response is for. + * - The body given in the expectation. + * - The headers given in the expectation. + * + * @return S3Client + */ + private function getS3ClientWithSequentialResponses( + array $responses, + callable $bodyBuilder + ): S3Client + { + $index = 0; + return new S3Client([ + 'region' => 'eu-west-1', + 'http_handler' => function (RequestInterface $request) + use ($bodyBuilder, $responses, &$index) { + $response = $responses[$index++]; + if ($response['status'] < 400) { + $headers = $response['headers'] ?? []; + $body = call_user_func_array( + $bodyBuilder, + [ + $response['operation'], + $response['body'] + ?? $response['contents'] + ?? null, + &$headers + ] + ); + + $this->parseCaseHeadersToAmzHeaders($headers); + + return new Response( + $response['status'], + $headers, + $body + ); + } else { + return new RejectedPromise( + new S3TransferException( + $response['errorMessage'] ?? "" + ) + ); + } + }, + ]); + } + + /** + * @param array $config + * + * @return void + */ + private function parseConfigFromCamelCaseToSnakeCase( + array &$config + ): void + { + foreach ($config as $key => $value) { + // Searches for lowercaseUPPERCASE occurrences + // Then it is replaced by using group1_group2 found. + $newKey = strtolower( + preg_replace( + "/([a-z0-9])([A-Z])/", + "$1_$2", + $key + ) + ); + unset($config[$key]); + $config[$newKey] = $value; + } + } + + /** + * @return Generator + */ + public function modeledDownloadCasesProvider(): Generator + { + $downloadCases = json_decode( + file_get_contents( + self::DOWNLOAD_BASE_CASES + ), + true + ); + foreach ($downloadCases as $case) { + yield $case['summary'] => [ + 'test_id' => $case['summary'], + 'config' => $case['config'], + 'download_request' => $case['downloadRequest'], + 'expectations' => $case['expectations'], + 'outcomes' => $case['outcomes'], + ]; + } + } + + /** + * @return Generator + */ + public function modeledUploadCasesProvider(): Generator + { + $downloadCases = json_decode( + file_get_contents( + self::UPLOAD_BASE_CASES + ), + true + ); + foreach ($downloadCases as $case) { + yield $case['summary'] => [ + 'test_id' => $case['summary'], + 'config' => $case['config'], + 'upload_request' => $case['uploadRequest'], + 'expectations' => $case['expectations'], + 'outcomes' => $case['outcomes'], + ]; + } + } + + /** + * @return Generator + */ + public function modeledUploadDirectoryCasesProvider(): Generator + { + $downloadCases = json_decode( + file_get_contents( + self::UPLOAD_DIRECTORY_BASE_CASES + ), + true + ); + foreach ($downloadCases as $case) { + yield $case['summary'] => [ + 'test_id' => $case['summary'], + 'config' => $case['config'], + 'upload_directory_request' => $case['uploadDirectoryRequest'], + 'source_structure' => $case['sourceStructure'] ?? null, + 'expectations' => $case['expectations'], + 'outcomes' => $case['outcomes'], + ]; + } + } + + /** + * @return Generator + */ + public function modeledDownloadDirectoryCasesProvider(): Generator + { + $downloadCases = json_decode( + file_get_contents( + self::DOWNLOAD_DIRECTORY_BASE_CASES + ), + true + ); + foreach ($downloadCases as $case) { + yield $case['summary'] => [ + 'test_id' => $case['summary'], + 'config' => $case['config'], + 'download_directory_request' => $case['downloadDirectoryRequest'], + 's3_objects' => $case['s3Objects'], + 'expectations' => $case['expectations'], + 'expected_files' => $case['expectedFiles'], + 'outcomes' => $case['outcomes'], + ]; + } + } + + /** + * @param array $requestArgs + * + * @return void + */ + private function parseRequestArgsFromCamelCaseToPascalCase( + array &$requestArgs + ): void + { + foreach ($requestArgs as $key => $value) { + $newKey = ucfirst($key); + unset($requestArgs[$key]); + $requestArgs[$newKey] = $value; + } + } + + /** + * @param array $caseHeaders + * + * @return void + */ + private function parseCaseHeadersToAmzHeaders(array &$caseHeaders): void + { + foreach ($caseHeaders as $key => $value) { + $newKey = $key; + switch ($key) { + case 'PartsCount': + $newKey = 'x-amz-mp-parts-count'; + break; + case 'ChecksumAlgorithm': + $newKey = 'x-amz-checksum-algorithm'; + break; + default: + if (preg_match('/Checksum[A-Z]+/', $key)) { + $newKey = 'x-amz-checksum-' . str_replace( + 'Checksum', + '', + $key + ); + } + } + + if ($newKey !== $key) { + $caseHeaders[$newKey] = $value; + unset($caseHeaders[$key]); + } + } + } + + /** + * @param array $methodsCallback If any from the callbacks below + * is not provided then a default implementation will be provided. + * - getCommand: (Closure, optional) This callable will + * receive as parameters: + * - $commandName: (string, optional) + * - $args: (array, optional) + * - executeAsync: (Closure, optional) This callable will + * receive as parameter: + * - $command: (CommandInterface, optional) + * + * @return S3Client + */ + private function getS3ClientMock( + array $methodsCallback = [] + ): S3Client + { + if (isset($methodsCallback['getCommand']) && !is_callable($methodsCallback['getCommand'])) { + throw new InvalidArgumentException("getCommand should be callable"); + } elseif (!isset($methodsCallback['getCommand'])) { + $methodsCallback['getCommand'] = function ( + string $commandName, + array $args + ) { + return new Command($commandName, $args); + }; + } + + if (isset($methodsCallback['executeAsync']) && !is_callable($methodsCallback['executeAsync'])) { + throw new InvalidArgumentException("getObject should be callable"); + } elseif (!isset($methodsCallback['executeAsync'])) { + $methodsCallback['executeAsync'] = function ($command) { + return match ($command->getName()) { + 'CreateMultipartUpload' => Create::promiseFor(new Result([ + 'UploadId' => 'FooUploadId', + ])), + 'UploadPart', + 'CompleteMultipartUpload', + 'AbortMultipartUpload', + 'PutObject' => Create::promiseFor(new Result([])), + default => null, + }; + }; + } + + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(array_keys($methodsCallback)) + ->getMock(); + foreach ($methodsCallback as $name => $callback) { + $client->method($name)->willReturnCallback($callback); + } + + return $client; + } +} diff --git a/tests/S3/S3Transfer/test-cases/download-directory-cross-platform-compatibility.json b/tests/S3/S3Transfer/test-cases/download-directory-cross-platform-compatibility.json new file mode 100644 index 0000000000..ee5e38f654 --- /dev/null +++ b/tests/S3/S3Transfer/test-cases/download-directory-cross-platform-compatibility.json @@ -0,0 +1,813 @@ +[ + { + "summary": "Test download directory - Windows happy case", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "maxConcurrency": 10 + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "C:\\Downloads", + "s3Prefix": "documents/" + }, + "s3Objects": [ + { + "key": "documents/report.pdf", + "size": 1048576 + }, + { + "key": "documents/data/spreadsheet.xlsx", + "size": 2097152 + }, + { + "key": "documents/readme.txt", + "size": 1024 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "prefix": "documents/", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "documents/report.pdf", "size": 1048576}, + {"key": "documents/data/spreadsheet.xlsx", "size": 2097152}, + {"key": "documents/readme.txt", "size": 1024} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "documents/report.pdf", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1048576", + "Content-Range": "bytes 0-1048575/1048576", + "PartsCount": 1 + } + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "documents/data/spreadsheet.xlsx", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "2097152", + "Content-Range": "bytes 0-2097151/2097152", + "PartsCount": 1 + } + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "documents/readme.txt", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1024", + "Content-Range": "bytes 0-1023/1024", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "C:\\Downloads\\report.pdf", + "size": 1048576 + }, + { + "path": "C:\\Downloads\\data\\spreadsheet.xlsx", + "size": 2097152 + }, + { + "path": "C:\\Downloads\\readme.txt", + "size": 1024 + } + ], + "outcomes": [ + { + "result": "success", + "objectsDownloaded": 3, + "objectsFailed": 0, + "note": "Windows normal operation with subdirectories" + } + ] + }, + { + "summary": "Test download directory - macOS happy case", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "maxConcurrency": 10 + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "/Users/user/Downloads", + "s3Prefix": "photos/" + }, + "s3Objects": [ + { + "key": "photos/2023/vacation.jpg", + "size": 3145728 + }, + { + "key": "photos/2023/family.jpg", + "size": 2621440 + }, + { + "key": "photos/metadata.json", + "size": 512 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "prefix": "photos/", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "photos/2023/vacation.jpg", "size": 3145728}, + {"key": "photos/2023/family.jpg", "size": 2621440}, + {"key": "photos/metadata.json", "size": 512} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "photos/2023/vacation.jpg", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "3145728", + "Content-Range": "bytes 0-3145727/3145728", + "PartsCount": 1 + } + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "photos/2023/family.jpg", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "2621440", + "Content-Range": "bytes 0-2621439/2621440", + "PartsCount": 1 + } + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "photos/metadata.json", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "512", + "Content-Range": "bytes 0-511/512", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "/Users/user/Downloads/2023/vacation.jpg", + "size": 3145728 + }, + { + "path": "/Users/user/Downloads/2023/family.jpg", + "size": 2621440 + }, + { + "path": "/Users/user/Downloads/metadata.json", + "size": 512 + } + ], + "outcomes": [ + { + "result": "success", + "objectsDownloaded": 3, + "objectsFailed": 0, + "note": "macOS normal operation with subdirectories" + } + ] + }, + { + "summary": "Test download directory - Linux happy case", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "maxConcurrency": 10 + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "/home/user/downloads", + "s3Prefix": "projects/" + }, + "s3Objects": [ + { + "key": "projects/src/main.py", + "size": 4096 + }, + { + "key": "projects/config/settings.json", + "size": 1024 + }, + { + "key": "projects/README.md", + "size": 2048 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "prefix": "projects/", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "projects/src/main.py", "size": 4096}, + {"key": "projects/config/settings.json", "size": 1024}, + {"key": "projects/README.md", "size": 2048} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "projects/src/main.py", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "4096", + "Content-Range": "bytes 0-4095/4096", + "PartsCount": 1 + } + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "projects/config/settings.json", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1024", + "Content-Range": "bytes 0-1023/1024", + "PartsCount": 1 + } + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "projects/README.md", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "2048", + "Content-Range": "bytes 0-2047/2048", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "/home/user/downloads/src/main.py", + "size": 4096 + }, + { + "path": "/home/user/downloads/config/settings.json", + "size": 1024 + }, + { + "path": "/home/user/downloads/README.md", + "size": 2048 + } + ], + "outcomes": [ + { + "result": "success", + "objectsDownloaded": 3, + "objectsFailed": 0, + "note": "Linux normal operation with subdirectories" + } + ] + }, + { + "summary": "Test download directory - Windows case insensitivity conflict with continue policy", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "maxConcurrency": 10 + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "C:\\Downloads", + "failurePolicy": "CONTINUE" + }, + "s3Objects": [ + { + "key": "readme.txt", + "size": 1024 + }, + { + "key": "README.TXT", + "size": 2048 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "readme.txt", "size": 1024}, + {"key": "README.TXT", "size": 2048} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "readme.txt", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1024", + "Content-Range": "bytes 0-1023/1024", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "C:\\Downloads\\readme.txt", + "size": 2048, + "note": "Second file overwrites first due to case insensitivity" + } + ], + "outcomes": [ + { + "result": "partial_success", + "objectsDownloaded": 1, + "objectsFailed": 1, + "note": "Case conflict detected and handled with continue policy" + } + ] + }, + { + "summary": "Test download directory - Windows case insensitivity conflict with default policy", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "maxConcurrency": 10 + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "C:\\Downloads" + }, + "s3Objects": [ + { + "key": "document.pdf", + "size": 1048576 + }, + { + "key": "DOCUMENT.PDF", + "size": 2097152 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "document.pdf", "size": 1048576}, + {"key": "DOCUMENT.PDF", "size": 2097152} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "document.pdf", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1048576", + "Content-Range": "bytes 0-1048575/1048576", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [], + "outcomes": [ + { + "result": "failure", + "objectsDownloaded": 0, + "objectsFailed": 2, + "error": "CaseConflictError", + "note": "Case conflict detected, operation terminated with default policy" + } + ] + }, + { + "summary": "Test download directory - Windows invalid filename characters", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "maxConcurrency": 10 + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "C:\\Downloads", + "failurePolicy": "CONTINUE" + }, + "s3Objects": [ + { + "key": "valid-file.txt", + "size": 1024 + }, + { + "key": "fileinvalid:chars?.txt", + "size": 2048 + }, + { + "key": "another|invalid*file.txt", + "size": 4096 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "valid-file.txt", "size": 1024}, + {"key": "fileinvalid:chars?.txt", "size": 2048}, + {"key": "another|invalid*file.txt", "size": 4096} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "valid-file.txt", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1024", + "Content-Range": "bytes 0-1023/1024", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "C:\\Downloads\\valid-file.txt", + "size": 1024 + } + ], + "outcomes": [ + { + "result": "partial_success", + "objectsDownloaded": 1, + "objectsFailed": 2, + "errors": [ + { + "key": "fileinvalid:chars?.txt", + "error": "InvalidFilenameError", + "message": "Object key contains characters invalid for Windows filesystem: < > : ?" + }, + { + "key": "another|invalid*file.txt", + "error": "InvalidFilenameError", + "message": "Object key contains characters invalid for Windows filesystem: | *" + } + ], + "note": "Invalid filename characters handled according to continue policy" + } + ] + }, + { + "summary": "Test download directory - Linux case sensitivity (no conflict)", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "maxConcurrency": 10 + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "/home/user/downloads" + }, + "s3Objects": [ + { + "key": "readme.txt", + "size": 1024 + }, + { + "key": "README.TXT", + "size": 2048 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "readme.txt", "size": 1024}, + {"key": "README.TXT", "size": 2048} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "readme.txt", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1024", + "Content-Range": "bytes 0-1023/1024", + "PartsCount": 1 + } + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "README.TXT", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "2048", + "Content-Range": "bytes 0-2047/2048", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "/home/user/downloads/readme.txt", + "size": 1024 + }, + { + "path": "/home/user/downloads/README.TXT", + "size": 2048 + } + ], + "outcomes": [ + { + "result": "success", + "objectsDownloaded": 2, + "objectsFailed": 0, + "note": "Linux case-sensitive filesystem allows both files" + } + ] + }, + { + "summary": "Test download directory - macOS case insensitivity conflict with continue policy", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "maxConcurrency": 10 + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "/Users/user/Downloads", + "failurePolicy": "CONTINUE" + }, + "s3Objects": [ + { + "key": "report.pdf", + "size": 1048576 + }, + { + "key": "REPORT.PDF", + "size": 2097152 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "report.pdf", "size": 1048576}, + {"key": "REPORT.PDF", "size": 2097152} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "report.pdf", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1048576", + "Content-Range": "bytes 0-1048575/1048576", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "/Users/user/Downloads/report.pdf", + "size": 2097152, + "note": "Second file overwrites first due to case insensitivity on macOS" + } + ], + "outcomes": [ + { + "result": "partial_success", + "objectsDownloaded": 1, + "objectsFailed": 1, + "note": "macOS case conflict detected and handled with continue policy" + } + ] + }, + { + "summary": "Test download directory - Linux special characters allowed", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "maxConcurrency": 10 + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "/home/user/downloads" + }, + "s3Objects": [ + { + "key": "file:with*special.txt", + "size": 1024 + }, + { + "key": "another|file?.txt", + "size": 2048 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "file:with*special.txt", "size": 1024}, + {"key": "another|file?.txt", "size": 2048} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "file:with*special.txt", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1024", + "Content-Range": "bytes 0-1023/1024", + "PartsCount": 1 + } + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "another|file?.txt", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "2048", + "Content-Range": "bytes 0-2047/2048", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "/home/user/downloads/file:with*special.txt", + "size": 1024 + }, + { + "path": "/home/user/downloads/another|file?.txt", + "size": 2048 + } + ], + "outcomes": [ + { + "result": "success", + "objectsDownloaded": 2, + "objectsFailed": 0, + "note": "Linux filesystem allows special characters that are invalid on Windows" + } + ] + } +] diff --git a/tests/S3/S3Transfer/test-cases/download-directory.json b/tests/S3/S3Transfer/test-cases/download-directory.json new file mode 100644 index 0000000000..df70989037 --- /dev/null +++ b/tests/S3/S3Transfer/test-cases/download-directory.json @@ -0,0 +1,808 @@ +[ + { + "summary": "Test download directory - basic download", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "maxConcurrency": 10 + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "/local/downloads" + }, + "s3Objects": [ + { + "key": "photo1.jpg", + "size": 2048576, + "lastModified": "2023-01-15T10:30:00Z" + }, + { + "key": "photo2.jpg", + "size": 1048576, + "lastModified": "2023-01-15T11:00:00Z" + }, + { + "key": "readme.txt", + "size": 1024, + "lastModified": "2023-01-10T09:00:00Z" + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "photo1.jpg", "size": 2048576}, + {"key": "photo2.jpg", "size": 1048576}, + {"key": "readme.txt", "size": 1024} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "photo1.jpg", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "2048576", + "Content-Range": "bytes 0-2048575/2048576", + "PartsCount": 1 + } + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "photo2.jpg", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1048576", + "Content-Range": "bytes 0-1048575/1048576", + "PartsCount": 1 + } + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "readme.txt", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1024", + "Content-Range": "bytes 0-1023/1024", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "/local/downloads/photo1.jpg", + "size": 2048576 + }, + { + "path": "/local/downloads/photo2.jpg", + "size": 1048576 + }, + { + "path": "/local/downloads/readme.txt", + "size": 1024 + } + ], + "outcomes": [ + { + "result": "success", + "objectsDownloaded": 3, + "objectsFailed": 0 + } + ] + }, + { + "summary": "Test download directory - with s3Prefix", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "maxConcurrency": 10 + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "/local/downloads", + "s3Prefix": "photos/" + }, + "s3Objects": [ + { + "key": "photos/2023/jan/photo1.jpg", + "size": 1048576, + "lastModified": "2023-01-15T10:30:00Z" + }, + { + "key": "photos/2023/jan/photo2.jpg", + "size": 1048576, + "lastModified": "2023-01-15T11:00:00Z" + }, + { + "key": "photos/readme.txt", + "size": 1024, + "lastModified": "2023-01-10T09:00:00Z" + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "prefix": "photos/", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "photos/2023/jan/photo1.jpg", "size": 1048576}, + {"key": "photos/2023/jan/photo2.jpg", "size": 1048576}, + {"key": "photos/readme.txt", "size": 1024} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "photos/2023/jan/photo1.jpg", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1048576", + "Content-Range": "bytes 0-1048575/1048576", + "PartsCount": 1 + } + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "photos/2023/jan/photo2.jpg", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1048576", + "Content-Range": "bytes 0-1048575/1048576", + "PartsCount": 1 + } + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "photos/readme.txt", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1024", + "Content-Range": "bytes 0-1023/1024", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "/local/downloads/2023/jan/photo1.jpg", + "size": 1048576 + }, + { + "path": "/local/downloads/2023/jan/photo2.jpg", + "size": 1048576 + }, + { + "path": "/local/downloads/readme.txt", + "size": 1024 + } + ], + "outcomes": [ + { + "result": "success", + "objectsDownloaded": 3, + "objectsFailed": 0 + } + ] + }, + { + "summary": "Test download directory - with filter callback", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "maxConcurrency": 10 + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "/local/downloads", + "s3Prefix": "mixed/", + "filter": "*.jpg" + }, + "s3Objects": [ + { + "key": "mixed/image.jpg", + "size": 1048576 + }, + { + "key": "mixed/document.txt", + "size": 2048 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "prefix": "mixed/", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "mixed/image.jpg", "size": 1048576}, + {"key": "mixed/document.txt", "size": 2048} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "mixed/image.jpg", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1048576", + "Content-Range": "bytes 0-1048575/1048576", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "/local/downloads/image.jpg", + "size": 1048576 + } + ], + "outcomes": [ + { + "result": "success", + "objectsDownloaded": 1, + "objectsFailed": 0, + "note": "document.txt filtered out" + } + ] + }, + { + "summary": "Test download directory - skip folder objects", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART" + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "/local/downloads", + "s3Prefix": "data/" + }, + "s3Objects": [ + { + "key": "data/folder1/", + "size": 0 + }, + { + "key": "data/file1.txt", + "size": 1024 + }, + { + "key": "data/folder2/", + "size": 0 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "prefix": "data/", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "data/folder1/", "size": 0}, + {"key": "data/file1.txt", "size": 1024}, + {"key": "data/folder2/", "size": 0} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "data/file1.txt", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1024", + "Content-Range": "bytes 0-1023/1024", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "/local/downloads/file1.txt", + "size": 1024 + } + ], + "outcomes": [ + { + "result": "success", + "objectsDownloaded": 1, + "objectsFailed": 0, + "note": "folder objects skipped" + } + ] + }, + { + "summary": "Test download directory - path traversal security validation", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART" + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "/local/downloads", + "recursive": true, + "failurePolicy": "CONTINUE_ON_FAILURE" + }, + "s3Objects": [ + { + "key": "../../../etc/passwd", + "size": 1024 + }, + { + "key": "safe-file.txt", + "size": 2048 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "../../../etc/passwd", "size": 1024}, + {"key": "safe-file.txt", "size": 2048} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "safe-file.txt", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "2048", + "Content-Range": "bytes 0-2047/2048", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "/local/downloads/safe-file.txt", + "size": 2048 + } + ], + "outcomes": [ + { + "result": "partial_success", + "objectsDownloaded": 1, + "objectsFailed": 1, + "note": "../../../etc/passwd rejected due to path traversal" + } + ] + }, + { + "summary": "Test download directory - path traversal with relative parent directory", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART" + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "/local/downloads", + "failurePolicy": "CONTINUE_ON_FAILURE" + }, + "s3Objects": [ + { + "key": "../2023/Jan/1.png", + "size": 1024 + }, + { + "key": "safe-file.txt", + "size": 2048 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "../2023/Jan/1.png", "size": 1024}, + {"key": "safe-file.txt", "size": 2048} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "safe-file.txt", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "2048", + "Content-Range": "bytes 0-2047/2048", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "/local/downloads/safe-file.txt", + "size": 2048 + } + ], + "outcomes": [ + { + "result": "partial_success", + "objectsDownloaded": 1, + "objectsFailed": 1, + "note": "../2023/Jan/1.png rejected due to path traversal" + } + ] + }, + { + "summary": "Test download directory - complex path traversal with multiple levels", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART" + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "/local/downloads", + "failurePolicy": "CONTINUE_ON_FAILURE" + }, + "s3Objects": [ + { + "key": "foo/../2023/../../Jan/1.png", + "size": 1024 + }, + { + "key": "normal/file.txt", + "size": 512 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "foo/../2023/../../Jan/1.png", "size": 1024}, + {"key": "normal/file.txt", "size": 512} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "normal/file.txt", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "512", + "Content-Range": "bytes 0-511/512", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "/local/downloads/normal/file.txt", + "size": 512 + } + ], + "outcomes": [ + { + "result": "partial_success", + "objectsDownloaded": 1, + "objectsFailed": 1, + "note": "foo/../2023/../../Jan/1.png rejected due to path traversal" + } + ] + }, + { + "summary": "Test download directory - failure handling with continue policy", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "maxConcurrency": 10 + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "/local/downloads", + "failurePolicy": "CONTINUE" + }, + "s3Objects": [ + { + "key": "file1.txt", + "size": 1024 + }, + { + "key": "file2.txt", + "size": 2048 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "file1.txt", "size": 1024}, + {"key": "file2.txt", "size": 2048} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "file1.txt", + "partNumber": 1 + }, + "response": { + "status": 404, + "error": "NoSuchKey" + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "file2.txt", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "2048", + "Content-Range": "bytes 0-2047/2048", + "PartsCount": 1 + } + } + } + ], + "expectedFiles": [ + { + "path": "/local/downloads/file2.txt", + "size": 2048 + } + ], + "outcomes": [ + { + "result": "partial_success", + "objectsDownloaded": 1, + "objectsFailed": 1, + "note": "file1.txt failed but operation continued" + } + ] + }, + { + "summary": "Test download directory - failure handling with default policy", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "maxConcurrency": 10 + }, + "downloadDirectoryRequest": { + "bucket": "example-bucket", + "destination": "/local/downloads" + }, + "s3Objects": [ + { + "key": "file1.txt", + "size": 1024 + }, + { + "key": "file2.txt", + "size": 2048 + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-bucket", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "file1.txt", "size": 1024}, + {"key": "file2.txt", "size": 2048} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-bucket", + "key": "file1.txt", + "partNumber": 1 + }, + "response": { + "status": 404, + "error": "NoSuchKey" + } + } + ], + "expectedFiles": [], + "outcomes": [ + { + "result": "failure", + "objectsDownloaded": 0, + "objectsFailed": 1, + "note": "operation stopped after first failure with default policy" + } + ] + }, + { + "summary": "Test download directory - S3 directory bucket", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "maxConcurrency": 10 + }, + "downloadDirectoryRequest": { + "bucket": "example-directory-bucket--use1-az1--x-s3", + "destination": "/local/downloads" + }, + "s3Objects": [ + { + "key": "report.pdf", + "size": 1048576, + "lastModified": "2023-01-15T10:30:00Z" + }, + { + "key": "data/metrics.csv", + "size": 2048576, + "lastModified": "2023-01-15T11:00:00Z" + } + ], + "expectations": [ + { + "request": { + "operation": "ListObjectsV2", + "bucket": "example-directory-bucket--use1-az1--x-s3", + "delimiter": null + }, + "response": { + "status": 200, + "contents": [ + {"key": "report.pdf", "size": 1048576}, + {"key": "data/metrics.csv", "size": 2048576} + ] + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-directory-bucket--use1-az1--x-s3", + "key": "report.pdf" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"dir123\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "bucket": "example-directory-bucket--use1-az1--x-s3", + "key": "data/metrics.csv" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"dir456\"" + } + } + } + ], + "expectedFiles": [ + { + "path": "/local/downloads/report.pdf", + "size": 1048576 + }, + { + "path": "/local/downloads/data/metrics.csv", + "size": 2048576 + } + ], + "outcomes": [ + { + "result": "success", + "objectsDownloaded": 2, + "objectsFailed": 0, + "note": "Successfully downloaded from S3 directory bucket" + } + ] + } +] diff --git a/tests/S3/S3Transfer/test-cases/download-single-object.json b/tests/S3/S3Transfer/test-cases/download-single-object.json new file mode 100644 index 0000000000..4e9537d4c4 --- /dev/null +++ b/tests/S3/S3Transfer/test-cases/download-single-object.json @@ -0,0 +1,582 @@ +[ + { + "summary": "Test download with part GET - single object download (object size < part size)", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1048576", + "Content-Range": "bytes 0-1048575/1048576", + "PartsCount": 1, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "downloadType": "single_object", + "totalBytes": 1048576, + "partCount": 1 + } + ] + }, + { + "summary": "Test download with part GET - multipart download (object size > part size)", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/25165824", + "PartsCount": 3, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "partNumber": 2, + "headers": { + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 8388608-16777215/25165824", + "PartsCount": 3, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "partNumber": 3, + "headers": { + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 16777216-25165823/25165824", + "PartsCount": 3, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "downloadType": "multipart", + "totalBytes": 25165824, + "partCount": 3 + } + ] + }, + { + "summary": "Test download with ranged GET - single object download (object size < part size)", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "RANGE", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=0-8388607" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1048576", + "Content-Range": "bytes 0-1048575/1048576", + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "downloadType": "single_object", + "totalBytes": 1048576, + "partCount": 1 + } + ] + }, + { + "summary": "Test download with ranged GET - multipart download (object size > part size)", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "RANGE", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=0-8388607" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/25165824", + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=8388608-16777215", + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 8388608-16777215/25165824", + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=16777216-25165823", + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 16777216-25165823/25165824", + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "downloadType": "multipart", + "totalBytes": 25165824, + "partCount": 3 + } + ] + }, + { + "summary": "Test download with part GET - error handling when part request fails", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/25165824", + "PartsCount": 3, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "partNumber": 2, + "headers": { + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 500, + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error" + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error" + } + ] + }, + { + "summary": "Test download with ranged GET - error handling when range request fails", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "RANGE", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=0-8388607" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/25165824", + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=8388608-16777215", + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 500, + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error" + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error" + } + ] + }, + { + "summary": "Test download with part GET - checksum validation success", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "partNumber": 1, + "checksumMode": "ENABLED" + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/8388608", + "PartsCount": 1, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"", + "ChecksumAlgorithm": "CRC32", + "ChecksumCRC32": "abcdef12" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "downloadType": "single_object", + "totalBytes": 8388608, + "partCount": 1, + "checksumValidated": true, + "checksumAlgorithm": "CRC32" + } + ] + }, + { + "summary": "Test download with part GET - checksum validation failure", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "partNumber": 1, + "checksumMode": "ENABLED" + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/8388608", + "PartsCount": 1, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"", + "ChecksumAlgorithm": "CRC32", + "ChecksumCRC32": "abcdef12" + }, + "body": "CORRUPTED_DATA" + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "ChecksumMismatch", + "errorMessage": "Calculated response checksum did not match the expected value" + } + ] + }, + { + "summary": "Test download with part GET - multipart download with uneven last part", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/10485760", + "PartsCount": 2, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "partNumber": 2, + "headers": { + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "2097152", + "Content-Range": "bytes 8388608-10485759/10485760", + "PartsCount": 2, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "downloadType": "multipart", + "totalBytes": 10485760, + "partCount": 2, + "lastPartSize": 2097152 + } + ] + }, + { + "summary": "Test download with ranged GET - multipart download with uneven last part", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "RANGE", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=0-8388607" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/10485760", + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=8388608-16777215", + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "2097152", + "Content-Range": "bytes 8388608-10485759/10485760", + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "downloadType": "multipart", + "totalBytes": 10485760, + "partCount": 2, + "lastPartSize": 2097152 + } + ] + }, + { + "summary": "Test download with part GET - validation failure when part count mismatch", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/25165824", + "PartsCount": 3, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "partNumber": 2, + "headers": { + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 8388608-16777215/25165824", + "PartsCount": 3, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "ValidationError", + "errorMessage": "Expected 3 parts but only received 2" + } + ] + } +] \ No newline at end of file diff --git a/tests/S3/S3Transfer/test-cases/upload-directory-cross-platform-compatibility.json b/tests/S3/S3Transfer/test-cases/upload-directory-cross-platform-compatibility.json new file mode 100644 index 0000000000..a0866dbc48 --- /dev/null +++ b/tests/S3/S3Transfer/test-cases/upload-directory-cross-platform-compatibility.json @@ -0,0 +1,315 @@ +[ + { + "summary": "Test upload directory - Windows happy case", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "maxConcurrency": 10 + }, + "uploadDirectoryRequest": { + "bucket": "example-bucket", + "source": "C:\\Users\\user\\Documents", + "recursive": true, + "s3Prefix": "backup" + }, + "sourceStructure": [ + { + "path": "C:\\Users\\user\\Documents\\report.docx", + "type": "file", + "size": 1048576 + }, + { + "path": "C:\\Users\\user\\Documents\\projects\\presentation.pptx", + "type": "file", + "size": 5242880 + }, + { + "path": "C:\\Users\\user\\Documents\\notes.txt", + "type": "file", + "size": 2048 + } + ], + "expectations": [ + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "backup/report.docx", + "filePath": "C:\\Users\\user\\Documents\\report.docx" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"winhappy123\"" + } + } + }, + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "backup/projects/presentation.pptx", + "filePath": "C:\\Users\\user\\Documents\\projects\\presentation.pptx" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"winhappy456\"" + } + } + }, + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "backup/notes.txt", + "filePath": "C:\\Users\\user\\Documents\\notes.txt" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"winhappy789\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "objectsUploaded": 3, + "objectsFailed": 0, + "note": "Windows normal operation with backslash to forward slash conversion" + } + ] + }, + { + "summary": "Test upload directory - macOS happy case", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "maxConcurrency": 10 + }, + "uploadDirectoryRequest": { + "bucket": "example-bucket", + "source": "/Users/user/Pictures", + "recursive": true, + "s3Prefix": "photos" + }, + "sourceStructure": [ + { + "path": "/Users/user/Pictures/IMG_001.jpg", + "type": "file", + "size": 4194304 + }, + { + "path": "/Users/user/Pictures/2023/summer/beach.jpg", + "type": "file", + "size": 6291456 + }, + { + "path": "/Users/user/Pictures/album.json", + "type": "file", + "size": 1024 + } + ], + "expectations": [ + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "photos/IMG_001.jpg", + "filePath": "/Users/user/Pictures/IMG_001.jpg" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"machappy123\"" + } + } + }, + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "photos/2023/summer/beach.jpg", + "filePath": "/Users/user/Pictures/2023/summer/beach.jpg" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"machappy456\"" + } + } + }, + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "photos/album.json", + "filePath": "/Users/user/Pictures/album.json" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"machappy789\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "objectsUploaded": 3, + "objectsFailed": 0, + "note": "macOS normal operation with forward slash preservation" + } + ] + }, + { + "summary": "Test upload directory - Linux happy case", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "maxConcurrency": 10 + }, + "uploadDirectoryRequest": { + "bucket": "example-bucket", + "source": "/home/user/workspace/project", + "recursive": true, + "s3Prefix": "code" + }, + "sourceStructure": [ + { + "path": "/home/user/workspace/project/main.c", + "type": "file", + "size": 8192 + }, + { + "path": "/home/user/workspace/project/lib/utils.h", + "type": "file", + "size": 2048 + }, + { + "path": "/home/user/workspace/project/Makefile", + "type": "file", + "size": 1024 + } + ], + "expectations": [ + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "code/main.c", + "filePath": "/home/user/workspace/project/main.c" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"linuxhappy123\"" + } + } + }, + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "code/lib/utils.h", + "filePath": "/home/user/workspace/project/lib/utils.h" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"linuxhappy456\"" + } + } + }, + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "code/Makefile", + "filePath": "/home/user/workspace/project/Makefile" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"linuxhappy789\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "objectsUploaded": 3, + "objectsFailed": 0, + "note": "Linux normal operation with forward slash preservation" + } + ] + }, + { + "summary": "Test upload directory - Linux case sensitivity (distinct files)", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "maxConcurrency": 10 + }, + "uploadDirectoryRequest": { + "bucket": "example-bucket", + "source": "/home/user/documents", + "recursive": true, + "s3Prefix": "backup" + }, + "sourceStructure": [ + { + "path": "/home/user/documents/readme.txt", + "type": "file", + "size": 1024 + }, + { + "path": "/home/user/documents/README.TXT", + "type": "file", + "size": 2048 + } + ], + "expectations": [ + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "backup/readme.txt", + "filePath": "/home/user/documents/readme.txt" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"linux123\"" + } + } + }, + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "backup/README.TXT", + "filePath": "/home/user/documents/README.TXT" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"linux456\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "objectsUploaded": 2, + "objectsFailed": 0, + "note": "Linux case-sensitive filesystem treats files as distinct" + } + ] + } +] diff --git a/tests/S3/S3Transfer/test-cases/upload-directory.json b/tests/S3/S3Transfer/test-cases/upload-directory.json new file mode 100644 index 0000000000..f49e8c0849 --- /dev/null +++ b/tests/S3/S3Transfer/test-cases/upload-directory.json @@ -0,0 +1,601 @@ +[ + { + "summary": "Test upload directory - basic recursive upload", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "maxConcurrency": 10 + }, + "uploadDirectoryRequest": { + "bucket": "example-bucket", + "source": "/local/photos", + "recursive": true, + "followSymbolicLinks": false + }, + "sourceStructure": [ + { + "path": "/local/photos/2023/jan/photo1.jpg", + "type": "file", + "size": 2048576 + }, + { + "path": "/local/photos/2023/jan/photo2.jpg", + "type": "file", + "size": 1048576 + }, + { + "path": "/local/photos/readme.txt", + "type": "file", + "size": 1024 + } + ], + "expectations": [ + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "2023/jan/photo1.jpg", + "filePath": "/local/photos/2023/jan/photo1.jpg" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"abc123\"" + } + } + }, + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "2023/jan/photo2.jpg", + "filePath": "/local/photos/2023/jan/photo2.jpg" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"def456\"" + } + } + }, + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "readme.txt", + "filePath": "/local/photos/readme.txt" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"ghi789\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "objectsUploaded": 3, + "objectsFailed": 0 + } + ] + }, + { + "summary": "Test upload directory - with s3Prefix", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "maxConcurrency": 10 + }, + "uploadDirectoryRequest": { + "bucket": "example-bucket", + "source": "/local/photos", + "recursive": true, + "s3Prefix": "backup", + "followSymbolicLinks": false + }, + "sourceStructure": [ + { + "path": "/local/photos/2023/jan/photo1.jpg", + "type": "file", + "size": 1048576 + }, + { + "path": "/local/photos/2023/jan/photo2.jpg", + "type": "file", + "size": 1048576 + }, + { + "path": "/local/photos/readme.txt", + "type": "file", + "size": 1024 + } + ], + "expectations": [ + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "backup/2023/jan/photo1.jpg", + "filePath": "/local/photos/2023/jan/photo1.jpg" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"abc123\"" + } + } + }, + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "backup/2023/jan/photo2.jpg", + "filePath": "/local/photos/2023/jan/photo2.jpg" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"def456\"" + } + } + }, + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "backup/readme.txt", + "filePath": "/local/photos/readme.txt" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"ghi789\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "objectsUploaded": 3, + "objectsFailed": 0 + } + ] + }, + { + "summary": "Test upload directory - non-recursive upload", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "maxConcurrency": 10 + }, + "uploadDirectoryRequest": { + "bucket": "example-bucket", + "source": "/local/docs", + "recursive": false, + "s3Prefix": null + }, + "sourceStructure": [ + { + "path": "/local/docs/file1.txt", + "type": "file", + "size": 1048576 + }, + { + "path": "/local/docs/subdir/file2.txt", + "type": "file", + "size": 2048576 + } + ], + "expectations": [ + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "file1.txt", + "filePath": "/local/docs/file1.txt" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"xyz123\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "objectsUploaded": 1, + "objectsFailed": 0, + "note": "subdir/file2.txt skipped due to recursive=false" + } + ] + }, + { + "summary": "Test upload directory - with filter callback", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "maxConcurrency": 10 + }, + "uploadDirectoryRequest": { + "bucket": "example-bucket", + "source": "/local/mixed", + "recursive": true, + "filter": "*.jpg" + }, + "sourceStructure": [ + { + "path": "/local/mixed/image.jpg", + "type": "file", + "size": 2048576 + }, + { + "path": "/local/mixed/document.txt", + "type": "file", + "size": 2048576 + } + ], + "expectations": [ + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "image.jpg", + "filePath": "/local/mixed/image.jpg" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"filtered123\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "objectsUploaded": 1, + "objectsFailed": 0, + "note": "document.txt filtered out" + } + ] + }, + { + "summary": "Test upload directory - failure handling with continue policy", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "maxConcurrency": 10 + }, + "uploadDirectoryRequest": { + "bucket": "example-bucket", + "source": "/local/files", + "recursive": true, + "failurePolicy": "CONTINUE_ON_FAILURE" + }, + "sourceStructure": [ + { + "path": "/local/files/good.txt", + "type": "file", + "size": 2048576 + }, + { + "path": "/local/files/bad.txt", + "type": "file", + "size": 2048576 + } + ], + "expectations": [ + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "good.txt", + "filePath": "/local/files/good.txt" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"good123\"" + } + } + }, + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "bad.txt", + "filePath": "/local/files/bad.txt" + }, + "response": { + "status": 403, + "error": "AccessDenied" + } + } + ], + "outcomes": [ + { + "result": "partial_success", + "objectsUploaded": 1, + "objectsFailed": 1 + } + ] + }, + { + "summary": "Test upload directory - failure handling with default policy", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "maxConcurrency": 1 + }, + "uploadDirectoryRequest": { + "bucket": "example-bucket", + "source": "/local/files", + "recursive": true + }, + "sourceStructure": [ + { + "path": "/local/files/file1.txt", + "type": "file", + "size": 2048576 + }, + { + "path": "/local/files/file2.txt", + "type": "file", + "size": 2048576 + } + ], + "expectations": [ + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "file1.txt", + "filePath": "/local/files/file1.txt" + }, + "response": { + "status": 403, + "error": "AccessDenied" + } + } + ], + "outcomes": [ + { + "result": "failure", + "objectsUploaded": 0, + "objectsFailed": 1, + "error": "AccessDenied", + "note": "Upload terminated on first failure due to default policy" + } + ] + }, + { + "summary": "Test upload directory - source directory does not exist", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216 + }, + "uploadDirectoryRequest": { + "bucket": "example-bucket", + "source": "/nonexistent/path", + "recursive": true + }, + "expectations": [], + "outcomes": [ + { + "result": "failure", + "error": "DirectoryNotFound", + "message": "Source directory does not exist" + } + ] + }, + { + "summary": "Test upload directory - empty directory", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "maxConcurrency": 10 + }, + "uploadDirectoryRequest": { + "bucket": "example-bucket", + "source": "/local/empty-dir", + "recursive": true, + "followSymbolicLinks": false + }, + "sourceStructure": [], + "expectations": [], + "outcomes": [ + { + "result": "success", + "objectsUploaded": 0, + "objectsFailed": 0, + "note": "Empty directory - no files to upload" + } + ] + }, + { + "summary": "Test upload directory - S3 directory bucket", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "maxConcurrency": 10 + }, + "uploadDirectoryRequest": { + "bucket": "example-directory-bucket--use1-az1--x-s3", + "source": "/local/docs", + "recursive": true, + "followSymbolicLinks": false + }, + "sourceStructure": [ + { + "path": "/local/docs/report.pdf", + "type": "file", + "size": 1048576 + }, + { + "path": "/local/docs/data/metrics.csv", + "type": "file", + "size": 2048576 + } + ], + "expectations": [ + { + "request": { + "operation": "PutObject", + "bucket": "example-directory-bucket--use1-az1--x-s3", + "key": "report.pdf", + "filePath": "/local/docs/report.pdf" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"dir123\"" + } + } + }, + { + "request": { + "operation": "PutObject", + "bucket": "example-directory-bucket--use1-az1--x-s3", + "key": "data/metrics.csv", + "filePath": "/local/docs/data/metrics.csv" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"dir456\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "objectsUploaded": 2, + "objectsFailed": 0, + "note": "Successfully uploaded to S3 directory bucket" + } + ] + }, + { + "summary": "Test upload directory - multipart upload for large file", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "maxConcurrency": 10 + }, + "uploadDirectoryRequest": { + "bucket": "example-bucket", + "source": "/local/large-files", + "recursive": false, + "followSymbolicLinks": false + }, + "sourceStructure": [ + { + "path": "/local/large-files/large-video.mp4", + "type": "file", + "size": 12582912 + }, + { + "path": "/local/large-files/metadata.txt", + "type": "file", + "size": 2048 + } + ], + "expectations": [ + { + "request": { + "operation": "PutObject", + "bucket": "example-bucket", + "key": "metadata.txt", + "filePath": "/local/large-files/metadata.txt" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"smalletag\"" + } + } + }, + { + "request": { + "operation": "CreateMultipartUpload", + "bucket": "example-bucket", + "key": "large-video.mp4" + }, + "response": { + "status": 200, + "body": { + "UploadId": "upload123", + "Bucket": "example-bucket", + "Key": "large-video.mp4" + } + } + }, + { + "request": { + "operation": "UploadPart", + "bucket": "example-bucket", + "key": "large-video.mp4", + "uploadId": "upload123", + "partNumber": 1, + "filePath": "/local/large-files/large-video.mp4", + "partSize": 8388608 + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"part1etag\"" + } + } + }, + { + "request": { + "operation": "UploadPart", + "bucket": "example-bucket", + "key": "large-video.mp4", + "uploadId": "upload123", + "partNumber": 2, + "filePath": "/local/large-files/large-video.mp4", + "partSize": 4194304 + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"part2etag\"" + } + } + }, + { + "request": { + "operation": "CompleteMultipartUpload", + "bucket": "example-bucket", + "key": "large-video.mp4", + "uploadId": "upload123", + "parts": [ + {"partNumber": 1, "etag": "\"part1etag\"", "checksumCRC32": "abc123"}, + {"partNumber": 2, "etag": "\"part2etag\"", "checksumCRC32": "def456"} + ] + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"finaletag\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "objectsUploaded": 2, + "objectsFailed": 0, + "note": "20MB file uploaded using multipart upload with 2 parts, small file uploaded normally" + } + ] + } +] diff --git a/tests/S3/S3Transfer/test-cases/upload-single-object.json b/tests/S3/S3Transfer/test-cases/upload-single-object.json new file mode 100644 index 0000000000..209bec9217 --- /dev/null +++ b/tests/S3/S3Transfer/test-cases/upload-single-object.json @@ -0,0 +1,746 @@ +[ + { + "summary": "Test upload with single object upload (object size < multipart threshold)", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 10485760, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "PutObject", + "contentLength": 10485760 + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"", + "ChecksumAlgorithm": "CRC32", + "ChecksumCRC32": "abcdef12" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "uploadType": "single_object", + "totalBytes": 10485760 + } + ] + }, + { + "summary": "Test upload with multipart upload (object size > multipart threshold)", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 25165824, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "CreateMultipartUpload", + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "body": { + "UploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 1, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 2, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 3, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"b54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "87654321" + } + } + }, + { + "request": { + "operation": "CompleteMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "multipartUpload": { + "Parts": [ + { + "PartNumber": 1, + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + }, + { + "PartNumber": 2, + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + }, + { + "PartNumber": 3, + "ETag": "\"b54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "87654321" + } + ] + }, + "checksumAlgorithm": "CRC32", + "mpuObjectSize": 25165824 + }, + "response": { + "status": 200, + "body": { + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"", + "Bucket": "example-bucket", + "Key": "example-object", + "Location": "https://example-bucket.s3.amazonaws.com/example-object" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "uploadType": "multipart", + "totalBytes": 25165824, + "partCount": 3, + "checksumAlgorithm": "CRC32" + } + ] + }, + { + "summary": "Test upload with multipart upload and uneven last part", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 10485760, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "CreateMultipartUpload", + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "body": { + "UploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 1, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 2, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 2097152, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + } + } + }, + { + "request": { + "operation": "CompleteMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "multipartUpload": { + "Parts": [ + { + "PartNumber": 1, + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + }, + { + "PartNumber": 2, + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + } + ] + }, + "checksumAlgorithm": "CRC32", + "mpuObjectSize": 10485760 + }, + "response": { + "status": 200, + "body": { + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "uploadType": "multipart", + "totalBytes": 10485760, + "partCount": 2, + "lastPartSize": 2097152, + "checksumAlgorithm": "CRC32" + } + ] + }, + { + "summary": "Test upload with multipart upload and full object checksum", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 16777216, + "checksumAlgorithm": "CRC32", + "checksumCRC32": "fedcba98", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "CreateMultipartUpload", + "checksumAlgorithm": "CRC32", + "checksumType": "FULL_OBJECT" + }, + "response": { + "status": 200, + "body": { + "UploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 1, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 2, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + } + } + }, + { + "request": { + "operation": "CompleteMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "multipartUpload": { + "Parts": [ + { + "PartNumber": 1, + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + }, + { + "PartNumber": 2, + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + } + ] + }, + "checksumAlgorithm": "CRC32", + "checksumType": "FULL_OBJECT", + "checksumCRC32": "fedcba98", + "mpuObjectSize": 16777216 + }, + "response": { + "status": 200, + "body": { + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"", + "Bucket": "example-bucket", + "Key": "example-object", + "Location": "https://example-bucket.s3.amazonaws.com/example-object", + "ChecksumCRC32": "fedcba98" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "uploadType": "multipart", + "totalBytes": 16777216, + "partCount": 2, + "checksumAlgorithm": "CRC32", + "checksumType": "FULL_OBJECT" + } + ] + }, + { + "summary": "Test upload with single object upload and checksum calculation", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 10485760, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "PutObject", + "contentLength": 10485760, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"", + "ChecksumCRC32": "abcdef12" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "uploadType": "single_object", + "totalBytes": 10485760, + "checksumAlgorithm": "CRC32" + } + ] + }, + { + "summary": "Test upload with multipart upload - error handling when part upload fails", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 25165824, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "CreateMultipartUpload", + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "body": { + "UploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 1, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 2, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 500, + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error" + } + }, + { + "request": { + "operation": "AbortMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA" + }, + "response": { + "status": 204 + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error", + "abortedMultipartUpload": true + } + ] + }, + { + "summary": "Test upload with multipart upload - error handling when CompleteMultipartUpload fails", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 16777216, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "CreateMultipartUpload", + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "body": { + "UploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 1, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 2, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + } + } + }, + { + "request": { + "operation": "CompleteMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "multipartUpload": { + "Parts": [ + { + "PartNumber": 1, + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + }, + { + "PartNumber": 2, + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + } + ] + }, + "checksumAlgorithm": "CRC32", + "mpuObjectSize": 16777216 + }, + "response": { + "status": 500, + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error" + } + }, + { + "request": { + "operation": "AbortMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA" + }, + "response": { + "status": 204 + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error", + "abortedMultipartUpload": true + } + ] + }, + { + "summary": "Test upload with multipart upload - validation failure when part count mismatch", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 25165824, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "CreateMultipartUpload", + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "body": { + "UploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 1, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + } + } + }, + { + "request": { + "operation": "AbortMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA" + }, + "response": { + "status": 204 + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "ValidationError", + "errorMessage": "Expected 3 parts but only received 1", + "abortedMultipartUpload": true + } + ] + }, + { + "summary": "Test upload with multipart upload - validation failure when part size mismatch", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 16777216, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "CreateMultipartUpload", + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "body": { + "UploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 1, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 4194304, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + } + } + }, + { + "request": { + "operation": "AbortMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA" + }, + "response": { + "status": 204 + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "ValidationError", + "errorMessage": "Part size mismatch: expected 8388608 bytes but got 4194304 bytes for part 1", + "abortedMultipartUpload": true + } + ] + } +] \ No newline at end of file diff --git a/tests/TestsUtility.php b/tests/TestsUtility.php new file mode 100644 index 0000000000..9e939d58e3 --- /dev/null +++ b/tests/TestsUtility.php @@ -0,0 +1,41 @@ +tempDir)) { - $it = new \RecursiveIteratorIterator( - new \RecursiveDirectoryIterator($this->tempDir, \FilesystemIterator::SKIP_DOTS), - \RecursiveIteratorIterator::CHILD_FIRST - ); - - foreach ($it as $file) { - if ($file->isDir() && !is_link($file->getPathname())) { - @rmdir($file->getPathname()); - } else { - @unlink($file->getPathname()); - } - } - - @rmdir($this->tempDir); - } + + TestsUtility::cleanUpDir($this->tempDir); } /** @@ -1179,35 +1163,6 @@ public function testUserAgentCaptureCredentialsProfileStsWebIdTokenMetric() $s3Client->listBuckets(); } - /** - * Helper method to clean up temporary dirs. - * - * @param $dirPath - * - * @return void - */ - private function cleanUpDir($dirPath): void - { - if (!is_dir($dirPath)) { - return; - } - - $files = dir_iterator($dirPath); - foreach ($files as $file) { - if (in_array($file, ['.', '..'])) { - continue; - } - - $filePath = $dirPath . '/' . $file; - if (is_file($filePath) || !is_dir($filePath)) { - unlink($filePath); - } elseif (is_dir($filePath)) { - $this->cleanUpDir($filePath); - } - } - - rmdir($dirPath); - } /** * Test user agent captures metric for credentials resolved from