Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Progress Tracker for MultipartUploader & MultipartCopy #2699

Merged
merged 34 commits into from
Feb 17, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
b2d70f6
multipart uploader method
cintiashamsu Apr 17, 2023
51f7574
update displayProgress method
cintiashamsu Apr 17, 2023
5cb12d5
removed comments
cintiashamsu Apr 17, 2023
ebd06c4
added ability to display progressBar for MultipartUpload and Multipar…
cintiashamsu Apr 19, 2023
6dc3597
Update UploadState.php
cintiashamsu Apr 19, 2023
7f05ff1
Update AbstractUploader.php
cintiashamsu Apr 19, 2023
657ab9a
MultipartUploader and MultipartCopy now sends total size directly to …
cintiashamsu Apr 19, 2023
89c9513
Update UploadStateTest.php
cintiashamsu Apr 24, 2023
d68597c
Added assert using expectOutputString and structure for data provider
cintiashamsu Apr 24, 2023
99738f8
added two tests
cintiashamsu Apr 26, 2023
604df58
Update UploadState.php
cintiashamsu Apr 26, 2023
12e328d
Update UploadStateTest.php
cintiashamsu Apr 28, 2023
e42ab91
Update UploadStateTest.php
cintiashamsu May 1, 2023
470627a
Combine progressThresholds and progressBar into one array
cintiashamsu May 1, 2023
8cfe7e9
Updated displayProgress to use progressBar
cintiashamsu May 1, 2023
9ade473
Update UploadState.php
cintiashamsu May 2, 2023
74e7938
added exceptions for non-int arguments
cintiashamsu May 3, 2023
5eded67
Added unit tests for failed upload and type checking
cintiashamsu May 3, 2023
735c9c4
Added config option
cintiashamsu May 5, 2023
cff396a
added config option to multipartUploader and multipartCopy
cintiashamsu May 10, 2023
776cc4d
added another parameter to markPartAsUploaded for possible Glacier su…
cintiashamsu May 15, 2023
465fc61
added descriptions for methods, cleaned up tests
cintiashamsu May 24, 2023
e2d4e81
PHPUnit Polyfills on UploadStateTest
cintiashamsu May 26, 2023
1a0a580
removed array_key_first to support php 5.5
cintiashamsu May 26, 2023
eee6a67
dev guide description added
cintiashamsu May 30, 2023
1156b75
added method setDisplayProgress, changelog
cintiashamsu Jun 9, 2023
da296ce
$displayProgress not declared dynamically
cintiashamsu Jun 9, 2023
9ea4f20
line-wrap in upload state
cintiashamsu Jun 10, 2023
102f26d
Merge branch 'aws:master' into new-feature
cintiashamsu Jun 10, 2023
367bc01
=== comparison operator
cintiashamsu Jun 27, 2023
d81c87b
Update MultipartUploaderTest.php
cintiashamsu Jun 27, 2023
39b1044
displayprogress set in multipartuploader/copy
cintiashamsu Jun 27, 2023
3fe8a59
Merge branch 'master' into new-feature
stobrien89 Jun 26, 2024
91dc501
Merge branch 'master' into new-feature
stobrien89 Feb 17, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
added ability to display progressBar for MultipartUpload and Multipar…
…tCopy
  • Loading branch information
cintiashamsu committed Apr 19, 2023
commit ebd06c4da9f2fcfd1b877da0fe6238b50311e670
2 changes: 0 additions & 2 deletions src/Multipart/AbstractUploader.php
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,6 @@ protected function getUploadCommands(callable $resultHandler)
$this->source->read($this->state->getPartSize());
}
}
//prints near the end of the handleResult print statements? Maybe the thing about "Or do we just create parts til we reach the end of the file?"-------------------------------------------------------------------------------------------------------------
// print "AbstractUploader Number of parts: " . $numberOfParts . "\n";
}

/**
Expand Down
29 changes: 29 additions & 0 deletions src/Multipart/UploadState.php
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,18 @@ class UploadState
const INITIATED = 1;
const COMPLETED = 2;

protected $progressBar = [
"Transfer initiated...\n| | 0.0%\n",
"|== | 12.5%\n",
"|===== | 25.0%\n",
"|======= | 37.5%\n",
"|========== | 50.0%\n",
"|============ | 62.5%\n",
"|=============== | 75.0%\n",
"|================= | 87.5%\n",
"|====================| 100.0%\nTransfer complete!\n"
];

/** @var array Params used to identity the upload. */
private $id;

Expand All @@ -31,6 +43,7 @@ class UploadState
public function __construct(array $id)
{
$this->id = $id;
echo array_shift($this->progressBar);
}

/**
Expand Down Expand Up @@ -76,6 +89,22 @@ public function setPartSize($partSize)
$this->partSize = $partSize;
}

public function setProgressThresholds($thresholds)
{
$this->progressThresholds = $thresholds;
}

public function displayProgress($totalUploaded)
{
while (!empty($this->progressThresholds)
&& !empty($this->progressBar)
&& $totalUploaded >= $this->progressThresholds[0])
{
array_shift($this->progressThresholds);
echo array_shift($this->progressBar);
}
}

/**
* Marks a part as being uploaded.
*
Expand Down
2 changes: 2 additions & 0 deletions src/S3/MultipartCopy.php
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ public function __construct(
$client,
array_change_key_case($config) + ['source_metadata' => null]
);

$this->createProgressThresholds($this->sourceMetadata["ContentLength"]);
}

/**
Expand Down
32 changes: 2 additions & 30 deletions src/S3/MultipartUploader.php
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,6 @@ class MultipartUploader extends AbstractUploader
const PART_MIN_SIZE = 5242880;
const PART_MAX_SIZE = 5368709120;
const PART_MAX_NUM = 10000;
private $uploadedBytes = 0;
private $progressBar = [
"Transfer initiated...\n| | 0.0%\n",
"|== | 12.5%\n",
"|===== | 25.0%\n",
"|======= | 37.5%\n",
"|========== | 50.0%\n",
"|============ | 62.5%\n",
"|=============== | 75.0%\n",
"|================= | 87.5%\n",
"|====================| 100.0%\nTransfer complete!\n"
];

/**
* Creates a multipart upload for an S3 object.
Expand Down Expand Up @@ -82,12 +70,8 @@ public function __construct(
'key' => null,
'exception_class' => S3MultipartUploadException::class,
]);
$totalSize = $this->source->getSize();
$this->progressThresholds = [];
for ($i=1;$i<=8;$i++) {
$this->progressThresholds []= round($totalSize*($i/8));
}
echo array_shift($this->progressBar);

$this->createProgressThresholds($this->source->getSize());
}

protected function loadUploadWorkflowInfo()
Expand Down Expand Up @@ -152,22 +136,10 @@ protected function createPart($seekable, $number)
}

$data['ContentLength'] = $contentLength;
$this->uploadedBytes += $contentLength;
$this->displayProgress();

return $data;
}

protected function displayProgress()
{
$threshold = $this->progressThresholds;

if (!empty($threshold) and !empty($this->progressBar) and $this->uploadedBytes >= $threshold[0]) {
array_shift($this->progressThresholds);
echo array_shift($this->progressBar);
}
}

protected function extractETag(ResultInterface $result)
{
return $result['ETag'];
Expand Down
14 changes: 14 additions & 0 deletions src/S3/MultipartUploadingTrait.php
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

trait MultipartUploadingTrait
{
private $uploadedBytes = 0;

/**
* Creates an UploadState object for a multipart upload by querying the
* service for the specified upload's information.
Expand Down Expand Up @@ -49,12 +51,24 @@ public static function getStateFromService(
return $state;
}

protected function createProgressThresholds($totalSize)
{
$this->progressThresholds = [];
for ($i=1;$i<=8;$i++) {
$this->progressThresholds []= round($totalSize*($i/8));
}
$this->getState()->setProgressThresholds($this->progressThresholds);
}

protected function handleResult(CommandInterface $command, ResultInterface $result)
{
$this->getState()->markPartAsUploaded($command['PartNumber'], [
'PartNumber' => $command['PartNumber'],
'ETag' => $this->extractETag($result),
]);

$this->uploadedBytes += $command["ContentLength"];
$this->getState()->displayProgress($this->uploadedBytes);
}

abstract protected function extractETag(ResultInterface $result);
Expand Down
8 changes: 2 additions & 6 deletions src/S3/ObjectUploader.php
Original file line number Diff line number Diff line change
Expand Up @@ -92,16 +92,12 @@ public function promise()
if (is_callable($this->options['before_upload'])) {
$this->options['before_upload']($command);
}
// return $this->client->executeAsync($command);
$test = $this->client->executeAsync($command);
return $test;

return $this->client->executeAsync($command);
}

public function upload()
{
$result = $this->promise()->wait();
return $result;
return $this->promise()->wait();
}

/**
Expand Down