fix(@aws-amplify/storage): fix s3 multipart upload for very large fil…

…es (aws-amplify#6509)
abdielou · Aug 6, 2020 · 621966e · 621966e
1 parent 7d17aee
commit 621966e
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 12 deletions.
diff --git a/packages/storage/__tests__/providers/AWSS3ProviderManagedUpload-unit-test.ts b/packages/storage/__tests__/providers/AWSS3ProviderManagedUpload-unit-test.ts
@@ -14,6 +14,7 @@ import {
 	AWSS3ProviderManagedUpload,
 	BodyPart,
 } from '../../src/providers/AWSS3ProviderManagedUpload';
+import { Credentials } from '@aws-amplify/core';
 import {
 	S3Client,
 	PutObjectCommand,
@@ -136,7 +137,7 @@ describe('multi part upload tests', () => {
 			protected minPartSize = testMinPartSize;
 			protected async uploadParts(uploadId: string, parts: BodyPart[]) {
 				// Make service calls and set the event listeners first from the base impl
-				super.uploadParts(uploadId, parts);
+				await super.uploadParts(uploadId, parts);
 				// Now trigger some notifications from the event listeners
 				for (const part of parts) {
 					part.emitter.emit('sendProgress', {

diff --git a/packages/storage/src/providers/AWSS3ProviderManagedUpload.ts b/packages/storage/src/providers/AWSS3ProviderManagedUpload.ts
@@ -15,6 +15,7 @@ import {
 	ConsoleLogger as Logger,
 	getAmplifyUserAgent,
 	Platform,
+	Credentials,
 } from '@aws-amplify/core';
 import {
 	S3Client,
@@ -76,7 +77,7 @@ export class AWSS3ProviderManagedUpload {
 			// Multipart upload is not required. Upload the sanitized body as is
 			this.params.Body = this.body;
 			const putObjectCommand = new PutObjectCommand(this.params);
-			const s3 = this._createNewS3Client(this.opts, this.emitter);
+			const s3 = await this._createNewS3Client(this.opts, this.emitter);
 			return s3.send(putObjectCommand);
 		} else {
 			// Step 1: Initiate the multi part upload
@@ -138,7 +139,7 @@ export class AWSS3ProviderManagedUpload {
 		const createMultiPartUploadCommand = new CreateMultipartUploadCommand(
 			this.params
 		);
-		const s3 = this._createNewS3Client(this.opts);
+		const s3 = await this._createNewS3Client(this.opts);
 		const response = await s3.send(createMultiPartUploadCommand);
 		logger.debug(response.UploadId);
 		return response.UploadId;
@@ -160,7 +161,7 @@ export class AWSS3ProviderManagedUpload {
 				Bucket: this.params.Bucket,
 			};
 			const uploadPartCommand = new UploadPartCommand(uploadPartCommandInput);
-			const s3 = this._createNewS3Client(this.opts, part.emitter);
+			const s3 = await this._createNewS3Client(this.opts, part.emitter);
 			promises.push(s3.send(uploadPartCommand));
 		}
 		try {
@@ -192,7 +193,7 @@ export class AWSS3ProviderManagedUpload {
 			MultipartUpload: { Parts: this.multiPartMap },
 		};
 		const completeUploadCommand = new CompleteMultipartUploadCommand(input);
-		const s3 = this._createNewS3Client(this.opts);
+		const s3 = await this._createNewS3Client(this.opts);
 		try {
 			const data = await s3.send(completeUploadCommand);
 			return data.Key;
@@ -235,7 +236,7 @@ export class AWSS3ProviderManagedUpload {
 			UploadId: uploadId,
 		};
 
-		const s3 = this._createNewS3Client(this.opts);
+		const s3 = await this._createNewS3Client(this.opts);
 		await s3.send(new AbortMultipartUploadCommand(input));
 
 		// verify that all parts are removed.
@@ -329,12 +330,9 @@ export class AWSS3ProviderManagedUpload {
 	 * @private
 	 * creates an S3 client with new V3 aws sdk
 	 */
-	protected _createNewS3Client(config, emitter?) {
-		const {
-			region,
-			credentials,
-			dangerouslyConnectToHttpEndpointForTesting,
-		} = config;
+	protected async _createNewS3Client(config, emitter?) {
+		const credentials = await this._getCredentials();
+		const { region, dangerouslyConnectToHttpEndpointForTesting } = config;
 		let localTestingConfig = {};
 
 		if (dangerouslyConnectToHttpEndpointForTesting) {
@@ -357,4 +355,21 @@ export class AWSS3ProviderManagedUpload {
 		client.middlewareStack.remove(SET_CONTENT_LENGTH_HEADER);
 		return client;
 	}
+
+	/**
+	 * @private
+	 */
+	_getCredentials() {
+		return Credentials.get()
+			.then(credentials => {
+				if (!credentials) return false;
+				const cred = Credentials.shear(credentials);
+				logger.debug('set credentials for storage', cred);
+				return cred;
+			})
+			.catch(error => {
+				logger.warn('ensure credentials error', error);
+				return false;
+			});
+	}
 }