From a719d3dcf0c2742cc7058668d5a830633c5fc126 Mon Sep 17 00:00:00 2001 From: Mike Stipicevic Date: Mon, 28 Nov 2016 13:24:16 -0800 Subject: [PATCH] Retry up to three times on upload timeout. BUG=668526 Change-Id: Id49f1c1394a72880410eb06ba7356307fff7e8c5 Reviewed-on: https://chromium-review.googlesource.com/414496 Commit-Queue: Mike Stipicevic Reviewed-by: Marc-Antoine Ruel --- third_party/upload.py | 57 +++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/third_party/upload.py b/third_party/upload.py index d56c2b518..94935bf40 100755 --- a/third_party/upload.py +++ b/third_party/upload.py @@ -133,6 +133,7 @@ VCS_ABBREVIATIONS.update((alias, vcs['name']) for alias in vcs['aliases']) UPLOAD_TIMEOUT = 120 +MAX_UPLOAD_ATTEMPTS = 3 # The result of parsing Subversion's [auto-props] setting. @@ -1018,27 +1019,41 @@ def UploadFile(filename, file_id, content, is_binary, status, is_base): patches = dict() [patches.setdefault(v, k) for k, v in patch_list] - threads = [] - thread_pool = ThreadPool(options.num_upload_threads) - - for filename in patches.keys(): - base_content, new_content, is_binary, status = files[filename] - file_id_str = patches.get(filename) - if file_id_str.find("nobase") != -1: - base_content = None - file_id_str = file_id_str[file_id_str.rfind("_") + 1:] - file_id = int(file_id_str) - if base_content != None: - t = thread_pool.apply_async(UploadFile, args=(filename, - file_id, base_content, is_binary, status, True)) - threads.append(t) - if new_content != None: - t = thread_pool.apply_async(UploadFile, args=(filename, - file_id, new_content, is_binary, status, False)) - threads.append(t) - - for t in threads: - print(t.get(timeout=UPLOAD_TIMEOUT)) + def uploadAttempt(): + threads = [] + thread_pool = ThreadPool(options.num_upload_threads) + + for filename in patches.keys(): + base_content, new_content, is_binary, status = files[filename] + file_id_str = patches.get(filename) + if file_id_str.find("nobase") != -1: + base_content = None + file_id_str = file_id_str[file_id_str.rfind("_") + 1:] + file_id = int(file_id_str) + if base_content != None: + t = thread_pool.apply_async(UploadFile, args=(filename, + file_id, base_content, is_binary, status, True)) + threads.append(t) + if new_content != None: + t = thread_pool.apply_async(UploadFile, args=(filename, + file_id, new_content, is_binary, status, False)) + threads.append(t) + + for t in threads: + print(t.get(timeout=UPLOAD_TIMEOUT)) + + success = False + for _ in range(MAX_UPLOAD_ATTEMPTS): + try: + uploadAttempt() + success = True + break + except multiprocessing.TimeoutError: + LOGGER.warning('Timeout error while uploading, retrying...') + + if not success: + raise IOError( + '%d consecutive timeout errors, aborting!' % MAX_UPLOAD_ATTEMPTS) def IsImage(self, filename): """Returns true if the filename has an image extension."""