Commit b12e775e authored by Dorota Czaplejewicz's avatar Dorota Czaplejewicz

devkit-flash: Allow for configureable number of retries

The main purpose of this is to allow infinite retries.
The discussion on the community channel revealed that people will have connectivity problems. This solution is not perfect for unreliable connections, but it's better. Perhaps the ideal solution would be to leave the file between runs and let the user control when to download (data caps, offline periods, etc).
parent 24725ef5
......@@ -3,6 +3,7 @@
import argparse
import datetime
import hashlib
import itertools
import jenkins
import logging
import lzma
......@@ -52,7 +53,6 @@ FB: Done
'''
BLOCK_SIZE = 8192
UNCOMPRESSED_SIZE = 3600000000
MAX_ATTEMPTS = 10
class VerifyImageException(Exception):
......@@ -86,13 +86,20 @@ def verify_image(image, meta):
"does not match {}".format(m.hexdigest(), hexdigest))
def resuming_stream(url, expected_size):
def resuming_stream(url, expected_size, max_attempts):
position = 0
for i in range(MAX_ATTEMPTS):
if max_attempts < 1:
retries = itertools.count()
else:
retries = range(max_attempts)
for i in retries:
try:
resp = requests.get(url,
stream=True,
headers={'Range': 'bytes={}-'.format(position)})
headers={'Range': 'bytes={}-'.format(position)},
proxies={'https': '[::1]:8080'})
resp.raise_for_status()
if resp.status_code != requests.codes.partial_content:
......@@ -107,22 +114,21 @@ def resuming_stream(url, expected_size):
raise PrematureEndException()
return
except (requests.exceptions.ConnectionError, PrematureEndException):
if i < MAX_ATTEMPTS - 1:
logging.info("Connection error, retrying")
time.sleep(5)
else:
if i == max_attempts - 1:
logging.error("Max connection errors reached, aborting")
raise
logging.info("Connection error, retrying")
time.sleep(5)
def stream_file(url):
def stream_file(url, attempts):
resp = requests.head(url, stream=True)
resp.raise_for_status()
ts = int(resp.headers.get('content-length', 0))
return resuming_stream(url, ts), ts
return resuming_stream(url, ts, attempts), ts
def download_image(url, target):
def download_image(url, target, attempts):
decomp = lzma.LZMADecompressor()
logging.info("Downloading image from {}".format(url))
......@@ -140,7 +146,7 @@ def download_image(url, target):
meta = None
uncompressed_size = UNCOMPRESSED_SIZE
stream, ts = stream_file(url)
stream, ts = stream_file(url, attempts)
download_bar = tqdm.tqdm(total=ts,
desc='Download',
leave=False)
......@@ -225,6 +231,9 @@ def main():
help="Download an image for this distribution, default is '{}'".format(DIST))
parser.add_argument('--skip-cleanup', action='store_true', default=False,
help='Skip temporary directory cleanup')
parser.add_argument('--download-attempts', type=int, default=10,
help="Maximum number of attempts to resume "
"devkit image download. 0-unlimited")
group = parser.add_argument_group(title='Testing and debugging options')
group.add_argument('--debug', action="store_true", default=False,
......@@ -270,7 +279,7 @@ def main():
uuu_target = os.path.join(outdir, UUU_SCRIPT)
download_image(urljoin(image_ref['url'], 'artifact/{}.xz').format(IMAGE),
image_target)
image_target, args.download_attempts)
download_uboot(urljoin(uboot_ref['url'], 'artifact/build/{}'.format(UBOOT)),
uboot_target)
write_uuu_script(uuu_target, image_target, uboot_target)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment