Commit ef4a109f authored by Guido Gunther's avatar Guido Gunther

Merge branch 'resume' into 'master'

Devkit: Resume image download on errors

See merge request !22
parents ff65b589 b12e775e
Pipeline #3786 passed with stage
in 2 minutes and 3 seconds
......@@ -3,6 +3,7 @@
import argparse
import datetime
import hashlib
import itertools
import jenkins
import logging
import lzma
......@@ -12,6 +13,7 @@ import shutil
import subprocess
import sys
import tempfile
import time
import tqdm
import yaml
......@@ -57,6 +59,10 @@ class VerifyImageException(Exception):
pass
class PrematureEndException(Exception):
pass
def verify_image(image, meta):
m = hashlib.sha256()
size = int(meta['image']['size'])
......@@ -80,7 +86,49 @@ def verify_image(image, meta):
"does not match {}".format(m.hexdigest(), hexdigest))
def download_image(url, target):
def resuming_stream(url, expected_size, max_attempts):
position = 0
if max_attempts < 1:
retries = itertools.count()
else:
retries = range(max_attempts)
for i in retries:
try:
resp = requests.get(url,
stream=True,
headers={'Range': 'bytes={}-'.format(position)},
proxies={'https': '[::1]:8080'})
resp.raise_for_status()
if resp.status_code != requests.codes.partial_content:
position = 0
logging.info('Proceeding from {} bytes'.format(position))
for data in resp.iter_content(BLOCK_SIZE):
position += len(data)
yield data
if position < expected_size:
raise PrematureEndException()
return
except (requests.exceptions.ConnectionError, PrematureEndException):
if i == max_attempts - 1:
logging.error("Max connection errors reached, aborting")
raise
logging.info("Connection error, retrying")
time.sleep(5)
def stream_file(url, attempts):
resp = requests.head(url, stream=True)
resp.raise_for_status()
ts = int(resp.headers.get('content-length', 0))
return resuming_stream(url, ts, attempts), ts
def download_image(url, target, attempts):
decomp = lzma.LZMADecompressor()
logging.info("Downloading image from {}".format(url))
......@@ -98,9 +146,7 @@ def download_image(url, target):
meta = None
uncompressed_size = UNCOMPRESSED_SIZE
resp = requests.get(url, stream=True)
resp.raise_for_status()
ts = int(resp.headers.get('content-length', 0))
stream, ts = stream_file(url, attempts)
download_bar = tqdm.tqdm(total=ts,
desc='Download',
leave=False)
......@@ -108,7 +154,7 @@ def download_image(url, target):
desc='Decompr.',
leave=False)
with open(target, 'wb+') as f:
for data in resp.iter_content(BLOCK_SIZE):
for data in stream:
if data:
out = decomp.decompress(data)
decompress_bar.update(len(out))
......@@ -185,6 +231,9 @@ def main():
help="Download an image for this distribution, default is '{}'".format(DIST))
parser.add_argument('--skip-cleanup', action='store_true', default=False,
help='Skip temporary directory cleanup')
parser.add_argument('--download-attempts', type=int, default=10,
help="Maximum number of attempts to resume "
"devkit image download. 0-unlimited")
group = parser.add_argument_group(title='Testing and debugging options')
group.add_argument('--debug', action="store_true", default=False,
......@@ -230,7 +279,7 @@ def main():
uuu_target = os.path.join(outdir, UUU_SCRIPT)
download_image(urljoin(image_ref['url'], 'artifact/{}.xz').format(IMAGE),
image_target)
image_target, args.download_attempts)
download_uboot(urljoin(uboot_ref['url'], 'artifact/build/{}'.format(UBOOT)),
uboot_target)
write_uuu_script(uuu_target, image_target, uboot_target)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment