Approach: use S3 range GETs to stream the object in fixed-size chunks, write to a temporary file at the right offset (so writes are atomic when we rename), compute SHA-256 incrementally (resuming by hashing existing bytes), and retry on transient errors with exponential backoff. Use boto3/botocore for S3 requests.python
import os
import hashlib
import time
import tempfile
import boto3
from botocore.exceptions import BotoCoreError, ClientError
CHUNK = 8 * 1024 * 1024 # 8 MB per request
def download_s3_with_resume(bucket, key, dest_path, expected_sha256, max_retries=5):
s3 = boto3.client('s3')
# 1) get object size
meta = s3.head_object(Bucket=bucket, Key=key)
total = meta['ContentLength']
tmp_path = dest_path + '.part'
# 2) determine starting offset (resume if tmp exists)
offset = 0
sha = hashlib.sha256()
if os.path.exists(tmp_path):
offset = os.path.getsize(tmp_path)
# re-hash existing bytes
with open(tmp_path, 'rb') as f:
while True:
b = f.read(CHUNK)
if not b: break
sha.update(b)
# 3) open file for append+binary and perform ranged GETs
with open(tmp_path, 'ab') as f:
while offset < total:
end = min(total - 1, offset + CHUNK - 1)
attempt = 0
while attempt <= max_retries:
try:
# Range header: bytes=start-end
resp = s3.get_object(Bucket=bucket, Key=key, Range=f'bytes={offset}-{end}')
stream = resp['Body']
for chunk in iter(lambda: stream.read(1024*1024), b''):
if not chunk: break
f.write(chunk)
sha.update(chunk)
offset += len(chunk)
stream.close()
break # success, break retry loop
except (BotoCoreError, ClientError, ConnectionError) as e:
attempt += 1
if attempt > max_retries:
raise
backoff = 2 ** attempt
time.sleep(backoff)
# flush to disk to ensure durability
f.flush()
os.fsync(f.fileno())
# 4) verify size and checksum
if offset != total:
raise IOError(f"Download incomplete: got {offset} of {total}")
digest = sha.hexdigest()
if digest != expected_sha256.lower():
raise ValueError(f"SHA256 mismatch: {digest} != {expected_sha256}")
# 5) atomic move to final destination
os.replace(tmp_path, dest_path)
return dest_path
