From ae9ee517aa225dd0d845955ddf111bf4ddf72e02 Mon Sep 17 00:00:00 2001 From: Max Horn Date: Wed, 21 Feb 2024 21:47:21 +0100 Subject: [PATCH] Release scripts: fix download_with_sha256() (#5650) This function failed to freshly download the .sha256 checksum files since we switched the `curl` flags to include `-C` for continuing downloads. A simple fix would be to just always delete local copies of those files and thus force a fresh download. This goes one step further and uses the `requests` package to fetch the checksum directly into a string, bypassing the file system. --- dev/releases/utils.py | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/dev/releases/utils.py b/dev/releases/utils.py index 8814a36367..2ddbef83ce 100644 --- a/dev/releases/utils.py +++ b/dev/releases/utils.py @@ -16,6 +16,8 @@ import sys from typing import Iterator, List, NoReturn, Optional +import requests + # print notices in green def notice(msg: str) -> None: @@ -116,32 +118,26 @@ def download(url: str, dst: str) -> None: error("failed downloading " + url) -def file_matches_checksumfile(filename: str) -> bool: - with open(filename + ".sha256", "r", encoding="utf-8") as f: - expected_checksum = f.read().strip() - return expected_checksum == sha256file(filename) - - -def verify_via_checksumfile(filename: str) -> None: - actual_checksum = sha256file(filename) - with open(filename + ".sha256", "r", encoding="utf-8") as f: - expected_checksum = f.read().strip() - if expected_checksum != actual_checksum: - error( - f"checksum for '{filename}' expected to be {expected_checksum} but got {actual_checksum}" - ) - - # Download file at the given URL to path `dst`, unless we detect that a file # already exists at `dst` with the expected checksum. def download_with_sha256(url: str, dst: str) -> None: - download(url + ".sha256", dst + ".sha256") + # fetch the checksum file directly into memory + r = requests.get(url + ".sha256") + if r.status_code != 200: + error(f"Failed to download sha256 file {url}.sha256") + expected_checksum = r.text.strip() if os.path.isfile(dst): - if file_matches_checksumfile(dst): + actual_checksum = sha256file(dst) + if expected_checksum == actual_checksum: return notice(f"{dst} exists but does not match the checksumfile; redownloading") + os.remove(dst) download(url, dst) - verify_via_checksumfile(dst) + actual_checksum = sha256file(dst) + if expected_checksum != actual_checksum: + error( + f"checksum for '{dst}' expected to be {expected_checksum} but got {actual_checksum}" + ) # Run what ever command and create appropriate log file