From 3ab25201909bece9066ac6191670bcee09791d54 Mon Sep 17 00:00:00 2001 From: Ben Westgate Date: Wed, 22 May 2024 22:37:31 -0500 Subject: [PATCH] contrib: Fixup verify-binaries OS platform parsing Parse platform strings with "-" or '.' correctly such as "linux-gnu" or "x86_64-linux-gnu.tar.gz" to download the matching files or file. String partition() is used to tolerate more dashes. Update `VERSION_EXAMPLE` with a new string parsed correctly now. Fix "-aarch64" interpreted as a release candidate due to sub-string "rc", causing all downloads to fail. Now "rc" must immediately follow first "-" to indicate an [-rc] string. Local variables `version_rc`, `version_os` renamed to `rc`, `platform`. If "-rcN" is specified, `platform` is reassigned to remove the '-rcN'. Changes are useful to only download one bitcoin core binary on slow connections. Making `verify.py pub` more intuitive, robust, and versatile. Closes #30145 When user types a platform string not found in any filename lets help and say the platform closest to what they typed in a `f"No files matched the platform specified. Did you mean: {closest_match}"` log. Improves UX when unaware how we name our files. Uses the difflib Python built-in which was already imported elsewhere. Update test.py to test single file verification verify-binaries/verify.py can accept an entire filename filter for its "-platform" parameter now so let us test that it succeeds and downloads and verifies only one file. `verify.py pub 22.0-x86_64-linux-gnu.tar.gz` should get and verify only the requested binary. It is placed before the existing wide verification as it is a faster test and possibly easier to break. Update doc with examples now possible after bugfix Add example to show release candidates now work with "-platform" strings containing "-" and string provided can be from the middle of filename: `./contrib/verify-binaries/verify.py --json pub 23.0-rc5-linux-gnu` Change example 5 to not match example 3. New examples to show platform can now be provided specifically enough to download only a single binary down to its file extension: `./contrib/verify-binaries/verify.py pub 25.2-x86_64-linux` `./contrib/verify-binaries/verify.py pub 24.1-rc1-darwin` `./contrib/verify-binaries/verify.py pub 27.0-win64-setup.exe` This is the most common use if not verifying all files so users see it as the first example for "only download the binaries for a certain architecture and/or platform". Downloading one file is intuitively what most will think this meant and this change delivers on that expectation. Co-authored-by: stickies-v --- contrib/verify-binaries/README.md | 10 ++++++---- contrib/verify-binaries/test.py | 15 +++++++++++++++ contrib/verify-binaries/verify.py | 26 +++++++++++--------------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/contrib/verify-binaries/README.md b/contrib/verify-binaries/README.md index 04d683e69b5..0f3e16a5bc9 100644 --- a/contrib/verify-binaries/README.md +++ b/contrib/verify-binaries/README.md @@ -50,6 +50,7 @@ Get JSON output and don't prompt for user input (no auto key import): ```sh ./contrib/verify-binaries/verify.py --json pub 22.0-x86 +./contrib/verify-binaries/verify.py --json pub 23.0-rc5-linux-gnu ``` Rely only on local GPG state and manually specified keys, while requiring a @@ -57,14 +58,15 @@ threshold of at least 10 trusted signatures: ```sh ./contrib/verify-binaries/verify.py \ --trusted-keys 74E2DEF5D77260B98BC19438099BAD163C70FBFA,9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C \ - --min-good-sigs 10 pub 22.0-x86 + --min-good-sigs 10 pub 22.0-linux ``` -If you only want to download the binaries for a certain platform, add the corresponding suffix, e.g.: +If you only want to download the binaries for a certain architecture and/or platform, add the corresponding suffix, e.g.: ```sh -./contrib/verify-binaries/verify.py pub 24.0.1-darwin -./contrib/verify-binaries/verify.py pub 23.1-rc1-win64 +./contrib/verify-binaries/verify.py pub 25.2-x86_64-linux +./contrib/verify-binaries/verify.py pub 24.1-rc1-darwin +./contrib/verify-binaries/verify.py pub 27.0-win64-setup.exe ``` If you do not want to keep the downloaded binaries, specify the cleanup option. diff --git a/contrib/verify-binaries/test.py b/contrib/verify-binaries/test.py index 22d718ece33..875606ec226 100755 --- a/contrib/verify-binaries/test.py +++ b/contrib/verify-binaries/test.py @@ -12,6 +12,21 @@ def main(): expect_code(run_verify("", "pub", '0.32.awefa.12f9h'), 11, "Malformed version should fail") expect_code(run_verify('--min-good-sigs 20', "pub", "22.0"), 9, "--min-good-sigs 20 should fail") + print("- testing verification (22.0-x86_64-linux-gnu.tar.gz)", flush=True) + _220_x86_64_linux_gnu = run_verify("--json", "pub", "22.0-x86_64-linux-gnu.tar.gz") + try: + result = json.loads(_220_x86_64_linux_gnu.stdout.decode()) + except Exception: + print("failed on 22.0-x86_64-linux-gnu.tar.gz --json:") + print_process_failure(_220_x86_64_linux_gnu) + raise + + expect_code(_220_x86_64_linux_gnu, 0, "22.0-x86_64-linux-gnu.tar.gz should succeed") + v = result['verified_binaries'] + assert result['good_trusted_sigs'] + assert len(v) == 1 + assert v['bitcoin-22.0-x86_64-linux-gnu.tar.gz'] == '59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16' + print("- testing verification (22.0)", flush=True) _220 = run_verify("--json", "pub", "22.0") try: diff --git a/contrib/verify-binaries/verify.py b/contrib/verify-binaries/verify.py index 12e6e10d8a4..6c07b36c9d8 100755 --- a/contrib/verify-binaries/verify.py +++ b/contrib/verify-binaries/verify.py @@ -97,23 +97,17 @@ def bool_from_env(key, default=False) -> bool: VERSION_FORMAT = ".[.][-rc[0-9]][-platform]" -VERSION_EXAMPLE = "22.0-x86_64 or 23.1-rc1-darwin" +VERSION_EXAMPLE = "22.0 or 23.1-rc1-darwin.dmg or 27.0-x86_64-linux-gnu" def parse_version_string(version_str): - parts = version_str.split('-') - version_base = parts[0] - version_rc = "" - version_os = "" - if len(parts) == 2: # "-rcN" or "version-platform" - if "rc" in parts[1]: - version_rc = parts[1] - else: - version_os = parts[1] - elif len(parts) == 3: # "-rcN-platform" - version_rc = parts[1] - version_os = parts[2] + # "[-rcN][-platform]" + version_base, _, platform = version_str.partition('-') + rc = "" + if platform.startswith("rc"): # "-rcN[-platform]" + rc, _, platform = platform.partition('-') + # else "" or "-platform" - return version_base, version_rc, version_os + return version_base, rc, platform def download_with_wget(remote_file, local_file): @@ -514,7 +508,9 @@ def verify_published_handler(args: argparse.Namespace) -> ReturnCode: # Extract hashes and filenames hashes_to_verify = parse_sums_file(SUMS_FILENAME, [os_filter]) if not hashes_to_verify: - log.error("no files matched the platform specified") + available_versions = ["-".join(line[1].split("-")[2:]) for line in parse_sums_file(SUMS_FILENAME, [])] + closest_match = difflib.get_close_matches(os_filter, available_versions, cutoff=0, n=1)[0] + log.error(f"No files matched the platform specified. Did you mean: {closest_match}") return ReturnCode.NO_BINARIES_MATCH # remove binaries that are known not to be hosted by bitcoincore.org