diff --git a/aqt/archives.py b/aqt/archives.py index b0f385a..bc3de1c 100644 --- a/aqt/archives.py +++ b/aqt/archives.py @@ -48,7 +48,6 @@ class QtPackage: archive_url: str archive: str package_desc: str - hashurl: str pkg_update_name: str version: Optional[Version] = field(default=None) @@ -61,7 +60,7 @@ class QtPackage: return ( f"QtPackage(name={self.name}, url={self.archive_url}, " f"archive={self.archive}, desc={self.package_desc}" - f"hashurl={self.hashurl}{v_info})" + f"{v_info})" ) @@ -278,14 +277,12 @@ class QtArchives: # 5.15.0-0-202005140804qtbase-Linux-RHEL_7_6-GCC-Linux-RHEL_7_6-X86_64.7z full_version + archive, ) - hashurl = package_url + ".sha1" self.archives.append( QtPackage( name=archive_name, archive_url=package_url, archive=archive, package_desc=package_desc, - hashurl=hashurl, pkg_update_name=pkg_name, # For testing purposes ) ) @@ -480,14 +477,12 @@ class ToolArchives(QtArchives): # 4.1.1-202105261130ifw-linux-x64.7z f"{named_version}{archive}", ) - hashurl = package_url + ".sha1" self.archives.append( QtPackage( name=name, archive_url=package_url, archive=archive, package_desc=package_desc, - hashurl=hashurl, pkg_update_name=name, # Redundant ) ) diff --git a/aqt/exceptions.py b/aqt/exceptions.py index 8c46d25..5f1ce8c 100644 --- a/aqt/exceptions.py +++ b/aqt/exceptions.py @@ -47,6 +47,10 @@ class ArchiveChecksumError(ArchiveDownloadError): pass +class ChecksumDownloadFailure(ArchiveDownloadError): + pass + + class ArchiveConnectionError(AqtException): pass diff --git a/aqt/helper.py b/aqt/helper.py index 2b3c3c8..ad4772f 100644 --- a/aqt/helper.py +++ b/aqt/helper.py @@ -24,18 +24,25 @@ import hashlib import json import logging.config import os +import posixpath import sys import xml.etree.ElementTree as ElementTree from logging import getLogger from logging.handlers import QueueListener from pathlib import Path -from typing import Callable, Dict, List, Tuple +from typing import Callable, Dict, Generator, List, Tuple from urllib.parse import urlparse import requests import requests.adapters -from aqt.exceptions import ArchiveChecksumError, ArchiveConnectionError, ArchiveDownloadError, ArchiveListError +from aqt.exceptions import ( + ArchiveChecksumError, + ArchiveConnectionError, + ArchiveDownloadError, + ArchiveListError, + ChecksumDownloadFailure, +) def _get_meta(url: str): @@ -134,6 +141,32 @@ def retry_on_errors(action: Callable[[], any], acceptable_errors: Tuple, num_ret raise e from e +def iter_list_reps(_list: List, num_reps: int) -> Generator: + list_index = 0 + for i in range(num_reps): + yield _list[list_index] + list_index += 1 + if list_index >= len(_list): + list_index = 0 + + +def get_hash(archive_url: str, algorithm: str, timeout) -> str: + path = urlparse(archive_url).path + while path.startswith("/"): + path = path[1:] + for base_url in iter_list_reps(Settings.trusted_mirrors, Settings.max_retries_to_retrieve_hash): + url = posixpath.join(base_url, f"{path}.{algorithm}") + try: + r = getUrl(url, timeout) + # sha256 & md5 files are: "some_hash archive_filename" + return r.split(" ")[0] + except (ArchiveConnectionError, ArchiveDownloadError): + pass + raise ChecksumDownloadFailure( + f"Failed to download checksum for the file '{path.split('/')[-1]}' from mirrors '{Settings.trusted_mirrors}" + ) + + def altlink(url: str, alt: str): """ Blacklisting redirected(alt) location based on Settings.blacklist configuration. diff --git a/aqt/installer.py b/aqt/installer.py index 31bfc25..fd24fb2 100644 --- a/aqt/installer.py +++ b/aqt/installer.py @@ -51,7 +51,7 @@ from aqt.exceptions import ( CliKeyboardInterrupt, OutOfMemory, ) -from aqt.helper import MyQueueListener, Settings, downloadBinaryFile, getUrl, retry_on_errors, setup_logging +from aqt.helper import MyQueueListener, Settings, downloadBinaryFile, get_hash, retry_on_errors, setup_logging from aqt.metadata import ArchiveId, MetadataFactory, QtRepoProperty, SimpleSpec, Version, show_list, suggested_follow_up from aqt.updater import Updater @@ -1002,7 +1002,6 @@ def installer( """ name = qt_archive.name url = qt_archive.archive_url - hashurl = qt_archive.hashurl archive: Path = archive_dest / qt_archive.archive start_time = time.perf_counter() # set defaults @@ -1021,9 +1020,9 @@ def installer( timeout = (Settings.connection_timeout, Settings.response_timeout) else: timeout = (Settings.connection_timeout, response_timeout) - hash = binascii.unhexlify(getUrl(hashurl, timeout)) + hash = binascii.unhexlify(get_hash(url, algorithm="sha256", timeout=timeout)) retry_on_errors( - action=lambda: downloadBinaryFile(url, archive, "sha1", hash, timeout), + action=lambda: downloadBinaryFile(url, archive, "sha256", hash, timeout), acceptable_errors=(ArchiveChecksumError,), num_retries=Settings.max_retries_on_checksum_error, name=f"Downloading {name}", diff --git a/tests/test_archives.py b/tests/test_archives.py index ea4add5..7ee1155 100644 --- a/tests/test_archives.py +++ b/tests/test_archives.py @@ -139,7 +139,6 @@ def test_qt_archives_modules(monkeypatch, arch, requested_module_names, has_none assert url_match mod_name, archive_name = url_match.groups() assert pkg.archive == archive_name - assert pkg.hashurl == pkg.archive_url + ".sha1" assert archive_name in expected_7z_files expected_7z_files.remove(archive_name) assert len(expected_7z_files) == 0, "Actual number of packages was fewer than expected" @@ -227,7 +226,6 @@ def test_tools_variants(monkeypatch, tool_name, tool_variant_name, is_expect_fai actual_variant_name, archive_name = url_match.groups() assert actual_variant_name == tool_variant_name assert pkg.archive == archive_name - assert pkg.hashurl == pkg.archive_url + ".sha1" assert archive_name in expected_7z_files expected_7z_files.remove(archive_name) assert len(expected_7z_files) == 0, f"Failed to produce QtPackages for {expected_7z_files}" diff --git a/tests/test_helper.py b/tests/test_helper.py index bc27068..c74ae24 100644 --- a/tests/test_helper.py +++ b/tests/test_helper.py @@ -9,8 +9,8 @@ import requests from requests.models import Response from aqt import helper -from aqt.exceptions import ArchiveChecksumError, ArchiveConnectionError, ArchiveDownloadError -from aqt.helper import getUrl, retry_on_errors +from aqt.exceptions import ArchiveChecksumError, ArchiveConnectionError, ArchiveDownloadError, ChecksumDownloadFailure +from aqt.helper import Settings, get_hash, getUrl, retry_on_errors from aqt.metadata import Version @@ -183,6 +183,41 @@ def test_helper_retry_on_error(num_attempts_before_success, num_retries_allowed) assert retry_on_errors(action, (RuntimeError,), num_retries_allowed, "do something") +@pytest.mark.parametrize( + "num_tries_required, num_retries_allowed", + ( + (2, 5), + (5, 5), + (6, 5), + ), +) +def test_helper_get_hash_retries(monkeypatch, num_tries_required, num_retries_allowed): + num_tries = 0 + + def mock_getUrl(url, *args, **kwargs): + nonlocal num_tries + num_tries += 1 + if num_tries < num_tries_required: + raise ArchiveConnectionError(f"Must retry {num_tries_required - num_tries} more times before success") + parsed = urlparse(url) + base = f"{parsed.scheme}://{parsed.netloc}" + assert base in Settings.trusted_mirrors + + hash_filename = str(parsed.path.split("/")[-1]) + assert hash_filename == "archive.7z.sha256" + return "MOCK_HASH archive.7z" + + monkeypatch.setattr("aqt.helper.getUrl", mock_getUrl) + + if num_tries_required > num_retries_allowed: + with pytest.raises(ChecksumDownloadFailure) as e: + result = get_hash("http://insecure.mirror.com/some/path/to/archive.7z", "sha256", (5, 5)) + assert e.type == ChecksumDownloadFailure + else: + result = get_hash("http://insecure.mirror.com/some/path/to/archive.7z", "sha256", (5, 5)) + assert result == "MOCK_HASH" + + @pytest.mark.parametrize( "version, expect", [ diff --git a/tests/test_install.py b/tests/test_install.py index 6c2ebf7..9da6416 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -137,7 +137,7 @@ def make_mock_geturl_download_archive( def mock_getUrl(url: str, *args) -> str: if url.endswith(updates_url): return "\n{}\n".format("\n".join([archive.xml_package_update() for archive in archives])) - elif url.endswith(".sha1"): + elif url.endswith(".sha256"): return "" # Skip the checksum assert False @@ -598,7 +598,7 @@ def test_install( mock_get_url, mock_download_archive = make_mock_geturl_download_archive(archives, arch, host, updates_url) monkeypatch.setattr("aqt.archives.getUrl", mock_get_url) - monkeypatch.setattr("aqt.installer.getUrl", mock_get_url) + monkeypatch.setattr("aqt.helper.getUrl", mock_get_url) monkeypatch.setattr("aqt.installer.downloadBinaryFile", mock_download_archive) with TemporaryDirectory() as output_dir: @@ -713,7 +713,7 @@ def test_install_nonexistent_archives(monkeypatch, capsys, cmd, xml_file: Option return (Path(__file__).parent / "data" / xml_file).read_text("utf-8") monkeypatch.setattr("aqt.archives.getUrl", mock_get_url) - monkeypatch.setattr("aqt.installer.getUrl", mock_get_url) + monkeypatch.setattr("aqt.helper.getUrl", mock_get_url) monkeypatch.setattr("aqt.metadata.getUrl", mock_get_url) cli = Cli() @@ -779,7 +779,7 @@ def test_install_pool_exception(monkeypatch, capsys, make_exception, settings_fi cmd = ["install-qt", host, target, ver, arch] mock_get_url, mock_download_archive = make_mock_geturl_download_archive(archives, arch, host, updates_url) monkeypatch.setattr("aqt.archives.getUrl", mock_get_url) - monkeypatch.setattr("aqt.installer.getUrl", mock_get_url) + monkeypatch.setattr("aqt.helper.getUrl", mock_get_url) monkeypatch.setattr("aqt.installer.installer", mock_installer_func) Settings.load_settings(str(Path(__file__).parent / settings_file)) @@ -793,7 +793,7 @@ def test_install_installer_archive_extraction_err(monkeypatch): def mock_extractor_that_fails(*args, **kwargs): raise subprocess.CalledProcessError(returncode=1, cmd="some command", output="out", stderr="err") - monkeypatch.setattr("aqt.installer.getUrl", lambda *args: "") + monkeypatch.setattr("aqt.installer.get_hash", lambda *args, **kwargs: "") monkeypatch.setattr("aqt.installer.downloadBinaryFile", lambda *args: None) monkeypatch.setattr("aqt.installer.subprocess.run", mock_extractor_that_fails) @@ -804,7 +804,6 @@ def test_install_installer_archive_extraction_err(monkeypatch): "archive-url", "archive", "package_desc", - "hashurl", "pkg_update_name", ), base_dir=temp_dir,