Use sha256 hashes only from trusted mirrors

To keep this commit small, `hashurl` was removed from QtPackage, and
`get_hash` constructs the hash url based on the url of the 7z archive
to download. I think that in the future, QtArchive and QtPackage could
be refactored to construct this url more appropriately. However, this
would be a complicated change that doesn't belong in this commit.

aqt/archives.py

@@ -48,7 +48,6 @@ class QtPackage:
     archive_url: str
     archive: str
     package_desc: str
-    hashurl: str
     pkg_update_name: str
     version: Optional[Version] = field(default=None)
 
@@ -61,7 +60,7 @@ class QtPackage:
         return (
             f"QtPackage(name={self.name}, url={self.archive_url}, "
             f"archive={self.archive}, desc={self.package_desc}"
-            f"hashurl={self.hashurl}{v_info})"
+            f"{v_info})"
         )
 
 
@@ -278,14 +277,12 @@ class QtArchives:
                     # 5.15.0-0-202005140804qtbase-Linux-RHEL_7_6-GCC-Linux-RHEL_7_6-X86_64.7z
                     full_version + archive,
                 )
-                hashurl = package_url + ".sha1"
                 self.archives.append(
                     QtPackage(
                         name=archive_name,
                         archive_url=package_url,
                         archive=archive,
                         package_desc=package_desc,
-                        hashurl=hashurl,
                         pkg_update_name=pkg_name,  # For testing purposes
                     )
                 )
@@ -480,14 +477,12 @@ class ToolArchives(QtArchives):
                 #  4.1.1-202105261130ifw-linux-x64.7z
                 f"{named_version}{archive}",
             )
-            hashurl = package_url + ".sha1"
             self.archives.append(
                 QtPackage(
                     name=name,
                     archive_url=package_url,
                     archive=archive,
                     package_desc=package_desc,
-                    hashurl=hashurl,
                     pkg_update_name=name,  # Redundant
                 )
             )

aqt/exceptions.py

@@ -47,6 +47,10 @@ class ArchiveChecksumError(ArchiveDownloadError):
     pass
 
 
+class ChecksumDownloadFailure(ArchiveDownloadError):
+    pass
+
+
 class ArchiveConnectionError(AqtException):
     pass
 

aqt/helper.py

@@ -24,18 +24,25 @@ import hashlib
 import json
 import logging.config
 import os
+import posixpath
 import sys
 import xml.etree.ElementTree as ElementTree
 from logging import getLogger
 from logging.handlers import QueueListener
 from pathlib import Path
-from typing import Callable, Dict, List, Tuple
+from typing import Callable, Dict, Generator, List, Tuple
 from urllib.parse import urlparse
 
 import requests
 import requests.adapters
 
-from aqt.exceptions import ArchiveChecksumError, ArchiveConnectionError, ArchiveDownloadError, ArchiveListError
+from aqt.exceptions import (
+    ArchiveChecksumError,
+    ArchiveConnectionError,
+    ArchiveDownloadError,
+    ArchiveListError,
+    ChecksumDownloadFailure,
+)
 
 
 def _get_meta(url: str):
@@ -134,6 +141,32 @@ def retry_on_errors(action: Callable[[], any], acceptable_errors: Tuple, num_ret
             raise e from e
 
 
+def iter_list_reps(_list: List, num_reps: int) -> Generator:
+    list_index = 0
+    for i in range(num_reps):
+        yield _list[list_index]
+        list_index += 1
+        if list_index >= len(_list):
+            list_index = 0
+
+
+def get_hash(archive_url: str, algorithm: str, timeout) -> str:
+    path = urlparse(archive_url).path
+    while path.startswith("/"):
+        path = path[1:]
+    for base_url in iter_list_reps(Settings.trusted_mirrors, Settings.max_retries_to_retrieve_hash):
+        url = posixpath.join(base_url, f"{path}.{algorithm}")
+        try:
+            r = getUrl(url, timeout)
+            # sha256 & md5 files are: "some_hash archive_filename"
+            return r.split(" ")[0]
+        except (ArchiveConnectionError, ArchiveDownloadError):
+            pass
+    raise ChecksumDownloadFailure(
+        f"Failed to download checksum for the file '{path.split('/')[-1]}' from mirrors '{Settings.trusted_mirrors}"
+    )
+
+
 def altlink(url: str, alt: str):
     """
     Blacklisting redirected(alt) location based on Settings.blacklist configuration.

aqt/installer.py

@@ -51,7 +51,7 @@ from aqt.exceptions import (
     CliKeyboardInterrupt,
     OutOfMemory,
 )
-from aqt.helper import MyQueueListener, Settings, downloadBinaryFile, getUrl, retry_on_errors, setup_logging
+from aqt.helper import MyQueueListener, Settings, downloadBinaryFile, get_hash, retry_on_errors, setup_logging
 from aqt.metadata import ArchiveId, MetadataFactory, QtRepoProperty, SimpleSpec, Version, show_list, suggested_follow_up
 from aqt.updater import Updater
 
@@ -1002,7 +1002,6 @@ def installer(
     """
     name = qt_archive.name
     url = qt_archive.archive_url
-    hashurl = qt_archive.hashurl
     archive: Path = archive_dest / qt_archive.archive
     start_time = time.perf_counter()
     # set defaults
@@ -1021,9 +1020,9 @@ def installer(
         timeout = (Settings.connection_timeout, Settings.response_timeout)
     else:
         timeout = (Settings.connection_timeout, response_timeout)
-    hash = binascii.unhexlify(getUrl(hashurl, timeout))
+    hash = binascii.unhexlify(get_hash(url, algorithm="sha256", timeout=timeout))
     retry_on_errors(
-        action=lambda: downloadBinaryFile(url, archive, "sha1", hash, timeout),
+        action=lambda: downloadBinaryFile(url, archive, "sha256", hash, timeout),
         acceptable_errors=(ArchiveChecksumError,),
         num_retries=Settings.max_retries_on_checksum_error,
         name=f"Downloading {name}",

tests/test_archives.py

@@ -139,7 +139,6 @@ def test_qt_archives_modules(monkeypatch, arch, requested_module_names, has_none
             assert url_match
             mod_name, archive_name = url_match.groups()
             assert pkg.archive == archive_name
-            assert pkg.hashurl == pkg.archive_url + ".sha1"
             assert archive_name in expected_7z_files
             expected_7z_files.remove(archive_name)
         assert len(expected_7z_files) == 0, "Actual number of packages was fewer than expected"
@@ -227,7 +226,6 @@ def test_tools_variants(monkeypatch, tool_name, tool_variant_name, is_expect_fai
         actual_variant_name, archive_name = url_match.groups()
         assert actual_variant_name == tool_variant_name
         assert pkg.archive == archive_name
-        assert pkg.hashurl == pkg.archive_url + ".sha1"
         assert archive_name in expected_7z_files
         expected_7z_files.remove(archive_name)
     assert len(expected_7z_files) == 0, f"Failed to produce QtPackages for {expected_7z_files}"

tests/test_helper.py

@@ -9,8 +9,8 @@ import requests
 from requests.models import Response
 
 from aqt import helper
-from aqt.exceptions import ArchiveChecksumError, ArchiveConnectionError, ArchiveDownloadError
+from aqt.exceptions import ArchiveChecksumError, ArchiveConnectionError, ArchiveDownloadError, ChecksumDownloadFailure
-from aqt.helper import getUrl, retry_on_errors
+from aqt.helper import Settings, get_hash, getUrl, retry_on_errors
 from aqt.metadata import Version
 
 
@@ -183,6 +183,41 @@ def test_helper_retry_on_error(num_attempts_before_success, num_retries_allowed)
         assert retry_on_errors(action, (RuntimeError,), num_retries_allowed, "do something")
 
 
+@pytest.mark.parametrize(
+    "num_tries_required, num_retries_allowed",
+    (
+        (2, 5),
+        (5, 5),
+        (6, 5),
+    ),
+)
+def test_helper_get_hash_retries(monkeypatch, num_tries_required, num_retries_allowed):
+    num_tries = 0
+
+    def mock_getUrl(url, *args, **kwargs):
+        nonlocal num_tries
+        num_tries += 1
+        if num_tries < num_tries_required:
+            raise ArchiveConnectionError(f"Must retry {num_tries_required - num_tries} more times before success")
+        parsed = urlparse(url)
+        base = f"{parsed.scheme}://{parsed.netloc}"
+        assert base in Settings.trusted_mirrors
+
+        hash_filename = str(parsed.path.split("/")[-1])
+        assert hash_filename == "archive.7z.sha256"
+        return "MOCK_HASH archive.7z"
+
+    monkeypatch.setattr("aqt.helper.getUrl", mock_getUrl)
+
+    if num_tries_required > num_retries_allowed:
+        with pytest.raises(ChecksumDownloadFailure) as e:
+            result = get_hash("http://insecure.mirror.com/some/path/to/archive.7z", "sha256", (5, 5))
+        assert e.type == ChecksumDownloadFailure
+    else:
+        result = get_hash("http://insecure.mirror.com/some/path/to/archive.7z", "sha256", (5, 5))
+        assert result == "MOCK_HASH"
+
+
 @pytest.mark.parametrize(
     "version, expect",
     [

tests/test_install.py

@@ -137,7 +137,7 @@ def make_mock_geturl_download_archive(
     def mock_getUrl(url: str, *args) -> str:
         if url.endswith(updates_url):
             return "<Updates>\n{}\n</Updates>".format("\n".join([archive.xml_package_update() for archive in archives]))
-        elif url.endswith(".sha1"):
+        elif url.endswith(".sha256"):
             return ""  # Skip the checksum
         assert False
 
@@ -598,7 +598,7 @@ def test_install(
 
     mock_get_url, mock_download_archive = make_mock_geturl_download_archive(archives, arch, host, updates_url)
     monkeypatch.setattr("aqt.archives.getUrl", mock_get_url)
-    monkeypatch.setattr("aqt.installer.getUrl", mock_get_url)
+    monkeypatch.setattr("aqt.helper.getUrl", mock_get_url)
     monkeypatch.setattr("aqt.installer.downloadBinaryFile", mock_download_archive)
 
     with TemporaryDirectory() as output_dir:
@@ -713,7 +713,7 @@ def test_install_nonexistent_archives(monkeypatch, capsys, cmd, xml_file: Option
         return (Path(__file__).parent / "data" / xml_file).read_text("utf-8")
 
     monkeypatch.setattr("aqt.archives.getUrl", mock_get_url)
-    monkeypatch.setattr("aqt.installer.getUrl", mock_get_url)
+    monkeypatch.setattr("aqt.helper.getUrl", mock_get_url)
     monkeypatch.setattr("aqt.metadata.getUrl", mock_get_url)
 
     cli = Cli()
@@ -779,7 +779,7 @@ def test_install_pool_exception(monkeypatch, capsys, make_exception, settings_fi
     cmd = ["install-qt", host, target, ver, arch]
     mock_get_url, mock_download_archive = make_mock_geturl_download_archive(archives, arch, host, updates_url)
     monkeypatch.setattr("aqt.archives.getUrl", mock_get_url)
-    monkeypatch.setattr("aqt.installer.getUrl", mock_get_url)
+    monkeypatch.setattr("aqt.helper.getUrl", mock_get_url)
     monkeypatch.setattr("aqt.installer.installer", mock_installer_func)
 
     Settings.load_settings(str(Path(__file__).parent / settings_file))
@@ -793,7 +793,7 @@ def test_install_installer_archive_extraction_err(monkeypatch):
     def mock_extractor_that_fails(*args, **kwargs):
         raise subprocess.CalledProcessError(returncode=1, cmd="some command", output="out", stderr="err")
 
-    monkeypatch.setattr("aqt.installer.getUrl", lambda *args: "")
+    monkeypatch.setattr("aqt.installer.get_hash", lambda *args, **kwargs: "")
     monkeypatch.setattr("aqt.installer.downloadBinaryFile", lambda *args: None)
     monkeypatch.setattr("aqt.installer.subprocess.run", mock_extractor_that_fails)
 
@@ -804,7 +804,6 @@ def test_install_installer_archive_extraction_err(monkeypatch):
                 "archive-url",
                 "archive",
                 "package_desc",
-                "hashurl",
                 "pkg_update_name",
             ),
             base_dir=temp_dir,