Mirrors/aqtinstall
1
0
Fork 0
mirror of https://github.com/miurahr/aqtinstall.git synced 2025-12-17 12:44:38 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use sha256 hashes only from trusted mirrors

Browse Source 
To keep this commit small, `hashurl` was removed from QtPackage, and
`get_hash` constructs the hash url based on the url of the 7z archive
to download. I think that in the future, QtArchive and QtPackage could
be refactored to construct this url more appropriately. However, this
would be a complicated change that doesn't belong in this commit.
This commit is contained in:
David Dalcino
2022-02-27 11:43:36 -08:00
parent b92ee9935d
commit 7ebd6aa34e
7 changed files with 85 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
aqt/archives.py
View File

@@ -48,7 +48,6 @@ class QtPackage:
archive_url: str
archive: str
package_desc: str
hashurl: str
pkg_update_name: str
version: Optional[Version] = field(default=None)
@@ -61,7 +60,7 @@ class QtPackage:
return (
f"QtPackage(name={self.name}, url={self.archive_url}, "
f"archive={self.archive}, desc={self.package_desc}"
f"hashurl={self.hashurl}{v_info})"
f"{v_info})"
)
@@ -278,14 +277,12 @@ class QtArchives:
# 5.15.0-0-202005140804qtbase-Linux-RHEL_7_6-GCC-Linux-RHEL_7_6-X86_64.7z
full_version + archive,
)
hashurl = package_url + ".sha1"
self.archives.append(
QtPackage(
name=archive_name,
archive_url=package_url,
archive=archive,
package_desc=package_desc,
hashurl=hashurl,
pkg_update_name=pkg_name, # For testing purposes
)
)
@@ -480,14 +477,12 @@ class ToolArchives(QtArchives):
# 4.1.1-202105261130ifw-linux-x64.7z
f"{named_version}{archive}",
)
hashurl = package_url + ".sha1"
self.archives.append(
QtPackage(
name=name,
archive_url=package_url,
archive=archive,
package_desc=package_desc,
hashurl=hashurl,
pkg_update_name=name, # Redundant
)
)

4
aqt/exceptions.py
View File

@@ -47,6 +47,10 @@ class ArchiveChecksumError(ArchiveDownloadError):
pass
class ChecksumDownloadFailure(ArchiveDownloadError):
pass
class ArchiveConnectionError(AqtException):
pass

37
aqt/helper.py
View File

@@ -24,18 +24,25 @@ import hashlib
import json
import logging.config
import os
import posixpath
import sys
import xml.etree.ElementTree as ElementTree
from logging import getLogger
from logging.handlers import QueueListener
from pathlib import Path
from typing import Callable, Dict, List, Tuple
from typing import Callable, Dict, Generator, List, Tuple
from urllib.parse import urlparse
import requests
import requests.adapters
from aqt.exceptions import ArchiveChecksumError, ArchiveConnectionError, ArchiveDownloadError, ArchiveListError
from aqt.exceptions import (
ArchiveChecksumError,
ArchiveConnectionError,
ArchiveDownloadError,
ArchiveListError,
ChecksumDownloadFailure,
)
def _get_meta(url: str):
@@ -134,6 +141,32 @@ def retry_on_errors(action: Callable[[], any], acceptable_errors: Tuple, num_ret
raise e from e
def iter_list_reps(_list: List, num_reps: int) -> Generator:
list_index = 0
for i in range(num_reps):
yield _list[list_index]
list_index += 1
if list_index >= len(_list):
list_index = 0
def get_hash(archive_url: str, algorithm: str, timeout) -> str:
path = urlparse(archive_url).path
while path.startswith("/"):
path = path[1:]
for base_url in iter_list_reps(Settings.trusted_mirrors, Settings.max_retries_to_retrieve_hash):
url = posixpath.join(base_url, f"{path}.{algorithm}")
try:
r = getUrl(url, timeout)
# sha256 & md5 files are: "some_hash archive_filename"
return r.split(" ")[0]
except (ArchiveConnectionError, ArchiveDownloadError):
pass
raise ChecksumDownloadFailure(
f"Failed to download checksum for the file '{path.split('/')[-1]}' from mirrors '{Settings.trusted_mirrors}"
)
def altlink(url: str, alt: str):
"""
Blacklisting redirected(alt) location based on Settings.blacklist configuration.

7
aqt/installer.py
View File

@@ -51,7 +51,7 @@ from aqt.exceptions import (
CliKeyboardInterrupt,
OutOfMemory,
)
from aqt.helper import MyQueueListener, Settings, downloadBinaryFile, getUrl, retry_on_errors, setup_logging
from aqt.helper import MyQueueListener, Settings, downloadBinaryFile, get_hash, retry_on_errors, setup_logging
from aqt.metadata import ArchiveId, MetadataFactory, QtRepoProperty, SimpleSpec, Version, show_list, suggested_follow_up
from aqt.updater import Updater
@@ -1002,7 +1002,6 @@ def installer(
"""
name = qt_archive.name
url = qt_archive.archive_url
hashurl = qt_archive.hashurl
archive: Path = archive_dest / qt_archive.archive
start_time = time.perf_counter()
# set defaults
@@ -1021,9 +1020,9 @@ def installer(
timeout = (Settings.connection_timeout, Settings.response_timeout)
else:
timeout = (Settings.connection_timeout, response_timeout)
hash = binascii.unhexlify(getUrl(hashurl, timeout))
hash = binascii.unhexlify(get_hash(url, algorithm="sha256", timeout=timeout))
retry_on_errors(
action=lambda: downloadBinaryFile(url, archive, "sha1", hash, timeout),
action=lambda: downloadBinaryFile(url, archive, "sha256", hash, timeout),
acceptable_errors=(ArchiveChecksumError,),
num_retries=Settings.max_retries_on_checksum_error,
name=f"Downloading {name}",

2
tests/test_archives.py
View File

@@ -139,7 +139,6 @@ def test_qt_archives_modules(monkeypatch, arch, requested_module_names, has_none
assert url_match
mod_name, archive_name = url_match.groups()
assert pkg.archive == archive_name
assert pkg.hashurl == pkg.archive_url + ".sha1"
assert archive_name in expected_7z_files
expected_7z_files.remove(archive_name)
assert len(expected_7z_files) == 0, "Actual number of packages was fewer than expected"
@@ -227,7 +226,6 @@ def test_tools_variants(monkeypatch, tool_name, tool_variant_name, is_expect_fai
actual_variant_name, archive_name = url_match.groups()
assert actual_variant_name == tool_variant_name
assert pkg.archive == archive_name
assert pkg.hashurl == pkg.archive_url + ".sha1"
assert archive_name in expected_7z_files
expected_7z_files.remove(archive_name)
assert len(expected_7z_files) == 0, f"Failed to produce QtPackages for {expected_7z_files}"

39
tests/test_helper.py
View File

@@ -9,8 +9,8 @@ import requests
from requests.models import Response
from aqt import helper
from aqt.exceptions import ArchiveChecksumError, ArchiveConnectionError, ArchiveDownloadError
from aqt.helper import getUrl, retry_on_errors
from aqt.exceptions import ArchiveChecksumError, ArchiveConnectionError, ArchiveDownloadError, ChecksumDownloadFailure
from aqt.helper import Settings, get_hash, getUrl, retry_on_errors
from aqt.metadata import Version
@@ -183,6 +183,41 @@ def test_helper_retry_on_error(num_attempts_before_success, num_retries_allowed)
assert retry_on_errors(action, (RuntimeError,), num_retries_allowed, "do something")
@pytest.mark.parametrize(
"num_tries_required, num_retries_allowed",
(
(2, 5),
(5, 5),
(6, 5),
),
)
def test_helper_get_hash_retries(monkeypatch, num_tries_required, num_retries_allowed):
num_tries = 0
def mock_getUrl(url, *args, **kwargs):
nonlocal num_tries
num_tries += 1
if num_tries < num_tries_required:
raise ArchiveConnectionError(f"Must retry {num_tries_required - num_tries} more times before success")
parsed = urlparse(url)
base = f"{parsed.scheme}://{parsed.netloc}"
assert base in Settings.trusted_mirrors
hash_filename = str(parsed.path.split("/")[-1])
assert hash_filename == "archive.7z.sha256"
return "MOCK_HASH archive.7z"
monkeypatch.setattr("aqt.helper.getUrl", mock_getUrl)
if num_tries_required > num_retries_allowed:
with pytest.raises(ChecksumDownloadFailure) as e:
result = get_hash("http://insecure.mirror.com/some/path/to/archive.7z", "sha256", (5, 5))
assert e.type == ChecksumDownloadFailure
else:
result = get_hash("http://insecure.mirror.com/some/path/to/archive.7z", "sha256", (5, 5))
assert result == "MOCK_HASH"
@pytest.mark.parametrize(
"version, expect",
[

11
tests/test_install.py
View File

@@ -137,7 +137,7 @@ def make_mock_geturl_download_archive(
def mock_getUrl(url: str, *args) -> str:
if url.endswith(updates_url):
return "<Updates>\n{}\n</Updates>".format("\n".join([archive.xml_package_update() for archive in archives]))
elif url.endswith(".sha1"):
elif url.endswith(".sha256"):
return "" # Skip the checksum
assert False
@@ -598,7 +598,7 @@ def test_install(
mock_get_url, mock_download_archive = make_mock_geturl_download_archive(archives, arch, host, updates_url)
monkeypatch.setattr("aqt.archives.getUrl", mock_get_url)
monkeypatch.setattr("aqt.installer.getUrl", mock_get_url)
monkeypatch.setattr("aqt.helper.getUrl", mock_get_url)
monkeypatch.setattr("aqt.installer.downloadBinaryFile", mock_download_archive)
with TemporaryDirectory() as output_dir:
@@ -713,7 +713,7 @@ def test_install_nonexistent_archives(monkeypatch, capsys, cmd, xml_file: Option
return (Path(__file__).parent / "data" / xml_file).read_text("utf-8")
monkeypatch.setattr("aqt.archives.getUrl", mock_get_url)
monkeypatch.setattr("aqt.installer.getUrl", mock_get_url)
monkeypatch.setattr("aqt.helper.getUrl", mock_get_url)
monkeypatch.setattr("aqt.metadata.getUrl", mock_get_url)
cli = Cli()
@@ -779,7 +779,7 @@ def test_install_pool_exception(monkeypatch, capsys, make_exception, settings_fi
cmd = ["install-qt", host, target, ver, arch]
mock_get_url, mock_download_archive = make_mock_geturl_download_archive(archives, arch, host, updates_url)
monkeypatch.setattr("aqt.archives.getUrl", mock_get_url)
monkeypatch.setattr("aqt.installer.getUrl", mock_get_url)
monkeypatch.setattr("aqt.helper.getUrl", mock_get_url)
monkeypatch.setattr("aqt.installer.installer", mock_installer_func)
Settings.load_settings(str(Path(__file__).parent / settings_file))
@@ -793,7 +793,7 @@ def test_install_installer_archive_extraction_err(monkeypatch):
def mock_extractor_that_fails(*args, **kwargs):
raise subprocess.CalledProcessError(returncode=1, cmd="some command", output="out", stderr="err")
monkeypatch.setattr("aqt.installer.getUrl", lambda *args: "")
monkeypatch.setattr("aqt.installer.get_hash", lambda *args, **kwargs: "")
monkeypatch.setattr("aqt.installer.downloadBinaryFile", lambda *args: None)
monkeypatch.setattr("aqt.installer.subprocess.run", mock_extractor_that_fails)
@@ -804,7 +804,6 @@ def test_install_installer_archive_extraction_err(monkeypatch):
"archive-url",
"archive",
"package_desc",
"hashurl",
"pkg_update_name",
),
base_dir=temp_dir,