Merge branch 'master' into topic-fix-453

CHANGELOG.rst

@@ -7,6 +7,22 @@ All notable changes to this project will be documented in this file.
 `Unreleased`_
 =============
 
+`v2.0.2`_ (1, Nov. 2021)
+=========================
+
+Added
+-----
+* Support Qt 6.2.1 (#441)
+
+Fixed
+-----
+* Degraded install-tool (#442,#443)
+
+Changed
+-------
+* Add suggestion to use ``--external`` for MemoryError (#439)
+
+
 `v2.0.1`_ (29, Oct. 2021)
 =========================
 
@@ -248,7 +264,7 @@ Fixed
 =========================
 
 Added
------
+-----.. _v2.0.1: https://github.com/miurahr/aqtinstall/compare/v2.0.0...v2.0.1
 
 * Patching android installation for Qt6
   - patch target_qt.conf
@@ -294,7 +310,8 @@ Fixed
 
 
 
-.. _Unreleased: https://github.com/miurahr/aqtinstall/compare/v2.0.1...HEAD
+.. _Unreleased: https://github.com/miurahr/aqtinstall/compare/v2.0.2...HEAD
+.. _v2.0.2: https://github.com/miurahr/aqtinstall/compare/v2.0.1...v2.0.2
 .. _v2.0.1: https://github.com/miurahr/aqtinstall/compare/v2.0.0...v2.0.1
 .. _v2.0.0: https://github.com/miurahr/aqtinstall/compare/v1.2.5...v2.0.0
 .. _v1.2.5: https://github.com/miurahr/aqtinstall/compare/v1.2.4...v1.2.5

aqt/helper.py

@@ -79,7 +79,7 @@ def getUrl(url: str, timeout) -> str:
     return result
 
 
-def downloadBinaryFile(url: str, out: str, hash_algo: str, exp: bytes, timeout):
+def downloadBinaryFile(url: str, out: Path, hash_algo: str, exp: bytes, timeout):
     logger = getLogger("aqt.helper")
     filename = Path(url).name
     with requests.Session() as session:
@@ -114,7 +114,7 @@ def downloadBinaryFile(url: str, out: str, hash_algo: str, exp: bytes, timeout):
                 raise ArchiveChecksumError(
                     f"Downloaded file {filename} is corrupted! Detect checksum error.\n"
                     f"Expect {exp.hex()}: {url}\n"
-                    f"Actual {hash.digest().hex()}: {out}"
+                    f"Actual {hash.digest().hex()}: {out.name}"
                 )
 
 
@@ -305,6 +305,14 @@ class SettingsClass:
                 result = record["modules"]
         return result
 
+    @property
+    def archive_download_location(self):
+        return self.config.get("aqt", "archive_download_location", fallback=".")
+
+    @property
+    def always_keep_archives(self):
+        return self.config.getboolean("aqt", "always_keep_archives", fallback=False)
+
     @property
     def concurrency(self):
         """concurrency configuration.

aqt/installer.py

@@ -33,6 +33,8 @@ import sys
 import time
 from logging import getLogger
 from logging.handlers import QueueHandler
+from pathlib import Path
+from tempfile import TemporaryDirectory
 from typing import Any, Callable, List, Optional
 
 import aqt
@@ -219,6 +221,24 @@ class Cli:
             getLogger("aqt.installer").info(f"Resolved spec '{qt_version_or_spec}' to {version}")
             return version
 
+    @staticmethod
+    def choose_archive_dest(archive_dest: Optional[str], keep: bool, temp_dir: str) -> Path:
+        """
+        Choose archive download destination, based on context.
+
+        There are three potential behaviors here:
+        1. By default, return a temp directory that will be removed on program exit.
+        2. If the user has asked to keep archives, but has not specified a destination,
+            we return Settings.archive_download_location ("." by default).
+        3. If the user has asked to keep archives and specified a destination,
+            we create the destination dir if it doesn't exist, and return that directory.
+        """
+        if not archive_dest:
+            return Path(Settings.archive_download_location if keep else temp_dir)
+        dest = Path(archive_dest)
+        dest.mkdir(parents=True, exist_ok=True)
+        return dest
+
     def run_install_qt(self, args):
         """Run install subcommand"""
         start_time = time.perf_counter()
@@ -235,7 +255,8 @@ class Cli:
         else:
             qt_version: str = args.qt_version
             Cli._validate_version_str(qt_version)
-        keep = args.keep
+        keep: bool = args.keep or Settings.always_keep_archives
+        archive_dest: Optional[str] = args.archive_dest
         output_dir = args.outputdir
         if output_dir is None:
             base_dir = os.getcwd()
@@ -295,7 +316,9 @@ class Cli:
             base,
         )
         target_config = qt_archives.get_target_config()
-        run_installer(qt_archives.get_packages(), base_dir, sevenzip, keep)
+        with TemporaryDirectory() as temp_dir:
+            _archive_dest = Cli.choose_archive_dest(archive_dest, keep, temp_dir)
+            run_installer(qt_archives.get_packages(), base_dir, sevenzip, keep, _archive_dest)
         if not nopatch:
             Updater.update(target_config, base_dir)
         self.logger.info("Finished installation")
@@ -322,7 +345,8 @@ class Cli:
             base_dir = os.getcwd()
         else:
             base_dir = output_dir
-        keep = args.keep
+        keep: bool = args.keep or Settings.always_keep_archives
+        archive_dest: Optional[str] = args.archive_dest
         if args.base is not None:
             base = args.base
         else:
@@ -355,7 +379,9 @@ class Cli:
             ),
             base,
         )
-        run_installer(srcdocexamples_archives.get_packages(), base_dir, sevenzip, keep)
+        with TemporaryDirectory() as temp_dir:
+            _archive_dest = Cli.choose_archive_dest(archive_dest, keep, temp_dir)
+            run_installer(srcdocexamples_archives.get_packages(), base_dir, sevenzip, keep, _archive_dest)
         self.logger.info("Finished installation")
 
     def run_install_src(self, args):
@@ -407,7 +433,8 @@ class Cli:
         version = getattr(args, "version", None)
         if version is not None:
             Cli._validate_version_str(version, allow_minus=True)
-        keep = args.keep
+        keep: bool = args.keep or Settings.always_keep_archives
+        archive_dest: Optional[str] = args.archive_dest
         if args.base is not None:
             base = args.base
         else:
@@ -444,7 +471,9 @@ class Cli:
                 ),
                 base,
             )
-            run_installer(tool_archives.get_packages(), base_dir, sevenzip, keep)
+            with TemporaryDirectory() as temp_dir:
+                _archive_dest = Cli.choose_archive_dest(archive_dest, keep, temp_dir)
+                run_installer(tool_archives.get_packages(), base_dir, sevenzip, keep, _archive_dest)
         self.logger.info("Finished installation")
         self.logger.info("Time elapsed: {time:.8f} second".format(time=time.perf_counter() - start_time))
 
@@ -805,6 +834,13 @@ class Cli:
             action="store_true",
             help="Keep downloaded archive when specified, otherwise remove after install",
         )
+        subparser.add_argument(
+            "-d",
+            "--archive-dest",
+            type=str,
+            default=None,
+            help="Set the destination path for downloaded archives (temp directory by default).",
+        )
 
     def _set_module_options(self, subparser):
         subparser.add_argument("-m", "--modules", nargs="*", help="Specify extra modules to install")
@@ -888,14 +924,14 @@ class Cli:
             return function(fallback_url)
 
 
-def run_installer(archives: List[QtPackage], base_dir: str, sevenzip: Optional[str], keep: bool):
+def run_installer(archives: List[QtPackage], base_dir: str, sevenzip: Optional[str], keep: bool, archive_dest: Path):
     queue = multiprocessing.Manager().Queue(-1)
     listener = MyQueueListener(queue)
     listener.start()
     #
     tasks = []
     for arc in archives:
-        tasks.append((arc, base_dir, sevenzip, queue, keep))
+        tasks.append((arc, base_dir, sevenzip, queue, archive_dest, keep))
     ctx = multiprocessing.get_context("spawn")
     pool = ctx.Pool(Settings.concurrency, init_worker_sh)
 
@@ -946,6 +982,7 @@ def installer(
     base_dir: str,
     command: Optional[str],
     queue: multiprocessing.Queue,
+    archive_dest: Path,
     keep: bool = False,
     response_timeout: Optional[int] = None,
 ):
@@ -956,7 +993,7 @@ def installer(
     name = qt_archive.name
     url = qt_archive.archive_url
     hashurl = qt_archive.hashurl
-    archive = qt_archive.archive
+    archive: Path = archive_dest / qt_archive.archive
     start_time = time.perf_counter()
     # set defaults
     Settings.load_settings()
@@ -993,10 +1030,10 @@ def installer(
                 "-bd",
                 "-y",
                 "-o{}".format(base_dir),
-                archive,
+                str(archive),
             ]
         else:
-            command_args = [command, "x", "-aoa", "-bd", "-y", archive]
+            command_args = [command, "x", "-aoa", "-bd", "-y", str(archive)]
         try:
             proc = subprocess.run(command_args, stdout=subprocess.PIPE, check=True)
             logger.debug(proc.stdout)
@@ -1005,7 +1042,7 @@ def installer(
             raise ArchiveExtractionError(msg) from cpe
     if not keep:
         os.unlink(archive)
-    logger.info("Finished installation of {} in {:.8f}".format(archive, time.perf_counter() - start_time))
+    logger.info("Finished installation of {} in {:.8f}".format(archive.name, time.perf_counter() - start_time))
     qh.flush()
     qh.close()
     logger.removeHandler(qh)

aqt/settings.ini

@@ -5,6 +5,8 @@ concurrency: 4
 baseurl: https://download.qt.io
 7zcmd: 7z
 print_stacktrace_on_error: False
+always_keep_archives: False
+archive_download_location: .
 
 [requests]
 connection_timeout: 3.5

azure-pipelines.yml

@@ -32,7 +32,7 @@ jobs:
 - job: Mac
   dependsOn: MatricesGenerator
   pool:
-    vmImage: 'macOS-10.14'
+    vmImage: 'macOS-10.15'
   strategy:
     matrix: $[ dependencies.MatricesGenerator.outputs['mtrx.mac'] ]
   steps:

docs/cli.rst

@@ -361,7 +361,20 @@ are described here:
 
 .. option:: --keep, -k
 
-    Keep downloaded archive when specified, otherwise remove after install
+    Keep downloaded archive when specified, otherwise remove after install.
+    Use ``--archive-dest <path>`` to choose where aqt will place these files.
+    If you do not specify a download destination, aqt will place these files in
+    the current working directory.
+
+.. option:: --archive-dest <path>
+
+    Set the destination path for downloaded archives (temp directory by default).
+    All downloaded archives will be automatically deleted unless you have
+    specified the ``--keep`` option above, or ``aqt`` crashes.
+
+    Note that this option refers to the intermediate ``.7z`` archives that ``aqt``
+    downloads and then extracts to ``--outputdir``.
+    Most users will not need to keep these files.
 
 .. option:: --modules, -m (<list of modules> | all)
 
@@ -440,6 +453,7 @@ install-qt command
         [-E | --external <7zip command>]
         [--internal]
         [-k | --keep]
+        [-d | --archive-dest] <path>
         [-m | --modules (all | <module> [<module>...])]
         [--archives <archive> [<archive>...]]
         [--noarchives]
@@ -515,6 +529,7 @@ install-src command
         [-E | --external <7zip command>]
         [--internal]
         [-k | --keep]
+        [-d | --archive-dest] <path>
         [--archives <archive> [<archive>...]]
         [--kde]
         <host> [<target>] (<Qt version> | <spec>)
@@ -574,6 +589,7 @@ install-doc command
         [-E | --external <7zip command>]
         [--internal]
         [-k | --keep]
+        [-d | --archive-dest] <path>
         [-m | --modules (all | <module> [<module>...])]
         [--archives <archive> [<archive>...]]
         <host> [<target>] (<Qt version> | <spec>)
@@ -625,6 +641,7 @@ install-example command
         [-E | --external <7zip command>]
         [--internal]
         [-k | --keep]
+        [-d | --archive-dest] <path>
         [-m | --modules (all | <module> [<module>...])]
         [--archives <archive> [<archive>...]]
         <host> [<target>] (<Qt version> | <spec>)
@@ -678,6 +695,7 @@ install-tool command
         [-E | --external <7zip command>]
         [--internal]
         [-k | --keep]
+        [-d | --archive-dest] <path>
         <host> <target> <tool name> [<tool variant name>]
 
 Install tools like QtIFW, mingw, Cmake, Conan, and vcredist.

docs/configuration.rst

@@ -20,6 +20,8 @@ A file is like as follows:
     baseurl: https://download.qt.io
     7zcmd: 7z
     print_stacktrace_on_error: False
+    always_keep_archives: False
+    archive_download_location: .
 
     [requests]
     connection_timeout: 3.5
@@ -71,6 +73,18 @@ print_stacktrace_on_error:
     The ``False`` setting will hide the stack trace, unless an unhandled
     exception occurs.
 
+always_keep_archives:
+    This is either ``True`` or ``False``.
+    The ``True`` setting turns on the ``--keep`` option every time you run aqt,
+    and cannot be overridden by command line options.
+    The ``False`` setting will require you to set ``--keep`` manually every time
+    you run aqt, unless you don't want to keep ``.7z`` archives.
+
+archive_download_location:
+    This is the relative or absolute path to the location in which ``.7z`` archives
+    will be downloaded, when ``--keep`` is turned on.
+    You can override this location with the ``--archives-dest`` option.
+
 
 The ``[requests]`` section controls the way that ``aqt`` makes network requests.
 

tests/test_cli.py

@@ -351,3 +351,26 @@ def test_cli_set_7zip(monkeypatch):
         cli._set_sevenzip("some_nonexistent_binary")
     assert err.type == CliInputError
     assert format(err.value) == "Specified 7zip command executable does not exist: 'some_nonexistent_binary'"
+
+
+@pytest.mark.parametrize(
+    "archive_dest, keep, temp_dir, expect, should_make_dir",
+    (
+        (None, False, "temp", "temp", False),
+        (None, True, "temp", ".", False),
+        ("my_archives", False, "temp", "my_archives", True),
+        ("my_archives", True, "temp", "my_archives", True),
+    ),
+)
+def test_cli_choose_archive_dest(
+    monkeypatch, archive_dest: Optional[str], keep: bool, temp_dir: str, expect: str, should_make_dir: bool
+):
+    enclosed = {"made_dir": False}
+
+    def mock_mkdir(*args, **kwargs):
+        enclosed["made_dir"] = True
+
+    monkeypatch.setattr("aqt.installer.Path.mkdir", mock_mkdir)
+
+    assert Cli.choose_archive_dest(archive_dest, keep, temp_dir) == Path(expect)
+    assert enclosed["made_dir"] == should_make_dir

tests/test_helper.py

@@ -127,10 +127,10 @@ def test_helper_downloadBinary_wrong_checksum(tmp_path, monkeypatch):
     expected_err = (
         f"Downloaded file test.xml is corrupted! Detect checksum error."
         f"\nExpect {wrong_hash.hex()}: {url}"
-        f"\nActual {actual_hash.hex()}: {out}"
+        f"\nActual {actual_hash.hex()}: {out.name}"
     )
     with pytest.raises(ArchiveChecksumError) as e:
-        helper.downloadBinaryFile(url, str(out), "md5", wrong_hash, 60)
+        helper.downloadBinaryFile(url, out, "md5", wrong_hash, 60)
     assert e.type == ArchiveChecksumError
     assert format(e.value) == expected_err
 

tests/test_install.py

@@ -146,11 +146,11 @@ def make_mock_geturl_download_archive(
 
         def locate_archive() -> MockArchive:
             for arc in archives:
-                if out == arc.filename_7z:
+                if Path(out).name == arc.filename_7z:
                     return arc
             assert False, "Requested an archive that was not mocked"
 
-        locate_archive().write_compressed_archive(Path("./"))
+        locate_archive().write_compressed_archive(Path(out).parent)
 
     return mock_getUrl, mock_download_archive
 
@@ -810,6 +810,7 @@ def test_install_installer_archive_extraction_err(monkeypatch):
             base_dir=temp_dir,
             command="some_nonexistent_7z_extractor",
             queue=MockMultiprocessingManager.Queue(),
+            archive_dest=Path(temp_dir),
         )
     assert err.type == ArchiveExtractionError
     err_msg = format(err.value).rstrip()