Merge pull request #498 from ddalcino/fix-sonatype-issues

Fix sonatype issues

aqt/archives.py

@@ -21,11 +21,12 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 import binascii
 import posixpath
-import xml.etree.ElementTree as ElementTree
 from dataclasses import dataclass, field
 from logging import getLogger
 from typing import Dict, Iterable, List, Optional, Tuple
 
+from defusedxml import ElementTree
+
 from aqt.exceptions import ArchiveDownloadError, ArchiveListError, ChecksumDownloadFailure, NoPackageFound
 from aqt.helper import Settings, get_hash, getUrl, ssplit
 from aqt.metadata import QtRepoProperty, Version

aqt/helper.py

@@ -27,15 +27,16 @@ import os
 import posixpath
 import secrets
 import sys
-import xml.etree.ElementTree as ElementTree
 from logging import getLogger
 from logging.handlers import QueueListener
 from pathlib import Path
 from typing import Callable, Dict, Generator, List, Optional, Tuple
 from urllib.parse import urlparse
+from xml.etree.ElementTree import Element
 
 import requests
 import requests.adapters
+from defusedxml import ElementTree
 
 from aqt.exceptions import (
     ArchiveChecksumError,
@@ -257,7 +258,7 @@ def ssplit(data: str):
 
 def xml_to_modules(
     xml_text: str,
-    predicate: Callable[[ElementTree.Element], bool],
+    predicate: Callable[[Element], bool],
 ) -> Dict[str, Dict[str, str]]:
     """Converts an XML document to a dict of `PackageUpdate` dicts, indexed by `Name` attribute.
     Only report elements that satisfy `predicate(element)`.

aqt/metadata.py

@@ -22,12 +22,12 @@ import binascii
 import itertools
 import operator
 import posixpath
-import random
 import re
+import secrets as random
 import shutil
 from logging import getLogger
 from typing import Callable, Dict, Generator, Iterable, Iterator, List, Optional, Tuple, Union
-from xml.etree import ElementTree as ElementTree
+from xml.etree.ElementTree import Element
 
 import bs4
 from semantic_version import SimpleSpec as SemanticSimpleSpec
@@ -622,7 +622,7 @@ class MetadataFactory:
         )
 
     @staticmethod
-    def _has_nonempty_downloads(element: ElementTree.Element) -> bool:
+    def _has_nonempty_downloads(element: Element) -> bool:
         """Returns True if the element has a nonempty '<DownloadableArchives/>' tag"""
         downloads = element.find("DownloadableArchives")
         update_file = element.find("UpdateFile")
@@ -641,7 +641,7 @@ class MetadataFactory:
         )
         return f"{version.major}{version.minor}{patch}"
 
-    def _fetch_module_metadata(self, folder: str, predicate: Optional[Callable[[ElementTree.Element], bool]] = None):
+    def _fetch_module_metadata(self, folder: str, predicate: Optional[Callable[[Element], bool]] = None):
         rest_of_url = posixpath.join(self.archive_id.to_url(), folder, "Updates.xml")
         xml = self.fetch_http(rest_of_url)
         return xml_to_modules(
@@ -705,15 +705,15 @@ class MetadataFactory:
         qt_version_str = self._get_qt_version_str(version)
         nonempty = MetadataFactory._has_nonempty_downloads
 
-        def all_modules(element: ElementTree.Element) -> bool:
+        def all_modules(element: Element) -> bool:
             _module, _arch = element.find("Name").text.split(".")[-2:]
             return _arch == arch and _module != qt_version_str and nonempty(element)
 
-        def specify_modules(element: ElementTree.Element) -> bool:
+        def specify_modules(element: Element) -> bool:
             _module, _arch = element.find("Name").text.split(".")[-2:]
             return _arch == arch and _module in modules and nonempty(element)
 
-        def no_modules(element: ElementTree.Element) -> bool:
+        def no_modules(element: Element) -> bool:
             name: Optional[str] = element.find("Name").text
             return name and name.endswith(f".{qt_version_str}.{arch}") and nonempty(element)
 

ci/generate_azure_pipelines_matrices.py

@@ -3,7 +3,7 @@ This sets variables for a matrix of QT versions to test downloading against with
 """
 import collections
 import json
-import random
+import secrets as random
 import re
 from itertools import product
 from typing import Dict, Optional

ci/generate_combinations.py

@@ -4,7 +4,7 @@ import argparse
 import json
 import logging
 from pathlib import Path
-from typing import Dict, Generator, Iterator, List, Optional, Tuple, Union
+from typing import Dict, Generator, Iterable, List, Optional, Tuple, Union
 
 from jsoncomparison import NO_DIFF, Compare
 
@@ -26,8 +26,8 @@ def is_blacklisted_tool(tool_name: str) -> bool:
 def iter_archive_ids(
     *,
     category: str,
-    hosts: Iterator[str] = ArchiveId.HOSTS,
-    targets: Optional[Iterator[str]] = None,
+    hosts: Iterable[str] = ArchiveId.HOSTS,
+    targets: Optional[Iterable[str]] = None,
     add_extensions: bool = False,
 ) -> Generator[ArchiveId, None, None]:
     def iter_extensions() -> Generator[str, None, None]:
@@ -76,7 +76,7 @@ def iter_arches() -> Generator[dict, None, None]:
 def iter_tool_variants() -> Generator[dict, None, None]:
     for archive_id in iter_archive_ids(category="tools"):
         logger.info("Fetching tool variants for {}".format(archive_id))
-        for tool_name in tqdm(sorted(MetadataFactory(archive_id).getList())):
+        for tool_name in tqdm(sorted(MetadataFactory(archive_id).fetch_tools())):
             if is_blacklisted_tool(tool_name):
                 continue
             for tool_variant in MetadataFactory(
@@ -93,7 +93,7 @@ def iter_tool_variants() -> Generator[dict, None, None]:
 def iter_qt_minor_groups(
     host: str = "linux", target: str = "desktop"
 ) -> Generator[Tuple[int, int], None, None]:
-    versions: Versions = MetadataFactory(ArchiveId("qt", host, target)).getList()
+    versions: Versions = MetadataFactory(ArchiveId("qt", host, target)).fetch_versions()
     for minor_group in versions:
         v = minor_group[0]
         yield v.major, v.minor

setup.cfg

@@ -46,6 +46,7 @@ install_requires =
     texttable
     bs4
     dataclasses;python_version<"3.7"
+    defusedxml
 setup_requires =
     setuptools-scm[toml]>=6.0.1
     setuptools>=45.0

tests/test_install.py

@@ -733,10 +733,10 @@ def test_install_nonexistent_archives(monkeypatch, capsys, cmd, xml_file: Option
 
 
 @pytest.mark.parametrize(
-    "make_exception, settings_file, expect_end_msg, expect_return",
+    "exception_class, settings_file, expect_end_msg, expect_return",
     (
         (
-            "RuntimeError()",
+            RuntimeError,
             "../aqt/settings.ini",
             "===========================PLEASE FILE A BUG REPORT===========================\n"
             "You have discovered a bug in aqt.\n"
@@ -745,13 +745,13 @@ def test_install_nonexistent_archives(monkeypatch, capsys, cmd, xml_file: Option
             Cli.UNHANDLED_EXCEPTION_CODE,
         ),
         (
-            "KeyboardInterrupt()",
+            KeyboardInterrupt,
             "../aqt/settings.ini",
             "Caught KeyboardInterrupt, terminating installer workers\nInstaller halted by keyboard interrupt.",
             1,
         ),
         (
-            "MemoryError()",
+            MemoryError,
             "../aqt/settings.ini",
             "Caught MemoryError, terminating installer workers\n"
             "Out of memory when downloading and extracting archives in parallel.\n"
@@ -763,7 +763,7 @@ def test_install_nonexistent_archives(monkeypatch, capsys, cmd, xml_file: Option
             1,
         ),
         (
-            "MemoryError()",
+            MemoryError,
             "data/settings_no_concurrency.ini",
             "Caught MemoryError, terminating installer workers\n"
             "Out of memory when downloading and extracting archives.\n"
@@ -775,9 +775,9 @@ def test_install_nonexistent_archives(monkeypatch, capsys, cmd, xml_file: Option
         ),
     ),
 )
-def test_install_pool_exception(monkeypatch, capsys, make_exception, settings_file, expect_end_msg, expect_return):
+def test_install_pool_exception(monkeypatch, capsys, exception_class, settings_file, expect_end_msg, expect_return):
     def mock_installer_func(*args):
-        raise eval(make_exception)
+        raise exception_class()
 
     host, target, ver, arch = "windows", "desktop", "6.1.0", "win64_mingw81"
     updates_url = "windows_x86/desktop/qt6_610/Updates.xml"