Move constants into settings.ini configuration

Signed-off-by: Hiroshi Miura <miurahr@linux.com>

aqt/helper.py

@@ -169,8 +169,6 @@ class Settings(object):
     _shared_state = {
         "config": None,
         "_combinations": None,
-        "_concurrency": None,
-        "_blacklist": None,
         "_lock": multiprocessing.Lock(),
     }
 
@@ -188,12 +186,6 @@ class Settings(object):
                     # load custom file
                     if config_path is not None:
                         self.config.read(config_path)
-                    self._concurrency = self.config.getint(
-                        "aqt", "concurrency", fallback=4
-                    )
-                    self._blacklist = ast.literal_eval(
-                        self.config.get("mirrors", "blacklist", fallback="[]")
-                    )
                     # load combinations
                     with open(
                         os.path.join(os.path.dirname(__file__), "combinations.json"),
@@ -241,7 +233,7 @@ class Settings(object):
         :return: concurrency
         :rtype: int
         """
-        return self._concurrency
+        return self.config.getint("aqt", "concurrency", fallback=4)
 
     @property
     def blacklist(self):
@@ -250,4 +242,24 @@ class Settings(object):
         :returns: list of site URLs(scheme and host part)
         :rtype: List[str]
         """
-        return self._blacklist
+        return ast.literal_eval(self.config.get("mirrors", "blacklist", fallback="[]"))
+
+    @property
+    def baseurl(self):
+        return self.config.get("aqt", "baseurl", fallback="https://download.qt.io")
+
+    @property
+    def connection_timeout(self):
+        return self.config.getint("aqt", "connection_timeout", fallback=3.5)
+
+    @property
+    def response_timeout(self):
+        return self.config.getint("aqt", "response_timeout", fallback=3.5)
+
+    @property
+    def fallbacks(self):
+        return ast.literal_eval(self.config.get("mirrors", "fallbacks", fallback="[]"))
+
+    @property
+    def zipcmd(self):
+        return self.config.get("aqt", "7zcmd", fallback="7z")

aqt/installer.py

@@ -70,15 +70,6 @@ class ExtractionError(Exception):
     pass
 
 
-BASE_URL = "https://download.qt.io"
-FALLBACK_URLS = [
-    "https://mirrors.ocf.berkeley.edu/qt",
-    "https://ftp.jaist.ac.jp/pub/qtproject",
-    "http://ftp1.nluug.nl/languages/qt",
-    "https://mirrors.dotsrc.org/qtproject",
-]
-
-
 class Cli:
     """CLI main class to parse command line argument and launch proper functions."""
 
@@ -228,7 +219,7 @@ class Cli:
         if args.timeout is not None:
             timeout = (args.timeout, args.timeout)
         else:
-            timeout = (5, 5)
+            timeout = (self.settings.connection_timeout, self.settings.response_timeout)
         arch = self._set_arch(args, arch, os_name, target, qt_version)
         modules = args.modules
         sevenzip = self._set_sevenzip(args.external)
@@ -241,7 +232,7 @@ class Cli:
                 exit(1)
             base = args.base
         else:
-            base = BASE_URL
+            base = self.settings.baseurl
         archives = args.archives
         if args.noarchives:
             if modules is None:
@@ -298,7 +289,7 @@ class Cli:
                     target,
                     qt_version,
                     arch,
-                    random.choice(FALLBACK_URLS),
+                    random.choice(self.settings.fallbacks),
                     subarchives=archives,
                     modules=modules,
                     logging=self.logger,
@@ -340,7 +331,7 @@ class Cli:
         if args.base is not None:
             base = args.base
         else:
-            base = BASE_URL
+            base = self.settings.baseurl
         qt_ver_num = qt_version.replace(".", "")
         packages = ["qt.qt5.{}.{}".format(qt_ver_num, arch)]
         if args.archives is not None:
@@ -378,15 +369,15 @@ class Cli:
         if args.base is not None:
             base = args.base
         else:
-            base = BASE_URL
+            base = self.settings.baseurl
         if args.timeout is not None:
             timeout = (args.timeout, args.timeout)
         else:
-            timeout = (5, 5)
+            timeout = (self.settings.connection_timeout, self.settings.response_timeout)
         sevenzip = self._set_sevenzip(args.external)
         if EXT7Z and sevenzip is None:
             # override when py7zr is not exist
-            sevenzip = self._set_sevenzip("7z")
+            sevenzip = self._set_sevenzip(self.settings.zipcmd)
         modules = args.modules
         archives = args.archives
         all_extra = True if modules is not None and "all" in modules else False
@@ -417,7 +408,7 @@ class Cli:
                     os_name,
                     target,
                     qt_version,
-                    random.choice(FALLBACK_URLS),
+                    random.choice(self.settings.fallbacks),
                     subarchives=archives,
                     modules=modules,
                     logging=self.logger,
@@ -469,11 +460,11 @@ class Cli:
         if args.base is not None:
             base = args.base
         else:
-            base = BASE_URL
+            base = self.settings.baseurl
         if args.timeout is not None:
             timeout = (args.timeout, args.timeout)
         else:
-            timeout = (5, 5)
+            timeout = (self.settings.connection_timeout, self.settings.response_timeout)
         if not self._check_tools_arg_combination(os_name, tool_name, arch):
             self.logger.warning(
                 "Specified target combination is not valid: {} {} {}".format(
@@ -500,7 +491,7 @@ class Cli:
                     tool_name,
                     version,
                     arch,
-                    random.choice(FALLBACK_URLS),
+                    random.choice(self.settings.fallbacks),
                     logging=self.logger,
                     timeout=timeout,
                 )
@@ -524,9 +515,9 @@ class Cli:
         host = args.host
         target = args.target
         try:
-            pl = PackagesList(qt_version, host, target, BASE_URL)
+            pl = PackagesList(qt_version, host, target, self.settings.baseurl)
         except (ArchiveConnectionError, ArchiveDownloadError):
-            pl = PackagesList(qt_version, host, target, random.choice(FALLBACK_URLS))
+            pl = PackagesList(qt_version, host, target, random.choice(self.settings.fallbacks))
         print("List Qt packages in %s for %s" % (args.qt_version, args.host))
         table = Texttable()
         table.set_deco(Texttable.HEADER)
@@ -771,7 +762,7 @@ class Cli:
         return args.func(args)
 
 
-def installer(qt_archive, base_dir, command, keep=False, response_timeout=30):
+def installer(qt_archive, base_dir, command, keep=False, response_timeout=None):
     """
     Installer function to download archive files and extract it.
     It is called through multiprocessing.Pool()
@@ -784,7 +775,11 @@ def installer(qt_archive, base_dir, command, keep=False, response_timeout=30):
     logger = getLogger("aqt")
     logger.info("Downloading {}...".format(name))
     logger.debug("Download URL: {}".format(url))
-    timeout = (3.5, response_timeout)
+    settings = Settings()
+    if response_timeout is None:
+        timeout = (settings.connection_timeout, settings.response_timeout)
+    else:
+        timeout = (settings.connection_timeout, response_timeout)
     hash = binascii.unhexlify(getUrl(hashurl, timeout, logger))
     downloadBinaryFile(url, archive, "sha1", hash, timeout, logger)
     if command is None:

aqt/settings.ini

@@ -2,6 +2,11 @@
 
 [aqt]
 concurrency: 4
+connection_timeout: 3.5
+response_timeout: 30
+baseurl: "https://download.qt.io"
+7zcmd: "7z"
 
 [mirrors]
 blacklist: ['http://mirrors.ustc.edu.cn', 'http://mirrors.tuna.tsinghua.edu.cn', 'http://mirrors.geekpie.club']
+fallbacks: ["https://mirrors.ocf.berkeley.edu/qt", "https://ftp.jaist.ac.jp/pub/qtproject", "http://ftp1.nluug.nl/languages/qt", "https://mirrors.dotsrc.org/qtproject"]

ci/settings.ini

@@ -2,6 +2,8 @@
 
 [aqt]
 concurrency: 2
+connection_timeout: 10
+response_timeout: 10
 
 [mirrors]
 blacklist: ['http://mirrors.ustc.edu.cn', 'http://mirrors.tuna.tsinghua.edu.cn', 'http://mirrors.geekpie.club', 'http://mirrors.sjtug.sjtu.edu.cn']

tests/test_archives.py

@@ -2,8 +2,8 @@ import os
 
 import pytest
 
+from aqt.helper import Settings
 from aqt.archives import QtArchives
-from aqt.installer import BASE_URL
 
 
 @pytest.mark.parametrize(
@@ -21,7 +21,7 @@ def test_parse_update_xml(monkeypatch, os_name, version, target, datafile):
 
     monkeypatch.setattr(QtArchives, "_download_update_xml", _mock)
 
-    qt_archives = QtArchives(os_name, "desktop", version, target, BASE_URL)
+    qt_archives = QtArchives(os_name, "desktop", version, target, Settings().baseurl)
     assert qt_archives.archives is not None
 
     # Get packages with all extra modules

tests/test_doc_archives.py

@@ -3,7 +3,7 @@ import os
 import pytest
 
 from aqt.archives import QtArchives
-from aqt.installer import BASE_URL
+from aqt.helper import Settings
 
 
 @pytest.mark.parametrize(
@@ -17,12 +17,13 @@ def test_parse_update_xml(monkeypatch, os_name, version, target, datafile):
 
     monkeypatch.setattr(QtArchives, "_download_update_xml", _mock)
 
-    qt_archives = QtArchives(os_name, "desktop", version, target, BASE_URL)
+    settings = Settings()
+    qt_archives = QtArchives(os_name, "desktop", version, target, settings.baseurl)
     assert qt_archives.archives is not None
 
     # Get packages with all extra modules
     qt_archives_all_modules = QtArchives(
-        os_name, "desktop", version, target, BASE_URL, None, ["all"], None, True
+        os_name, "desktop", version, target, settings.baseurl, None, ["all"], None, True
     )
     assert qt_archives_all_modules.archives is not None