diff --git a/aqt/helper.py b/aqt/helper.py index 871330f..b8bc184 100644 --- a/aqt/helper.py +++ b/aqt/helper.py @@ -169,8 +169,6 @@ class Settings(object): _shared_state = { "config": None, "_combinations": None, - "_concurrency": None, - "_blacklist": None, "_lock": multiprocessing.Lock(), } @@ -188,12 +186,6 @@ class Settings(object): # load custom file if config_path is not None: self.config.read(config_path) - self._concurrency = self.config.getint( - "aqt", "concurrency", fallback=4 - ) - self._blacklist = ast.literal_eval( - self.config.get("mirrors", "blacklist", fallback="[]") - ) # load combinations with open( os.path.join(os.path.dirname(__file__), "combinations.json"), @@ -241,7 +233,7 @@ class Settings(object): :return: concurrency :rtype: int """ - return self._concurrency + return self.config.getint("aqt", "concurrency", fallback=4) @property def blacklist(self): @@ -250,4 +242,24 @@ class Settings(object): :returns: list of site URLs(scheme and host part) :rtype: List[str] """ - return self._blacklist + return ast.literal_eval(self.config.get("mirrors", "blacklist", fallback="[]")) + + @property + def baseurl(self): + return self.config.get("aqt", "baseurl", fallback="https://download.qt.io") + + @property + def connection_timeout(self): + return self.config.getint("aqt", "connection_timeout", fallback=3.5) + + @property + def response_timeout(self): + return self.config.getint("aqt", "response_timeout", fallback=3.5) + + @property + def fallbacks(self): + return ast.literal_eval(self.config.get("mirrors", "fallbacks", fallback="[]")) + + @property + def zipcmd(self): + return self.config.get("aqt", "7zcmd", fallback="7z") diff --git a/aqt/installer.py b/aqt/installer.py index 8bafa5b..8db07ab 100644 --- a/aqt/installer.py +++ b/aqt/installer.py @@ -70,15 +70,6 @@ class ExtractionError(Exception): pass -BASE_URL = "https://download.qt.io" -FALLBACK_URLS = [ - "https://mirrors.ocf.berkeley.edu/qt", - "https://ftp.jaist.ac.jp/pub/qtproject", - "http://ftp1.nluug.nl/languages/qt", - "https://mirrors.dotsrc.org/qtproject", -] - - class Cli: """CLI main class to parse command line argument and launch proper functions.""" @@ -228,7 +219,7 @@ class Cli: if args.timeout is not None: timeout = (args.timeout, args.timeout) else: - timeout = (5, 5) + timeout = (self.settings.connection_timeout, self.settings.response_timeout) arch = self._set_arch(args, arch, os_name, target, qt_version) modules = args.modules sevenzip = self._set_sevenzip(args.external) @@ -241,7 +232,7 @@ class Cli: exit(1) base = args.base else: - base = BASE_URL + base = self.settings.baseurl archives = args.archives if args.noarchives: if modules is None: @@ -298,7 +289,7 @@ class Cli: target, qt_version, arch, - random.choice(FALLBACK_URLS), + random.choice(self.settings.fallbacks), subarchives=archives, modules=modules, logging=self.logger, @@ -340,7 +331,7 @@ class Cli: if args.base is not None: base = args.base else: - base = BASE_URL + base = self.settings.baseurl qt_ver_num = qt_version.replace(".", "") packages = ["qt.qt5.{}.{}".format(qt_ver_num, arch)] if args.archives is not None: @@ -378,15 +369,15 @@ class Cli: if args.base is not None: base = args.base else: - base = BASE_URL + base = self.settings.baseurl if args.timeout is not None: timeout = (args.timeout, args.timeout) else: - timeout = (5, 5) + timeout = (self.settings.connection_timeout, self.settings.response_timeout) sevenzip = self._set_sevenzip(args.external) if EXT7Z and sevenzip is None: # override when py7zr is not exist - sevenzip = self._set_sevenzip("7z") + sevenzip = self._set_sevenzip(self.settings.zipcmd) modules = args.modules archives = args.archives all_extra = True if modules is not None and "all" in modules else False @@ -417,7 +408,7 @@ class Cli: os_name, target, qt_version, - random.choice(FALLBACK_URLS), + random.choice(self.settings.fallbacks), subarchives=archives, modules=modules, logging=self.logger, @@ -469,11 +460,11 @@ class Cli: if args.base is not None: base = args.base else: - base = BASE_URL + base = self.settings.baseurl if args.timeout is not None: timeout = (args.timeout, args.timeout) else: - timeout = (5, 5) + timeout = (self.settings.connection_timeout, self.settings.response_timeout) if not self._check_tools_arg_combination(os_name, tool_name, arch): self.logger.warning( "Specified target combination is not valid: {} {} {}".format( @@ -500,7 +491,7 @@ class Cli: tool_name, version, arch, - random.choice(FALLBACK_URLS), + random.choice(self.settings.fallbacks), logging=self.logger, timeout=timeout, ) @@ -524,9 +515,9 @@ class Cli: host = args.host target = args.target try: - pl = PackagesList(qt_version, host, target, BASE_URL) + pl = PackagesList(qt_version, host, target, self.settings.baseurl) except (ArchiveConnectionError, ArchiveDownloadError): - pl = PackagesList(qt_version, host, target, random.choice(FALLBACK_URLS)) + pl = PackagesList(qt_version, host, target, random.choice(self.settings.fallbacks)) print("List Qt packages in %s for %s" % (args.qt_version, args.host)) table = Texttable() table.set_deco(Texttable.HEADER) @@ -771,7 +762,7 @@ class Cli: return args.func(args) -def installer(qt_archive, base_dir, command, keep=False, response_timeout=30): +def installer(qt_archive, base_dir, command, keep=False, response_timeout=None): """ Installer function to download archive files and extract it. It is called through multiprocessing.Pool() @@ -784,7 +775,11 @@ def installer(qt_archive, base_dir, command, keep=False, response_timeout=30): logger = getLogger("aqt") logger.info("Downloading {}...".format(name)) logger.debug("Download URL: {}".format(url)) - timeout = (3.5, response_timeout) + settings = Settings() + if response_timeout is None: + timeout = (settings.connection_timeout, settings.response_timeout) + else: + timeout = (settings.connection_timeout, response_timeout) hash = binascii.unhexlify(getUrl(hashurl, timeout, logger)) downloadBinaryFile(url, archive, "sha1", hash, timeout, logger) if command is None: diff --git a/aqt/settings.ini b/aqt/settings.ini index f31a959..99de379 100644 --- a/aqt/settings.ini +++ b/aqt/settings.ini @@ -2,6 +2,11 @@ [aqt] concurrency: 4 +connection_timeout: 3.5 +response_timeout: 30 +baseurl: "https://download.qt.io" +7zcmd: "7z" [mirrors] blacklist: ['http://mirrors.ustc.edu.cn', 'http://mirrors.tuna.tsinghua.edu.cn', 'http://mirrors.geekpie.club'] +fallbacks: ["https://mirrors.ocf.berkeley.edu/qt", "https://ftp.jaist.ac.jp/pub/qtproject", "http://ftp1.nluug.nl/languages/qt", "https://mirrors.dotsrc.org/qtproject"] diff --git a/ci/settings.ini b/ci/settings.ini index 262e7dc..e88efc3 100644 --- a/ci/settings.ini +++ b/ci/settings.ini @@ -2,6 +2,8 @@ [aqt] concurrency: 2 +connection_timeout: 10 +response_timeout: 10 [mirrors] blacklist: ['http://mirrors.ustc.edu.cn', 'http://mirrors.tuna.tsinghua.edu.cn', 'http://mirrors.geekpie.club', 'http://mirrors.sjtug.sjtu.edu.cn'] diff --git a/tests/test_archives.py b/tests/test_archives.py index e30ae24..37a9f96 100644 --- a/tests/test_archives.py +++ b/tests/test_archives.py @@ -2,8 +2,8 @@ import os import pytest +from aqt.helper import Settings from aqt.archives import QtArchives -from aqt.installer import BASE_URL @pytest.mark.parametrize( @@ -21,7 +21,7 @@ def test_parse_update_xml(monkeypatch, os_name, version, target, datafile): monkeypatch.setattr(QtArchives, "_download_update_xml", _mock) - qt_archives = QtArchives(os_name, "desktop", version, target, BASE_URL) + qt_archives = QtArchives(os_name, "desktop", version, target, Settings().baseurl) assert qt_archives.archives is not None # Get packages with all extra modules diff --git a/tests/test_doc_archives.py b/tests/test_doc_archives.py index 1ae1837..e925804 100644 --- a/tests/test_doc_archives.py +++ b/tests/test_doc_archives.py @@ -3,7 +3,7 @@ import os import pytest from aqt.archives import QtArchives -from aqt.installer import BASE_URL +from aqt.helper import Settings @pytest.mark.parametrize( @@ -17,12 +17,13 @@ def test_parse_update_xml(monkeypatch, os_name, version, target, datafile): monkeypatch.setattr(QtArchives, "_download_update_xml", _mock) - qt_archives = QtArchives(os_name, "desktop", version, target, BASE_URL) + settings = Settings() + qt_archives = QtArchives(os_name, "desktop", version, target, settings.baseurl) assert qt_archives.archives is not None # Get packages with all extra modules qt_archives_all_modules = QtArchives( - os_name, "desktop", version, target, BASE_URL, None, ["all"], None, True + os_name, "desktop", version, target, settings.baseurl, None, ["all"], None, True ) assert qt_archives_all_modules.archives is not None