Mirrors/aqtinstall
1
0
Fork 0
mirror of https://github.com/miurahr/aqtinstall.git synced 2025-12-17 04:34:37 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #812 from miurahr/topic/miurahr/ci/combination-delete

Browse Source 
fix: delete combination check
This commit is contained in:
Hiroshi Miura
2024-08-17 10:52:55 +09:00
committed by GitHub
parent cc132299ce 253349b133
commit 73fc45e12a
12 changed files with 6 additions and 1725 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
.github/workflows/check-combinations.yml vendored
View File

@@ -1,59 +0,0 @@
name: "Check combinations.json"
on:
schedule:
# Run at midnight on the first of every month
# https://crontab.guru/once-a-month
- cron: "0 0 1 * *"
workflow_dispatch:
push:
paths:
- 'ci/**'
jobs:
check_combinations:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 20
- name: Update the "update-combinations" branch (ff merge to master)
uses: MaximeHeckel/github-action-merge-fast-forward@b4e9b28dce30a682e5fbe3135be4053ea2a75e15
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
branchtomerge: "origin/master"
branch: "update-combinations"
- name: Set up Python 3.9
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Build and install
run: |
python -m pip install ./ --user
- name: Install generate_combinations dependencies
run: pip install -U jsoncomparison
- name: Check combinations.json
run: PYTHONPATH=$(pwd) python3 ci/generate_combinations.py --write --no-tqdm
- name: Commit and make pull request
uses: gr2m/create-or-update-pull-request-action@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
title: "Update `aqt/combinations.json`"
body: |
The `ci/generate_combinations.py` script has detected changes to the repo at https://download.qt.io.
This PR will update `aqt/combinations.json` to account for those changes.
Posted from [the `check_combinations` action](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
branch: "update-combinations"
path: "aqt/combinations.json"
commit-message: "Update aqt/combinations.json"
author: "Qt Repo Watchbot <qtrepowatchbot[bot]@users.noreply.github.com>"

1
MANIFEST.in
View File

@@ -4,7 +4,6 @@ include LICENSE
include pyproject.toml
include .flake8
recursive-include aqt *.ini
recursive-include aqt *.json
recursive-include docs *.bat
recursive-include docs *.py
recursive-include docs *.rst

1
aqt/archives.py
View File

@@ -288,7 +288,6 @@ class QtArchives:
self.arch: str = arch
self.os_name: str = os_name
self.all_extra: bool = all_extra
self.arch_list: List[str] = [item.get("arch") for item in Settings.qt_combinations]
self.base: str = base
self.logger = getLogger("aqt.archives")
self.archives: List[QtPackage] = []

1310
aqt/combinations.json
View File

File diff suppressed because it is too large Load Diff

41
aqt/helper.py
View File

@@ -21,7 +21,6 @@
import binascii
import configparser
import hashlib
import json
import logging.config
import os
import posixpath
@@ -333,7 +332,6 @@ class SettingsClass:
"""
Class to hold configuration and settings.
Actual values are stored in 'settings.ini' file.
It also holds a `combinations` database.
"""
# this class is Borg
@@ -341,7 +339,6 @@ class SettingsClass:
"config": None,
"configfile": None,
"loggingconf": None,
"_combinations": None,
"_lock": threading.Lock(),
}
@@ -359,11 +356,6 @@ class SettingsClass:
self.loggingconf = os.path.join(os.path.dirname(__file__), "logging.ini")
def load_settings(self, file: Optional[Union[str, TextIO]] = None) -> None:
with open(
os.path.join(os.path.dirname(__file__), "combinations.json"),
"r",
) as j:
self._combinations = json.load(j)[0]
if file is not None:
if isinstance(file, str):
result = self.config.read(file)
@@ -379,39 +371,6 @@ class SettingsClass:
with open(self.configfile, "r") as f:
self.config.read_file(f)
@property
def qt_combinations(self):
return self._combinations["qt"]
@property
def tools_combinations(self):
return self._combinations["tools"]
@property
def available_versions(self):
return self._combinations["versions"]
@property
def available_offline_installer_version(self):
res = self._combinations["new_archive"]
res.extend(self._combinations["versions"])
return res
def available_modules(self, qt_version):
"""Known module names
:returns: dictionary of qt_version and module names
:rtype: List[str]
"""
modules = self._combinations["modules"]
versions = qt_version.split(".")
version = "{}.{}".format(versions[0], versions[1])
result = None
for record in modules:
if record["qt_version"] == version:
result = record["modules"]
return result
@property
def archive_download_location(self):
return self.config.get("aqt", "archive_download_location", fallback=".")

51
aqt/installer.py
View File

@@ -200,39 +200,6 @@ class Cli:
)
return Cli.UNHANDLED_EXCEPTION_CODE
def _check_tools_arg_combination(self, os_name, tool_name, arch):
for c in Settings.tools_combinations:
if c["os_name"] == os_name and c["tool_name"] == tool_name and c["arch"] == arch:
return True
return False
def _check_qt_arg_combination(self, qt_version, os_name, target, arch):
for c in Settings.qt_combinations:
if c["os_name"] == os_name and c["target"] == target and c["arch"] == arch:
return True
return False
def _check_qt_arg_versions(self, version):
return version in Settings.available_versions
def _check_qt_arg_version_offline(self, version):
return version in Settings.available_offline_installer_version
def _warning_unknown_qt_version(self, qt_version: str) -> str:
return self._warning_on_bad_combination(f'Qt version "{qt_version}"')
def _warning_unknown_target_arch_combo(self, args: List[str]) -> str:
return self._warning_on_bad_combination(f"target combination \"{' '.join(args)}\"")
def _warning_unexpected_modules(self, unexpected_modules: List[str]) -> str:
return self._warning_on_bad_combination(f"modules {unexpected_modules}")
def _warning_on_bad_combination(self, combo_message: str) -> str:
return (
f"Specified {combo_message} did not exist when this version of aqtinstall was released. "
"This may not install properly, but we will try our best."
)
def _set_sevenzip(self, external: Optional[str]) -> Optional[str]:
sevenzip = external
fallback = Settings.zipcmd
@@ -293,11 +260,6 @@ class Cli:
return False
return True
def _select_unexpected_modules(self, qt_version: str, modules: Optional[List[str]]) -> List[str]:
"""Returns a sorted list of all the requested modules that do not exist in the combinations.json file."""
available = Settings.available_modules(qt_version)
return sorted(set(modules or []) - set(available or []))
@staticmethod
def _determine_qt_version(
qt_version_or_spec: str, host: str, target: str, arch: str, base_url: str = Settings.baseurl
@@ -417,15 +379,7 @@ class Cli:
auto_desktop_archives: List[QtPackage] = get_auto_desktop_archives()
if not self._check_qt_arg_versions(qt_version):
self.logger.warning(self._warning_unknown_qt_version(qt_version))
if not self._check_qt_arg_combination(qt_version, os_name, target, arch):
self.logger.warning(self._warning_unknown_target_arch_combo([os_name, target, arch]))
all_extra = True if modules is not None and "all" in modules else False
if not all_extra:
unexpected_modules = self._select_unexpected_modules(qt_version, modules)
if unexpected_modules:
self.logger.warning(self._warning_unexpected_modules(unexpected_modules))
qt_archives: QtArchives = retry_on_bad_connection(
lambda base_url: QtArchives(
@@ -495,8 +449,6 @@ class Cli:
modules = getattr(args, "modules", None) # `--modules` is invalid for `install-src`
archives = args.archives
all_extra = True if modules is not None and "all" in modules else False
if not self._check_qt_arg_versions(qt_version):
self.logger.warning(self._warning_unknown_qt_version(qt_version))
srcdocexamples_archives: SrcDocExamplesArchives = retry_on_bad_connection(
lambda base_url: SrcDocExamplesArchives(
@@ -589,9 +541,6 @@ class Cli:
archs = [args.tool_variant]
for arch in archs:
if not self._check_tools_arg_combination(os_name, tool_name, arch):
self.logger.warning(self._warning_unknown_target_arch_combo([os_name, tool_name, arch]))
tool_archives: ToolArchives = retry_on_bad_connection(
lambda base_url: ToolArchives(
os_name=os_name,

216
ci/generate_combinations.py
View File

@@ -1,216 +0,0 @@
#!/usr/bin/env python3
import argparse
import json
import logging
import sys
from pathlib import Path
from typing import Dict, Generator, Iterable, List, Optional, Tuple, Union, NamedTuple
from jsoncomparison import NO_DIFF, Compare
from aqt.exceptions import ArchiveConnectionError, ArchiveDownloadError
from aqt.helper import Settings, setup_logging
from aqt.metadata import ArchiveId, MetadataFactory, Versions
def is_blacklisted_tool(tool_name: str) -> bool:
for prefix in ("tools_qt3dstudio_",):
if tool_name.startswith(prefix):
return True
for suffix in ("_preview", "_early_access"):
if tool_name.endswith(suffix):
return True
return False
def iter_archive_ids(
*,
category: str,
hosts: Iterable[str] = ArchiveId.HOSTS,
targets: Optional[Iterable[str]] = None,
) -> Generator[ArchiveId, None, None]:
for host in sorted(hosts):
use_targets = targets
if use_targets is None:
use_targets = ArchiveId.TARGETS_FOR_HOST[host]
for target in use_targets:
yield ArchiveId(category, host, target)
def iter_arches() -> Generator[dict, None, None]:
logger.info("Fetching arches")
archive_ids = list(iter_archive_ids(category="qt"))
for archive_id in tqdm(archive_ids):
for version in ("latest", "6.5.3", "6.2.4", "5.15.2", "5.13.2", "5.9.9"):
if archive_id.target == "winrt" and (version == "latest" or version.startswith("6")):
continue
for arch_name in MetadataFactory(archive_id, architectures_ver=version).getList():
yield {
"os_name": archive_id.host,
"target": archive_id.target,
"arch": arch_name,
}
def iter_tool_variants() -> Generator[dict, None, None]:
for archive_id in iter_archive_ids(category="tools"):
logger.info("Fetching tool variants for {}".format(archive_id))
for tool_name in tqdm(sorted(MetadataFactory(archive_id).fetch_tools())):
if is_blacklisted_tool(tool_name):
continue
for tool_variant in MetadataFactory(
archive_id, tool_name=tool_name
).getList():
yield {
"os_name": archive_id.host,
"target": archive_id.target,
"tool_name": tool_name,
"arch": tool_variant,
}
def iter_qt_minor_groups(
host: str = "linux", target: str = "desktop"
) -> Generator[Tuple[int, int], None, None]:
versions: Versions = MetadataFactory(ArchiveId("qt", host, target)).fetch_versions()
for minor_group in versions:
v = minor_group[0]
yield v.major, v.minor
def iter_modules_for_qt_minor_groups(
host: str = "linux", target: str = "desktop", arch: str = "gcc_64"
) -> Generator[Dict, None, None]:
logger.info("Fetching qt modules for {}/{}".format(host, target))
for major, minor in tqdm(list(iter_qt_minor_groups(host, target))):
use_linux_gcc = (host == "linux" and arch == "gcc_64" and major == 6 and minor >= 7)
use_arch = "linux_gcc_64" if use_linux_gcc else arch
yield {
"qt_version": f"{major}.{minor}",
"modules": MetadataFactory(
ArchiveId("qt", host, target), modules_query=MetadataFactory.ModulesQuery(f"{major}.{minor}.0", use_arch)
).getList(),
}
def list_qt_versions(host: str = "linux", target: str = "desktop") -> List[str]:
all_versions = list()
versions: Versions = MetadataFactory(ArchiveId("qt", host, target)).getList()
for minor_group in versions:
all_versions.extend([str(ver) for ver in minor_group])
return all_versions
def merge_records(arch_records) -> List[Dict]:
all_records: List[Dict] = []
hashes = set()
for record in arch_records:
_hash = record["os_name"], record["target"], record["arch"]
if _hash not in hashes:
all_records.append(record)
hashes.add(_hash)
for sorting_key in ("arch", "target", "os_name"):
all_records = sorted(all_records, key=lambda d: d[sorting_key])
return all_records
def generate_combos(new_archive: List[str]):
return {
"qt": merge_records(iter_arches()),
"tools": list(iter_tool_variants()),
"modules": list(iter_modules_for_qt_minor_groups()),
"versions": list_qt_versions(),
"new_archive": new_archive,
}
def alphabetize_modules(combos: Dict[str, Union[List[Dict], List[str]]]):
for i, item in enumerate(combos["modules"]):
combos["modules"][i]["modules"] = sorted(item["modules"])
def write_combinations_json(
combos: List[Dict[str, Union[List[Dict], List[str]]]],
filename: Path,
):
logger.info(f"Write file {filename}")
json_text = json.dumps(combos, sort_keys=True, indent=2)
if filename.write_text(json_text, encoding="utf_8") == 0:
raise RuntimeError("Failed to write file!")
def main(filename: Path, is_write_file: bool, is_verbose: bool) -> int:
try:
expect = json.loads(filename.read_text())
alphabetize_modules(expect[0])
actual = [generate_combos(new_archive=expect[0]["new_archive"])]
diff = Compare().check(expect, actual)
if is_verbose:
logger.info("=" * 80)
logger.info("Program Output:")
logger.info(json.dumps(actual, sort_keys=True, indent=2))
logger.info("=" * 80)
logger.info(f"Comparison with existing '{filename}':")
logger.info(json.dumps(diff, sort_keys=True, indent=2))
logger.info("=" * 80)
if diff == NO_DIFF:
logger.info(f"{filename} is up to date! No PR is necessary this time!")
return 0 # no difference
if is_write_file:
logger.info(f"{filename} has changed; writing changes to file...")
write_combinations_json(actual, filename)
return 0 # File written successfully
logger.warning(f"{filename} is out of date, but no changes were written")
return 1 # difference reported
except (ArchiveConnectionError, ArchiveDownloadError) as e:
logger.error(format(e))
return 1
def get_tqdm(disable: bool):
if disable:
return lambda x: x
from tqdm import tqdm as base_tqdm
return lambda *a: base_tqdm(*a, disable=disable)
if __name__ == "__main__":
Settings.load_settings()
setup_logging()
logger = logging.getLogger("aqt.generate_combos")
json_filename = Path(__file__).parent.parent / "aqt/combinations.json"
parser = argparse.ArgumentParser(
description="Generate combinations.json from download.qt.io, "
"compare with existing file, and write file to correct differences"
)
parser.add_argument(
"--write",
help="write to combinations.json if changes detected",
action="store_true",
)
parser.add_argument(
"--no-tqdm",
help="disable progress bars (makes CI logs easier to read)",
action="store_true",
)
parser.add_argument(
"--verbose",
help="Print a json dump of the new file, and an abbreviated diff with the old file",
action="store_true",
)
args = parser.parse_args()
tqdm = get_tqdm(args.no_tqdm)
sys.exit(
main(filename=json_filename, is_write_file=args.write, is_verbose=args.verbose)
)

4
docs/CHANGELOG.rst
View File

@@ -13,6 +13,10 @@ All notable changes to this project will be documented in this file.
`Unreleased`_
=============
Deleted
-------
* Qt version/architecture/modules checking (#748, #807, #812)
`v3.1.18`_ (3, August 2024)
===========================

2
pyproject.toml
View File

@@ -40,7 +40,7 @@ packages = ["aqt"]
readme = {file = ["README.rst"]}
[tool.setuptools.package-data]
aqt = ["*.yml", "*.json", "*.ini"]
aqt = ["*.yml", "*.ini"]
[project.scripts]
aqt = "aqt.__main__:main"

34
tests/test_cli.py
View File

@@ -54,40 +54,6 @@ def test_cli_help(capsys):
assert expected_help(out)
@pytest.mark.parametrize(
"qt_version, modules, unexpected_modules",
(
("5.11.3", ["qtcharts", "qtwebengine"], []),
("5.11.3", ["not_exist"], ["not_exist"]),
("5.11.3", ["qtcharts", "qtwebengine", "not_exist"], ["not_exist"]),
("5.11.3", None, []),
("5.15.0", ["Unknown"], ["Unknown"]),
),
)
def test_cli_select_unexpected_modules(qt_version: str, modules: Optional[List[str]], unexpected_modules: List[str]):
cli = Cli()
cli._setup_settings()
assert cli._select_unexpected_modules(qt_version, modules) == unexpected_modules
nonexistent_qt = "5.16.0"
assert cli._select_unexpected_modules(nonexistent_qt, modules) == sorted(modules or [])
def test_cli_check_combination():
cli = Cli()
cli._setup_settings()
assert cli._check_qt_arg_combination("5.11.3", "linux", "desktop", "gcc_64")
assert cli._check_qt_arg_combination("5.11.3", "mac", "desktop", "clang_64")
assert not cli._check_qt_arg_combination("5.14.0", "android", "desktop", "clang_64")
def test_cli_check_version():
cli = Cli()
cli._setup_settings()
assert cli._check_qt_arg_versions("5.12.0")
assert not cli._check_qt_arg_versions("5.12")
@pytest.mark.parametrize(
"host, target, arch, version_or_spec, expected_version, is_bad_spec",
(

10
tests/test_install.py
View File

@@ -1194,8 +1194,6 @@ def test_install(
(
"install-qt windows desktop 5.16.0 win32_mingw73",
None,
'WARNING : Specified Qt version "5.16.0" did not exist when this version of aqtinstall was released. '
"This may not install properly, but we will try our best.\n"
"ERROR : Failed to locate XML data for Qt version '5.16.0'.\n"
"==============================Suggested follow-up:==============================\n"
"* Please use 'aqt list-qt windows desktop' to show versions available.\n",
@@ -1203,8 +1201,6 @@ def test_install(
(
"install-qt windows desktop 5.15.0 bad_arch",
"windows-5150-update.xml",
'WARNING : Specified target combination "windows desktop bad_arch" did not exist when this version of '
"aqtinstall was released. This may not install properly, but we will try our best.\n"
"ERROR : The packages ['qt_base'] were not found while parsing XML of package information!\n"
"==============================Suggested follow-up:==============================\n"
"* Please use 'aqt list-qt windows desktop --arch 5.15.0' to show architectures available.\n",
@@ -1212,8 +1208,6 @@ def test_install(
(
"install-qt windows desktop 5.15.0 win32_mingw73 -m nonexistent foo",
"windows-5150-update.xml",
"WARNING : Specified modules ['foo', 'nonexistent'] did not exist when this version of aqtinstall "
"was released. This may not install properly, but we will try our best.\n"
"ERROR : The packages ['foo', 'nonexistent', 'qt_base'] were not found"
" while parsing XML of package information!\n"
"==============================Suggested follow-up:==============================\n"
@@ -1251,8 +1245,6 @@ def test_install(
(
"install-tool windows desktop tools_vcredist nonexistent",
"windows-desktop-tools_vcredist-update.xml",
'WARNING : Specified target combination "windows tools_vcredist nonexistent" did not exist when this version of '
"aqtinstall was released. This may not install properly, but we will try our best.\n"
"ERROR : The package 'nonexistent' was not found while parsing XML of package information!\n"
"==============================Suggested follow-up:==============================\n"
"* Please use 'aqt list-tool windows desktop tools_vcredist' to show tool variants available.\n",
@@ -1260,8 +1252,6 @@ def test_install(
(
"install-tool windows desktop tools_nonexistent nonexistent",
None,
'WARNING : Specified target combination "windows tools_nonexistent nonexistent" did not exist when this '
"version of aqtinstall was released. This may not install properly, but we will try our best.\n"
"ERROR : Failed to locate XML data for the tool 'tools_nonexistent'.\n"
"==============================Suggested follow-up:==============================\n"
"* Please use 'aqt list-tool windows desktop' to show tools available.\n",

2
tools/build_standalone.py
View File

@@ -24,7 +24,7 @@ if __name__ == "__main__":
adddata_arg = "{src:s};aqt"
else:
adddata_arg = "{src:s}:aqt"
for data in ["aqt/logging.ini", "aqt/settings.ini", "aqt/combinations.json"]:
for data in ["aqt/logging.ini", "aqt/settings.ini"]:
args.append('--add-data')
args.append(adddata_arg.format(src=data))
args.append(os.path.join(tools_dir, "launch_aqt.py"))