mirror of
https://github.com/miurahr/aqtinstall.git
synced 2025-12-16 20:27:05 +03:00
301ff5173b
* fix generate_combinations issue with Linux Qt 6.7 Qt 6.7.0 changes the default arch for Linux from gcc_64 to linux_gcc_64. This allows the generate_combinations.py script to account for that. Also, Qt 6.7.0 is missing a few architectures that were present in earlier versions of Qt6. This makes sure that the script can pick those up. * black (v 24.1.1) New version of black formats metadata.py differently than before * Fix broken generate_combinations.py script Versions 6.5 and 6.2 must be complete versions Don't try to fetch Qt6 for target==winrt; it doesn't exist
217 lines
7.3 KiB
Python
217 lines
7.3 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Dict, Generator, Iterable, List, Optional, Tuple, Union, NamedTuple
|
|
|
|
from jsoncomparison import NO_DIFF, Compare
|
|
|
|
from aqt.exceptions import ArchiveConnectionError, ArchiveDownloadError
|
|
from aqt.helper import Settings, setup_logging
|
|
from aqt.metadata import ArchiveId, MetadataFactory, Versions
|
|
|
|
|
|
def is_blacklisted_tool(tool_name: str) -> bool:
|
|
for prefix in ("tools_qt3dstudio_",):
|
|
if tool_name.startswith(prefix):
|
|
return True
|
|
for suffix in ("_preview", "_early_access"):
|
|
if tool_name.endswith(suffix):
|
|
return True
|
|
return False
|
|
|
|
|
|
def iter_archive_ids(
|
|
*,
|
|
category: str,
|
|
hosts: Iterable[str] = ArchiveId.HOSTS,
|
|
targets: Optional[Iterable[str]] = None,
|
|
) -> Generator[ArchiveId, None, None]:
|
|
for host in sorted(hosts):
|
|
use_targets = targets
|
|
if use_targets is None:
|
|
use_targets = ArchiveId.TARGETS_FOR_HOST[host]
|
|
for target in use_targets:
|
|
yield ArchiveId(category, host, target)
|
|
|
|
|
|
def iter_arches() -> Generator[dict, None, None]:
|
|
logger.info("Fetching arches")
|
|
archive_ids = list(iter_archive_ids(category="qt"))
|
|
for archive_id in tqdm(archive_ids):
|
|
for version in ("latest", "6.5.3", "6.2.4", "5.15.2", "5.13.2", "5.9.9"):
|
|
if archive_id.target == "winrt" and (version == "latest" or version.startswith("6")):
|
|
continue
|
|
for arch_name in MetadataFactory(archive_id, architectures_ver=version).getList():
|
|
yield {
|
|
"os_name": archive_id.host,
|
|
"target": archive_id.target,
|
|
"arch": arch_name,
|
|
}
|
|
|
|
|
|
def iter_tool_variants() -> Generator[dict, None, None]:
|
|
for archive_id in iter_archive_ids(category="tools"):
|
|
logger.info("Fetching tool variants for {}".format(archive_id))
|
|
for tool_name in tqdm(sorted(MetadataFactory(archive_id).fetch_tools())):
|
|
if is_blacklisted_tool(tool_name):
|
|
continue
|
|
for tool_variant in MetadataFactory(
|
|
archive_id, tool_name=tool_name
|
|
).getList():
|
|
yield {
|
|
"os_name": archive_id.host,
|
|
"target": archive_id.target,
|
|
"tool_name": tool_name,
|
|
"arch": tool_variant,
|
|
}
|
|
|
|
|
|
def iter_qt_minor_groups(
|
|
host: str = "linux", target: str = "desktop"
|
|
) -> Generator[Tuple[int, int], None, None]:
|
|
versions: Versions = MetadataFactory(ArchiveId("qt", host, target)).fetch_versions()
|
|
for minor_group in versions:
|
|
v = minor_group[0]
|
|
yield v.major, v.minor
|
|
|
|
|
|
def iter_modules_for_qt_minor_groups(
|
|
host: str = "linux", target: str = "desktop", arch: str = "gcc_64"
|
|
) -> Generator[Dict, None, None]:
|
|
logger.info("Fetching qt modules for {}/{}".format(host, target))
|
|
for major, minor in tqdm(list(iter_qt_minor_groups(host, target))):
|
|
use_linux_gcc = (host == "linux" and arch == "gcc_64" and major == 6 and minor >= 7)
|
|
use_arch = "linux_gcc_64" if use_linux_gcc else arch
|
|
yield {
|
|
"qt_version": f"{major}.{minor}",
|
|
"modules": MetadataFactory(
|
|
ArchiveId("qt", host, target), modules_query=MetadataFactory.ModulesQuery(f"{major}.{minor}.0", use_arch)
|
|
).getList(),
|
|
}
|
|
|
|
|
|
def list_qt_versions(host: str = "linux", target: str = "desktop") -> List[str]:
|
|
all_versions = list()
|
|
versions: Versions = MetadataFactory(ArchiveId("qt", host, target)).getList()
|
|
for minor_group in versions:
|
|
all_versions.extend([str(ver) for ver in minor_group])
|
|
return all_versions
|
|
|
|
|
|
def merge_records(arch_records) -> List[Dict]:
|
|
all_records: List[Dict] = []
|
|
hashes = set()
|
|
for record in arch_records:
|
|
_hash = record["os_name"], record["target"], record["arch"]
|
|
if _hash not in hashes:
|
|
all_records.append(record)
|
|
hashes.add(_hash)
|
|
for sorting_key in ("arch", "target", "os_name"):
|
|
all_records = sorted(all_records, key=lambda d: d[sorting_key])
|
|
return all_records
|
|
|
|
|
|
def generate_combos(new_archive: List[str]):
|
|
return {
|
|
"qt": merge_records(iter_arches()),
|
|
"tools": list(iter_tool_variants()),
|
|
"modules": list(iter_modules_for_qt_minor_groups()),
|
|
"versions": list_qt_versions(),
|
|
"new_archive": new_archive,
|
|
}
|
|
|
|
|
|
def alphabetize_modules(combos: Dict[str, Union[List[Dict], List[str]]]):
|
|
for i, item in enumerate(combos["modules"]):
|
|
combos["modules"][i]["modules"] = sorted(item["modules"])
|
|
|
|
|
|
def write_combinations_json(
|
|
combos: List[Dict[str, Union[List[Dict], List[str]]]],
|
|
filename: Path,
|
|
):
|
|
logger.info(f"Write file {filename}")
|
|
json_text = json.dumps(combos, sort_keys=True, indent=2)
|
|
if filename.write_text(json_text, encoding="utf_8") == 0:
|
|
raise RuntimeError("Failed to write file!")
|
|
|
|
|
|
def main(filename: Path, is_write_file: bool, is_verbose: bool) -> int:
|
|
try:
|
|
expect = json.loads(filename.read_text())
|
|
alphabetize_modules(expect[0])
|
|
actual = [generate_combos(new_archive=expect[0]["new_archive"])]
|
|
diff = Compare().check(expect, actual)
|
|
|
|
if is_verbose:
|
|
logger.info("=" * 80)
|
|
logger.info("Program Output:")
|
|
logger.info(json.dumps(actual, sort_keys=True, indent=2))
|
|
|
|
logger.info("=" * 80)
|
|
logger.info(f"Comparison with existing '{filename}':")
|
|
logger.info(json.dumps(diff, sort_keys=True, indent=2))
|
|
logger.info("=" * 80)
|
|
|
|
if diff == NO_DIFF:
|
|
logger.info(f"{filename} is up to date! No PR is necessary this time!")
|
|
return 0 # no difference
|
|
if is_write_file:
|
|
logger.info(f"{filename} has changed; writing changes to file...")
|
|
write_combinations_json(actual, filename)
|
|
return 0 # File written successfully
|
|
logger.warning(f"{filename} is out of date, but no changes were written")
|
|
return 1 # difference reported
|
|
|
|
except (ArchiveConnectionError, ArchiveDownloadError) as e:
|
|
logger.error(format(e))
|
|
return 1
|
|
|
|
|
|
def get_tqdm(disable: bool):
|
|
if disable:
|
|
return lambda x: x
|
|
|
|
from tqdm import tqdm as base_tqdm
|
|
|
|
return lambda *a: base_tqdm(*a, disable=disable)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
Settings.load_settings()
|
|
setup_logging()
|
|
logger = logging.getLogger("aqt.generate_combos")
|
|
|
|
json_filename = Path(__file__).parent.parent / "aqt/combinations.json"
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description="Generate combinations.json from download.qt.io, "
|
|
"compare with existing file, and write file to correct differences"
|
|
)
|
|
parser.add_argument(
|
|
"--write",
|
|
help="write to combinations.json if changes detected",
|
|
action="store_true",
|
|
)
|
|
parser.add_argument(
|
|
"--no-tqdm",
|
|
help="disable progress bars (makes CI logs easier to read)",
|
|
action="store_true",
|
|
)
|
|
parser.add_argument(
|
|
"--verbose",
|
|
help="Print a json dump of the new file, and an abbreviated diff with the old file",
|
|
action="store_true",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
tqdm = get_tqdm(args.no_tqdm)
|
|
|
|
sys.exit(
|
|
main(filename=json_filename, is_write_file=args.write, is_verbose=args.verbose)
|
|
)
|