Fix patching of Qt6.2.2-ios (#510)

* Add tests for proper patching of Qt6.2.2-ios
* Add a build target for ios installations
* Add test coverage for mobile patching on linux

This change folds in some refactoring as well, in an attempt to reduce
repetitive logic, and make sure that this logic is used everywhere appropriate.

aqt/updater.py

@@ -24,6 +24,7 @@ import pathlib
 import re
 import subprocess
 from logging import getLogger
+from typing import Union
 
 import patch
 
@@ -33,6 +34,26 @@ from aqt.helper import Settings
 from aqt.metadata import SimpleSpec, Version
 
 
+def default_desktop_arch_dir(host: str, version: Union[Version, str]) -> str:
+    version: Version = version if isinstance(version, Version) else Version(version)
+    if host == "linux":
+        return "gcc_64"
+    elif host == "mac":
+        return "macos" if version in SimpleSpec(">=6.1.2") else "clang_64"
+    else:  # Windows
+        # This is a temporary solution. This arch directory cannot exist for many versions of Qt.
+        # TODO: determine this dynamically
+        return "mingw81_64"
+
+
+def dir_for_version(ver: Version) -> str:
+    return "5.9" if ver == Version("5.9.0") else f"{ver.major}.{ver.minor}.{ver.patch}"
+
+
+def unpatched_path(os_name: str, final_component: str) -> str:
+    return ("/home/qt/work/install" if os_name == "linux" else "/Users/qt/work/install") + "/" + final_component
+
+
 class Updater:
     def __init__(self, prefix: pathlib.Path, logger):
         self.logger = logger
@@ -160,30 +181,14 @@ class Updater:
                 newpath=bytes(str(self.prefix), "UTF-8"),
             )
 
-    def patch_qmake_script(self, base_dir, qt_version, os_name):
+    def patch_qmake_script(self, base_dir, qt_version: str, os_name):
-        if os_name == "linux":
+        arch_dir = default_desktop_arch_dir(os_name, qt_version)
-            self.logger.info("Patching {}/bin/qmake".format(self.prefix))
+        sep = "\\" if os_name == "windows" else "/"
-            self._patch_textfile(
+        patched = sep.join([base_dir, qt_version, arch_dir, "bin"])
-                self.prefix / "bin" / "qmake",
+        unpatched = unpatched_path(os_name, "bin")
-                "/home/qt/work/install/bin",
+        qmake_path = self.prefix / "bin" / ("qmake.bat" if os_name == "windows" else "qmake")
-                "{}/{}/{}/bin".format(base_dir, qt_version, "gcc_64"),
+        self.logger.info(f"Patching {qmake_path}")
-            )
+        self._patch_textfile(qmake_path, unpatched, patched)
-        elif os_name == "mac":
-            self.logger.info("Patching {}/bin/qmake".format(self.prefix))
-            self._patch_textfile(
-                self.prefix / "bin" / "qmake",
-                "/Users/qt/work/install/bin",
-                "{}/{}/{}/bin".format(base_dir, qt_version, "clang_64"),
-            )
-        elif os_name == "windows":
-            self.logger.info("Patching {}/bin/qmake.bat".format(self.prefix))
-            self._patch_textfile(
-                self.prefix / "bin" / "qmake.bat",
-                "/Users/qt/work/install/bin",
-                "{}\\{}\\{}\\bin".format(base_dir, qt_version, "mingw81_64"),
-            )
-        else:
-            pass
 
     def patch_qtcore(self, target):
         """patch to QtCore"""
@@ -235,15 +240,8 @@ class Updater:
 
     def patch_target_qt_conf(self, base_dir, qt_version, arch_dir, os_name):
         target_qt_conf = self.prefix / "bin" / "target_qt.conf"
-        if os_name == "linux":
+        old_targetprefix = f'Prefix={unpatched_path(os_name, "target")}'
-            old_targetprefix = "Prefix=/home/qt/work/install/target"
+        new_hostprefix = f"HostPrefix=../../{default_desktop_arch_dir(os_name, qt_version)}"
-            new_hostprefix = "HostPrefix=../../gcc_64"
-        elif os_name == "mac":
-            old_targetprefix = "Prefix=/Users/qt/work/install/target"
-            new_hostprefix = "HostPrefix=../../clang_64"
-        else:
-            old_targetprefix = "Prefix=/Users/qt/work/install/target"
-            new_hostprefix = "HostPrefix=../../mingw81_64"
         new_targetprefix = "Prefix={}".format(str(pathlib.Path(base_dir).joinpath(qt_version, arch_dir, "target")))
         new_hostdata = "HostData=../{}".format(arch_dir)
         self._patch_textfile(target_qt_conf, old_targetprefix, new_targetprefix)
@@ -260,7 +258,7 @@ class Updater:
         arch = target.arch
         version = Version(target.version)
         os_name = target.os_name
-        version_dir = "5.9" if version == Version("5.9.0") else target.version
+        version_dir = dir_for_version(version)
         if arch is None:
             arch_dir = ""
         elif arch.startswith("win64_mingw"):
@@ -274,8 +272,8 @@ class Updater:
                 arch_dir = b + "_" + a
             else:
                 arch_dir = arch[6:]
-        elif version in SimpleSpec(">=6.1.2") and os_name == "mac" and arch == "clang_64":
+        elif os_name == "mac" and arch == "clang_64":
-            arch_dir = "macos"
+            arch_dir = default_desktop_arch_dir(os_name, version)
         else:
             arch_dir = arch
         try:

ci/generate_azure_pipelines_matrices.py

@@ -269,6 +269,7 @@ mac_build_jobs.append(
 mac_build_jobs.extend(
     [
         BuildJob("install-qt", "5.15.2", "mac", "ios", "ios", "ios"),
+        BuildJob("install-qt", "6.2.2", "mac", "ios", "ios", "ios", module="qtsensors"),
         BuildJob(
             "install-qt", "6.1.0", "mac", "android", "android_armv7", "android_armv7"
         ),

ci/steps.yml

@@ -54,7 +54,8 @@ steps:
           if [[ "$(HOST)" == "windows" ]]; then
             python -m aqt install-qt $(HOST) desktop $(QT_VERSION) mingw81_64 --archives qtbase
           else
-            python -m aqt install-qt $(HOST) desktop $(QT_VERSION) --archives qtbase
+            # qtdeclarative contains `qmlimportscanner`: necessary for ios builds, Qt6+
+            python -m aqt install-qt $(HOST) desktop $(QT_VERSION) --archives qtbase qtdeclarative
           fi
         fi
         if [[ "$(OUTPUT_DIR)" != "" ]]; then
@@ -174,6 +175,21 @@ steps:
     condition: and(eq(variables['TARGET'], 'android'), or(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OS'], 'Darwin')), ne(variables['SUBCOMMAND'], 'list'), ne(variables['SUBCOMMAND'], 'install-tool'))
     displayName: Build accelbubble example application to test for android
 
+  ##----------------------------------------------------
+  # for iOS target
+  - bash: |
+      set -ex
+      mkdir $(Build.BinariesDirectory)/tests
+      (cd $(Build.BinariesDirectory)/tests; 7zr x $(Build.SourcesDirectory)/ci/accelbubble.7z)
+      export PATH=$(QT_BINDIR):$PATH
+      qmake $(Build.BinariesDirectory)/tests/accelbubble
+      make
+    condition: |
+        and(eq(variables['TARGET'], 'ios'), 
+            ne(variables['SUBCOMMAND'], 'list'), 
+            ne(variables['SUBCOMMAND'], 'install-tool'))
+    displayName: Build accelbubble example application to test for ios
+
   ##----------------------------------------------------
   # Cache Powershell Modules in $MODULES_FOLDER
   - task: Cache@2

tests/test_install.py

@@ -574,7 +574,117 @@ def tool_archive(host: str, tool_name: str, variant: str, date: datetime = datet
                 r"^aqtinstall\(aqt\) v.* on Python 3.*\n"
                 r"Downloading qtbase...\n"
                 r"Finished installation of qtbase-windows-android_armv7.7z in .*\n"
-                r"Patching .*/bin/qmake.bat\n"
+                r"Patching .*6\.1\.0[/\\]android_armv7[/\\]bin[/\\]qmake.bat\n"
+                r"Finished installation\n"
+                r"Time elapsed: .* second"
+            ),
+        ),
+        (
+            "install-qt linux android 6.3.0 android_arm64_v8a".split(),
+            "linux",
+            "android",
+            "6.3.0",
+            "android_arm64_v8a",
+            "android_arm64_v8a",
+            "linux_x64/android/qt6_630_arm64_v8a/Updates.xml",
+            [
+                MockArchive(
+                    filename_7z="qtbase-linux-android_arm64_v8a.7z",
+                    update_xml_name="qt.qt6.630.android_arm64_v8a",
+                    contents=(
+                        # Qt 6 non-desktop should patch qconfig.pri, qmake script and target_qt.conf
+                        PatchedFile(
+                            filename="mkspecs/qconfig.pri",
+                            unpatched_content="... blah blah blah ...\n"
+                            "QT_EDITION = Not OpenSource\n"
+                            "QT_LICHECK = Not Empty\n"
+                            "... blah blah blah ...\n",
+                            patched_content="... blah blah blah ...\n"
+                            "QT_EDITION = OpenSource\n"
+                            "QT_LICHECK =\n"
+                            "... blah blah blah ...\n",
+                        ),
+                        PatchedFile(
+                            filename="bin/target_qt.conf",
+                            unpatched_content="Prefix=/home/qt/work/install/target\n"
+                            "HostPrefix=../../\n"
+                            "HostData=target\n",
+                            patched_content="Prefix={base_dir}{sep}6.3.0{sep}android_arm64_v8a{sep}target\n"
+                            "HostPrefix=../../gcc_64\n"
+                            "HostData=../android_arm64_v8a\n",
+                        ),
+                        PatchedFile(
+                            filename="bin/qmake",
+                            unpatched_content="... blah blah blah ...\n"
+                            "/home/qt/work/install/bin\n"
+                            "... blah blah blah ...\n",
+                            patched_content="... blah blah blah ...\n"
+                            "{base_dir}/6.3.0/gcc_64/bin\n"
+                            "... blah blah blah ...\n",
+                        ),
+                    ),
+                ),
+            ],
+            re.compile(
+                r"^aqtinstall\(aqt\) v.* on Python 3.*\n"
+                r"Downloading qtbase...\n"
+                r"Finished installation of qtbase-linux-android_arm64_v8a.7z in .*\n"
+                r"Patching .*6\.3\.0[/\\]android_arm64_v8a[/\\]bin[/\\]qmake\n"
+                r"Finished installation\n"
+                r"Time elapsed: .* second"
+            ),
+        ),
+        (
+            "install-qt mac ios 6.1.2".split(),
+            "mac",
+            "ios",
+            "6.1.2",
+            "ios",
+            "ios",
+            "mac_x64/ios/qt6_612/Updates.xml",
+            [
+                MockArchive(
+                    filename_7z="qtbase-mac-ios.7z",
+                    update_xml_name="qt.qt6.612.ios",
+                    contents=(
+                        # Qt 6 non-desktop should patch qconfig.pri, qmake script and target_qt.conf
+                        PatchedFile(
+                            filename="mkspecs/qconfig.pri",
+                            unpatched_content="... blah blah blah ...\n"
+                            "QT_EDITION = Not OpenSource\n"
+                            "QT_LICHECK = Not Empty\n"
+                            "... blah blah blah ...\n",
+                            patched_content="... blah blah blah ...\n"
+                            "QT_EDITION = OpenSource\n"
+                            "QT_LICHECK =\n"
+                            "... blah blah blah ...\n",
+                        ),
+                        PatchedFile(
+                            filename="bin/target_qt.conf",
+                            unpatched_content="Prefix=/Users/qt/work/install/target\n"
+                            "HostPrefix=../../\n"
+                            "HostData=target\n",
+                            patched_content="Prefix={base_dir}{sep}6.1.2{sep}ios{sep}target\n"
+                            "HostPrefix=../../macos\n"
+                            "HostData=../ios\n",
+                        ),
+                        PatchedFile(
+                            filename="bin/qmake",
+                            unpatched_content="... blah blah blah ...\n"
+                            "/Users/qt/work/install/bin\n"
+                            "... blah blah blah ...\n",
+                            patched_content="... blah blah blah ...\n"
+                            "{base_dir}/6.1.2/macos/bin\n"
+                            "... blah blah blah ...\n",
+                        ),
+                    ),
+                ),
+            ],
+            re.compile(
+                r"^aqtinstall\(aqt\) v.* on Python 3.*\n"
+                r"Downloading qtbase...\n"
+                r"Finished installation of qtbase-mac-ios.7z in .*\n"
+                r"Patching .*6\.1\.2[/\\]ios[/\\]bin[/\\]qmake\n"
                 r"Finished installation\n"
                 r"Time elapsed: .* second"
             ),