Patch qmake as finalize process (#130)

* Patch qmake hard coded path with installed prefix(#100)
* AP: Update tests
 - Test android target with 5.14.x
 - Test ios target on mac
* Does not patch mac framework when android, ios and wasm
* Update changelog
* GHA: Update to run qmake to check patch
  - Check qmake works well if patched.

Signed-off-by: Hiroshi Miura <miurahr@linux.com>

.github/workflows/test-install-qt.yml

@@ -1,6 +1,6 @@
 name: Test on GH actions environment
 
-on: [push, pull_request]
+on: push
 
 jobs:
   test:
@@ -9,7 +9,7 @@ jobs:
       matrix:
         os: [windows-latest, macOS-latest, ubuntu-latest]
         py: [3.6, 3.8]
-        qtver: [5.14.2, 5.15.0]
+        qtver: [5.14.1, 5.15.0]
     steps:
       - uses: actions/checkout@v1
         with:
@@ -29,7 +29,6 @@ jobs:
           python -m pip install ./ --user
       - name: Run Aqt
         run: |
-          import os
           import subprocess
           timeout = 300
           command_line = ["python", "-m", "aqt", "install"]
@@ -46,7 +45,34 @@ jobs:
             args = [qtver, "linux", "desktop", "gcc_64"]
           command_line.extend(args)
           try:
-            subprocess.run(command_line, timeout=timeout, check=True)
+            res = subprocess.run(command_line, timeout=timeout, check=True)
           except subprocess.CalledProcessError as cpe:
             exit(cpe.returncode)
+          assert res.returncode == 0
+        shell: python
+      - name: Test qmake -query
+        run: |
+          import pathlib
+          from subprocess import CalledProcessError, PIPE, run
+          platform = "${{ matrix.os }}"
+          qtver = "${{ matrix.qtver }}"
+          if platform == "windows-latest":
+            if qtver.startswith('5.15'):
+              arch_dir = 'msvc2019_64'
+            else:
+              arch_dir = 'msvc2017_64'
+          elif platform == "macOS-latest":
+            arch_dir = 'clang_64'
+          else:
+            arch_dir = 'gcc_64'
+          try:
+            res = run([f"{qtver}/{arch_dir}/bin/qmake", "-query"], timeout=15, check=True, stdout=PIPE)
+          except CalledProcessError as cpe:
+            exit(cpe.returncode)
+          if res.returncode == 0:
+            qt_prefix_path = pathlib.Path.cwd() / qtver / arch_dir
+            for line in res.stdout.splitlines():
+              if line.startswith(b'QT_INSTALL_PREFIX'):
+                result = line[18:].decode('UTF-8')
+                assert qt_prefix_path.samefile(result)
         shell: python

CHANGELOG.rst

@@ -17,6 +17,14 @@ Added
 Changed
 -------
 
+* Patch qmake when finishing installation.(#100)
+  qmake has a hard-coded prefix path, and aqt modify binary in finish phase.
+  it is not necessary for Qt 5.14.2, 5.15.0 and later.
+  This behavior try to be as same as a Qt installer framework doing.
+* Patch Framework.QtCore when finishing installation.(#100)
+  As same as qmake, framework also has a hard-coded prefix path.
+  (Suggestions from @agateau)
+
 Fixed
 -----
 

aqt/archives.py

@@ -34,6 +34,14 @@ class ArchiveDownloadError(Exception):
     pass
 
 
+class TargetConfig:
+    def __init__(self, version, target, arch, os_name):
+        self.version = version
+        self.target = target
+        self.arch = arch
+        self.os_name = os_name
+
+
 class QtPackage:
     """
       Hold package information.
@@ -146,13 +154,13 @@ class QtArchives:
          """
         return self.archives
 
-    def get_target_config(self):
+    def get_target_config(self) -> TargetConfig:
         """Get target configuration
 
         :return: configured target and its version with arch
-        :rtype: tuple(version, target, arch)
+        :rtype: TargetConfig object
         """
-        return self.version, self.target, self.arch
+        return TargetConfig(self.version, self.target, self.arch, self.os_name)
 
 
 class ToolArchives(QtArchives):
@@ -198,9 +206,9 @@ class ToolArchives(QtArchives):
                 self.archives.append(QtPackage(name, package_url, archive, package_desc,
                                                has_mirror=(self.mirror is not None)))
 
-    def get_target_config(self):
+    def get_target_config(self) -> TargetConfig:
         """Get target configuration.
 
         :return tuple of three parameter, "Tools", target and arch
         """
-        return "Tools", self.target, self.arch
+        return TargetConfig("Tools", self.target, self.arch, self.os_name)

aqt/combinations.json

@@ -9,6 +9,7 @@
   {"os_name": "mac",     "target": "desktop", "arch": "clang_64"},
   {"os_name": "mac",     "target": "desktop", "arch": "wasm_32"},
   {"os_name": "mac",     "target": "ios",     "arch": "ios"},
+  {"os_name": "mac",     "target": "android", "arch": "android"},
   {"os_name": "windows", "target": "desktop", "arch": "win64_msvc2019_64"},
   {"os_name": "windows", "target": "desktop", "arch": "win32_msvc2019"},
   {"os_name": "windows", "target": "desktop", "arch": "win64_msvc2017_64"},

aqt/helper.py

@@ -51,3 +51,7 @@ def altlink(url: str, alt: str, logger=None):
             # Return first priority item which is not blacklist in mirrors list,
             # if not found then return alt in default
             return next(filter(lambda mirror: not any(mirror.startswith(b) for b in blacklist), mirrors), alt)
+
+
+def versiontuple(v: str):
+    return tuple(map(int, (v.split("."))))

aqt/installer.py

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2018 Linus Jahn <lnj@kaidan.im>
 # Copyright (C) 2019,2020 Hiroshi Miura <miurahr@linux.com>
+# Copyright (C) 2020, Aurélien Gâteau
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in
@@ -22,6 +23,7 @@
 
 import concurrent.futures
 import os
+import pathlib
 import subprocess
 import sys
 import time
@@ -33,7 +35,8 @@ from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
 from aqt.archives import QtPackage
-from aqt.helper import altlink
+from aqt.helper import altlink, versiontuple
+from aqt.qtpatch import Updater
 from aqt.settings import Settings
 
 
@@ -119,8 +122,20 @@ class QtInstaller:
                 self.logger.error(cpe.stderr)
             raise cpe
 
-    def make_conf_files(self, qt_version, arch_dir):
+    def get_arch_dir(self, arch):
+        if arch.startswith('win64_mingw'):
+            arch_dir = arch[6:] + '_64'
+        elif arch.startswith('win32_mingw'):
+            arch_dir = arch[6:] + '_32'
+        elif arch.startswith('win'):
+            arch_dir = arch[6:]
+        else:
+            arch_dir = arch
+        return arch_dir
+
+    def make_conf_files(self, qt_version, arch):
         """Make Qt configuration files, qt.conf and qtconfig.pri"""
+        arch_dir = self.get_arch_dir(arch)
         try:
             # prepare qt.conf
             with open(os.path.join(self.base_dir, qt_version, arch_dir, 'bin', 'qt.conf'), 'w') as f:
@@ -148,17 +163,14 @@ class QtInstaller:
             if len(not_done) > 0:
                 self.logger.error("Installation error detected.")
                 exit(1)
-
         # finalize
-        qt_version, target, arch = self.qt_archives.get_target_config()
+        target = self.qt_archives.get_target_config()
-        if qt_version != "Tools":  # tools installation
+        if target.version == "Tools":
-            if arch.startswith('win64_mingw'):
+            pass
-                arch_dir = arch[6:] + '_64'
+        else:
-            elif arch.startswith('win32_mingw'):
+            self.make_conf_files(target.version, target.arch)
-                arch_dir = arch[6:] + '_32'
+            prefix = pathlib.Path(self.base_dir) / target.version / target.arch
-            elif arch.startswith('win'):
+            updater = Updater(prefix, self.logger)
-                arch_dir = arch[6:]
+            if versiontuple(target.version) < (5, 14, 2):
-            else:
+                updater.patch_qt(target)
-                arch_dir = arch
-            self.make_conf_files(qt_version, arch_dir)
         self.logger.info("Finished installation")

aqt/qtpatch.py

@@ -0,0 +1,57 @@
+import os
+import pathlib
+import subprocess
+
+
+class Updater:
+
+    def __init__(self, prefix: pathlib.Path, logger):
+        self.logger = logger
+        self.prefix = prefix
+        self.qmake_path = None
+        self.qconfigs = {}
+        self._detect_qmake(prefix)
+
+    def _patch_qtcore(self):
+        framework_dir = self.prefix.joinpath("lib", "QtCore.framework")
+        assert framework_dir.exists(), "Invalid installation prefix"
+        for component in ["QtCore", "QtCore_debug"]:
+            if framework_dir.joinpath(component).exists():
+                qtcore_path = framework_dir.joinpath(component).resolve()
+                self.logger.info("Patching {}".format(qtcore_path))
+                self._patch_file(qtcore_path, bytes(str(self.prefix), "ascii"))
+
+    def _patch_file(self, file: pathlib.Path, newpath: bytes):
+        PREFIX_VAR = b"qt_prfxpath="
+        st = file.stat()
+        data = file.read_bytes()
+        idx = data.find(PREFIX_VAR)
+        if idx > 0:
+            return
+        assert len(newpath) < 256, "Qt Prefix path is too long(255)."
+        data = data[:idx] + PREFIX_VAR + newpath + data[idx + len(newpath):]
+        file.write_bytes(data)
+        os.chmod(str(file), st.st_mode)
+
+    def _detect_qmake(self, prefix):
+        ''' detect Qt configurations from qmake
+        '''
+        for qmake_path in [prefix.joinpath('bin', 'qmake'), prefix.joinpath('bin', 'qmake.exe')]:
+            if qmake_path.exists():
+                result = subprocess.run([str(qmake_path), '-query'], stdout=subprocess.PIPE)
+                if result.returncode == 0:
+                    self.qmake_path = qmake_path
+                    for line in result.stdout.splitlines():
+                        vals = line.decode('UTF-8').split(':')
+                        self.qconfigs[vals[0]] = vals[1]
+                break
+
+    def patch_qt(self, target):
+        ''' patch works '''
+        self.logger.info("Patching qmake")
+        mac_exceptions = ['ios', 'android', 'wasm_32',
+                          'android_x86_64', 'android_arm64_v8a', 'android_x86', 'android_armv7']
+        if target.os_name == 'mac' and target.arch not in mac_exceptions:
+            self._patch_qtcore()
+        if self.qmake_path is not None:
+            self._patch_file(self.qmake_path, bytes(str(self.prefix), 'UTF-8'))

ci/generate_azure_pipelines_matrices.py

@@ -53,9 +53,12 @@ for qt_version in qt_versions:
         BuildJob(qt_version, 'mac', 'desktop', 'clang_64', "clang_64")
     )
 
-# Mac iOS
+# Mac iOS, android
-mac_build_jobs.append(
+mac_build_jobs.extend(
-    BuildJob('5.13.2', 'mac', 'ios', 'ios', 'ios')
+    [
+        BuildJob('5.13.2', 'mac', 'ios', 'ios', 'ios'),
+        BuildJob('5.14.1', 'mac', 'android', 'android', 'android')
+    ]
 )
 
 # Windows Desktop
@@ -93,15 +96,10 @@ windows_build_jobs.append(
     BuildJob('5.14.2', 'windows', 'desktop', 'wasm_32', "wasm_32")
 )
 
-# Androids for Linux platforms
+# android
-# aqt is for CI/CD systems!
+linux_build_jobs.append(
-# Users might develop on Win/Mac, but are most likely to use Linux for CI/CD with
+    BuildJob('5.14.1', 'linux', 'android', 'android', 'android')
-# the Android ecosystem.
+)
-
-for android_arch in ['android_x86_64', 'android_arm64_v8a', 'android_x86', 'android_armv7']:
-    linux_build_jobs.append(
-        BuildJob('5.13.2', 'linux', 'android', android_arch, android_arch)
-    )
 
 matrices = {}