[Bug] Fixed AArch64 calling convention handling on Apple platforms

The standard AArch64 calling convention specifies that the minimum
size of each argument passed via stack is 8 bytes. However, Apple
doesn't follow this convention and only aligns stack to 8 bytes
when an argument has 8 bytes or more. This makes it incompatible
with the standard calling convention as two consecutive 32-bit
integers would occupy only 8 bytes instead of 16 when passed via
stack.

src/asmjit/arm/a64func.cpp

@@ -41,6 +41,7 @@ static RegType regTypeFromFpOrVecTypeId(TypeId typeId) noexcept {
 
 ASMJIT_FAVOR_SIZE Error initCallConv(CallConv& cc, CallConvId ccId, const Environment& environment) noexcept {
   cc.setArch(environment.arch());
+  cc.setStrategy(environment.isDarwin() ? CallConvStrategy::kAArch64Apple : CallConvStrategy::kDefault);
 
   cc.setSaveRestoreRegSize(RegGroup::kGp, 8);
   cc.setSaveRestoreRegSize(RegGroup::kVec, 8);
@@ -68,7 +69,7 @@ ASMJIT_FAVOR_SIZE Error initCallConv(CallConv& cc, CallConvId ccId, const Enviro
   return kErrorOk;
 }
 
-ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& signature, uint32_t registerSize) noexcept {
+ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& signature) noexcept {
   DebugUtils::unused(signature);
 
   const CallConv& cc = func.callConv();
@@ -77,6 +78,13 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si
   uint32_t i;
   uint32_t argCount = func.argCount();
 
+  // Minimum stack size of a single argument passed via stack. The standard AArch64 calling convention
+  // specifies 8 bytes, so each function argument would occupy at least 8 bytes even if it needs less.
+  // However, Apple has decided to not follow this rule and function argument can occupy less, for
+  // example two consecutive 32-bit arguments would occupy 8 bytes total, instead of 16 as specified
+  // by ARM.
+  uint32_t minStackArgSize = cc.strategy() == CallConvStrategy::kAArch64Apple ? 4u : 8u;
+
   if (func.hasRet()) {
     for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
       TypeId typeId = func._rets[valueIndex].typeId();
@@ -119,7 +127,8 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si
   }
 
   switch (cc.strategy()) {
-    case CallConvStrategy::kDefault: {
+    case CallConvStrategy::kDefault:
+    case CallConvStrategy::kAArch64Apple: {
       uint32_t gpzPos = 0;
       uint32_t vecPos = 0;
 
@@ -140,7 +149,9 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si
             gpzPos++;
           }
           else {
-            uint32_t size = Support::max<uint32_t>(TypeUtils::sizeOf(typeId), registerSize);
+            uint32_t size = Support::max<uint32_t>(TypeUtils::sizeOf(typeId), minStackArgSize);
+            if (size >= 8)
+              stackOffset = Support::alignUp(stackOffset, 8);
             arg.assignStackOffset(int32_t(stackOffset));
             stackOffset += size;
           }
@@ -164,7 +175,9 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si
             vecPos++;
           }
           else {
-            uint32_t size = TypeUtils::sizeOf(typeId);
+            uint32_t size = Support::max<uint32_t>(TypeUtils::sizeOf(typeId), minStackArgSize);
+            if (size >= 8)
+              stackOffset = Support::alignUp(stackOffset, 8);
             arg.assignStackOffset(int32_t(stackOffset));
             stackOffset += size;
           }
@@ -178,7 +191,7 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si
       return DebugUtils::errored(kErrorInvalidState);
   }
 
-  func._argStackSize = stackOffset;
+  func._argStackSize = Support::alignUp(stackOffset, 8u);
   return kErrorOk;
 }
 

src/asmjit/arm/a64func_p.h

@@ -21,7 +21,7 @@ namespace FuncInternal {
 Error initCallConv(CallConv& cc, CallConvId ccId, const Environment& environment) noexcept;
 
 //! Initialize `FuncDetail` (AArch64 specific).
-Error initFuncDetail(FuncDetail& func, const FuncSignature& signature, uint32_t registerSize) noexcept;
+Error initFuncDetail(FuncDetail& func, const FuncSignature& signature) noexcept;
 
 } // {FuncInternal}
 

src/asmjit/arm/a64rapass.cpp

@@ -612,7 +612,9 @@ void ARMRAPass::onInit() noexcept {
   // make unavailable all registers that are special and cannot be used in general.
   bool hasFP = _func->frame().hasPreservedFP();
 
-  if (hasFP)
+  // Apple ABI requires that the frame-pointer register is not changed by leaf functions and properly updated
+  // by non-leaf functions. So, let's make this register unavailable as it's just not safe to update it.
+  if (hasFP || cc()->environment().isDarwin())
     makeUnavailable(RegGroup::kGp, Gp::kIdFp);
 
   makeUnavailable(RegGroup::kGp, Gp::kIdSp);

src/asmjit/core/environment.h

@@ -126,6 +126,8 @@ enum class PlatformABI : uint8_t {
   kAndroid,
   //! Cygwin ABI.
   kCygwin,
+  //! Darwin ABI.
+  kDarwin,
 
   //! Maximum value of `PlatformABI`.
   kMaxValue,
@@ -142,6 +144,8 @@ enum class PlatformABI : uint8_t {
     kGNU
 #elif defined(__ANDROID__)
     kAndroid
+#elif defined(__APPLE__)
+    kDarwin
 #else
     kUnknown
 #endif
@@ -399,6 +403,8 @@ public:
   ASMJIT_INLINE_NODEBUG bool isMSVC() const noexcept { return _platformABI == PlatformABI::kMSVC; }
   //! Tests whether the ABI is GNU.
   ASMJIT_INLINE_NODEBUG bool isGNU() const noexcept { return _platformABI == PlatformABI::kGNU; }
+  //! Tests whether the ABI is GNU.
+  ASMJIT_INLINE_NODEBUG bool isDarwin() const noexcept { return _platformABI == PlatformABI::kDarwin; }
 
   //! Returns a calculated stack alignment for this environment.
   ASMJIT_API uint32_t stackAlignment() const noexcept;

src/asmjit/core/func.cpp

@@ -75,7 +75,7 @@ ASMJIT_FAVOR_SIZE Error FuncDetail::init(const FuncSignature& signature, const E
 
 #if !defined(ASMJIT_NO_AARCH64)
   if (environment.isFamilyAArch64())
-    return a64::FuncInternal::initFuncDetail(*this, signature, registerSize);
+    return a64::FuncInternal::initFuncDetail(*this, signature);
 #endif
 
   // We should never bubble here as if `cc.init()` succeeded then there has to be an implementation for the current

src/asmjit/core/func.h

@@ -119,6 +119,8 @@ enum class CallConvStrategy : uint8_t {
   kX64Windows = 1,
   //! Windows 64-bit __vectorcall register assignment strategy.
   kX64VectorCall = 2,
+  //! Apple's AArch64 calling convention (differs compared to AArch64 calling convention used by Linux).
+  kAArch64Apple = 3,
 
   //! Maximum value of `CallConvStrategy`.
   kMaxValue = kX64VectorCall

src/asmjit/x86/x86func.cpp

@@ -327,7 +327,8 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si
   }
 
   switch (cc.strategy()) {
-    case CallConvStrategy::kDefault: {
+    case CallConvStrategy::kDefault:
+    default: {
       uint32_t gpzPos = 0;
       uint32_t vecPos = 0;