diff --git a/src/asmjit/base/assembler.cpp b/src/asmjit/base/assembler.cpp index 014885a..d4a6162 100644 --- a/src/asmjit/base/assembler.cpp +++ b/src/asmjit/base/assembler.cpp @@ -27,7 +27,9 @@ Assembler::Assembler() noexcept _section(nullptr), _bufferData(nullptr), _bufferEnd(nullptr), - _bufferPtr(nullptr) {} + _bufferPtr(nullptr), + _op4(), + _op5() {} Assembler::~Assembler() noexcept { if (_code) sync(); @@ -45,6 +47,10 @@ Error Assembler::onAttach(CodeHolder* code) noexcept { _bufferData = p; _bufferEnd = p + _section->_buffer._capacity; _bufferPtr = p + _section->_buffer._length; + + _op4.reset(); + _op5.reset(); + return Base::onAttach(code); } @@ -53,9 +59,50 @@ Error Assembler::onDetach(CodeHolder* code) noexcept { _bufferData = nullptr; _bufferEnd = nullptr; _bufferPtr = nullptr; + + _op4.reset(); + _op5.reset(); + return Base::onDetach(code); } +// ============================================================================ +// [asmjit::Assembler - Code-Generation] +// ============================================================================ + +Error Assembler::_emit(uint32_t instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3, const Operand_& o4, const Operand_& o5) { + _op4 = o4; + _op5 = o5; + _options |= kOptionOp4Op5Used; + return _emit(instId, o0, o1, o2, o3); +} + +Error Assembler::_emitOpArray(uint32_t instId, const Operand_* opArray, size_t opCount) { + const Operand_* op = opArray; + switch (opCount) { + case 0: return _emit(instId, _none, _none, _none, _none); + case 1: return _emit(instId, op[0], _none, _none, _none); + case 2: return _emit(instId, op[0], op[1], _none, _none); + case 3: return _emit(instId, op[0], op[1], op[2], _none); + case 4: return _emit(instId, op[0], op[1], op[2], op[3]); + + case 5: + _op4 = op[4]; + _op5.reset(); + _options |= kOptionOp4Op5Used; + return _emit(instId, op[0], op[1], op[2], op[3]); + + case 6: + _op4 = op[4]; + _op5 = op[5]; + _options |= kOptionOp4Op5Used; + return _emit(instId, op[0], op[1], op[2], op[3]); + + default: + return DebugUtils::errored(kErrorInvalidArgument); + } +} + // ============================================================================ // [asmjit::Assembler - Sync] // ============================================================================ @@ -333,15 +380,20 @@ void Assembler::_emitLog( opArray[1].copyFrom(o1); opArray[2].copyFrom(o2); opArray[3].copyFrom(o3); - opArray[4].copyFrom(_op4); - opArray[5].copyFrom(_op5); - if (!(options & CodeEmitter::kOptionOp4)) opArray[4].reset(); - if (!(options & CodeEmitter::kOptionOp5)) opArray[5].reset(); + + if (options & kOptionOp4Op5Used) { + opArray[4].copyFrom(_op4); + opArray[5].copyFrom(_op5); + } + else { + opArray[4].reset(); + opArray[5].reset(); + } Logging::formatInstruction( sb, logOptions, this, getArchType(), - instId, options, _opExtra, opArray, 6); + instId, options, _extraOp, opArray, 6); if ((logOptions & Logger::kOptionBinaryForm) != 0) Logging::formatLine(sb, _bufferPtr, emittedSize, relSize, imLen, getInlineComment()); @@ -364,18 +416,23 @@ Error Assembler::_emitFailed( opArray[1].copyFrom(o1); opArray[2].copyFrom(o2); opArray[3].copyFrom(o3); - opArray[4].copyFrom(_op4); - opArray[5].copyFrom(_op5); - if (!(options & CodeEmitter::kOptionOp4)) opArray[4].reset(); - if (!(options & CodeEmitter::kOptionOp5)) opArray[5].reset(); + if (options & kOptionOp4Op5Used) { + opArray[4].copyFrom(_op4); + opArray[5].copyFrom(_op5); + } + else { + opArray[4].reset(); + opArray[5].reset(); + } Logging::formatInstruction( sb, 0, this, getArchType(), - instId, options, _opExtra, opArray, 6); + instId, options, _extraOp, opArray, 6); resetOptions(); + resetExtraOp(); resetInlineComment(); return setLastError(err, sb.getData()); } diff --git a/src/asmjit/base/assembler.h b/src/asmjit/base/assembler.h index f48c150..55fbb14 100644 --- a/src/asmjit/base/assembler.h +++ b/src/asmjit/base/assembler.h @@ -53,6 +53,15 @@ public: ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + // -------------------------------------------------------------------------- + // [Code-Generation] + // -------------------------------------------------------------------------- + + using CodeEmitter::_emit; + + ASMJIT_API Error _emit(uint32_t instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3, const Operand_& o4, const Operand_& o5) override; + ASMJIT_API Error _emitOpArray(uint32_t instId, const Operand_* opArray, size_t opCount) override; + // -------------------------------------------------------------------------- // [Code-Buffer] // -------------------------------------------------------------------------- @@ -129,6 +138,9 @@ public: uint8_t* _bufferData; //!< Start of the CodeBuffer of the current section. uint8_t* _bufferEnd; //!< End (first invalid byte) of the current section. uint8_t* _bufferPtr; //!< Pointer in the CodeBuffer of the current section. + + Operand_ _op4; //!< 5th operand data, used only temporarily. + Operand_ _op5; //!< 6th operand data, used only temporarily. }; //! \} diff --git a/src/asmjit/base/codebuilder.cpp b/src/asmjit/base/codebuilder.cpp index 4bc726f..2f92dc8 100644 --- a/src/asmjit/base/codebuilder.cpp +++ b/src/asmjit/base/codebuilder.cpp @@ -542,31 +542,10 @@ Error CodeBuilder::serialize(CodeEmitter* dst) { case CBNode::kNodeInst: case CBNode::kNodeFuncCall: { - CBInst* node = static_cast(node_); - - uint32_t instId = node->getInstId(); - uint32_t options = node->getOptions(); - - const Operand* opArray = node->getOpArray(); - uint32_t opCount = node->getOpCount(); - - const Operand_* o0 = &dst->_none; - const Operand_* o1 = &dst->_none; - const Operand_* o2 = &dst->_none; - const Operand_* o3 = &dst->_none; - - switch (opCount) { - case 6: dst->_op5 = opArray[5]; options |= CodeEmitter::kOptionOp5; ASMJIT_FALLTHROUGH; - case 5: dst->_op4 = opArray[4]; options |= CodeEmitter::kOptionOp4; ASMJIT_FALLTHROUGH; - case 4: o3 = &opArray[3]; ASMJIT_FALLTHROUGH; - case 3: o2 = &opArray[2]; ASMJIT_FALLTHROUGH; - case 2: o1 = &opArray[1]; ASMJIT_FALLTHROUGH; - case 1: o0 = &opArray[0]; ASMJIT_FALLTHROUGH; - case 0: break; - } - - dst->setOptions(options); - err = dst->_emit(instId, *o0, *o1, *o2, *o3); + CBInst* node = node_->as(); + dst->setOptions(node->getOptions()); + dst->setExtraOp(node->getExtraOp()); + err = dst->emitOpArray(node->getInstId(), node->getOpArray(), node->getOpCount()); break; } diff --git a/src/asmjit/base/codebuilder.h b/src/asmjit/base/codebuilder.h index 365070a..fea0379 100644 --- a/src/asmjit/base/codebuilder.h +++ b/src/asmjit/base/codebuilder.h @@ -510,12 +510,14 @@ public: //! Clear emit options. ASMJIT_INLINE void delOptions(uint32_t options) noexcept { _options &= ~options; } - //! Get op-mask operand (used to represent AVX-512 op-mask selector). - ASMJIT_INLINE Operand& getOpExtra() noexcept { return _opExtra; } + //! Get if the node has extra operand. + ASMJIT_INLINE bool hasExtraOp() const noexcept { return !_extraOp.isNone(); } + //! Get extra operand operand. + ASMJIT_INLINE Operand& getExtraOp() noexcept { return _extraOp; } //! \overload - ASMJIT_INLINE const Operand& getOpExtra() const noexcept { return _opExtra; } - //1 Set op-mask operand. - ASMJIT_INLINE void setOpExtra(const Operand& opExtra) noexcept { _opExtra = opExtra; } + ASMJIT_INLINE const Operand& getExtraOp() const noexcept { return _extraOp; } + //! Set extra operand. + ASMJIT_INLINE void setExtraOp(const Operand& extraOp) noexcept { _extraOp = extraOp; } //! Get operands count. ASMJIT_INLINE uint32_t getOpCount() const noexcept { return _opCount; } @@ -572,7 +574,7 @@ Update: uint8_t _memOpIndex; //!< \internal uint8_t _reserved; //!< \internal uint32_t _options; //!< Instruction options. - Operand _opExtra; //!< Extra operand (op-mask {k} on AVX-512). + Operand _extraOp; //!< Extra operand (REP {cx} or op-mask {k} on AVX-512). Operand* _opArray; //!< Instruction operands. }; @@ -583,7 +585,6 @@ Update: struct CBInstEx : public CBInst { Operand _op4; Operand _op5; - Operand _opExtra; }; // ============================================================================ diff --git a/src/asmjit/base/codeemitter.cpp b/src/asmjit/base/codeemitter.cpp index 6bfaad7..c4bdfd6 100644 --- a/src/asmjit/base/codeemitter.cpp +++ b/src/asmjit/base/codeemitter.cpp @@ -43,9 +43,7 @@ CodeEmitter::CodeEmitter(uint32_t type) noexcept _globalOptions(kOptionMaybeFailureCase), _options(0), _inlineComment(nullptr), - _op4(), - _op5(), - _opExtra(), + _extraOp(), _none(), _nativeGpReg(), _nativeGpArray(nullptr) {} @@ -82,15 +80,34 @@ Error CodeEmitter::onDetach(CodeHolder* code) noexcept { _options = 0; _inlineComment = nullptr; - _op4.reset(); - _op5.reset(); - _opExtra.reset(); + + _extraOp.reset(); _nativeGpReg.reset(); _nativeGpArray = nullptr; return kErrorOk; } +// ============================================================================ +// [asmjit::CodeEmitter - Code-Generation] +// ============================================================================ + +Error CodeEmitter::_emitOpArray(uint32_t instId, const Operand_* opArray, size_t opCount) { + const Operand_* op = opArray; + switch (opCount) { + case 0: return _emit(instId, _none, _none, _none, _none); + case 1: return _emit(instId, op[0], _none, _none, _none); + case 2: return _emit(instId, op[0], op[1], _none, _none); + case 3: return _emit(instId, op[0], op[1], op[2], _none); + case 4: return _emit(instId, op[0], op[1], op[2], op[3]); + case 5: return _emit(instId, op[0], op[1], op[2], op[3], op[4], _none); + case 6: return _emit(instId, op[0], op[1], op[2], op[3], op[4], op[5]); + + default: + return DebugUtils::errored(kErrorInvalidArgument); + } +} + // ============================================================================ // [asmjit::CodeEmitter - Finalize] // ============================================================================ @@ -187,99 +204,49 @@ Error CodeEmitter::commentv(const char* fmt, va_list ap) { // ============================================================================ #define OP const Operand_& -#define NO _none -Error CodeEmitter::emit(uint32_t instId) { return _emit(instId, NO, NO, NO, NO); } -Error CodeEmitter::emit(uint32_t instId, OP o0) { return _emit(instId, o0, NO, NO, NO); } -Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1) { return _emit(instId, o0, o1, NO, NO); } -Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2) { return _emit(instId, o0, o1, o2, NO); } +Error CodeEmitter::emit(uint32_t instId) { return _emit(instId, _none, _none, _none, _none); } +Error CodeEmitter::emit(uint32_t instId, OP o0) { return _emit(instId, o0, _none, _none, _none); } +Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1) { return _emit(instId, o0, o1, _none, _none); } +Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2) { return _emit(instId, o0, o1, o2, _none); } Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2, OP o3) { return _emit(instId, o0, o1, o2, o3); } +Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2, OP o3, OP o4) { return _emit(instId, o0, o1, o2, o3, o4, _none); } +Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2, OP o3, OP o4, OP o5) { return _emit(instId, o0, o1, o2, o3, o4, o5); } -Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2, OP o3, OP o4) { - _op4 = o4; - - if (!o4.isNone()) _options |= kOptionOp4; - return _emit(instId, o0, o1, o2, o3); -} - -Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2, OP o3, OP o4, OP o5) { - _op4 = o4; - _op5 = o5; - - if (!o4.isNone()) _options |= kOptionOp4; - if (!o5.isNone()) _options |= kOptionOp5; - return _emit(instId, o0, o1, o2, o3); -} - -Error CodeEmitter::emit(uint32_t instId, int o0) { return _emit(instId, Imm(o0), NO, NO, NO); } -Error CodeEmitter::emit(uint32_t instId, OP o0, int o1) { return _emit(instId, o0, Imm(o1), NO, NO); } -Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, int o2) { return _emit(instId, o0, o1, Imm(o2), NO); } +Error CodeEmitter::emit(uint32_t instId, int o0) { return _emit(instId, Imm(o0), _none, _none, _none); } +Error CodeEmitter::emit(uint32_t instId, OP o0, int o1) { return _emit(instId, o0, Imm(o1), _none, _none); } +Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, int o2) { return _emit(instId, o0, o1, Imm(o2), _none); } Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2, int o3) { return _emit(instId, o0, o1, o2, Imm(o3)); } +Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2, OP o3, int o4) { return _emit(instId, o0, o1, o2, o3, Imm(o4), _none); } +Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2, OP o3, OP o4, int o5) { return _emit(instId, o0, o1, o2, o3, o4, Imm(o5)); } -Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2, OP o3, int o4) { - _options |= kOptionOp4; - _op4 = Imm(o4); - return _emit(instId, o0, o1, o2, o3); -} - -Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2, OP o3, OP o4, int o5) { - _op4 = o4; - _op5 = Imm(o5); - - _options |= kOptionOp4 | kOptionOp5; - return _emit(instId, o0, o1, o2, o3); -} - -Error CodeEmitter::emit(uint32_t instId, int64_t o0) { return _emit(instId, Imm(o0), NO, NO, NO); } -Error CodeEmitter::emit(uint32_t instId, OP o0, int64_t o1) { return _emit(instId, o0, Imm(o1), NO, NO); } -Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, int64_t o2) { return _emit(instId, o0, o1, Imm(o2), NO); } +Error CodeEmitter::emit(uint32_t instId, int64_t o0) { return _emit(instId, Imm(o0), _none, _none, _none); } +Error CodeEmitter::emit(uint32_t instId, OP o0, int64_t o1) { return _emit(instId, o0, Imm(o1), _none, _none); } +Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, int64_t o2) { return _emit(instId, o0, o1, Imm(o2), _none); } Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2, int64_t o3) { return _emit(instId, o0, o1, o2, Imm(o3)); } -Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2, OP o3, int64_t o4) { - _options |= kOptionOp4; - _op4 = Imm(o4); - return _emit(instId, o0, o1, o2, o3); -} +Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2, OP o3, int64_t o4) { return _emit(instId, o0, o1, o2, o3, Imm(o4), _none); } +Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2, OP o3, OP o4, int64_t o5) { return _emit(instId, o0, o1, o2, o3, o4, Imm(o5)); } -Error CodeEmitter::emit(uint32_t instId, OP o0, OP o1, OP o2, OP o3, OP o4, int64_t o5) { - _op4 = o4; - _op5 = Imm(o5); - - _options |= kOptionOp4 | kOptionOp5; - return _emit(instId, o0, o1, o2, o3); -} - -#undef NO #undef OP // ============================================================================ // [asmjit::CodeEmitter - Validation] // ============================================================================ -Error CodeEmitter::_validate(uint32_t instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3) const noexcept { +Error CodeEmitter::_validate(uint32_t instId, const Operand_* opArray, uint32_t opCount) const noexcept { #if !defined(ASMJIT_DISABLE_VALIDATION) - Operand_ opArray[6]; - opArray[0].copyFrom(o0); - opArray[1].copyFrom(o1); - opArray[2].copyFrom(o2); - opArray[3].copyFrom(o3); - opArray[4].copyFrom(_op4); - opArray[5].copyFrom(_op5); - uint32_t archType = getArchType(); uint32_t options = getGlobalOptions() | getOptions(); - if (!(options & CodeEmitter::kOptionOp4)) opArray[4].reset(); - if (!(options & CodeEmitter::kOptionOp5)) opArray[5].reset(); - #if defined(ASMJIT_BUILD_X86) if (ArchInfo::isX86Family(archType)) - return X86Inst::validate(archType, instId, options, _opExtra, opArray, 6); + return X86Inst::validate(archType, instId, options, _extraOp, opArray, 6); #endif #if defined(ASMJIT_BUILD_ARM) if (ArchInfo::isArmFamily(archType)) - return ArmInst::validate(archType, instId, options, _opExtra, opArray, 6); + return ArmInst::validate(archType, instId, options, _extraOp, opArray, 6); #endif return DebugUtils::errored(kErrorInvalidArch); diff --git a/src/asmjit/base/codeemitter.h b/src/asmjit/base/codeemitter.h index fab60ea..0087a82 100644 --- a/src/asmjit/base/codeemitter.h +++ b/src/asmjit/base/codeemitter.h @@ -101,15 +101,11 @@ public: //! NOTE: Reserved options should never appear in `CBInst` options. kOptionReservedMask = 0x00000007U, - //! Instruction has `_op4` (5th operand, indexed from zero). - kOptionOp4 = 0x0000008U, - //! Instruction has `_op5` (6th operand, indexed from zero). - kOptionOp5 = 0x0000010U, - //! Instruction has `_opExtra` operand (mask-op {k} operand when using AVX-512). - kOptionOpExtra = 0x00000020U, + //! Used only by Assembler to mark `_op4` and `_op5` are used. + kOptionOp4Op5Used = 0x00000008U, //! Prevents following a jump during compilation (CodeCompiler). - kOptionUnfollow = 0x00000040U, + kOptionUnfollow = 0x00000010U, //! Overwrite the destination operand (CodeCompiler). //! @@ -149,7 +145,7 @@ public: //! //! - `sqrtss x, y` - only LO element of `x` is changed, if you don't use //! HI elements, use `X86Compiler.overwrite().sqrtss(x, y)`. - kOptionOverwrite = 0x00000080U + kOptionOverwrite = 0x00000020U }; // -------------------------------------------------------------------------- @@ -172,8 +168,12 @@ public: // [Code-Generation] // -------------------------------------------------------------------------- - //! Emit instruction. + //! Emit instruction having max 4 operands. virtual Error _emit(uint32_t instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3) = 0; + //! Emit instruction having max 6 operands. + virtual Error _emit(uint32_t instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3, const Operand_& o4, const Operand_& o5) = 0; + //! Emit instruction having operands stored in array. + virtual Error _emitOpArray(uint32_t instId, const Operand_* opArray, size_t opCount); //! Create a new label. virtual Label newLabel() = 0; @@ -315,20 +315,12 @@ public: //! Reset options of the next instruction. ASMJIT_INLINE void resetOptions() noexcept { _options = 0; } - //! Get if the 5th operand (indexed from zero) of the next instruction is used. - ASMJIT_INLINE bool hasOp4() const noexcept { return (_options & kOptionOp4) != 0; } - //! Get if the 6th operand (indexed from zero) of the next instruction is used. - ASMJIT_INLINE bool hasOp5() const noexcept { return (_options & kOptionOp5) != 0; } - //! Get if the op-mask operand of the next instruction is used. - ASMJIT_INLINE bool hasOpExtra() const noexcept { return (_options & kOptionOpExtra) != 0; } - - ASMJIT_INLINE const Operand& getOp4() const noexcept { return static_cast(_op4); } - ASMJIT_INLINE const Operand& getOp5() const noexcept { return static_cast(_op5); } - ASMJIT_INLINE const Operand& getOpExtra() const noexcept { return static_cast(_opExtra); } - - ASMJIT_INLINE void setOp4(const Operand_& op4) noexcept { _options |= kOptionOp4; _op4 = op4; } - ASMJIT_INLINE void setOp5(const Operand_& op5) noexcept { _options |= kOptionOp5; _op5 = op5; } - ASMJIT_INLINE void setOpExtra(const Operand_& opExtra) noexcept { _options |= kOptionOpExtra; _opExtra = opExtra; } + //! Get an extra operand that will be used by the next instruction (architecture specific). + ASMJIT_INLINE const Operand& getExtraOp() const noexcept { return static_cast(_extraOp); } + //! Set an extra operand that will be used by the next instruction (architecture specific). + ASMJIT_INLINE void setExtraOp(const Operand_& extraOp) noexcept { _extraOp = extraOp; } + //! Reset an extra operand that will be used by the next instruction (architecture specific). + ASMJIT_INLINE void resetExtraOp() noexcept { _extraOp.setSignature(0); } //! Get annotation of the next instruction. ASMJIT_INLINE const char* getInlineComment() const noexcept { return _inlineComment; } @@ -461,12 +453,16 @@ public: return emit(instId, o0, o1, o2, o3, o4, static_cast(o5)); } + ASMJIT_INLINE Error emitOpArray(uint32_t instId, const Operand_* opArray, size_t opCount) { + return _emitOpArray(instId, opArray, opCount); + } + // -------------------------------------------------------------------------- // [Validation] // -------------------------------------------------------------------------- //! Validate instruction with current options, called by `_emit()` if validation is enabled. - ASMJIT_API Error _validate(uint32_t instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3) const noexcept; + ASMJIT_API Error _validate(uint32_t instId, const Operand_* opArray, uint32_t opCount) const noexcept; // -------------------------------------------------------------------------- // [Members] @@ -488,10 +484,7 @@ public: uint32_t _options; //!< Used to pass instruction options (affects the next instruction). const char* _inlineComment; //!< Inline comment of the next instruction (affects the next instruction). - Operand_ _op4; //!< 5th operand data (indexed from zero) (affects the next instruction). - Operand_ _op5; //!< 6th operand data (indexed from zero) (affects the next instruction). - Operand_ _opExtra; //!< Extra operand (op-mask {k} on AVX-512) (affects the next instruction). - + Operand_ _extraOp; //!< Extra operand (op-mask {k} on AVX-512) (affects the next instruction). Operand_ _none; //!< Used to pass unused operands to `_emit()` instead of passing null. Reg _nativeGpReg; //!< Native GP register with zero id. const Reg* _nativeGpArray; //!< Array of native registers indexed from zero. diff --git a/src/asmjit/base/cpuinfo.cpp b/src/asmjit/base/cpuinfo.cpp index 30cf52f..3e0a9aa 100644 --- a/src/asmjit/base/cpuinfo.cpp +++ b/src/asmjit/base/cpuinfo.cpp @@ -503,6 +503,9 @@ ASMJIT_FAVOR_SIZE static void x86DetectCpuInfo(CpuInfo* cpuInfo) noexcept { if (regs.ebx & 0x20000000U) cpuInfo->addFeature(CpuInfo::kX86FeatureSHA); if (regs.ecx & 0x00000001U) cpuInfo->addFeature(CpuInfo::kX86FeaturePREFETCHWT1); + // TSX is supported if at least one of `HLE` and `RTM` is supported. + if (regs.ebx & 0x00000810U) cpuInfo->addFeature(CpuInfo::kX86FeatureTSX); + // Detect AVX2. if (cpuInfo->hasFeature(CpuInfo::kX86FeatureAVX)) { if (regs.ebx & 0x00000020U) cpuInfo->addFeature(CpuInfo::kX86FeatureAVX2); diff --git a/src/asmjit/base/cpuinfo.h b/src/asmjit/base/cpuinfo.h index 099c668..a7a647d 100644 --- a/src/asmjit/base/cpuinfo.h +++ b/src/asmjit/base/cpuinfo.h @@ -107,7 +107,7 @@ public: kX86FeatureSHA, //!< CPU has SHA-1 and SHA-256. kX86FeatureXSAVE, //!< CPU has XSAVE support - XSAVE/XRSTOR, XSETBV/XGETBV, and XCR. kX86FeatureXSAVEOPT, //!< CPU has XSAVEOPT support - XSAVEOPT/XSAVEOPT64. - kX86FeatureOSXSAVE, //!< OS has enabled XSAVE, you can call XGETBV to get XCR content. + kX86FeatureOSXSAVE, //!< CPU has XSAVE enabled by OS. kX86FeatureAVX, //!< CPU has AVX. kX86FeatureAVX2, //!< CPU has AVX2. kX86FeatureF16C, //!< CPU has F16C. @@ -121,6 +121,7 @@ public: kX86FeatureMPX, //!< CPU has MPX (memory protection extensions). kX86FeatureHLE, //!< CPU has HLE. kX86FeatureRTM, //!< CPU has RTM. + kX86FeatureTSX, //!< CPU has TSX. kX86FeatureERMS, //!< CPU has ERMS (enhanced REP MOVSB/STOSB). kX86FeatureFSGSBASE, //!< CPU has FSGSBASE. kX86FeatureAVX512_F, //!< CPU has AVX512-F (foundation). diff --git a/src/asmjit/base/globals.cpp b/src/asmjit/base/globals.cpp index f0cc594..b4612e5 100644 --- a/src/asmjit/base/globals.cpp +++ b/src/asmjit/base/globals.cpp @@ -49,9 +49,12 @@ static const char errorMessages[] = "Invalid register kind\0" "Invalid register's physical id\0" "Invalid register's virtual id\0" - "Invalid LOCK prefix\0" - "Invalid REP prefix\0" - "Invalid REX prefix\0" + "Invalid prefix combination\0" + "Invalid lock prefix\0" + "Invalid xacquire prefix\0" + "Invalid xrelease prefix\0" + "Invalid rep prefix\0" + "Invalid rex prefix\0" "Invalid mask, expected {k}\0" "Invalid use of {k}\0" "Invalid use of {k}{z}\0" diff --git a/src/asmjit/base/globals.h b/src/asmjit/base/globals.h index 8d9e75c..7923d75 100644 --- a/src/asmjit/base/globals.h +++ b/src/asmjit/base/globals.h @@ -177,8 +177,14 @@ ASMJIT_ENUM(ErrorCode) { kErrorInvalidPhysId, //! Invalid register's virtual id. kErrorInvalidVirtId, + //! Invalid prefix combination. + kErrorInvalidPrefixCombination, //! Invalid LOCK prefix. kErrorInvalidLockPrefix, + //! Invalid XACQUIRE prefix. + kErrorInvalidXAcquirePrefix, + //! Invalid XACQUIRE prefix. + kErrorInvalidXReleasePrefix, //! Invalid REP prefix. kErrorInvalidRepPrefix, //! Invalid REX prefix. diff --git a/src/asmjit/base/logging.cpp b/src/asmjit/base/logging.cpp index f1198a6..67b9baf 100644 --- a/src/asmjit/base/logging.cpp +++ b/src/asmjit/base/logging.cpp @@ -235,15 +235,15 @@ Error Logging::formatInstruction( uint32_t archType, uint32_t instId, uint32_t options, - const Operand_& opExtra, + const Operand_& extraOp, const Operand_* opArray, uint32_t opCount) noexcept { #if defined(ASMJIT_BUILD_X86) - return X86Logging::formatInstruction(sb, logOptions, emitter, archType, instId, options, opExtra, opArray, opCount); + return X86Logging::formatInstruction(sb, logOptions, emitter, archType, instId, options, extraOp, opArray, opCount); #endif // ASMJIT_BUILD_X86 #if defined(ASMJIT_BUILD_ARM) - return ArmLogging::formatInstruction(sb, logOptions, emitter, archType, instId, options, opExtra, opArray, opCount); + return ArmLogging::formatInstruction(sb, logOptions, emitter, archType, instId, options, extraOp, opArray, opCount); #endif // ASMJIT_BUILD_ARM return kErrorInvalidArch; @@ -374,7 +374,7 @@ Error Logging::formatNode( cb->getArchType(), node->getInstId(), node->getOptions(), - node->getOpExtra(), + node->getExtraOp(), node->getOpArray(), node->getOpCount())); break; } @@ -438,7 +438,7 @@ Error Logging::formatNode( cb->getArchType(), node->getInstId(), node->getOptions(), - node->getOpExtra(), + node->getExtraOp(), node->getOpArray(), node->getOpCount())); break; } diff --git a/src/asmjit/base/logging.h b/src/asmjit/base/logging.h index 03ca1a2..12a680b 100644 --- a/src/asmjit/base/logging.h +++ b/src/asmjit/base/logging.h @@ -251,7 +251,7 @@ struct Logging { uint32_t archType, uint32_t instId, uint32_t options, - const Operand_& opExtra, + const Operand_& extraOp, const Operand_* opArray, uint32_t opCount) noexcept; #if !defined(ASMJIT_DISABLE_BUILDER) diff --git a/src/asmjit/x86/x86assembler.cpp b/src/asmjit/x86/x86assembler.cpp index 5947584..6a0f76a 100644 --- a/src/asmjit/x86/x86assembler.cpp +++ b/src/asmjit/x86/x86assembler.cpp @@ -538,7 +538,9 @@ Error X86Assembler::_emit(uint32_t instId, const Operand_& o0, const Operand_& o CodeEmitter::kOptionStrictValidation | // Strict validation. X86Inst::kOptionRep | // REP/REPZ prefix. X86Inst::kOptionRepnz | // REPNZ prefix. - X86Inst::kOptionLock ; // LOCK prefix. + X86Inst::kOptionLock | // LOCK prefix. + X86Inst::kOptionXAcquire | // XACQUIRE prefix. + X86Inst::kOptionXRelease ; // XRELEASE prefix. // Signature of the first 3 operands. uint32_t isign3 = o0.getOp() + (o1.getOp() << 3) + (o2.getOp() << 6); @@ -564,26 +566,55 @@ Error X86Assembler::_emit(uint32_t instId, const Operand_& o0, const Operand_& o // Strict validation. #if !defined(ASMJIT_DISABLE_VALIDATION) if (options & CodeEmitter::kOptionStrictValidation) { - err = _validate(instId, o0, o1, o2, o3); + Operand_ opArray[6]; + + opArray[0].copyFrom(o0); + opArray[1].copyFrom(o1); + opArray[2].copyFrom(o2); + opArray[3].copyFrom(o3); + + if (options & kOptionOp4Op5Used) { + opArray[4].copyFrom(_op4); + opArray[5].copyFrom(_op5); + } + else { + opArray[4].reset(); + opArray[5].reset(); + } + + err = _validate(instId, opArray, 6); if (ASMJIT_UNLIKELY(err)) goto Failed; } #endif // !ASMJIT_DISABLE_VALIDATION - uint32_t instFlags = instData->getFlags(); + uint32_t iFlags = instData->getFlags(); - // LOCK prefix. + // LOCK, XACQUIRE, and XRELEASE prefixes. if (options & X86Inst::kOptionLock) { - if (ASMJIT_UNLIKELY(!(instFlags & X86Inst::kFlagLock))) + bool xAcqRel = (options & (X86Inst::kOptionXAcquire | X86Inst::kOptionXRelease)) != 0; + + if (ASMJIT_UNLIKELY(!(iFlags & (X86Inst::kFlagLock)) && !xAcqRel)) goto InvalidLockPrefix; + + if (xAcqRel) { + if (ASMJIT_UNLIKELY((options & X86Inst::kOptionXAcquire) && !(iFlags & X86Inst::kFlagXAcquire))) + goto InvalidXAcquirePrefix; + + if (ASMJIT_UNLIKELY((options & X86Inst::kOptionXRelease) && !(iFlags & X86Inst::kFlagXRelease))) + goto InvalidXReleasePrefix; + + EMIT_BYTE((options & X86Inst::kOptionXAcquire) ? 0xF2 : 0xF3); + } + EMIT_BYTE(0xF0); } - // REP / REPNZ prefix. + // REP and REPNZ prefixes. if (options & (X86Inst::kOptionRep | X86Inst::kOptionRepnz)) { - if (ASMJIT_UNLIKELY(!(instFlags & (X86Inst::kFlagRep | X86Inst::kFlagRepnz)))) + if (ASMJIT_UNLIKELY(!(iFlags & (X86Inst::kFlagRep | X86Inst::kFlagRepnz)))) goto InvalidRepPrefix; - if (!_opExtra.isNone() && ASMJIT_UNLIKELY(!X86Reg::isGp(_opExtra, X86Gp::kIdCx))) + if (!_extraOp.isNone() && ASMJIT_UNLIKELY(!X86Reg::isGp(_extraOp, X86Gp::kIdCx))) goto InvalidRepPrefix; EMIT_BYTE((options & X86Inst::kOptionRepnz) ? 0xF2 : 0xF3); @@ -609,6 +640,14 @@ Error X86Assembler::_emit(uint32_t instId, const Operand_& o0, const Operand_& o case X86Inst::kEncodingX86Op: goto EmitX86Op; + case X86Inst::kEncodingX86Op_O_I8: + if (ASMJIT_UNLIKELY(isign3 != ENC_OPS1(Imm))) + goto InvalidInstruction; + + imVal = o0.as().getUInt8(); + imLen = 1; + ASMJIT_FALLTHROUGH; + case X86Inst::kEncodingX86Op_O: rbReg = 0; goto EmitX86R; @@ -1011,6 +1050,7 @@ CaseX86M_GPB_MulDiv: // displacement is not encodable so the alternative opcode field // in X86DB must be zero. opCode = 0xE8; + opReg = 0; goto EmitJmpCall; case X86Inst::kEncodingX86Cmpxchg: { @@ -1282,6 +1322,7 @@ CaseX86M_GPB_MulDiv: } rmRel = &o0; + opReg = 0; goto EmitJmpCall; case X86Inst::kEncodingX86JecxzLoop: @@ -1296,6 +1337,8 @@ CaseX86M_GPB_MulDiv: rmRel = &o1; } + + opReg = 0; goto EmitJmpCall; case X86Inst::kEncodingX86Jmp: @@ -1311,6 +1354,11 @@ CaseX86M_GPB_MulDiv: // Jump encoded with 32-bit displacement use 0xE9 opcode. Jump encoded // with 8-bit displacement's opcode is stored as an alternative opcode. opCode = 0xE9; + opReg = 0; + goto EmitJmpCall; + + case X86Inst::kEncodingX86JmpRel: + rmRel = &o0; goto EmitJmpCall; case X86Inst::kEncodingX86Lea: @@ -2815,6 +2863,9 @@ CaseVexRm: break; case X86Inst::kEncodingVexRm_T1_4X: { + if (!(options & kOptionOp4Op5Used)) + goto InvalidInstruction; + if (X86Reg::isZmm(o0 ) && X86Reg::isZmm(o1) && X86Reg::isZmm(o2 ) && X86Reg::isZmm(o3) && X86Reg::isZmm(_op4) && _op5.isMem()) { @@ -3374,7 +3425,7 @@ CaseVexRvm_R: } case X86Inst::kEncodingVexRvrmiRvmri_Lx: { - if (!(options & CodeEmitter::kOptionOp4) || !_op4.isImm()) + if (!(options & CodeEmitter::kOptionOp4Op5Used) || !_op4.isImm()) goto InvalidInstruction; const uint32_t isign4 = isign3 + (o3.getOp() << 9); @@ -3987,26 +4038,21 @@ EmitVexEvexR: // Mark invalid VEX (force EVEX) case: // [@.......|.LL.....|Vvvvv..R|RBBmmmmm]. x |= (~commonData->getFlags() & X86Inst::kFlagVex) << (31 - Utils::firstBitOfT()); + if (X86Reg::isK(_extraOp)) + x |= _extraOp.as().getId() << 16; // [@.......|.LL..aaa|Vvvvv..R|RBBmmmmm]. + // Handle AVX512 options by a single branch. - const uint32_t kAvx512Options = X86Inst::kOptionOpExtra | - X86Inst::kOptionKZ | - X86Inst::kOption1ToX | - X86Inst::kOptionSAE | - X86Inst::kOptionER ; + const uint32_t kAvx512Options = X86Inst::kOptionZMask | + X86Inst::kOption1ToX | + X86Inst::kOptionSAE | + X86Inst::kOptionER ; if (options & kAvx512Options) { // Memory broadcast without a memory operand is invalid. if (ASMJIT_UNLIKELY(options & X86Inst::kOption1ToX)) goto InvalidBroadcast; // TODO: {sae} and {er} - - // NOTE: We consider a valid construct internally even when {kz} was - // specified without specifying the register. In that case it would be - // `k0` and basically everything should be zeroed. It's valid EVEX. - if (options & X86Inst::kOptionOpExtra) - x |= _opExtra.getId() << 16; - - x |= options & X86Inst::kOptionKZ; // [@.......|zLL..aaa|Vvvvv..R|RBBmmmmm]. + x |= options & X86Inst::kOptionZMask; // [@.......|zLL..aaa|Vvvvv..R|RBBmmmmm]. } // Check if EVEX is required by checking bits in `x` : [@.......|xx...xxx|x......x|.x.x....]. @@ -4101,25 +4147,21 @@ EmitVexEvexM: // Mark invalid VEX (force EVEX) case: // [@.......|.LL.X...|Vvvvv..R|RXBmmmmm]. x |= (~commonData->getFlags() & X86Inst::kFlagVex) << (31 - Utils::firstBitOfT()); + if (X86Reg::isK(_extraOp)) + x |= _extraOp.as().getId() << 16; // [@.......|.LL.Xaaa|Vvvvv..R|RXBmmmmm]. + // Handle AVX512 options by a single branch. - const uint32_t kAvx512Options = X86Inst::kOptionOpExtra | - X86Inst::kOption1ToX | - X86Inst::kOptionKZ | - X86Inst::kOptionSAE | - X86Inst::kOptionER ; + const uint32_t kAvx512Options = X86Inst::kOption1ToX | + X86Inst::kOptionZMask | + X86Inst::kOptionSAE | + X86Inst::kOptionER ; if (options & kAvx512Options) { // {er} and {sae} are both invalid if memory operand is used. if (ASMJIT_UNLIKELY(options & (X86Inst::kOptionSAE | X86Inst::kOptionER))) goto InvalidEROrSAE; - // NOTE: We consider a valid construct internally even when {kz} was - // specified without specifying the register. In that case it would be - // `k0` and basically everything would be zeroed. It's a valid EVEX. - if (options & X86Inst::kOptionOpExtra) - x |= _opExtra.getId() << 16; - x |= options & (X86Inst::kOption1ToX | // [@.......|.LLbXaaa|Vvvvv..R|RXBmmmmm]. - X86Inst::kOptionKZ ); // [@.......|zLLbXaaa|Vvvvv..R|RXBmmmmm]. + X86Inst::kOptionZMask); // [@.......|zLLbXaaa|Vvvvv..R|RXBmmmmm]. } // Check if EVEX is required by checking bits in `x` : [@.......|xx.xxxxx|x......x|...x....]. @@ -4214,11 +4256,12 @@ EmitJmpCall: // Jcc instructions with 32-bit displacement use 0x0F prefix, // other instructions don't. No other prefixes are used by X86. ASMJIT_ASSERT((opCode8 & X86Inst::kOpCode_MM_Mask) == 0); - ASMJIT_ASSERT((opCode & X86Inst::kOpCode_MM_Mask) == 0 || - (opCode & X86Inst::kOpCode_MM_Mask) == X86Inst::kOpCode_MM_0F); + ASMJIT_ASSERT((opCode & X86Inst::kOpCode_MM_Mask) == 0 || + (opCode & X86Inst::kOpCode_MM_Mask) == X86Inst::kOpCode_MM_0F); - inst32Size += static_cast( - (opCode & X86Inst::kOpCode_MM_Mask) == X86Inst::kOpCode_MM_0F); + // Only one of these should be used at the same time. + inst32Size += static_cast(opReg != 0); + inst32Size += static_cast((opCode & X86Inst::kOpCode_MM_Mask) == X86Inst::kOpCode_MM_0F); if (rmRel->isLabel()) { label = _code->getLabelEntry(rmRel->as