From 2ff454d41555d16d0759e4d0e95ade3c875b615e Mon Sep 17 00:00:00 2001 From: kobalicek Date: Sun, 15 Jun 2025 16:33:41 +0200 Subject: [PATCH] [abi] AsmJit v1.17 - cumulative & breaking changes * Reworked register operands - all vector registers are now platform::Vec deriving from UniVec (universal vector operand), additionally, there is no platform::Reg, instead asmjit::Reg provides all necessary features to make it a base register for each target architecture * Reworked casting between registers - now architecture agnostic names are preferred - use Gp32 instead of Gpd or GpW, Gp64 instead of Gpq and GpX, etc... * Reworked vector registers and their names - architecture agnostic naming is now preferred Vec32, Vec64, Vec128, etc... * Reworked naming conventions used across AsmJit - for clarity Identifiers are now prefixed with the type, like sectionId(), labelId(), etc... * Reworked how Zone and ZoneAllocator are used across AsmJit, prefering Zone in most cases and ZoneAllocator only for containers - this change alone achieves around 5% better performance of Builder and Compiler * Reworked LabelEntry - decreased the size of the base entry to 16 bytes for anonymous and unnamed labels. Avoided an indirection when using labelEntries() - LabelEntry is now a value and not a pointer * Renamed LabelLink to Fixup * Added a new header which would include + target tools for the host architecture, if enabled and supported * Added new AArch64 instructions (BTI, CSSC, CHKFEAT) * Added a mvn_ alternative of mvn instruction (fix for Windows ARM64 SDK) * Added more AArch64 CPU features to CpuInfo * Added better support for Apple CPU detection (Apple M3, M4) * Added a new benchmarking tool asmjit_bench_overhead, which benchmarks the overhead of CodeHolder::init()/reset() and creating/attaching emitters to it. Thanks to the benchmark the most common code-paths were optimized * Added a new benchmarking tool asmjit_bench_regalloc, which aims to benchmark the cost and complexity of register allocation. * Renamed asmjit_test_perf to asmjit_bench_codegen to make it clear what is a test and what is a benchmark --- .github/workflows/build-config.json | 3 +- .github/workflows/build.yml | 194 +- CMakeLists.txt | 76 +- db/aarch32.js | 2 +- db/aarch64.js | 2 +- db/base.js | 2 +- db/exp.js | 2 +- db/index.js | 2 +- db/x86.js | 2 +- src/asmjit.natvis | 10 +- src/asmjit/a64.h | 18 +- src/asmjit/arm.h | 22 +- src/asmjit/arm/a64archtraits_p.h | 56 +- src/asmjit/arm/a64assembler.cpp | 408 ++-- src/asmjit/arm/a64assembler.h | 6 +- src/asmjit/arm/a64builder.cpp | 11 +- src/asmjit/arm/a64builder.h | 6 +- src/asmjit/arm/a64compiler.cpp | 18 +- src/asmjit/arm/a64compiler.h | 18 +- src/asmjit/arm/a64emithelper.cpp | 34 +- src/asmjit/arm/a64emithelper_p.h | 13 +- src/asmjit/arm/a64emitter.h | 96 +- src/asmjit/arm/a64formatter.cpp | 2 +- src/asmjit/arm/a64formatter_p.h | 2 +- src/asmjit/arm/a64func.cpp | 28 +- src/asmjit/arm/a64func_p.h | 2 +- src/asmjit/arm/a64globals.h | 24 +- src/asmjit/arm/a64instapi.cpp | 6 +- src/asmjit/arm/a64instapi_p.h | 2 +- src/asmjit/arm/a64instdb.cpp | 1682 +++++++++-------- src/asmjit/arm/a64instdb.h | 2 +- src/asmjit/arm/a64instdb_p.h | 52 +- src/asmjit/arm/a64operand.cpp | 42 +- src/asmjit/arm/a64operand.h | 1395 ++++++++------ src/asmjit/arm/a64rapass.cpp | 76 +- src/asmjit/arm/a64rapass_p.h | 2 +- src/asmjit/arm/armformatter.cpp | 94 +- src/asmjit/arm/armformatter_p.h | 2 +- src/asmjit/arm/armglobals.h | 2 +- src/asmjit/arm/armoperand.h | 399 ---- src/asmjit/arm/armutils.h | 2 +- src/asmjit/asmjit-scope-begin.h | 2 +- src/asmjit/asmjit-scope-end.h | 2 +- src/asmjit/asmjit.h | 2 +- src/asmjit/core.h | 168 +- src/asmjit/core/api-build_p.h | 2 +- src/asmjit/core/api-config.h | 136 +- src/asmjit/core/archcommons.h | 2 +- src/asmjit/core/archtraits.cpp | 35 +- src/asmjit/core/archtraits.h | 52 +- src/asmjit/core/assembler.cpp | 99 +- src/asmjit/core/assembler.h | 7 +- src/asmjit/core/builder.cpp | 233 +-- src/asmjit/core/builder.h | 257 ++- src/asmjit/core/builder_p.h | 7 +- src/asmjit/core/codebuffer.h | 2 +- src/asmjit/core/codeholder.cpp | 768 +++++--- src/asmjit/core/codeholder.h | 800 ++++---- src/asmjit/core/codewriter.cpp | 10 +- src/asmjit/core/codewriter_p.h | 10 +- src/asmjit/core/compiler.cpp | 139 +- src/asmjit/core/compiler.h | 61 +- src/asmjit/core/compilerdefs.h | 10 +- src/asmjit/core/constpool.cpp | 5 +- src/asmjit/core/constpool.h | 17 +- src/asmjit/core/cpuinfo.cpp | 479 +++-- src/asmjit/core/cpuinfo.h | 46 +- src/asmjit/core/emithelper.cpp | 32 +- src/asmjit/core/emithelper_p.h | 8 +- src/asmjit/core/emitter.cpp | 35 +- src/asmjit/core/emitter.h | 147 +- src/asmjit/core/emitterutils.cpp | 2 +- src/asmjit/core/emitterutils_p.h | 2 +- src/asmjit/core/environment.cpp | 2 +- src/asmjit/core/environment.h | 2 +- src/asmjit/core/errorhandler.cpp | 2 +- src/asmjit/core/errorhandler.h | 2 +- src/asmjit/core/fixup.h | 282 +++ src/asmjit/core/formatter.cpp | 67 +- src/asmjit/core/formatter.h | 2 +- src/asmjit/core/formatter_p.h | 8 +- src/asmjit/core/func.cpp | 8 +- src/asmjit/core/func.h | 58 +- src/asmjit/core/funcargscontext.cpp | 26 +- src/asmjit/core/funcargscontext_p.h | 18 +- src/asmjit/core/globals.cpp | 2 +- src/asmjit/core/globals.h | 45 +- src/asmjit/core/inst.cpp | 2 +- src/asmjit/core/inst.h | 42 +- src/asmjit/core/instdb.cpp | 2 +- src/asmjit/core/instdb_p.h | 2 +- src/asmjit/core/jitallocator.cpp | 293 ++- src/asmjit/core/jitallocator.h | 2 +- src/asmjit/core/jitruntime.cpp | 13 +- src/asmjit/core/jitruntime.h | 7 +- src/asmjit/core/logger.cpp | 2 +- src/asmjit/core/logger.h | 2 +- src/asmjit/core/misc_p.h | 2 +- src/asmjit/core/operand.cpp | 28 +- src/asmjit/core/operand.h | 1512 ++++++++++----- src/asmjit/core/osutils.cpp | 2 +- src/asmjit/core/osutils.h | 2 +- src/asmjit/core/osutils_p.h | 2 +- src/asmjit/core/raassignment_p.h | 20 +- src/asmjit/core/rabuilders_p.h | 4 +- src/asmjit/core/radefs_p.h | 18 +- src/asmjit/core/ralocal.cpp | 2 +- src/asmjit/core/ralocal_p.h | 2 +- src/asmjit/core/rapass.cpp | 18 +- src/asmjit/core/rapass_p.h | 27 +- src/asmjit/core/rastack.cpp | 6 +- src/asmjit/core/rastack_p.h | 15 +- src/asmjit/core/string.cpp | 2 +- src/asmjit/core/string.h | 3 +- src/asmjit/core/support.cpp | 106 +- src/asmjit/core/support.h | 555 +++--- src/asmjit/core/support_p.h | 2 +- src/asmjit/core/target.cpp | 2 +- src/asmjit/core/target.h | 2 +- src/asmjit/core/type.cpp | 2 +- src/asmjit/core/type.h | 14 +- src/asmjit/core/virtmem.cpp | 4 +- src/asmjit/core/virtmem.h | 2 +- src/asmjit/core/zone.cpp | 129 +- src/asmjit/core/zone.h | 395 ++-- src/asmjit/core/zonehash.cpp | 2 +- src/asmjit/core/zonehash.h | 2 +- src/asmjit/core/zonelist.cpp | 2 +- src/asmjit/core/zonelist.h | 10 +- src/asmjit/core/zonestack.cpp | 11 +- src/asmjit/core/zonestack.h | 2 +- src/asmjit/core/zonestring.h | 2 +- src/asmjit/core/zonetree.cpp | 2 +- src/asmjit/core/zonetree.h | 2 +- src/asmjit/core/zonevector.cpp | 2 +- src/asmjit/core/zonevector.h | 34 +- src/asmjit/host.h | 33 + src/asmjit/x86.h | 35 +- src/asmjit/x86/x86archtraits_p.h | 125 +- src/asmjit/x86/x86assembler.cpp | 240 +-- src/asmjit/x86/x86assembler.h | 12 +- src/asmjit/x86/x86builder.cpp | 10 +- src/asmjit/x86/x86builder.h | 10 +- src/asmjit/x86/x86compiler.cpp | 18 +- src/asmjit/x86/x86compiler.h | 52 +- src/asmjit/x86/x86emithelper.cpp | 69 +- src/asmjit/x86/x86emithelper_p.h | 23 +- src/asmjit/x86/x86emitter.h | 4 +- src/asmjit/x86/x86formatter.cpp | 129 +- src/asmjit/x86/x86formatter_p.h | 2 +- src/asmjit/x86/x86func.cpp | 60 +- src/asmjit/x86/x86func_p.h | 2 +- src/asmjit/x86/x86globals.h | 2 +- src/asmjit/x86/x86instapi.cpp | 203 +- src/asmjit/x86/x86instapi_p.h | 2 +- src/asmjit/x86/x86instdb.cpp | 2 +- src/asmjit/x86/x86instdb.h | 6 +- src/asmjit/x86/x86instdb_p.h | 4 +- src/asmjit/x86/x86opcode_p.h | 2 +- src/asmjit/x86/x86operand.cpp | 98 +- src/asmjit/x86/x86operand.h | 926 ++++----- src/asmjit/x86/x86rapass.cpp | 169 +- src/asmjit/x86/x86rapass_p.h | 2 +- ...test_perf.cpp => asmjit_bench_codegen.cpp} | 20 +- ...jit_test_perf.h => asmjit_bench_codegen.h} | 27 +- ...f_a64.cpp => asmjit_bench_codegen_a64.cpp} | 12 +- ...f_x86.cpp => asmjit_bench_codegen_x86.cpp} | 305 ++- test/asmjit_bench_overhead.cpp | 479 +++++ test/asmjit_bench_regalloc.cpp | 497 +++++ test/asmjit_test_assembler.cpp | 38 +- test/asmjit_test_assembler.h | 2 +- test/asmjit_test_assembler_a64.cpp | 39 +- test/asmjit_test_assembler_x64.cpp | 2 +- test/asmjit_test_assembler_x86.cpp | 2 +- test/asmjit_test_compiler.cpp | 11 +- test/asmjit_test_compiler.h | 2 +- test/asmjit_test_compiler_a64.cpp | 2 +- test/asmjit_test_compiler_x86.cpp | 90 +- test/asmjit_test_emitters.cpp | 62 +- test/asmjit_test_environment.cpp | 112 +- test/asmjit_test_execute.cpp | 103 - test/asmjit_test_instinfo.cpp | 24 +- test/asmjit_test_misc.h | 44 +- test/asmjit_test_random.h | 77 + test/asmjit_test_unit.cpp | 17 +- test/asmjit_test_x86_sections.cpp | 32 +- test/asmjitutils.h | 30 +- test/cmdline.h | 2 +- test/performancetimer.h | 2 +- tools/generator-commons.js | 2 +- tools/generator-cxx.js | 2 +- tools/tablegen-a64.js | 2 +- tools/tablegen-x86.js | 2 +- tools/tablegen.js | 2 +- 194 files changed, 9604 insertions(+), 6913 deletions(-) delete mode 100644 src/asmjit/arm/armoperand.h create mode 100644 src/asmjit/core/fixup.h create mode 100644 src/asmjit/host.h rename test/{asmjit_test_perf.cpp => asmjit_bench_codegen.cpp} (82%) rename test/{asmjit_test_perf.h => asmjit_bench_codegen.h} (79%) rename test/{asmjit_test_perf_a64.cpp => asmjit_bench_codegen_a64.cpp} (98%) rename test/{asmjit_test_perf_x86.cpp => asmjit_bench_codegen_x86.cpp} (95%) create mode 100644 test/asmjit_bench_overhead.cpp create mode 100644 test/asmjit_bench_regalloc.cpp delete mode 100644 test/asmjit_test_execute.cpp create mode 100644 test/asmjit_test_random.h diff --git a/.github/workflows/build-config.json b/.github/workflows/build-config.json index 7506bef..de54054 100644 --- a/.github/workflows/build-config.json +++ b/.github/workflows/build-config.json @@ -17,10 +17,9 @@ { "optional": true, "cmd": ["asmjit_test_assembler"] }, { "optional": true, "cmd": ["asmjit_test_assembler", "--validate"] }, { "optional": true, "cmd": ["asmjit_test_emitters"] }, - { "optional": true, "cmd": ["asmjit_test_execute"] }, { "optional": true, "cmd": ["asmjit_test_compiler"] }, { "optional": true, "cmd": ["asmjit_test_instinfo"] }, { "optional": true, "cmd": ["asmjit_test_x86_sections"] }, - { "optional": true, "cmd": ["asmjit_test_perf", "--quick"] } + { "optional": true, "cmd": ["asmjit_bench_codegen", "--quick"] } ] } diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 90ce935..6b803b0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,109 +35,105 @@ jobs: fail-fast: false matrix: include: - - { title: "diag-analyze" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Debug" , diagnostics: "analyze-build" } - - { title: "diag-asan" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", diagnostics: "asan", defs: "ASMJIT_TEST=1" } - - { title: "diag-msan" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", diagnostics: "msan", defs: "ASMJIT_TEST=1" } - - { title: "diag-ubsan" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", diagnostics: "ubsan", defs: "ASMJIT_TEST=1" } - - { title: "diag-hardened" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", diagnostics: "hardened", defs: "ASMJIT_TEST=1" } - - { title: "diag-valgrind" , host: "ubuntu-24.04" , arch: "x64" , cc: "clang-19", conf: "Release", diagnostics: "valgrind", defs: "ASMJIT_TEST=1" } - - - { title: "no-deprecated" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_DEPRECATED=1" } - - { title: "no-intrinsics" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_INTRINSICS=1" } - - { title: "no-logging" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_LOGGING=1" } - - { title: "no-logging-text" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_LOGGING=1,ASMJIT_NO_TEXT=1" } - - { title: "no-builder" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_BUILDER=1" } - - { title: "no-compiler" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_COMPILER=1" } - - { title: "no-introspection", host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_COMPILER=1,ASMJIT_NO_INTROSPECTION=1" } - - { title: "no-jit" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_JIT=1" } - - { title: "no-validation" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_VALIDATION=1" } - - { title: "no-x86" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_X86=1" } - - { title: "no-aarch64" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_AARCH64=1" } - - - { title: "lang-c++20" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Debug" , defs: "ASMJIT_TEST=1,CMAKE_CXX_FLAGS=-std=c++20" } - - { title: "lang-c++23" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Debug" , defs: "ASMJIT_TEST=1,CMAKE_CXX_FLAGS=-std=c++23" } - - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-9" , conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-9" , conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-9" , conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-9" , conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-10" , conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-10" , conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-10" , conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-10" , conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-11" , conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-11" , conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-11" , conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-11" , conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-12" , conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-12" , conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-12" , conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-12" , conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-13" , conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-13" , conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-13" , conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-13" , conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-11", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-11", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-11", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-11", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-12", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-12", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-12", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-12", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-13", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-13", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-13", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-13", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-14", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-14", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-14", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-14", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-15", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-15", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-15", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-15", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-16", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-16", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-16", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-16", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-17", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-17", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-17", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-17", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-18", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-18", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-18", conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-18", conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "macos" , host: "macos-13" , arch: "x64" , cc: "gcc-14" , conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "macos" , host: "macos-13" , arch: "x64" , cc: "gcc-14" , conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "macos" , host: "macos-13" , arch: "x64" , cc: "clang" , conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "macos" , host: "macos-13" , arch: "x64" , cc: "clang" , conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "macos" , host: "macos-14" , arch: "arm64" , cc: "clang" , conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "macos" , host: "macos-14" , arch: "arm64" , cc: "clang" , conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "windows" , host: "windows-2022" , arch: "x86" , cc: "vs2022" , conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "windows" , host: "windows-2022" , arch: "x86" , cc: "vs2022" , conf: "Release", defs: "ASMJIT_TEST=1" } - - { title: "windows" , host: "windows-2022" , arch: "x64" , cc: "vs2022" , conf: "Debug" , defs: "ASMJIT_TEST=1" } - - { title: "windows" , host: "windows-2022" , arch: "x64" , cc: "vs2022" , conf: "Release", defs: "ASMJIT_TEST=1" } - - # Cross compiled, cannot run tests (Windows/ARM64). - - { title: "windows" , host: "windows-2022" , arch: "arm64" , cc: "vs2022" , conf: "Debug" , defs: "ASMJIT_TEST=0" } - - { title: "windows" , host: "windows-2022" , arch: "arm64" , cc: "vs2022" , conf: "Release", defs: "ASMJIT_TEST=0" } + - { title: "diag-analyze" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Debug" , defs: "ASMJIT_TEST=0", diagnostics: "analyze-build"} + - { title: "diag-asan" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1", diagnostics: "asan", } + - { title: "diag-msan" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1", diagnostics: "msan", } + - { title: "diag-ubsan" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1", diagnostics: "ubsan", } + - { title: "diag-hardened" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1", diagnostics: "hardened", } + - { title: "diag-valgrind" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1", diagnostics: "valgrind", } + - { title: "no-deprecated" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_DEPRECATED=1" } + - { title: "no-intrinsics" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_INTRINSICS=1" } + - { title: "no-logging" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_LOGGING=1" } + - { title: "no-logging-text" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_LOGGING=1,ASMJIT_NO_TEXT=1" } + - { title: "no-builder" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_BUILDER=1" } + - { title: "no-compiler" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_COMPILER=1" } + - { title: "no-introspection", host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_COMPILER=1,ASMJIT_NO_INTROSPECTION=1" } + - { title: "no-jit" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_JIT=1" } + - { title: "no-validation" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_VALIDATION=1" } + - { title: "no-x86" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_X86=1" } + - { title: "no-aarch64" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1,ASMJIT_NO_AARCH64=1" } + - { title: "lang-c++20" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Debug" , defs: "ASMJIT_TEST=1,CMAKE_CXX_FLAGS=-std=c++20" } + - { title: "lang-c++23" , host: "ubuntu-latest" , arch: "x64" , cc: "clang-19", conf: "Debug" , defs: "ASMJIT_TEST=1,CMAKE_CXX_FLAGS=-std=c++23" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-9" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-9" , conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-9" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-9" , conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-10" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-10" , conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-10" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-10" , conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-11" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-11" , conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-11" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-11" , conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-12" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-12" , conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-12" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-12" , conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-13" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "gcc-13" , conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-13" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "gcc-13" , conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-11", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-11", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-11", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-11", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-12", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-12", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-12", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-12", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-13", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-13", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-13", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-13", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-14", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-14", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-14", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-14", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-15", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-15", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-15", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-15", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-16", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-16", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-16", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-16", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-17", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-17", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-17", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-17", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-18", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x86" , cc: "clang-18", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-18", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-22.04" , arch: "x64" , cc: "clang-18", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-24.04-arm", arch: "arm64" , cc: "clang-19", conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "linux" , host: "ubuntu-24.04-arm", arch: "arm64" , cc: "clang-19", conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "macos" , host: "macos-13" , arch: "x64" , cc: "gcc-14" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "macos" , host: "macos-13" , arch: "x64" , cc: "gcc-14" , conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "macos" , host: "macos-13" , arch: "x64" , cc: "clang" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "macos" , host: "macos-13" , arch: "x64" , cc: "clang" , conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "macos" , host: "macos-14" , arch: "arm64" , cc: "clang" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "macos" , host: "macos-14" , arch: "arm64" , cc: "clang" , conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "windows" , host: "windows-2022" , arch: "x86" , cc: "vs2022" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "windows" , host: "windows-2022" , arch: "x86" , cc: "vs2022" , conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "windows" , host: "windows-2022" , arch: "x64" , cc: "vs2022" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "windows" , host: "windows-2022" , arch: "x64" , cc: "vs2022" , conf: "Release", defs: "ASMJIT_TEST=1" } + - { title: "windows" , host: "windows-11-arm" , arch: "arm64" , cc: "vs2022" , conf: "Debug" , defs: "ASMJIT_TEST=1" } + - { title: "windows" , host: "windows-11-arm" , arch: "arm64" , cc: "vs2022" , conf: "Release", defs: "ASMJIT_TEST=1" } # Cross compiled, cannot run tests (Windows/UWP). - - { title: "windows/uwp" , host: "windows-2022" , arch: "x64" , cc: "vs2022" , conf: "Release", defs: "ASMJIT_TEST=0,CMAKE_SYSTEM_NAME=WindowsStore,CMAKE_SYSTEM_VERSION=10.0,CMAKE_CXX_FLAGS=-D_WIN32_WINNT=0x0A00" } + - { title: "windows/uwp" , host: "windows-2022" , arch: "x64" , cc: "vs2022" , conf: "Release", defs: "ASMJIT_TEST=0,CMAKE_SYSTEM_NAME=WindowsStore,CMAKE_SYSTEM_VERSION=10.0,CMAKE_CXX_FLAGS=-D_WIN32_WINNT=0x0A00" } - - { title: "freebsd" , host: "ubuntu-latest" , arch: "x64" , cc: "clang" , conf: "Release", vm: "freebsd", vm_ver: "14.1", defs: "ASMJIT_TEST=1" } - - { title: "freebsd" , host: "ubuntu-latest" , arch: "arm64" , cc: "clang" , conf: "Release", vm: "freebsd", vm_ver: "14.1", defs: "ASMJIT_TEST=1" } - - { title: "netbsd" , host: "ubuntu-latest" , arch: "x64" , cc: "clang" , conf: "Release", vm: "netbsd" , vm_ver: "10.0", defs: "ASMJIT_TEST=1" } - - { title: "netbsd" , host: "ubuntu-latest" , arch: "arm64" , cc: "clang" , conf: "Release", vm: "netbsd" , vm_ver: "10.0", defs: "ASMJIT_TEST=1" } - - { title: "openbsd" , host: "ubuntu-latest" , arch: "x64" , cc: "clang" , conf: "Release", vm: "openbsd", vm_ver: "7.4" , defs: "ASMJIT_TEST=1" } - - { title: "openbsd" , host: "ubuntu-latest" , arch: "arm64" , cc: "clang" , conf: "Release", vm: "openbsd", vm_ver: "7.4" , defs: "ASMJIT_TEST=1" } + - { title: "freebsd" , host: "ubuntu-latest" , arch: "x64" , cc: "clang" , conf: "Release", vm: "freebsd", vm_ver: "14.2", defs: "ASMJIT_TEST=1" } + - { title: "freebsd" , host: "ubuntu-latest" , arch: "arm64" , cc: "clang" , conf: "Release", vm: "freebsd", vm_ver: "14.2", defs: "ASMJIT_TEST=1" } + - { title: "netbsd" , host: "ubuntu-latest" , arch: "x64" , cc: "clang" , conf: "Release", vm: "netbsd" , vm_ver: "10.1", defs: "ASMJIT_TEST=1" } + - { title: "netbsd" , host: "ubuntu-latest" , arch: "arm64" , cc: "clang" , conf: "Release", vm: "netbsd" , vm_ver: "10.1", defs: "ASMJIT_TEST=1" } + - { title: "openbsd" , host: "ubuntu-latest" , arch: "x64" , cc: "clang" , conf: "Release", vm: "openbsd", vm_ver: "7.4" , defs: "ASMJIT_TEST=1" } + - { title: "openbsd" , host: "ubuntu-latest" , arch: "arm64" , cc: "clang" , conf: "Release", vm: "openbsd", vm_ver: "7.4" , defs: "ASMJIT_TEST=1" } - - { title: "debian" , host: "ubuntu-latest" , arch: "arm/v7" , cc: "clang" , conf: "Release", vm: "debian:unstable", defs: "ASMJIT_TEST=1" } - - { title: "debian" , host: "ubuntu-latest" , arch: "arm64" , cc: "clang" , conf: "Release", vm: "debian:unstable", defs: "ASMJIT_TEST=1" } - - { title: "debian" , host: "ubuntu-latest" , arch: "riscv64", cc: "clang" , conf: "Release", vm: "debian:unstable", defs: "ASMJIT_TEST=1" } - - { title: "debian" , host: "ubuntu-latest" , arch: "ppc64le", cc: "clang" , conf: "Release", vm: "debian:unstable", defs: "ASMJIT_TEST=1" } + - { title: "debian" , host: "ubuntu-latest" , arch: "arm/v7" , cc: "clang" , conf: "Release", vm: "debian:unstable", defs: "ASMJIT_TEST=1" } + - { title: "debian" , host: "ubuntu-latest" , arch: "riscv64", cc: "clang" , conf: "Release", vm: "debian:unstable", defs: "ASMJIT_TEST=1" } + - { title: "debian" , host: "ubuntu-latest" , arch: "ppc64le", cc: "clang" , conf: "Release", vm: "debian:unstable", defs: "ASMJIT_TEST=1" } name: "${{matrix.title}}/${{matrix.arch}}, ${{matrix.cc}} ${{matrix.conf}}" runs-on: "${{matrix.host}}" diff --git a/CMakeLists.txt b/CMakeLists.txt index cec651e..61f928c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,6 +43,14 @@ if (ASMJIT_EMBED AND NOT ASMJIT_STATIC) set(ASMJIT_STATIC TRUE) endif() +if (NOT DEFINED ASMJIT_NO_DEPRECATED) + set(ASMJIT_NO_DEPRECATED FALSE) +endif() + +if (NOT DEFINED ASMJIT_NO_ABI_NAMESPACE) + set(ASMJIT_NO_ABI_NAMESPACE FALSE) +endif() + # AsmJit - Configuration - Backend # ================================ @@ -61,10 +69,6 @@ endif() # AsmJit - Configuration - Features # ================================= -if (NOT DEFINED ASMJIT_NO_DEPRECATED) - set(ASMJIT_NO_DEPRECATED FALSE) -endif() - if (NOT DEFINED ASMJIT_NO_SHM_OPEN) set(ASMJIT_NO_SHM_OPEN FALSE) endif() @@ -117,14 +121,16 @@ set(ASMJIT_NO_AARCH64 "${ASMJIT_NO_AARCH64}" CACHE BOOL "Disable AAr set(ASMJIT_NO_FOREIGN "${ASMJIT_NO_FOREIGN}" CACHE BOOL "Disable all foreign architectures (enables only a target architecture)") set(ASMJIT_NO_DEPRECATED "${ASMJIT_NO_DEPRECATED}" CACHE BOOL "Disable deprecated API at build time") +set(ASMJIT_NO_ABI_NAMESPACE "${ASMJIT_NO_ABI_NAMESPACE}" CACHE BOOL "Disable the use of ABI namespace (inline namespace in {asmjit} adding ABI version)") + set(ASMJIT_NO_SHM_OPEN "${ASMJIT_NO_SHM_OPEN}" CACHE BOOL "Disable the use of shm_open() even on platforms where it's supported") set(ASMJIT_NO_JIT "${ASMJIT_NO_JIT}" CACHE BOOL "Disable VirtMem, JitAllocator, and JitRuntime at build time") set(ASMJIT_NO_TEXT "${ASMJIT_NO_TEXT}" CACHE BOOL "Disable textual representation of instructions, enums, cpu features, ...") set(ASMJIT_NO_LOGGING "${ASMJIT_NO_LOGGING}" CACHE BOOL "Disable logging features at build time") set(ASMJIT_NO_VALIDATION "${ASMJIT_NO_VALIDATION}" CACHE BOOL "Disable instruction validation API at build time") set(ASMJIT_NO_INTROSPECTION "${ASMJIT_NO_INTROSPECTION}" CACHE BOOL "Disable instruction introspection API at build time") -set(ASMJIT_NO_BUILDER "${ASMJIT_NO_BUILDER}" CACHE BOOL "Disable Builder emitter at build time") -set(ASMJIT_NO_COMPILER "${ASMJIT_NO_COMPILER}" CACHE BOOL "Disable Compiler emitter at build time") +set(ASMJIT_NO_BUILDER "${ASMJIT_NO_BUILDER}" CACHE BOOL "Disable Builder at build time") +set(ASMJIT_NO_COMPILER "${ASMJIT_NO_COMPILER}" CACHE BOOL "Disable Compiler at build time") # AsmJit - Project # ================ @@ -333,11 +339,13 @@ endif() foreach(build_option # AsmJit build options. ASMJIT_STATIC ASMJIT_NO_DEPRECATED + ASMJIT_NO_ABI_NAMESPACE # AsmJit backends selection. ASMJIT_NO_X86 ASMJIT_NO_AARCH64 ASMJIT_NO_FOREIGN # AsmJit features selection. + ASMJIT_NO_SHM_OPEN ASMJIT_NO_JIT ASMJIT_NO_TEXT ASMJIT_NO_LOGGING @@ -400,6 +408,7 @@ set(ASMJIT_SRC_LIST asmjit/core/environment.h asmjit/core/errorhandler.cpp asmjit/core/errorhandler.h + asmjit/core/fixup.h asmjit/core/formatter.cpp asmjit/core/formatter.h asmjit/core/func.cpp @@ -462,7 +471,6 @@ set(ASMJIT_SRC_LIST asmjit/arm/armformatter.cpp asmjit/arm/armformatter_p.h asmjit/arm/armglobals.h - asmjit/arm/armoperand.h asmjit/arm/armutils.h asmjit/arm/a64archtraits_p.h asmjit/arm/a64assembler.cpp @@ -616,19 +624,20 @@ if (NOT ASMJIT_EMBED) CFLAGS_DBG ${ASMJIT_PRIVATE_CFLAGS_DBG} CFLAGS_REL ${ASMJIT_PRIVATE_CFLAGS_REL}) - asmjit_add_target(asmjit_test_perf EXECUTABLE - SOURCES test/asmjit_test_perf.cpp - test/asmjit_test_perf_a64.cpp - test/asmjit_test_perf_x86.cpp - SOURCES test/asmjit_test_perf.h + asmjit_add_target(asmjit_bench_codegen EXECUTABLE + SOURCES test/asmjit_bench_codegen.cpp + test/asmjit_bench_codegen_a64.cpp + test/asmjit_bench_codegen_x86.cpp + SOURCES test/asmjit_bench_codegen.h LIBRARIES asmjit::asmjit CFLAGS ${ASMJIT_PRIVATE_CFLAGS} CFLAGS_DBG ${ASMJIT_PRIVATE_CFLAGS_DBG} CFLAGS_REL ${ASMJIT_PRIVATE_CFLAGS_REL}) - foreach(_target asmjit_test_environment + foreach(_target asmjit_bench_overhead + asmjit_bench_regalloc + asmjit_test_environment asmjit_test_emitters - asmjit_test_execute asmjit_test_x86_sections) asmjit_add_target(${_target} TEST SOURCES test/${_target}.cpp @@ -655,24 +664,37 @@ if (NOT ASMJIT_EMBED) set(ASMJIT_SSE2_CFLAGS "") check_cxx_source_compiles(" - #if defined(_M_IX86) || defined(__X86__) || defined(__i386__) - int target_is_32_bit_x86() { return 1; } - #else - // Compile error... + #if defined(_M_X64) || defined(__x86_64__) + // Skip... + #elif defined(_M_IX86) || defined(__X86__) || defined(__i386__) + int target_arch_is_x86() { return 1; } #endif + int main() { return target_arch_is_x86(); } + " ASMJIT_TARGET_ARCH_X86) - int main() { - return target_is_32_bit_x86(); - } - " ASMJIT_TARGET_IS_32_BIT_X86) + check_cxx_source_compiles(" + #if defined(_M_X64) || defined(__x86_64__) + int target_arch_is_x86_64() { return 1; } + #endif + int main() { return target_arch_is_x86_64(); } + " ASMJIT_TARGET_ARCH_X86_64) - if (ASMJIT_TARGET_IS_32_BIT_X86) - if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC" OR "x${CMAKE_CXX_COMPILER_FRONTEND_VARIANT}" STREQUAL "xMSVC") - asmjit_detect_cflags(ASMJIT_SSE2_CFLAGS "-arch:SSE2") - else() - asmjit_detect_cflags(ASMJIT_SSE2_CFLAGS "-msse2") + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC" OR "x${CMAKE_CXX_COMPILER_FRONTEND_VARIANT}" STREQUAL "xMSVC") + if (ASMJIT_TARGET_ARCH_X86) + asmjit_detect_cflags(ASMJIT_SSE2_CFLAGS -arch:SSE2) + endif() + if (ASMJIT_TARGET_ARCH_X86 OR ASMJIT_TARGET_ARCH_X86_64) + asmjit_detect_cflags(ASMJIT_AVX2FMA_CFLAGS -arch:AVX2) + endif() + else() + if (ASMJIT_TARGET_ARCH_X86) + asmjit_detect_cflags(ASMJIT_SSE2_CFLAGS -msse2) + endif() + if (ASMJIT_TARGET_ARCH_X86 OR ASMJIT_TARGET_ARCH_X86_64) + asmjit_detect_cflags(ASMJIT_AVX2FMA_CFLAGS -mavx2 -mfma) endif() endif() + asmjit_add_target(asmjit_test_compiler TEST SOURCES test/asmjit_test_compiler.cpp test/asmjit_test_compiler.h diff --git a/db/aarch32.js b/db/aarch32.js index 7d80d41..edb4ded 100644 --- a/db/aarch32.js +++ b/db/aarch32.js @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib (function($scope, $as) { diff --git a/db/aarch64.js b/db/aarch64.js index 070d9fa..d014ab6 100644 --- a/db/aarch64.js +++ b/db/aarch64.js @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib (function($scope, $as) { diff --git a/db/base.js b/db/base.js index bedb13f..68b4a04 100644 --- a/db/base.js +++ b/db/base.js @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib (function($scope, $as) { diff --git a/db/exp.js b/db/exp.js index 956912a..4f0ff18 100644 --- a/db/exp.js +++ b/db/exp.js @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib (function($scope, $as) { diff --git a/db/index.js b/db/index.js index 61d8eb8..bf55e6f 100644 --- a/db/index.js +++ b/db/index.js @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib "use strict"; diff --git a/db/x86.js b/db/x86.js index b15489d..a40d03a 100644 --- a/db/x86.js +++ b/db/x86.js @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib (function($scope, $as) { diff --git a/src/asmjit.natvis b/src/asmjit.natvis index 68012e0..e40e395 100644 --- a/src/asmjit.natvis +++ b/src/asmjit.natvis @@ -72,7 +72,7 @@ immType() - + @@ -143,14 +143,14 @@ _data (asmjit::TypeId)(typeId()) - (asmjit::BaseReg::RegType)regType() + (asmjit::RegType)regType() regId() stackOffset() - + @@ -190,8 +190,8 @@ _prev _next - _any._nodeType - _any._nodeFlags + _nodeType + _nodeFlags _position _userDataU64 diff --git a/src/asmjit/a64.h b/src/asmjit/a64.h index 12a4af9..2aabf23 100644 --- a/src/asmjit/a64.h +++ b/src/asmjit/a64.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_A64_H_INCLUDED @@ -26,21 +26,13 @@ //! //! ### Register Operands //! -//! - \ref arm::Reg - Base class of all AArch32/AArch64 registers. -//! - \ref a64::Gp - General purpose register (AArch64): -//! - \ref a64::GpW - 32-bit general purpose register (AArch64). -//! - \ref a64::GpX - 64-bit general purpose register (AArch64). -//! - \ref a64::Vec - Vector (SIMD) register: -//! - \ref a64::VecB - 8-bit SIMD register. -//! - \ref a64::VecH - 16-bit SIMD register. -//! - \ref a64::VecS - 32-bit SIMD register. -//! - \ref a64::VecD - 64-bit SIMD register. -//! - \ref a64::VecV - 128-bit SIMD register. +//! - \ref a64::Gp - General purpose register (AArch64). +//! - \ref a64::Vec - Vector (SIMD) register. //! //! ### Memory Operands //! -//! - \ref arm::Mem - AArch32/AArch64 memory operand that provides support for all ARM addressing features -//! including base, index, pre/post increment, and ARM-specific shift addressing and index extending. +//! - \ref a64::Mem - AArch64 memory operand that provides support for all ARM addressing features including base, +//! index, pre/post increment, and ARM-specific shift addressing + index extending. //! //! ### Other //! diff --git a/src/asmjit/arm.h b/src/asmjit/arm.h index 0e4e673..ddc1e4f 100644 --- a/src/asmjit/arm.h +++ b/src/asmjit/arm.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_H_INCLUDED @@ -48,22 +48,13 @@ //! //! ### Register Operands //! -//! - \ref arm::Reg - Base class of all AArch32/AArch64 registers. +//! - AArch32: //! - \ref a32::Gp - 32-bit general purpose register used by AArch32: +//! - \ref a32::Vec - Vector (SIMD) register. +//! +//! - AArch64: //! - \ref a64::Gp - 32-bit or 64-bit general purpose register used by AArch64: -//! - \ref a64::GpW - 32-bit register (AArch64). -//! - \ref a64::GpX - 64-bit register (AArch64). -//! - \ref arm::BaseVec - Base vector (SIMD) register. -//! - \ref a32::Vec - Vector (SIMD) register (AArch32): -//! - \ref a32::VecS - 32-bit SIMD register (AArch32). -//! - \ref a32::VecD - 64-bit SIMD register (AArch32). -//! - \ref a32::VecV - 128-bit SIMD register (AArch32). -//! - \ref a64::Vec - Vector (SIMD) register (AArch64): -//! - \ref a64::VecB - 8-bit SIMD register (AArch64). -//! - \ref a64::VecH - 16-bit SIMD register (AArch64). -//! - \ref a64::VecS - 32-bit SIMD register (AArch64). -//! - \ref a64::VecD - 64-bit SIMD register (AArch64). -//! - \ref a64::VecV - 128-bit SIMD register (AArch64). +//! - \ref a64::Vec - Vector (SIMD) register. //! //! ### Memory Operands //! @@ -80,7 +71,6 @@ #include "asmjit-scope-begin.h" #include "arm/armglobals.h" -#include "arm/armoperand.h" #include "arm/armutils.h" #include "asmjit-scope-end.h" diff --git a/src/asmjit/arm/a64archtraits_p.h b/src/asmjit/arm/a64archtraits_p.h index 4b5bde6..f5dea59 100644 --- a/src/asmjit/arm/a64archtraits_p.h +++ b/src/asmjit/arm/a64archtraits_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64ARCHTRAITS_P_H_INCLUDED @@ -20,18 +20,28 @@ ASMJIT_BEGIN_SUB_NAMESPACE(a64) static const constexpr ArchTraits a64ArchTraits = { // SP/FP/LR/PC. - Gp::kIdSp, Gp::kIdFp, Gp::kIdLr, 0xFF, + Gp::kIdSp, Gp::kIdFp, Gp::kIdLr, 0xFFu, // Reserved. - { 0, 0, 0 }, + { 0u, 0u, 0u }, // HW stack alignment (AArch64 requires stack aligned to 16 bytes at HW level). - 16, + 16u, - // Min/max stack offset - byte addressing is the worst, VecQ addressing the best. + // Min/max stack offset - byte addressing is the worst, vec.q addressing the best. 4095, 65520, - // Instruction hints [Gp, Vec, ExtraVirt2, ExtraVirt3]. + // Supported register types. + 0u | (1u << uint32_t(RegType::kGp32 )) + | (1u << uint32_t(RegType::kGp64 )) + | (1u << uint32_t(RegType::kVec8 )) + | (1u << uint32_t(RegType::kVec16 )) + | (1u << uint32_t(RegType::kVec32 )) + | (1u << uint32_t(RegType::kVec64 )) + | (1u << uint32_t(RegType::kVec128)) + | (1u << uint32_t(RegType::kMask )), + + // Instruction hints [Gp, Vec, Mask, Extra]. {{ InstHints::kPushPop, InstHints::kPushPop, @@ -39,29 +49,19 @@ static const constexpr ArchTraits a64ArchTraits = { InstHints::kNoHints }}, - // RegInfo. - #define V(index) OperandSignature{RegTraits::kSignature} - {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, - #undef V - - // RegTypeToTypeId. - #define V(index) TypeId(RegTraits::kTypeId) - {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, - #undef V - // TypeIdToRegType. - #define V(index) (index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt8) ? RegType::kARM_GpW : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt8) ? RegType::kARM_GpW : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt16) ? RegType::kARM_GpW : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt16) ? RegType::kARM_GpW : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt32) ? RegType::kARM_GpW : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt32) ? RegType::kARM_GpW : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt64) ? RegType::kARM_GpX : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt64) ? RegType::kARM_GpX : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kIntPtr) ? RegType::kARM_GpX : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUIntPtr) ? RegType::kARM_GpX : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat32) ? RegType::kARM_VecS : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat64) ? RegType::kARM_VecD : RegType::kNone) + #define V(index) (index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt8) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt8) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt16) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt16) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt32) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt32) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt64) ? RegType::kGp64 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt64) ? RegType::kGp64 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kIntPtr) ? RegType::kGp64 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUIntPtr) ? RegType::kGp64 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat32) ? RegType::kVec32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat64) ? RegType::kVec64 : RegType::kNone) {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, #undef V diff --git a/src/asmjit/arm/a64assembler.cpp b/src/asmjit/arm/a64assembler.cpp index e4395ea..fc7d4f8 100644 --- a/src/asmjit/arm/a64assembler.cpp +++ b/src/asmjit/arm/a64assembler.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -57,14 +57,14 @@ static const uint8_t armShiftOpToLdStOptMap[] = { ASMJIT_LOOKUP_TABLE_16(VALUE, // ================================== static inline RegType extendOptionToRegType(uint32_t option) noexcept { - uint32_t pred = (uint32_t(RegType::kARM_GpW) << (0x0 * 4)) | // 0b000 - UXTB. - (uint32_t(RegType::kARM_GpW) << (0x1 * 4)) | // 0b001 - UXTH. - (uint32_t(RegType::kARM_GpW) << (0x2 * 4)) | // 0b010 - UXTW. - (uint32_t(RegType::kARM_GpX) << (0x3 * 4)) | // 0b011 - UXTX|LSL. - (uint32_t(RegType::kARM_GpW) << (0x4 * 4)) | // 0b100 - SXTB. - (uint32_t(RegType::kARM_GpW) << (0x5 * 4)) | // 0b101 - SXTH. - (uint32_t(RegType::kARM_GpW) << (0x6 * 4)) | // 0b110 - SXTW. - (uint32_t(RegType::kARM_GpX) << (0x7 * 4)) ; // 0b111 - SXTX. + uint32_t pred = (uint32_t(RegType::kGp32) << (0x0 * 4)) | // 0b000 - UXTB. + (uint32_t(RegType::kGp32) << (0x1 * 4)) | // 0b001 - UXTH. + (uint32_t(RegType::kGp32) << (0x2 * 4)) | // 0b010 - UXTW. + (uint32_t(RegType::kGp64) << (0x3 * 4)) | // 0b011 - UXTX|LSL. + (uint32_t(RegType::kGp32) << (0x4 * 4)) | // 0b100 - SXTB. + (uint32_t(RegType::kGp32) << (0x5 * 4)) | // 0b101 - SXTH. + (uint32_t(RegType::kGp32) << (0x6 * 4)) | // 0b110 - SXTW. + (uint32_t(RegType::kGp64) << (0x7 * 4)) ; // 0b111 - SXTX. return RegType((pred >> (option * 4u)) & 0xFu); } @@ -142,29 +142,29 @@ struct SizeOpTable { }; // 40 elements for each combination. - SizeOp array[(uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB) + 1) * 8]; + SizeOp array[(uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8) + 1) * 8]; }; #define VALUE_BIN(x) { \ - x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k00 : \ - x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k00Q : \ - x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00 : \ - x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00Q : SizeOp::kInvalid \ + x == (((uint32_t(RegType::kVec64 ) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k00 : \ + x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k00Q : \ + x == (((uint32_t(RegType::kVec64 ) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00 : \ + x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00Q : SizeOp::kInvalid \ } #define VALUE_ANY(x) { \ - x == (((uint32_t(RegType::kARM_VecB) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k00S : \ - x == (((uint32_t(RegType::kARM_VecH) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k01S : \ - x == (((uint32_t(RegType::kARM_VecS) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k10S : \ - x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k11S : \ - x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00 : \ - x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00Q : \ - x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kH )) ? SizeOp::k01 : \ - x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kH )) ? SizeOp::k01Q : \ - x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kS )) ? SizeOp::k10 : \ - x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kS )) ? SizeOp::k10Q : \ - x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kD )) ? SizeOp::k11S : \ - x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | uint32_t(VecElementType::kD )) ? SizeOp::k11Q : SizeOp::kInvalid \ + x == (((uint32_t(RegType::kVec8) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k00S : \ + x == (((uint32_t(RegType::kVec16) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k01S : \ + x == (((uint32_t(RegType::kVec32) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k10S : \ + x == (((uint32_t(RegType::kVec64) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k11S : \ + x == (((uint32_t(RegType::kVec64) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00 : \ + x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00Q : \ + x == (((uint32_t(RegType::kVec64) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kH )) ? SizeOp::k01 : \ + x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kH )) ? SizeOp::k01Q : \ + x == (((uint32_t(RegType::kVec64) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kS )) ? SizeOp::k10 : \ + x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kS )) ? SizeOp::k10Q : \ + x == (((uint32_t(RegType::kVec64) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kD )) ? SizeOp::k11S : \ + x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kD )) ? SizeOp::k11Q : SizeOp::kInvalid \ } static const SizeOpTable sizeOpTable[SizeOpTable::kCount] = { @@ -291,7 +291,7 @@ static inline SizeOp armElementTypeToSizeOp(uint32_t vecOpType, RegType regType, const SizeOpMap& map = sizeOpMap[vecOpType]; const SizeOpTable& table = sizeOpTable[map.tableId]; - size_t index = (Support::min(diff(regType, RegType::kARM_VecB), diff(RegType::kARM_VecV, RegType::kARM_VecB) + 1) << 3) | uint32_t(elementType); + size_t index = (Support::min(diff(regType, RegType::kVec8), diff(RegType::kVec128, RegType::kVec8) + 1) << 3) | uint32_t(elementType); SizeOp op = table.array[index]; SizeOp modifiedOp { uint8_t(op.value & map.sizeOpMask) }; @@ -506,15 +506,10 @@ static inline bool matchSignature(const Operand_& o0, const Operand_& o1, const // 2. Relative displacement (Label). // 3. Base register + either offset or index. static inline bool armCheckMemBaseIndexRel(const Mem& mem) noexcept { - // Allowed base types (Nothing, Label, and GpX). - constexpr uint32_t kBaseMask = B(0) | - B(RegType::kLabelTag) | - B(RegType::kARM_GpX); - - // Allowed index types (Nothing, GpW, and GpX). - constexpr uint32_t kIndexMask = B(0) | - B(RegType::kARM_GpW) | - B(RegType::kARM_GpX) ; + // Allowed base types (Nothing, Label, and Gp64). + constexpr uint32_t kBaseMask = B(0) | B(RegType::kLabelTag) | B(RegType::kGp64); + // Allowed index types (Nothing, Gp32, and Gp64). + constexpr uint32_t kIndexMask = B(0) | B(RegType::kGp32) | B(RegType::kGp64) ; RegType baseType = mem.baseType(); RegType indexType = mem.indexType(); @@ -524,7 +519,7 @@ static inline bool armCheckMemBaseIndexRel(const Mem& mem) noexcept { } if (baseType > RegType::kLabelTag) { - // Index allows either GpW or GpX. + // Index allows either Gp32 or Gp64. if (!Support::bitTest(kIndexMask, indexType)) { return false; } @@ -561,7 +556,7 @@ static inline bool pickFpOpcode(const Vec& reg, uint32_t sOp, uint32_t sHf, uint if (!reg.hasElementType()) { // Scalar operation [HSD]. - uint32_t sz = diff(reg.type(), RegType::kARM_VecH); + uint32_t sz = diff(reg.regType(), RegType::kVec16); if (sz > 2u || !Support::bitTest(szBits[sHf].sizeMask, sz)) { return false; } @@ -572,7 +567,7 @@ static inline bool pickFpOpcode(const Vec& reg, uint32_t sOp, uint32_t sHf, uint } else { // Vector operation [HSD]. - uint32_t q = diff(reg.type(), RegType::kARM_VecD); + uint32_t q = diff(reg.regType(), RegType::kVec64); uint32_t sz = diff(reg.elementType(), VecElementType::kH); if (q > 1u || sz > 2u || !Support::bitTest(szBits[vHf].sizeMask, sz)) { @@ -611,22 +606,21 @@ static inline bool checkSignature(const Operand_& o0, const Operand_& o1, const // Checks whether the register is GP register of the allowed types. // -// Allowed is a 2-bit mask, where the first bits allows GpW and the second bit -// allows GpX. These bits are usually stored within the instruction, but could -// be also hardcoded in the assembler for instructions where GP types are not -// selectable. +// Allowed is a 2-bit mask, where the first bits allows Gp32 and the second bit allows Gp64. These bits are usually +// stored within the instruction, but could be also hardcoded in the assembler for instructions where GP types are +// not selectable. static inline bool checkGpType(const Operand_& op, uint32_t allowed) noexcept { - RegType type = op.as().type(); - return Support::bitTest(allowed << uint32_t(RegType::kARM_GpW), type); + RegType type = op.as().regType(); + return Support::bitTest(allowed << uint32_t(RegType::kGp32), type); } static inline bool checkGpType(const Operand_& op, uint32_t allowed, uint32_t* x) noexcept { - // NOTE: We set 'x' to one only when GpW is allowed, otherwise the X is part + // NOTE: We set 'x' to one only when Gp32 is allowed, otherwise the X is part // of the opcode and we cannot set it. This is why this works without requiring // additional logic. - RegType type = op.as().type(); - *x = diff(type, RegType::kARM_GpW) & allowed; - return Support::bitTest(allowed << uint32_t(RegType::kARM_GpW), type); + RegType type = op.as().regType(); + *x = diff(type, RegType::kGp32) & allowed; + return Support::bitTest(allowed << uint32_t(RegType::kGp32), type); } static inline bool checkGpType(const Operand_& o0, const Operand_& o1, uint32_t allowed, uint32_t* x) noexcept { @@ -689,7 +683,7 @@ static inline bool checkVecId(const Operand_& o0, const Operand_& o1, const Oper */ static inline bool checkMemBase(const Mem& mem) noexcept { - return mem.baseType() == RegType::kARM_GpX && mem.baseId() <= 31; + return mem.baseType() == RegType::kGp64 && mem.baseId() <= 31; } static inline bool checkEven(const Operand_& o0, const Operand_& o1) noexcept { @@ -714,38 +708,38 @@ static inline bool checkConsecutive(const Operand_& o0, const Operand_& o1, cons // a64::Assembler - CheckReg // ========================= -#define V(index) (index == uint32_t(RegType::kARM_GpW) ? Gp::kIdZr : \ - index == uint32_t(RegType::kARM_GpX) ? Gp::kIdZr : \ - index == uint32_t(RegType::kARM_VecB) ? 31u : \ - index == uint32_t(RegType::kARM_VecH) ? 31u : \ - index == uint32_t(RegType::kARM_VecS) ? 31u : \ - index == uint32_t(RegType::kARM_VecD) ? 31u : \ - index == uint32_t(RegType::kARM_VecV) ? 31u : 0) +#define V(index) (index == uint32_t(RegType::kGp32) ? Gp::kIdZr : \ + index == uint32_t(RegType::kGp64) ? Gp::kIdZr : \ + index == uint32_t(RegType::kVec8) ? 31u : \ + index == uint32_t(RegType::kVec16) ? 31u : \ + index == uint32_t(RegType::kVec32) ? 31u : \ + index == uint32_t(RegType::kVec64) ? 31u : \ + index == uint32_t(RegType::kVec128) ? 31u : 0) static const Support::Array commonHiRegIdOfType = {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}; #undef V static inline bool checkValidRegs(const Operand_& o0) noexcept { - return bool(unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().type()])); + return bool(unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().regType()])); } static inline bool checkValidRegs(const Operand_& o0, const Operand_& o1) noexcept { - return bool((unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().type()])) & - (unsigned(o1.id() < 31) | unsigned(o1.id() == commonHiRegIdOfType[o1.as().type()]))); + return bool((unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().regType()])) & + (unsigned(o1.id() < 31) | unsigned(o1.id() == commonHiRegIdOfType[o1.as().regType()]))); } static inline bool checkValidRegs(const Operand_& o0, const Operand_& o1, const Operand_& o2) noexcept { - return bool((unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().type()])) & - (unsigned(o1.id() < 31) | unsigned(o1.id() == commonHiRegIdOfType[o1.as().type()])) & - (unsigned(o2.id() < 31) | unsigned(o2.id() == commonHiRegIdOfType[o2.as().type()]))); + return bool((unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().regType()])) & + (unsigned(o1.id() < 31) | unsigned(o1.id() == commonHiRegIdOfType[o1.as().regType()])) & + (unsigned(o2.id() < 31) | unsigned(o2.id() == commonHiRegIdOfType[o2.as().regType()]))); } static inline bool checkValidRegs(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3) noexcept { - return bool((unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().type()])) & - (unsigned(o1.id() < 31) | unsigned(o1.id() == commonHiRegIdOfType[o1.as().type()])) & - (unsigned(o2.id() < 31) | unsigned(o2.id() == commonHiRegIdOfType[o2.as().type()])) & - (unsigned(o3.id() < 31) | unsigned(o3.id() == commonHiRegIdOfType[o3.as().type()]))); + return bool((unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().regType()])) & + (unsigned(o1.id() < 31) | unsigned(o1.id() == commonHiRegIdOfType[o1.as().regType()])) & + (unsigned(o2.id() < 31) | unsigned(o2.id() == commonHiRegIdOfType[o2.as().regType()])) & + (unsigned(o3.id() < 31) | unsigned(o3.id() == commonHiRegIdOfType[o3.as().regType()]))); } // a64::Assembler - Construction & Destruction @@ -753,8 +747,11 @@ static inline bool checkValidRegs(const Operand_& o0, const Operand_& o1, const Assembler::Assembler(CodeHolder* code) noexcept : BaseAssembler() { _archMask = uint64_t(1) << uint32_t(Arch::kAArch64); - if (code) + initEmitterFuncs(this); + + if (code) { code->attach(this); + } } Assembler::~Assembler() noexcept {} @@ -874,6 +871,17 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co break; } + case InstDB::kEncodingBaseOpX16: { + const InstDB::EncodingData::BaseOpX16& opData = InstDB::EncodingData::baseOpX16[encodingIndex]; + + if (isign4 == ENC_OPS1(Reg) && o0.as().isGp64(16)) { + opcode.reset(opData.opcode); + goto EmitOp; + } + + break; + } + case InstDB::kEncodingBaseOpImm: { const InstDB::EncodingData::BaseOpImm& opData = InstDB::EncodingData::baseOpImm[encodingIndex]; @@ -1065,7 +1073,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co case InstDB::kEncodingBaseMov: { // MOV is a pseudo instruction that uses various instructions depending on its signature. - uint32_t x = diff(o0.as().type(), RegType::kARM_GpW); + uint32_t x = diff(o0.as().regType(), RegType::kGp32); if (x > 1) goto InvalidInstruction; @@ -1144,7 +1152,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co case InstDB::kEncodingBaseMovKNZ: { const InstDB::EncodingData::BaseMovKNZ& opData = InstDB::EncodingData::baseMovKNZ[encodingIndex]; - uint32_t x = diff(o0.as().type(), RegType::kARM_GpW); + uint32_t x = diff(o0.as().regType(), RegType::kGp32); if (x > 1) goto InvalidInstruction; @@ -1198,7 +1206,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co const InstDB::EncodingData::BaseAdr& opData = InstDB::EncodingData::baseAdr[encodingIndex]; if (isign4 == ENC_OPS2(Reg, Label) || isign4 == ENC_OPS2(Reg, Imm)) { - if (!o0.as().isGpX()) + if (!o0.as().isGp64()) goto InvalidInstruction; if (!checkGpId(o0, kZR)) @@ -1337,7 +1345,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co } // Validate whether the register operands match extend option. - if (o2.as().type() != extendOptionToRegType(sType) || o1.as().type() < o2.as().type()) { + if (o2.as().regType() != extendOptionToRegType(sType) || o1.as().regType() < o2.as().regType()) { goto InvalidInstruction; } @@ -1512,7 +1520,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co } // Validate whether the register operands match extend option. - if (o1.as().type() != extendOptionToRegType(sType) || o0.as().type() < o1.as().type()) { + if (o1.as().regType() != extendOptionToRegType(sType) || o0.as().regType() < o1.as().regType()) { goto InvalidInstruction; } @@ -1784,7 +1792,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co if (!checkGpType(o0, opData.rType, &x)) goto InvalidInstruction; - if (!o1.as().isGpW()) + if (!o1.as().isGp32()) goto InvalidInstruction; if (!checkGpId(o0, o1)) @@ -2058,6 +2066,63 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co break; } + // ------------------------------------------------------------------------ + // [Base - Min/Max] + // ------------------------------------------------------------------------ + + case InstDB::kEncodingBaseMinMax: { + const InstDB::EncodingData::BaseMinMax& opData = InstDB::EncodingData::baseMinMax[encodingIndex]; + + if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { + uint32_t x; + if (!checkGpType(o0, InstDB::kWX, &x)) + goto InvalidInstruction; + + if (!checkSignature(o0, o1, o2)) + goto InvalidInstruction; + + opcode.reset(opData.regOp); + opcode.addImm(x, 31); + opcode.addReg(o2, 16); + opcode.addReg(o1, 5); + opcode.addReg(o0, 0); + goto EmitOp; + } + + if (isign4 == ENC_OPS3(Reg, Reg, Imm)) { + uint32_t x; + if (!checkGpType(o0, InstDB::kWX, &x)) + goto InvalidInstruction; + + if (!checkSignature(o0, o1)) + goto InvalidInstruction; + + uint64_t imm = o2.as().valueAs(); + + if (opData.immOp & (1u << 18)) { + // Zero extend imm. + if (!Support::isUInt8(imm)) { + goto InvalidImmediate; + } + } + else { + // Sign extend imm. + if (!Support::isInt8(int64_t(imm))) { + goto InvalidImmediate; + } + } + + opcode.reset(opData.immOp); + opcode.addImm(x, 31); + opcode.addImm(uint32_t(imm & 0xFFu), 10); + opcode.addReg(o1, 5); + opcode.addReg(o0, 0); + goto EmitOp; + } + + break; + } + // ------------------------------------------------------------------------ // [Base - Special] // ------------------------------------------------------------------------ @@ -2078,7 +2143,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co uint32_t rt = 31; if (o1.isReg()) { - if (!o1.as().isGpX()) + if (!o1.as().isGp64()) goto InvalidInstruction; if (!checkGpId(o1, kZR)) @@ -2097,7 +2162,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co case InstDB::kEncodingBaseMrs: { if (isign4 == ENC_OPS2(Reg, Imm)) { - if (!o0.as().isGpX()) + if (!o0.as().isGp64()) goto InvalidInstruction; if (!checkGpId(o0, kZR)) @@ -2121,7 +2186,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co case InstDB::kEncodingBaseMsr: { if (isign4 == ENC_OPS2(Imm, Reg)) { - if (!o1.as().isGpX()) + if (!o1.as().isGp64()) goto InvalidInstruction; if (o0.as().valueAs() > 0xFFFFu) @@ -2179,7 +2244,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co const Operand_& o4 = opExt[EmitterUtils::kOp4]; if (o4.isReg()) { - if (!o4.as().isGpX()) + if (!o4.as().isGp64()) goto InvalidInstruction; if (!checkGpId(o4, kZR)) @@ -2211,7 +2276,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co const InstDB::EncodingData::BaseBranchReg& opData = InstDB::EncodingData::baseBranchReg[encodingIndex]; if (isign4 == ENC_OPS1(Reg)) { - if (!o0.as().isGpX()) + if (!o0.as().isGp64()) goto InvalidInstruction; if (!checkGpId(o0, kZR)) @@ -2232,7 +2297,13 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co opcode.reset(opData.opcode); rmRel = &o0; - if (instCC != CondCode::kAL) { + // A variation that uses Cond code (or where Cond code is forced like BC.). + if (instCC != CondCode::kAL || Support::bitTest(opcode.v, 30)) { + if (opcode.hasX()) { + // Condition code cannot be applied when the instruction has X bit set (this would be BL instruction). + goto InvalidInstruction; + } + opcode |= B(30); opcode.addImm(condCodeToOpcodeCond(uint32_t(instCC)), 0); offsetFormat.resetToImmValue(OffsetType::kSignedOffset, 4, 5, 19, 2); @@ -2497,7 +2568,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co if (!checkGpId(o0, o1, kZR)) goto InvalidPhysId; - if (m.baseType() != RegType::kARM_GpX || m.hasIndex()) + if (m.baseType() != RegType::kGp64 || m.hasIndex()) goto InvalidAddress; if (m.isOffset64Bit()) @@ -2542,7 +2613,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co const Mem& m = o2.as(); uint32_t x; - if (!o0.as().isGpW() || !checkGpType(o1, opData.rType, &x)) + if (!o0.as().isGp32() || !checkGpType(o1, opData.rType, &x)) goto InvalidInstruction; if (!checkGpId(o0, o1, kZR)) @@ -2592,7 +2663,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co const Mem& m = o3.as(); uint32_t x; - if (!o0.as().isGpW() || !checkGpType(o1, opData.rType, &x) || !checkSignature(o1, o2)) + if (!o0.as().isGp32() || !checkGpType(o1, opData.rType, &x) || !checkSignature(o1, o2)) goto InvalidInstruction; if (!checkGpId(o0, o1, o2, kZR)) @@ -2819,7 +2890,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::FSimdSV& opData = InstDB::EncodingData::fSimdSV[encodingIndex]; if (isign4 == ENC_OPS2(Reg, Reg)) { - uint32_t q = diff(o1.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o1.as().regType(), RegType::kVec64); if (q > 1) goto InvalidInstruction; @@ -2829,7 +2900,7 @@ Case_BaseLdurStur: // This operation is only defined for: // hD, vS.{4|8}h (16-bit) // sD, vS.4s (32-bit) - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); uint32_t elementSz = diff(o1.as().elementType(), VecElementType::kH); // Size greater than 1 means 64-bit elements, not supported. @@ -2901,7 +2972,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, instFlags)) goto InvalidInstruction; - uint32_t q = o1.as().isVecQ(); + uint32_t q = o1.as().isVec128(); uint32_t sz; if (!pickFpOpcode(o0.as(), opData.elementScalarOp(), InstDB::kHF_D, opData.elementVectorOp(), InstDB::kHF_D, &opcode, &sz)) @@ -2948,7 +3019,7 @@ Case_BaseLdurStur: if (!checkSignature(o0, o1, o2) || o0.as().hasElementIndex()) goto InvalidInstruction; - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); if (q > 1) goto InvalidInstruction; @@ -2978,7 +3049,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdFccmpFccmpe& opData = InstDB::EncodingData::simdFccmpFccmpe[encodingIndex]; if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) { - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); if (sz > 2) goto InvalidInstruction; @@ -3040,7 +3111,7 @@ Case_BaseLdurStur: if (!checkSignature(o0, o1)) goto InvalidInstruction; - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); if (q > 1) goto InvalidInstruction; @@ -3105,7 +3176,7 @@ Case_BaseLdurStur: case InstDB::kEncodingSimdFcmpFcmpe: { const InstDB::EncodingData::SimdFcmpFcmpe& opData = InstDB::EncodingData::simdFcmpFcmpe[encodingIndex]; - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); uint32_t type = (sz - 1) & 0x3u; if (sz > 2) @@ -3140,7 +3211,7 @@ Case_BaseLdurStur: if (!checkSignature(o0, o1, o2)) goto InvalidInstruction; - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); uint32_t type = (sz - 1) & 0x3u; if (sz > 2 || o0.as().hasElementType()) @@ -3161,8 +3232,8 @@ Case_BaseLdurStur: case InstDB::kEncodingSimdFcvt: { if (isign4 == ENC_OPS2(Reg, Reg)) { - uint32_t dstSz = diff(o0.as().type(), RegType::kARM_VecH); - uint32_t srcSz = diff(o1.as().type(), RegType::kARM_VecH); + uint32_t dstSz = diff(o0.as().regType(), RegType::kVec16); + uint32_t srcSz = diff(o1.as().regType(), RegType::kVec16); if ((dstSz | srcSz) > 3) goto InvalidInstruction; @@ -3205,7 +3276,7 @@ Case_BaseLdurStur: if (isign4 == ENC_OPS2(Reg, Reg)) { // Scalar form - only FCVTXN. - if (o0.as().isVecS() && o1.as().isVecD()) { + if (o0.as().isVec32() && o1.as().isVec64()) { if (!opData.hasScalar()) goto InvalidInstruction; @@ -3222,7 +3293,7 @@ Case_BaseLdurStur: const Vec& rL = (instFlags & InstDB::kInstFlagLong) ? o0.as() : o1.as(); const Vec& rN = (instFlags & InstDB::kInstFlagLong) ? o1.as() : o0.as(); - uint32_t q = diff(rN.type(), RegType::kARM_VecD); + uint32_t q = diff(rN.regType(), RegType::kVec64); if (uint32_t(opcode.hasQ()) != q) goto InvalidInstruction; @@ -3248,8 +3319,8 @@ Case_BaseLdurStur: if (isign4 == ENC_OPS2(Reg, Reg)) { if (oGp.as().isGp() && oVec.as().isVec()) { - uint32_t x = oGp.as().isGpX(); - uint32_t type = diff(oVec.as().type(), RegType::kARM_VecH); + uint32_t x = oGp.as().isGp64(); + uint32_t type = diff(oVec.as().regType(), RegType::kVec16); if (type > 2u) goto InvalidInstruction; @@ -3281,8 +3352,8 @@ Case_BaseLdurStur: goto InvalidInstruction; if (oGp.as().isGp() && oVec.as().isVec()) { - uint32_t x = oGp.as().isGpX(); - uint32_t type = diff(oVec.as().type(), RegType::kARM_VecH); + uint32_t x = oGp.as().isGp64(); + uint32_t type = diff(oVec.as().regType(), RegType::kVec16); uint32_t scaleLimit = 32u << x; if (scale > scaleLimit) @@ -3321,7 +3392,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdFmlal& opData = InstDB::EncodingData::simdFmlal[encodingIndex]; if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); uint32_t qIsOptional = opData.optionalQ(); if (qIsOptional) { @@ -3340,7 +3411,7 @@ Case_BaseLdurStur: q = 0; } - if (uint32_t(o0.as().type()) != uint32_t(o1.as().type()) + qIsOptional || + if (uint32_t(o0.as().regType()) != uint32_t(o1.as().regType()) + qIsOptional || uint32_t(o0.as().elementType()) != opData.tA || uint32_t(o1.as().elementType()) != opData.tB) goto InvalidInstruction; @@ -3386,8 +3457,8 @@ Case_BaseLdurStur: // FMOV Wd, Sn (sf=0 type=00 rmode=00 op=110) // FMOV Xd, Dn (sf=1 type=11 rmode=00 op=110) // FMOV Xd, Vn.d[1] (sf=1 type=10 rmode=01 op=110) - uint32_t x = o0.as().isGpX(); - uint32_t sz = diff(o1.as().type(), RegType::kARM_VecH); + uint32_t x = o0.as().isGp64(); + uint32_t sz = diff(o1.as().regType(), RegType::kVec16); uint32_t type = (sz - 1) & 0x3u; uint32_t rModeOp = 0b00110; @@ -3407,10 +3478,10 @@ Case_BaseLdurStur: if (o1.as().hasElementType()) goto InvalidInstruction; - if (o1.as().isVecS() && x) + if (o1.as().isVec32() && x) goto InvalidInstruction; - if (o1.as().isVecD() && !x) + if (o1.as().isVec64() && !x) goto InvalidInstruction; } @@ -3426,8 +3497,8 @@ Case_BaseLdurStur: // FMOV Sd, Wn (sf=0 type=00 rmode=00 op=111) // FMOV Dd, Xn (sf=1 type=11 rmode=00 op=111) // FMOV Vd.d[1], Xn (sf=1 type=10 rmode=01 op=111) - uint32_t x = o1.as().isGpX(); - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); + uint32_t x = o1.as().isGp64(); + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); uint32_t type = (sz - 1) & 0x3u; uint32_t rModeOp = 0b00111; @@ -3447,10 +3518,10 @@ Case_BaseLdurStur: if (o0.as().hasElementType()) goto InvalidInstruction; - if (o0.as().isVecS() && x) + if (o0.as().isVec32() && x) goto InvalidInstruction; - if (o0.as().isVecD() && !x) + if (o0.as().isVec64() && !x) goto InvalidInstruction; } @@ -3461,7 +3532,7 @@ Case_BaseLdurStur: } if (checkSignature(o0, o1)) { - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); if (sz > 2) goto InvalidInstruction; @@ -3491,7 +3562,7 @@ Case_BaseLdurStur: uint32_t imm8 = Utils::encodeFP64ToImm8(fpValue); if (!o0.as().hasElementType()) { // FMOV (scalar, immediate). - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); uint32_t type = (sz - 1u) & 0x3u; if (sz > 2) @@ -3503,7 +3574,7 @@ Case_BaseLdurStur: goto EmitOp_Rd0; } else { - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); uint32_t sz = diff(o0.as().elementType(), VecElementType::kH); if (q > 1 || sz > 2) @@ -3531,14 +3602,14 @@ Case_BaseLdurStur: // hD, vS.2h (16-bit) // sD, vS.2s (32-bit) // dD, vS.2d (64-bit) - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); if (sz > 2) goto InvalidInstruction; static const uint32_t szSignatures[3] = { - VecS::kSignature | (Vec::kSignatureElementH), - VecD::kSignature | (Vec::kSignatureElementS), - VecV::kSignature | (Vec::kSignatureElementD) + RegTraits::kSignature | (Vec::kSignatureElementH), + RegTraits::kSignature | (Vec::kSignatureElementS), + RegTraits::kSignature | (Vec::kSignatureElementD) }; if (o1.signature() != szSignatures[sz]) @@ -3554,7 +3625,7 @@ Case_BaseLdurStur: if (!checkSignature(o0, o1, o2)) goto InvalidInstruction; - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); if (q > 1) goto InvalidInstruction; @@ -3582,10 +3653,10 @@ Case_BaseLdurStur: if (isign4 == ENC_OPS2(Reg, Reg)) { // The first destination operand is scalar, which matches element-type of source vectors. uint32_t L = (instFlags & InstDB::kInstFlagLong) != 0; - if (diff(o0.as().type(), RegType::kARM_VecB) != diff(o1.as().elementType(), VecElementType::kB) + L) + if (diff(o0.as().regType(), RegType::kVec8) != diff(o1.as().elementType(), VecElementType::kB) + L) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().type(), o1.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().regType(), o1.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3606,7 +3677,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, instFlags)) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().type(), sop.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().regType(), sop.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3643,7 +3714,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, o2, instFlags)) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().type(), sop.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().regType(), sop.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3678,11 +3749,11 @@ Case_BaseLdurStur: const InstDB::EncodingData::ISimdWWV& opData = InstDB::EncodingData::iSimdWWV[encodingIndex]; if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o2.as().type(), o2.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o2.as().regType(), o2.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; - if (!checkSignature(o0, o1) || !o0.as().isVecV() || uint32_t(o0.as().elementType()) != uint32_t(o2.as().elementType()) + 1u) + if (!checkSignature(o0, o1) || !o0.as().isVec128() || uint32_t(o0.as().elementType()) != uint32_t(o2.as().elementType()) + 1u) goto InvalidInstruction; opcode.reset(opData.opcode()); @@ -3704,7 +3775,7 @@ Case_BaseLdurStur: goto InvalidInstruction; if (!o2.as().hasElementIndex()) { - SizeOp sizeOp = armElementTypeToSizeOp(opData.regularVecType, sop.as().type(), sop.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.regularVecType, sop.as().regType(), sop.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3718,7 +3789,7 @@ Case_BaseLdurStur: goto EmitOp_Rd0_Rn5_Rm16; } else { - SizeOp sizeOp = armElementTypeToSizeOp(opData.elementVecType, sop.as().type(), sop.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.elementVecType, sop.as().regType(), sop.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3751,7 +3822,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, o2, instFlags)) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().type(), sop.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().regType(), sop.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3784,7 +3855,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, o2, o3, instFlags)) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().type(), sop.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().regType(), sop.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3831,7 +3902,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, o2, instFlags)) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(opData.opType3, o0.as().type(), o0.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.opType3, o0.as().regType(), o0.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3852,7 +3923,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, o2, instFlags)) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(InstDB::kVO_V_B, o0.as().type(), o0.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(InstDB::kVO_V_B, o0.as().regType(), o0.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3862,7 +3933,7 @@ Case_BaseLdurStur: } if (isign4 == ENC_OPS2(Reg, Imm) || isign4 == ENC_OPS3(Reg, Imm, Imm)) { - SizeOp sizeOp = armElementTypeToSizeOp(InstDB::kVO_V_HS, o0.as().type(), o0.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(InstDB::kVO_V_HS, o0.as().regType(), o0.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3918,7 +3989,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, o2, instFlags)) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o0.as().type(), o0.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o0.as().regType(), o0.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3936,7 +4007,7 @@ Case_BaseLdurStur: if (o2.as().value() != 0) goto InvalidImmediate; - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o0.as().type(), o0.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o0.as().regType(), o0.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3954,7 +4025,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdDot& opData = InstDB::EncodingData::simdDot[encodingIndex]; if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); uint32_t size = 2; if (q > 1u) @@ -3964,7 +4035,7 @@ Case_BaseLdurStur: if (!opData.vectorOp) goto InvalidInstruction; - if (o0.as().type() != o1.as().type() || o1.as().type() != o2.as().type()) + if (o0.as().regType() != o1.as().regType() || o1.as().regType() != o2.as().regType()) goto InvalidInstruction; if (uint32_t(o0.as().elementType()) != opData.tA || @@ -3980,7 +4051,7 @@ Case_BaseLdurStur: if (!opData.elementOp) goto InvalidInstruction; - if (o0.as().type() != o1.as().type() || !o2.as().isVecV()) + if (o0.as().regType() != o1.as().regType() || !o2.as().isVec128()) goto InvalidInstruction; if (uint32_t(o0.as().elementType()) != opData.tA || @@ -4019,7 +4090,7 @@ Case_BaseLdurStur: B(uint32_t(VecElementType::kS) + 8) | B(uint32_t(VecElementType::kD) + 8) ; - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); if (o1.as().isGp()) { // DUP - Vec (scalar|vector) <- GP register. @@ -4045,7 +4116,7 @@ Case_BaseLdurStur: uint32_t dstIndex = o1.as().elementIndex(); if (!o0.as().hasElementType()) { // DUP - Vec (scalar) <- Vec[N]. - uint32_t lsbIndex = diff(o0.as().type(), RegType::kARM_VecB); + uint32_t lsbIndex = diff(o0.as().regType(), RegType::kVec8); if (lsbIndex != diff(o1.as().elementType(), VecElementType::kB) || lsbIndex > 3) goto InvalidInstruction; @@ -4081,7 +4152,7 @@ Case_BaseLdurStur: } case InstDB::kEncodingSimdIns: SimdIns: { - if (isign4 == ENC_OPS2(Reg, Reg) && o0.as().isVecV()) { + if (isign4 == ENC_OPS2(Reg, Reg) && o0.as().isVec128()) { if (!o0.as().hasElementIndex()) goto InvalidInstruction; @@ -4099,13 +4170,13 @@ Case_BaseLdurStur: opcode.addImm(imm5, 16); goto EmitOp_Rd0_Rn5; } - else if (o1.as().isVecV() && o1.as().hasElementIndex()) { + else if (o1.as().isVec128() && o1.as().hasElementIndex()) { // INS - Vec[N] <- Vec[M]. if (o0.as().elementType() != o1.as().elementType()) goto InvalidInstruction; uint32_t srcIndex = o1.as().elementIndex(); - if (o0.as().type() != o1.as().type()) + if (o0.as().regType() != o1.as().regType()) goto InvalidInstruction; uint32_t imm4 = srcIndex << lsbIndex; @@ -4137,7 +4208,7 @@ Case_BaseLdurStur: goto InvalidInstruction; // ORR Vd, Vn, Vm - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); if (q > 1) goto InvalidInstruction; @@ -4169,7 +4240,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdMoviMvni& opData = InstDB::EncodingData::simdMoviMvni[encodingIndex]; if (isign4 == ENC_OPS2(Reg, Imm) || isign4 == ENC_OPS3(Reg, Imm, Imm)) { - SizeOp sizeOp = armElementTypeToSizeOp(InstDB::kVO_V_Any, o0.as().type(), o0.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(InstDB::kVO_V_Any, o0.as().regType(), o0.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -4313,7 +4384,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdShift& opData = InstDB::EncodingData::simdShift[encodingIndex]; const Operand_& sop = significantSimdOp(o0, o1, instFlags); - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().type(), sop.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().regType(), sop.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -4365,7 +4436,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdShiftES& opData = InstDB::EncodingData::simdShiftES[encodingIndex]; if (isign4 == ENC_OPS3(Reg, Reg, Imm)) { - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().type(), o1.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().regType(), o1.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -4411,14 +4482,14 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdSmovUmov& opData = InstDB::EncodingData::simdSmovUmov[encodingIndex]; if (isign4 == ENC_OPS2(Reg, Reg) && o0.as().isGp() && o1.as().isVec()) { - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().type(), o1.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().regType(), o1.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; if (!o1.as().hasElementIndex()) goto InvalidInstruction; - uint32_t x = o0.as().isGpX(); + uint32_t x = o0.as().isGp64(); uint32_t gpMustBeX = uint32_t(sizeOp.size() >= 3u - opData.isSigned); if (opData.isSigned) { @@ -4451,7 +4522,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdSxtlUxtl& opData = InstDB::EncodingData::simdSxtlUxtl[encodingIndex]; if (isign4 == ENC_OPS2(Reg, Reg)) { - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().type(), o1.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().regType(), o1.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -4480,7 +4551,7 @@ Case_BaseLdurStur: const Operand_& o4 = opExt[EmitterUtils::kOp4]; const Operand_& o5 = opExt[EmitterUtils::kOp5]; - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); if (q > 1 || o0.as().hasElementIndex()) goto InvalidInstruction; @@ -4559,7 +4630,7 @@ Case_BaseLdurStur: // 32-bit | size==10 | opc == 01 | 010 // 64-bit | size==11 | opc == 01 | 011 // 128-bit| size==00 | opc == 11 | 100 - uint32_t xsz = diff(o0.as().type(), RegType::kARM_VecB); + uint32_t xsz = diff(o0.as().regType(), RegType::kVec8); if (xsz > 4u || o0.as().hasElementIndex()) goto InvalidRegType; @@ -4656,7 +4727,7 @@ Case_BaseLdurStur: const Mem& m = o2.as(); rmRel = &m; - uint32_t opc = diff(o0.as().type(), RegType::kARM_VecS); + uint32_t opc = diff(o0.as().regType(), RegType::kVec32); if (opc > 2u || o0.as().hasElementTypeOrIndex()) goto InvalidInstruction; @@ -4666,7 +4737,7 @@ Case_BaseLdurStur: if (!checkVecId(o0, o1)) goto InvalidPhysId; - if (m.baseType() != RegType::kARM_GpX || m.hasIndex()) + if (m.baseType() != RegType::kGp64 || m.hasIndex()) goto InvalidAddress; if (m.isOffset64Bit()) @@ -4712,7 +4783,7 @@ Case_SimdLdurStur: const Mem& m = o1.as(); rmRel = &m; - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecB); + uint32_t sz = diff(o0.as().regType(), RegType::kVec8); if (sz > 4 || o0.as().hasElementTypeOrIndex()) goto InvalidInstruction; @@ -4804,7 +4875,7 @@ Case_SimdLdurStur: if (sz > 3) goto InvalidInstruction; - if (m.baseType() != RegType::kARM_GpX) + if (m.baseType() != RegType::kGp64) goto InvalidAddress; // Rn cannot be ZR, but can be SP. @@ -4821,7 +4892,7 @@ Case_SimdLdurStur: if (v.hasElementIndex()) goto InvalidInstruction; - q = diff(v.type(), RegType::kARM_VecD); + q = diff(v.regType(), RegType::kVec64); if (q > 1) goto InvalidInstruction; @@ -4853,7 +4924,7 @@ Case_SimdLdurStur: // LDx/STx (multiple structures). static const uint8_t opcSsizeByN[] = { 0u, 0x7u << 2, 0xAu << 2, 0x6u << 2, 0x2u << 2 }; - q = diff(v.type(), RegType::kARM_VecD); + q = diff(v.regType(), RegType::kVec64); if (q > 1) goto InvalidInstruction; @@ -5026,26 +5097,27 @@ EmitOp_Rel: labelOffset = rmRel->as().offset(); } - LabelEntry* label = _code->labelEntry(labelId); - if (ASMJIT_UNLIKELY(!label)) { + if (ASMJIT_UNLIKELY(!_code->isLabelValid(labelId))) { goto InvalidLabel; } + LabelEntry& le = _code->labelEntry(labelId); + if (offsetFormat.type() == OffsetType::kAArch64_ADRP) { // TODO: [ARM] Always create relocation entry. } - if (label->isBoundTo(_section)) { + if (le.isBoundTo(_section)) { // Label bound to the current section. - offsetValue = label->offset() - uint64_t(offset()) + uint64_t(labelOffset); + offsetValue = le.offset() - uint64_t(offset()) + uint64_t(labelOffset); goto EmitOp_DispImm; } else { - // Record non-bound label. + // Create a fixup referencing an non-bound label. size_t codeOffset = writer.offsetFrom(_bufferData); - LabelLink* link = _code->newLabelLink(label, _section->id(), codeOffset, intptr_t(labelOffset), offsetFormat); + Fixup* fixup = _code->newFixup(&le, _section->sectionId(), codeOffset, intptr_t(labelOffset), offsetFormat); - if (ASMJIT_UNLIKELY(!link)) { + if (ASMJIT_UNLIKELY(!fixup)) { goto OutOfMemory; } @@ -5060,7 +5132,7 @@ EmitOp_Rel: size_t codeOffset = writer.offsetFrom(_bufferData); - if (baseAddress == Globals::kNoBaseAddress || _section->id() != 0) { + if (baseAddress == Globals::kNoBaseAddress || _section->sectionId() != 0) { // Create a new RelocEntry as we cannot calculate the offset right now. RelocEntry* re; err = _code->newRelocEntry(&re, RelocType::kAbsToRel); @@ -5068,7 +5140,7 @@ EmitOp_Rel: goto Failed; } - re->_sourceSectionId = _section->id(); + re->_sourceSectionId = _section->sectionId(); re->_sourceOffset = codeOffset; re->_format = offsetFormat; re->_payload = rmRel->as().valueAs() + 4u; @@ -5195,7 +5267,7 @@ Error Assembler::align(AlignMode alignMode, uint32_t alignment) { return kErrorOk; } - if (ASMJIT_UNLIKELY(alignment > Globals::kMaxAlignment || !Support::isPowerOf2(alignment))) { + if (ASMJIT_UNLIKELY(!Support::isPowerOf2UpTo(alignment, Globals::kMaxAlignment))) { return reportError(DebugUtils::errored(kErrorInvalidArgument)); } @@ -5247,16 +5319,16 @@ Error Assembler::align(AlignMode alignMode, uint32_t alignment) { // a64::Assembler - Events // ======================= -Error Assembler::onAttach(CodeHolder* code) noexcept { +Error Assembler::onAttach(CodeHolder& code) noexcept { ASMJIT_PROPAGATE(Base::onAttach(code)); _instructionAlignment = uint8_t(4); - assignEmitterFuncs(this); + updateEmitterFuncs(this); return kErrorOk; } -Error Assembler::onDetach(CodeHolder* code) noexcept { +Error Assembler::onDetach(CodeHolder& code) noexcept { return Base::onDetach(code); } diff --git a/src/asmjit/arm/a64assembler.h b/src/asmjit/arm/a64assembler.h index a660ddd..e1e154a 100644 --- a/src/asmjit/arm/a64assembler.h +++ b/src/asmjit/arm/a64assembler.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64ASSEMBLER_H_INCLUDED @@ -48,8 +48,8 @@ public: //! \name Events //! \{ - ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; - ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + ASMJIT_API Error onAttach(CodeHolder& code) noexcept override; + ASMJIT_API Error onDetach(CodeHolder& code) noexcept override; //! \} }; diff --git a/src/asmjit/arm/a64builder.cpp b/src/asmjit/arm/a64builder.cpp index 9396789..7b114cb 100644 --- a/src/asmjit/arm/a64builder.cpp +++ b/src/asmjit/arm/a64builder.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -17,6 +17,8 @@ ASMJIT_BEGIN_SUB_NAMESPACE(a64) Builder::Builder(CodeHolder* code) noexcept : BaseBuilder() { _archMask = uint64_t(1) << uint32_t(Arch::kAArch64); + initEmitterFuncs(this); + if (code) { code->attach(this); } @@ -26,20 +28,19 @@ Builder::~Builder() noexcept {} // a64::Builder - Events // ===================== -Error Builder::onAttach(CodeHolder* code) noexcept { +Error Builder::onAttach(CodeHolder& code) noexcept { ASMJIT_PROPAGATE(Base::onAttach(code)); _instructionAlignment = uint8_t(4); - assignEmitterFuncs(this); + updateEmitterFuncs(this); return kErrorOk; } -Error Builder::onDetach(CodeHolder* code) noexcept { +Error Builder::onDetach(CodeHolder& code) noexcept { return Base::onDetach(code); } - // a64::Builder - Finalize // ======================= diff --git a/src/asmjit/arm/a64builder.h b/src/asmjit/arm/a64builder.h index dd15dd1..9a468ac 100644 --- a/src/asmjit/arm/a64builder.h +++ b/src/asmjit/arm/a64builder.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64BUILDER_H_INCLUDED @@ -36,8 +36,8 @@ public: //! \name Events //! \{ - ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; - ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + ASMJIT_API Error onAttach(CodeHolder& code) noexcept override; + ASMJIT_API Error onDetach(CodeHolder& code) noexcept override; //! \} diff --git a/src/asmjit/arm/a64compiler.cpp b/src/asmjit/arm/a64compiler.cpp index 8d889d8..cc9fc7f 100644 --- a/src/asmjit/arm/a64compiler.cpp +++ b/src/asmjit/arm/a64compiler.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -18,6 +18,8 @@ ASMJIT_BEGIN_SUB_NAMESPACE(a64) Compiler::Compiler(CodeHolder* code) noexcept : BaseCompiler() { _archMask = uint64_t(1) << uint32_t(Arch::kAArch64); + initEmitterFuncs(this); + if (code) { code->attach(this); } @@ -27,7 +29,7 @@ Compiler::~Compiler() noexcept {} // a64::Compiler - Events // ====================== -Error Compiler::onAttach(CodeHolder* code) noexcept { +Error Compiler::onAttach(CodeHolder& code) noexcept { ASMJIT_PROPAGATE(Base::onAttach(code)); Error err = addPassT(); @@ -37,15 +39,23 @@ Error Compiler::onAttach(CodeHolder* code) noexcept { } _instructionAlignment = uint8_t(4); - assignEmitterFuncs(this); + updateEmitterFuncs(this); return kErrorOk; } -Error Compiler::onDetach(CodeHolder* code) noexcept { +Error Compiler::onDetach(CodeHolder& code) noexcept { return Base::onDetach(code); } +Error Compiler::onReinit(CodeHolder& code) noexcept { + Error err = Base::onReinit(code); + if (err == kErrorOk) { + err = addPassT(); + } + return err; +} + // a64::Compiler - Finalize // ======================== diff --git a/src/asmjit/arm/a64compiler.h b/src/asmjit/arm/a64compiler.h index 661e58d..1b104aa 100644 --- a/src/asmjit/arm/a64compiler.h +++ b/src/asmjit/arm/a64compiler.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64COMPILER_H_INCLUDED @@ -99,6 +99,11 @@ public: template ASMJIT_INLINE_NODEBUG Gp newUIntPtr(Args&&... args) { return _newRegInternal(TypeId::kUIntPtr, std::forward(args)...); } + template + ASMJIT_INLINE_NODEBUG Gp newGp32(Args&&... args) { return _newRegInternal(TypeId::kUInt32, std::forward(args)...); } + template + ASMJIT_INLINE_NODEBUG Gp newGp64(Args&&... args) { return _newRegInternal(TypeId::kUInt64, std::forward(args)...); } + template ASMJIT_INLINE_NODEBUG Gp newGpw(Args&&... args) { return _newRegInternal(TypeId::kUInt32, std::forward(args)...); } template @@ -214,9 +219,9 @@ public: //! Return. ASMJIT_INLINE_NODEBUG Error ret() { return addRet(Operand(), Operand()); } //! \overload - ASMJIT_INLINE_NODEBUG Error ret(const BaseReg& o0) { return addRet(o0, Operand()); } + ASMJIT_INLINE_NODEBUG Error ret(const Reg& o0) { return addRet(o0, Operand()); } //! \overload - ASMJIT_INLINE_NODEBUG Error ret(const BaseReg& o0, const BaseReg& o1) { return addRet(o0, o1); } + ASMJIT_INLINE_NODEBUG Error ret(const Reg& o0, const Reg& o1) { return addRet(o0, o1); } //! \} @@ -226,15 +231,16 @@ public: using EmitterExplicitT::br; //! Adds a jump to the given `target` with the provided jump `annotation`. - ASMJIT_INLINE_NODEBUG Error br(const BaseReg& target, JumpAnnotation* annotation) { return emitAnnotatedJump(Inst::kIdBr, target, annotation); } + ASMJIT_INLINE_NODEBUG Error br(const Reg& target, JumpAnnotation* annotation) { return emitAnnotatedJump(Inst::kIdBr, target, annotation); } //! \} //! \name Events //! \{ - ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; - ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + ASMJIT_API Error onAttach(CodeHolder& code) noexcept override; + ASMJIT_API Error onDetach(CodeHolder& code) noexcept override; + ASMJIT_API Error onReinit(CodeHolder& code) noexcept override; //! \} diff --git a/src/asmjit/arm/a64emithelper.cpp b/src/asmjit/arm/a64emithelper.cpp index 2606172..3537761 100644 --- a/src/asmjit/arm/a64emithelper.cpp +++ b/src/asmjit/arm/a64emithelper.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -148,8 +148,8 @@ ASMJIT_FAVOR_SIZE Error EmitHelper::emitRegMove( } Error EmitHelper::emitRegSwap( - const BaseReg& a, - const BaseReg& b, const char* comment) { + const Reg& a, + const Reg& b, const char* comment) { DebugUtils::unused(a, b, comment); return DebugUtils::errored(kErrorInvalidState); @@ -157,13 +157,12 @@ Error EmitHelper::emitRegSwap( // TODO: [ARM] EmitArgMove is unfinished. Error EmitHelper::emitArgMove( - const BaseReg& dst_, TypeId dstTypeId, + const Reg& dst_, TypeId dstTypeId, const Operand_& src_, TypeId srcTypeId, const char* comment) { // Deduce optional `dstTypeId`, which may be `TypeId::kVoid` in some cases. if (dstTypeId == TypeId::kVoid) { - const ArchTraits& archTraits = ArchTraits::byArch(_emitter->arch()); - dstTypeId = archTraits.regTypeToTypeId(dst_.type()); + dstTypeId = RegUtils::typeIdOf(dst_.regType()); } // Invalid or abstract TypeIds are not allowed. @@ -180,7 +179,7 @@ Error EmitHelper::emitArgMove( if (TypeUtils::isInt(srcTypeId)) { uint32_t x = uint32_t(dstSize == 8); - dst.setSignature(OperandSignature{x ? uint32_t(GpX::kSignature) : uint32_t(GpW::kSignature)}); + dst.setSignature(OperandSignature{x ? RegTraits::kSignature : RegTraits::kSignature}); _emitter->setInlineComment(comment); if (src.isReg()) { @@ -209,10 +208,10 @@ Error EmitHelper::emitArgMove( if (TypeUtils::isFloat(dstTypeId) || TypeUtils::isVec(dstTypeId)) { if (TypeUtils::isFloat(srcTypeId) || TypeUtils::isVec(srcTypeId)) { switch (srcSize) { - case 2: dst.as().setSignature(OperandSignature{VecH::kSignature}); break; - case 4: dst.as().setSignature(OperandSignature{VecS::kSignature}); break; - case 8: dst.as().setSignature(OperandSignature{VecD::kSignature}); break; - case 16: dst.as().setSignature(OperandSignature{VecV::kSignature}); break; + case 2: dst.as().setSignature(RegTraits::kSignature); break; + case 4: dst.as().setSignature(RegTraits::kSignature); break; + case 8: dst.as().setSignature(RegTraits::kSignature); break; + case 16: dst.as().setSignature(RegTraits::kSignature); break; default: return DebugUtils::errored(kErrorInvalidState); } @@ -295,7 +294,7 @@ struct PrologEpilogInfo { } if (n == 1) { - pairs[pairCount].ids[1] = uint8_t(BaseReg::kIdBad); + pairs[pairCount].ids[1] = uint8_t(Reg::kIdBad); pairs[pairCount].offset = uint16_t(offset); offset += slotSize * 2; pairCount++; @@ -321,10 +320,9 @@ ASMJIT_FAVOR_SIZE Error EmitHelper::emitProlog(const FuncFrame& frame) { { Inst::kIdStr_v, Inst::kIdStp_v } }}; - // Emit: 'bti' (indirect branch protection). + // Emit: 'bti {jc}' (indirect branch protection). if (frame.hasIndirectBranchProtection()) { - // TODO: The instruction is not available at the moment (would be ABI break). - // ASMJIT_PROPAGATE(emitter->bti()); + ASMJIT_PROPAGATE(emitter->bti(Predicate::BTI::kJC)); } uint32_t adjustInitialOffset = pei.sizeTotal; @@ -349,7 +347,7 @@ ASMJIT_FAVOR_SIZE Error EmitHelper::emitProlog(const FuncFrame& frame) { mem.makePreIndex(); } - if (pair.ids[1] == BaseReg::kIdBad) { + if (pair.ids[1] == Reg::kIdBad) { ASMJIT_PROPAGATE(emitter->emit(insts.singleInstId, regs[0], mem)); } else { @@ -433,7 +431,7 @@ ASMJIT_FAVOR_SIZE Error EmitHelper::emitEpilog(const FuncFrame& frame) { mem.makePostIndex(); } - if (pair.ids[1] == BaseReg::kIdBad) { + if (pair.ids[1] == Reg::kIdBad) { ASMJIT_PROPAGATE(emitter->emit(insts.singleInstId, regs[0], mem)); } else { @@ -464,7 +462,7 @@ static Error ASMJIT_CDECL Emitter_emitArgsAssignment(BaseEmitter* emitter, const return emitHelper.emitArgsAssignment(frame, args); } -void assignEmitterFuncs(BaseEmitter* emitter) { +void initEmitterFuncs(BaseEmitter* emitter) { emitter->_funcs.emitProlog = Emitter_emitProlog; emitter->_funcs.emitEpilog = Emitter_emitEpilog; emitter->_funcs.emitArgsAssignment = Emitter_emitArgsAssignment; diff --git a/src/asmjit/arm/a64emithelper_p.h b/src/asmjit/arm/a64emithelper_p.h index 85e0c74..1b034fe 100644 --- a/src/asmjit/arm/a64emithelper_p.h +++ b/src/asmjit/arm/a64emithelper_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64EMITHELPER_P_H_INCLUDED @@ -31,18 +31,21 @@ public: const Operand_& src_, TypeId typeId, const char* comment = nullptr) override; Error emitRegSwap( - const BaseReg& a, - const BaseReg& b, const char* comment = nullptr) override; + const Reg& a, + const Reg& b, const char* comment = nullptr) override; Error emitArgMove( - const BaseReg& dst_, TypeId dstTypeId, + const Reg& dst_, TypeId dstTypeId, const Operand_& src_, TypeId srcTypeId, const char* comment = nullptr) override; Error emitProlog(const FuncFrame& frame); Error emitEpilog(const FuncFrame& frame); }; -void assignEmitterFuncs(BaseEmitter* emitter); +void initEmitterFuncs(BaseEmitter* emitter); + +[[maybe_unused]] +static inline void updateEmitterFuncs(BaseEmitter* emitter) noexcept { DebugUtils::unused(emitter); } //! \} //! \endcond diff --git a/src/asmjit/arm/a64emitter.h b/src/asmjit/arm/a64emitter.h index 15c91e0..876af77 100644 --- a/src/asmjit/arm/a64emitter.h +++ b/src/asmjit/arm/a64emitter.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64EMITTER_H_INCLUDED @@ -226,6 +226,9 @@ struct EmitterExplicitT { ASMJIT_INST_2x(mvn, Mvn, Gp, Gp) ASMJIT_INST_3x(mvn, Mvn, Gp, Gp, Imm) + ASMJIT_INST_2x(mvn_, Mvn, Gp, Gp) + ASMJIT_INST_3x(mvn_, Mvn, Gp, Gp, Imm) + ASMJIT_INST_2x(neg, Neg, Gp, Gp) ASMJIT_INST_3x(neg, Neg, Gp, Gp, Imm) ASMJIT_INST_2x(negs, Negs, Gp, Gp) @@ -311,27 +314,9 @@ struct EmitterExplicitT { ASMJIT_INST_1x(dcps1, Dcps1, Imm) ASMJIT_INST_1x(dcps2, Dcps2, Imm) ASMJIT_INST_1x(dcps3, Dcps3, Imm) - ASMJIT_INST_0x(dgh, Dgh) ASMJIT_INST_0x(pssbb, Pssbb) ASMJIT_INST_0x(ssbb, Ssbb) ASMJIT_INST_1x(udf, Udf, Imm) - ASMJIT_INST_1x(setf8, Setf8, Gp) - ASMJIT_INST_1x(setf16, Setf16, Gp) - - //! \} - - //! \name ARMv8.4 Instructions - //! \{ - - ASMJIT_INST_0x(cfinv, Cfinv) - - //! \} - - //! \name ARMv8.5 Instructions - //! \{ - - ASMJIT_INST_0x(axflag, Axflag) - ASMJIT_INST_0x(xaflag, Xaflag) //! \} @@ -631,6 +616,27 @@ struct EmitterExplicitT { ASMJIT_INST_3x(swplh, Swplh, Gp, Gp, Mem) //! \} + //! \name BTI Instructions + //! \{ + + ASMJIT_INST_1x(bti, Bti, Imm); + + //! \} + + //! \name CHK Instructions + //! \{ + + ASMJIT_INST_1x(chkfeat, Chkfeat, Gp); + + //! \} + + //! \name CLRBHB Instructions + //! \{ + + ASMJIT_INST_0x(clrbhb, Clrbhb); + + //! \} + //! \name CRC Instructions (ARMv8.1-A, optional in ARMv8.0-A) //! \{ @@ -646,6 +652,55 @@ struct EmitterExplicitT { //! \} + //! \name CSSC Instructions + //! \{ + + ASMJIT_INST_2x(abs, Abs, Gp, Gp); + ASMJIT_INST_2x(cnt, Cnt, Gp, Gp); + ASMJIT_INST_2x(ctz, Ctz, Gp, Gp); + ASMJIT_INST_3x(smax, Smax, Gp, Gp, Gp); + ASMJIT_INST_3x(smax, Smax, Gp, Gp, Imm); + ASMJIT_INST_3x(smin, Smin, Gp, Gp, Gp); + ASMJIT_INST_3x(smin, Smin, Gp, Gp, Imm); + ASMJIT_INST_3x(umax, Umax, Gp, Gp, Gp); + ASMJIT_INST_3x(umax, Umax, Gp, Gp, Imm); + ASMJIT_INST_3x(umin, Umin, Gp, Gp, Gp); + ASMJIT_INST_3x(umin, Umin, Gp, Gp, Imm); + + //! \} + + //! \name DGH Instructions + //! \{ + + ASMJIT_INST_0x(dgh, Dgh) + + //! \} + + //! \name FLAGM Instructions + //! \{ + + ASMJIT_INST_0x(cfinv, Cfinv) + ASMJIT_INST_1x(setf8, Setf8, Gp) + ASMJIT_INST_1x(setf16, Setf16, Gp) + + //! \} + + //! \name FLAGM2 Instructions + //! \{ + + ASMJIT_INST_0x(axflag, Axflag) + ASMJIT_INST_0x(xaflag, Xaflag) + + //! \} + + //! \name HBC Instructions + //! \{ + + ASMJIT_INST_1cc(bc, Bc, Imm) + ASMJIT_INST_1cc(bc, Bc, Label) + + //! \} + //! \name MTE Instructions //! \{ @@ -874,6 +929,7 @@ struct EmitterExplicitT { ASMJIT_INST_3x(movi, Movi_v, Vec, Imm, Imm); ASMJIT_INST_3x(mul, Mul_v, Vec, Vec, Vec); ASMJIT_INST_2x(mvn, Mvn_v, Vec, Vec); + ASMJIT_INST_2x(mvn_, Mvn_v, Vec, Vec); ASMJIT_INST_2x(mvni, Mvni_v, Vec, Imm); ASMJIT_INST_3x(mvni, Mvni_v, Vec, Imm, Imm); ASMJIT_INST_2x(neg, Neg_v, Vec, Vec); @@ -1213,7 +1269,7 @@ struct EmitterExplicitT { //! \} }; -//! Emitter (ARM). +//! Emitter (AArch64). //! //! \note This class cannot be instantiated, you can only cast to it and use it as emitter that emits to either //! `a64::Assembler`, `a64::Builder`, or `a64::Compiler` (use with caution with `a64::Compiler` as it requires diff --git a/src/asmjit/arm/a64formatter.cpp b/src/asmjit/arm/a64formatter.cpp index b687e92..069f6af 100644 --- a/src/asmjit/arm/a64formatter.cpp +++ b/src/asmjit/arm/a64formatter.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/arm/a64formatter_p.h b/src/asmjit/arm/a64formatter_p.h index d0adde3..1da4936 100644 --- a/src/asmjit/arm/a64formatter_p.h +++ b/src/asmjit/arm/a64formatter_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64FORMATTER_P_H_INCLUDED diff --git a/src/asmjit/arm/a64func.cpp b/src/asmjit/arm/a64func.cpp index 9e36074..d4a54d6 100644 --- a/src/asmjit/arm/a64func.cpp +++ b/src/asmjit/arm/a64func.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -26,19 +26,19 @@ static inline bool shouldTreatAsCDecl(CallConvId ccId) noexcept { static RegType regTypeFromFpOrVecTypeId(TypeId typeId) noexcept { if (typeId == TypeId::kFloat32) { - return RegType::kARM_VecS; + return RegType::kVec32; } else if (typeId == TypeId::kFloat64) { - return RegType::kARM_VecD; + return RegType::kVec64; } else if (TypeUtils::isVec32(typeId)) { - return RegType::kARM_VecS; + return RegType::kVec32; } else if (TypeUtils::isVec64(typeId)) { - return RegType::kARM_VecD; + return RegType::kVec64; } else if (TypeUtils::isVec128(typeId)) { - return RegType::kARM_VecV; + return RegType::kVec128; } else { return RegType::kNone; @@ -103,20 +103,20 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si case TypeId::kInt8: case TypeId::kInt16: case TypeId::kInt32: { - func._rets[valueIndex].initReg(RegType::kARM_GpW, valueIndex, TypeId::kInt32); + func._rets[valueIndex].initReg(RegType::kGp32, valueIndex, TypeId::kInt32); break; } case TypeId::kUInt8: case TypeId::kUInt16: case TypeId::kUInt32: { - func._rets[valueIndex].initReg(RegType::kARM_GpW, valueIndex, TypeId::kUInt32); + func._rets[valueIndex].initReg(RegType::kGp32, valueIndex, TypeId::kUInt32); break; } case TypeId::kInt64: case TypeId::kUInt64: { - func._rets[valueIndex].initReg(RegType::kARM_GpX, valueIndex, typeId); + func._rets[valueIndex].initReg(RegType::kGp64, valueIndex, typeId); break; } @@ -144,14 +144,14 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si TypeId typeId = arg.typeId(); if (TypeUtils::isInt(typeId)) { - uint32_t regId = BaseReg::kIdBad; + uint32_t regId = Reg::kIdBad; if (gpzPos < CallConv::kMaxRegArgsPerGroup) { regId = cc._passedOrder[RegGroup::kGp].id[gpzPos]; } - if (regId != BaseReg::kIdBad) { - RegType regType = typeId <= TypeId::kUInt32 ? RegType::kARM_GpW : RegType::kARM_GpX; + if (regId != Reg::kIdBad) { + RegType regType = typeId <= TypeId::kUInt32 ? RegType::kGp32 : RegType::kGp64; arg.assignRegData(regType, regId); func.addUsedRegs(RegGroup::kGp, Support::bitMask(regId)); gpzPos++; @@ -168,13 +168,13 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si } if (TypeUtils::isFloat(typeId) || TypeUtils::isVec(typeId)) { - uint32_t regId = BaseReg::kIdBad; + uint32_t regId = Reg::kIdBad; if (vecPos < CallConv::kMaxRegArgsPerGroup) { regId = cc._passedOrder[RegGroup::kVec].id[vecPos]; } - if (regId != BaseReg::kIdBad) { + if (regId != Reg::kIdBad) { RegType regType = regTypeFromFpOrVecTypeId(typeId); if (regType == RegType::kNone) { return DebugUtils::errored(kErrorInvalidRegType); diff --git a/src/asmjit/arm/a64func_p.h b/src/asmjit/arm/a64func_p.h index 7f2221c..2239f65 100644 --- a/src/asmjit/arm/a64func_p.h +++ b/src/asmjit/arm/a64func_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64FUNC_P_H_INCLUDED diff --git a/src/asmjit/arm/a64globals.h b/src/asmjit/arm/a64globals.h index b4dac03..e7d2a26 100644 --- a/src/asmjit/arm/a64globals.h +++ b/src/asmjit/arm/a64globals.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64GLOBALS_H_INCLUDED @@ -26,6 +26,7 @@ namespace Inst { enum Id : uint32_t { // ${InstId:Begin} kIdNone = 0, //!< Instruction ''. + kIdAbs, //!< Instruction 'abs'. kIdAdc, //!< Instruction 'adc'. kIdAdcs, //!< Instruction 'adcs'. kIdAdd, //!< Instruction 'add'. @@ -54,6 +55,7 @@ namespace Inst { kIdAutizb, //!< Instruction 'autizb'. kIdAxflag, //!< Instruction 'axflag'. kIdB, //!< Instruction 'b'. + kIdBc, //!< Instruction 'bc'. kIdBfc, //!< Instruction 'bfc'. kIdBfi, //!< Instruction 'bfi'. kIdBfm, //!< Instruction 'bfm'. @@ -64,6 +66,7 @@ namespace Inst { kIdBlr, //!< Instruction 'blr'. kIdBr, //!< Instruction 'br'. kIdBrk, //!< Instruction 'brk'. + kIdBti, //!< Instruction 'bti'. kIdCas, //!< Instruction 'cas'. kIdCasa, //!< Instruction 'casa'. kIdCasab, //!< Instruction 'casab'. @@ -85,8 +88,10 @@ namespace Inst { kIdCcmn, //!< Instruction 'ccmn'. kIdCcmp, //!< Instruction 'ccmp'. kIdCfinv, //!< Instruction 'cfinv'. + kIdChkfeat, //!< Instruction 'chkfeat'. kIdCinc, //!< Instruction 'cinc'. kIdCinv, //!< Instruction 'cinv'. + kIdClrbhb, //!< Instruction 'clrbhb'. kIdClrex, //!< Instruction 'clrex'. kIdCls, //!< Instruction 'cls'. kIdClz, //!< Instruction 'clz'. @@ -94,6 +99,7 @@ namespace Inst { kIdCmp, //!< Instruction 'cmp'. kIdCmpp, //!< Instruction 'cmpp'. kIdCneg, //!< Instruction 'cneg'. + kIdCnt, //!< Instruction 'cnt'. kIdCrc32b, //!< Instruction 'crc32b'. kIdCrc32cb, //!< Instruction 'crc32cb'. kIdCrc32ch, //!< Instruction 'crc32ch'. @@ -109,6 +115,7 @@ namespace Inst { kIdCsinc, //!< Instruction 'csinc'. kIdCsinv, //!< Instruction 'csinv'. kIdCsneg, //!< Instruction 'csneg'. + kIdCtz, //!< Instruction 'ctz'. kIdDc, //!< Instruction 'dc'. kIdDcps1, //!< Instruction 'dcps1'. kIdDcps2, //!< Instruction 'dcps2'. @@ -311,7 +318,9 @@ namespace Inst { kIdSev, //!< Instruction 'sev'. kIdSevl, //!< Instruction 'sevl'. kIdSmaddl, //!< Instruction 'smaddl'. + kIdSmax, //!< Instruction 'smax'. kIdSmc, //!< Instruction 'smc'. + kIdSmin, //!< Instruction 'smin'. kIdSmnegl, //!< Instruction 'smnegl'. kIdSmsubl, //!< Instruction 'smsubl'. kIdSmulh, //!< Instruction 'smulh'. @@ -429,6 +438,8 @@ namespace Inst { kIdUdf, //!< Instruction 'udf'. kIdUdiv, //!< Instruction 'udiv'. kIdUmaddl, //!< Instruction 'umaddl'. + kIdUmax, //!< Instruction 'umax'. + kIdUmin, //!< Instruction 'umin'. kIdUmnegl, //!< Instruction 'umnegl'. kIdUmull, //!< Instruction 'umull'. kIdUmulh, //!< Instruction 'umulh'. @@ -824,6 +835,17 @@ namespace AT { }; } +//! Branch target identification targets (BTI). +namespace BTI { + //! Branch target identification targets + enum Value : uint32_t { + kNone = 0u, + kC = 1u, + kJ = 2u, + kJC = 3u + }; +} + //! Data barrier options (DMB/DSB). namespace DB { //! Data barrier immediate values. diff --git a/src/asmjit/arm/a64instapi.cpp b/src/asmjit/arm/a64instapi.cpp index d655acf..003db32 100644 --- a/src/asmjit/arm/a64instapi.cpp +++ b/src/asmjit/arm/a64instapi.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -128,7 +128,7 @@ Error queryRWInfo(const BaseInst& inst, const Operand_* operands, size_t opCount OpRWFlags rwFlags = i < opCount - 1 ? (OpRWFlags)rwInfo.rwx[0] : (OpRWFlags)rwInfo.rwx[1]; op._opFlags = rwFlags & ~(OpRWFlags::kZExt); - op._physId = BaseReg::kIdBad; + op._physId = Reg::kIdBad; op._rmSize = 0; op._resetReserved(); @@ -177,7 +177,7 @@ Error queryRWInfo(const BaseInst& inst, const Operand_* operands, size_t opCount OpRWFlags rwFlags = (OpRWFlags)rwInfo.rwx[i]; op._opFlags = rwFlags & ~(OpRWFlags::kZExt); - op._physId = BaseReg::kIdBad; + op._physId = Reg::kIdBad; op._rmSize = 0; op._resetReserved(); diff --git a/src/asmjit/arm/a64instapi_p.h b/src/asmjit/arm/a64instapi_p.h index f294db6..772bb42 100644 --- a/src/asmjit/arm/a64instapi_p.h +++ b/src/asmjit/arm/a64instapi_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64INSTAPI_P_H_INCLUDED diff --git a/src/asmjit/arm/a64instdb.cpp b/src/asmjit/arm/a64instdb.cpp index bc6646e..4b2170b 100644 --- a/src/asmjit/arm/a64instdb.cpp +++ b/src/asmjit/arm/a64instdb.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -43,9 +43,6 @@ ERETAA, ERETAB: Exception Return, with pointer authentication. LDAPxxx PACIA, PACIA1716, PACIASP, PACIAZ, PACIZA: Pointer Authentication Code for Instruction address, using key A. PACIB, PACIB1716, PACIBSP, PACIBZ, PACIZB: Pointer Authentication Code for Instruction address, using key B. -PRFM (immediate): Prefetch Memory (immediate). -PRFM (literal): Prefetch Memory (literal). -PRFM (register): Prefetch Memory (register). PRFUM: Prefetch Memory (unscaled offset). RETAA, RETAB: Return from subroutine, with pointer authentication. RMIF: Rotate, Mask Insert Flags. @@ -59,770 +56,781 @@ const InstInfo _instInfoTable[] = { // +------------------+---------------------+--------------------------------------------------------------------------------------+-----------+---------------------------+----+ // ${InstInfo:Begin} INST(None , None , (_) , 0 , 0 , 0 ), // #0 - INST(Adc , BaseRRR , (0b0001101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 0 ), // #1 - INST(Adcs , BaseRRR , (0b0011101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 1 ), // #2 - INST(Add , BaseAddSub , (0b0001011000, 0b0001011001, 0b0010001) , kRWI_W , 0 , 0 ), // #3 - INST(Addg , BaseRRII , (0b1001000110000000000000, kX, kSP, kX, kSP, 6, 4, 16, 4, 0, 10) , kRWI_W , 0 , 0 ), // #4 - INST(Adds , BaseAddSub , (0b0101011000, 0b0101011001, 0b0110001) , kRWI_W , 0 , 1 ), // #5 - INST(Adr , BaseAdr , (0b0001000000000000000000, OffsetType::kAArch64_ADR) , kRWI_W , 0 , 0 ), // #6 - INST(Adrp , BaseAdr , (0b1001000000000000000000, OffsetType::kAArch64_ADRP) , kRWI_W , 0 , 1 ), // #7 - INST(And , BaseLogical , (0b0001010000, 0b00100100, 0) , kRWI_W , 0 , 0 ), // #8 - INST(Ands , BaseLogical , (0b1101010000, 0b11100100, 0) , kRWI_W , 0 , 1 ), // #9 - INST(Asr , BaseShift , (0b0001101011000000001010, 0b0001001100000000011111, 0) , kRWI_W , 0 , 0 ), // #10 - INST(Asrv , BaseShift , (0b0001101011000000001010, 0b0000000000000000000000, 0) , kRWI_W , 0 , 1 ), // #11 - INST(At , BaseAtDcIcTlbi , (0b00011111110000, 0b00001111000000, true) , kRWI_RX , 0 , 0 ), // #12 - INST(Autda , BaseRR , (0b11011010110000010001100000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 0 ), // #13 - INST(Autdza , BaseR , (0b11011010110000010011101111100000, kX, kZR, 0) , kRWI_X , 0 , 0 ), // #14 - INST(Autdb , BaseRR , (0b11011010110000010001110000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 1 ), // #15 - INST(Autdzb , BaseR , (0b11011010110000010011111111100000, kX, kZR, 0) , kRWI_X , 0 , 1 ), // #16 - INST(Autia , BaseRR , (0b11011010110000010001000000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 2 ), // #17 - INST(Autia1716 , BaseOp , (0b11010101000000110010000110011111) , 0 , 0 , 0 ), // #18 - INST(Autiasp , BaseOp , (0b11010101000000110010001110111111) , 0 , 0 , 1 ), // #19 - INST(Autiaz , BaseOp , (0b11010101000000110010001110011111) , 0 , 0 , 2 ), // #20 - INST(Autib , BaseRR , (0b11011010110000010001010000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 3 ), // #21 - INST(Autib1716 , BaseOp , (0b11010101000000110010000111011111) , 0 , 0 , 3 ), // #22 - INST(Autibsp , BaseOp , (0b11010101000000110010001111111111) , 0 , 0 , 4 ), // #23 - INST(Autibz , BaseOp , (0b11010101000000110010001111011111) , 0 , 0 , 5 ), // #24 - INST(Autiza , BaseR , (0b11011010110000010011001111100000, kX, kZR, 0) , kRWI_X , 0 , 2 ), // #25 - INST(Autizb , BaseR , (0b11011010110000010011011111100000, kX, kZR, 0) , kRWI_X , 0 , 3 ), // #26 - INST(Axflag , BaseOp , (0b11010101000000000100000001011111) , 0 , 0 , 6 ), // #27 - INST(B , BaseBranchRel , (0b00010100000000000000000000000000) , 0 , F(Cond) , 0 ), // #28 - INST(Bfc , BaseBfc , (0b00110011000000000000001111100000) , kRWI_X , 0 , 0 ), // #29 - INST(Bfi , BaseBfi , (0b00110011000000000000000000000000) , kRWI_X , 0 , 0 ), // #30 - INST(Bfm , BaseBfm , (0b00110011000000000000000000000000) , kRWI_X , 0 , 0 ), // #31 - INST(Bfxil , BaseBfx , (0b00110011000000000000000000000000) , kRWI_X , 0 , 0 ), // #32 - INST(Bic , BaseLogical , (0b0001010001, 0b00100100, 1) , kRWI_W , 0 , 2 ), // #33 - INST(Bics , BaseLogical , (0b1101010001, 0b11100100, 1) , kRWI_W , 0 , 3 ), // #34 - INST(Bl , BaseBranchRel , (0b10010100000000000000000000000000) , 0 , 0 , 1 ), // #35 - INST(Blr , BaseBranchReg , (0b11010110001111110000000000000000) , kRWI_R , 0 , 0 ), // #36 - INST(Br , BaseBranchReg , (0b11010110000111110000000000000000) , kRWI_R , 0 , 1 ), // #37 - INST(Brk , BaseOpImm , (0b11010100001000000000000000000000, 16, 5) , 0 , 0 , 0 ), // #38 - INST(Cas , BaseAtomicOp , (0b1000100010100000011111, kWX, 30, 0) , kRWI_XRX , 0 , 0 ), // #39 - INST(Casa , BaseAtomicOp , (0b1000100011100000011111, kWX, 30, 1) , kRWI_XRX , 0 , 1 ), // #40 - INST(Casab , BaseAtomicOp , (0b0000100011100000011111, kW , 0 , 1) , kRWI_XRX , 0 , 2 ), // #41 - INST(Casah , BaseAtomicOp , (0b0100100011100000011111, kW , 0 , 1) , kRWI_XRX , 0 , 3 ), // #42 - INST(Casal , BaseAtomicOp , (0b1000100011100000111111, kWX, 30, 1) , kRWI_XRX , 0 , 4 ), // #43 - INST(Casalb , BaseAtomicOp , (0b0000100011100000111111, kW , 0 , 1) , kRWI_XRX , 0 , 5 ), // #44 - INST(Casalh , BaseAtomicOp , (0b0100100011100000111111, kW , 0 , 1) , kRWI_XRX , 0 , 6 ), // #45 - INST(Casb , BaseAtomicOp , (0b0000100010100000011111, kW , 0 , 0) , kRWI_XRX , 0 , 7 ), // #46 - INST(Cash , BaseAtomicOp , (0b0100100010100000011111, kW , 0 , 0) , kRWI_XRX , 0 , 8 ), // #47 - INST(Casl , BaseAtomicOp , (0b1000100010100000111111, kWX, 30, 0) , kRWI_XRX , 0 , 9 ), // #48 - INST(Caslb , BaseAtomicOp , (0b0000100010100000111111, kW , 0 , 0) , kRWI_XRX , 0 , 10 ), // #49 - INST(Caslh , BaseAtomicOp , (0b0100100010100000111111, kW , 0 , 0) , kRWI_XRX , 0 , 11 ), // #50 - INST(Casp , BaseAtomicCasp , (0b0000100000100000011111, kWX, 30) , kRWI_XXRRX, 0 , 0 ), // #51 - INST(Caspa , BaseAtomicCasp , (0b0000100001100000011111, kWX, 30) , kRWI_XXRRX, 0 , 1 ), // #52 - INST(Caspal , BaseAtomicCasp , (0b0000100001100000111111, kWX, 30) , kRWI_XXRRX, 0 , 2 ), // #53 - INST(Caspl , BaseAtomicCasp , (0b0000100000100000111111, kWX, 30) , kRWI_XXRRX, 0 , 3 ), // #54 - INST(Cbnz , BaseBranchCmp , (0b00110101000000000000000000000000) , kRWI_R , 0 , 0 ), // #55 - INST(Cbz , BaseBranchCmp , (0b00110100000000000000000000000000) , kRWI_R , 0 , 1 ), // #56 - INST(Ccmn , BaseCCmp , (0b00111010010000000000000000000000) , kRWI_R , 0 , 0 ), // #57 - INST(Ccmp , BaseCCmp , (0b01111010010000000000000000000000) , kRWI_R , 0 , 1 ), // #58 - INST(Cfinv , BaseOp , (0b11010101000000000100000000011111) , 0 , 0 , 7 ), // #59 - INST(Cinc , BaseCInc , (0b00011010100000000000010000000000) , kRWI_W , 0 , 0 ), // #60 - INST(Cinv , BaseCInc , (0b01011010100000000000000000000000) , kRWI_W , 0 , 1 ), // #61 - INST(Clrex , BaseOpImm , (0b11010101000000110011000001011111, 4, 8) , 0 , 0 , 1 ), // #62 - INST(Cls , BaseRR , (0b01011010110000000001010000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 4 ), // #63 - INST(Clz , BaseRR , (0b01011010110000000001000000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 5 ), // #64 - INST(Cmn , BaseCmpCmn , (0b0101011000, 0b0101011001, 0b0110001) , kRWI_R , 0 , 0 ), // #65 - INST(Cmp , BaseCmpCmn , (0b1101011000, 0b1101011001, 0b1110001) , kRWI_R , 0 , 1 ), // #66 - INST(Cmpp , BaseRR , (0b10111010110000000000000000011111, kX, kSP, 5, kX, kSP, 16, true) , kRWI_R , 0 , 6 ), // #67 - INST(Cneg , BaseCInc , (0b01011010100000000000010000000000) , kRWI_W , 0 , 2 ), // #68 - INST(Crc32b , BaseRRR , (0b0001101011000000010000, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 2 ), // #69 - INST(Crc32cb , BaseRRR , (0b0001101011000000010100, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 3 ), // #70 - INST(Crc32ch , BaseRRR , (0b0001101011000000010101, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 4 ), // #71 - INST(Crc32cw , BaseRRR , (0b0001101011000000010110, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 5 ), // #72 - INST(Crc32cx , BaseRRR , (0b1001101011000000010111, kW, kZR, kW, kZR, kX, kZR, false) , kRWI_W , 0 , 6 ), // #73 - INST(Crc32h , BaseRRR , (0b0001101011000000010001, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 7 ), // #74 - INST(Crc32w , BaseRRR , (0b0001101011000000010010, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 8 ), // #75 - INST(Crc32x , BaseRRR , (0b1001101011000000010011, kW, kZR, kW, kZR, kX, kZR, false) , kRWI_W , 0 , 9 ), // #76 - INST(Csdb , BaseOp , (0b11010101000000110010001010011111) , 0 , 0 , 8 ), // #77 - INST(Csel , BaseCSel , (0b00011010100000000000000000000000) , kRWI_W , 0 , 0 ), // #78 - INST(Cset , BaseCSet , (0b00011010100111110000011111100000) , kRWI_W , 0 , 0 ), // #79 - INST(Csetm , BaseCSet , (0b01011010100111110000001111100000) , kRWI_W , 0 , 1 ), // #80 - INST(Csinc , BaseCSel , (0b00011010100000000000010000000000) , kRWI_W , 0 , 1 ), // #81 - INST(Csinv , BaseCSel , (0b01011010100000000000000000000000) , kRWI_W , 0 , 2 ), // #82 - INST(Csneg , BaseCSel , (0b01011010100000000000010000000000) , kRWI_W , 0 , 3 ), // #83 - INST(Dc , BaseAtDcIcTlbi , (0b00011110000000, 0b00001110000000, true) , kRWI_RX , 0 , 1 ), // #84 - INST(Dcps1 , BaseOpImm , (0b11010100101000000000000000000001, 16, 5) , 0 , 0 , 2 ), // #85 - INST(Dcps2 , BaseOpImm , (0b11010100101000000000000000000010, 16, 5) , 0 , 0 , 3 ), // #86 - INST(Dcps3 , BaseOpImm , (0b11010100101000000000000000000011, 16, 5) , 0 , 0 , 4 ), // #87 - INST(Dgh , BaseOp , (0b11010101000000110010000011011111) , 0 , 0 , 9 ), // #88 - INST(Dmb , BaseOpImm , (0b11010101000000110011000010111111, 4, 8) , 0 , 0 , 5 ), // #89 - INST(Drps , BaseOp , (0b11010110101111110000001111100000) , 0 , 0 , 10 ), // #90 - INST(Dsb , BaseOpImm , (0b11010101000000110011000010011111, 4, 8) , 0 , 0 , 6 ), // #91 - INST(Eon , BaseLogical , (0b1001010001, 0b10100100, 1) , kRWI_W , 0 , 4 ), // #92 - INST(Eor , BaseLogical , (0b1001010000, 0b10100100, 0) , kRWI_W , 0 , 5 ), // #93 - INST(Esb , BaseOp , (0b11010101000000110010001000011111) , 0 , 0 , 11 ), // #94 - INST(Extr , BaseExtract , (0b00010011100000000000000000000000) , kRWI_W , 0 , 0 ), // #95 - INST(Eret , BaseOp , (0b11010110100111110000001111100000) , 0 , 0 , 12 ), // #96 - INST(Gmi , BaseRRR , (0b1001101011000000000101, kX , kZR, kX , kSP, kX , kZR, true) , kRWI_W , 0 , 10 ), // #97 - INST(Hint , BaseOpImm , (0b11010101000000110010000000011111, 7, 5) , 0 , 0 , 7 ), // #98 - INST(Hlt , BaseOpImm , (0b11010100010000000000000000000000, 16, 5) , 0 , 0 , 8 ), // #99 - INST(Hvc , BaseOpImm , (0b11010100000000000000000000000010, 16, 5) , 0 , 0 , 9 ), // #100 - INST(Ic , BaseAtDcIcTlbi , (0b00011110000000, 0b00001110000000, false) , kRWI_RX , 0 , 2 ), // #101 - INST(Isb , BaseOpImm , (0b11010101000000110011000011011111, 4, 8) , 0 , 0 , 10 ), // #102 - INST(Ldadd , BaseAtomicOp , (0b1011100000100000000000, kWX, 30, 0) , kRWI_WRX , 0 , 12 ), // #103 - INST(Ldadda , BaseAtomicOp , (0b1011100010100000000000, kWX, 30, 1) , kRWI_WRX , 0 , 13 ), // #104 - INST(Ldaddab , BaseAtomicOp , (0b0011100010100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 14 ), // #105 - INST(Ldaddah , BaseAtomicOp , (0b0111100010100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 15 ), // #106 - INST(Ldaddal , BaseAtomicOp , (0b1011100011100000000000, kWX, 30, 1) , kRWI_WRX , 0 , 16 ), // #107 - INST(Ldaddalb , BaseAtomicOp , (0b0011100011100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 17 ), // #108 - INST(Ldaddalh , BaseAtomicOp , (0b0111100011100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 18 ), // #109 - INST(Ldaddb , BaseAtomicOp , (0b0011100000100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 19 ), // #110 - INST(Ldaddh , BaseAtomicOp , (0b0111100000100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 20 ), // #111 - INST(Ldaddl , BaseAtomicOp , (0b1011100001100000000000, kWX, 30, 0) , kRWI_WRX , 0 , 21 ), // #112 - INST(Ldaddlb , BaseAtomicOp , (0b0011100001100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 22 ), // #113 - INST(Ldaddlh , BaseAtomicOp , (0b0111100001100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 23 ), // #114 - INST(Ldar , BaseRM_NoImm , (0b1000100011011111111111, kWX, kZR, 30) , kRWI_W , 0 , 0 ), // #115 - INST(Ldarb , BaseRM_NoImm , (0b0000100011011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 1 ), // #116 - INST(Ldarh , BaseRM_NoImm , (0b0100100011011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 2 ), // #117 - INST(Ldaxp , BaseLdxp , (0b1000100001111111100000, kWX, 30) , kRWI_WW , 0 , 0 ), // #118 - INST(Ldaxr , BaseRM_NoImm , (0b1000100001011111111111, kWX, kZR, 30) , kRWI_W , 0 , 3 ), // #119 - INST(Ldaxrb , BaseRM_NoImm , (0b0000100001011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 4 ), // #120 - INST(Ldaxrh , BaseRM_NoImm , (0b0100100001011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 5 ), // #121 - INST(Ldclr , BaseAtomicOp , (0b1011100000100000000100, kWX, 30, 0) , kRWI_WRX , 0 , 24 ), // #122 - INST(Ldclra , BaseAtomicOp , (0b1011100010100000000100, kWX, 30, 1) , kRWI_WRX , 0 , 25 ), // #123 - INST(Ldclrab , BaseAtomicOp , (0b0011100010100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 26 ), // #124 - INST(Ldclrah , BaseAtomicOp , (0b0111100010100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 27 ), // #125 - INST(Ldclral , BaseAtomicOp , (0b1011100011100000000100, kWX, 30, 1) , kRWI_WRX , 0 , 28 ), // #126 - INST(Ldclralb , BaseAtomicOp , (0b0011100011100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 29 ), // #127 - INST(Ldclralh , BaseAtomicOp , (0b0111100011100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 30 ), // #128 - INST(Ldclrb , BaseAtomicOp , (0b0011100000100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 31 ), // #129 - INST(Ldclrh , BaseAtomicOp , (0b0111100000100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 32 ), // #130 - INST(Ldclrl , BaseAtomicOp , (0b1011100001100000000100, kWX, 30, 0) , kRWI_WRX , 0 , 33 ), // #131 - INST(Ldclrlb , BaseAtomicOp , (0b0011100001100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 34 ), // #132 - INST(Ldclrlh , BaseAtomicOp , (0b0111100001100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 35 ), // #133 - INST(Ldeor , BaseAtomicOp , (0b1011100000100000001000, kWX, 30, 0) , kRWI_WRX , 0 , 36 ), // #134 - INST(Ldeora , BaseAtomicOp , (0b1011100010100000001000, kWX, 30, 1) , kRWI_WRX , 0 , 37 ), // #135 - INST(Ldeorab , BaseAtomicOp , (0b0011100010100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 38 ), // #136 - INST(Ldeorah , BaseAtomicOp , (0b0111100010100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 39 ), // #137 - INST(Ldeoral , BaseAtomicOp , (0b1011100011100000001000, kWX, 30, 1) , kRWI_WRX , 0 , 40 ), // #138 - INST(Ldeoralb , BaseAtomicOp , (0b0011100011100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 41 ), // #139 - INST(Ldeoralh , BaseAtomicOp , (0b0111100011100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 42 ), // #140 - INST(Ldeorb , BaseAtomicOp , (0b0011100000100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 43 ), // #141 - INST(Ldeorh , BaseAtomicOp , (0b0111100000100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 44 ), // #142 - INST(Ldeorl , BaseAtomicOp , (0b1011100001100000001000, kWX, 30, 0) , kRWI_WRX , 0 , 45 ), // #143 - INST(Ldeorlb , BaseAtomicOp , (0b0011100001100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 46 ), // #144 - INST(Ldeorlh , BaseAtomicOp , (0b0111100001100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 47 ), // #145 - INST(Ldg , BaseRM_SImm9 , (0b1101100101100000000000, 0b0000000000000000000000, kX , kZR, 0, 4) , kRWI_W , 0 , 0 ), // #146 - INST(Ldgm , BaseRM_NoImm , (0b1101100111100000000000, kX , kZR, 0 ) , kRWI_W , 0 , 6 ), // #147 - INST(Ldlar , BaseRM_NoImm , (0b1000100011011111011111, kWX, kZR, 30) , kRWI_W , 0 , 7 ), // #148 - INST(Ldlarb , BaseRM_NoImm , (0b0000100011011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 8 ), // #149 - INST(Ldlarh , BaseRM_NoImm , (0b0100100011011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 9 ), // #150 - INST(Ldnp , BaseLdpStp , (0b0010100001, 0 , kWX, 31, 2) , kRWI_WW , 0 , 0 ), // #151 - INST(Ldp , BaseLdpStp , (0b0010100101, 0b0010100011, kWX, 31, 2) , kRWI_WW , 0 , 1 ), // #152 - INST(Ldpsw , BaseLdpStp , (0b0110100101, 0b0110100011, kX , 0 , 2) , kRWI_WW , 0 , 2 ), // #153 - INST(Ldr , BaseLdSt , (0b1011100101, 0b10111000010, 0b10111000011, 0b00011000, kWX, 30, 2, Inst::kIdLdur) , kRWI_W , 0 , 0 ), // #154 - INST(Ldraa , BaseRM_SImm10 , (0b1111100000100000000001, kX , kZR, 0, 3) , kRWI_W , 0 , 0 ), // #155 - INST(Ldrab , BaseRM_SImm10 , (0b1111100010100000000001, kX , kZR, 0, 3) , kRWI_W , 0 , 1 ), // #156 - INST(Ldrb , BaseLdSt , (0b0011100101, 0b00111000010, 0b00111000011, 0 , kW , 0 , 0, Inst::kIdLdurb) , kRWI_W , 0 , 1 ), // #157 - INST(Ldrh , BaseLdSt , (0b0111100101, 0b01111000010, 0b01111000011, 0 , kW , 0 , 1, Inst::kIdLdurh) , kRWI_W , 0 , 2 ), // #158 - INST(Ldrsb , BaseLdSt , (0b0011100111, 0b00111000100, 0b00111000111, 0 , kWX, 22, 0, Inst::kIdLdursb) , kRWI_W , 0 , 3 ), // #159 - INST(Ldrsh , BaseLdSt , (0b0111100111, 0b01111000100, 0b01111000111, 0 , kWX, 22, 1, Inst::kIdLdursh) , kRWI_W , 0 , 4 ), // #160 - INST(Ldrsw , BaseLdSt , (0b1011100110, 0b10111000100, 0b10111000101, 0b10011000, kX , 0 , 2, Inst::kIdLdursw) , kRWI_W , 0 , 5 ), // #161 - INST(Ldset , BaseAtomicOp , (0b1011100000100000001100, kWX, 30, 0) , kRWI_WRX , 0 , 48 ), // #162 - INST(Ldseta , BaseAtomicOp , (0b1011100010100000001100, kWX, 30, 1) , kRWI_WRX , 0 , 49 ), // #163 - INST(Ldsetab , BaseAtomicOp , (0b0011100010100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 50 ), // #164 - INST(Ldsetah , BaseAtomicOp , (0b0111100010100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 51 ), // #165 - INST(Ldsetal , BaseAtomicOp , (0b1011100011100000001100, kWX, 30, 1) , kRWI_WRX , 0 , 52 ), // #166 - INST(Ldsetalb , BaseAtomicOp , (0b0011100011100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 53 ), // #167 - INST(Ldsetalh , BaseAtomicOp , (0b0111100011100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 54 ), // #168 - INST(Ldsetb , BaseAtomicOp , (0b0011100000100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 55 ), // #169 - INST(Ldseth , BaseAtomicOp , (0b0111100000100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 56 ), // #170 - INST(Ldsetl , BaseAtomicOp , (0b1011100001100000001100, kWX, 30, 0) , kRWI_WRX , 0 , 57 ), // #171 - INST(Ldsetlb , BaseAtomicOp , (0b0011100001100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 58 ), // #172 - INST(Ldsetlh , BaseAtomicOp , (0b0111100001100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 59 ), // #173 - INST(Ldsmax , BaseAtomicOp , (0b1011100000100000010000, kWX, 30, 0) , kRWI_WRX , 0 , 60 ), // #174 - INST(Ldsmaxa , BaseAtomicOp , (0b1011100010100000010000, kWX, 30, 1) , kRWI_WRX , 0 , 61 ), // #175 - INST(Ldsmaxab , BaseAtomicOp , (0b0011100010100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 62 ), // #176 - INST(Ldsmaxah , BaseAtomicOp , (0b0111100010100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 63 ), // #177 - INST(Ldsmaxal , BaseAtomicOp , (0b1011100011100000010000, kWX, 30, 1) , kRWI_WRX , 0 , 64 ), // #178 - INST(Ldsmaxalb , BaseAtomicOp , (0b0011100011100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 65 ), // #179 - INST(Ldsmaxalh , BaseAtomicOp , (0b0111100011100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 66 ), // #180 - INST(Ldsmaxb , BaseAtomicOp , (0b0011100000100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 67 ), // #181 - INST(Ldsmaxh , BaseAtomicOp , (0b0111100000100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 68 ), // #182 - INST(Ldsmaxl , BaseAtomicOp , (0b1011100001100000010000, kWX, 30, 0) , kRWI_WRX , 0 , 69 ), // #183 - INST(Ldsmaxlb , BaseAtomicOp , (0b0011100001100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 70 ), // #184 - INST(Ldsmaxlh , BaseAtomicOp , (0b0111100001100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 71 ), // #185 - INST(Ldsmin , BaseAtomicOp , (0b1011100000100000010100, kWX, 30, 0) , kRWI_WRX , 0 , 72 ), // #186 - INST(Ldsmina , BaseAtomicOp , (0b1011100010100000010100, kWX, 30, 1) , kRWI_WRX , 0 , 73 ), // #187 - INST(Ldsminab , BaseAtomicOp , (0b0011100010100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 74 ), // #188 - INST(Ldsminah , BaseAtomicOp , (0b0111100010100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 75 ), // #189 - INST(Ldsminal , BaseAtomicOp , (0b1011100011100000010100, kWX, 30, 1) , kRWI_WRX , 0 , 76 ), // #190 - INST(Ldsminalb , BaseAtomicOp , (0b0011100011100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 77 ), // #191 - INST(Ldsminalh , BaseAtomicOp , (0b0111100011100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 78 ), // #192 - INST(Ldsminb , BaseAtomicOp , (0b0011100000100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 79 ), // #193 - INST(Ldsminh , BaseAtomicOp , (0b0111100000100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 80 ), // #194 - INST(Ldsminl , BaseAtomicOp , (0b1011100001100000010100, kWX, 30, 0) , kRWI_WRX , 0 , 81 ), // #195 - INST(Ldsminlb , BaseAtomicOp , (0b0011100001100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 82 ), // #196 - INST(Ldsminlh , BaseAtomicOp , (0b0111100001100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 83 ), // #197 - INST(Ldtr , BaseRM_SImm9 , (0b1011100001000000000010, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_W , 0 , 1 ), // #198 - INST(Ldtrb , BaseRM_SImm9 , (0b0011100001000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 2 ), // #199 - INST(Ldtrh , BaseRM_SImm9 , (0b0111100001000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 3 ), // #200 - INST(Ldtrsb , BaseRM_SImm9 , (0b0011100011000000000010, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 4 ), // #201 - INST(Ldtrsh , BaseRM_SImm9 , (0b0111100011000000000010, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 5 ), // #202 - INST(Ldtrsw , BaseRM_SImm9 , (0b1011100010000000000010, 0b0000000000000000000000, kX , kZR, 0 , 0) , kRWI_W , 0 , 6 ), // #203 - INST(Ldumax , BaseAtomicOp , (0b1011100000100000011000, kWX, 30, 0) , kRWI_WRX , 0 , 84 ), // #204 - INST(Ldumaxa , BaseAtomicOp , (0b1011100010100000011000, kWX, 30, 1) , kRWI_WRX , 0 , 85 ), // #205 - INST(Ldumaxab , BaseAtomicOp , (0b0011100010100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 86 ), // #206 - INST(Ldumaxah , BaseAtomicOp , (0b0111100010100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 87 ), // #207 - INST(Ldumaxal , BaseAtomicOp , (0b1011100011100000011000, kWX, 30, 1) , kRWI_WRX , 0 , 88 ), // #208 - INST(Ldumaxalb , BaseAtomicOp , (0b0011100011100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 89 ), // #209 - INST(Ldumaxalh , BaseAtomicOp , (0b0111100011100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 90 ), // #210 - INST(Ldumaxb , BaseAtomicOp , (0b0011100000100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 91 ), // #211 - INST(Ldumaxh , BaseAtomicOp , (0b0111100000100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 92 ), // #212 - INST(Ldumaxl , BaseAtomicOp , (0b1011100001100000011000, kWX, 30, 0) , kRWI_WRX , 0 , 93 ), // #213 - INST(Ldumaxlb , BaseAtomicOp , (0b0011100001100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 94 ), // #214 - INST(Ldumaxlh , BaseAtomicOp , (0b0111100001100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 95 ), // #215 - INST(Ldumin , BaseAtomicOp , (0b1011100000100000011100, kWX, 30, 0) , kRWI_WRX , 0 , 96 ), // #216 - INST(Ldumina , BaseAtomicOp , (0b1011100010100000011100, kWX, 30, 1) , kRWI_WRX , 0 , 97 ), // #217 - INST(Lduminab , BaseAtomicOp , (0b0011100010100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 98 ), // #218 - INST(Lduminah , BaseAtomicOp , (0b0111100010100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 99 ), // #219 - INST(Lduminal , BaseAtomicOp , (0b1011100011100000011100, kWX, 30, 1) , kRWI_WRX , 0 , 100), // #220 - INST(Lduminalb , BaseAtomicOp , (0b0011100011100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 101), // #221 - INST(Lduminalh , BaseAtomicOp , (0b0111100011100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 102), // #222 - INST(Lduminb , BaseAtomicOp , (0b0011100000100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 103), // #223 - INST(Lduminh , BaseAtomicOp , (0b0111100000100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 104), // #224 - INST(Lduminl , BaseAtomicOp , (0b1011100001100000011100, kWX, 30, 0) , kRWI_WRX , 0 , 105), // #225 - INST(Lduminlb , BaseAtomicOp , (0b0011100001100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 106), // #226 - INST(Lduminlh , BaseAtomicOp , (0b0111100001100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 107), // #227 - INST(Ldur , BaseRM_SImm9 , (0b1011100001000000000000, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_W , 0 , 7 ), // #228 - INST(Ldurb , BaseRM_SImm9 , (0b0011100001000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 8 ), // #229 - INST(Ldurh , BaseRM_SImm9 , (0b0111100001000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 9 ), // #230 - INST(Ldursb , BaseRM_SImm9 , (0b0011100011000000000000, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 10 ), // #231 - INST(Ldursh , BaseRM_SImm9 , (0b0111100011000000000000, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 11 ), // #232 - INST(Ldursw , BaseRM_SImm9 , (0b1011100010000000000000, 0b0000000000000000000000, kX , kZR, 0 , 0) , kRWI_W , 0 , 12 ), // #233 - INST(Ldxp , BaseLdxp , (0b1000100001111111000000, kWX, 30) , kRWI_WW , 0 , 1 ), // #234 - INST(Ldxr , BaseRM_NoImm , (0b1000100001011111011111, kWX, kZR, 30) , kRWI_W , 0 , 10 ), // #235 - INST(Ldxrb , BaseRM_NoImm , (0b0000100001011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 11 ), // #236 - INST(Ldxrh , BaseRM_NoImm , (0b0100100001011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 12 ), // #237 - INST(Lsl , BaseShift , (0b0001101011000000001000, 0b0101001100000000000000, 0) , kRWI_W , 0 , 2 ), // #238 - INST(Lslv , BaseShift , (0b0001101011000000001000, 0b0000000000000000000000, 0) , kRWI_W , 0 , 3 ), // #239 - INST(Lsr , BaseShift , (0b0001101011000000001001, 0b0101001100000000011111, 0) , kRWI_W , 0 , 4 ), // #240 - INST(Lsrv , BaseShift , (0b0001101011000000001001, 0b0000000000000000000000, 0) , kRWI_W , 0 , 5 ), // #241 - INST(Madd , BaseRRRR , (0b0001101100000000000000, kWX, kZR, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 0 ), // #242 - INST(Mneg , BaseRRR , (0b0001101100000000111111, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 11 ), // #243 - INST(Mov , BaseMov , (_) , kRWI_W , 0 , 0 ), // #244 - INST(Movk , BaseMovKNZ , (0b01110010100000000000000000000000) , kRWI_X , 0 , 0 ), // #245 - INST(Movn , BaseMovKNZ , (0b00010010100000000000000000000000) , kRWI_W , 0 , 1 ), // #246 - INST(Movz , BaseMovKNZ , (0b01010010100000000000000000000000) , kRWI_W , 0 , 2 ), // #247 - INST(Mrs , BaseMrs , (_) , kRWI_W , 0 , 0 ), // #248 - INST(Msr , BaseMsr , (_) , kRWI_W , 0 , 0 ), // #249 - INST(Msub , BaseRRRR , (0b0001101100000000100000, kWX, kZR, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 1 ), // #250 - INST(Mul , BaseRRR , (0b0001101100000000011111, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 12 ), // #251 - INST(Mvn , BaseMvnNeg , (0b00101010001000000000001111100000) , kRWI_W , 0 , 0 ), // #252 - INST(Neg , BaseMvnNeg , (0b01001011000000000000001111100000) , kRWI_W , 0 , 1 ), // #253 - INST(Negs , BaseMvnNeg , (0b01101011000000000000001111100000) , kRWI_W , 0 , 2 ), // #254 - INST(Ngc , BaseRR , (0b01011010000000000000001111100000, kWX, kZR, 0, kWX, kZR, 16, true) , kRWI_W , 0 , 7 ), // #255 - INST(Ngcs , BaseRR , (0b01111010000000000000001111100000, kWX, kZR, 0, kWX, kZR, 16, true) , kRWI_W , 0 , 8 ), // #256 - INST(Nop , BaseOp , (0b11010101000000110010000000011111) , 0 , 0 , 13 ), // #257 - INST(Orn , BaseLogical , (0b0101010001, 0b01100100, 1) , kRWI_W , 0 , 6 ), // #258 - INST(Orr , BaseLogical , (0b0101010000, 0b01100100, 0) , kRWI_W , 0 , 7 ), // #259 - INST(Pacda , BaseRR , (0b11011010110000010000100000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 9 ), // #260 - INST(Pacdb , BaseRR , (0b11011010110000010000110000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 10 ), // #261 - INST(Pacdza , BaseR , (0b11011010110000010010101111100000, kX, kZR, 0) , kRWI_X , 0 , 4 ), // #262 - INST(Pacdzb , BaseR , (0b11011010110000010010111111100000, kX, kZR, 0) , kRWI_X , 0 , 5 ), // #263 - INST(Pacga , BaseRRR , (0b1001101011000000001100, kX, kZR, kX, kZR, kX, kSP, false) , kRWI_W , 0 , 13 ), // #264 - INST(Prfm , BasePrfm , (0b11111000101, 0b1111100110, 0b11111000100, 0b11011000) , kRWI_R , 0 , 0 ), // #265 - INST(Pssbb , BaseOp , (0b11010101000000110011010010011111) , 0 , 0 , 14 ), // #266 - INST(Rbit , BaseRR , (0b01011010110000000000000000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 11 ), // #267 - INST(Ret , BaseBranchReg , (0b11010110010111110000000000000000) , kRWI_R , 0 , 2 ), // #268 - INST(Rev , BaseRev , (_) , kRWI_W , 0 , 0 ), // #269 - INST(Rev16 , BaseRR , (0b01011010110000000000010000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 12 ), // #270 - INST(Rev32 , BaseRR , (0b11011010110000000000100000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 13 ), // #271 - INST(Rev64 , BaseRR , (0b11011010110000000000110000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 14 ), // #272 - INST(Ror , BaseShift , (0b0001101011000000001011, 0b0001001110000000000000, 1) , kRWI_W , 0 , 6 ), // #273 - INST(Rorv , BaseShift , (0b0001101011000000001011, 0b0000000000000000000000, 1) , kRWI_W , 0 , 7 ), // #274 - INST(Sbc , BaseRRR , (0b0101101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 14 ), // #275 - INST(Sbcs , BaseRRR , (0b0111101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 15 ), // #276 - INST(Sbfiz , BaseBfi , (0b00010011000000000000000000000000) , kRWI_W , 0 , 1 ), // #277 - INST(Sbfm , BaseBfm , (0b00010011000000000000000000000000) , kRWI_W , 0 , 1 ), // #278 - INST(Sbfx , BaseBfx , (0b00010011000000000000000000000000) , kRWI_W , 0 , 1 ), // #279 - INST(Sdiv , BaseRRR , (0b0001101011000000000011, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 16 ), // #280 - INST(Setf8 , BaseR , (0b00111010000000000000100000001101, kW, kZR, 5) , 0 , 0 , 6 ), // #281 - INST(Setf16 , BaseR , (0b00111010000000000100100000001101, kW, kZR, 5) , 0 , 0 , 7 ), // #282 - INST(Sev , BaseOp , (0b11010101000000110010000010011111) , 0 , 0 , 15 ), // #283 - INST(Sevl , BaseOp , (0b11010101000000110010000010111111) , 0 , 0 , 16 ), // #284 - INST(Smaddl , BaseRRRR , (0b1001101100100000000000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 2 ), // #285 - INST(Smc , BaseOpImm , (0b11010100000000000000000000000011, 16, 5) , 0 , 0 , 11 ), // #286 - INST(Smnegl , BaseRRR , (0b1001101100100000111111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 17 ), // #287 - INST(Smsubl , BaseRRRR , (0b1001101100100000100000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 3 ), // #288 - INST(Smulh , BaseRRR , (0b1001101101000000011111, kX , kZR, kX , kZR, kX , kZR, true) , kRWI_W , 0 , 18 ), // #289 - INST(Smull , BaseRRR , (0b1001101100100000011111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 19 ), // #290 - INST(Ssbb , BaseOp , (0b11010101000000110011000010011111) , 0 , 0 , 17 ), // #291 - INST(St2g , BaseRM_SImm9 , (0b1101100110100000000010, 0b1101100110100000000001, kX, kSP, 0, 4) , kRWI_RW , 0 , 13 ), // #292 - INST(Stadd , BaseAtomicSt , (0b1011100000100000000000, kWX, 30) , kRWI_RX , 0 , 0 ), // #293 - INST(Staddl , BaseAtomicSt , (0b1011100001100000000000, kWX, 30) , kRWI_RX , 0 , 1 ), // #294 - INST(Staddb , BaseAtomicSt , (0b0011100000100000000000, kW , 0 ) , kRWI_RX , 0 , 2 ), // #295 - INST(Staddlb , BaseAtomicSt , (0b0011100001100000000000, kW , 0 ) , kRWI_RX , 0 , 3 ), // #296 - INST(Staddh , BaseAtomicSt , (0b0111100000100000000000, kW , 0 ) , kRWI_RX , 0 , 4 ), // #297 - INST(Staddlh , BaseAtomicSt , (0b0111100001100000000000, kW , 0 ) , kRWI_RX , 0 , 5 ), // #298 - INST(Stclr , BaseAtomicSt , (0b1011100000100000000100, kWX, 30) , kRWI_RX , 0 , 6 ), // #299 - INST(Stclrl , BaseAtomicSt , (0b1011100001100000000100, kWX, 30) , kRWI_RX , 0 , 7 ), // #300 - INST(Stclrb , BaseAtomicSt , (0b0011100000100000000100, kW , 0 ) , kRWI_RX , 0 , 8 ), // #301 - INST(Stclrlb , BaseAtomicSt , (0b0011100001100000000100, kW , 0 ) , kRWI_RX , 0 , 9 ), // #302 - INST(Stclrh , BaseAtomicSt , (0b0111100000100000000100, kW , 0 ) , kRWI_RX , 0 , 10 ), // #303 - INST(Stclrlh , BaseAtomicSt , (0b0111100001100000000100, kW , 0 ) , kRWI_RX , 0 , 11 ), // #304 - INST(Steor , BaseAtomicSt , (0b1011100000100000001000, kWX, 30) , kRWI_RX , 0 , 12 ), // #305 - INST(Steorl , BaseAtomicSt , (0b1011100001100000001000, kWX, 30) , kRWI_RX , 0 , 13 ), // #306 - INST(Steorb , BaseAtomicSt , (0b0011100000100000001000, kW , 0 ) , kRWI_RX , 0 , 14 ), // #307 - INST(Steorlb , BaseAtomicSt , (0b0011100001100000001000, kW , 0 ) , kRWI_RX , 0 , 15 ), // #308 - INST(Steorh , BaseAtomicSt , (0b0111100000100000001000, kW , 0 ) , kRWI_RX , 0 , 16 ), // #309 - INST(Steorlh , BaseAtomicSt , (0b0111100001100000001000, kW , 0 ) , kRWI_RX , 0 , 17 ), // #310 - INST(Stg , BaseRM_SImm9 , (0b1101100100100000000010, 0b1101100100100000000001, kX, kSP, 0, 4) , kRWI_RW , 0 , 14 ), // #311 - INST(Stgm , BaseRM_NoImm , (0b1101100110100000000000, kX , kZR, 0 ) , kRWI_RW , 0 , 13 ), // #312 - INST(Stgp , BaseLdpStp , (0b0110100100, 0b0110100010, kX, 0, 4) , kRWI_RRW , 0 , 3 ), // #313 - INST(Stllr , BaseRM_NoImm , (0b1000100010011111011111, kWX, kZR, 30) , kRWI_RW , 0 , 14 ), // #314 - INST(Stllrb , BaseRM_NoImm , (0b0000100010011111011111, kW , kZR, 0 ) , kRWI_RW , 0 , 15 ), // #315 - INST(Stllrh , BaseRM_NoImm , (0b0100100010011111011111, kW , kZR, 0 ) , kRWI_RW , 0 , 16 ), // #316 - INST(Stlr , BaseRM_NoImm , (0b1000100010011111111111, kWX, kZR, 30) , kRWI_RW , 0 , 17 ), // #317 - INST(Stlrb , BaseRM_NoImm , (0b0000100010011111111111, kW , kZR, 0 ) , kRWI_RW , 0 , 18 ), // #318 - INST(Stlrh , BaseRM_NoImm , (0b0100100010011111111111, kW , kZR, 0 ) , kRWI_RW , 0 , 19 ), // #319 - INST(Stlxp , BaseStxp , (0b1000100000100000100000, kWX, 30) , kRWI_WRRX , 0 , 0 ), // #320 - INST(Stlxr , BaseAtomicOp , (0b1000100000000000111111, kWX, 30, 1) , kRWI_WRX , 0 , 108), // #321 - INST(Stlxrb , BaseAtomicOp , (0b0000100000000000111111, kW , 0 , 1) , kRWI_WRX , 0 , 109), // #322 - INST(Stlxrh , BaseAtomicOp , (0b0100100000000000111111, kW , 0 , 1) , kRWI_WRX , 0 , 110), // #323 - INST(Stnp , BaseLdpStp , (0b0010100000, 0 , kWX, 31, 2) , kRWI_RRW , 0 , 4 ), // #324 - INST(Stp , BaseLdpStp , (0b0010100100, 0b0010100010, kWX, 31, 2) , kRWI_RRW , 0 , 5 ), // #325 - INST(Str , BaseLdSt , (0b1011100100, 0b10111000000, 0b10111000001, 0 , kWX, 30, 2, Inst::kIdStur) , kRWI_RW , 0 , 6 ), // #326 - INST(Strb , BaseLdSt , (0b0011100100, 0b00111000000, 0b00111000001, 0 , kW , 30, 0, Inst::kIdSturb) , kRWI_RW , 0 , 7 ), // #327 - INST(Strh , BaseLdSt , (0b0111100100, 0b01111000000, 0b01111000001, 0 , kWX, 30, 1, Inst::kIdSturh) , kRWI_RW , 0 , 8 ), // #328 - INST(Stset , BaseAtomicSt , (0b1011100000100000001100, kWX, 30) , kRWI_RX , 0 , 18 ), // #329 - INST(Stsetl , BaseAtomicSt , (0b1011100001100000001100, kWX, 30) , kRWI_RX , 0 , 19 ), // #330 - INST(Stsetb , BaseAtomicSt , (0b0011100000100000001100, kW , 0 ) , kRWI_RX , 0 , 20 ), // #331 - INST(Stsetlb , BaseAtomicSt , (0b0011100001100000001100, kW , 0 ) , kRWI_RX , 0 , 21 ), // #332 - INST(Stseth , BaseAtomicSt , (0b0111100000100000001100, kW , 0 ) , kRWI_RX , 0 , 22 ), // #333 - INST(Stsetlh , BaseAtomicSt , (0b0111100001100000001100, kW , 0 ) , kRWI_RX , 0 , 23 ), // #334 - INST(Stsmax , BaseAtomicSt , (0b1011100000100000010000, kWX, 30) , kRWI_RX , 0 , 24 ), // #335 - INST(Stsmaxl , BaseAtomicSt , (0b1011100001100000010000, kWX, 30) , kRWI_RX , 0 , 25 ), // #336 - INST(Stsmaxb , BaseAtomicSt , (0b0011100000100000010000, kW , 0 ) , kRWI_RX , 0 , 26 ), // #337 - INST(Stsmaxlb , BaseAtomicSt , (0b0011100001100000010000, kW , 0 ) , kRWI_RX , 0 , 27 ), // #338 - INST(Stsmaxh , BaseAtomicSt , (0b0111100000100000010000, kW , 0 ) , kRWI_RX , 0 , 28 ), // #339 - INST(Stsmaxlh , BaseAtomicSt , (0b0111100001100000010000, kW , 0 ) , kRWI_RX , 0 , 29 ), // #340 - INST(Stsmin , BaseAtomicSt , (0b1011100000100000010100, kWX, 30) , kRWI_RX , 0 , 30 ), // #341 - INST(Stsminl , BaseAtomicSt , (0b1011100001100000010100, kWX, 30) , kRWI_RX , 0 , 31 ), // #342 - INST(Stsminb , BaseAtomicSt , (0b0011100000100000010100, kW , 0 ) , kRWI_RX , 0 , 32 ), // #343 - INST(Stsminlb , BaseAtomicSt , (0b0011100001100000010100, kW , 0 ) , kRWI_RX , 0 , 33 ), // #344 - INST(Stsminh , BaseAtomicSt , (0b0111100000100000010100, kW , 0 ) , kRWI_RX , 0 , 34 ), // #345 - INST(Stsminlh , BaseAtomicSt , (0b0111100001100000010100, kW , 0 ) , kRWI_RX , 0 , 35 ), // #346 - INST(Sttr , BaseRM_SImm9 , (0b1011100000000000000010, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_RW , 0 , 15 ), // #347 - INST(Sttrb , BaseRM_SImm9 , (0b0011100000000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 16 ), // #348 - INST(Sttrh , BaseRM_SImm9 , (0b0111100000000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 17 ), // #349 - INST(Stumax , BaseAtomicSt , (0b1011100000100000011000, kWX, 30) , kRWI_RX , 0 , 36 ), // #350 - INST(Stumaxl , BaseAtomicSt , (0b1011100001100000011000, kWX, 30) , kRWI_RX , 0 , 37 ), // #351 - INST(Stumaxb , BaseAtomicSt , (0b0011100000100000011000, kW , 0 ) , kRWI_RX , 0 , 38 ), // #352 - INST(Stumaxlb , BaseAtomicSt , (0b0011100001100000011000, kW , 0 ) , kRWI_RX , 0 , 39 ), // #353 - INST(Stumaxh , BaseAtomicSt , (0b0111100000100000011000, kW , 0 ) , kRWI_RX , 0 , 40 ), // #354 - INST(Stumaxlh , BaseAtomicSt , (0b0111100001100000011000, kW , 0 ) , kRWI_RX , 0 , 41 ), // #355 - INST(Stumin , BaseAtomicSt , (0b1011100000100000011100, kWX, 30) , kRWI_RX , 0 , 42 ), // #356 - INST(Stuminl , BaseAtomicSt , (0b1011100001100000011100, kWX, 30) , kRWI_RX , 0 , 43 ), // #357 - INST(Stuminb , BaseAtomicSt , (0b0011100000100000011100, kW , 0 ) , kRWI_RX , 0 , 44 ), // #358 - INST(Stuminlb , BaseAtomicSt , (0b0011100001100000011100, kW , 0 ) , kRWI_RX , 0 , 45 ), // #359 - INST(Stuminh , BaseAtomicSt , (0b0111100000100000011100, kW , 0 ) , kRWI_RX , 0 , 46 ), // #360 - INST(Stuminlh , BaseAtomicSt , (0b0111100001100000011100, kW , 0 ) , kRWI_RX , 0 , 47 ), // #361 - INST(Stur , BaseRM_SImm9 , (0b1011100000000000000000, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_RW , 0 , 18 ), // #362 - INST(Sturb , BaseRM_SImm9 , (0b0011100000000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 19 ), // #363 - INST(Sturh , BaseRM_SImm9 , (0b0111100000000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 20 ), // #364 - INST(Stxp , BaseStxp , (0b1000100000100000000000, kWX, 30) , kRWI_WRRW , 0 , 1 ), // #365 - INST(Stxr , BaseStx , (0b1000100000000000011111, kWX, 30) , kRWI_WRW , 0 , 0 ), // #366 - INST(Stxrb , BaseStx , (0b0000100000000000011111, kW , 0 ) , kRWI_WRW , 0 , 1 ), // #367 - INST(Stxrh , BaseStx , (0b0100100000000000011111, kW , 0 ) , kRWI_WRW , 0 , 2 ), // #368 - INST(Stz2g , BaseRM_SImm9 , (0b1101100111100000000010, 0b1101100111100000000001, kX , kSP, 0, 4) , kRWI_RW , 0 , 21 ), // #369 - INST(Stzg , BaseRM_SImm9 , (0b1101100101100000000010, 0b1101100101100000000001, kX , kSP, 0, 4) , kRWI_RW , 0 , 22 ), // #370 - INST(Stzgm , BaseRM_NoImm , (0b1101100100100000000000, kX , kZR, 0) , kRWI_RW , 0 , 20 ), // #371 - INST(Sub , BaseAddSub , (0b1001011000, 0b1001011001, 0b1010001) , kRWI_W , 0 , 2 ), // #372 - INST(Subg , BaseRRII , (0b1101000110000000000000, kX, kSP, kX, kSP, 6, 4, 16, 4, 0, 10) , kRWI_W , 0 , 1 ), // #373 - INST(Subp , BaseRRR , (0b1001101011000000000000, kX, kZR, kX, kSP, kX, kSP, false) , kRWI_W , 0 , 20 ), // #374 - INST(Subps , BaseRRR , (0b1011101011000000000000, kX, kZR, kX, kSP, kX, kSP, false) , kRWI_W , 0 , 21 ), // #375 - INST(Subs , BaseAddSub , (0b1101011000, 0b1101011001, 0b1110001) , kRWI_W , 0 , 3 ), // #376 - INST(Svc , BaseOpImm , (0b11010100000000000000000000000001, 16, 5) , 0 , 0 , 12 ), // #377 - INST(Swp , BaseAtomicOp , (0b1011100000100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 111), // #378 - INST(Swpa , BaseAtomicOp , (0b1011100010100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 112), // #379 - INST(Swpab , BaseAtomicOp , (0b0011100010100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 113), // #380 - INST(Swpah , BaseAtomicOp , (0b0111100010100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 114), // #381 - INST(Swpal , BaseAtomicOp , (0b1011100011100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 115), // #382 - INST(Swpalb , BaseAtomicOp , (0b0011100011100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 116), // #383 - INST(Swpalh , BaseAtomicOp , (0b0111100011100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 117), // #384 - INST(Swpb , BaseAtomicOp , (0b0011100000100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 118), // #385 - INST(Swph , BaseAtomicOp , (0b0111100000100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 119), // #386 - INST(Swpl , BaseAtomicOp , (0b1011100001100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 120), // #387 - INST(Swplb , BaseAtomicOp , (0b0011100001100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 121), // #388 - INST(Swplh , BaseAtomicOp , (0b0111100001100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 122), // #389 - INST(Sxtb , BaseExtend , (0b0001001100000000000111, kWX, 0) , kRWI_W , 0 , 0 ), // #390 - INST(Sxth , BaseExtend , (0b0001001100000000001111, kWX, 0) , kRWI_W , 0 , 1 ), // #391 - INST(Sxtw , BaseExtend , (0b1001001101000000011111, kX , 0) , kRWI_W , 0 , 2 ), // #392 - INST(Sys , BaseSys , (_) , kRWI_W , 0 , 0 ), // #393 - INST(Tlbi , BaseAtDcIcTlbi , (0b00011110000000, 0b00010000000000, false) , kRWI_RX , 0 , 3 ), // #394 - INST(Tst , BaseTst , (0b1101010000, 0b111001000) , kRWI_R , 0 , 0 ), // #395 - INST(Tbnz , BaseBranchTst , (0b00110111000000000000000000000000) , kRWI_R , 0 , 0 ), // #396 - INST(Tbz , BaseBranchTst , (0b00110110000000000000000000000000) , kRWI_R , 0 , 1 ), // #397 - INST(Ubfiz , BaseBfi , (0b01010011000000000000000000000000) , kRWI_W , 0 , 2 ), // #398 - INST(Ubfm , BaseBfm , (0b01010011000000000000000000000000) , kRWI_W , 0 , 2 ), // #399 - INST(Ubfx , BaseBfx , (0b01010011000000000000000000000000) , kRWI_W , 0 , 2 ), // #400 - INST(Udf , BaseOpImm , (0b00000000000000000000000000000000, 16, 0) , 0 , 0 , 13 ), // #401 - INST(Udiv , BaseRRR , (0b0001101011000000000010, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 22 ), // #402 - INST(Umaddl , BaseRRRR , (0b1001101110100000000000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 4 ), // #403 - INST(Umnegl , BaseRRR , (0b1001101110100000111111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 23 ), // #404 - INST(Umull , BaseRRR , (0b1001101110100000011111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 24 ), // #405 - INST(Umulh , BaseRRR , (0b1001101111000000011111, kX , kZR, kX , kZR, kX , kZR, false) , kRWI_W , 0 , 25 ), // #406 - INST(Umsubl , BaseRRRR , (0b1001101110100000100000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 5 ), // #407 - INST(Uxtb , BaseExtend , (0b0101001100000000000111, kW, 1) , kRWI_W , 0 , 3 ), // #408 - INST(Uxth , BaseExtend , (0b0101001100000000001111, kW, 1) , kRWI_W , 0 , 4 ), // #409 - INST(Wfe , BaseOp , (0b11010101000000110010000001011111) , 0 , 0 , 18 ), // #410 - INST(Wfi , BaseOp , (0b11010101000000110010000001111111) , 0 , 0 , 19 ), // #411 - INST(Xaflag , BaseOp , (0b11010101000000000100000000111111) , 0 , 0 , 20 ), // #412 - INST(Xpacd , BaseR , (0b11011010110000010100011111100000, kX, kZR, 0) , kRWI_X , 0 , 8 ), // #413 - INST(Xpaci , BaseR , (0b11011010110000010100001111100000, kX, kZR, 0) , kRWI_X , 0 , 9 ), // #414 - INST(Xpaclri , BaseOp , (0b11010101000000110010000011111111) , kRWI_X , 0 , 21 ), // #415 - INST(Yield , BaseOp , (0b11010101000000110010000000111111) , 0 , 0 , 22 ), // #416 - INST(Abs_v , ISimdVV , (0b0000111000100000101110, kVO_V_Any) , kRWI_W , 0 , 0 ), // #417 - INST(Add_v , ISimdVVV , (0b0000111000100000100001, kVO_V_Any) , kRWI_W , 0 , 0 ), // #418 - INST(Addhn_v , ISimdVVV , (0b0000111000100000010000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 1 ), // #419 - INST(Addhn2_v , ISimdVVV , (0b0100111000100000010000, kVO_V_B16H8S4) , kRWI_W , F(Narrow) , 2 ), // #420 - INST(Addp_v , ISimdPair , (0b0101111000110001101110, 0b0000111000100000101111, kVO_V_Any) , kRWI_W , F(Pair) , 0 ), // #421 - INST(Addv_v , ISimdSV , (0b0000111000110001101110, kVO_V_BH_4S) , kRWI_W , 0 , 0 ), // #422 - INST(Aesd_v , ISimdVVx , (0b0100111000101000010110, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 0 ), // #423 - INST(Aese_v , ISimdVVx , (0b0100111000101000010010, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 1 ), // #424 - INST(Aesimc_v , ISimdVVx , (0b0100111000101000011110, kOp_V16B, kOp_V16B) , kRWI_W , 0 , 2 ), // #425 - INST(Aesmc_v , ISimdVVx , (0b0100111000101000011010, kOp_V16B, kOp_V16B) , kRWI_W , 0 , 3 ), // #426 - INST(And_v , ISimdVVV , (0b0000111000100000000111, kVO_V_B) , kRWI_W , 0 , 3 ), // #427 - INST(Bcax_v , ISimdVVVV , (0b1100111000100000000000, kVO_V_B16) , kRWI_W , 0 , 0 ), // #428 - INST(Bfcvt_v , ISimdVVx , (0b0001111001100011010000, kOp_H, kOp_S) , kRWI_W , 0 , 4 ), // #429 - INST(Bfcvtn_v , ISimdVVx , (0b0000111010100001011010, kOp_V4H, kOp_V4S) , kRWI_W , F(Narrow) , 5 ), // #430 - INST(Bfcvtn2_v , ISimdVVx , (0b0100111010100001011010, kOp_V8H, kOp_V4S) , kRWI_W , F(Narrow) , 6 ), // #431 - INST(Bfdot_v , SimdDot , (0b0010111001000000111111, 0b0000111101000000111100, kET_S, kET_H, kET_2H) , kRWI_X , 0 , 0 ), // #432 - INST(Bfmlalb_v , SimdFmlal , (0b0010111011000000111111, 0b0000111111000000111100, 0, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 0 ), // #433 - INST(Bfmlalt_v , SimdFmlal , (0b0110111011000000111111, 0b0100111111000000111100, 0, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 1 ), // #434 - INST(Bfmmla_v , ISimdVVVx , (0b0110111001000000111011, kOp_V4S, kOp_V8H, kOp_V8H) , kRWI_X , F(Long) , 0 ), // #435 - INST(Bic_v , SimdBicOrr , (0b0000111001100000000111, 0b0010111100000000000001) , kRWI_W , 0 , 0 ), // #436 - INST(Bif_v , ISimdVVV , (0b0010111011100000000111, kVO_V_B) , kRWI_X , 0 , 4 ), // #437 - INST(Bit_v , ISimdVVV , (0b0010111010100000000111, kVO_V_B) , kRWI_X , 0 , 5 ), // #438 - INST(Bsl_v , ISimdVVV , (0b0010111001100000000111, kVO_V_B) , kRWI_X , 0 , 6 ), // #439 - INST(Cls_v , ISimdVV , (0b0000111000100000010010, kVO_V_BHS) , kRWI_W , 0 , 1 ), // #440 - INST(Clz_v , ISimdVV , (0b0010111000100000010010, kVO_V_BHS) , kRWI_W , 0 , 2 ), // #441 - INST(Cmeq_v , SimdCmp , (0b0010111000100000100011, 0b0000111000100000100110, kVO_V_Any) , kRWI_W , 0 , 0 ), // #442 - INST(Cmge_v , SimdCmp , (0b0000111000100000001111, 0b0010111000100000100010, kVO_V_Any) , kRWI_W , 0 , 1 ), // #443 - INST(Cmgt_v , SimdCmp , (0b0000111000100000001101, 0b0000111000100000100010, kVO_V_Any) , kRWI_W , 0 , 2 ), // #444 - INST(Cmhi_v , SimdCmp , (0b0010111000100000001101, 0b0000000000000000000000, kVO_V_Any) , kRWI_W , 0 , 3 ), // #445 - INST(Cmhs_v , SimdCmp , (0b0010111000100000001111, 0b0000000000000000000000, kVO_V_Any) , kRWI_W , 0 , 4 ), // #446 - INST(Cmle_v , SimdCmp , (0b0000000000000000000000, 0b0010111000100000100110, kVO_V_Any) , kRWI_W , 0 , 5 ), // #447 - INST(Cmlt_v , SimdCmp , (0b0000000000000000000000, 0b0000111000100000101010, kVO_V_Any) , kRWI_W , 0 , 6 ), // #448 - INST(Cmtst_v , ISimdVVV , (0b0000111000100000100011, kVO_V_Any) , kRWI_W , 0 , 7 ), // #449 - INST(Cnt_v , ISimdVV , (0b0000111000100000010110, kVO_V_B) , kRWI_W , 0 , 3 ), // #450 - INST(Dup_v , SimdDup , (_) , kRWI_W , 0 , 0 ), // #451 - INST(Eor_v , ISimdVVV , (0b0010111000100000000111, kVO_V_B) , kRWI_W , 0 , 8 ), // #452 - INST(Eor3_v , ISimdVVVV , (0b1100111000000000000000, kVO_V_B16) , kRWI_W , 0 , 1 ), // #453 - INST(Ext_v , ISimdVVVI , (0b0010111000000000000000, kVO_V_B, 4, 11, 1) , kRWI_W , 0 , 0 ), // #454 - INST(Fabd_v , FSimdVVV , (0b0111111010100000110101, kHF_C, 0b0010111010100000110101, kHF_C) , kRWI_W , 0 , 0 ), // #455 - INST(Fabs_v , FSimdVV , (0b0001111000100000110000, kHF_A, 0b0000111010100000111110, kHF_B) , kRWI_W , 0 , 0 ), // #456 - INST(Facge_v , FSimdVVV , (0b0111111000100000111011, kHF_C, 0b0010111000100000111011, kHF_C) , kRWI_W , 0 , 1 ), // #457 - INST(Facgt_v , FSimdVVV , (0b0111111010100000111011, kHF_C, 0b0010111010100000111011, kHF_C) , kRWI_W , 0 , 2 ), // #458 - INST(Fadd_v , FSimdVVV , (0b0001111000100000001010, kHF_A, 0b0000111000100000110101, kHF_C) , kRWI_W , 0 , 3 ), // #459 - INST(Faddp_v , FSimdPair , (0b0111111000110000110110, 0b0010111000100000110101) , kRWI_W , 0 , 0 ), // #460 - INST(Fcadd_v , SimdFcadd , (0b0010111000000000111001) , kRWI_W , 0 , 0 ), // #461 - INST(Fccmp_v , SimdFccmpFccmpe , (0b00011110001000000000010000000000) , kRWI_R , 0 , 0 ), // #462 - INST(Fccmpe_v , SimdFccmpFccmpe , (0b00011110001000000000010000010000) , kRWI_R , 0 , 1 ), // #463 - INST(Fcmeq_v , SimdFcm , (0b0000111000100000111001, kHF_C, 0b0000111010100000110110) , kRWI_W , 0 , 0 ), // #464 - INST(Fcmge_v , SimdFcm , (0b0010111000100000111001, kHF_C, 0b0010111010100000110010) , kRWI_W , 0 , 1 ), // #465 - INST(Fcmgt_v , SimdFcm , (0b0010111010100000111001, kHF_C, 0b0000111010100000110010) , kRWI_W , 0 , 2 ), // #466 - INST(Fcmla_v , SimdFcmla , (0b0010111000000000110001, 0b0010111100000000000100) , kRWI_X , 0 , 0 ), // #467 - INST(Fcmle_v , SimdFcm , (0b0000000000000000000000, kHF_C, 0b0010111010100000110110) , kRWI_W , 0 , 3 ), // #468 - INST(Fcmlt_v , SimdFcm , (0b0000000000000000000000, kHF_C, 0b0000111010100000111010) , kRWI_W , 0 , 4 ), // #469 - INST(Fcmp_v , SimdFcmpFcmpe , (0b00011110001000000010000000000000) , kRWI_R , 0 , 0 ), // #470 - INST(Fcmpe_v , SimdFcmpFcmpe , (0b00011110001000000010000000010000) , kRWI_R , 0 , 1 ), // #471 - INST(Fcsel_v , SimdFcsel , (_) , kRWI_W , 0 , 0 ), // #472 - INST(Fcvt_v , SimdFcvt , (_) , kRWI_W , 0 , 0 ), // #473 - INST(Fcvtas_v , SimdFcvtSV , (0b0000111000100001110010, 0b0000000000000000000000, 0b0001111000100100000000, 1) , kRWI_W , 0 , 0 ), // #474 - INST(Fcvtau_v , SimdFcvtSV , (0b0010111000100001110010, 0b0000000000000000000000, 0b0001111000100101000000, 1) , kRWI_W , 0 , 1 ), // #475 - INST(Fcvtl_v , SimdFcvtLN , (0b0000111000100001011110, 0, 0) , kRWI_W , F(Long) , 0 ), // #476 - INST(Fcvtl2_v , SimdFcvtLN , (0b0100111000100001011110, 0, 0) , kRWI_W , F(Long) , 1 ), // #477 - INST(Fcvtms_v , SimdFcvtSV , (0b0000111000100001101110, 0b0000000000000000000000, 0b0001111000110000000000, 1) , kRWI_W , 0 , 2 ), // #478 - INST(Fcvtmu_v , SimdFcvtSV , (0b0010111000100001101110, 0b0000000000000000000000, 0b0001111000110001000000, 1) , kRWI_W , 0 , 3 ), // #479 - INST(Fcvtn_v , SimdFcvtLN , (0b0000111000100001011010, 0, 0) , kRWI_W , F(Narrow) , 2 ), // #480 - INST(Fcvtn2_v , SimdFcvtLN , (0b0100111000100001011010, 0, 0) , kRWI_X , F(Narrow) , 3 ), // #481 - INST(Fcvtns_v , SimdFcvtSV , (0b0000111000100001101010, 0b0000000000000000000000, 0b0001111000100000000000, 1) , kRWI_W , 0 , 4 ), // #482 - INST(Fcvtnu_v , SimdFcvtSV , (0b0010111000100001101010, 0b0000000000000000000000, 0b0001111000100001000000, 1) , kRWI_W , 0 , 5 ), // #483 - INST(Fcvtps_v , SimdFcvtSV , (0b0000111010100001101010, 0b0000000000000000000000, 0b0001111000101000000000, 1) , kRWI_W , 0 , 6 ), // #484 - INST(Fcvtpu_v , SimdFcvtSV , (0b0010111010100001101010, 0b0000000000000000000000, 0b0001111000101001000000, 1) , kRWI_W , 0 , 7 ), // #485 - INST(Fcvtxn_v , SimdFcvtLN , (0b0010111000100001011010, 1, 1) , kRWI_W , F(Narrow) , 4 ), // #486 - INST(Fcvtxn2_v , SimdFcvtLN , (0b0110111000100001011010, 1, 0) , kRWI_X , F(Narrow) , 5 ), // #487 - INST(Fcvtzs_v , SimdFcvtSV , (0b0000111010100001101110, 0b0000111100000000111111, 0b0001111000111000000000, 1) , kRWI_W , 0 , 8 ), // #488 - INST(Fcvtzu_v , SimdFcvtSV , (0b0010111010100001101110, 0b0010111100000000111111, 0b0001111000111001000000, 1) , kRWI_W , 0 , 9 ), // #489 - INST(Fdiv_v , FSimdVVV , (0b0001111000100000000110, kHF_A, 0b0010111000100000111111, kHF_C) , kRWI_W , 0 , 4 ), // #490 - INST(Fjcvtzs_v , ISimdVVx , (0b0001111001111110000000, kOp_GpW, kOp_D) , kRWI_W , 0 , 7 ), // #491 - INST(Fmadd_v , FSimdVVVV , (0b0001111100000000000000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 0 ), // #492 - INST(Fmax_v , FSimdVVV , (0b0001111000100000010010, kHF_A, 0b0000111000100000111101, kHF_C) , kRWI_W , 0 , 5 ), // #493 - INST(Fmaxnm_v , FSimdVVV , (0b0001111000100000011010, kHF_A, 0b0000111000100000110001, kHF_C) , kRWI_W , 0 , 6 ), // #494 - INST(Fmaxnmp_v , FSimdPair , (0b0111111000110000110010, 0b0010111000100000110001) , kRWI_W , 0 , 1 ), // #495 - INST(Fmaxnmv_v , FSimdSV , (0b0010111000110000110010) , kRWI_W , 0 , 0 ), // #496 - INST(Fmaxp_v , FSimdPair , (0b0111111000110000111110, 0b0010111000100000111101) , kRWI_W , 0 , 2 ), // #497 - INST(Fmaxv_v , FSimdSV , (0b0010111000110000111110) , kRWI_W , 0 , 1 ), // #498 - INST(Fmin_v , FSimdVVV , (0b0001111000100000010110, kHF_A, 0b0000111010100000111101, kHF_C) , kRWI_W , 0 , 7 ), // #499 - INST(Fminnm_v , FSimdVVV , (0b0001111000100000011110, kHF_A, 0b0000111010100000110001, kHF_C) , kRWI_W , 0 , 8 ), // #500 - INST(Fminnmp_v , FSimdPair , (0b0111111010110000110010, 0b0010111010100000110001) , kRWI_W , 0 , 3 ), // #501 - INST(Fminnmv_v , FSimdSV , (0b0010111010110000110010) , kRWI_W , 0 , 2 ), // #502 - INST(Fminp_v , FSimdPair , (0b0111111010110000111110, 0b0010111010100000111101) , kRWI_W , 0 , 4 ), // #503 - INST(Fminv_v , FSimdSV , (0b0010111010110000111110) , kRWI_W , 0 , 3 ), // #504 - INST(Fmla_v , FSimdVVVe , (0b0000000000000000000000, kHF_N, 0b0000111000100000110011, 0b0000111110000000000100) , kRWI_X , F(VH0_15) , 0 ), // #505 - INST(Fmlal_v , SimdFmlal , (0b0000111000100000111011, 0b0000111110000000000000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 2 ), // #506 - INST(Fmlal2_v , SimdFmlal , (0b0010111000100000110011, 0b0010111110000000100000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 3 ), // #507 - INST(Fmls_v , FSimdVVVe , (0b0000000000000000000000, kHF_N, 0b0000111010100000110011, 0b0000111110000000010100) , kRWI_X , F(VH0_15) , 1 ), // #508 - INST(Fmlsl_v , SimdFmlal , (0b0000111010100000111011, 0b0000111110000000010000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 4 ), // #509 - INST(Fmlsl2_v , SimdFmlal , (0b0010111010100000110011, 0b0010111110000000110000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 5 ), // #510 - INST(Fmov_v , SimdFmov , (_) , kRWI_W , 0 , 0 ), // #511 - INST(Fmsub_v , FSimdVVVV , (0b0001111100000000100000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 1 ), // #512 - INST(Fmul_v , FSimdVVVe , (0b0001111000100000000010, kHF_A, 0b0010111000100000110111, 0b0000111110000000100100) , kRWI_W , F(VH0_15) , 2 ), // #513 - INST(Fmulx_v , FSimdVVVe , (0b0101111000100000110111, kHF_C, 0b0000111000100000110111, 0b0010111110000000100100) , kRWI_W , F(VH0_15) , 3 ), // #514 - INST(Fneg_v , FSimdVV , (0b0001111000100001010000, kHF_A, 0b0010111010100000111110, kHF_B) , kRWI_W , 0 , 1 ), // #515 - INST(Fnmadd_v , FSimdVVVV , (0b0001111100100000000000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 2 ), // #516 - INST(Fnmsub_v , FSimdVVVV , (0b0001111100100000100000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 3 ), // #517 - INST(Fnmul_v , FSimdVVV , (0b0001111000100000100010, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 9 ), // #518 - INST(Frecpe_v , FSimdVV , (0b0101111010100001110110, kHF_B, 0b0000111010100001110110, kHF_B) , kRWI_W , 0 , 2 ), // #519 - INST(Frecps_v , FSimdVVV , (0b0101111000100000111111, kHF_C, 0b0000111000100000111111, kHF_C) , kRWI_W , 0 , 10 ), // #520 - INST(Frecpx_v , FSimdVV , (0b0101111010100001111110, kHF_B, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 3 ), // #521 - INST(Frint32x_v , FSimdVV , (0b0001111000101000110000, kHF_N, 0b0010111000100001111010, kHF_N) , kRWI_W , 0 , 4 ), // #522 - INST(Frint32z_v , FSimdVV , (0b0001111000101000010000, kHF_N, 0b0000111000100001111010, kHF_N) , kRWI_W , 0 , 5 ), // #523 - INST(Frint64x_v , FSimdVV , (0b0001111000101001110000, kHF_N, 0b0010111000100001111110, kHF_N) , kRWI_W , 0 , 6 ), // #524 - INST(Frint64z_v , FSimdVV , (0b0001111000101001010000, kHF_N, 0b0000111000100001111110, kHF_N) , kRWI_W , 0 , 7 ), // #525 - INST(Frinta_v , FSimdVV , (0b0001111000100110010000, kHF_A, 0b0010111000100001100010, kHF_B) , kRWI_W , 0 , 8 ), // #526 - INST(Frinti_v , FSimdVV , (0b0001111000100111110000, kHF_A, 0b0010111010100001100110, kHF_B) , kRWI_W , 0 , 9 ), // #527 - INST(Frintm_v , FSimdVV , (0b0001111000100101010000, kHF_A, 0b0000111000100001100110, kHF_B) , kRWI_W , 0 , 10 ), // #528 - INST(Frintn_v , FSimdVV , (0b0001111000100100010000, kHF_A, 0b0000111000100001100010, kHF_B) , kRWI_W , 0 , 11 ), // #529 - INST(Frintp_v , FSimdVV , (0b0001111000100100110000, kHF_A, 0b0000111010100001100010, kHF_B) , kRWI_W , 0 , 12 ), // #530 - INST(Frintx_v , FSimdVV , (0b0001111000100111010000, kHF_A, 0b0010111000100001100110, kHF_B) , kRWI_W , 0 , 13 ), // #531 - INST(Frintz_v , FSimdVV , (0b0001111000100101110000, kHF_A, 0b0000111010100001100110, kHF_B) , kRWI_W , 0 , 14 ), // #532 - INST(Frsqrte_v , FSimdVV , (0b0111111010100001110110, kHF_B, 0b0010111010100001110110, kHF_B) , kRWI_W , 0 , 15 ), // #533 - INST(Frsqrts_v , FSimdVVV , (0b0101111010100000111111, kHF_C, 0b0000111010100000111111, kHF_C) , kRWI_W , 0 , 11 ), // #534 - INST(Fsqrt_v , FSimdVV , (0b0001111000100001110000, kHF_A, 0b0010111010100001111110, kHF_B) , kRWI_W , 0 , 16 ), // #535 - INST(Fsub_v , FSimdVVV , (0b0001111000100000001110, kHF_A, 0b0000111010100000110101, kHF_C) , kRWI_W , 0 , 12 ), // #536 - INST(Ins_v , SimdIns , (_) , kRWI_X , 0 , 0 ), // #537 - INST(Ld1_v , SimdLdNStN , (0b0000110101000000000000, 0b0000110001000000001000, 1, 0) , kRWI_LDn , F(Consecutive) , 0 ), // #538 - INST(Ld1r_v , SimdLdNStN , (0b0000110101000000110000, 0b0000000000000000000000, 1, 1) , kRWI_LDn , F(Consecutive) , 1 ), // #539 - INST(Ld2_v , SimdLdNStN , (0b0000110101100000000000, 0b0000110001000000100000, 2, 0) , kRWI_LDn , F(Consecutive) , 2 ), // #540 - INST(Ld2r_v , SimdLdNStN , (0b0000110101100000110000, 0b0000000000000000000000, 2, 1) , kRWI_LDn , F(Consecutive) , 3 ), // #541 - INST(Ld3_v , SimdLdNStN , (0b0000110101000000001000, 0b0000110001000000010000, 3, 0) , kRWI_LDn , F(Consecutive) , 4 ), // #542 - INST(Ld3r_v , SimdLdNStN , (0b0000110101000000111000, 0b0000000000000000000000, 3, 1) , kRWI_LDn , F(Consecutive) , 5 ), // #543 - INST(Ld4_v , SimdLdNStN , (0b0000110101100000001000, 0b0000110001000000000000, 4, 0) , kRWI_LDn , F(Consecutive) , 6 ), // #544 - INST(Ld4r_v , SimdLdNStN , (0b0000110101100000111000, 0b0000000000000000000000, 4, 1) , kRWI_LDn , F(Consecutive) , 7 ), // #545 - INST(Ldnp_v , SimdLdpStp , (0b0010110001, 0b0000000000) , kRWI_WW , 0 , 0 ), // #546 - INST(Ldp_v , SimdLdpStp , (0b0010110101, 0b0010110011) , kRWI_WW , 0 , 1 ), // #547 - INST(Ldr_v , SimdLdSt , (0b0011110101, 0b00111100010, 0b00111100011, 0b00011100, Inst::kIdLdur_v) , kRWI_W , 0 , 0 ), // #548 - INST(Ldur_v , SimdLdurStur , (0b0011110001000000000000) , kRWI_W , 0 , 0 ), // #549 - INST(Mla_v , ISimdVVVe , (0b0000111000100000100101, kVO_V_BHS, 0b0010111100000000000000, kVO_V_HS) , kRWI_X , F(VH0_15) , 0 ), // #550 - INST(Mls_v , ISimdVVVe , (0b0010111000100000100101, kVO_V_BHS, 0b0010111100000000010000, kVO_V_HS) , kRWI_X , F(VH0_15) , 1 ), // #551 - INST(Mov_v , SimdMov , (_) , kRWI_W , 0 , 0 ), // #552 - INST(Movi_v , SimdMoviMvni , (0b0000111100000000000001, 0) , kRWI_W , 0 , 0 ), // #553 - INST(Mul_v , ISimdVVVe , (0b0000111000100000100111, kVO_V_BHS, 0b0000111100000000100000, kVO_V_HS) , kRWI_W , F(VH0_15) , 2 ), // #554 - INST(Mvn_v , ISimdVV , (0b0010111000100000010110, kVO_V_B) , kRWI_W , 0 , 4 ), // #555 - INST(Mvni_v , SimdMoviMvni , (0b0000111100000000000001, 1) , kRWI_W , 0 , 1 ), // #556 - INST(Neg_v , ISimdVV , (0b0010111000100000101110, kVO_V_Any) , kRWI_W , 0 , 5 ), // #557 - INST(Not_v , ISimdVV , (0b0010111000100000010110, kVO_V_B) , kRWI_W , 0 , 6 ), // #558 - INST(Orn_v , ISimdVVV , (0b0000111011100000000111, kVO_V_B) , kRWI_W , 0 , 9 ), // #559 - INST(Orr_v , SimdBicOrr , (0b0000111010100000000111, 0b0000111100000000000001) , kRWI_W , 0 , 1 ), // #560 - INST(Pmul_v , ISimdVVV , (0b0010111000100000100111, kVO_V_B) , kRWI_W , 0 , 10 ), // #561 - INST(Pmull_v , ISimdVVV , (0b0000111000100000111000, kVO_V_B8D1) , kRWI_W , F(Long) , 11 ), // #562 - INST(Pmull2_v , ISimdVVV , (0b0100111000100000111000, kVO_V_B16D2) , kRWI_W , F(Long) , 12 ), // #563 - INST(Raddhn_v , ISimdVVV , (0b0010111000100000010000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 13 ), // #564 - INST(Raddhn2_v , ISimdVVV , (0b0110111000100000010000, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 14 ), // #565 - INST(Rax1_v , ISimdVVV , (0b1100111001100000100011, kVO_V_D2) , kRWI_W , 0 , 15 ), // #566 - INST(Rbit_v , ISimdVV , (0b0010111001100000010110, kVO_V_B) , kRWI_W , 0 , 7 ), // #567 - INST(Rev16_v , ISimdVV , (0b0000111000100000000110, kVO_V_B) , kRWI_W , 0 , 8 ), // #568 - INST(Rev32_v , ISimdVV , (0b0010111000100000000010, kVO_V_BH) , kRWI_W , 0 , 9 ), // #569 - INST(Rev64_v , ISimdVV , (0b0000111000100000000010, kVO_V_BHS) , kRWI_W , 0 , 10 ), // #570 - INST(Rshrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100011, 1, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 0 ), // #571 - INST(Rshrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100011, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 1 ), // #572 - INST(Rsubhn_v , ISimdVVV , (0b0010111000100000011000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 16 ), // #573 - INST(Rsubhn2_v , ISimdVVV , (0b0110111000100000011000, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 17 ), // #574 - INST(Saba_v , ISimdVVV , (0b0000111000100000011111, kVO_V_BHS) , kRWI_X , 0 , 18 ), // #575 - INST(Sabal_v , ISimdVVV , (0b0000111000100000010100, kVO_V_B8H4S2) , kRWI_X , F(Long) , 19 ), // #576 - INST(Sabal2_v , ISimdVVV , (0b0100111000100000010100, kVO_V_B16H8S4) , kRWI_X , F(Long) , 20 ), // #577 - INST(Sabd_v , ISimdVVV , (0b0000111000100000011101, kVO_V_BHS) , kRWI_W , 0 , 21 ), // #578 - INST(Sabdl_v , ISimdVVV , (0b0000111000100000011100, kVO_V_B8H4S2) , kRWI_W , F(Long) , 22 ), // #579 - INST(Sabdl2_v , ISimdVVV , (0b0100111000100000011100, kVO_V_B16H8S4) , kRWI_W , F(Long) , 23 ), // #580 - INST(Sadalp_v , ISimdVV , (0b0000111000100000011010, kVO_V_BHS) , kRWI_X , F(Long) | F(Pair) , 11 ), // #581 - INST(Saddl_v , ISimdVVV , (0b0000111000100000000000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 24 ), // #582 - INST(Saddl2_v , ISimdVVV , (0b0100111000100000000000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 25 ), // #583 - INST(Saddlp_v , ISimdVV , (0b0000111000100000001010, kVO_V_BHS) , kRWI_W , F(Long) | F(Pair) , 12 ), // #584 - INST(Saddlv_v , ISimdSV , (0b0000111000110000001110, kVO_V_BH_4S) , kRWI_W , F(Long) , 1 ), // #585 - INST(Saddw_v , ISimdWWV , (0b0000111000100000000100, kVO_V_B8H4S2) , kRWI_W , 0 , 0 ), // #586 - INST(Saddw2_v , ISimdWWV , (0b0000111000100000000100, kVO_V_B16H8S4) , kRWI_W , 0 , 1 ), // #587 - INST(Scvtf_v , SimdFcvtSV , (0b0000111000100001110110, 0b0000111100000000111001, 0b0001111000100010000000, 0) , kRWI_W , 0 , 10 ), // #588 - INST(Sdot_v , SimdDot , (0b0000111010000000100101, 0b0000111110000000111000, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 1 ), // #589 - INST(Sha1c_v , ISimdVVVx , (0b0101111000000000000000, kOp_Q, kOp_S, kOp_V4S) , kRWI_X , 0 , 1 ), // #590 - INST(Sha1h_v , ISimdVVx , (0b0101111000101000000010, kOp_S, kOp_S) , kRWI_W , 0 , 8 ), // #591 - INST(Sha1m_v , ISimdVVVx , (0b0101111000000000001000, kOp_Q, kOp_S, kOp_V4S) , kRWI_X , 0 , 2 ), // #592 - INST(Sha1p_v , ISimdVVVx , (0b0101111000000000000100, kOp_Q, kOp_S, kOp_V4S) , kRWI_X , 0 , 3 ), // #593 - INST(Sha1su0_v , ISimdVVVx , (0b0101111000000000001100, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 4 ), // #594 - INST(Sha1su1_v , ISimdVVx , (0b0101111000101000000110, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 9 ), // #595 - INST(Sha256h_v , ISimdVVVx , (0b0101111000000000010000, kOp_Q, kOp_Q, kOp_V4S) , kRWI_X , 0 , 5 ), // #596 - INST(Sha256h2_v , ISimdVVVx , (0b0101111000000000010100, kOp_Q, kOp_Q, kOp_V4S) , kRWI_X , 0 , 6 ), // #597 - INST(Sha256su0_v , ISimdVVx , (0b0101111000101000001010, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 10 ), // #598 - INST(Sha256su1_v , ISimdVVVx , (0b0101111000000000011000, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 7 ), // #599 - INST(Sha512h_v , ISimdVVVx , (0b1100111001100000100000, kOp_Q, kOp_Q, kOp_V2D) , kRWI_X , 0 , 8 ), // #600 - INST(Sha512h2_v , ISimdVVVx , (0b1100111001100000100001, kOp_Q, kOp_Q, kOp_V2D) , kRWI_X , 0 , 9 ), // #601 - INST(Sha512su0_v , ISimdVVx , (0b1100111011000000100000, kOp_V2D, kOp_V2D) , kRWI_X , 0 , 11 ), // #602 - INST(Sha512su1_v , ISimdVVVx , (0b1100111001100000100010, kOp_V2D, kOp_V2D, kOp_V2D) , kRWI_X , 0 , 10 ), // #603 - INST(Shadd_v , ISimdVVV , (0b0000111000100000000001, kVO_V_BHS) , kRWI_W , 0 , 26 ), // #604 - INST(Shl_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000010101, 0, kVO_V_Any) , kRWI_W , 0 , 2 ), // #605 - INST(Shll_v , SimdShiftES , (0b0010111000100001001110, kVO_V_B8H4S2) , kRWI_W , F(Long) , 0 ), // #606 - INST(Shll2_v , SimdShiftES , (0b0110111000100001001110, kVO_V_B16H8S4) , kRWI_W , F(Long) , 1 ), // #607 - INST(Shrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100001, 1, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 3 ), // #608 - INST(Shrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100001, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 4 ), // #609 - INST(Shsub_v , ISimdVVV , (0b0000111000100000001001, kVO_V_BHS) , kRWI_W , 0 , 27 ), // #610 - INST(Sli_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000010101, 0, kVO_V_Any) , kRWI_X , 0 , 5 ), // #611 - INST(Sm3partw1_v , ISimdVVVx , (0b1100111001100000110000, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 11 ), // #612 - INST(Sm3partw2_v , ISimdVVVx , (0b1100111001100000110001, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 12 ), // #613 - INST(Sm3ss1_v , ISimdVVVVx , (0b1100111001000000000000, kOp_V4S, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_W , 0 , 0 ), // #614 - INST(Sm3tt1a_v , SimdSm3tt , (0b1100111001000000100000) , kRWI_X , 0 , 0 ), // #615 - INST(Sm3tt1b_v , SimdSm3tt , (0b1100111001000000100001) , kRWI_X , 0 , 1 ), // #616 - INST(Sm3tt2a_v , SimdSm3tt , (0b1100111001000000100010) , kRWI_X , 0 , 2 ), // #617 - INST(Sm3tt2b_v , SimdSm3tt , (0b1100111001000000100011) , kRWI_X , 0 , 3 ), // #618 - INST(Sm4e_v , ISimdVVx , (0b1100111011000000100001, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 12 ), // #619 - INST(Sm4ekey_v , ISimdVVVx , (0b1100111001100000110010, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 13 ), // #620 - INST(Smax_v , ISimdVVV , (0b0000111000100000011001, kVO_V_BHS) , kRWI_W , 0 , 28 ), // #621 - INST(Smaxp_v , ISimdVVV , (0b0000111000100000101001, kVO_V_BHS) , kRWI_W , 0 , 29 ), // #622 - INST(Smaxv_v , ISimdSV , (0b0000111000110000101010, kVO_V_BH_4S) , kRWI_W , 0 , 2 ), // #623 - INST(Smin_v , ISimdVVV , (0b0000111000100000011011, kVO_V_BHS) , kRWI_W , 0 , 30 ), // #624 - INST(Sminp_v , ISimdVVV , (0b0000111000100000101011, kVO_V_BHS) , kRWI_W , 0 , 31 ), // #625 - INST(Sminv_v , ISimdSV , (0b0000111000110001101010, kVO_V_BH_4S) , kRWI_W , 0 , 3 ), // #626 - INST(Smlal_v , ISimdVVVe , (0b0000111000100000100000, kVO_V_B8H4S2, 0b0000111100000000001000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 3 ), // #627 - INST(Smlal2_v , ISimdVVVe , (0b0100111000100000100000, kVO_V_B16H8S4, 0b0100111100000000001000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 4 ), // #628 - INST(Smlsl_v , ISimdVVVe , (0b0000111000100000101000, kVO_V_B8H4S2, 0b0000111100000000011000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 5 ), // #629 - INST(Smlsl2_v , ISimdVVVe , (0b0100111000100000101000, kVO_V_B16H8S4, 0b0100111100000000011000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 6 ), // #630 - INST(Smmla_v , ISimdVVVx , (0b0100111010000000101001, kOp_V4S, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 14 ), // #631 - INST(Smov_v , SimdSmovUmov , (0b0000111000000000001011, kVO_V_BHS, 1) , kRWI_W , 0 , 0 ), // #632 - INST(Smull_v , ISimdVVVe , (0b0000111000100000110000, kVO_V_B8H4S2, 0b0000111100000000101000, kVO_V_H4S2) , kRWI_W , F(Long) | F(VH0_15) , 7 ), // #633 - INST(Smull2_v , ISimdVVVe , (0b0100111000100000110000, kVO_V_B16H8S4, 0b0100111100000000101000, kVO_V_H8S4) , kRWI_W , F(Long) | F(VH0_15) , 8 ), // #634 - INST(Sqabs_v , ISimdVV , (0b0000111000100000011110, kVO_SV_Any) , kRWI_W , 0 , 13 ), // #635 - INST(Sqadd_v , ISimdVVV , (0b0000111000100000000011, kVO_SV_Any) , kRWI_W , 0 , 32 ), // #636 - INST(Sqdmlal_v , ISimdVVVe , (0b0000111000100000100100, kVO_SV_BHS, 0b0000111100000000001100, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 9 ), // #637 - INST(Sqdmlal2_v , ISimdVVVe , (0b0100111000100000100100, kVO_V_B16H8S4, 0b0100111100000000001100, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 10 ), // #638 - INST(Sqdmlsl_v , ISimdVVVe , (0b0000111000100000101100, kVO_SV_BHS, 0b0000111100000000011100, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 11 ), // #639 - INST(Sqdmlsl2_v , ISimdVVVe , (0b0100111000100000101100, kVO_V_B16H8S4, 0b0100111100000000011100, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 12 ), // #640 - INST(Sqdmulh_v , ISimdVVVe , (0b0000111000100000101101, kVO_SV_HS, 0b0000111100000000110000, kVO_SV_HS) , kRWI_W , F(VH0_15) , 13 ), // #641 - INST(Sqdmull_v , ISimdVVVe , (0b0000111000100000110100, kVO_SV_BHS, 0b0000111100000000101100, kVO_V_H4S2) , kRWI_W , F(Long) | F(VH0_15) , 14 ), // #642 - INST(Sqdmull2_v , ISimdVVVe , (0b0100111000100000110100, kVO_V_B16H8S4, 0b0100111100000000101100, kVO_V_H8S4) , kRWI_W , F(Long) | F(VH0_15) , 15 ), // #643 - INST(Sqneg_v , ISimdVV , (0b0010111000100000011110, kVO_SV_Any) , kRWI_W , 0 , 14 ), // #644 - INST(Sqrdmlah_v , ISimdVVVe , (0b0010111000000000100001, kVO_SV_HS, 0b0010111100000000110100, kVO_SV_HS) , kRWI_X , F(VH0_15) , 16 ), // #645 - INST(Sqrdmlsh_v , ISimdVVVe , (0b0010111000000000100011, kVO_SV_HS, 0b0010111100000000111100, kVO_SV_HS) , kRWI_X , F(VH0_15) , 17 ), // #646 - INST(Sqrdmulh_v , ISimdVVVe , (0b0010111000100000101101, kVO_SV_HS, 0b0000111100000000110100, kVO_SV_HS) , kRWI_W , F(VH0_15) , 18 ), // #647 - INST(Sqrshl_v , SimdShift , (0b0000111000100000010111, 0b0000000000000000000000, 1, kVO_SV_Any) , kRWI_W , 0 , 6 ), // #648 - INST(Sqrshrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100111, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 7 ), // #649 - INST(Sqrshrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100111, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 8 ), // #650 - INST(Sqrshrun_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100011, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 9 ), // #651 - INST(Sqrshrun2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100011, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 10 ), // #652 - INST(Sqshl_v , SimdShift , (0b0000111000100000010011, 0b0000111100000000011101, 0, kVO_SV_Any) , kRWI_W , 0 , 11 ), // #653 - INST(Sqshlu_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000011001, 0, kVO_SV_Any) , kRWI_W , 0 , 12 ), // #654 - INST(Sqshrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100101, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 13 ), // #655 - INST(Sqshrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100101, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 14 ), // #656 - INST(Sqshrun_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100001, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 15 ), // #657 - INST(Sqshrun2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100001, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 16 ), // #658 - INST(Sqsub_v , ISimdVVV , (0b0000111000100000001011, kVO_SV_Any) , kRWI_W , 0 , 33 ), // #659 - INST(Sqxtn_v , ISimdVV , (0b0000111000100001010010, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 15 ), // #660 - INST(Sqxtn2_v , ISimdVV , (0b0100111000100001010010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 16 ), // #661 - INST(Sqxtun_v , ISimdVV , (0b0010111000100001001010, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 17 ), // #662 - INST(Sqxtun2_v , ISimdVV , (0b0110111000100001001010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 18 ), // #663 - INST(Srhadd_v , ISimdVVV , (0b0000111000100000000101, kVO_V_BHS) , kRWI_W , 0 , 34 ), // #664 - INST(Sri_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000010001, 1, kVO_V_Any) , kRWI_W , 0 , 17 ), // #665 - INST(Srshl_v , SimdShift , (0b0000111000100000010101, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 18 ), // #666 - INST(Srshr_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000001001, 1, kVO_V_Any) , kRWI_W , 0 , 19 ), // #667 - INST(Srsra_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000001101, 1, kVO_V_Any) , kRWI_X , 0 , 20 ), // #668 - INST(Sshl_v , SimdShift , (0b0000111000100000010001, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 21 ), // #669 - INST(Sshll_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000101001, 0, kVO_V_B8H4S2) , kRWI_W , F(Long) , 22 ), // #670 - INST(Sshll2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000101001, 0, kVO_V_B16H8S4) , kRWI_W , F(Long) , 23 ), // #671 - INST(Sshr_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000000001, 1, kVO_V_Any) , kRWI_W , 0 , 24 ), // #672 - INST(Ssra_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000000101, 1, kVO_V_Any) , kRWI_X , 0 , 25 ), // #673 - INST(Ssubl_v , ISimdVVV , (0b0000111000100000001000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 35 ), // #674 - INST(Ssubl2_v , ISimdVVV , (0b0100111000100000001000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 36 ), // #675 - INST(Ssubw_v , ISimdWWV , (0b0000111000100000001100, kVO_V_B8H4S2) , kRWI_W , 0 , 2 ), // #676 - INST(Ssubw2_v , ISimdWWV , (0b0000111000100000001100, kVO_V_B16H8S4) , kRWI_X , 0 , 3 ), // #677 - INST(St1_v , SimdLdNStN , (0b0000110100000000000000, 0b0000110000000000001000, 1, 0) , kRWI_STn , F(Consecutive) , 8 ), // #678 - INST(St2_v , SimdLdNStN , (0b0000110100100000000000, 0b0000110000000000100000, 2, 0) , kRWI_STn , F(Consecutive) , 9 ), // #679 - INST(St3_v , SimdLdNStN , (0b0000110100000000001000, 0b0000110000000000010000, 3, 0) , kRWI_STn , F(Consecutive) , 10 ), // #680 - INST(St4_v , SimdLdNStN , (0b0000110100100000001000, 0b0000110000000000000000, 4, 0) , kRWI_STn , F(Consecutive) , 11 ), // #681 - INST(Stnp_v , SimdLdpStp , (0b0010110000, 0b0000000000) , kRWI_RRW , 0 , 2 ), // #682 - INST(Stp_v , SimdLdpStp , (0b0010110100, 0b0010110010) , kRWI_RRW , 0 , 3 ), // #683 - INST(Str_v , SimdLdSt , (0b0011110100, 0b00111100000, 0b00111100001, 0b00000000, Inst::kIdStur_v) , kRWI_RW , 0 , 1 ), // #684 - INST(Stur_v , SimdLdurStur , (0b0011110000000000000000) , kRWI_RW , 0 , 1 ), // #685 - INST(Sub_v , ISimdVVV , (0b0010111000100000100001, kVO_V_Any) , kRWI_W , 0 , 37 ), // #686 - INST(Subhn_v , ISimdVVV , (0b0000111000100000011000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 38 ), // #687 - INST(Subhn2_v , ISimdVVV , (0b0000111000100000011000, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 39 ), // #688 - INST(Sudot_v , SimdDot , (0b0000000000000000000000, 0b0000111100000000111100, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 2 ), // #689 - INST(Suqadd_v , ISimdVV , (0b0000111000100000001110, kVO_SV_Any) , kRWI_X , 0 , 19 ), // #690 - INST(Sxtl_v , SimdSxtlUxtl , (0b0000111100000000101001, kVO_V_B8H4S2) , kRWI_W , F(Long) , 0 ), // #691 - INST(Sxtl2_v , SimdSxtlUxtl , (0b0100111100000000101001, kVO_V_B16H8S4) , kRWI_W , F(Long) , 1 ), // #692 - INST(Tbl_v , SimdTblTbx , (0b0000111000000000000000) , kRWI_W , 0 , 0 ), // #693 - INST(Tbx_v , SimdTblTbx , (0b0000111000000000000100) , kRWI_W , 0 , 1 ), // #694 - INST(Trn1_v , ISimdVVV , (0b0000111000000000001010, kVO_V_BHS_D2) , kRWI_W , 0 , 40 ), // #695 - INST(Trn2_v , ISimdVVV , (0b0000111000000000011010, kVO_V_BHS_D2) , kRWI_W , 0 , 41 ), // #696 - INST(Uaba_v , ISimdVVV , (0b0010111000100000011111, kVO_V_BHS) , kRWI_X , 0 , 42 ), // #697 - INST(Uabal_v , ISimdVVV , (0b0010111000100000010100, kVO_V_B8H4S2) , kRWI_X , F(Long) , 43 ), // #698 - INST(Uabal2_v , ISimdVVV , (0b0110111000100000010100, kVO_V_B16H8S4) , kRWI_X , F(Long) , 44 ), // #699 - INST(Uabd_v , ISimdVVV , (0b0010111000100000011101, kVO_V_BHS) , kRWI_W , 0 , 45 ), // #700 - INST(Uabdl_v , ISimdVVV , (0b0010111000100000011100, kVO_V_B8H4S2) , kRWI_W , F(Long) , 46 ), // #701 - INST(Uabdl2_v , ISimdVVV , (0b0110111000100000011100, kVO_V_B16H8S4) , kRWI_W , F(Long) , 47 ), // #702 - INST(Uadalp_v , ISimdVV , (0b0010111000100000011010, kVO_V_BHS) , kRWI_X , F(Long) | F(Pair) , 20 ), // #703 - INST(Uaddl_v , ISimdVVV , (0b0010111000100000000000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 48 ), // #704 - INST(Uaddl2_v , ISimdVVV , (0b0110111000100000000000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 49 ), // #705 - INST(Uaddlp_v , ISimdVV , (0b0010111000100000001010, kVO_V_BHS) , kRWI_W , F(Long) | F(Pair) , 21 ), // #706 - INST(Uaddlv_v , ISimdSV , (0b0010111000110000001110, kVO_V_BH_4S) , kRWI_W , F(Long) , 4 ), // #707 - INST(Uaddw_v , ISimdWWV , (0b0010111000100000000100, kVO_V_B8H4S2) , kRWI_W , 0 , 4 ), // #708 - INST(Uaddw2_v , ISimdWWV , (0b0010111000100000000100, kVO_V_B16H8S4) , kRWI_W , 0 , 5 ), // #709 - INST(Ucvtf_v , SimdFcvtSV , (0b0010111000100001110110, 0b0010111100000000111001, 0b0001111000100011000000, 0) , kRWI_W , 0 , 11 ), // #710 - INST(Udot_v , SimdDot , (0b0010111010000000100101, 0b0010111110000000111000, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 3 ), // #711 - INST(Uhadd_v , ISimdVVV , (0b0010111000100000000001, kVO_V_BHS) , kRWI_W , 0 , 50 ), // #712 - INST(Uhsub_v , ISimdVVV , (0b0010111000100000001001, kVO_V_BHS) , kRWI_W , 0 , 51 ), // #713 - INST(Umax_v , ISimdVVV , (0b0010111000100000011001, kVO_V_BHS) , kRWI_W , 0 , 52 ), // #714 - INST(Umaxp_v , ISimdVVV , (0b0010111000100000101001, kVO_V_BHS) , kRWI_W , 0 , 53 ), // #715 - INST(Umaxv_v , ISimdSV , (0b0010111000110000101010, kVO_V_BH_4S) , kRWI_W , 0 , 5 ), // #716 - INST(Umin_v , ISimdVVV , (0b0010111000100000011011, kVO_V_BHS) , kRWI_W , 0 , 54 ), // #717 - INST(Uminp_v , ISimdVVV , (0b0010111000100000101011, kVO_V_BHS) , kRWI_W , 0 , 55 ), // #718 - INST(Uminv_v , ISimdSV , (0b0010111000110001101010, kVO_V_BH_4S) , kRWI_W , 0 , 6 ), // #719 - INST(Umlal_v , ISimdVVVe , (0b0010111000100000100000, kVO_V_B8H4S2, 0b0010111100000000001000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 19 ), // #720 - INST(Umlal2_v , ISimdVVVe , (0b0110111000100000100000, kVO_V_B16H8S4, 0b0010111100000000001000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 20 ), // #721 - INST(Umlsl_v , ISimdVVVe , (0b0010111000100000101000, kVO_V_B8H4S2, 0b0010111100000000011000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 21 ), // #722 - INST(Umlsl2_v , ISimdVVVe , (0b0110111000100000101000, kVO_V_B16H8S4, 0b0110111100000000011000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 22 ), // #723 - INST(Ummla_v , ISimdVVVx , (0b0110111010000000101001, kOp_V4S, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 15 ), // #724 - INST(Umov_v , SimdSmovUmov , (0b0000111000000000001111, kVO_V_Any, 0) , kRWI_W , 0 , 1 ), // #725 - INST(Umull_v , ISimdVVVe , (0b0010111000100000110000, kVO_V_B8H4S2, 0b0010111100000000101000, kVO_V_H4S2) , kRWI_W , F(Long) | F(VH0_15) , 23 ), // #726 - INST(Umull2_v , ISimdVVVe , (0b0110111000100000110000, kVO_V_B16H8S4, 0b0110111100000000101000, kVO_V_H8S4) , kRWI_W , F(Long) | F(VH0_15) , 24 ), // #727 - INST(Uqadd_v , ISimdVVV , (0b0010111000100000000011, kVO_SV_Any) , kRWI_W , 0 , 56 ), // #728 - INST(Uqrshl_v , SimdShift , (0b0010111000100000010111, 0b0000000000000000000000, 0, kVO_SV_Any) , kRWI_W , 0 , 26 ), // #729 - INST(Uqrshrn_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100111, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 27 ), // #730 - INST(Uqrshrn2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100111, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 28 ), // #731 - INST(Uqshl_v , SimdShift , (0b0010111000100000010011, 0b0010111100000000011101, 0, kVO_SV_Any) , kRWI_W , 0 , 29 ), // #732 - INST(Uqshrn_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100101, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 30 ), // #733 - INST(Uqshrn2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100101, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 31 ), // #734 - INST(Uqsub_v , ISimdVVV , (0b0010111000100000001011, kVO_SV_Any) , kRWI_W , 0 , 57 ), // #735 - INST(Uqxtn_v , ISimdVV , (0b0010111000100001010010, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 22 ), // #736 - INST(Uqxtn2_v , ISimdVV , (0b0110111000100001010010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 23 ), // #737 - INST(Urecpe_v , ISimdVV , (0b0000111010100001110010, kVO_V_S) , kRWI_W , 0 , 24 ), // #738 - INST(Urhadd_v , ISimdVVV , (0b0010111000100000000101, kVO_V_BHS) , kRWI_W , 0 , 58 ), // #739 - INST(Urshl_v , SimdShift , (0b0010111000100000010101, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 32 ), // #740 - INST(Urshr_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000001001, 1, kVO_V_Any) , kRWI_W , 0 , 33 ), // #741 - INST(Ursqrte_v , ISimdVV , (0b0010111010100001110010, kVO_V_S) , kRWI_W , 0 , 25 ), // #742 - INST(Ursra_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000001101, 1, kVO_V_Any) , kRWI_X , 0 , 34 ), // #743 - INST(Usdot_v , SimdDot , (0b0000111010000000100111, 0b0000111110000000111100, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 4 ), // #744 - INST(Ushl_v , SimdShift , (0b0010111000100000010001, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 35 ), // #745 - INST(Ushll_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000101001, 0, kVO_V_B8H4S2) , kRWI_W , F(Long) , 36 ), // #746 - INST(Ushll2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000101001, 0, kVO_V_B16H8S4) , kRWI_W , F(Long) , 37 ), // #747 - INST(Ushr_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000000001, 1, kVO_V_Any) , kRWI_W , 0 , 38 ), // #748 - INST(Usmmla_v , ISimdVVVx , (0b0100111010000000101011, kOp_V4S, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 16 ), // #749 - INST(Usqadd_v , ISimdVV , (0b0010111000100000001110, kVO_SV_Any) , kRWI_X , 0 , 26 ), // #750 - INST(Usra_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000000101, 1, kVO_V_Any) , kRWI_X , 0 , 39 ), // #751 - INST(Usubl_v , ISimdVVV , (0b0010111000100000001000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 59 ), // #752 - INST(Usubl2_v , ISimdVVV , (0b0110111000100000001000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 60 ), // #753 - INST(Usubw_v , ISimdWWV , (0b0010111000100000001100, kVO_V_B8H4S2) , kRWI_W , 0 , 6 ), // #754 - INST(Usubw2_v , ISimdWWV , (0b0010111000100000001100, kVO_V_B16H8S4) , kRWI_W , 0 , 7 ), // #755 - INST(Uxtl_v , SimdSxtlUxtl , (0b0010111100000000101001, kVO_V_B8H4S2) , kRWI_W , F(Long) , 2 ), // #756 - INST(Uxtl2_v , SimdSxtlUxtl , (0b0110111100000000101001, kVO_V_B16H8S4) , kRWI_W , F(Long) , 3 ), // #757 - INST(Uzp1_v , ISimdVVV , (0b0000111000000000000110, kVO_V_BHS_D2) , kRWI_W , 0 , 61 ), // #758 - INST(Uzp2_v , ISimdVVV , (0b0000111000000000010110, kVO_V_BHS_D2) , kRWI_W , 0 , 62 ), // #759 - INST(Xar_v , ISimdVVVI , (0b1100111001100000100011, kVO_V_D2, 6, 10, 0) , kRWI_W , 0 , 1 ), // #760 - INST(Xtn_v , ISimdVV , (0b0000111000100001001010, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 27 ), // #761 - INST(Xtn2_v , ISimdVV , (0b0100111000100001001010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 28 ), // #762 - INST(Zip1_v , ISimdVVV , (0b0000111000000000001110, kVO_V_BHS_D2) , kRWI_W , 0 , 63 ), // #763 - INST(Zip2_v , ISimdVVV , (0b0000111000000000011110, kVO_V_BHS_D2) , kRWI_W , 0 , 64 ) // #764 + INST(Abs , BaseRR , (0b01011010110000000010000000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 0 ), // #1 + INST(Adc , BaseRRR , (0b0001101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 0 ), // #2 + INST(Adcs , BaseRRR , (0b0011101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 1 ), // #3 + INST(Add , BaseAddSub , (0b0001011000, 0b0001011001, 0b0010001) , kRWI_W , 0 , 0 ), // #4 + INST(Addg , BaseRRII , (0b1001000110000000000000, kX, kSP, kX, kSP, 6, 4, 16, 4, 0, 10) , kRWI_W , 0 , 0 ), // #5 + INST(Adds , BaseAddSub , (0b0101011000, 0b0101011001, 0b0110001) , kRWI_W , 0 , 1 ), // #6 + INST(Adr , BaseAdr , (0b0001000000000000000000, OffsetType::kAArch64_ADR) , kRWI_W , 0 , 0 ), // #7 + INST(Adrp , BaseAdr , (0b1001000000000000000000, OffsetType::kAArch64_ADRP) , kRWI_W , 0 , 1 ), // #8 + INST(And , BaseLogical , (0b0001010000, 0b00100100, 0) , kRWI_W , 0 , 0 ), // #9 + INST(Ands , BaseLogical , (0b1101010000, 0b11100100, 0) , kRWI_W , 0 , 1 ), // #10 + INST(Asr , BaseShift , (0b0001101011000000001010, 0b0001001100000000011111, 0) , kRWI_W , 0 , 0 ), // #11 + INST(Asrv , BaseShift , (0b0001101011000000001010, 0b0000000000000000000000, 0) , kRWI_W , 0 , 1 ), // #12 + INST(At , BaseAtDcIcTlbi , (0b00011111110000, 0b00001111000000, true) , kRWI_RX , 0 , 0 ), // #13 + INST(Autda , BaseRR , (0b11011010110000010001100000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 1 ), // #14 + INST(Autdza , BaseR , (0b11011010110000010011101111100000, kX, kZR, 0) , kRWI_X , 0 , 0 ), // #15 + INST(Autdb , BaseRR , (0b11011010110000010001110000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 2 ), // #16 + INST(Autdzb , BaseR , (0b11011010110000010011111111100000, kX, kZR, 0) , kRWI_X , 0 , 1 ), // #17 + INST(Autia , BaseRR , (0b11011010110000010001000000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 3 ), // #18 + INST(Autia1716 , BaseOp , (0b11010101000000110010000110011111) , 0 , 0 , 0 ), // #19 + INST(Autiasp , BaseOp , (0b11010101000000110010001110111111) , 0 , 0 , 1 ), // #20 + INST(Autiaz , BaseOp , (0b11010101000000110010001110011111) , 0 , 0 , 2 ), // #21 + INST(Autib , BaseRR , (0b11011010110000010001010000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 4 ), // #22 + INST(Autib1716 , BaseOp , (0b11010101000000110010000111011111) , 0 , 0 , 3 ), // #23 + INST(Autibsp , BaseOp , (0b11010101000000110010001111111111) , 0 , 0 , 4 ), // #24 + INST(Autibz , BaseOp , (0b11010101000000110010001111011111) , 0 , 0 , 5 ), // #25 + INST(Autiza , BaseR , (0b11011010110000010011001111100000, kX, kZR, 0) , kRWI_X , 0 , 2 ), // #26 + INST(Autizb , BaseR , (0b11011010110000010011011111100000, kX, kZR, 0) , kRWI_X , 0 , 3 ), // #27 + INST(Axflag , BaseOp , (0b11010101000000000100000001011111) , 0 , 0 , 6 ), // #28 + INST(B , BaseBranchRel , (0b00010100000000000000000000000000) , 0 , F(Cond) , 0 ), // #29 + INST(Bc , BaseBranchRel , (0b00010100000000000000000000010000) , 0 , F(Cond) , 1 ), // #30 + INST(Bfc , BaseBfc , (0b00110011000000000000001111100000) , kRWI_X , 0 , 0 ), // #31 + INST(Bfi , BaseBfi , (0b00110011000000000000000000000000) , kRWI_X , 0 , 0 ), // #32 + INST(Bfm , BaseBfm , (0b00110011000000000000000000000000) , kRWI_X , 0 , 0 ), // #33 + INST(Bfxil , BaseBfx , (0b00110011000000000000000000000000) , kRWI_X , 0 , 0 ), // #34 + INST(Bic , BaseLogical , (0b0001010001, 0b00100100, 1) , kRWI_W , 0 , 2 ), // #35 + INST(Bics , BaseLogical , (0b1101010001, 0b11100100, 1) , kRWI_W , 0 , 3 ), // #36 + INST(Bl , BaseBranchRel , (0b10010100000000000000000000000000) , 0 , 0 , 2 ), // #37 + INST(Blr , BaseBranchReg , (0b11010110001111110000000000000000) , kRWI_R , 0 , 0 ), // #38 + INST(Br , BaseBranchReg , (0b11010110000111110000000000000000) , kRWI_R , 0 , 1 ), // #39 + INST(Brk , BaseOpImm , (0b11010100001000000000000000000000, 16, 5) , 0 , 0 , 0 ), // #40 + INST(Bti , BaseOpImm , (0b11010101000000110010010000011111, 2, 6) , 0 , 0 , 1 ), // #41 + INST(Cas , BaseAtomicOp , (0b1000100010100000011111, kWX, 30, 0) , kRWI_XRX , 0 , 0 ), // #42 + INST(Casa , BaseAtomicOp , (0b1000100011100000011111, kWX, 30, 1) , kRWI_XRX , 0 , 1 ), // #43 + INST(Casab , BaseAtomicOp , (0b0000100011100000011111, kW , 0 , 1) , kRWI_XRX , 0 , 2 ), // #44 + INST(Casah , BaseAtomicOp , (0b0100100011100000011111, kW , 0 , 1) , kRWI_XRX , 0 , 3 ), // #45 + INST(Casal , BaseAtomicOp , (0b1000100011100000111111, kWX, 30, 1) , kRWI_XRX , 0 , 4 ), // #46 + INST(Casalb , BaseAtomicOp , (0b0000100011100000111111, kW , 0 , 1) , kRWI_XRX , 0 , 5 ), // #47 + INST(Casalh , BaseAtomicOp , (0b0100100011100000111111, kW , 0 , 1) , kRWI_XRX , 0 , 6 ), // #48 + INST(Casb , BaseAtomicOp , (0b0000100010100000011111, kW , 0 , 0) , kRWI_XRX , 0 , 7 ), // #49 + INST(Cash , BaseAtomicOp , (0b0100100010100000011111, kW , 0 , 0) , kRWI_XRX , 0 , 8 ), // #50 + INST(Casl , BaseAtomicOp , (0b1000100010100000111111, kWX, 30, 0) , kRWI_XRX , 0 , 9 ), // #51 + INST(Caslb , BaseAtomicOp , (0b0000100010100000111111, kW , 0 , 0) , kRWI_XRX , 0 , 10 ), // #52 + INST(Caslh , BaseAtomicOp , (0b0100100010100000111111, kW , 0 , 0) , kRWI_XRX , 0 , 11 ), // #53 + INST(Casp , BaseAtomicCasp , (0b0000100000100000011111, kWX, 30) , kRWI_XXRRX, 0 , 0 ), // #54 + INST(Caspa , BaseAtomicCasp , (0b0000100001100000011111, kWX, 30) , kRWI_XXRRX, 0 , 1 ), // #55 + INST(Caspal , BaseAtomicCasp , (0b0000100001100000111111, kWX, 30) , kRWI_XXRRX, 0 , 2 ), // #56 + INST(Caspl , BaseAtomicCasp , (0b0000100000100000111111, kWX, 30) , kRWI_XXRRX, 0 , 3 ), // #57 + INST(Cbnz , BaseBranchCmp , (0b00110101000000000000000000000000) , kRWI_R , 0 , 0 ), // #58 + INST(Cbz , BaseBranchCmp , (0b00110100000000000000000000000000) , kRWI_R , 0 , 1 ), // #59 + INST(Ccmn , BaseCCmp , (0b00111010010000000000000000000000) , kRWI_R , 0 , 0 ), // #60 + INST(Ccmp , BaseCCmp , (0b01111010010000000000000000000000) , kRWI_R , 0 , 1 ), // #61 + INST(Cfinv , BaseOp , (0b11010101000000000100000000011111) , 0 , 0 , 7 ), // #62 + INST(Chkfeat , BaseOpX16 , (0b11010101000000110010010100011111) , 0 , 0 , 0 ), // #63 + INST(Cinc , BaseCInc , (0b00011010100000000000010000000000) , kRWI_W , 0 , 0 ), // #64 + INST(Cinv , BaseCInc , (0b01011010100000000000000000000000) , kRWI_W , 0 , 1 ), // #65 + INST(Clrbhb , BaseOp , (0b11010101000000110010001011011111) , 0 , 0 , 8 ), // #66 + INST(Clrex , BaseOpImm , (0b11010101000000110011000001011111, 4, 8) , 0 , 0 , 2 ), // #67 + INST(Cls , BaseRR , (0b01011010110000000001010000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 5 ), // #68 + INST(Clz , BaseRR , (0b01011010110000000001000000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 6 ), // #69 + INST(Cmn , BaseCmpCmn , (0b0101011000, 0b0101011001, 0b0110001) , kRWI_R , 0 , 0 ), // #70 + INST(Cmp , BaseCmpCmn , (0b1101011000, 0b1101011001, 0b1110001) , kRWI_R , 0 , 1 ), // #71 + INST(Cmpp , BaseRR , (0b10111010110000000000000000011111, kX, kSP, 5, kX, kSP, 16, true) , kRWI_R , 0 , 7 ), // #72 + INST(Cneg , BaseCInc , (0b01011010100000000000010000000000) , kRWI_W , 0 , 2 ), // #73 + INST(Cnt , BaseRR , (0b01011010110000000001110000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 8 ), // #74 + INST(Crc32b , BaseRRR , (0b0001101011000000010000, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 2 ), // #75 + INST(Crc32cb , BaseRRR , (0b0001101011000000010100, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 3 ), // #76 + INST(Crc32ch , BaseRRR , (0b0001101011000000010101, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 4 ), // #77 + INST(Crc32cw , BaseRRR , (0b0001101011000000010110, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 5 ), // #78 + INST(Crc32cx , BaseRRR , (0b1001101011000000010111, kW, kZR, kW, kZR, kX, kZR, false) , kRWI_W , 0 , 6 ), // #79 + INST(Crc32h , BaseRRR , (0b0001101011000000010001, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 7 ), // #80 + INST(Crc32w , BaseRRR , (0b0001101011000000010010, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 8 ), // #81 + INST(Crc32x , BaseRRR , (0b1001101011000000010011, kW, kZR, kW, kZR, kX, kZR, false) , kRWI_W , 0 , 9 ), // #82 + INST(Csdb , BaseOp , (0b11010101000000110010001010011111) , 0 , 0 , 9 ), // #83 + INST(Csel , BaseCSel , (0b00011010100000000000000000000000) , kRWI_W , 0 , 0 ), // #84 + INST(Cset , BaseCSet , (0b00011010100111110000011111100000) , kRWI_W , 0 , 0 ), // #85 + INST(Csetm , BaseCSet , (0b01011010100111110000001111100000) , kRWI_W , 0 , 1 ), // #86 + INST(Csinc , BaseCSel , (0b00011010100000000000010000000000) , kRWI_W , 0 , 1 ), // #87 + INST(Csinv , BaseCSel , (0b01011010100000000000000000000000) , kRWI_W , 0 , 2 ), // #88 + INST(Csneg , BaseCSel , (0b01011010100000000000010000000000) , kRWI_W , 0 , 3 ), // #89 + INST(Ctz , BaseRR , (0b01011010110000000001100000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 9 ), // #90 + INST(Dc , BaseAtDcIcTlbi , (0b00011110000000, 0b00001110000000, true) , kRWI_RX , 0 , 1 ), // #91 + INST(Dcps1 , BaseOpImm , (0b11010100101000000000000000000001, 16, 5) , 0 , 0 , 3 ), // #92 + INST(Dcps2 , BaseOpImm , (0b11010100101000000000000000000010, 16, 5) , 0 , 0 , 4 ), // #93 + INST(Dcps3 , BaseOpImm , (0b11010100101000000000000000000011, 16, 5) , 0 , 0 , 5 ), // #94 + INST(Dgh , BaseOp , (0b11010101000000110010000011011111) , 0 , 0 , 10 ), // #95 + INST(Dmb , BaseOpImm , (0b11010101000000110011000010111111, 4, 8) , 0 , 0 , 6 ), // #96 + INST(Drps , BaseOp , (0b11010110101111110000001111100000) , 0 , 0 , 11 ), // #97 + INST(Dsb , BaseOpImm , (0b11010101000000110011000010011111, 4, 8) , 0 , 0 , 7 ), // #98 + INST(Eon , BaseLogical , (0b1001010001, 0b10100100, 1) , kRWI_W , 0 , 4 ), // #99 + INST(Eor , BaseLogical , (0b1001010000, 0b10100100, 0) , kRWI_W , 0 , 5 ), // #100 + INST(Esb , BaseOp , (0b11010101000000110010001000011111) , 0 , 0 , 12 ), // #101 + INST(Extr , BaseExtract , (0b00010011100000000000000000000000) , kRWI_W , 0 , 0 ), // #102 + INST(Eret , BaseOp , (0b11010110100111110000001111100000) , 0 , 0 , 13 ), // #103 + INST(Gmi , BaseRRR , (0b1001101011000000000101, kX , kZR, kX , kSP, kX , kZR, true) , kRWI_W , 0 , 10 ), // #104 + INST(Hint , BaseOpImm , (0b11010101000000110010000000011111, 7, 5) , 0 , 0 , 8 ), // #105 + INST(Hlt , BaseOpImm , (0b11010100010000000000000000000000, 16, 5) , 0 , 0 , 9 ), // #106 + INST(Hvc , BaseOpImm , (0b11010100000000000000000000000010, 16, 5) , 0 , 0 , 10 ), // #107 + INST(Ic , BaseAtDcIcTlbi , (0b00011110000000, 0b00001110000000, false) , kRWI_RX , 0 , 2 ), // #108 + INST(Isb , BaseOpImm , (0b11010101000000110011000011011111, 4, 8) , 0 , 0 , 11 ), // #109 + INST(Ldadd , BaseAtomicOp , (0b1011100000100000000000, kWX, 30, 0) , kRWI_WRX , 0 , 12 ), // #110 + INST(Ldadda , BaseAtomicOp , (0b1011100010100000000000, kWX, 30, 1) , kRWI_WRX , 0 , 13 ), // #111 + INST(Ldaddab , BaseAtomicOp , (0b0011100010100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 14 ), // #112 + INST(Ldaddah , BaseAtomicOp , (0b0111100010100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 15 ), // #113 + INST(Ldaddal , BaseAtomicOp , (0b1011100011100000000000, kWX, 30, 1) , kRWI_WRX , 0 , 16 ), // #114 + INST(Ldaddalb , BaseAtomicOp , (0b0011100011100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 17 ), // #115 + INST(Ldaddalh , BaseAtomicOp , (0b0111100011100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 18 ), // #116 + INST(Ldaddb , BaseAtomicOp , (0b0011100000100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 19 ), // #117 + INST(Ldaddh , BaseAtomicOp , (0b0111100000100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 20 ), // #118 + INST(Ldaddl , BaseAtomicOp , (0b1011100001100000000000, kWX, 30, 0) , kRWI_WRX , 0 , 21 ), // #119 + INST(Ldaddlb , BaseAtomicOp , (0b0011100001100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 22 ), // #120 + INST(Ldaddlh , BaseAtomicOp , (0b0111100001100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 23 ), // #121 + INST(Ldar , BaseRM_NoImm , (0b1000100011011111111111, kWX, kZR, 30) , kRWI_W , 0 , 0 ), // #122 + INST(Ldarb , BaseRM_NoImm , (0b0000100011011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 1 ), // #123 + INST(Ldarh , BaseRM_NoImm , (0b0100100011011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 2 ), // #124 + INST(Ldaxp , BaseLdxp , (0b1000100001111111100000, kWX, 30) , kRWI_WW , 0 , 0 ), // #125 + INST(Ldaxr , BaseRM_NoImm , (0b1000100001011111111111, kWX, kZR, 30) , kRWI_W , 0 , 3 ), // #126 + INST(Ldaxrb , BaseRM_NoImm , (0b0000100001011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 4 ), // #127 + INST(Ldaxrh , BaseRM_NoImm , (0b0100100001011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 5 ), // #128 + INST(Ldclr , BaseAtomicOp , (0b1011100000100000000100, kWX, 30, 0) , kRWI_WRX , 0 , 24 ), // #129 + INST(Ldclra , BaseAtomicOp , (0b1011100010100000000100, kWX, 30, 1) , kRWI_WRX , 0 , 25 ), // #130 + INST(Ldclrab , BaseAtomicOp , (0b0011100010100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 26 ), // #131 + INST(Ldclrah , BaseAtomicOp , (0b0111100010100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 27 ), // #132 + INST(Ldclral , BaseAtomicOp , (0b1011100011100000000100, kWX, 30, 1) , kRWI_WRX , 0 , 28 ), // #133 + INST(Ldclralb , BaseAtomicOp , (0b0011100011100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 29 ), // #134 + INST(Ldclralh , BaseAtomicOp , (0b0111100011100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 30 ), // #135 + INST(Ldclrb , BaseAtomicOp , (0b0011100000100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 31 ), // #136 + INST(Ldclrh , BaseAtomicOp , (0b0111100000100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 32 ), // #137 + INST(Ldclrl , BaseAtomicOp , (0b1011100001100000000100, kWX, 30, 0) , kRWI_WRX , 0 , 33 ), // #138 + INST(Ldclrlb , BaseAtomicOp , (0b0011100001100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 34 ), // #139 + INST(Ldclrlh , BaseAtomicOp , (0b0111100001100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 35 ), // #140 + INST(Ldeor , BaseAtomicOp , (0b1011100000100000001000, kWX, 30, 0) , kRWI_WRX , 0 , 36 ), // #141 + INST(Ldeora , BaseAtomicOp , (0b1011100010100000001000, kWX, 30, 1) , kRWI_WRX , 0 , 37 ), // #142 + INST(Ldeorab , BaseAtomicOp , (0b0011100010100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 38 ), // #143 + INST(Ldeorah , BaseAtomicOp , (0b0111100010100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 39 ), // #144 + INST(Ldeoral , BaseAtomicOp , (0b1011100011100000001000, kWX, 30, 1) , kRWI_WRX , 0 , 40 ), // #145 + INST(Ldeoralb , BaseAtomicOp , (0b0011100011100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 41 ), // #146 + INST(Ldeoralh , BaseAtomicOp , (0b0111100011100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 42 ), // #147 + INST(Ldeorb , BaseAtomicOp , (0b0011100000100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 43 ), // #148 + INST(Ldeorh , BaseAtomicOp , (0b0111100000100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 44 ), // #149 + INST(Ldeorl , BaseAtomicOp , (0b1011100001100000001000, kWX, 30, 0) , kRWI_WRX , 0 , 45 ), // #150 + INST(Ldeorlb , BaseAtomicOp , (0b0011100001100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 46 ), // #151 + INST(Ldeorlh , BaseAtomicOp , (0b0111100001100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 47 ), // #152 + INST(Ldg , BaseRM_SImm9 , (0b1101100101100000000000, 0b0000000000000000000000, kX , kZR, 0, 4) , kRWI_W , 0 , 0 ), // #153 + INST(Ldgm , BaseRM_NoImm , (0b1101100111100000000000, kX , kZR, 0 ) , kRWI_W , 0 , 6 ), // #154 + INST(Ldlar , BaseRM_NoImm , (0b1000100011011111011111, kWX, kZR, 30) , kRWI_W , 0 , 7 ), // #155 + INST(Ldlarb , BaseRM_NoImm , (0b0000100011011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 8 ), // #156 + INST(Ldlarh , BaseRM_NoImm , (0b0100100011011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 9 ), // #157 + INST(Ldnp , BaseLdpStp , (0b0010100001, 0 , kWX, 31, 2) , kRWI_WW , 0 , 0 ), // #158 + INST(Ldp , BaseLdpStp , (0b0010100101, 0b0010100011, kWX, 31, 2) , kRWI_WW , 0 , 1 ), // #159 + INST(Ldpsw , BaseLdpStp , (0b0110100101, 0b0110100011, kX , 0 , 2) , kRWI_WW , 0 , 2 ), // #160 + INST(Ldr , BaseLdSt , (0b1011100101, 0b10111000010, 0b10111000011, 0b00011000, kWX, 30, 2, Inst::kIdLdur) , kRWI_W , 0 , 0 ), // #161 + INST(Ldraa , BaseRM_SImm10 , (0b1111100000100000000001, kX , kZR, 0, 3) , kRWI_W , 0 , 0 ), // #162 + INST(Ldrab , BaseRM_SImm10 , (0b1111100010100000000001, kX , kZR, 0, 3) , kRWI_W , 0 , 1 ), // #163 + INST(Ldrb , BaseLdSt , (0b0011100101, 0b00111000010, 0b00111000011, 0 , kW , 0 , 0, Inst::kIdLdurb) , kRWI_W , 0 , 1 ), // #164 + INST(Ldrh , BaseLdSt , (0b0111100101, 0b01111000010, 0b01111000011, 0 , kW , 0 , 1, Inst::kIdLdurh) , kRWI_W , 0 , 2 ), // #165 + INST(Ldrsb , BaseLdSt , (0b0011100111, 0b00111000100, 0b00111000111, 0 , kWX, 22, 0, Inst::kIdLdursb) , kRWI_W , 0 , 3 ), // #166 + INST(Ldrsh , BaseLdSt , (0b0111100111, 0b01111000100, 0b01111000111, 0 , kWX, 22, 1, Inst::kIdLdursh) , kRWI_W , 0 , 4 ), // #167 + INST(Ldrsw , BaseLdSt , (0b1011100110, 0b10111000100, 0b10111000101, 0b10011000, kX , 0 , 2, Inst::kIdLdursw) , kRWI_W , 0 , 5 ), // #168 + INST(Ldset , BaseAtomicOp , (0b1011100000100000001100, kWX, 30, 0) , kRWI_WRX , 0 , 48 ), // #169 + INST(Ldseta , BaseAtomicOp , (0b1011100010100000001100, kWX, 30, 1) , kRWI_WRX , 0 , 49 ), // #170 + INST(Ldsetab , BaseAtomicOp , (0b0011100010100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 50 ), // #171 + INST(Ldsetah , BaseAtomicOp , (0b0111100010100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 51 ), // #172 + INST(Ldsetal , BaseAtomicOp , (0b1011100011100000001100, kWX, 30, 1) , kRWI_WRX , 0 , 52 ), // #173 + INST(Ldsetalb , BaseAtomicOp , (0b0011100011100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 53 ), // #174 + INST(Ldsetalh , BaseAtomicOp , (0b0111100011100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 54 ), // #175 + INST(Ldsetb , BaseAtomicOp , (0b0011100000100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 55 ), // #176 + INST(Ldseth , BaseAtomicOp , (0b0111100000100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 56 ), // #177 + INST(Ldsetl , BaseAtomicOp , (0b1011100001100000001100, kWX, 30, 0) , kRWI_WRX , 0 , 57 ), // #178 + INST(Ldsetlb , BaseAtomicOp , (0b0011100001100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 58 ), // #179 + INST(Ldsetlh , BaseAtomicOp , (0b0111100001100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 59 ), // #180 + INST(Ldsmax , BaseAtomicOp , (0b1011100000100000010000, kWX, 30, 0) , kRWI_WRX , 0 , 60 ), // #181 + INST(Ldsmaxa , BaseAtomicOp , (0b1011100010100000010000, kWX, 30, 1) , kRWI_WRX , 0 , 61 ), // #182 + INST(Ldsmaxab , BaseAtomicOp , (0b0011100010100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 62 ), // #183 + INST(Ldsmaxah , BaseAtomicOp , (0b0111100010100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 63 ), // #184 + INST(Ldsmaxal , BaseAtomicOp , (0b1011100011100000010000, kWX, 30, 1) , kRWI_WRX , 0 , 64 ), // #185 + INST(Ldsmaxalb , BaseAtomicOp , (0b0011100011100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 65 ), // #186 + INST(Ldsmaxalh , BaseAtomicOp , (0b0111100011100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 66 ), // #187 + INST(Ldsmaxb , BaseAtomicOp , (0b0011100000100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 67 ), // #188 + INST(Ldsmaxh , BaseAtomicOp , (0b0111100000100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 68 ), // #189 + INST(Ldsmaxl , BaseAtomicOp , (0b1011100001100000010000, kWX, 30, 0) , kRWI_WRX , 0 , 69 ), // #190 + INST(Ldsmaxlb , BaseAtomicOp , (0b0011100001100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 70 ), // #191 + INST(Ldsmaxlh , BaseAtomicOp , (0b0111100001100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 71 ), // #192 + INST(Ldsmin , BaseAtomicOp , (0b1011100000100000010100, kWX, 30, 0) , kRWI_WRX , 0 , 72 ), // #193 + INST(Ldsmina , BaseAtomicOp , (0b1011100010100000010100, kWX, 30, 1) , kRWI_WRX , 0 , 73 ), // #194 + INST(Ldsminab , BaseAtomicOp , (0b0011100010100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 74 ), // #195 + INST(Ldsminah , BaseAtomicOp , (0b0111100010100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 75 ), // #196 + INST(Ldsminal , BaseAtomicOp , (0b1011100011100000010100, kWX, 30, 1) , kRWI_WRX , 0 , 76 ), // #197 + INST(Ldsminalb , BaseAtomicOp , (0b0011100011100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 77 ), // #198 + INST(Ldsminalh , BaseAtomicOp , (0b0111100011100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 78 ), // #199 + INST(Ldsminb , BaseAtomicOp , (0b0011100000100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 79 ), // #200 + INST(Ldsminh , BaseAtomicOp , (0b0111100000100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 80 ), // #201 + INST(Ldsminl , BaseAtomicOp , (0b1011100001100000010100, kWX, 30, 0) , kRWI_WRX , 0 , 81 ), // #202 + INST(Ldsminlb , BaseAtomicOp , (0b0011100001100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 82 ), // #203 + INST(Ldsminlh , BaseAtomicOp , (0b0111100001100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 83 ), // #204 + INST(Ldtr , BaseRM_SImm9 , (0b1011100001000000000010, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_W , 0 , 1 ), // #205 + INST(Ldtrb , BaseRM_SImm9 , (0b0011100001000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 2 ), // #206 + INST(Ldtrh , BaseRM_SImm9 , (0b0111100001000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 3 ), // #207 + INST(Ldtrsb , BaseRM_SImm9 , (0b0011100011000000000010, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 4 ), // #208 + INST(Ldtrsh , BaseRM_SImm9 , (0b0111100011000000000010, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 5 ), // #209 + INST(Ldtrsw , BaseRM_SImm9 , (0b1011100010000000000010, 0b0000000000000000000000, kX , kZR, 0 , 0) , kRWI_W , 0 , 6 ), // #210 + INST(Ldumax , BaseAtomicOp , (0b1011100000100000011000, kWX, 30, 0) , kRWI_WRX , 0 , 84 ), // #211 + INST(Ldumaxa , BaseAtomicOp , (0b1011100010100000011000, kWX, 30, 1) , kRWI_WRX , 0 , 85 ), // #212 + INST(Ldumaxab , BaseAtomicOp , (0b0011100010100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 86 ), // #213 + INST(Ldumaxah , BaseAtomicOp , (0b0111100010100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 87 ), // #214 + INST(Ldumaxal , BaseAtomicOp , (0b1011100011100000011000, kWX, 30, 1) , kRWI_WRX , 0 , 88 ), // #215 + INST(Ldumaxalb , BaseAtomicOp , (0b0011100011100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 89 ), // #216 + INST(Ldumaxalh , BaseAtomicOp , (0b0111100011100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 90 ), // #217 + INST(Ldumaxb , BaseAtomicOp , (0b0011100000100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 91 ), // #218 + INST(Ldumaxh , BaseAtomicOp , (0b0111100000100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 92 ), // #219 + INST(Ldumaxl , BaseAtomicOp , (0b1011100001100000011000, kWX, 30, 0) , kRWI_WRX , 0 , 93 ), // #220 + INST(Ldumaxlb , BaseAtomicOp , (0b0011100001100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 94 ), // #221 + INST(Ldumaxlh , BaseAtomicOp , (0b0111100001100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 95 ), // #222 + INST(Ldumin , BaseAtomicOp , (0b1011100000100000011100, kWX, 30, 0) , kRWI_WRX , 0 , 96 ), // #223 + INST(Ldumina , BaseAtomicOp , (0b1011100010100000011100, kWX, 30, 1) , kRWI_WRX , 0 , 97 ), // #224 + INST(Lduminab , BaseAtomicOp , (0b0011100010100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 98 ), // #225 + INST(Lduminah , BaseAtomicOp , (0b0111100010100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 99 ), // #226 + INST(Lduminal , BaseAtomicOp , (0b1011100011100000011100, kWX, 30, 1) , kRWI_WRX , 0 , 100), // #227 + INST(Lduminalb , BaseAtomicOp , (0b0011100011100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 101), // #228 + INST(Lduminalh , BaseAtomicOp , (0b0111100011100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 102), // #229 + INST(Lduminb , BaseAtomicOp , (0b0011100000100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 103), // #230 + INST(Lduminh , BaseAtomicOp , (0b0111100000100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 104), // #231 + INST(Lduminl , BaseAtomicOp , (0b1011100001100000011100, kWX, 30, 0) , kRWI_WRX , 0 , 105), // #232 + INST(Lduminlb , BaseAtomicOp , (0b0011100001100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 106), // #233 + INST(Lduminlh , BaseAtomicOp , (0b0111100001100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 107), // #234 + INST(Ldur , BaseRM_SImm9 , (0b1011100001000000000000, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_W , 0 , 7 ), // #235 + INST(Ldurb , BaseRM_SImm9 , (0b0011100001000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 8 ), // #236 + INST(Ldurh , BaseRM_SImm9 , (0b0111100001000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 9 ), // #237 + INST(Ldursb , BaseRM_SImm9 , (0b0011100011000000000000, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 10 ), // #238 + INST(Ldursh , BaseRM_SImm9 , (0b0111100011000000000000, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 11 ), // #239 + INST(Ldursw , BaseRM_SImm9 , (0b1011100010000000000000, 0b0000000000000000000000, kX , kZR, 0 , 0) , kRWI_W , 0 , 12 ), // #240 + INST(Ldxp , BaseLdxp , (0b1000100001111111000000, kWX, 30) , kRWI_WW , 0 , 1 ), // #241 + INST(Ldxr , BaseRM_NoImm , (0b1000100001011111011111, kWX, kZR, 30) , kRWI_W , 0 , 10 ), // #242 + INST(Ldxrb , BaseRM_NoImm , (0b0000100001011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 11 ), // #243 + INST(Ldxrh , BaseRM_NoImm , (0b0100100001011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 12 ), // #244 + INST(Lsl , BaseShift , (0b0001101011000000001000, 0b0101001100000000000000, 0) , kRWI_W , 0 , 2 ), // #245 + INST(Lslv , BaseShift , (0b0001101011000000001000, 0b0000000000000000000000, 0) , kRWI_W , 0 , 3 ), // #246 + INST(Lsr , BaseShift , (0b0001101011000000001001, 0b0101001100000000011111, 0) , kRWI_W , 0 , 4 ), // #247 + INST(Lsrv , BaseShift , (0b0001101011000000001001, 0b0000000000000000000000, 0) , kRWI_W , 0 , 5 ), // #248 + INST(Madd , BaseRRRR , (0b0001101100000000000000, kWX, kZR, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 0 ), // #249 + INST(Mneg , BaseRRR , (0b0001101100000000111111, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 11 ), // #250 + INST(Mov , BaseMov , (_) , kRWI_W , 0 , 0 ), // #251 + INST(Movk , BaseMovKNZ , (0b01110010100000000000000000000000) , kRWI_X , 0 , 0 ), // #252 + INST(Movn , BaseMovKNZ , (0b00010010100000000000000000000000) , kRWI_W , 0 , 1 ), // #253 + INST(Movz , BaseMovKNZ , (0b01010010100000000000000000000000) , kRWI_W , 0 , 2 ), // #254 + INST(Mrs , BaseMrs , (_) , kRWI_W , 0 , 0 ), // #255 + INST(Msr , BaseMsr , (_) , kRWI_W , 0 , 0 ), // #256 + INST(Msub , BaseRRRR , (0b0001101100000000100000, kWX, kZR, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 1 ), // #257 + INST(Mul , BaseRRR , (0b0001101100000000011111, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 12 ), // #258 + INST(Mvn , BaseMvnNeg , (0b00101010001000000000001111100000) , kRWI_W , 0 , 0 ), // #259 + INST(Neg , BaseMvnNeg , (0b01001011000000000000001111100000) , kRWI_W , 0 , 1 ), // #260 + INST(Negs , BaseMvnNeg , (0b01101011000000000000001111100000) , kRWI_W , 0 , 2 ), // #261 + INST(Ngc , BaseRR , (0b01011010000000000000001111100000, kWX, kZR, 0, kWX, kZR, 16, true) , kRWI_W , 0 , 10 ), // #262 + INST(Ngcs , BaseRR , (0b01111010000000000000001111100000, kWX, kZR, 0, kWX, kZR, 16, true) , kRWI_W , 0 , 11 ), // #263 + INST(Nop , BaseOp , (0b11010101000000110010000000011111) , 0 , 0 , 14 ), // #264 + INST(Orn , BaseLogical , (0b0101010001, 0b01100100, 1) , kRWI_W , 0 , 6 ), // #265 + INST(Orr , BaseLogical , (0b0101010000, 0b01100100, 0) , kRWI_W , 0 , 7 ), // #266 + INST(Pacda , BaseRR , (0b11011010110000010000100000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 12 ), // #267 + INST(Pacdb , BaseRR , (0b11011010110000010000110000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 13 ), // #268 + INST(Pacdza , BaseR , (0b11011010110000010010101111100000, kX, kZR, 0) , kRWI_X , 0 , 4 ), // #269 + INST(Pacdzb , BaseR , (0b11011010110000010010111111100000, kX, kZR, 0) , kRWI_X , 0 , 5 ), // #270 + INST(Pacga , BaseRRR , (0b1001101011000000001100, kX, kZR, kX, kZR, kX, kSP, false) , kRWI_W , 0 , 13 ), // #271 + INST(Prfm , BasePrfm , (0b11111000101, 0b1111100110, 0b11111000100, 0b11011000) , kRWI_R , 0 , 0 ), // #272 + INST(Pssbb , BaseOp , (0b11010101000000110011010010011111) , 0 , 0 , 15 ), // #273 + INST(Rbit , BaseRR , (0b01011010110000000000000000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 14 ), // #274 + INST(Ret , BaseBranchReg , (0b11010110010111110000000000000000) , kRWI_R , 0 , 2 ), // #275 + INST(Rev , BaseRev , (_) , kRWI_W , 0 , 0 ), // #276 + INST(Rev16 , BaseRR , (0b01011010110000000000010000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 15 ), // #277 + INST(Rev32 , BaseRR , (0b11011010110000000000100000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 16 ), // #278 + INST(Rev64 , BaseRR , (0b11011010110000000000110000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 17 ), // #279 + INST(Ror , BaseShift , (0b0001101011000000001011, 0b0001001110000000000000, 1) , kRWI_W , 0 , 6 ), // #280 + INST(Rorv , BaseShift , (0b0001101011000000001011, 0b0000000000000000000000, 1) , kRWI_W , 0 , 7 ), // #281 + INST(Sbc , BaseRRR , (0b0101101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 14 ), // #282 + INST(Sbcs , BaseRRR , (0b0111101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 15 ), // #283 + INST(Sbfiz , BaseBfi , (0b00010011000000000000000000000000) , kRWI_W , 0 , 1 ), // #284 + INST(Sbfm , BaseBfm , (0b00010011000000000000000000000000) , kRWI_W , 0 , 1 ), // #285 + INST(Sbfx , BaseBfx , (0b00010011000000000000000000000000) , kRWI_W , 0 , 1 ), // #286 + INST(Sdiv , BaseRRR , (0b0001101011000000000011, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 16 ), // #287 + INST(Setf8 , BaseR , (0b00111010000000000000100000001101, kW, kZR, 5) , 0 , 0 , 6 ), // #288 + INST(Setf16 , BaseR , (0b00111010000000000100100000001101, kW, kZR, 5) , 0 , 0 , 7 ), // #289 + INST(Sev , BaseOp , (0b11010101000000110010000010011111) , 0 , 0 , 16 ), // #290 + INST(Sevl , BaseOp , (0b11010101000000110010000010111111) , 0 , 0 , 17 ), // #291 + INST(Smaddl , BaseRRRR , (0b1001101100100000000000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 2 ), // #292 + INST(Smax , BaseMinMax , (0b00011010110000000110000000000000, 0b00010001110000000000000000000000) , kRWI_W , 0 , 0 ), // #293 + INST(Smc , BaseOpImm , (0b11010100000000000000000000000011, 16, 5) , 0 , 0 , 12 ), // #294 + INST(Smin , BaseMinMax , (0b00011010110000000110100000000000, 0b00010001110010000000000000000000) , kRWI_W , 0 , 1 ), // #295 + INST(Smnegl , BaseRRR , (0b1001101100100000111111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 17 ), // #296 + INST(Smsubl , BaseRRRR , (0b1001101100100000100000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 3 ), // #297 + INST(Smulh , BaseRRR , (0b1001101101000000011111, kX , kZR, kX , kZR, kX , kZR, true) , kRWI_W , 0 , 18 ), // #298 + INST(Smull , BaseRRR , (0b1001101100100000011111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 19 ), // #299 + INST(Ssbb , BaseOp , (0b11010101000000110011000010011111) , 0 , 0 , 18 ), // #300 + INST(St2g , BaseRM_SImm9 , (0b1101100110100000000010, 0b1101100110100000000001, kX, kSP, 0, 4) , kRWI_RW , 0 , 13 ), // #301 + INST(Stadd , BaseAtomicSt , (0b1011100000100000000000, kWX, 30) , kRWI_RX , 0 , 0 ), // #302 + INST(Staddl , BaseAtomicSt , (0b1011100001100000000000, kWX, 30) , kRWI_RX , 0 , 1 ), // #303 + INST(Staddb , BaseAtomicSt , (0b0011100000100000000000, kW , 0 ) , kRWI_RX , 0 , 2 ), // #304 + INST(Staddlb , BaseAtomicSt , (0b0011100001100000000000, kW , 0 ) , kRWI_RX , 0 , 3 ), // #305 + INST(Staddh , BaseAtomicSt , (0b0111100000100000000000, kW , 0 ) , kRWI_RX , 0 , 4 ), // #306 + INST(Staddlh , BaseAtomicSt , (0b0111100001100000000000, kW , 0 ) , kRWI_RX , 0 , 5 ), // #307 + INST(Stclr , BaseAtomicSt , (0b1011100000100000000100, kWX, 30) , kRWI_RX , 0 , 6 ), // #308 + INST(Stclrl , BaseAtomicSt , (0b1011100001100000000100, kWX, 30) , kRWI_RX , 0 , 7 ), // #309 + INST(Stclrb , BaseAtomicSt , (0b0011100000100000000100, kW , 0 ) , kRWI_RX , 0 , 8 ), // #310 + INST(Stclrlb , BaseAtomicSt , (0b0011100001100000000100, kW , 0 ) , kRWI_RX , 0 , 9 ), // #311 + INST(Stclrh , BaseAtomicSt , (0b0111100000100000000100, kW , 0 ) , kRWI_RX , 0 , 10 ), // #312 + INST(Stclrlh , BaseAtomicSt , (0b0111100001100000000100, kW , 0 ) , kRWI_RX , 0 , 11 ), // #313 + INST(Steor , BaseAtomicSt , (0b1011100000100000001000, kWX, 30) , kRWI_RX , 0 , 12 ), // #314 + INST(Steorl , BaseAtomicSt , (0b1011100001100000001000, kWX, 30) , kRWI_RX , 0 , 13 ), // #315 + INST(Steorb , BaseAtomicSt , (0b0011100000100000001000, kW , 0 ) , kRWI_RX , 0 , 14 ), // #316 + INST(Steorlb , BaseAtomicSt , (0b0011100001100000001000, kW , 0 ) , kRWI_RX , 0 , 15 ), // #317 + INST(Steorh , BaseAtomicSt , (0b0111100000100000001000, kW , 0 ) , kRWI_RX , 0 , 16 ), // #318 + INST(Steorlh , BaseAtomicSt , (0b0111100001100000001000, kW , 0 ) , kRWI_RX , 0 , 17 ), // #319 + INST(Stg , BaseRM_SImm9 , (0b1101100100100000000010, 0b1101100100100000000001, kX, kSP, 0, 4) , kRWI_RW , 0 , 14 ), // #320 + INST(Stgm , BaseRM_NoImm , (0b1101100110100000000000, kX , kZR, 0 ) , kRWI_RW , 0 , 13 ), // #321 + INST(Stgp , BaseLdpStp , (0b0110100100, 0b0110100010, kX, 0, 4) , kRWI_RRW , 0 , 3 ), // #322 + INST(Stllr , BaseRM_NoImm , (0b1000100010011111011111, kWX, kZR, 30) , kRWI_RW , 0 , 14 ), // #323 + INST(Stllrb , BaseRM_NoImm , (0b0000100010011111011111, kW , kZR, 0 ) , kRWI_RW , 0 , 15 ), // #324 + INST(Stllrh , BaseRM_NoImm , (0b0100100010011111011111, kW , kZR, 0 ) , kRWI_RW , 0 , 16 ), // #325 + INST(Stlr , BaseRM_NoImm , (0b1000100010011111111111, kWX, kZR, 30) , kRWI_RW , 0 , 17 ), // #326 + INST(Stlrb , BaseRM_NoImm , (0b0000100010011111111111, kW , kZR, 0 ) , kRWI_RW , 0 , 18 ), // #327 + INST(Stlrh , BaseRM_NoImm , (0b0100100010011111111111, kW , kZR, 0 ) , kRWI_RW , 0 , 19 ), // #328 + INST(Stlxp , BaseStxp , (0b1000100000100000100000, kWX, 30) , kRWI_WRRX , 0 , 0 ), // #329 + INST(Stlxr , BaseAtomicOp , (0b1000100000000000111111, kWX, 30, 1) , kRWI_WRX , 0 , 108), // #330 + INST(Stlxrb , BaseAtomicOp , (0b0000100000000000111111, kW , 0 , 1) , kRWI_WRX , 0 , 109), // #331 + INST(Stlxrh , BaseAtomicOp , (0b0100100000000000111111, kW , 0 , 1) , kRWI_WRX , 0 , 110), // #332 + INST(Stnp , BaseLdpStp , (0b0010100000, 0 , kWX, 31, 2) , kRWI_RRW , 0 , 4 ), // #333 + INST(Stp , BaseLdpStp , (0b0010100100, 0b0010100010, kWX, 31, 2) , kRWI_RRW , 0 , 5 ), // #334 + INST(Str , BaseLdSt , (0b1011100100, 0b10111000000, 0b10111000001, 0 , kWX, 30, 2, Inst::kIdStur) , kRWI_RW , 0 , 6 ), // #335 + INST(Strb , BaseLdSt , (0b0011100100, 0b00111000000, 0b00111000001, 0 , kW , 30, 0, Inst::kIdSturb) , kRWI_RW , 0 , 7 ), // #336 + INST(Strh , BaseLdSt , (0b0111100100, 0b01111000000, 0b01111000001, 0 , kWX, 30, 1, Inst::kIdSturh) , kRWI_RW , 0 , 8 ), // #337 + INST(Stset , BaseAtomicSt , (0b1011100000100000001100, kWX, 30) , kRWI_RX , 0 , 18 ), // #338 + INST(Stsetl , BaseAtomicSt , (0b1011100001100000001100, kWX, 30) , kRWI_RX , 0 , 19 ), // #339 + INST(Stsetb , BaseAtomicSt , (0b0011100000100000001100, kW , 0 ) , kRWI_RX , 0 , 20 ), // #340 + INST(Stsetlb , BaseAtomicSt , (0b0011100001100000001100, kW , 0 ) , kRWI_RX , 0 , 21 ), // #341 + INST(Stseth , BaseAtomicSt , (0b0111100000100000001100, kW , 0 ) , kRWI_RX , 0 , 22 ), // #342 + INST(Stsetlh , BaseAtomicSt , (0b0111100001100000001100, kW , 0 ) , kRWI_RX , 0 , 23 ), // #343 + INST(Stsmax , BaseAtomicSt , (0b1011100000100000010000, kWX, 30) , kRWI_RX , 0 , 24 ), // #344 + INST(Stsmaxl , BaseAtomicSt , (0b1011100001100000010000, kWX, 30) , kRWI_RX , 0 , 25 ), // #345 + INST(Stsmaxb , BaseAtomicSt , (0b0011100000100000010000, kW , 0 ) , kRWI_RX , 0 , 26 ), // #346 + INST(Stsmaxlb , BaseAtomicSt , (0b0011100001100000010000, kW , 0 ) , kRWI_RX , 0 , 27 ), // #347 + INST(Stsmaxh , BaseAtomicSt , (0b0111100000100000010000, kW , 0 ) , kRWI_RX , 0 , 28 ), // #348 + INST(Stsmaxlh , BaseAtomicSt , (0b0111100001100000010000, kW , 0 ) , kRWI_RX , 0 , 29 ), // #349 + INST(Stsmin , BaseAtomicSt , (0b1011100000100000010100, kWX, 30) , kRWI_RX , 0 , 30 ), // #350 + INST(Stsminl , BaseAtomicSt , (0b1011100001100000010100, kWX, 30) , kRWI_RX , 0 , 31 ), // #351 + INST(Stsminb , BaseAtomicSt , (0b0011100000100000010100, kW , 0 ) , kRWI_RX , 0 , 32 ), // #352 + INST(Stsminlb , BaseAtomicSt , (0b0011100001100000010100, kW , 0 ) , kRWI_RX , 0 , 33 ), // #353 + INST(Stsminh , BaseAtomicSt , (0b0111100000100000010100, kW , 0 ) , kRWI_RX , 0 , 34 ), // #354 + INST(Stsminlh , BaseAtomicSt , (0b0111100001100000010100, kW , 0 ) , kRWI_RX , 0 , 35 ), // #355 + INST(Sttr , BaseRM_SImm9 , (0b1011100000000000000010, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_RW , 0 , 15 ), // #356 + INST(Sttrb , BaseRM_SImm9 , (0b0011100000000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 16 ), // #357 + INST(Sttrh , BaseRM_SImm9 , (0b0111100000000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 17 ), // #358 + INST(Stumax , BaseAtomicSt , (0b1011100000100000011000, kWX, 30) , kRWI_RX , 0 , 36 ), // #359 + INST(Stumaxl , BaseAtomicSt , (0b1011100001100000011000, kWX, 30) , kRWI_RX , 0 , 37 ), // #360 + INST(Stumaxb , BaseAtomicSt , (0b0011100000100000011000, kW , 0 ) , kRWI_RX , 0 , 38 ), // #361 + INST(Stumaxlb , BaseAtomicSt , (0b0011100001100000011000, kW , 0 ) , kRWI_RX , 0 , 39 ), // #362 + INST(Stumaxh , BaseAtomicSt , (0b0111100000100000011000, kW , 0 ) , kRWI_RX , 0 , 40 ), // #363 + INST(Stumaxlh , BaseAtomicSt , (0b0111100001100000011000, kW , 0 ) , kRWI_RX , 0 , 41 ), // #364 + INST(Stumin , BaseAtomicSt , (0b1011100000100000011100, kWX, 30) , kRWI_RX , 0 , 42 ), // #365 + INST(Stuminl , BaseAtomicSt , (0b1011100001100000011100, kWX, 30) , kRWI_RX , 0 , 43 ), // #366 + INST(Stuminb , BaseAtomicSt , (0b0011100000100000011100, kW , 0 ) , kRWI_RX , 0 , 44 ), // #367 + INST(Stuminlb , BaseAtomicSt , (0b0011100001100000011100, kW , 0 ) , kRWI_RX , 0 , 45 ), // #368 + INST(Stuminh , BaseAtomicSt , (0b0111100000100000011100, kW , 0 ) , kRWI_RX , 0 , 46 ), // #369 + INST(Stuminlh , BaseAtomicSt , (0b0111100001100000011100, kW , 0 ) , kRWI_RX , 0 , 47 ), // #370 + INST(Stur , BaseRM_SImm9 , (0b1011100000000000000000, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_RW , 0 , 18 ), // #371 + INST(Sturb , BaseRM_SImm9 , (0b0011100000000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 19 ), // #372 + INST(Sturh , BaseRM_SImm9 , (0b0111100000000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 20 ), // #373 + INST(Stxp , BaseStxp , (0b1000100000100000000000, kWX, 30) , kRWI_WRRW , 0 , 1 ), // #374 + INST(Stxr , BaseStx , (0b1000100000000000011111, kWX, 30) , kRWI_WRW , 0 , 0 ), // #375 + INST(Stxrb , BaseStx , (0b0000100000000000011111, kW , 0 ) , kRWI_WRW , 0 , 1 ), // #376 + INST(Stxrh , BaseStx , (0b0100100000000000011111, kW , 0 ) , kRWI_WRW , 0 , 2 ), // #377 + INST(Stz2g , BaseRM_SImm9 , (0b1101100111100000000010, 0b1101100111100000000001, kX , kSP, 0, 4) , kRWI_RW , 0 , 21 ), // #378 + INST(Stzg , BaseRM_SImm9 , (0b1101100101100000000010, 0b1101100101100000000001, kX , kSP, 0, 4) , kRWI_RW , 0 , 22 ), // #379 + INST(Stzgm , BaseRM_NoImm , (0b1101100100100000000000, kX , kZR, 0) , kRWI_RW , 0 , 20 ), // #380 + INST(Sub , BaseAddSub , (0b1001011000, 0b1001011001, 0b1010001) , kRWI_W , 0 , 2 ), // #381 + INST(Subg , BaseRRII , (0b1101000110000000000000, kX, kSP, kX, kSP, 6, 4, 16, 4, 0, 10) , kRWI_W , 0 , 1 ), // #382 + INST(Subp , BaseRRR , (0b1001101011000000000000, kX, kZR, kX, kSP, kX, kSP, false) , kRWI_W , 0 , 20 ), // #383 + INST(Subps , BaseRRR , (0b1011101011000000000000, kX, kZR, kX, kSP, kX, kSP, false) , kRWI_W , 0 , 21 ), // #384 + INST(Subs , BaseAddSub , (0b1101011000, 0b1101011001, 0b1110001) , kRWI_W , 0 , 3 ), // #385 + INST(Svc , BaseOpImm , (0b11010100000000000000000000000001, 16, 5) , 0 , 0 , 13 ), // #386 + INST(Swp , BaseAtomicOp , (0b1011100000100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 111), // #387 + INST(Swpa , BaseAtomicOp , (0b1011100010100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 112), // #388 + INST(Swpab , BaseAtomicOp , (0b0011100010100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 113), // #389 + INST(Swpah , BaseAtomicOp , (0b0111100010100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 114), // #390 + INST(Swpal , BaseAtomicOp , (0b1011100011100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 115), // #391 + INST(Swpalb , BaseAtomicOp , (0b0011100011100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 116), // #392 + INST(Swpalh , BaseAtomicOp , (0b0111100011100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 117), // #393 + INST(Swpb , BaseAtomicOp , (0b0011100000100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 118), // #394 + INST(Swph , BaseAtomicOp , (0b0111100000100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 119), // #395 + INST(Swpl , BaseAtomicOp , (0b1011100001100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 120), // #396 + INST(Swplb , BaseAtomicOp , (0b0011100001100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 121), // #397 + INST(Swplh , BaseAtomicOp , (0b0111100001100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 122), // #398 + INST(Sxtb , BaseExtend , (0b0001001100000000000111, kWX, 0) , kRWI_W , 0 , 0 ), // #399 + INST(Sxth , BaseExtend , (0b0001001100000000001111, kWX, 0) , kRWI_W , 0 , 1 ), // #400 + INST(Sxtw , BaseExtend , (0b1001001101000000011111, kX , 0) , kRWI_W , 0 , 2 ), // #401 + INST(Sys , BaseSys , (_) , kRWI_W , 0 , 0 ), // #402 + INST(Tlbi , BaseAtDcIcTlbi , (0b00011110000000, 0b00010000000000, false) , kRWI_RX , 0 , 3 ), // #403 + INST(Tst , BaseTst , (0b1101010000, 0b111001000) , kRWI_R , 0 , 0 ), // #404 + INST(Tbnz , BaseBranchTst , (0b00110111000000000000000000000000) , kRWI_R , 0 , 0 ), // #405 + INST(Tbz , BaseBranchTst , (0b00110110000000000000000000000000) , kRWI_R , 0 , 1 ), // #406 + INST(Ubfiz , BaseBfi , (0b01010011000000000000000000000000) , kRWI_W , 0 , 2 ), // #407 + INST(Ubfm , BaseBfm , (0b01010011000000000000000000000000) , kRWI_W , 0 , 2 ), // #408 + INST(Ubfx , BaseBfx , (0b01010011000000000000000000000000) , kRWI_W , 0 , 2 ), // #409 + INST(Udf , BaseOpImm , (0b00000000000000000000000000000000, 16, 0) , 0 , 0 , 14 ), // #410 + INST(Udiv , BaseRRR , (0b0001101011000000000010, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 22 ), // #411 + INST(Umaddl , BaseRRRR , (0b1001101110100000000000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 4 ), // #412 + INST(Umax , BaseMinMax , (0b00011010110000000110010000000000, 0b00010001110001000000000000000000) , kRWI_W , 0 , 2 ), // #413 + INST(Umin , BaseMinMax , (0b00011010110000000110110000000000, 0b00010001110011000000000000000000) , kRWI_W , 0 , 3 ), // #414 + INST(Umnegl , BaseRRR , (0b1001101110100000111111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 23 ), // #415 + INST(Umull , BaseRRR , (0b1001101110100000011111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 24 ), // #416 + INST(Umulh , BaseRRR , (0b1001101111000000011111, kX , kZR, kX , kZR, kX , kZR, false) , kRWI_W , 0 , 25 ), // #417 + INST(Umsubl , BaseRRRR , (0b1001101110100000100000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 5 ), // #418 + INST(Uxtb , BaseExtend , (0b0101001100000000000111, kW, 1) , kRWI_W , 0 , 3 ), // #419 + INST(Uxth , BaseExtend , (0b0101001100000000001111, kW, 1) , kRWI_W , 0 , 4 ), // #420 + INST(Wfe , BaseOp , (0b11010101000000110010000001011111) , 0 , 0 , 19 ), // #421 + INST(Wfi , BaseOp , (0b11010101000000110010000001111111) , 0 , 0 , 20 ), // #422 + INST(Xaflag , BaseOp , (0b11010101000000000100000000111111) , 0 , 0 , 21 ), // #423 + INST(Xpacd , BaseR , (0b11011010110000010100011111100000, kX, kZR, 0) , kRWI_X , 0 , 8 ), // #424 + INST(Xpaci , BaseR , (0b11011010110000010100001111100000, kX, kZR, 0) , kRWI_X , 0 , 9 ), // #425 + INST(Xpaclri , BaseOp , (0b11010101000000110010000011111111) , kRWI_X , 0 , 22 ), // #426 + INST(Yield , BaseOp , (0b11010101000000110010000000111111) , 0 , 0 , 23 ), // #427 + INST(Abs_v , ISimdVV , (0b0000111000100000101110, kVO_V_Any) , kRWI_W , 0 , 0 ), // #428 + INST(Add_v , ISimdVVV , (0b0000111000100000100001, kVO_V_Any) , kRWI_W , 0 , 0 ), // #429 + INST(Addhn_v , ISimdVVV , (0b0000111000100000010000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 1 ), // #430 + INST(Addhn2_v , ISimdVVV , (0b0100111000100000010000, kVO_V_B16H8S4) , kRWI_W , F(Narrow) , 2 ), // #431 + INST(Addp_v , ISimdPair , (0b0101111000110001101110, 0b0000111000100000101111, kVO_V_Any) , kRWI_W , F(Pair) , 0 ), // #432 + INST(Addv_v , ISimdSV , (0b0000111000110001101110, kVO_V_BH_4S) , kRWI_W , 0 , 0 ), // #433 + INST(Aesd_v , ISimdVVx , (0b0100111000101000010110, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 0 ), // #434 + INST(Aese_v , ISimdVVx , (0b0100111000101000010010, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 1 ), // #435 + INST(Aesimc_v , ISimdVVx , (0b0100111000101000011110, kOp_V16B, kOp_V16B) , kRWI_W , 0 , 2 ), // #436 + INST(Aesmc_v , ISimdVVx , (0b0100111000101000011010, kOp_V16B, kOp_V16B) , kRWI_W , 0 , 3 ), // #437 + INST(And_v , ISimdVVV , (0b0000111000100000000111, kVO_V_B) , kRWI_W , 0 , 3 ), // #438 + INST(Bcax_v , ISimdVVVV , (0b1100111000100000000000, kVO_V_B16) , kRWI_W , 0 , 0 ), // #439 + INST(Bfcvt_v , ISimdVVx , (0b0001111001100011010000, kOp_H, kOp_S) , kRWI_W , 0 , 4 ), // #440 + INST(Bfcvtn_v , ISimdVVx , (0b0000111010100001011010, kOp_V4H, kOp_V4S) , kRWI_W , F(Narrow) , 5 ), // #441 + INST(Bfcvtn2_v , ISimdVVx , (0b0100111010100001011010, kOp_V8H, kOp_V4S) , kRWI_W , F(Narrow) , 6 ), // #442 + INST(Bfdot_v , SimdDot , (0b0010111001000000111111, 0b0000111101000000111100, kET_S, kET_H, kET_2H) , kRWI_X , 0 , 0 ), // #443 + INST(Bfmlalb_v , SimdFmlal , (0b0010111011000000111111, 0b0000111111000000111100, 0, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 0 ), // #444 + INST(Bfmlalt_v , SimdFmlal , (0b0110111011000000111111, 0b0100111111000000111100, 0, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 1 ), // #445 + INST(Bfmmla_v , ISimdVVVx , (0b0110111001000000111011, kOp_V4S, kOp_V8H, kOp_V8H) , kRWI_X , F(Long) , 0 ), // #446 + INST(Bic_v , SimdBicOrr , (0b0000111001100000000111, 0b0010111100000000000001) , kRWI_W , 0 , 0 ), // #447 + INST(Bif_v , ISimdVVV , (0b0010111011100000000111, kVO_V_B) , kRWI_X , 0 , 4 ), // #448 + INST(Bit_v , ISimdVVV , (0b0010111010100000000111, kVO_V_B) , kRWI_X , 0 , 5 ), // #449 + INST(Bsl_v , ISimdVVV , (0b0010111001100000000111, kVO_V_B) , kRWI_X , 0 , 6 ), // #450 + INST(Cls_v , ISimdVV , (0b0000111000100000010010, kVO_V_BHS) , kRWI_W , 0 , 1 ), // #451 + INST(Clz_v , ISimdVV , (0b0010111000100000010010, kVO_V_BHS) , kRWI_W , 0 , 2 ), // #452 + INST(Cmeq_v , SimdCmp , (0b0010111000100000100011, 0b0000111000100000100110, kVO_V_Any) , kRWI_W , 0 , 0 ), // #453 + INST(Cmge_v , SimdCmp , (0b0000111000100000001111, 0b0010111000100000100010, kVO_V_Any) , kRWI_W , 0 , 1 ), // #454 + INST(Cmgt_v , SimdCmp , (0b0000111000100000001101, 0b0000111000100000100010, kVO_V_Any) , kRWI_W , 0 , 2 ), // #455 + INST(Cmhi_v , SimdCmp , (0b0010111000100000001101, 0b0000000000000000000000, kVO_V_Any) , kRWI_W , 0 , 3 ), // #456 + INST(Cmhs_v , SimdCmp , (0b0010111000100000001111, 0b0000000000000000000000, kVO_V_Any) , kRWI_W , 0 , 4 ), // #457 + INST(Cmle_v , SimdCmp , (0b0000000000000000000000, 0b0010111000100000100110, kVO_V_Any) , kRWI_W , 0 , 5 ), // #458 + INST(Cmlt_v , SimdCmp , (0b0000000000000000000000, 0b0000111000100000101010, kVO_V_Any) , kRWI_W , 0 , 6 ), // #459 + INST(Cmtst_v , ISimdVVV , (0b0000111000100000100011, kVO_V_Any) , kRWI_W , 0 , 7 ), // #460 + INST(Cnt_v , ISimdVV , (0b0000111000100000010110, kVO_V_B) , kRWI_W , 0 , 3 ), // #461 + INST(Dup_v , SimdDup , (_) , kRWI_W , 0 , 0 ), // #462 + INST(Eor_v , ISimdVVV , (0b0010111000100000000111, kVO_V_B) , kRWI_W , 0 , 8 ), // #463 + INST(Eor3_v , ISimdVVVV , (0b1100111000000000000000, kVO_V_B16) , kRWI_W , 0 , 1 ), // #464 + INST(Ext_v , ISimdVVVI , (0b0010111000000000000000, kVO_V_B, 4, 11, 1) , kRWI_W , 0 , 0 ), // #465 + INST(Fabd_v , FSimdVVV , (0b0111111010100000110101, kHF_C, 0b0010111010100000110101, kHF_C) , kRWI_W , 0 , 0 ), // #466 + INST(Fabs_v , FSimdVV , (0b0001111000100000110000, kHF_A, 0b0000111010100000111110, kHF_B) , kRWI_W , 0 , 0 ), // #467 + INST(Facge_v , FSimdVVV , (0b0111111000100000111011, kHF_C, 0b0010111000100000111011, kHF_C) , kRWI_W , 0 , 1 ), // #468 + INST(Facgt_v , FSimdVVV , (0b0111111010100000111011, kHF_C, 0b0010111010100000111011, kHF_C) , kRWI_W , 0 , 2 ), // #469 + INST(Fadd_v , FSimdVVV , (0b0001111000100000001010, kHF_A, 0b0000111000100000110101, kHF_C) , kRWI_W , 0 , 3 ), // #470 + INST(Faddp_v , FSimdPair , (0b0111111000110000110110, 0b0010111000100000110101) , kRWI_W , 0 , 0 ), // #471 + INST(Fcadd_v , SimdFcadd , (0b0010111000000000111001) , kRWI_W , 0 , 0 ), // #472 + INST(Fccmp_v , SimdFccmpFccmpe , (0b00011110001000000000010000000000) , kRWI_R , 0 , 0 ), // #473 + INST(Fccmpe_v , SimdFccmpFccmpe , (0b00011110001000000000010000010000) , kRWI_R , 0 , 1 ), // #474 + INST(Fcmeq_v , SimdFcm , (0b0000111000100000111001, kHF_C, 0b0000111010100000110110) , kRWI_W , 0 , 0 ), // #475 + INST(Fcmge_v , SimdFcm , (0b0010111000100000111001, kHF_C, 0b0010111010100000110010) , kRWI_W , 0 , 1 ), // #476 + INST(Fcmgt_v , SimdFcm , (0b0010111010100000111001, kHF_C, 0b0000111010100000110010) , kRWI_W , 0 , 2 ), // #477 + INST(Fcmla_v , SimdFcmla , (0b0010111000000000110001, 0b0010111100000000000100) , kRWI_X , 0 , 0 ), // #478 + INST(Fcmle_v , SimdFcm , (0b0000000000000000000000, kHF_C, 0b0010111010100000110110) , kRWI_W , 0 , 3 ), // #479 + INST(Fcmlt_v , SimdFcm , (0b0000000000000000000000, kHF_C, 0b0000111010100000111010) , kRWI_W , 0 , 4 ), // #480 + INST(Fcmp_v , SimdFcmpFcmpe , (0b00011110001000000010000000000000) , kRWI_R , 0 , 0 ), // #481 + INST(Fcmpe_v , SimdFcmpFcmpe , (0b00011110001000000010000000010000) , kRWI_R , 0 , 1 ), // #482 + INST(Fcsel_v , SimdFcsel , (_) , kRWI_W , 0 , 0 ), // #483 + INST(Fcvt_v , SimdFcvt , (_) , kRWI_W , 0 , 0 ), // #484 + INST(Fcvtas_v , SimdFcvtSV , (0b0000111000100001110010, 0b0000000000000000000000, 0b0001111000100100000000, 1) , kRWI_W , 0 , 0 ), // #485 + INST(Fcvtau_v , SimdFcvtSV , (0b0010111000100001110010, 0b0000000000000000000000, 0b0001111000100101000000, 1) , kRWI_W , 0 , 1 ), // #486 + INST(Fcvtl_v , SimdFcvtLN , (0b0000111000100001011110, 0, 0) , kRWI_W , F(Long) , 0 ), // #487 + INST(Fcvtl2_v , SimdFcvtLN , (0b0100111000100001011110, 0, 0) , kRWI_W , F(Long) , 1 ), // #488 + INST(Fcvtms_v , SimdFcvtSV , (0b0000111000100001101110, 0b0000000000000000000000, 0b0001111000110000000000, 1) , kRWI_W , 0 , 2 ), // #489 + INST(Fcvtmu_v , SimdFcvtSV , (0b0010111000100001101110, 0b0000000000000000000000, 0b0001111000110001000000, 1) , kRWI_W , 0 , 3 ), // #490 + INST(Fcvtn_v , SimdFcvtLN , (0b0000111000100001011010, 0, 0) , kRWI_W , F(Narrow) , 2 ), // #491 + INST(Fcvtn2_v , SimdFcvtLN , (0b0100111000100001011010, 0, 0) , kRWI_X , F(Narrow) , 3 ), // #492 + INST(Fcvtns_v , SimdFcvtSV , (0b0000111000100001101010, 0b0000000000000000000000, 0b0001111000100000000000, 1) , kRWI_W , 0 , 4 ), // #493 + INST(Fcvtnu_v , SimdFcvtSV , (0b0010111000100001101010, 0b0000000000000000000000, 0b0001111000100001000000, 1) , kRWI_W , 0 , 5 ), // #494 + INST(Fcvtps_v , SimdFcvtSV , (0b0000111010100001101010, 0b0000000000000000000000, 0b0001111000101000000000, 1) , kRWI_W , 0 , 6 ), // #495 + INST(Fcvtpu_v , SimdFcvtSV , (0b0010111010100001101010, 0b0000000000000000000000, 0b0001111000101001000000, 1) , kRWI_W , 0 , 7 ), // #496 + INST(Fcvtxn_v , SimdFcvtLN , (0b0010111000100001011010, 1, 1) , kRWI_W , F(Narrow) , 4 ), // #497 + INST(Fcvtxn2_v , SimdFcvtLN , (0b0110111000100001011010, 1, 0) , kRWI_X , F(Narrow) , 5 ), // #498 + INST(Fcvtzs_v , SimdFcvtSV , (0b0000111010100001101110, 0b0000111100000000111111, 0b0001111000111000000000, 1) , kRWI_W , 0 , 8 ), // #499 + INST(Fcvtzu_v , SimdFcvtSV , (0b0010111010100001101110, 0b0010111100000000111111, 0b0001111000111001000000, 1) , kRWI_W , 0 , 9 ), // #500 + INST(Fdiv_v , FSimdVVV , (0b0001111000100000000110, kHF_A, 0b0010111000100000111111, kHF_C) , kRWI_W , 0 , 4 ), // #501 + INST(Fjcvtzs_v , ISimdVVx , (0b0001111001111110000000, kOp_GpW, kOp_D) , kRWI_W , 0 , 7 ), // #502 + INST(Fmadd_v , FSimdVVVV , (0b0001111100000000000000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 0 ), // #503 + INST(Fmax_v , FSimdVVV , (0b0001111000100000010010, kHF_A, 0b0000111000100000111101, kHF_C) , kRWI_W , 0 , 5 ), // #504 + INST(Fmaxnm_v , FSimdVVV , (0b0001111000100000011010, kHF_A, 0b0000111000100000110001, kHF_C) , kRWI_W , 0 , 6 ), // #505 + INST(Fmaxnmp_v , FSimdPair , (0b0111111000110000110010, 0b0010111000100000110001) , kRWI_W , 0 , 1 ), // #506 + INST(Fmaxnmv_v , FSimdSV , (0b0010111000110000110010) , kRWI_W , 0 , 0 ), // #507 + INST(Fmaxp_v , FSimdPair , (0b0111111000110000111110, 0b0010111000100000111101) , kRWI_W , 0 , 2 ), // #508 + INST(Fmaxv_v , FSimdSV , (0b0010111000110000111110) , kRWI_W , 0 , 1 ), // #509 + INST(Fmin_v , FSimdVVV , (0b0001111000100000010110, kHF_A, 0b0000111010100000111101, kHF_C) , kRWI_W , 0 , 7 ), // #510 + INST(Fminnm_v , FSimdVVV , (0b0001111000100000011110, kHF_A, 0b0000111010100000110001, kHF_C) , kRWI_W , 0 , 8 ), // #511 + INST(Fminnmp_v , FSimdPair , (0b0111111010110000110010, 0b0010111010100000110001) , kRWI_W , 0 , 3 ), // #512 + INST(Fminnmv_v , FSimdSV , (0b0010111010110000110010) , kRWI_W , 0 , 2 ), // #513 + INST(Fminp_v , FSimdPair , (0b0111111010110000111110, 0b0010111010100000111101) , kRWI_W , 0 , 4 ), // #514 + INST(Fminv_v , FSimdSV , (0b0010111010110000111110) , kRWI_W , 0 , 3 ), // #515 + INST(Fmla_v , FSimdVVVe , (0b0000000000000000000000, kHF_N, 0b0000111000100000110011, 0b0000111110000000000100) , kRWI_X , F(VH0_15) , 0 ), // #516 + INST(Fmlal_v , SimdFmlal , (0b0000111000100000111011, 0b0000111110000000000000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 2 ), // #517 + INST(Fmlal2_v , SimdFmlal , (0b0010111000100000110011, 0b0010111110000000100000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 3 ), // #518 + INST(Fmls_v , FSimdVVVe , (0b0000000000000000000000, kHF_N, 0b0000111010100000110011, 0b0000111110000000010100) , kRWI_X , F(VH0_15) , 1 ), // #519 + INST(Fmlsl_v , SimdFmlal , (0b0000111010100000111011, 0b0000111110000000010000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 4 ), // #520 + INST(Fmlsl2_v , SimdFmlal , (0b0010111010100000110011, 0b0010111110000000110000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 5 ), // #521 + INST(Fmov_v , SimdFmov , (_) , kRWI_W , 0 , 0 ), // #522 + INST(Fmsub_v , FSimdVVVV , (0b0001111100000000100000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 1 ), // #523 + INST(Fmul_v , FSimdVVVe , (0b0001111000100000000010, kHF_A, 0b0010111000100000110111, 0b0000111110000000100100) , kRWI_W , F(VH0_15) , 2 ), // #524 + INST(Fmulx_v , FSimdVVVe , (0b0101111000100000110111, kHF_C, 0b0000111000100000110111, 0b0010111110000000100100) , kRWI_W , F(VH0_15) , 3 ), // #525 + INST(Fneg_v , FSimdVV , (0b0001111000100001010000, kHF_A, 0b0010111010100000111110, kHF_B) , kRWI_W , 0 , 1 ), // #526 + INST(Fnmadd_v , FSimdVVVV , (0b0001111100100000000000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 2 ), // #527 + INST(Fnmsub_v , FSimdVVVV , (0b0001111100100000100000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 3 ), // #528 + INST(Fnmul_v , FSimdVVV , (0b0001111000100000100010, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 9 ), // #529 + INST(Frecpe_v , FSimdVV , (0b0101111010100001110110, kHF_B, 0b0000111010100001110110, kHF_B) , kRWI_W , 0 , 2 ), // #530 + INST(Frecps_v , FSimdVVV , (0b0101111000100000111111, kHF_C, 0b0000111000100000111111, kHF_C) , kRWI_W , 0 , 10 ), // #531 + INST(Frecpx_v , FSimdVV , (0b0101111010100001111110, kHF_B, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 3 ), // #532 + INST(Frint32x_v , FSimdVV , (0b0001111000101000110000, kHF_N, 0b0010111000100001111010, kHF_N) , kRWI_W , 0 , 4 ), // #533 + INST(Frint32z_v , FSimdVV , (0b0001111000101000010000, kHF_N, 0b0000111000100001111010, kHF_N) , kRWI_W , 0 , 5 ), // #534 + INST(Frint64x_v , FSimdVV , (0b0001111000101001110000, kHF_N, 0b0010111000100001111110, kHF_N) , kRWI_W , 0 , 6 ), // #535 + INST(Frint64z_v , FSimdVV , (0b0001111000101001010000, kHF_N, 0b0000111000100001111110, kHF_N) , kRWI_W , 0 , 7 ), // #536 + INST(Frinta_v , FSimdVV , (0b0001111000100110010000, kHF_A, 0b0010111000100001100010, kHF_B) , kRWI_W , 0 , 8 ), // #537 + INST(Frinti_v , FSimdVV , (0b0001111000100111110000, kHF_A, 0b0010111010100001100110, kHF_B) , kRWI_W , 0 , 9 ), // #538 + INST(Frintm_v , FSimdVV , (0b0001111000100101010000, kHF_A, 0b0000111000100001100110, kHF_B) , kRWI_W , 0 , 10 ), // #539 + INST(Frintn_v , FSimdVV , (0b0001111000100100010000, kHF_A, 0b0000111000100001100010, kHF_B) , kRWI_W , 0 , 11 ), // #540 + INST(Frintp_v , FSimdVV , (0b0001111000100100110000, kHF_A, 0b0000111010100001100010, kHF_B) , kRWI_W , 0 , 12 ), // #541 + INST(Frintx_v , FSimdVV , (0b0001111000100111010000, kHF_A, 0b0010111000100001100110, kHF_B) , kRWI_W , 0 , 13 ), // #542 + INST(Frintz_v , FSimdVV , (0b0001111000100101110000, kHF_A, 0b0000111010100001100110, kHF_B) , kRWI_W , 0 , 14 ), // #543 + INST(Frsqrte_v , FSimdVV , (0b0111111010100001110110, kHF_B, 0b0010111010100001110110, kHF_B) , kRWI_W , 0 , 15 ), // #544 + INST(Frsqrts_v , FSimdVVV , (0b0101111010100000111111, kHF_C, 0b0000111010100000111111, kHF_C) , kRWI_W , 0 , 11 ), // #545 + INST(Fsqrt_v , FSimdVV , (0b0001111000100001110000, kHF_A, 0b0010111010100001111110, kHF_B) , kRWI_W , 0 , 16 ), // #546 + INST(Fsub_v , FSimdVVV , (0b0001111000100000001110, kHF_A, 0b0000111010100000110101, kHF_C) , kRWI_W , 0 , 12 ), // #547 + INST(Ins_v , SimdIns , (_) , kRWI_X , 0 , 0 ), // #548 + INST(Ld1_v , SimdLdNStN , (0b0000110101000000000000, 0b0000110001000000001000, 1, 0) , kRWI_LDn , F(Consecutive) , 0 ), // #549 + INST(Ld1r_v , SimdLdNStN , (0b0000110101000000110000, 0b0000000000000000000000, 1, 1) , kRWI_LDn , F(Consecutive) , 1 ), // #550 + INST(Ld2_v , SimdLdNStN , (0b0000110101100000000000, 0b0000110001000000100000, 2, 0) , kRWI_LDn , F(Consecutive) , 2 ), // #551 + INST(Ld2r_v , SimdLdNStN , (0b0000110101100000110000, 0b0000000000000000000000, 2, 1) , kRWI_LDn , F(Consecutive) , 3 ), // #552 + INST(Ld3_v , SimdLdNStN , (0b0000110101000000001000, 0b0000110001000000010000, 3, 0) , kRWI_LDn , F(Consecutive) , 4 ), // #553 + INST(Ld3r_v , SimdLdNStN , (0b0000110101000000111000, 0b0000000000000000000000, 3, 1) , kRWI_LDn , F(Consecutive) , 5 ), // #554 + INST(Ld4_v , SimdLdNStN , (0b0000110101100000001000, 0b0000110001000000000000, 4, 0) , kRWI_LDn , F(Consecutive) , 6 ), // #555 + INST(Ld4r_v , SimdLdNStN , (0b0000110101100000111000, 0b0000000000000000000000, 4, 1) , kRWI_LDn , F(Consecutive) , 7 ), // #556 + INST(Ldnp_v , SimdLdpStp , (0b0010110001, 0b0000000000) , kRWI_WW , 0 , 0 ), // #557 + INST(Ldp_v , SimdLdpStp , (0b0010110101, 0b0010110011) , kRWI_WW , 0 , 1 ), // #558 + INST(Ldr_v , SimdLdSt , (0b0011110101, 0b00111100010, 0b00111100011, 0b00011100, Inst::kIdLdur_v) , kRWI_W , 0 , 0 ), // #559 + INST(Ldur_v , SimdLdurStur , (0b0011110001000000000000) , kRWI_W , 0 , 0 ), // #560 + INST(Mla_v , ISimdVVVe , (0b0000111000100000100101, kVO_V_BHS, 0b0010111100000000000000, kVO_V_HS) , kRWI_X , F(VH0_15) , 0 ), // #561 + INST(Mls_v , ISimdVVVe , (0b0010111000100000100101, kVO_V_BHS, 0b0010111100000000010000, kVO_V_HS) , kRWI_X , F(VH0_15) , 1 ), // #562 + INST(Mov_v , SimdMov , (_) , kRWI_W , 0 , 0 ), // #563 + INST(Movi_v , SimdMoviMvni , (0b0000111100000000000001, 0) , kRWI_W , 0 , 0 ), // #564 + INST(Mul_v , ISimdVVVe , (0b0000111000100000100111, kVO_V_BHS, 0b0000111100000000100000, kVO_V_HS) , kRWI_W , F(VH0_15) , 2 ), // #565 + INST(Mvn_v , ISimdVV , (0b0010111000100000010110, kVO_V_B) , kRWI_W , 0 , 4 ), // #566 + INST(Mvni_v , SimdMoviMvni , (0b0000111100000000000001, 1) , kRWI_W , 0 , 1 ), // #567 + INST(Neg_v , ISimdVV , (0b0010111000100000101110, kVO_V_Any) , kRWI_W , 0 , 5 ), // #568 + INST(Not_v , ISimdVV , (0b0010111000100000010110, kVO_V_B) , kRWI_W , 0 , 6 ), // #569 + INST(Orn_v , ISimdVVV , (0b0000111011100000000111, kVO_V_B) , kRWI_W , 0 , 9 ), // #570 + INST(Orr_v , SimdBicOrr , (0b0000111010100000000111, 0b0000111100000000000001) , kRWI_W , 0 , 1 ), // #571 + INST(Pmul_v , ISimdVVV , (0b0010111000100000100111, kVO_V_B) , kRWI_W , 0 , 10 ), // #572 + INST(Pmull_v , ISimdVVV , (0b0000111000100000111000, kVO_V_B8D1) , kRWI_W , F(Long) , 11 ), // #573 + INST(Pmull2_v , ISimdVVV , (0b0100111000100000111000, kVO_V_B16D2) , kRWI_W , F(Long) , 12 ), // #574 + INST(Raddhn_v , ISimdVVV , (0b0010111000100000010000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 13 ), // #575 + INST(Raddhn2_v , ISimdVVV , (0b0110111000100000010000, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 14 ), // #576 + INST(Rax1_v , ISimdVVV , (0b1100111001100000100011, kVO_V_D2) , kRWI_W , 0 , 15 ), // #577 + INST(Rbit_v , ISimdVV , (0b0010111001100000010110, kVO_V_B) , kRWI_W , 0 , 7 ), // #578 + INST(Rev16_v , ISimdVV , (0b0000111000100000000110, kVO_V_B) , kRWI_W , 0 , 8 ), // #579 + INST(Rev32_v , ISimdVV , (0b0010111000100000000010, kVO_V_BH) , kRWI_W , 0 , 9 ), // #580 + INST(Rev64_v , ISimdVV , (0b0000111000100000000010, kVO_V_BHS) , kRWI_W , 0 , 10 ), // #581 + INST(Rshrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100011, 1, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 0 ), // #582 + INST(Rshrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100011, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 1 ), // #583 + INST(Rsubhn_v , ISimdVVV , (0b0010111000100000011000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 16 ), // #584 + INST(Rsubhn2_v , ISimdVVV , (0b0110111000100000011000, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 17 ), // #585 + INST(Saba_v , ISimdVVV , (0b0000111000100000011111, kVO_V_BHS) , kRWI_X , 0 , 18 ), // #586 + INST(Sabal_v , ISimdVVV , (0b0000111000100000010100, kVO_V_B8H4S2) , kRWI_X , F(Long) , 19 ), // #587 + INST(Sabal2_v , ISimdVVV , (0b0100111000100000010100, kVO_V_B16H8S4) , kRWI_X , F(Long) , 20 ), // #588 + INST(Sabd_v , ISimdVVV , (0b0000111000100000011101, kVO_V_BHS) , kRWI_W , 0 , 21 ), // #589 + INST(Sabdl_v , ISimdVVV , (0b0000111000100000011100, kVO_V_B8H4S2) , kRWI_W , F(Long) , 22 ), // #590 + INST(Sabdl2_v , ISimdVVV , (0b0100111000100000011100, kVO_V_B16H8S4) , kRWI_W , F(Long) , 23 ), // #591 + INST(Sadalp_v , ISimdVV , (0b0000111000100000011010, kVO_V_BHS) , kRWI_X , F(Long) | F(Pair) , 11 ), // #592 + INST(Saddl_v , ISimdVVV , (0b0000111000100000000000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 24 ), // #593 + INST(Saddl2_v , ISimdVVV , (0b0100111000100000000000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 25 ), // #594 + INST(Saddlp_v , ISimdVV , (0b0000111000100000001010, kVO_V_BHS) , kRWI_W , F(Long) | F(Pair) , 12 ), // #595 + INST(Saddlv_v , ISimdSV , (0b0000111000110000001110, kVO_V_BH_4S) , kRWI_W , F(Long) , 1 ), // #596 + INST(Saddw_v , ISimdWWV , (0b0000111000100000000100, kVO_V_B8H4S2) , kRWI_W , 0 , 0 ), // #597 + INST(Saddw2_v , ISimdWWV , (0b0000111000100000000100, kVO_V_B16H8S4) , kRWI_W , 0 , 1 ), // #598 + INST(Scvtf_v , SimdFcvtSV , (0b0000111000100001110110, 0b0000111100000000111001, 0b0001111000100010000000, 0) , kRWI_W , 0 , 10 ), // #599 + INST(Sdot_v , SimdDot , (0b0000111010000000100101, 0b0000111110000000111000, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 1 ), // #600 + INST(Sha1c_v , ISimdVVVx , (0b0101111000000000000000, kOp_Q, kOp_S, kOp_V4S) , kRWI_X , 0 , 1 ), // #601 + INST(Sha1h_v , ISimdVVx , (0b0101111000101000000010, kOp_S, kOp_S) , kRWI_W , 0 , 8 ), // #602 + INST(Sha1m_v , ISimdVVVx , (0b0101111000000000001000, kOp_Q, kOp_S, kOp_V4S) , kRWI_X , 0 , 2 ), // #603 + INST(Sha1p_v , ISimdVVVx , (0b0101111000000000000100, kOp_Q, kOp_S, kOp_V4S) , kRWI_X , 0 , 3 ), // #604 + INST(Sha1su0_v , ISimdVVVx , (0b0101111000000000001100, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 4 ), // #605 + INST(Sha1su1_v , ISimdVVx , (0b0101111000101000000110, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 9 ), // #606 + INST(Sha256h_v , ISimdVVVx , (0b0101111000000000010000, kOp_Q, kOp_Q, kOp_V4S) , kRWI_X , 0 , 5 ), // #607 + INST(Sha256h2_v , ISimdVVVx , (0b0101111000000000010100, kOp_Q, kOp_Q, kOp_V4S) , kRWI_X , 0 , 6 ), // #608 + INST(Sha256su0_v , ISimdVVx , (0b0101111000101000001010, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 10 ), // #609 + INST(Sha256su1_v , ISimdVVVx , (0b0101111000000000011000, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 7 ), // #610 + INST(Sha512h_v , ISimdVVVx , (0b1100111001100000100000, kOp_Q, kOp_Q, kOp_V2D) , kRWI_X , 0 , 8 ), // #611 + INST(Sha512h2_v , ISimdVVVx , (0b1100111001100000100001, kOp_Q, kOp_Q, kOp_V2D) , kRWI_X , 0 , 9 ), // #612 + INST(Sha512su0_v , ISimdVVx , (0b1100111011000000100000, kOp_V2D, kOp_V2D) , kRWI_X , 0 , 11 ), // #613 + INST(Sha512su1_v , ISimdVVVx , (0b1100111001100000100010, kOp_V2D, kOp_V2D, kOp_V2D) , kRWI_X , 0 , 10 ), // #614 + INST(Shadd_v , ISimdVVV , (0b0000111000100000000001, kVO_V_BHS) , kRWI_W , 0 , 26 ), // #615 + INST(Shl_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000010101, 0, kVO_V_Any) , kRWI_W , 0 , 2 ), // #616 + INST(Shll_v , SimdShiftES , (0b0010111000100001001110, kVO_V_B8H4S2) , kRWI_W , F(Long) , 0 ), // #617 + INST(Shll2_v , SimdShiftES , (0b0110111000100001001110, kVO_V_B16H8S4) , kRWI_W , F(Long) , 1 ), // #618 + INST(Shrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100001, 1, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 3 ), // #619 + INST(Shrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100001, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 4 ), // #620 + INST(Shsub_v , ISimdVVV , (0b0000111000100000001001, kVO_V_BHS) , kRWI_W , 0 , 27 ), // #621 + INST(Sli_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000010101, 0, kVO_V_Any) , kRWI_X , 0 , 5 ), // #622 + INST(Sm3partw1_v , ISimdVVVx , (0b1100111001100000110000, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 11 ), // #623 + INST(Sm3partw2_v , ISimdVVVx , (0b1100111001100000110001, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 12 ), // #624 + INST(Sm3ss1_v , ISimdVVVVx , (0b1100111001000000000000, kOp_V4S, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_W , 0 , 0 ), // #625 + INST(Sm3tt1a_v , SimdSm3tt , (0b1100111001000000100000) , kRWI_X , 0 , 0 ), // #626 + INST(Sm3tt1b_v , SimdSm3tt , (0b1100111001000000100001) , kRWI_X , 0 , 1 ), // #627 + INST(Sm3tt2a_v , SimdSm3tt , (0b1100111001000000100010) , kRWI_X , 0 , 2 ), // #628 + INST(Sm3tt2b_v , SimdSm3tt , (0b1100111001000000100011) , kRWI_X , 0 , 3 ), // #629 + INST(Sm4e_v , ISimdVVx , (0b1100111011000000100001, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 12 ), // #630 + INST(Sm4ekey_v , ISimdVVVx , (0b1100111001100000110010, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 13 ), // #631 + INST(Smax_v , ISimdVVV , (0b0000111000100000011001, kVO_V_BHS) , kRWI_W , 0 , 28 ), // #632 + INST(Smaxp_v , ISimdVVV , (0b0000111000100000101001, kVO_V_BHS) , kRWI_W , 0 , 29 ), // #633 + INST(Smaxv_v , ISimdSV , (0b0000111000110000101010, kVO_V_BH_4S) , kRWI_W , 0 , 2 ), // #634 + INST(Smin_v , ISimdVVV , (0b0000111000100000011011, kVO_V_BHS) , kRWI_W , 0 , 30 ), // #635 + INST(Sminp_v , ISimdVVV , (0b0000111000100000101011, kVO_V_BHS) , kRWI_W , 0 , 31 ), // #636 + INST(Sminv_v , ISimdSV , (0b0000111000110001101010, kVO_V_BH_4S) , kRWI_W , 0 , 3 ), // #637 + INST(Smlal_v , ISimdVVVe , (0b0000111000100000100000, kVO_V_B8H4S2, 0b0000111100000000001000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 3 ), // #638 + INST(Smlal2_v , ISimdVVVe , (0b0100111000100000100000, kVO_V_B16H8S4, 0b0100111100000000001000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 4 ), // #639 + INST(Smlsl_v , ISimdVVVe , (0b0000111000100000101000, kVO_V_B8H4S2, 0b0000111100000000011000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 5 ), // #640 + INST(Smlsl2_v , ISimdVVVe , (0b0100111000100000101000, kVO_V_B16H8S4, 0b0100111100000000011000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 6 ), // #641 + INST(Smmla_v , ISimdVVVx , (0b0100111010000000101001, kOp_V4S, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 14 ), // #642 + INST(Smov_v , SimdSmovUmov , (0b0000111000000000001011, kVO_V_BHS, 1) , kRWI_W , 0 , 0 ), // #643 + INST(Smull_v , ISimdVVVe , (0b0000111000100000110000, kVO_V_B8H4S2, 0b0000111100000000101000, kVO_V_H4S2) , kRWI_W , F(Long) | F(VH0_15) , 7 ), // #644 + INST(Smull2_v , ISimdVVVe , (0b0100111000100000110000, kVO_V_B16H8S4, 0b0100111100000000101000, kVO_V_H8S4) , kRWI_W , F(Long) | F(VH0_15) , 8 ), // #645 + INST(Sqabs_v , ISimdVV , (0b0000111000100000011110, kVO_SV_Any) , kRWI_W , 0 , 13 ), // #646 + INST(Sqadd_v , ISimdVVV , (0b0000111000100000000011, kVO_SV_Any) , kRWI_W , 0 , 32 ), // #647 + INST(Sqdmlal_v , ISimdVVVe , (0b0000111000100000100100, kVO_SV_BHS, 0b0000111100000000001100, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 9 ), // #648 + INST(Sqdmlal2_v , ISimdVVVe , (0b0100111000100000100100, kVO_V_B16H8S4, 0b0100111100000000001100, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 10 ), // #649 + INST(Sqdmlsl_v , ISimdVVVe , (0b0000111000100000101100, kVO_SV_BHS, 0b0000111100000000011100, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 11 ), // #650 + INST(Sqdmlsl2_v , ISimdVVVe , (0b0100111000100000101100, kVO_V_B16H8S4, 0b0100111100000000011100, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 12 ), // #651 + INST(Sqdmulh_v , ISimdVVVe , (0b0000111000100000101101, kVO_SV_HS, 0b0000111100000000110000, kVO_SV_HS) , kRWI_W , F(VH0_15) , 13 ), // #652 + INST(Sqdmull_v , ISimdVVVe , (0b0000111000100000110100, kVO_SV_BHS, 0b0000111100000000101100, kVO_V_H4S2) , kRWI_W , F(Long) | F(VH0_15) , 14 ), // #653 + INST(Sqdmull2_v , ISimdVVVe , (0b0100111000100000110100, kVO_V_B16H8S4, 0b0100111100000000101100, kVO_V_H8S4) , kRWI_W , F(Long) | F(VH0_15) , 15 ), // #654 + INST(Sqneg_v , ISimdVV , (0b0010111000100000011110, kVO_SV_Any) , kRWI_W , 0 , 14 ), // #655 + INST(Sqrdmlah_v , ISimdVVVe , (0b0010111000000000100001, kVO_SV_HS, 0b0010111100000000110100, kVO_SV_HS) , kRWI_X , F(VH0_15) , 16 ), // #656 + INST(Sqrdmlsh_v , ISimdVVVe , (0b0010111000000000100011, kVO_SV_HS, 0b0010111100000000111100, kVO_SV_HS) , kRWI_X , F(VH0_15) , 17 ), // #657 + INST(Sqrdmulh_v , ISimdVVVe , (0b0010111000100000101101, kVO_SV_HS, 0b0000111100000000110100, kVO_SV_HS) , kRWI_W , F(VH0_15) , 18 ), // #658 + INST(Sqrshl_v , SimdShift , (0b0000111000100000010111, 0b0000000000000000000000, 1, kVO_SV_Any) , kRWI_W , 0 , 6 ), // #659 + INST(Sqrshrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100111, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 7 ), // #660 + INST(Sqrshrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100111, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 8 ), // #661 + INST(Sqrshrun_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100011, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 9 ), // #662 + INST(Sqrshrun2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100011, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 10 ), // #663 + INST(Sqshl_v , SimdShift , (0b0000111000100000010011, 0b0000111100000000011101, 0, kVO_SV_Any) , kRWI_W , 0 , 11 ), // #664 + INST(Sqshlu_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000011001, 0, kVO_SV_Any) , kRWI_W , 0 , 12 ), // #665 + INST(Sqshrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100101, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 13 ), // #666 + INST(Sqshrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100101, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 14 ), // #667 + INST(Sqshrun_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100001, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 15 ), // #668 + INST(Sqshrun2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100001, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 16 ), // #669 + INST(Sqsub_v , ISimdVVV , (0b0000111000100000001011, kVO_SV_Any) , kRWI_W , 0 , 33 ), // #670 + INST(Sqxtn_v , ISimdVV , (0b0000111000100001010010, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 15 ), // #671 + INST(Sqxtn2_v , ISimdVV , (0b0100111000100001010010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 16 ), // #672 + INST(Sqxtun_v , ISimdVV , (0b0010111000100001001010, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 17 ), // #673 + INST(Sqxtun2_v , ISimdVV , (0b0110111000100001001010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 18 ), // #674 + INST(Srhadd_v , ISimdVVV , (0b0000111000100000000101, kVO_V_BHS) , kRWI_W , 0 , 34 ), // #675 + INST(Sri_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000010001, 1, kVO_V_Any) , kRWI_W , 0 , 17 ), // #676 + INST(Srshl_v , SimdShift , (0b0000111000100000010101, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 18 ), // #677 + INST(Srshr_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000001001, 1, kVO_V_Any) , kRWI_W , 0 , 19 ), // #678 + INST(Srsra_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000001101, 1, kVO_V_Any) , kRWI_X , 0 , 20 ), // #679 + INST(Sshl_v , SimdShift , (0b0000111000100000010001, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 21 ), // #680 + INST(Sshll_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000101001, 0, kVO_V_B8H4S2) , kRWI_W , F(Long) , 22 ), // #681 + INST(Sshll2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000101001, 0, kVO_V_B16H8S4) , kRWI_W , F(Long) , 23 ), // #682 + INST(Sshr_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000000001, 1, kVO_V_Any) , kRWI_W , 0 , 24 ), // #683 + INST(Ssra_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000000101, 1, kVO_V_Any) , kRWI_X , 0 , 25 ), // #684 + INST(Ssubl_v , ISimdVVV , (0b0000111000100000001000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 35 ), // #685 + INST(Ssubl2_v , ISimdVVV , (0b0100111000100000001000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 36 ), // #686 + INST(Ssubw_v , ISimdWWV , (0b0000111000100000001100, kVO_V_B8H4S2) , kRWI_W , 0 , 2 ), // #687 + INST(Ssubw2_v , ISimdWWV , (0b0000111000100000001100, kVO_V_B16H8S4) , kRWI_X , 0 , 3 ), // #688 + INST(St1_v , SimdLdNStN , (0b0000110100000000000000, 0b0000110000000000001000, 1, 0) , kRWI_STn , F(Consecutive) , 8 ), // #689 + INST(St2_v , SimdLdNStN , (0b0000110100100000000000, 0b0000110000000000100000, 2, 0) , kRWI_STn , F(Consecutive) , 9 ), // #690 + INST(St3_v , SimdLdNStN , (0b0000110100000000001000, 0b0000110000000000010000, 3, 0) , kRWI_STn , F(Consecutive) , 10 ), // #691 + INST(St4_v , SimdLdNStN , (0b0000110100100000001000, 0b0000110000000000000000, 4, 0) , kRWI_STn , F(Consecutive) , 11 ), // #692 + INST(Stnp_v , SimdLdpStp , (0b0010110000, 0b0000000000) , kRWI_RRW , 0 , 2 ), // #693 + INST(Stp_v , SimdLdpStp , (0b0010110100, 0b0010110010) , kRWI_RRW , 0 , 3 ), // #694 + INST(Str_v , SimdLdSt , (0b0011110100, 0b00111100000, 0b00111100001, 0b00000000, Inst::kIdStur_v) , kRWI_RW , 0 , 1 ), // #695 + INST(Stur_v , SimdLdurStur , (0b0011110000000000000000) , kRWI_RW , 0 , 1 ), // #696 + INST(Sub_v , ISimdVVV , (0b0010111000100000100001, kVO_V_Any) , kRWI_W , 0 , 37 ), // #697 + INST(Subhn_v , ISimdVVV , (0b0000111000100000011000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 38 ), // #698 + INST(Subhn2_v , ISimdVVV , (0b0000111000100000011000, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 39 ), // #699 + INST(Sudot_v , SimdDot , (0b0000000000000000000000, 0b0000111100000000111100, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 2 ), // #700 + INST(Suqadd_v , ISimdVV , (0b0000111000100000001110, kVO_SV_Any) , kRWI_X , 0 , 19 ), // #701 + INST(Sxtl_v , SimdSxtlUxtl , (0b0000111100000000101001, kVO_V_B8H4S2) , kRWI_W , F(Long) , 0 ), // #702 + INST(Sxtl2_v , SimdSxtlUxtl , (0b0100111100000000101001, kVO_V_B16H8S4) , kRWI_W , F(Long) , 1 ), // #703 + INST(Tbl_v , SimdTblTbx , (0b0000111000000000000000) , kRWI_W , 0 , 0 ), // #704 + INST(Tbx_v , SimdTblTbx , (0b0000111000000000000100) , kRWI_W , 0 , 1 ), // #705 + INST(Trn1_v , ISimdVVV , (0b0000111000000000001010, kVO_V_BHS_D2) , kRWI_W , 0 , 40 ), // #706 + INST(Trn2_v , ISimdVVV , (0b0000111000000000011010, kVO_V_BHS_D2) , kRWI_W , 0 , 41 ), // #707 + INST(Uaba_v , ISimdVVV , (0b0010111000100000011111, kVO_V_BHS) , kRWI_X , 0 , 42 ), // #708 + INST(Uabal_v , ISimdVVV , (0b0010111000100000010100, kVO_V_B8H4S2) , kRWI_X , F(Long) , 43 ), // #709 + INST(Uabal2_v , ISimdVVV , (0b0110111000100000010100, kVO_V_B16H8S4) , kRWI_X , F(Long) , 44 ), // #710 + INST(Uabd_v , ISimdVVV , (0b0010111000100000011101, kVO_V_BHS) , kRWI_W , 0 , 45 ), // #711 + INST(Uabdl_v , ISimdVVV , (0b0010111000100000011100, kVO_V_B8H4S2) , kRWI_W , F(Long) , 46 ), // #712 + INST(Uabdl2_v , ISimdVVV , (0b0110111000100000011100, kVO_V_B16H8S4) , kRWI_W , F(Long) , 47 ), // #713 + INST(Uadalp_v , ISimdVV , (0b0010111000100000011010, kVO_V_BHS) , kRWI_X , F(Long) | F(Pair) , 20 ), // #714 + INST(Uaddl_v , ISimdVVV , (0b0010111000100000000000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 48 ), // #715 + INST(Uaddl2_v , ISimdVVV , (0b0110111000100000000000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 49 ), // #716 + INST(Uaddlp_v , ISimdVV , (0b0010111000100000001010, kVO_V_BHS) , kRWI_W , F(Long) | F(Pair) , 21 ), // #717 + INST(Uaddlv_v , ISimdSV , (0b0010111000110000001110, kVO_V_BH_4S) , kRWI_W , F(Long) , 4 ), // #718 + INST(Uaddw_v , ISimdWWV , (0b0010111000100000000100, kVO_V_B8H4S2) , kRWI_W , 0 , 4 ), // #719 + INST(Uaddw2_v , ISimdWWV , (0b0010111000100000000100, kVO_V_B16H8S4) , kRWI_W , 0 , 5 ), // #720 + INST(Ucvtf_v , SimdFcvtSV , (0b0010111000100001110110, 0b0010111100000000111001, 0b0001111000100011000000, 0) , kRWI_W , 0 , 11 ), // #721 + INST(Udot_v , SimdDot , (0b0010111010000000100101, 0b0010111110000000111000, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 3 ), // #722 + INST(Uhadd_v , ISimdVVV , (0b0010111000100000000001, kVO_V_BHS) , kRWI_W , 0 , 50 ), // #723 + INST(Uhsub_v , ISimdVVV , (0b0010111000100000001001, kVO_V_BHS) , kRWI_W , 0 , 51 ), // #724 + INST(Umax_v , ISimdVVV , (0b0010111000100000011001, kVO_V_BHS) , kRWI_W , 0 , 52 ), // #725 + INST(Umaxp_v , ISimdVVV , (0b0010111000100000101001, kVO_V_BHS) , kRWI_W , 0 , 53 ), // #726 + INST(Umaxv_v , ISimdSV , (0b0010111000110000101010, kVO_V_BH_4S) , kRWI_W , 0 , 5 ), // #727 + INST(Umin_v , ISimdVVV , (0b0010111000100000011011, kVO_V_BHS) , kRWI_W , 0 , 54 ), // #728 + INST(Uminp_v , ISimdVVV , (0b0010111000100000101011, kVO_V_BHS) , kRWI_W , 0 , 55 ), // #729 + INST(Uminv_v , ISimdSV , (0b0010111000110001101010, kVO_V_BH_4S) , kRWI_W , 0 , 6 ), // #730 + INST(Umlal_v , ISimdVVVe , (0b0010111000100000100000, kVO_V_B8H4S2, 0b0010111100000000001000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 19 ), // #731 + INST(Umlal2_v , ISimdVVVe , (0b0110111000100000100000, kVO_V_B16H8S4, 0b0010111100000000001000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 20 ), // #732 + INST(Umlsl_v , ISimdVVVe , (0b0010111000100000101000, kVO_V_B8H4S2, 0b0010111100000000011000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 21 ), // #733 + INST(Umlsl2_v , ISimdVVVe , (0b0110111000100000101000, kVO_V_B16H8S4, 0b0110111100000000011000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 22 ), // #734 + INST(Ummla_v , ISimdVVVx , (0b0110111010000000101001, kOp_V4S, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 15 ), // #735 + INST(Umov_v , SimdSmovUmov , (0b0000111000000000001111, kVO_V_Any, 0) , kRWI_W , 0 , 1 ), // #736 + INST(Umull_v , ISimdVVVe , (0b0010111000100000110000, kVO_V_B8H4S2, 0b0010111100000000101000, kVO_V_H4S2) , kRWI_W , F(Long) | F(VH0_15) , 23 ), // #737 + INST(Umull2_v , ISimdVVVe , (0b0110111000100000110000, kVO_V_B16H8S4, 0b0110111100000000101000, kVO_V_H8S4) , kRWI_W , F(Long) | F(VH0_15) , 24 ), // #738 + INST(Uqadd_v , ISimdVVV , (0b0010111000100000000011, kVO_SV_Any) , kRWI_W , 0 , 56 ), // #739 + INST(Uqrshl_v , SimdShift , (0b0010111000100000010111, 0b0000000000000000000000, 0, kVO_SV_Any) , kRWI_W , 0 , 26 ), // #740 + INST(Uqrshrn_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100111, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 27 ), // #741 + INST(Uqrshrn2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100111, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 28 ), // #742 + INST(Uqshl_v , SimdShift , (0b0010111000100000010011, 0b0010111100000000011101, 0, kVO_SV_Any) , kRWI_W , 0 , 29 ), // #743 + INST(Uqshrn_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100101, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 30 ), // #744 + INST(Uqshrn2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100101, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 31 ), // #745 + INST(Uqsub_v , ISimdVVV , (0b0010111000100000001011, kVO_SV_Any) , kRWI_W , 0 , 57 ), // #746 + INST(Uqxtn_v , ISimdVV , (0b0010111000100001010010, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 22 ), // #747 + INST(Uqxtn2_v , ISimdVV , (0b0110111000100001010010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 23 ), // #748 + INST(Urecpe_v , ISimdVV , (0b0000111010100001110010, kVO_V_S) , kRWI_W , 0 , 24 ), // #749 + INST(Urhadd_v , ISimdVVV , (0b0010111000100000000101, kVO_V_BHS) , kRWI_W , 0 , 58 ), // #750 + INST(Urshl_v , SimdShift , (0b0010111000100000010101, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 32 ), // #751 + INST(Urshr_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000001001, 1, kVO_V_Any) , kRWI_W , 0 , 33 ), // #752 + INST(Ursqrte_v , ISimdVV , (0b0010111010100001110010, kVO_V_S) , kRWI_W , 0 , 25 ), // #753 + INST(Ursra_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000001101, 1, kVO_V_Any) , kRWI_X , 0 , 34 ), // #754 + INST(Usdot_v , SimdDot , (0b0000111010000000100111, 0b0000111110000000111100, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 4 ), // #755 + INST(Ushl_v , SimdShift , (0b0010111000100000010001, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 35 ), // #756 + INST(Ushll_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000101001, 0, kVO_V_B8H4S2) , kRWI_W , F(Long) , 36 ), // #757 + INST(Ushll2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000101001, 0, kVO_V_B16H8S4) , kRWI_W , F(Long) , 37 ), // #758 + INST(Ushr_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000000001, 1, kVO_V_Any) , kRWI_W , 0 , 38 ), // #759 + INST(Usmmla_v , ISimdVVVx , (0b0100111010000000101011, kOp_V4S, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 16 ), // #760 + INST(Usqadd_v , ISimdVV , (0b0010111000100000001110, kVO_SV_Any) , kRWI_X , 0 , 26 ), // #761 + INST(Usra_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000000101, 1, kVO_V_Any) , kRWI_X , 0 , 39 ), // #762 + INST(Usubl_v , ISimdVVV , (0b0010111000100000001000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 59 ), // #763 + INST(Usubl2_v , ISimdVVV , (0b0110111000100000001000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 60 ), // #764 + INST(Usubw_v , ISimdWWV , (0b0010111000100000001100, kVO_V_B8H4S2) , kRWI_W , 0 , 6 ), // #765 + INST(Usubw2_v , ISimdWWV , (0b0010111000100000001100, kVO_V_B16H8S4) , kRWI_W , 0 , 7 ), // #766 + INST(Uxtl_v , SimdSxtlUxtl , (0b0010111100000000101001, kVO_V_B8H4S2) , kRWI_W , F(Long) , 2 ), // #767 + INST(Uxtl2_v , SimdSxtlUxtl , (0b0110111100000000101001, kVO_V_B16H8S4) , kRWI_W , F(Long) , 3 ), // #768 + INST(Uzp1_v , ISimdVVV , (0b0000111000000000000110, kVO_V_BHS_D2) , kRWI_W , 0 , 61 ), // #769 + INST(Uzp2_v , ISimdVVV , (0b0000111000000000010110, kVO_V_BHS_D2) , kRWI_W , 0 , 62 ), // #770 + INST(Xar_v , ISimdVVVI , (0b1100111001100000100011, kVO_V_D2, 6, 10, 0) , kRWI_W , 0 , 1 ), // #771 + INST(Xtn_v , ISimdVV , (0b0000111000100001001010, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 27 ), // #772 + INST(Xtn2_v , ISimdVV , (0b0100111000100001001010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 28 ), // #773 + INST(Zip1_v , ISimdVVV , (0b0000111000000000001110, kVO_V_BHS_D2) , kRWI_W , 0 , 63 ), // #774 + INST(Zip2_v , ISimdVVV , (0b0000111000000000011110, kVO_V_BHS_D2) , kRWI_W , 0 , 64 ) // #775 // ${InstInfo:End} }; @@ -1070,8 +1078,9 @@ const BaseBranchReg baseBranchReg[3] = { { 0b11010110010111110000000000000000 } // ret }; -const BaseBranchRel baseBranchRel[2] = { +const BaseBranchRel baseBranchRel[3] = { { 0b00010100000000000000000000000000 }, // b + { 0b00010100000000000000000000010000 }, // bc { 0b10010100000000000000000000000000 } // bl }; @@ -1157,6 +1166,13 @@ const BaseLogical baseLogical[8] = { { 0b0101010000, 0b01100100, 0 } // orr }; +const BaseMinMax baseMinMax[4] = { + { 0b00011010110000000110000000000000, 0b00010001110000000000000000000000 }, // smax + { 0b00011010110000000110100000000000, 0b00010001110010000000000000000000 }, // smin + { 0b00011010110000000110010000000000, 0b00010001110001000000000000000000 }, // umax + { 0b00011010110000000110110000000000, 0b00010001110011000000000000000000 } // umin +}; + const BaseMovKNZ baseMovKNZ[3] = { { 0b01110010100000000000000000000000 }, // movk { 0b00010010100000000000000000000000 }, // movn @@ -1169,7 +1185,7 @@ const BaseMvnNeg baseMvnNeg[3] = { { 0b01101011000000000000001111100000 } // negs }; -const BaseOp baseOp[23] = { +const BaseOp baseOp[24] = { { 0b11010101000000110010000110011111 }, // autia1716 { 0b11010101000000110010001110111111 }, // autiasp { 0b11010101000000110010001110011111 }, // autiaz @@ -1178,6 +1194,7 @@ const BaseOp baseOp[23] = { { 0b11010101000000110010001111011111 }, // autibz { 0b11010101000000000100000001011111 }, // axflag { 0b11010101000000000100000000011111 }, // cfinv + { 0b11010101000000110010001011011111 }, // clrbhb { 0b11010101000000110010001010011111 }, // csdb { 0b11010101000000110010000011011111 }, // dgh { 0b11010110101111110000001111100000 }, // drps @@ -1195,8 +1212,9 @@ const BaseOp baseOp[23] = { { 0b11010101000000110010000000111111 } // yield }; -const BaseOpImm baseOpImm[14] = { +const BaseOpImm baseOpImm[15] = { { 0b11010100001000000000000000000000, 16, 5 }, // brk + { 0b11010101000000110010010000011111, 2, 6 }, // bti { 0b11010101000000110011000001011111, 4, 8 }, // clrex { 0b11010100101000000000000000000001, 16, 5 }, // dcps1 { 0b11010100101000000000000000000010, 16, 5 }, // dcps2 @@ -1212,6 +1230,10 @@ const BaseOpImm baseOpImm[14] = { { 0b00000000000000000000000000000000, 16, 0 } // udf }; +const BaseOpX16 baseOpX16[1] = { + { 0b11010101000000110010010100011111 } // chkfeat +}; + const BasePrfm basePrfm[1] = { { 0b11111000101, 0b1111100110, 0b11111000100, 0b11011000 } // prfm }; @@ -1284,7 +1306,8 @@ const BaseRM_SImm9 baseRM_SImm9[23] = { { 0b1101100101100000000010, 0b1101100101100000000001, kX , kSP, 0, 4 } // stzg }; -const BaseRR baseRR[15] = { +const BaseRR baseRR[18] = { + { 0b01011010110000000010000000000000, kWX, kZR, 0, kWX, kZR, 5, true }, // abs { 0b11011010110000010001100000000000, kX, kZR, 0, kX, kSP, 5, true }, // autda { 0b11011010110000010001110000000000, kX, kZR, 0, kX, kSP, 5, true }, // autdb { 0b11011010110000010001000000000000, kX, kZR, 0, kX, kSP, 5, true }, // autia @@ -1292,6 +1315,8 @@ const BaseRR baseRR[15] = { { 0b01011010110000000001010000000000, kWX, kZR, 0, kWX, kZR, 5, true }, // cls { 0b01011010110000000001000000000000, kWX, kZR, 0, kWX, kZR, 5, true }, // clz { 0b10111010110000000000000000011111, kX, kSP, 5, kX, kSP, 16, true }, // cmpp + { 0b01011010110000000001110000000000, kWX, kZR, 0, kWX, kZR, 5, true }, // cnt + { 0b01011010110000000001100000000000, kWX, kZR, 0, kWX, kZR, 5, true }, // ctz { 0b01011010000000000000001111100000, kWX, kZR, 0, kWX, kZR, 16, true }, // ngc { 0b01111010000000000000001111100000, kWX, kZR, 0, kWX, kZR, 16, true }, // ngcs { 0b11011010110000010000100000000000, kX, kZR, 0, kX, kSP, 5, true }, // pacda @@ -1852,7 +1877,7 @@ const InstDB::CommonInfo InstDB::commonData[] = { // ${NameData:Begin} // ------------------- Automatically generated, do not edit ------------------- const InstNameIndex InstDB::instNameIndex = {{ - { Inst::kIdAdc , Inst::kIdAnd_v + 1 }, + { Inst::kIdAbs , Inst::kIdAnd_v + 1 }, { Inst::kIdB , Inst::kIdBsl_v + 1 }, { Inst::kIdCas , Inst::kIdCnt_v + 1 }, { Inst::kIdDc , Inst::kIdDup_v + 1 }, @@ -1881,14 +1906,23 @@ const InstNameIndex InstDB::instNameIndex = {{ }, uint16_t(9)}; const char InstDB::_instNameStringTable[] = - "autia1716autibldsmaxalhldsminalldumaxallduminalsha256su0sha512su1sm3partwsqrshru" - "nldaddalldclralldeoralldsetallbstsmaxstsminstumaxstuminfrint32z64x64zh2sqdmlalsl" - "2sqdmulsqrdmlaulhn2sqshruuqrshrspcrc32cstaddstclrsteorstsetxpaclbfcvtbfmlaltfcvt" - "xfjcvtzfmaxnmfminnmfrsqrraddrsubsha1sm3tt12a2bsm4ekeysqxtuuqshrursqrsetfrev8"; + "\x61\x75\x74\x69\x61\x31\x37\x31\x36\x61\x75\x74\x69\x62\x6C\x64\x73\x6D\x61\x78\x61\x6C\x68\x6C\x64\x73\x6D\x69\x6E" + "\x61\x6C\x6C\x64\x75\x6D\x61\x78\x61\x6C\x6C\x64\x75\x6D\x69\x6E\x61\x6C\x73\x68\x61\x32\x35\x36\x73\x75\x30\x73\x68" + "\x61\x35\x31\x32\x73\x75\x31\x73\x6D\x33\x70\x61\x72\x74\x77\x73\x71\x72\x73\x68\x72\x75\x6E\x6C\x64\x61\x64\x64\x61" + "\x6C\x6C\x64\x63\x6C\x72\x61\x6C\x6C\x64\x65\x6F\x72\x61\x6C\x6C\x64\x73\x65\x74\x61\x6C\x6C\x62\x73\x74\x73\x6D\x61" + "\x78\x73\x74\x73\x6D\x69\x6E\x73\x74\x75\x6D\x61\x78\x73\x74\x75\x6D\x69\x6E\x66\x72\x69\x6E\x74\x33\x32\x7A\x36\x34" + "\x78\x36\x34\x7A\x68\x32\x73\x71\x64\x6D\x6C\x61\x6C\x73\x6C\x32\x73\x71\x64\x6D\x75\x6C\x73\x71\x72\x64\x6D\x6C\x61" + "\x75\x6C\x68\x6E\x32\x73\x71\x73\x68\x72\x75\x75\x71\x72\x73\x68\x72\x73\x70\x63\x68\x6B\x66\x65\x61\x63\x72\x63\x33" + "\x32\x63\x73\x74\x61\x64\x64\x73\x74\x63\x6C\x72\x73\x74\x65\x6F\x72\x73\x74\x73\x65\x74\x78\x70\x61\x63\x6C\x62\x66" + "\x63\x76\x74\x62\x66\x6D\x6C\x61\x6C\x74\x66\x63\x76\x74\x78\x66\x6A\x63\x76\x74\x7A\x66\x6D\x61\x78\x6E\x6D\x66\x6D" + "\x69\x6E\x6E\x6D\x66\x72\x73\x71\x72\x72\x61\x64\x64\x72\x73\x75\x62\x73\x68\x61\x31\x73\x6D\x33\x74\x74\x31\x32\x61" + "\x32\x62\x73\x6D\x34\x65\x6B\x65\x79\x73\x71\x78\x74\x75\x75\x71\x73\x68\x72\x75\x72\x73\x71\x72\x73\x65\x74\x66\x72" + "\x65\x76\x38"; const uint32_t InstDB::_instNameIndexTable[] = { 0x80000000, // Small ''. + 0x80004C41, // Small 'abs'. 0x80000C81, // Small 'adc'. 0x80098C81, // Small 'adcs'. 0x80001081, // Small 'add'. @@ -1917,6 +1951,7 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x85A4D2A1, // Small 'autizb'. 0x8E161B01, // Small 'axflag'. 0x80000002, // Small 'b'. + 0x80000062, // Small 'bc'. 0x80000CC2, // Small 'bfc'. 0x800024C2, // Small 'bfi'. 0x800034C2, // Small 'bfm'. @@ -1927,6 +1962,7 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80004982, // Small 'blr'. 0x80000242, // Small 'br'. 0x80002E42, // Small 'brk'. + 0x80002682, // Small 'bti'. 0x80004C23, // Small 'cas'. 0x8000CC23, // Small 'casa'. 0x8020CC23, // Small 'casab'. @@ -1948,8 +1984,10 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80073463, // Small 'ccmn'. 0x80083463, // Small 'ccmp'. 0x816724C3, // Small 'cfinv'. + 0x100260C1, // Large 'chkfea|t'. 0x8001B923, // Small 'cinc'. 0x800B3923, // Small 'cinv'. + 0x84814983, // Small 'clrbhb'. 0x8182C983, // Small 'clrex'. 0x80004D83, // Small 'cls'. 0x80006983, // Small 'clz'. @@ -1957,11 +1995,12 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x800041A3, // Small 'cmp'. 0x800841A3, // Small 'cmpp'. 0x800395C3, // Small 'cneg'. + 0x800051C3, // Small 'cnt'. 0x85DF0E43, // Small 'crc32b'. - 0x100D60C1, // Large 'crc32c|b'. - 0x101660C1, // Large 'crc32c|h'. - 0x104860C1, // Large 'crc32c|w'. - 0x101360C1, // Large 'crc32c|x'. + 0x100D60C7, // Large 'crc32c|b'. + 0x101660C7, // Large 'crc32c|h'. + 0x104860C7, // Large 'crc32c|w'. + 0x101360C7, // Large 'crc32c|x'. 0x91DF0E43, // Small 'crc32h'. 0xAFDF0E43, // Small 'crc32w'. 0xB1DF0E43, // Small 'crc32x'. @@ -1972,6 +2011,7 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80372663, // Small 'csinc'. 0x81672663, // Small 'csinv'. 0x8072BA63, // Small 'csneg'. + 0x80006A83, // Small 'ctz'. 0x80000064, // Small 'dc'. 0x81C9C064, // Small 'dcps1'. 0x81D9C064, // Small 'dcps2'. @@ -2158,9 +2198,9 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x800A2452, // Small 'rbit'. 0x800050B2, // Small 'ret'. 0x800058B2, // Small 'rev'. - 0x20073138, // Large 'rev|16'. + 0x2007313E, // Large 'rev|16'. 0x81DF58B2, // Small 'rev32'. - 0x208F3138, // Large 'rev|64'. + 0x208F313E, // Large 'rev|64'. 0x800049F2, // Small 'ror'. 0x800B49F2, // Small 'rorv'. 0x80000C53, // Small 'sbc'. @@ -2169,12 +2209,14 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80069853, // Small 'sbfm'. 0x800C1853, // Small 'sbfx'. 0x800B2493, // Small 'sdiv'. - 0x113B4134, // Large 'setf|8'. - 0x20074134, // Large 'setf|16'. + 0x1141413A, // Large 'setf|8'. + 0x2007413A, // Large 'setf|16'. 0x800058B3, // Small 'sev'. 0x800658B3, // Small 'sevl'. 0x984205B3, // Small 'smaddl'. + 0x800C05B3, // Small 'smax'. 0x80000DB3, // Small 'smc'. + 0x800725B3, // Small 'smin'. 0x9872B9B3, // Small 'smnegl'. 0x982ACDB3, // Small 'smsubl'. 0x808655B3, // Small 'smulh'. @@ -2184,21 +2226,21 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80420693, // Small 'stadd'. 0x98420693, // Small 'staddl'. 0x84420693, // Small 'staddb'. - 0x206D50C7, // Large 'stadd|lb'. + 0x206D50CD, // Large 'stadd|lb'. 0x90420693, // Small 'staddh'. - 0x201550C7, // Large 'stadd|lh'. + 0x201550CD, // Large 'stadd|lh'. 0x81260E93, // Small 'stclr'. 0x99260E93, // Small 'stclrl'. 0x85260E93, // Small 'stclrb'. - 0x206D50CC, // Large 'stclr|lb'. + 0x206D50D2, // Large 'stclr|lb'. 0x91260E93, // Small 'stclrh'. - 0x201550CC, // Large 'stclr|lh'. + 0x201550D2, // Large 'stclr|lh'. 0x81279693, // Small 'steor'. 0x99279693, // Small 'steorl'. 0x85279693, // Small 'steorb'. - 0x206D50D1, // Large 'steor|lb'. + 0x206D50D7, // Large 'steor|lb'. 0x91279693, // Small 'steorh'. - 0x201550D1, // Large 'steor|lh'. + 0x201550D7, // Large 'steor|lh'. 0x80001E93, // Small 'stg'. 0x80069E93, // Small 'stgm'. 0x80081E93, // Small 'stgp'. @@ -2220,9 +2262,9 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x8142CE93, // Small 'stset'. 0x9942CE93, // Small 'stsetl'. 0x8542CE93, // Small 'stsetb'. - 0x206D50D6, // Large 'stset|lb'. + 0x206D50DC, // Large 'stset|lb'. 0x9142CE93, // Small 'stseth'. - 0x201550D6, // Large 'stset|lh'. + 0x201550DC, // Large 'stset|lh'. 0xB016CE93, // Small 'stsmax'. 0x100E606F, // Large 'stsmax|l'. 0x100D606F, // Large 'stsmax|b'. @@ -2292,6 +2334,8 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80001895, // Small 'udf'. 0x800B2495, // Small 'udiv'. 0x984205B5, // Small 'umaddl'. + 0x800C05B5, // Small 'umax'. + 0x800725B5, // Small 'umin'. 0x9872B9B5, // Small 'umnegl'. 0x80C655B5, // Small 'umull'. 0x808655B5, // Small 'umulh'. @@ -2303,7 +2347,7 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x8E161838, // Small 'xaflag'. 0x80418618, // Small 'xpacd'. 0x80918618, // Small 'xpaci'. - 0x208850DB, // Large 'xpacl|ri'. + 0x208850E1, // Large 'xpacl|ri'. 0x80461539, // Small 'yield'. 0x80004C41, // Small 'abs'. 0x80001081, // Small 'add'. @@ -2319,10 +2363,10 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x800C0462, // Small 'bcax'. 0x814B0CC2, // Small 'bfcvt'. 0x9D4B0CC2, // Small 'bfcvtn'. - 0x20B150E0, // Large 'bfcvt|n2'. + 0x20B150E6, // Large 'bfcvt|n2'. 0x814790C2, // Small 'bfdot'. - 0x206D50E5, // Large 'bfmla|lb'. - 0x20EA50E5, // Large 'bfmla|lt'. + 0x206D50EB, // Large 'bfmla|lb'. + 0x20F050EB, // Large 'bfmla|lt'. 0x82C6B4C2, // Small 'bfmmla'. 0x80000D22, // Small 'bic'. 0x80001922, // Small 'bif'. @@ -2375,22 +2419,22 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0xA70A5866, // Small 'fcvtps'. 0xAB0A5866, // Small 'fcvtpu'. 0x9D8A5866, // Small 'fcvtxn'. - 0x20B150EC, // Large 'fcvtx|n2'. + 0x20B150F2, // Large 'fcvtx|n2'. 0xA7AA5866, // Small 'fcvtzs'. 0xABAA5866, // Small 'fcvtzu'. 0x800B2486, // Small 'fdiv'. - 0x101060F1, // Large 'fjcvtz|s'. + 0x101060F7, // Large 'fjcvtz|s'. 0x804205A6, // Small 'fmadd'. 0x800C05A6, // Small 'fmax'. 0x9AEC05A6, // Small 'fmaxnm'. - 0x104460F7, // Large 'fmaxnm|p'. - 0x10E360F7, // Large 'fmaxnm|v'. + 0x104460FD, // Large 'fmaxnm|p'. + 0x10E960FD, // Large 'fmaxnm|v'. 0x810C05A6, // Small 'fmaxp'. 0x816C05A6, // Small 'fmaxv'. 0x800725A6, // Small 'fmin'. 0x9AE725A6, // Small 'fminnm'. - 0x104460FD, // Large 'fminnm|p'. - 0x10E360FD, // Large 'fminnm|v'. + 0x10446103, // Large 'fminnm|p'. + 0x10E96103, // Large 'fminnm|v'. 0x810725A6, // Small 'fminp'. 0x816725A6, // Small 'fminv'. 0x8000B1A6, // Small 'fmla'. @@ -2421,8 +2465,8 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0xA1472646, // Small 'frintp'. 0xB1472646, // Small 'frintx'. 0xB5472646, // Small 'frintz'. - 0x20D25103, // Large 'frsqr|te'. - 0x20705103, // Large 'frsqr|ts'. + 0x20D85109, // Large 'frsqr|te'. + 0x20705109, // Large 'frsqr|ts'. 0x81494666, // Small 'fsqrt'. 0x80015666, // Small 'fsub'. 0x80004DC9, // Small 'ins'. @@ -2453,16 +2497,16 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80C655B0, // Small 'pmull'. 0xBAC655B0, // Small 'pmull2'. 0x9C821032, // Small 'raddhn'. - 0x30B04108, // Large 'radd|hn2'. + 0x30B0410E, // Large 'radd|hn2'. 0x800E6032, // Small 'rax1'. 0x800A2452, // Small 'rbit'. - 0x20073138, // Large 'rev|16'. + 0x2007313E, // Large 'rev|16'. 0x81DF58B2, // Small 'rev32'. - 0x208F3138, // Large 'rev|64'. + 0x208F313E, // Large 'rev|64'. 0x80E92272, // Small 'rshrn'. 0xBAE92272, // Small 'rshrn2'. 0x9C815672, // Small 'rsubhn'. - 0x30B0410C, // Large 'rsub|hn2'. + 0x30B04112, // Large 'rsub|hn2'. 0x80008833, // Small 'saba'. 0x80C08833, // Small 'sabal'. 0xBAC08833, // Small 'sabal2'. @@ -2482,8 +2526,8 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x808E0513, // Small 'sha1h'. 0x80DE0513, // Small 'sha1m'. 0x810E0513, // Small 'sha1p'. - 0x30354110, // Large 'sha1|su0'. - 0x303E4110, // Large 'sha1|su1'. + 0x30354116, // Large 'sha1|su0'. + 0x303E4116, // Large 'sha1|su1'. 0x1016602F, // Large 'sha256|h'. 0x2095602F, // Large 'sha256|h2'. 0x0000902F, // Large 'sha256su0'. @@ -2503,12 +2547,12 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x10058041, // Large 'sm3partw|1'. 0x10328041, // Large 'sm3partw|2'. 0xB939F9B3, // Small 'sm3ss1'. - 0x10006114, // Large 'sm3tt1|a'. - 0x100D6114, // Large 'sm3tt1|b'. - 0x211A5114, // Large 'sm3tt|2a'. - 0x211C5114, // Large 'sm3tt|2b'. + 0x1000611A, // Large 'sm3tt1|a'. + 0x100D611A, // Large 'sm3tt1|b'. + 0x2120511A, // Large 'sm3tt|2a'. + 0x2122511A, // Large 'sm3tt|2b'. 0x8002FDB3, // Small 'sm4e'. - 0x0000711E, // Large 'sm4ekey'. + 0x00007124, // Large 'sm4ekey'. 0x800C05B3, // Small 'smax'. 0x810C05B3, // Small 'smaxp'. 0x816C05B3, // Small 'smaxv'. @@ -2551,7 +2595,7 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80EA6233, // Small 'sqxtn'. 0xBAEA6233, // Small 'sqxtn2'. 0x9D5A6233, // Small 'sqxtun'. - 0x20B15125, // Large 'sqxtu|n2'. + 0x20B1512B, // Large 'sqxtu|n2'. 0x8840A253, // Small 'srhadd'. 0x80002653, // Small 'sri'. 0x80C44E53, // Small 'srshl'. @@ -2622,7 +2666,7 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x20B160B9, // Large 'uqrshr|n2'. 0x80C44E35, // Small 'uqshl'. 0x9D244E35, // Small 'uqshrn'. - 0x20B1512A, // Large 'uqshr|n2'. + 0x20B15130, // Large 'uqshr|n2'. 0x802ACE35, // Small 'uqsub'. 0x80EA6235, // Small 'uqxtn'. 0xBAEA6235, // Small 'uqxtn2'. @@ -2630,7 +2674,7 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x8840A255, // Small 'urhadd'. 0x80C44E55, // Small 'urshl'. 0x81244E55, // Small 'urshr'. - 0x20D2512F, // Large 'ursqr|te'. + 0x20D85135, // Large 'ursqr|te'. 0x80194E55, // Small 'ursra'. 0x81479275, // Small 'usdot'. 0x80062275, // Small 'ushl'. diff --git a/src/asmjit/arm/a64instdb.h b/src/asmjit/arm/a64instdb.h index 89773ee..50e4178 100644 --- a/src/asmjit/arm/a64instdb.h +++ b/src/asmjit/arm/a64instdb.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64INSTDB_H_INCLUDED diff --git a/src/asmjit/arm/a64instdb_p.h b/src/asmjit/arm/a64instdb_p.h index 322e5a8..2c971e1 100644 --- a/src/asmjit/arm/a64instdb_p.h +++ b/src/asmjit/arm/a64instdb_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64INSTDB_H_P_INCLUDED @@ -82,23 +82,22 @@ enum GpType : uint8_t { // =================== enum kOpSignature : uint32_t { - kOp_GpW = GpW::kSignature, - kOp_GpX = GpX::kSignature, + kOp_GpW = RegTraits::kSignature, + kOp_GpX = RegTraits::kSignature, + kOp_B = RegTraits::kSignature, + kOp_H = RegTraits::kSignature, + kOp_S = RegTraits::kSignature, + kOp_D = RegTraits::kSignature, + kOp_Q = RegTraits::kSignature, - kOp_B = VecB::kSignature, - kOp_H = VecH::kSignature, - kOp_S = VecS::kSignature, - kOp_D = VecD::kSignature, - kOp_Q = VecV::kSignature, + kOp_V8B = kOp_D | Vec::kSignatureElementB, + kOp_V4H = kOp_D | Vec::kSignatureElementH, + kOp_V2S = kOp_D | Vec::kSignatureElementS, - kOp_V8B = VecD::kSignature | Vec::kSignatureElementB, - kOp_V4H = VecD::kSignature | Vec::kSignatureElementH, - kOp_V2S = VecD::kSignature | Vec::kSignatureElementS, - - kOp_V16B = VecV::kSignature | Vec::kSignatureElementB, - kOp_V8H = VecV::kSignature | Vec::kSignatureElementH, - kOp_V4S = VecV::kSignature | Vec::kSignatureElementS, - kOp_V2D = VecV::kSignature | Vec::kSignatureElementD + kOp_V16B = kOp_Q | Vec::kSignatureElementB, + kOp_V8H = kOp_Q | Vec::kSignatureElementH, + kOp_V4S = kOp_Q | Vec::kSignatureElementS, + kOp_V2D = kOp_Q | Vec::kSignatureElementD }; // a64::InstDB - HFConv @@ -186,6 +185,7 @@ enum EncodingId : uint32_t { kEncodingBaseLdpStp, kEncodingBaseLdxp, kEncodingBaseLogical, + kEncodingBaseMinMax, kEncodingBaseMov, kEncodingBaseMovKNZ, kEncodingBaseMrs, @@ -193,6 +193,7 @@ enum EncodingId : uint32_t { kEncodingBaseMvnNeg, kEncodingBaseOp, kEncodingBaseOpImm, + kEncodingBaseOpX16, kEncodingBasePrfm, kEncodingBaseR, kEncodingBaseRM_NoImm, @@ -270,6 +271,10 @@ struct BaseOp { uint32_t opcode; }; +struct BaseOpX16 { + uint32_t opcode; +}; + struct BaseOpImm { uint32_t opcode; uint16_t immBits; @@ -342,6 +347,11 @@ struct BaseAdcSbc { uint32_t opcode; }; +struct BaseMinMax { + uint32_t regOp; + uint32_t immOp; +}; + struct BaseAddSub { uint32_t shiftedOp : 10; // sf|.......|Sh|.|Rm| Imm:6 |Rn|Rd| uint32_t extendedOp : 10; // sf|.......|..|.|Rm|Opt|Imm3|Rn|Rd| @@ -779,7 +789,7 @@ extern const BaseBfm baseBfm[3]; extern const BaseBfx baseBfx[3]; extern const BaseBranchCmp baseBranchCmp[2]; extern const BaseBranchReg baseBranchReg[3]; -extern const BaseBranchRel baseBranchRel[2]; +extern const BaseBranchRel baseBranchRel[3]; extern const BaseBranchTst baseBranchTst[2]; extern const BaseCCmp baseCCmp[2]; extern const BaseCInc baseCInc[3]; @@ -792,16 +802,18 @@ extern const BaseLdSt baseLdSt[9]; extern const BaseLdpStp baseLdpStp[6]; extern const BaseLdxp baseLdxp[2]; extern const BaseLogical baseLogical[8]; +extern const BaseMinMax baseMinMax[4]; extern const BaseMovKNZ baseMovKNZ[3]; extern const BaseMvnNeg baseMvnNeg[3]; -extern const BaseOp baseOp[23]; -extern const BaseOpImm baseOpImm[14]; +extern const BaseOp baseOp[24]; +extern const BaseOpImm baseOpImm[15]; +extern const BaseOpX16 baseOpX16[1]; extern const BasePrfm basePrfm[1]; extern const BaseR baseR[10]; extern const BaseRM_NoImm baseRM_NoImm[21]; extern const BaseRM_SImm10 baseRM_SImm10[2]; extern const BaseRM_SImm9 baseRM_SImm9[23]; -extern const BaseRR baseRR[15]; +extern const BaseRR baseRR[18]; extern const BaseRRII baseRRII[2]; extern const BaseRRR baseRRR[26]; extern const BaseRRRR baseRRRR[6]; diff --git a/src/asmjit/arm/a64operand.cpp b/src/asmjit/arm/a64operand.cpp index e8c6100..b2c2949 100644 --- a/src/asmjit/arm/a64operand.cpp +++ b/src/asmjit/arm/a64operand.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -32,28 +32,28 @@ UNIT(a64_operand) { EXPECT_EQ(sp.id(), Gp::kIdSp); EXPECT_EQ(w0.size(), 4u); EXPECT_EQ(x0.size(), 8u); - EXPECT_EQ(w0.type(), RegType::kARM_GpW); - EXPECT_EQ(x0.type(), RegType::kARM_GpX); - EXPECT_EQ(w0.group(), RegGroup::kGp); - EXPECT_EQ(x0.group(), RegGroup::kGp); + EXPECT_EQ(w0.regType(), RegType::kGp32); + EXPECT_EQ(x0.regType(), RegType::kGp64); + EXPECT_EQ(w0.regGroup(), RegGroup::kGp); + EXPECT_EQ(x0.regGroup(), RegGroup::kGp); INFO("Checking Vec register properties"); - EXPECT_EQ(v0.type(), RegType::kARM_VecV); - EXPECT_EQ(d0.type(), RegType::kARM_VecD); - EXPECT_EQ(s0.type(), RegType::kARM_VecS); - EXPECT_EQ(h0.type(), RegType::kARM_VecH); - EXPECT_EQ(b0.type(), RegType::kARM_VecB); + EXPECT_EQ(v0.regType(), RegType::kVec128); + EXPECT_EQ(d0.regType(), RegType::kVec64); + EXPECT_EQ(s0.regType(), RegType::kVec32); + EXPECT_EQ(h0.regType(), RegType::kVec16); + EXPECT_EQ(b0.regType(), RegType::kVec8); - EXPECT_EQ(v0.group(), RegGroup::kVec); - EXPECT_EQ(d0.group(), RegGroup::kVec); - EXPECT_EQ(s0.group(), RegGroup::kVec); - EXPECT_EQ(h0.group(), RegGroup::kVec); - EXPECT_EQ(b0.group(), RegGroup::kVec); + EXPECT_EQ(v0.regGroup(), RegGroup::kVec); + EXPECT_EQ(d0.regGroup(), RegGroup::kVec); + EXPECT_EQ(s0.regGroup(), RegGroup::kVec); + EXPECT_EQ(h0.regGroup(), RegGroup::kVec); + EXPECT_EQ(b0.regGroup(), RegGroup::kVec); INFO("Checking Vec register element[] access"); Vec vd_1 = v15.d(1); - EXPECT_EQ(vd_1.type(), RegType::kARM_VecV); - EXPECT_EQ(vd_1.group(), RegGroup::kVec); + EXPECT_EQ(vd_1.regType(), RegType::kVec128); + EXPECT_EQ(vd_1.regGroup(), RegGroup::kVec); EXPECT_EQ(vd_1.id(), 15u); EXPECT_TRUE(vd_1.isVecD2()); EXPECT_EQ(vd_1.elementType(), VecElementType::kD); @@ -61,8 +61,8 @@ UNIT(a64_operand) { EXPECT_EQ(vd_1.elementIndex(), 1u); Vec vs_3 = v15.s(3); - EXPECT_EQ(vs_3.type(), RegType::kARM_VecV); - EXPECT_EQ(vs_3.group(), RegGroup::kVec); + EXPECT_EQ(vs_3.regType(), RegType::kVec128); + EXPECT_EQ(vs_3.regGroup(), RegGroup::kVec); EXPECT_EQ(vs_3.id(), 15u); EXPECT_TRUE(vs_3.isVecS4()); EXPECT_EQ(vs_3.elementType(), VecElementType::kS); @@ -70,8 +70,8 @@ UNIT(a64_operand) { EXPECT_EQ(vs_3.elementIndex(), 3u); Vec vb_4 = v15.b4(3); - EXPECT_EQ(vb_4.type(), RegType::kARM_VecV); - EXPECT_EQ(vb_4.group(), RegGroup::kVec); + EXPECT_EQ(vb_4.regType(), RegType::kVec128); + EXPECT_EQ(vb_4.regGroup(), RegGroup::kVec); EXPECT_EQ(vb_4.id(), 15u); EXPECT_TRUE(vb_4.isVecB4x4()); EXPECT_EQ(vb_4.elementType(), VecElementType::kB4); diff --git a/src/asmjit/arm/a64operand.h b/src/asmjit/arm/a64operand.h index 32a8890..646e9bf 100644 --- a/src/asmjit/arm/a64operand.h +++ b/src/asmjit/arm/a64operand.h @@ -1,31 +1,26 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64OPERAND_H_INCLUDED #define ASMJIT_ARM_A64OPERAND_H_INCLUDED -#include "../arm/armoperand.h" +#include "../core/operand.h" +#include "../arm/armglobals.h" ASMJIT_BEGIN_SUB_NAMESPACE(a64) //! \addtogroup asmjit_a64 //! \{ -class GpW; -class GpX; - -class VecB; -class VecH; -class VecS; -class VecD; -class VecV; - //! General purpose register (AArch64). -class Gp : public Reg { +class Gp : public UniGp { public: - ASMJIT_DEFINE_ABSTRACT_REG(Gp, Reg) + ASMJIT_DEFINE_ABSTRACT_REG(Gp, UniGp) + + //! \name Constants + //! \{ //! Special register id. enum Id : uint32_t { @@ -48,6 +43,32 @@ public: kIdZr = 63 }; + //! \} + + //! \name Static Constructors + //! \{ + + //! Creates a new 32-bit low general purpose register (W) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Gp make_r32(uint32_t regId) noexcept { return Gp(_signatureOf(), regId); } + + //! Creates a new 64-bit low general purpose register (X) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Gp make_r64(uint32_t regId) noexcept { return Gp(_signatureOf(), regId); } + + //! Creates a new 32-bit low general purpose register (W) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Gp make_w(uint32_t regId) noexcept { return make_r32(regId); } + + //! Creates a new 64-bit low general purpose register (X) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Gp make_x(uint32_t regId) noexcept { return make_r64(regId); } + + //! \} + + //! \name Gp Register Accessors + //! \{ + //! Test whether this register is ZR register. [[nodiscard]] ASMJIT_INLINE_CONSTEXPR bool isZR() const noexcept { return id() == kIdZr; } @@ -56,35 +77,25 @@ public: [[nodiscard]] ASMJIT_INLINE_CONSTEXPR bool isSP() const noexcept { return id() == kIdSp; } - //! Cast this register to a 32-bit W register (returns a new operand). + //! Clones and casts this register to a 32-bit (W) register. [[nodiscard]] - ASMJIT_INLINE_NODEBUG GpW w() const noexcept; + ASMJIT_INLINE_CONSTEXPR Gp r32() const noexcept { return make_r32(id()); } - //! \overload + //! Clones and casts this register to a 64-bit (X) register. [[nodiscard]] - ASMJIT_INLINE_NODEBUG GpW r32() const noexcept; + ASMJIT_INLINE_CONSTEXPR Gp r64() const noexcept { return make_r64(id()); } - //! Cast this register to a 64-bit X register (returns a new operand). + //! Clones and casts this register to a 32-bit (W) register. [[nodiscard]] - ASMJIT_INLINE_NODEBUG GpX x() const noexcept; + ASMJIT_INLINE_CONSTEXPR Gp w() const noexcept { return r32(); } - //! \overload + //! Clones and casts this register to a 64-bit (X) register. [[nodiscard]] - ASMJIT_INLINE_NODEBUG GpX r64() const noexcept; + ASMJIT_INLINE_CONSTEXPR Gp x() const noexcept { return r64(); } + + //! \} }; -//! 32-bit general purpose W register (AArch64). -class GpW : public Gp { ASMJIT_DEFINE_FINAL_REG(GpW, Gp, RegTraits); }; -//! 64-bit general purpose X register (AArch64). -class GpX : public Gp { ASMJIT_DEFINE_FINAL_REG(GpX, Gp, RegTraits); }; - -#ifndef _DOXYGEN -ASMJIT_INLINE_NODEBUG GpW Gp::w() const noexcept { return GpW(id()); } -ASMJIT_INLINE_NODEBUG GpX Gp::x() const noexcept { return GpX(id()); } -ASMJIT_INLINE_NODEBUG GpW Gp::r32() const noexcept { return GpW(id()); } -ASMJIT_INLINE_NODEBUG GpX Gp::r64() const noexcept { return GpX(id()); } -#endif - //! Vector element type (AArch64). enum class VecElementType : uint32_t { //! No element type specified. @@ -111,264 +122,552 @@ enum class VecElementType : uint32_t { }; //! Vector register (AArch64). -class Vec : public BaseVec { +class Vec : public UniVec { public: - ASMJIT_DEFINE_ABSTRACT_REG(Vec, BaseVec) + ASMJIT_DEFINE_ABSTRACT_REG(Vec, UniVec) //! \cond - //! Shortcuts. - enum SignatureReg : uint32_t { - kSignatureElementB = uint32_t(VecElementType::kB) << kSignatureRegElementTypeShift, - kSignatureElementH = uint32_t(VecElementType::kH) << kSignatureRegElementTypeShift, - kSignatureElementS = uint32_t(VecElementType::kS) << kSignatureRegElementTypeShift, - kSignatureElementD = uint32_t(VecElementType::kD) << kSignatureRegElementTypeShift, - kSignatureElementB4 = uint32_t(VecElementType::kB4) << kSignatureRegElementTypeShift, - kSignatureElementH2 = uint32_t(VecElementType::kH2) << kSignatureRegElementTypeShift - }; - //! \endcond - //! Returns whether the register has element type or element index (or both). - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool hasElementTypeOrIndex() const noexcept { return _signature.hasField(); } + // Register element type (3 bits). + // |........|........|.XXX....|........| + static inline constexpr uint32_t kSignatureRegElementTypeShift = 12; + static inline constexpr uint32_t kSignatureRegElementTypeMask = 0x07 << kSignatureRegElementTypeShift; - //! Returns whether the vector register has associated a vector element type. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool hasElementType() const noexcept { return _signature.hasField(); } + // Register has element index (1 bit). + // |........|........|X.......|........| + static inline constexpr uint32_t kSignatureRegElementFlagShift = 15; + static inline constexpr uint32_t kSignatureRegElementFlagMask = 0x01 << kSignatureRegElementFlagShift; - //! Returns vector element type of the register. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR VecElementType elementType() const noexcept { return VecElementType(_signature.getField()); } + // Register element index (4 bits). + // |........|....XXXX|........|........| + static inline constexpr uint32_t kSignatureRegElementIndexShift = 16; + static inline constexpr uint32_t kSignatureRegElementIndexMask = 0x0F << kSignatureRegElementIndexShift; - //! Sets vector element type of the register to `elementType`. - ASMJIT_INLINE_NODEBUG void setElementType(VecElementType elementType) noexcept { _signature.setField(uint32_t(elementType)); } - - //! Resets vector element type to none. - ASMJIT_INLINE_NODEBUG void resetElementType() noexcept { _signature.setField(0); } - - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVecB8() const noexcept { - return _signature.subset(uint32_t(kBaseSignatureMask) | uint32_t(kSignatureRegElementTypeMask)) == (RegTraits::kSignature | kSignatureElementB); - } - - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVecH4() const noexcept { - return _signature.subset(uint32_t(kBaseSignatureMask) | uint32_t(kSignatureRegElementTypeMask)) == (RegTraits::kSignature | kSignatureElementH); - } - - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVecS2() const noexcept { - return _signature.subset(uint32_t(kBaseSignatureMask) | uint32_t(kSignatureRegElementTypeMask)) == (RegTraits::kSignature | kSignatureElementS); - } - - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVecD1() const noexcept { - return _signature.subset(uint32_t(kBaseSignatureMask) | uint32_t(kSignatureRegElementTypeMask)) == (RegTraits::kSignature); - } - - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVecB16() const noexcept { - return _signature.subset(uint32_t(kBaseSignatureMask) | uint32_t(kSignatureRegElementTypeMask)) == (RegTraits::kSignature | kSignatureElementB); - } - - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVecH8() const noexcept { - return _signature.subset(uint32_t(kBaseSignatureMask) | uint32_t(kSignatureRegElementTypeMask)) == (RegTraits::kSignature | kSignatureElementH); - } - - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVecS4() const noexcept { - return _signature.subset(uint32_t(kBaseSignatureMask) | uint32_t(kSignatureRegElementTypeMask)) == (RegTraits::kSignature | kSignatureElementS); - } - - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVecD2() const noexcept { - return _signature.subset(uint32_t(kBaseSignatureMask) | uint32_t(kSignatureRegElementTypeMask)) == (RegTraits::kSignature | kSignatureElementD); - } - - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVecB4x4() const noexcept { - return _signature.subset(uint32_t(kBaseSignatureMask) | uint32_t(kSignatureRegElementTypeMask)) == (RegTraits::kSignature | kSignatureElementB4); - } - - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVecH2x4() const noexcept { - return _signature.subset(uint32_t(kBaseSignatureMask) | uint32_t(kSignatureRegElementTypeMask)) == (RegTraits::kSignature | kSignatureElementH2); - } - - //! Creates a cloned register with element access. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG Vec at(uint32_t elementIndex) const noexcept { - return Vec((signature() & ~kSignatureRegElementIndexMask) | (elementIndex << kSignatureRegElementIndexShift) | kSignatureRegElementFlagMask, id()); - } - - //! Cast this register to an 8-bit B register (AArch64 only). - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecB b() const noexcept; - - //! Cast this register to a 16-bit H register (AArch64 only). - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecH h() const noexcept; - - //! Cast this register to a 32-bit S register. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecS s() const noexcept; - - //! Cast this register to a 64-bit D register. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecD d() const noexcept; - - //! Cast this register to a 128-bit Q register. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecV q() const noexcept; - - //! Cast this register to a 128-bit V register. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecV v() const noexcept; - - //! Casts this register to b (clone). - [[nodiscard]] - ASMJIT_INLINE_NODEBUG Vec v8() const noexcept; - - //! Casts this register to h (clone). - [[nodiscard]] - ASMJIT_INLINE_NODEBUG Vec v16() const noexcept; - - //! Casts this register to s (clone). - [[nodiscard]] - ASMJIT_INLINE_NODEBUG Vec v32() const noexcept; - - //! Casts this register to d (clone). - [[nodiscard]] - ASMJIT_INLINE_NODEBUG Vec v64() const noexcept; - - //! Casts this register to q (clone). - [[nodiscard]] - ASMJIT_INLINE_NODEBUG Vec v128() const noexcept; - - //! Cast this register to a 128-bit V.B[elementIndex] register. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecV b(uint32_t elementIndex) const noexcept; - - //! Cast this register to a 128-bit V.H[elementIndex] register. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecV h(uint32_t elementIndex) const noexcept; - - //! Cast this register to a 128-bit V.S[elementIndex] register. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecV s(uint32_t elementIndex) const noexcept; - - //! Cast this register to a 128-bit V.D[elementIndex] register. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecV d(uint32_t elementIndex) const noexcept; - - //! Cast this register to a 128-bit V.H2[elementIndex] register. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecV h2(uint32_t elementIndex) const noexcept; - - //! Cast this register to a 128-bit V.B4[elementIndex] register. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecV b4(uint32_t elementIndex) const noexcept; - - //! Cast this register to V.8B. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecD b8() const noexcept; - - //! Cast this register to V.16B. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecV b16() const noexcept; - - //! Cast this register to V.2H. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecS h2() const noexcept; - - //! Cast this register to V.4H. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecD h4() const noexcept; - - //! Cast this register to V.8H. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecV h8() const noexcept; - - //! Cast this register to V.2S. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecD s2() const noexcept; - - //! Cast this register to V.4S. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecV s4() const noexcept; - - //! Cast this register to V.2D. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG VecV d2() const noexcept; + static inline constexpr uint32_t kSignatureElementB = uint32_t(VecElementType::kB) << kSignatureRegElementTypeShift; + static inline constexpr uint32_t kSignatureElementH = uint32_t(VecElementType::kH) << kSignatureRegElementTypeShift; + static inline constexpr uint32_t kSignatureElementS = uint32_t(VecElementType::kS) << kSignatureRegElementTypeShift; + static inline constexpr uint32_t kSignatureElementD = uint32_t(VecElementType::kD) << kSignatureRegElementTypeShift; + static inline constexpr uint32_t kSignatureElementB4 = uint32_t(VecElementType::kB4) << kSignatureRegElementTypeShift; + static inline constexpr uint32_t kSignatureElementH2 = uint32_t(VecElementType::kH2) << kSignatureRegElementTypeShift; [[nodiscard]] static ASMJIT_INLINE_CONSTEXPR OperandSignature _makeElementAccessSignature(VecElementType elementType, uint32_t elementIndex) noexcept { return OperandSignature{ - uint32_t(RegTraits::kSignature) | + uint32_t(RegTraits::kSignature) | uint32_t(kSignatureRegElementFlagMask) | (uint32_t(elementType) << kSignatureRegElementTypeShift) | - (uint32_t(elementIndex << kSignatureRegElementIndexShift))}; + (uint32_t(elementIndex << kSignatureRegElementIndexShift)) + }; } + + //! \endcond + + //! \name Static Constructors + //! \{ + + //! Creates a new 8-bit vector register (B) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v8(uint32_t regId) noexcept { return Vec(_signatureOf(), regId); } + + //! Creates a new 16-bit vector register (H) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v16(uint32_t regId) noexcept { return Vec(_signatureOf(), regId); } + + //! Creates a new 32-bit vector register (S) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v32(uint32_t regId) noexcept { return Vec(_signatureOf(), regId); } + + //! Creates a new 64-bit vector register (D) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v64(uint32_t regId) noexcept { return Vec(_signatureOf(), regId); } + + //! Creates a new 128-bit vector register (Q) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v128(uint32_t regId) noexcept { return Vec(_signatureOf(), regId); } + + //! Creates a new 8-bit vector register (B) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_b(uint32_t regId) noexcept { return make_v8(regId); } + + //! Creates a new 16-bit vector register (H) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_h(uint32_t regId) noexcept { return make_v16(regId); } + + //! Creates a new 32-bit vector register (S) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_s(uint32_t regId) noexcept { return make_v32(regId); } + + //! Creates a new 64-bit vector register (D) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_d(uint32_t regId) noexcept { return make_v64(regId); } + + //! Creates a new 128-bit vector register (Q) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_q(uint32_t regId) noexcept { return make_v128(regId); } + + //! Creates a new 32-bit vector register (S) having the given vector `elementType` and register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v32_with_element_type(VecElementType elementType, uint32_t regId) noexcept { + uint32_t signature = RegTraits::kSignature | uint32_t(elementType) << kSignatureRegElementTypeShift; + return Vec(OperandSignature{signature}, regId); + } + + //! Creates a new 64-bit vector register (D) having the given vector `elementType` and register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v64_with_element_type(VecElementType elementType, uint32_t regId) noexcept { + uint32_t signature = RegTraits::kSignature | uint32_t(elementType) << kSignatureRegElementTypeShift; + return Vec(OperandSignature{signature}, regId); + } + + //! Creates a new 128-bit vector register (Q) having the given vector `elementType` and register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v128_with_element_type(VecElementType elementType, uint32_t regId) noexcept { + uint32_t signature = RegTraits::kSignature | uint32_t(elementType) << kSignatureRegElementTypeShift; + return Vec(OperandSignature{signature}, regId); + } + + //! Creates a new 128-bit vector of type specified by `elementType` and `elementIndex`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v128_with_element_index(VecElementType elementType, uint32_t elementIndex, uint32_t regId) noexcept { + return Vec(_makeElementAccessSignature(elementType, elementIndex), regId); + } + + //! \} + + //! \name Vector Register Accessors + //! \{ + + //! Returns whether the register has element type or element index (or both). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasElementTypeOrIndex() const noexcept { + return _signature.hasField(); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecB8() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementB); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecH4() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementH); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecS2() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementS); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecD1() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecB16() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementB); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecH8() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementH); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecS4() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementS); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecD2() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementD); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecB4x4() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementB4); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecH2x4() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementH2); + } + + //! Clones and casts the register to an 8-bit B register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec v8() const noexcept { return make_v8(id()); } + + //! Clones and casts the register to a 16-bit H register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec v16() const noexcept { return make_v16(id()); } + + //! Clones and casts the register to a 32-bit S register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec v32() const noexcept { return make_v32(id()); } + + //! Clones and casts the register to a 64-bit D register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec v64() const noexcept { return make_v64(id()); } + + //! Clones and casts the register to a 128-bit Q register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec v128() const noexcept { return make_v128(id()); } + + //! Clones and casts the register to an 8-bit B register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec b() const noexcept { return make_v8(id()); } + + //! Clones and casts the register to a 16-bit H register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec h() const noexcept { return make_v16(id()); } + + //! Clones and casts the register to a 32-bit S register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec s() const noexcept { return make_v32(id()); } + + //! Clones and casts the register to a 64-bit D register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec d() const noexcept { return make_v64(id()); } + + //! Clones and casts the register to a 128-bit Q register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec q() const noexcept { return make_v128(id()); } + + //! Clones and casts the register to a 128-bit V.B[elementIndex] register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec b(uint32_t elementIndex) const noexcept { return make_v128_with_element_index(VecElementType::kB, elementIndex, id()); } + + //! Clones and casts the register to a 128-bit V.H[elementIndex] register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec h(uint32_t elementIndex) const noexcept { return make_v128_with_element_index(VecElementType::kH, elementIndex, id()); } + + //! Clones and casts the register to a 128-bit V.S[elementIndex] register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec s(uint32_t elementIndex) const noexcept { return make_v128_with_element_index(VecElementType::kS, elementIndex, id()); } + + //! Clones and casts the register to a 128-bit V.D[elementIndex] register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec d(uint32_t elementIndex) const noexcept { return make_v128_with_element_index(VecElementType::kD, elementIndex, id()); } + + //! Clones and casts the register to a 128-bit V.H2[elementIndex] register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec h2(uint32_t elementIndex) const noexcept { return make_v128_with_element_index(VecElementType::kH2, elementIndex, id()); } + + //! Clones and casts the register to a 128-bit V.B4[elementIndex] register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec b4(uint32_t elementIndex) const noexcept { return make_v128_with_element_index(VecElementType::kB4, elementIndex, id()); } + + //! Clones and casts the register to V.8B. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec b8() const noexcept { return make_v64_with_element_type(VecElementType::kB, id()); } + + //! Clones and casts the register to V.16B. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec b16() const noexcept { return make_v128_with_element_type(VecElementType::kB, id()); } + + //! Clones and casts the register to V.2H. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec h2() const noexcept { return make_v32_with_element_type(VecElementType::kH, id()); } + + //! Clones and casts the register to V.4H. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec h4() const noexcept { return make_v64_with_element_type(VecElementType::kH, id()); } + + //! Clones and casts the register to V.8H. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec h8() const noexcept { return make_v128_with_element_type(VecElementType::kH, id()); } + + //! Clones and casts the register to V.2S. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec s2() const noexcept { return make_v64_with_element_type(VecElementType::kS, id()); } + + //! Clones and casts the register to V.4S. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec s4() const noexcept { return make_v128_with_element_type(VecElementType::kS, id()); } + + //! Clones and casts the register to V.2D. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec d2() const noexcept { return make_v128_with_element_type(VecElementType::kD, id()); } + + //! \} + + //! \name Element Type Accessors + //! \{ + + //! Returns whether the vector register has associated a vector element type. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasElementType() const noexcept { + return _signature.hasField(); + } + + //! Returns vector element type of the register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR VecElementType elementType() const noexcept { + return VecElementType(_signature.getField()); + } + + //! Sets vector element type of the register to `elementType`. + ASMJIT_INLINE_CONSTEXPR void setElementType(VecElementType elementType) noexcept { + _signature.setField(uint32_t(elementType)); + } + + //! Resets vector element type to none. + ASMJIT_INLINE_CONSTEXPR void resetElementType() noexcept { + _signature.setField(0); + } + + //! \} + + //! \name Element Index Accessors + //! \{ + + //! Returns whether the register has element index (it's an element index access). + ASMJIT_INLINE_CONSTEXPR bool hasElementIndex() const noexcept { + return _signature.hasField(); + } + + //! Returns element index of the register. + ASMJIT_INLINE_CONSTEXPR uint32_t elementIndex() const noexcept { + return _signature.getField(); + } + + //! Sets element index of the register to `elementType`. + ASMJIT_INLINE_CONSTEXPR void setElementIndex(uint32_t elementIndex) noexcept { + _signature |= kSignatureRegElementFlagMask; + _signature.setField(elementIndex); + } + + //! Resets element index of the register. + ASMJIT_INLINE_CONSTEXPR void resetElementIndex() noexcept { + _signature &= ~(kSignatureRegElementFlagMask | kSignatureRegElementIndexMask); + } + + //! Clones a vector register with element access enabled at the given `elementIndex`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec at(uint32_t elementIndex) const noexcept { + return Vec((signature() & ~kSignatureRegElementIndexMask) | (elementIndex << kSignatureRegElementIndexShift) | kSignatureRegElementFlagMask, id()); + } + + //! \} }; -//! 8-bit view (S) of VFP/SIMD register. -class VecB : public Vec { +//! Memory operand (AArch64). +class Mem : public BaseMem { public: - ASMJIT_DEFINE_FINAL_REG(VecB, Vec, RegTraits) + //! \cond INTERNAL + + // Index shift value (5 bits). + // |........|.....XXX|XX......|........| + static inline constexpr uint32_t kSignatureMemShiftValueShift = 14; + static inline constexpr uint32_t kSignatureMemShiftValueMask = 0x1Fu << kSignatureMemShiftValueShift; + + // Index shift operation (4 bits). + // |........|XXXX....|........|........| + static inline constexpr uint32_t kSignatureMemShiftOpShift = 20; + static inline constexpr uint32_t kSignatureMemShiftOpMask = 0x0Fu << kSignatureMemShiftOpShift; + + // Offset mode type (2 bits). + // |......XX|........|........|........| + static inline constexpr uint32_t kSignatureMemOffsetModeShift = 24; + static inline constexpr uint32_t kSignatureMemOffsetModeMask = 0x03u << kSignatureMemOffsetModeShift; + + //! \endcond + + //! \name Construction & Destruction + //! \{ + + //! Construct a default `Mem` operand, that points to [0]. + ASMJIT_INLINE_CONSTEXPR Mem() noexcept + : BaseMem() {} + + ASMJIT_INLINE_CONSTEXPR Mem(const Mem& other) noexcept + : BaseMem(other) {} + + ASMJIT_INLINE_NODEBUG explicit Mem(Globals::NoInit_) noexcept + : BaseMem(Globals::NoInit) {} + + ASMJIT_INLINE_CONSTEXPR Mem(const Signature& signature, uint32_t baseId, uint32_t indexId, int32_t offset) noexcept + : BaseMem(signature, baseId, indexId, offset) {} + + ASMJIT_INLINE_CONSTEXPR explicit Mem(const Label& base, int32_t off = 0, Signature signature = Signature{0}) noexcept + : BaseMem(Signature::fromOpType(OperandType::kMem) | + Signature::fromMemBaseType(RegType::kLabelTag) | + signature, base.id(), 0, off) {} + + ASMJIT_INLINE_CONSTEXPR explicit Mem(const Reg& base, int32_t off = 0, Signature signature = Signature{0}) noexcept + : BaseMem(Signature::fromOpType(OperandType::kMem) | + Signature::fromMemBaseType(base.regType()) | + signature, base.id(), 0, off) {} + + ASMJIT_INLINE_CONSTEXPR Mem(const Reg& base, const Reg& index, Signature signature = Signature{0}) noexcept + : BaseMem(Signature::fromOpType(OperandType::kMem) | + Signature::fromMemBaseType(base.regType()) | + Signature::fromMemIndexType(index.regType()) | + signature, base.id(), index.id(), 0) {} + + ASMJIT_INLINE_CONSTEXPR Mem(const Reg& base, const Reg& index, const Shift& shift, Signature signature = Signature{0}) noexcept + : BaseMem(Signature::fromOpType(OperandType::kMem) | + Signature::fromMemBaseType(base.regType()) | + Signature::fromMemIndexType(index.regType()) | + Signature::fromValue(uint32_t(shift.op())) | + Signature::fromValue(shift.value()) | + signature, base.id(), index.id(), 0) {} + + ASMJIT_INLINE_CONSTEXPR explicit Mem(uint64_t base, Signature signature = Signature{0}) noexcept + : BaseMem(Signature::fromOpType(OperandType::kMem) | + signature, uint32_t(base >> 32), 0, int32_t(uint32_t(base & 0xFFFFFFFFu))) {} + + //! \} + + //! \name Overloaded Operators + //! \{ + + ASMJIT_INLINE_CONSTEXPR Mem& operator=(const Mem& other) noexcept { + copyFrom(other); + return *this; + } + + //! \} + + //! \name Clone + //! \{ + + //! Clones the memory operand. + ASMJIT_INLINE_CONSTEXPR Mem clone() const noexcept { return Mem(*this); } + + //! Gets new memory operand adjusted by `off`. + ASMJIT_INLINE_CONSTEXPR Mem cloneAdjusted(int64_t off) const noexcept { + Mem result(*this); + result.addOffset(off); + return result; + } + + //! Clones the memory operand and makes it pre-index. + ASMJIT_INLINE_CONSTEXPR Mem pre() const noexcept { + Mem result(*this); + result.setOffsetMode(OffsetMode::kPreIndex); + return result; + } + + //! Clones the memory operand, applies a given offset `off` and makes it pre-index. + ASMJIT_INLINE_CONSTEXPR Mem pre(int64_t off) const noexcept { + Mem result(*this); + result.setOffsetMode(OffsetMode::kPreIndex); + result.addOffset(off); + return result; + } + + //! Clones the memory operand and makes it post-index. + ASMJIT_INLINE_CONSTEXPR Mem post() const noexcept { + Mem result(*this); + result.setOffsetMode(OffsetMode::kPostIndex); + return result; + } + + //! Clones the memory operand, applies a given offset `off` and makes it post-index. + ASMJIT_INLINE_CONSTEXPR Mem post(int64_t off) const noexcept { + Mem result(*this); + result.setOffsetMode(OffsetMode::kPostIndex); + result.addOffset(off); + return result; + } + + //! \} + + //! \name Base & Index + //! \{ + + //! Converts memory `baseType` and `baseId` to `arm::Reg` instance. + //! + //! The memory must have a valid base register otherwise the result will be wrong. + ASMJIT_INLINE_NODEBUG Reg baseReg() const noexcept { return Reg::fromTypeAndId(baseType(), baseId()); } + + //! Converts memory `indexType` and `indexId` to `arm::Reg` instance. + //! + //! The memory must have a valid index register otherwise the result will be wrong. + ASMJIT_INLINE_NODEBUG Reg indexReg() const noexcept { return Reg::fromTypeAndId(indexType(), indexId()); } + + using BaseMem::setIndex; + + ASMJIT_INLINE_CONSTEXPR void setIndex(const Reg& index, uint32_t shift) noexcept { + setIndex(index); + setShift(shift); + } + + ASMJIT_INLINE_CONSTEXPR void setIndex(const Reg& index, Shift shift) noexcept { + setIndex(index); + setShift(shift); + } + + //! \} + + //! \name ARM Specific Features + //! \{ + + //! Gets offset mode. + ASMJIT_INLINE_CONSTEXPR OffsetMode offsetMode() const noexcept { return OffsetMode(_signature.getField()); } + //! Sets offset mode to `mode`. + ASMJIT_INLINE_CONSTEXPR void setOffsetMode(OffsetMode mode) noexcept { _signature.setField(uint32_t(mode)); } + //! Resets offset mode to default (fixed offset, without write-back). + ASMJIT_INLINE_CONSTEXPR void resetOffsetMode() noexcept { _signature.setField(uint32_t(OffsetMode::kFixed)); } + + //! Tests whether the current memory offset mode is fixed (see \ref OffsetMode::kFixed). + ASMJIT_INLINE_CONSTEXPR bool isFixedOffset() const noexcept { return offsetMode() == OffsetMode::kFixed; } + //! Tests whether the current memory offset mode is either pre-index or post-index (write-back is used). + ASMJIT_INLINE_CONSTEXPR bool isPreOrPost() const noexcept { return offsetMode() != OffsetMode::kFixed; } + //! Tests whether the current memory offset mode is pre-index (write-back is used). + ASMJIT_INLINE_CONSTEXPR bool isPreIndex() const noexcept { return offsetMode() == OffsetMode::kPreIndex; } + //! Tests whether the current memory offset mode is post-index (write-back is used). + ASMJIT_INLINE_CONSTEXPR bool isPostIndex() const noexcept { return offsetMode() == OffsetMode::kPostIndex; } + + //! Sets offset mode of this memory operand to pre-index (write-back is used). + ASMJIT_INLINE_CONSTEXPR void makePreIndex() noexcept { setOffsetMode(OffsetMode::kPreIndex); } + //! Sets offset mode of this memory operand to post-index (write-back is used). + ASMJIT_INLINE_CONSTEXPR void makePostIndex() noexcept { setOffsetMode(OffsetMode::kPostIndex); } + + //! Gets shift operation that is used by index register. + ASMJIT_INLINE_CONSTEXPR ShiftOp shiftOp() const noexcept { return ShiftOp(_signature.getField()); } + //! Sets shift operation that is used by index register. + ASMJIT_INLINE_CONSTEXPR void setShiftOp(ShiftOp sop) noexcept { _signature.setField(uint32_t(sop)); } + //! Resets shift operation that is used by index register to LSL (default value). + ASMJIT_INLINE_CONSTEXPR void resetShiftOp() noexcept { _signature.setField(uint32_t(ShiftOp::kLSL)); } + + //! Gets whether the memory operand has shift (aka scale) constant. + ASMJIT_INLINE_CONSTEXPR bool hasShift() const noexcept { return _signature.hasField(); } + //! Gets the memory operand's shift (aka scale) constant. + ASMJIT_INLINE_CONSTEXPR uint32_t shift() const noexcept { return _signature.getField(); } + //! Sets the memory operand's shift (aka scale) constant. + ASMJIT_INLINE_CONSTEXPR void setShift(uint32_t shift) noexcept { _signature.setField(shift); } + + //! Sets the memory operand's shift and shift operation. + ASMJIT_INLINE_CONSTEXPR void setShift(Shift shift) noexcept { + _signature.setField(uint32_t(shift.op())); + _signature.setField(shift.value()); + } + + //! Resets the memory operand's shift (aka scale) constant to zero. + ASMJIT_INLINE_CONSTEXPR void resetShift() noexcept { _signature.setField(0); } + + //! \} }; -//! 16-bit view (S) of VFP/SIMD register. -class VecH : public Vec { -public: - ASMJIT_DEFINE_FINAL_REG(VecH, Vec, RegTraits) -}; +//! \name Shift Operation Construction +//! \{ -//! 32-bit view (S) of VFP/SIMD register. -class VecS : public Vec { -public: - ASMJIT_DEFINE_FINAL_REG(VecS, Vec, RegTraits) -}; +//! Constructs a `LSL #value` shift (logical shift left). +static ASMJIT_INLINE_CONSTEXPR Shift lsl(uint32_t value) noexcept { return Shift(ShiftOp::kLSL, value); } +//! Constructs a `LSR #value` shift (logical shift right). +static ASMJIT_INLINE_CONSTEXPR Shift lsr(uint32_t value) noexcept { return Shift(ShiftOp::kLSR, value); } +//! Constructs a `ASR #value` shift (arithmetic shift right). +static ASMJIT_INLINE_CONSTEXPR Shift asr(uint32_t value) noexcept { return Shift(ShiftOp::kASR, value); } +//! Constructs a `ROR #value` shift (rotate right). +static ASMJIT_INLINE_CONSTEXPR Shift ror(uint32_t value) noexcept { return Shift(ShiftOp::kROR, value); } +//! Constructs a `RRX` shift (rotate with carry by 1). +static ASMJIT_INLINE_CONSTEXPR Shift rrx() noexcept { return Shift(ShiftOp::kRRX, 0); } +//! Constructs a `MSL #value` shift (logical shift left filling ones). +static ASMJIT_INLINE_CONSTEXPR Shift msl(uint32_t value) noexcept { return Shift(ShiftOp::kMSL, value); } -//! 64-bit view (D) of VFP/SIMD register. -class VecD : public Vec { -public: - ASMJIT_DEFINE_FINAL_REG(VecD, Vec, RegTraits) -}; - -//! 128-bit vector register (Q or V). -class VecV : public Vec { -public: - ASMJIT_DEFINE_FINAL_REG(VecV, Vec, RegTraits) -}; - -ASMJIT_INLINE_NODEBUG VecB Vec::b() const noexcept { return VecB(id()); } -ASMJIT_INLINE_NODEBUG VecH Vec::h() const noexcept { return VecH(id()); } -ASMJIT_INLINE_NODEBUG VecS Vec::s() const noexcept { return VecS(id()); } -ASMJIT_INLINE_NODEBUG VecD Vec::d() const noexcept { return VecD(id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::q() const noexcept { return VecV(id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::v() const noexcept { return VecV(id()); } - -ASMJIT_INLINE_NODEBUG Vec Vec::v8() const noexcept { return VecB(id()); } -ASMJIT_INLINE_NODEBUG Vec Vec::v16() const noexcept { return VecH(id()); } -ASMJIT_INLINE_NODEBUG Vec Vec::v32() const noexcept { return VecS(id()); } -ASMJIT_INLINE_NODEBUG Vec Vec::v64() const noexcept { return VecD(id()); } -ASMJIT_INLINE_NODEBUG Vec Vec::v128() const noexcept { return VecV(id()); } - -ASMJIT_INLINE_NODEBUG VecV Vec::b(uint32_t elementIndex) const noexcept { return VecV(_makeElementAccessSignature(VecElementType::kB, elementIndex), id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::h(uint32_t elementIndex) const noexcept { return VecV(_makeElementAccessSignature(VecElementType::kH, elementIndex), id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::s(uint32_t elementIndex) const noexcept { return VecV(_makeElementAccessSignature(VecElementType::kS, elementIndex), id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::d(uint32_t elementIndex) const noexcept { return VecV(_makeElementAccessSignature(VecElementType::kD, elementIndex), id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::h2(uint32_t elementIndex) const noexcept { return VecV(_makeElementAccessSignature(VecElementType::kH2, elementIndex), id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::b4(uint32_t elementIndex) const noexcept { return VecV(_makeElementAccessSignature(VecElementType::kB4, elementIndex), id()); } - -ASMJIT_INLINE_NODEBUG VecD Vec::b8() const noexcept { return VecD(OperandSignature{VecD::kSignature | kSignatureElementB}, id()); } -ASMJIT_INLINE_NODEBUG VecS Vec::h2() const noexcept { return VecS(OperandSignature{VecS::kSignature | kSignatureElementH}, id()); } -ASMJIT_INLINE_NODEBUG VecD Vec::h4() const noexcept { return VecD(OperandSignature{VecD::kSignature | kSignatureElementH}, id()); } -ASMJIT_INLINE_NODEBUG VecD Vec::s2() const noexcept { return VecD(OperandSignature{VecD::kSignature | kSignatureElementS}, id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::b16() const noexcept { return VecV(OperandSignature{VecV::kSignature | kSignatureElementB}, id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::h8() const noexcept { return VecV(OperandSignature{VecV::kSignature | kSignatureElementH}, id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::s4() const noexcept { return VecV(OperandSignature{VecV::kSignature | kSignatureElementS}, id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::d2() const noexcept { return VecV(OperandSignature{VecV::kSignature | kSignatureElementD}, id()); } +//! \} #ifndef _DOXYGEN namespace regs { @@ -376,289 +675,301 @@ namespace regs { //! Creates a 32-bit W register operand. [[nodiscard]] -static ASMJIT_INLINE_CONSTEXPR GpW w(uint32_t id) noexcept { return GpW(id); } +static ASMJIT_INLINE_CONSTEXPR Gp w(uint32_t id) noexcept { return Gp::make_r32(id); } //! Creates a 64-bit X register operand. [[nodiscard]] -static ASMJIT_INLINE_CONSTEXPR GpX x(uint32_t id) noexcept { return GpX(id); } +static ASMJIT_INLINE_CONSTEXPR Gp x(uint32_t id) noexcept { return Gp::make_r64(id); } + +//! Creates an 8-bit B register operand. +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Vec b(uint32_t id) noexcept { return Vec::make_v8(id); } + +//! Creates a 16-bit H register operand. +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Vec h(uint32_t id) noexcept { return Vec::make_v16(id); } //! Creates a 32-bit S register operand. [[nodiscard]] -static ASMJIT_INLINE_CONSTEXPR VecS s(uint32_t id) noexcept { return VecS(id); } +static ASMJIT_INLINE_CONSTEXPR Vec s(uint32_t id) noexcept { return Vec::make_v32(id); } //! Creates a 64-bit D register operand. [[nodiscard]] -static ASMJIT_INLINE_CONSTEXPR VecD d(uint32_t id) noexcept { return VecD(id); } +static ASMJIT_INLINE_CONSTEXPR Vec d(uint32_t id) noexcept { return Vec::make_v64(id); } //! Creates a 1282-bit V register operand. [[nodiscard]] -static ASMJIT_INLINE_CONSTEXPR VecV v(uint32_t id) noexcept { return VecV(id); } +static ASMJIT_INLINE_CONSTEXPR Vec q(uint32_t id) noexcept { return Vec::make_v128(id); } -static constexpr GpW w0 = GpW(0); -static constexpr GpW w1 = GpW(1); -static constexpr GpW w2 = GpW(2); -static constexpr GpW w3 = GpW(3); -static constexpr GpW w4 = GpW(4); -static constexpr GpW w5 = GpW(5); -static constexpr GpW w6 = GpW(6); -static constexpr GpW w7 = GpW(7); -static constexpr GpW w8 = GpW(8); -static constexpr GpW w9 = GpW(9); -static constexpr GpW w10 = GpW(10); -static constexpr GpW w11 = GpW(11); -static constexpr GpW w12 = GpW(12); -static constexpr GpW w13 = GpW(13); -static constexpr GpW w14 = GpW(14); -static constexpr GpW w15 = GpW(15); -static constexpr GpW w16 = GpW(16); -static constexpr GpW w17 = GpW(17); -static constexpr GpW w18 = GpW(18); -static constexpr GpW w19 = GpW(19); -static constexpr GpW w20 = GpW(20); -static constexpr GpW w21 = GpW(21); -static constexpr GpW w22 = GpW(22); -static constexpr GpW w23 = GpW(23); -static constexpr GpW w24 = GpW(24); -static constexpr GpW w25 = GpW(25); -static constexpr GpW w26 = GpW(26); -static constexpr GpW w27 = GpW(27); -static constexpr GpW w28 = GpW(28); -static constexpr GpW w29 = GpW(29); -static constexpr GpW w30 = GpW(30); -static constexpr GpW wzr = GpW(Gp::kIdZr); -static constexpr GpW wsp = GpW(Gp::kIdSp); +//! Creates a 1282-bit V register operand. +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Vec v(uint32_t id) noexcept { return Vec::make_v128(id); } -static constexpr GpX x0 = GpX(0); -static constexpr GpX x1 = GpX(1); -static constexpr GpX x2 = GpX(2); -static constexpr GpX x3 = GpX(3); -static constexpr GpX x4 = GpX(4); -static constexpr GpX x5 = GpX(5); -static constexpr GpX x6 = GpX(6); -static constexpr GpX x7 = GpX(7); -static constexpr GpX x8 = GpX(8); -static constexpr GpX x9 = GpX(9); -static constexpr GpX x10 = GpX(10); -static constexpr GpX x11 = GpX(11); -static constexpr GpX x12 = GpX(12); -static constexpr GpX x13 = GpX(13); -static constexpr GpX x14 = GpX(14); -static constexpr GpX x15 = GpX(15); -static constexpr GpX x16 = GpX(16); -static constexpr GpX x17 = GpX(17); -static constexpr GpX x18 = GpX(18); -static constexpr GpX x19 = GpX(19); -static constexpr GpX x20 = GpX(20); -static constexpr GpX x21 = GpX(21); -static constexpr GpX x22 = GpX(22); -static constexpr GpX x23 = GpX(23); -static constexpr GpX x24 = GpX(24); -static constexpr GpX x25 = GpX(25); -static constexpr GpX x26 = GpX(26); -static constexpr GpX x27 = GpX(27); -static constexpr GpX x28 = GpX(28); -static constexpr GpX x29 = GpX(29); -static constexpr GpX x30 = GpX(30); -static constexpr GpX xzr = GpX(Gp::kIdZr); -static constexpr GpX sp = GpX(Gp::kIdSp); +static constexpr Gp w0 = Gp::make_r32(0); +static constexpr Gp w1 = Gp::make_r32(1); +static constexpr Gp w2 = Gp::make_r32(2); +static constexpr Gp w3 = Gp::make_r32(3); +static constexpr Gp w4 = Gp::make_r32(4); +static constexpr Gp w5 = Gp::make_r32(5); +static constexpr Gp w6 = Gp::make_r32(6); +static constexpr Gp w7 = Gp::make_r32(7); +static constexpr Gp w8 = Gp::make_r32(8); +static constexpr Gp w9 = Gp::make_r32(9); +static constexpr Gp w10 = Gp::make_r32(10); +static constexpr Gp w11 = Gp::make_r32(11); +static constexpr Gp w12 = Gp::make_r32(12); +static constexpr Gp w13 = Gp::make_r32(13); +static constexpr Gp w14 = Gp::make_r32(14); +static constexpr Gp w15 = Gp::make_r32(15); +static constexpr Gp w16 = Gp::make_r32(16); +static constexpr Gp w17 = Gp::make_r32(17); +static constexpr Gp w18 = Gp::make_r32(18); +static constexpr Gp w19 = Gp::make_r32(19); +static constexpr Gp w20 = Gp::make_r32(20); +static constexpr Gp w21 = Gp::make_r32(21); +static constexpr Gp w22 = Gp::make_r32(22); +static constexpr Gp w23 = Gp::make_r32(23); +static constexpr Gp w24 = Gp::make_r32(24); +static constexpr Gp w25 = Gp::make_r32(25); +static constexpr Gp w26 = Gp::make_r32(26); +static constexpr Gp w27 = Gp::make_r32(27); +static constexpr Gp w28 = Gp::make_r32(28); +static constexpr Gp w29 = Gp::make_r32(29); +static constexpr Gp w30 = Gp::make_r32(30); +static constexpr Gp wzr = Gp::make_r32(Gp::kIdZr); +static constexpr Gp wsp = Gp::make_r32(Gp::kIdSp); -static constexpr VecB b0 = VecB(0); -static constexpr VecB b1 = VecB(1); -static constexpr VecB b2 = VecB(2); -static constexpr VecB b3 = VecB(3); -static constexpr VecB b4 = VecB(4); -static constexpr VecB b5 = VecB(5); -static constexpr VecB b6 = VecB(6); -static constexpr VecB b7 = VecB(7); -static constexpr VecB b8 = VecB(8); -static constexpr VecB b9 = VecB(9); -static constexpr VecB b10 = VecB(10); -static constexpr VecB b11 = VecB(11); -static constexpr VecB b12 = VecB(12); -static constexpr VecB b13 = VecB(13); -static constexpr VecB b14 = VecB(14); -static constexpr VecB b15 = VecB(15); -static constexpr VecB b16 = VecB(16); -static constexpr VecB b17 = VecB(17); -static constexpr VecB b18 = VecB(18); -static constexpr VecB b19 = VecB(19); -static constexpr VecB b20 = VecB(20); -static constexpr VecB b21 = VecB(21); -static constexpr VecB b22 = VecB(22); -static constexpr VecB b23 = VecB(23); -static constexpr VecB b24 = VecB(24); -static constexpr VecB b25 = VecB(25); -static constexpr VecB b26 = VecB(26); -static constexpr VecB b27 = VecB(27); -static constexpr VecB b28 = VecB(28); -static constexpr VecB b29 = VecB(29); -static constexpr VecB b30 = VecB(30); -static constexpr VecB b31 = VecB(31); +static constexpr Gp x0 = Gp::make_r64(0); +static constexpr Gp x1 = Gp::make_r64(1); +static constexpr Gp x2 = Gp::make_r64(2); +static constexpr Gp x3 = Gp::make_r64(3); +static constexpr Gp x4 = Gp::make_r64(4); +static constexpr Gp x5 = Gp::make_r64(5); +static constexpr Gp x6 = Gp::make_r64(6); +static constexpr Gp x7 = Gp::make_r64(7); +static constexpr Gp x8 = Gp::make_r64(8); +static constexpr Gp x9 = Gp::make_r64(9); +static constexpr Gp x10 = Gp::make_r64(10); +static constexpr Gp x11 = Gp::make_r64(11); +static constexpr Gp x12 = Gp::make_r64(12); +static constexpr Gp x13 = Gp::make_r64(13); +static constexpr Gp x14 = Gp::make_r64(14); +static constexpr Gp x15 = Gp::make_r64(15); +static constexpr Gp x16 = Gp::make_r64(16); +static constexpr Gp x17 = Gp::make_r64(17); +static constexpr Gp x18 = Gp::make_r64(18); +static constexpr Gp x19 = Gp::make_r64(19); +static constexpr Gp x20 = Gp::make_r64(20); +static constexpr Gp x21 = Gp::make_r64(21); +static constexpr Gp x22 = Gp::make_r64(22); +static constexpr Gp x23 = Gp::make_r64(23); +static constexpr Gp x24 = Gp::make_r64(24); +static constexpr Gp x25 = Gp::make_r64(25); +static constexpr Gp x26 = Gp::make_r64(26); +static constexpr Gp x27 = Gp::make_r64(27); +static constexpr Gp x28 = Gp::make_r64(28); +static constexpr Gp x29 = Gp::make_r64(29); +static constexpr Gp x30 = Gp::make_r64(30); +static constexpr Gp xzr = Gp::make_r64(Gp::kIdZr); +static constexpr Gp sp = Gp::make_r64(Gp::kIdSp); -static constexpr VecH h0 = VecH(0); -static constexpr VecH h1 = VecH(1); -static constexpr VecH h2 = VecH(2); -static constexpr VecH h3 = VecH(3); -static constexpr VecH h4 = VecH(4); -static constexpr VecH h5 = VecH(5); -static constexpr VecH h6 = VecH(6); -static constexpr VecH h7 = VecH(7); -static constexpr VecH h8 = VecH(8); -static constexpr VecH h9 = VecH(9); -static constexpr VecH h10 = VecH(10); -static constexpr VecH h11 = VecH(11); -static constexpr VecH h12 = VecH(12); -static constexpr VecH h13 = VecH(13); -static constexpr VecH h14 = VecH(14); -static constexpr VecH h15 = VecH(15); -static constexpr VecH h16 = VecH(16); -static constexpr VecH h17 = VecH(17); -static constexpr VecH h18 = VecH(18); -static constexpr VecH h19 = VecH(19); -static constexpr VecH h20 = VecH(20); -static constexpr VecH h21 = VecH(21); -static constexpr VecH h22 = VecH(22); -static constexpr VecH h23 = VecH(23); -static constexpr VecH h24 = VecH(24); -static constexpr VecH h25 = VecH(25); -static constexpr VecH h26 = VecH(26); -static constexpr VecH h27 = VecH(27); -static constexpr VecH h28 = VecH(28); -static constexpr VecH h29 = VecH(29); -static constexpr VecH h30 = VecH(30); -static constexpr VecH h31 = VecH(31); +static constexpr Vec b0 = Vec::make_v8(0); +static constexpr Vec b1 = Vec::make_v8(1); +static constexpr Vec b2 = Vec::make_v8(2); +static constexpr Vec b3 = Vec::make_v8(3); +static constexpr Vec b4 = Vec::make_v8(4); +static constexpr Vec b5 = Vec::make_v8(5); +static constexpr Vec b6 = Vec::make_v8(6); +static constexpr Vec b7 = Vec::make_v8(7); +static constexpr Vec b8 = Vec::make_v8(8); +static constexpr Vec b9 = Vec::make_v8(9); +static constexpr Vec b10 = Vec::make_v8(10); +static constexpr Vec b11 = Vec::make_v8(11); +static constexpr Vec b12 = Vec::make_v8(12); +static constexpr Vec b13 = Vec::make_v8(13); +static constexpr Vec b14 = Vec::make_v8(14); +static constexpr Vec b15 = Vec::make_v8(15); +static constexpr Vec b16 = Vec::make_v8(16); +static constexpr Vec b17 = Vec::make_v8(17); +static constexpr Vec b18 = Vec::make_v8(18); +static constexpr Vec b19 = Vec::make_v8(19); +static constexpr Vec b20 = Vec::make_v8(20); +static constexpr Vec b21 = Vec::make_v8(21); +static constexpr Vec b22 = Vec::make_v8(22); +static constexpr Vec b23 = Vec::make_v8(23); +static constexpr Vec b24 = Vec::make_v8(24); +static constexpr Vec b25 = Vec::make_v8(25); +static constexpr Vec b26 = Vec::make_v8(26); +static constexpr Vec b27 = Vec::make_v8(27); +static constexpr Vec b28 = Vec::make_v8(28); +static constexpr Vec b29 = Vec::make_v8(29); +static constexpr Vec b30 = Vec::make_v8(30); +static constexpr Vec b31 = Vec::make_v8(31); -static constexpr VecS s0 = VecS(0); -static constexpr VecS s1 = VecS(1); -static constexpr VecS s2 = VecS(2); -static constexpr VecS s3 = VecS(3); -static constexpr VecS s4 = VecS(4); -static constexpr VecS s5 = VecS(5); -static constexpr VecS s6 = VecS(6); -static constexpr VecS s7 = VecS(7); -static constexpr VecS s8 = VecS(8); -static constexpr VecS s9 = VecS(9); -static constexpr VecS s10 = VecS(10); -static constexpr VecS s11 = VecS(11); -static constexpr VecS s12 = VecS(12); -static constexpr VecS s13 = VecS(13); -static constexpr VecS s14 = VecS(14); -static constexpr VecS s15 = VecS(15); -static constexpr VecS s16 = VecS(16); -static constexpr VecS s17 = VecS(17); -static constexpr VecS s18 = VecS(18); -static constexpr VecS s19 = VecS(19); -static constexpr VecS s20 = VecS(20); -static constexpr VecS s21 = VecS(21); -static constexpr VecS s22 = VecS(22); -static constexpr VecS s23 = VecS(23); -static constexpr VecS s24 = VecS(24); -static constexpr VecS s25 = VecS(25); -static constexpr VecS s26 = VecS(26); -static constexpr VecS s27 = VecS(27); -static constexpr VecS s28 = VecS(28); -static constexpr VecS s29 = VecS(29); -static constexpr VecS s30 = VecS(30); -static constexpr VecS s31 = VecS(31); +static constexpr Vec h0 = Vec::make_v16(0); +static constexpr Vec h1 = Vec::make_v16(1); +static constexpr Vec h2 = Vec::make_v16(2); +static constexpr Vec h3 = Vec::make_v16(3); +static constexpr Vec h4 = Vec::make_v16(4); +static constexpr Vec h5 = Vec::make_v16(5); +static constexpr Vec h6 = Vec::make_v16(6); +static constexpr Vec h7 = Vec::make_v16(7); +static constexpr Vec h8 = Vec::make_v16(8); +static constexpr Vec h9 = Vec::make_v16(9); +static constexpr Vec h10 = Vec::make_v16(10); +static constexpr Vec h11 = Vec::make_v16(11); +static constexpr Vec h12 = Vec::make_v16(12); +static constexpr Vec h13 = Vec::make_v16(13); +static constexpr Vec h14 = Vec::make_v16(14); +static constexpr Vec h15 = Vec::make_v16(15); +static constexpr Vec h16 = Vec::make_v16(16); +static constexpr Vec h17 = Vec::make_v16(17); +static constexpr Vec h18 = Vec::make_v16(18); +static constexpr Vec h19 = Vec::make_v16(19); +static constexpr Vec h20 = Vec::make_v16(20); +static constexpr Vec h21 = Vec::make_v16(21); +static constexpr Vec h22 = Vec::make_v16(22); +static constexpr Vec h23 = Vec::make_v16(23); +static constexpr Vec h24 = Vec::make_v16(24); +static constexpr Vec h25 = Vec::make_v16(25); +static constexpr Vec h26 = Vec::make_v16(26); +static constexpr Vec h27 = Vec::make_v16(27); +static constexpr Vec h28 = Vec::make_v16(28); +static constexpr Vec h29 = Vec::make_v16(29); +static constexpr Vec h30 = Vec::make_v16(30); +static constexpr Vec h31 = Vec::make_v16(31); -static constexpr VecD d0 = VecD(0); -static constexpr VecD d1 = VecD(1); -static constexpr VecD d2 = VecD(2); -static constexpr VecD d3 = VecD(3); -static constexpr VecD d4 = VecD(4); -static constexpr VecD d5 = VecD(5); -static constexpr VecD d6 = VecD(6); -static constexpr VecD d7 = VecD(7); -static constexpr VecD d8 = VecD(8); -static constexpr VecD d9 = VecD(9); -static constexpr VecD d10 = VecD(10); -static constexpr VecD d11 = VecD(11); -static constexpr VecD d12 = VecD(12); -static constexpr VecD d13 = VecD(13); -static constexpr VecD d14 = VecD(14); -static constexpr VecD d15 = VecD(15); -static constexpr VecD d16 = VecD(16); -static constexpr VecD d17 = VecD(17); -static constexpr VecD d18 = VecD(18); -static constexpr VecD d19 = VecD(19); -static constexpr VecD d20 = VecD(20); -static constexpr VecD d21 = VecD(21); -static constexpr VecD d22 = VecD(22); -static constexpr VecD d23 = VecD(23); -static constexpr VecD d24 = VecD(24); -static constexpr VecD d25 = VecD(25); -static constexpr VecD d26 = VecD(26); -static constexpr VecD d27 = VecD(27); -static constexpr VecD d28 = VecD(28); -static constexpr VecD d29 = VecD(29); -static constexpr VecD d30 = VecD(30); -static constexpr VecD d31 = VecD(31); +static constexpr Vec s0 = Vec::make_v32(0); +static constexpr Vec s1 = Vec::make_v32(1); +static constexpr Vec s2 = Vec::make_v32(2); +static constexpr Vec s3 = Vec::make_v32(3); +static constexpr Vec s4 = Vec::make_v32(4); +static constexpr Vec s5 = Vec::make_v32(5); +static constexpr Vec s6 = Vec::make_v32(6); +static constexpr Vec s7 = Vec::make_v32(7); +static constexpr Vec s8 = Vec::make_v32(8); +static constexpr Vec s9 = Vec::make_v32(9); +static constexpr Vec s10 = Vec::make_v32(10); +static constexpr Vec s11 = Vec::make_v32(11); +static constexpr Vec s12 = Vec::make_v32(12); +static constexpr Vec s13 = Vec::make_v32(13); +static constexpr Vec s14 = Vec::make_v32(14); +static constexpr Vec s15 = Vec::make_v32(15); +static constexpr Vec s16 = Vec::make_v32(16); +static constexpr Vec s17 = Vec::make_v32(17); +static constexpr Vec s18 = Vec::make_v32(18); +static constexpr Vec s19 = Vec::make_v32(19); +static constexpr Vec s20 = Vec::make_v32(20); +static constexpr Vec s21 = Vec::make_v32(21); +static constexpr Vec s22 = Vec::make_v32(22); +static constexpr Vec s23 = Vec::make_v32(23); +static constexpr Vec s24 = Vec::make_v32(24); +static constexpr Vec s25 = Vec::make_v32(25); +static constexpr Vec s26 = Vec::make_v32(26); +static constexpr Vec s27 = Vec::make_v32(27); +static constexpr Vec s28 = Vec::make_v32(28); +static constexpr Vec s29 = Vec::make_v32(29); +static constexpr Vec s30 = Vec::make_v32(30); +static constexpr Vec s31 = Vec::make_v32(31); -static constexpr VecV q0 = VecV(0); -static constexpr VecV q1 = VecV(1); -static constexpr VecV q2 = VecV(2); -static constexpr VecV q3 = VecV(3); -static constexpr VecV q4 = VecV(4); -static constexpr VecV q5 = VecV(5); -static constexpr VecV q6 = VecV(6); -static constexpr VecV q7 = VecV(7); -static constexpr VecV q8 = VecV(8); -static constexpr VecV q9 = VecV(9); -static constexpr VecV q10 = VecV(10); -static constexpr VecV q11 = VecV(11); -static constexpr VecV q12 = VecV(12); -static constexpr VecV q13 = VecV(13); -static constexpr VecV q14 = VecV(14); -static constexpr VecV q15 = VecV(15); -static constexpr VecV q16 = VecV(16); -static constexpr VecV q17 = VecV(17); -static constexpr VecV q18 = VecV(18); -static constexpr VecV q19 = VecV(19); -static constexpr VecV q20 = VecV(20); -static constexpr VecV q21 = VecV(21); -static constexpr VecV q22 = VecV(22); -static constexpr VecV q23 = VecV(23); -static constexpr VecV q24 = VecV(24); -static constexpr VecV q25 = VecV(25); -static constexpr VecV q26 = VecV(26); -static constexpr VecV q27 = VecV(27); -static constexpr VecV q28 = VecV(28); -static constexpr VecV q29 = VecV(29); -static constexpr VecV q30 = VecV(30); -static constexpr VecV q31 = VecV(31); +static constexpr Vec d0 = Vec::make_v64(0); +static constexpr Vec d1 = Vec::make_v64(1); +static constexpr Vec d2 = Vec::make_v64(2); +static constexpr Vec d3 = Vec::make_v64(3); +static constexpr Vec d4 = Vec::make_v64(4); +static constexpr Vec d5 = Vec::make_v64(5); +static constexpr Vec d6 = Vec::make_v64(6); +static constexpr Vec d7 = Vec::make_v64(7); +static constexpr Vec d8 = Vec::make_v64(8); +static constexpr Vec d9 = Vec::make_v64(9); +static constexpr Vec d10 = Vec::make_v64(10); +static constexpr Vec d11 = Vec::make_v64(11); +static constexpr Vec d12 = Vec::make_v64(12); +static constexpr Vec d13 = Vec::make_v64(13); +static constexpr Vec d14 = Vec::make_v64(14); +static constexpr Vec d15 = Vec::make_v64(15); +static constexpr Vec d16 = Vec::make_v64(16); +static constexpr Vec d17 = Vec::make_v64(17); +static constexpr Vec d18 = Vec::make_v64(18); +static constexpr Vec d19 = Vec::make_v64(19); +static constexpr Vec d20 = Vec::make_v64(20); +static constexpr Vec d21 = Vec::make_v64(21); +static constexpr Vec d22 = Vec::make_v64(22); +static constexpr Vec d23 = Vec::make_v64(23); +static constexpr Vec d24 = Vec::make_v64(24); +static constexpr Vec d25 = Vec::make_v64(25); +static constexpr Vec d26 = Vec::make_v64(26); +static constexpr Vec d27 = Vec::make_v64(27); +static constexpr Vec d28 = Vec::make_v64(28); +static constexpr Vec d29 = Vec::make_v64(29); +static constexpr Vec d30 = Vec::make_v64(30); +static constexpr Vec d31 = Vec::make_v64(31); -static constexpr VecV v0 = VecV(0); -static constexpr VecV v1 = VecV(1); -static constexpr VecV v2 = VecV(2); -static constexpr VecV v3 = VecV(3); -static constexpr VecV v4 = VecV(4); -static constexpr VecV v5 = VecV(5); -static constexpr VecV v6 = VecV(6); -static constexpr VecV v7 = VecV(7); -static constexpr VecV v8 = VecV(8); -static constexpr VecV v9 = VecV(9); -static constexpr VecV v10 = VecV(10); -static constexpr VecV v11 = VecV(11); -static constexpr VecV v12 = VecV(12); -static constexpr VecV v13 = VecV(13); -static constexpr VecV v14 = VecV(14); -static constexpr VecV v15 = VecV(15); -static constexpr VecV v16 = VecV(16); -static constexpr VecV v17 = VecV(17); -static constexpr VecV v18 = VecV(18); -static constexpr VecV v19 = VecV(19); -static constexpr VecV v20 = VecV(20); -static constexpr VecV v21 = VecV(21); -static constexpr VecV v22 = VecV(22); -static constexpr VecV v23 = VecV(23); -static constexpr VecV v24 = VecV(24); -static constexpr VecV v25 = VecV(25); -static constexpr VecV v26 = VecV(26); -static constexpr VecV v27 = VecV(27); -static constexpr VecV v28 = VecV(28); -static constexpr VecV v29 = VecV(29); -static constexpr VecV v30 = VecV(30); -static constexpr VecV v31 = VecV(31); +static constexpr Vec q0 = Vec::make_v128(0); +static constexpr Vec q1 = Vec::make_v128(1); +static constexpr Vec q2 = Vec::make_v128(2); +static constexpr Vec q3 = Vec::make_v128(3); +static constexpr Vec q4 = Vec::make_v128(4); +static constexpr Vec q5 = Vec::make_v128(5); +static constexpr Vec q6 = Vec::make_v128(6); +static constexpr Vec q7 = Vec::make_v128(7); +static constexpr Vec q8 = Vec::make_v128(8); +static constexpr Vec q9 = Vec::make_v128(9); +static constexpr Vec q10 = Vec::make_v128(10); +static constexpr Vec q11 = Vec::make_v128(11); +static constexpr Vec q12 = Vec::make_v128(12); +static constexpr Vec q13 = Vec::make_v128(13); +static constexpr Vec q14 = Vec::make_v128(14); +static constexpr Vec q15 = Vec::make_v128(15); +static constexpr Vec q16 = Vec::make_v128(16); +static constexpr Vec q17 = Vec::make_v128(17); +static constexpr Vec q18 = Vec::make_v128(18); +static constexpr Vec q19 = Vec::make_v128(19); +static constexpr Vec q20 = Vec::make_v128(20); +static constexpr Vec q21 = Vec::make_v128(21); +static constexpr Vec q22 = Vec::make_v128(22); +static constexpr Vec q23 = Vec::make_v128(23); +static constexpr Vec q24 = Vec::make_v128(24); +static constexpr Vec q25 = Vec::make_v128(25); +static constexpr Vec q26 = Vec::make_v128(26); +static constexpr Vec q27 = Vec::make_v128(27); +static constexpr Vec q28 = Vec::make_v128(28); +static constexpr Vec q29 = Vec::make_v128(29); +static constexpr Vec q30 = Vec::make_v128(30); +static constexpr Vec q31 = Vec::make_v128(31); + +static constexpr Vec v0 = Vec::make_v128(0); +static constexpr Vec v1 = Vec::make_v128(1); +static constexpr Vec v2 = Vec::make_v128(2); +static constexpr Vec v3 = Vec::make_v128(3); +static constexpr Vec v4 = Vec::make_v128(4); +static constexpr Vec v5 = Vec::make_v128(5); +static constexpr Vec v6 = Vec::make_v128(6); +static constexpr Vec v7 = Vec::make_v128(7); +static constexpr Vec v8 = Vec::make_v128(8); +static constexpr Vec v9 = Vec::make_v128(9); +static constexpr Vec v10 = Vec::make_v128(10); +static constexpr Vec v11 = Vec::make_v128(11); +static constexpr Vec v12 = Vec::make_v128(12); +static constexpr Vec v13 = Vec::make_v128(13); +static constexpr Vec v14 = Vec::make_v128(14); +static constexpr Vec v15 = Vec::make_v128(15); +static constexpr Vec v16 = Vec::make_v128(16); +static constexpr Vec v17 = Vec::make_v128(17); +static constexpr Vec v18 = Vec::make_v128(18); +static constexpr Vec v19 = Vec::make_v128(19); +static constexpr Vec v20 = Vec::make_v128(20); +static constexpr Vec v21 = Vec::make_v128(21); +static constexpr Vec v22 = Vec::make_v128(22); +static constexpr Vec v23 = Vec::make_v128(23); +static constexpr Vec v24 = Vec::make_v128(24); +static constexpr Vec v25 = Vec::make_v128(25); +static constexpr Vec v26 = Vec::make_v128(26); +static constexpr Vec v27 = Vec::make_v128(27); +static constexpr Vec v28 = Vec::make_v128(28); +static constexpr Vec v29 = Vec::make_v128(29); +static constexpr Vec v30 = Vec::make_v128(30); +static constexpr Vec v31 = Vec::make_v128(31); #ifndef _DOXYGEN } // {regs} @@ -755,6 +1066,14 @@ static ASMJIT_INLINE_CONSTEXPR Mem ptr(const Label& base, int32_t offset = 0) no return Mem(base, offset); } +//! Creates `[base]` absolute memory operand (AArch32 or AArch64). +//! +//! \note The concept of absolute memory operands doesn't exist on ARM, the ISA only provides PC relative addressing. +//! Absolute memory operands can only be used if it's known that the PC relative offset is encodable and that it +//! would be within the limits. Absolute address is also often output from disassemblers, so AsmJit supports it to +//! make it possible to assemble such output back. +static ASMJIT_INLINE_CONSTEXPR Mem ptr(uint64_t base) noexcept { return Mem(base); } + // TODO: [ARM] PC + offset address. #if 0 //! Creates `[PC + offset]` (relative) memory operand. @@ -769,14 +1088,4 @@ static ASMJIT_INLINE_CONSTEXPR Mem ptr(const PC& pc, int32_t offset = 0) noexcep ASMJIT_END_SUB_NAMESPACE -//! \cond INTERNAL -ASMJIT_BEGIN_NAMESPACE -ASMJIT_DEFINE_TYPE_ID(a64::GpW, TypeId::kInt32); -ASMJIT_DEFINE_TYPE_ID(a64::GpX, TypeId::kInt64); -ASMJIT_DEFINE_TYPE_ID(a64::VecS, TypeId::kFloat32x1); -ASMJIT_DEFINE_TYPE_ID(a64::VecD, TypeId::kFloat64x1); -ASMJIT_DEFINE_TYPE_ID(a64::VecV, TypeId::kInt32x4); -ASMJIT_END_NAMESPACE -//! \endcond - #endif // ASMJIT_ARM_A64OPERAND_H_INCLUDED diff --git a/src/asmjit/arm/a64rapass.cpp b/src/asmjit/arm/a64rapass.cpp index 005d45b..d681214 100644 --- a/src/asmjit/arm/a64rapass.cpp +++ b/src/asmjit/arm/a64rapass.cpp @@ -1,12 +1,13 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" #if !defined(ASMJIT_NO_AARCH64) && !defined(ASMJIT_NO_COMPILER) #include "../core/cpuinfo.h" +#include "../core/formatter_p.h" #include "../core/support.h" #include "../core/type.h" #include "../arm/a64assembler.h" @@ -99,13 +100,13 @@ public: Error onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept; [[nodiscard]] - Error moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_, BaseReg* out) noexcept; + Error moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_, Reg* out) noexcept; [[nodiscard]] Error moveImmToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_) noexcept; [[nodiscard]] - Error moveRegToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const BaseReg& reg) noexcept; + Error moveRegToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const Reg& reg) noexcept; [[nodiscard]] Error onBeforeRet(FuncRetNode* funcRet) noexcept; @@ -195,8 +196,8 @@ Error RACFGBuilder::onInst(InstNode* inst, InstControlFlow& controlType, RAInstB RegMask useRegs = _pass->_availableRegs[group]; RegMask outRegs = useRegs; - uint32_t useId = BaseReg::kIdBad; - uint32_t outId = BaseReg::kIdBad; + uint32_t useId = Reg::kIdBad; + uint32_t outId = Reg::kIdBad; uint32_t useRewriteMask = 0; uint32_t outRewriteMask = 0; @@ -227,7 +228,7 @@ Error RACFGBuilder::onInst(InstNode* inst, InstControlFlow& controlType, RAInstB } if (Support::test(flags, RATiedFlags::kUse)) { - useRewriteMask = Support::bitMask(inst->getRewriteIndex(®._baseId)); + useRewriteMask = Support::bitMask(inst->_getRewriteIndex(®._baseId)); if (opRwInfo.hasOpFlag(OpRWFlags::kRegPhysId)) { useId = opRwInfo.physId(); flags |= RATiedFlags::kUseFixed; @@ -240,7 +241,7 @@ Error RACFGBuilder::onInst(InstNode* inst, InstControlFlow& controlType, RAInstB } } else { - outRewriteMask = Support::bitMask(inst->getRewriteIndex(®._baseId)); + outRewriteMask = Support::bitMask(inst->_getRewriteIndex(®._baseId)); if (opRwInfo.hasOpFlag(OpRWFlags::kRegPhysId)) { outId = opRwInfo.physId(); flags |= RATiedFlags::kOutFixed; @@ -300,17 +301,17 @@ Error RACFGBuilder::onInst(InstNode* inst, InstControlFlow& controlType, RAInstB RegMask allocable = _pass->_availableRegs[group]; // Base registers have never fixed id on ARM. - const uint32_t useId = BaseReg::kIdBad; - const uint32_t outId = BaseReg::kIdBad; + const uint32_t useId = Reg::kIdBad; + const uint32_t outId = Reg::kIdBad; uint32_t useRewriteMask = 0; uint32_t outRewriteMask = 0; if (Support::test(flags, RATiedFlags::kUse)) { - useRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._baseId)); + useRewriteMask = Support::bitMask(inst->_getRewriteIndex(&mem._baseId)); } else { - outRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._baseId)); + outRewriteMask = Support::bitMask(inst->_getRewriteIndex(&mem._baseId)); } ASMJIT_PROPAGATE(ib.add(workReg, flags, allocable, useId, useRewriteMask, allocable, outId, outRewriteMask)); @@ -328,17 +329,17 @@ Error RACFGBuilder::onInst(InstNode* inst, InstControlFlow& controlType, RAInstB RegMask allocable = _pass->_availableRegs[group]; // Index registers have never fixed id on ARM. - const uint32_t useId = BaseReg::kIdBad; - const uint32_t outId = BaseReg::kIdBad; + const uint32_t useId = Reg::kIdBad; + const uint32_t outId = Reg::kIdBad; uint32_t useRewriteMask = 0; uint32_t outRewriteMask = 0; if (Support::test(flags, RATiedFlags::kUse)) { - useRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._data[Operand::kDataMemIndexId])); + useRewriteMask = Support::bitMask(inst->_getRewriteIndex(&mem._data[Operand::kDataMemIndexId])); } else { - outRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._data[Operand::kDataMemIndexId])); + outRewriteMask = Support::bitMask(inst->_getRewriteIndex(&mem._data[Operand::kDataMemIndexId])); } ASMJIT_PROPAGATE(ib.add(workReg, RATiedFlags::kUse | RATiedFlags::kRead, allocable, useId, useRewriteMask, allocable, outId, outRewriteMask)); @@ -382,7 +383,7 @@ Error RACFGBuilder::onBeforeInvoke(InvokeNode* invokeNode) noexcept { if (arg.isReg()) { RegGroup regGroup = workReg->group(); - RegGroup argGroup = Reg::groupOf(arg.regType()); + RegGroup argGroup = RegUtils::groupOf(arg.regType()); if (regGroup != argGroup) { // TODO: [ARM] Conversion is not supported. @@ -395,7 +396,7 @@ Error RACFGBuilder::onBeforeInvoke(InvokeNode* invokeNode) noexcept { } else if (op.isImm()) { if (arg.isReg()) { - BaseReg reg; + Reg reg; ASMJIT_PROPAGATE(moveImmToRegArg(invokeNode, arg, op.as(), ®)); invokeNode->_args[argIndex][valueIndex] = reg; } @@ -423,7 +424,7 @@ Error RACFGBuilder::onBeforeInvoke(InvokeNode* invokeNode) noexcept { if (ret.isReg()) { RegGroup regGroup = workReg->group(); - RegGroup retGroup = Reg::groupOf(ret.regType()); + RegGroup retGroup = RegUtils::groupOf(ret.regType()); if (regGroup != retGroup) { // TODO: [ARM] Conversion is not supported. @@ -474,7 +475,7 @@ Error RACFGBuilder::onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept } else if (arg.isReg()) { RegGroup regGroup = workReg->group(); - RegGroup argGroup = Reg::groupOf(arg.regType()); + RegGroup argGroup = RegUtils::groupOf(arg.regType()); if (regGroup == argGroup) { ASMJIT_PROPAGATE(ib.addCallArg(workReg, arg.regId())); @@ -498,7 +499,7 @@ Error RACFGBuilder::onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept if (ret.isReg()) { RegGroup regGroup = workReg->group(); - RegGroup retGroup = Reg::groupOf(ret.regType()); + RegGroup retGroup = RegUtils::groupOf(ret.regType()); if (regGroup == retGroup) { ASMJIT_PROPAGATE(ib.addCallRet(workReg, ret.regId())); @@ -522,7 +523,7 @@ Error RACFGBuilder::onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept // a64::RACFGBuilder - MoveImmToRegArg // =================================== -Error RACFGBuilder::moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_, BaseReg* out) noexcept { +Error RACFGBuilder::moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_, Reg* out) noexcept { DebugUtils::unused(invokeNode); ASMJIT_ASSERT(arg.isReg()); @@ -552,7 +553,7 @@ Error RACFGBuilder::moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg // ===================================== Error RACFGBuilder::moveImmToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_) noexcept { - BaseReg reg; + Reg reg; ASMJIT_PROPAGATE(moveImmToRegArg(invokeNode, arg, imm_, ®)); ASMJIT_PROPAGATE(moveRegToStackArg(invokeNode, arg, reg)); @@ -563,7 +564,7 @@ Error RACFGBuilder::moveImmToStackArg(InvokeNode* invokeNode, const FuncValue& a // a64::RACFGBuilder - MoveRegToStackArg // ===================================== -Error RACFGBuilder::moveRegToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const BaseReg& reg) noexcept { +Error RACFGBuilder::moveRegToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const Reg& reg) noexcept { DebugUtils::unused(invokeNode); Mem stackPtr = ptr(_pass->_sp.as(), arg.stackOffset()); @@ -613,7 +614,7 @@ Error RACFGBuilder::onRet(FuncRetNode* funcRet, RAInstBuilder& ib) noexcept { RegGroup group = workReg->group(); RegMask allocable = _pass->_availableRegs[group]; - ASMJIT_PROPAGATE(ib.add(workReg, RATiedFlags::kUse | RATiedFlags::kRead, allocable, ret.regId(), 0, 0, BaseReg::kIdBad, 0)); + ASMJIT_PROPAGATE(ib.add(workReg, RATiedFlags::kUse | RATiedFlags::kRead, allocable, ret.regId(), 0, 0, Reg::kIdBad, 0)); } } else { @@ -714,14 +715,14 @@ ASMJIT_FAVOR_SPEED Error ARMRAPass::_rewrite(BaseNode* first, BaseNode* stop) no uint32_t useId = tiedReg->useId(); while (useIt.hasNext()) { - inst->rewriteIdAtIndex(useIt.next(), useId); + inst->_rewriteIdAtIndex(useIt.next(), useId); } Support::BitWordIterator outIt(tiedReg->outRewriteMask()); uint32_t outId = tiedReg->outId(); while (outIt.hasNext()) { - inst->rewriteIdAtIndex(outIt.next(), outId); + inst->_rewriteIdAtIndex(outIt.next(), outId); } } @@ -767,7 +768,7 @@ ASMJIT_FAVOR_SPEED Error ARMRAPass::_rewrite(BaseNode* first, BaseNode* stop) no RAStackSlot* slot = workReg->stackSlot(); int32_t offset = slot->offset(); - mem._setBase(_sp.type(), slot->baseRegId()); + mem._setBase(_sp.regType(), slot->baseRegId()); mem.clearRegHome(); mem.addOffsetLo32(offset); } @@ -788,8 +789,8 @@ ASMJIT_FAVOR_SPEED Error ARMRAPass::_rewrite(BaseNode* first, BaseNode* stop) no return DebugUtils::errored(kErrorInvalidAddressIndex); } - GpX dst(inst->op(0).as().id()); - GpX base(mem.baseId()); + Gp dst = Gp::make_r64(inst->op(0).as().id()); + Gp base = Gp::make_r64(mem.baseId()); InstId arithInstId = offset < 0 ? Inst::kIdSub : Inst::kIdAdd; uint64_t absOffset = offset < 0 ? Support::neg(uint64_t(offset)) : uint64_t(offset); @@ -842,14 +843,15 @@ Error ARMRAPass::updateStackFrame() noexcept { Error ARMRAPass::emitMove(uint32_t workId, uint32_t dstPhysId, uint32_t srcPhysId) noexcept { RAWorkReg* wReg = workRegById(workId); - BaseReg dst(wReg->signature(), dstPhysId); - BaseReg src(wReg->signature(), srcPhysId); + Reg dst(wReg->signature(), dstPhysId); + Reg src(wReg->signature(), srcPhysId); const char* comment = nullptr; #ifndef ASMJIT_NO_LOGGING if (hasDiagnosticOption(DiagnosticOptions::kRAAnnotate)) { - _tmpString.assignFormat(" %s", workRegById(workId)->name()); + _tmpString.clear(); + Formatter::formatVirtRegNameWithPrefix(_tmpString, " ", 7u, wReg->virtReg()); comment = _tmpString.data(); } #endif @@ -864,14 +866,15 @@ Error ARMRAPass::emitSwap(uint32_t aWorkId, uint32_t aPhysId, uint32_t bWorkId, Error ARMRAPass::emitLoad(uint32_t workId, uint32_t dstPhysId) noexcept { RAWorkReg* wReg = workRegById(workId); - BaseReg dstReg(wReg->signature(), dstPhysId); + Reg dstReg(wReg->signature(), dstPhysId); BaseMem srcMem(workRegAsMem(wReg)); const char* comment = nullptr; #ifndef ASMJIT_NO_LOGGING if (hasDiagnosticOption(DiagnosticOptions::kRAAnnotate)) { - _tmpString.assignFormat(" %s", workRegById(workId)->name()); + _tmpString.clear(); + Formatter::formatVirtRegNameWithPrefix(_tmpString, " ", 7u, wReg->virtReg()); comment = _tmpString.data(); } #endif @@ -882,13 +885,14 @@ Error ARMRAPass::emitLoad(uint32_t workId, uint32_t dstPhysId) noexcept { Error ARMRAPass::emitSave(uint32_t workId, uint32_t srcPhysId) noexcept { RAWorkReg* wReg = workRegById(workId); BaseMem dstMem(workRegAsMem(wReg)); - BaseReg srcReg(wReg->signature(), srcPhysId); + Reg srcReg(wReg->signature(), srcPhysId); const char* comment = nullptr; #ifndef ASMJIT_NO_LOGGING if (hasDiagnosticOption(DiagnosticOptions::kRAAnnotate)) { - _tmpString.assignFormat(" %s", workRegById(workId)->name()); + _tmpString.clear(); + Formatter::formatVirtRegNameWithPrefix(_tmpString, " ", 7u, wReg->virtReg()); comment = _tmpString.data(); } #endif diff --git a/src/asmjit/arm/a64rapass_p.h b/src/asmjit/arm/a64rapass_p.h index f4ccdd1..908f661 100644 --- a/src/asmjit/arm/a64rapass_p.h +++ b/src/asmjit/arm/a64rapass_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64RAPASS_P_H_INCLUDED diff --git a/src/asmjit/arm/armformatter.cpp b/src/asmjit/arm/armformatter.cpp index 79e1d62..c4e6712 100644 --- a/src/asmjit/arm/armformatter.cpp +++ b/src/asmjit/arm/armformatter.cpp @@ -1,11 +1,12 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" #ifndef ASMJIT_NO_LOGGING +#include "../core/formatter_p.h" #include "../core/misc_p.h" #include "../core/support.h" #include "../arm/armformatter_p.h" @@ -48,6 +49,7 @@ Error FormatterInternal::formatFeature(String& sb, uint32_t featureId) noexcept "CHK\0" "CLRBHB\0" "CMOW\0" + "CMPBR\0" "CONSTPACFIELD\0" "CPA\0" "CPA2\0" @@ -69,6 +71,10 @@ Error FormatterInternal::formatFeature(String& sb, uint32_t featureId) noexcept "ECV\0" "EDHSR\0" "EDSP\0" + "F8E4M3\0" + "F8E5M2\0" + "F8F16MM\0" + "F8F32MM\0" "FAMINMAX\0" "FCMA\0" "FGT\0" @@ -85,6 +91,7 @@ Error FormatterInternal::formatFeature(String& sb, uint32_t featureId) noexcept "FP8DOT4\0" "FP8FMA\0" "FPMR\0" + "FPRCVT\0" "FRINTTS\0" "GCS\0" "HACDBS\0" @@ -107,9 +114,12 @@ Error FormatterInternal::formatFeature(String& sb, uint32_t featureId) noexcept "LS64\0" "LS64_ACCDATA\0" "LS64_V\0" + "LS64WB\0" "LSE\0" "LSE128\0" "LSE2\0" + "LSFE\0" + "LSUI\0" "LUT\0" "LVA\0" "LVA3\0" @@ -131,6 +141,7 @@ Error FormatterInternal::formatFeature(String& sb, uint32_t featureId) noexcept "NMI\0" "NV\0" "NV2\0" + "OCCMO\0" "PAN\0" "PAN2\0" "PAN3\0" @@ -166,6 +177,8 @@ Error FormatterInternal::formatFeature(String& sb, uint32_t featureId) noexcept "SME\0" "SME2\0" "SME2_1\0" + "SME2_2\0" + "SME_AES\0" "SME_B16B16\0" "SME_B16F32\0" "SME_BI32I32\0" @@ -180,6 +193,8 @@ Error FormatterInternal::formatFeature(String& sb, uint32_t featureId) noexcept "SME_I16I64\0" "SME_I8I32\0" "SME_LUTv2\0" + "SME_MOP4\0" + "SME_TMOP\0" "SPE\0" "SPE1_1\0" "SPE1_2\0" @@ -196,17 +211,25 @@ Error FormatterInternal::formatFeature(String& sb, uint32_t featureId) noexcept "SPMU\0" "SSBS\0" "SSBS2\0" + "SSVE_AES\0" + "SSVE_BITPERM\0" + "SSVE_FEXPA\0" "SSVE_FP8DOT2\0" "SSVE_FP8DOT4\0" "SSVE_FP8FMA\0" "SVE\0" "SVE2\0" "SVE2_1\0" + "SVE2_2\0" "SVE_AES\0" + "SVE_AES2\0" "SVE_B16B16\0" "SVE_BF16\0" + "SVE_BFSCALE\0" "SVE_BITPERM\0" "SVE_EBF16\0" + "SVE_ELTPERM\0" + "SVE_F16MM\0" "SVE_F32MM\0" "SVE_F64MM\0" "SVE_I8MM\0" @@ -232,19 +255,21 @@ Error FormatterInternal::formatFeature(String& sb, uint32_t featureId) noexcept static const uint16_t sFeatureIndex[] = { 0, 5, 11, 17, 24, 30, 38, 43, 49, 53, 57, 61, 66, 73, 79, 85, 90, 95, 99, - 103, 109, 113, 120, 125, 139, 143, 148, 154, 160, 165, 170, 177, 182, 187, - 191, 195, 203, 207, 212, 217, 223, 229, 233, 239, 244, 253, 258, 262, 267, - 271, 277, 284, 289, 292, 297, 306, 310, 318, 326, 333, 338, 346, 350, 357, - 364, 369, 375, 379, 383, 388, 394, 399, 405, 411, 415, 421, 425, 431, 438, - 445, 450, 463, 470, 474, 481, 486, 490, 494, 499, 503, 508, 513, 517, 522, - 527, 532, 547, 557, 576, 596, 608, 623, 638, 644, 648, 651, 655, 659, 664, - 669, 675, 680, 684, 690, 698, 702, 709, 714, 721, 725, 729, 733, 742, 748, - 754, 760, 766, 772, 778, 781, 788, 794, 799, 804, 811, 816, 823, 827, 831, - 835, 840, 847, 858, 869, 881, 892, 903, 914, 925, 935, 945, 954, 965, 976, - 986, 996, 1000, 1007, 1014, 1021, 1028, 1039, 1047, 1055, 1063, 1071, 1079, - 1087, 1096, 1101, 1106, 1112, 1125, 1138, 1150, 1154, 1159, 1166, 1174, 1185, - 1194, 1206, 1216, 1226, 1236, 1245, 1258, 1267, 1275, 1287, 1297, 1301, 1308, - 1318, 1324, 1328, 1332, 1336, 1344, 1348, 1355, 1360, 1364, 1367 + 103, 109, 113, 120, 125, 131, 145, 149, 154, 160, 166, 171, 176, 183, 188, + 193, 197, 201, 209, 213, 218, 223, 229, 235, 239, 245, 250, 257, 264, 272, + 280, 289, 294, 298, 303, 307, 313, 320, 325, 328, 333, 342, 346, 354, 362, + 369, 374, 381, 389, 393, 400, 407, 412, 418, 422, 426, 431, 437, 442, 448, + 454, 458, 464, 468, 474, 481, 488, 493, 506, 513, 520, 524, 531, 536, 541, + 546, 550, 554, 559, 563, 568, 573, 577, 582, 587, 592, 607, 617, 636, 656, + 668, 683, 698, 704, 708, 711, 715, 721, 725, 730, 735, 741, 746, 750, 756, + 764, 768, 775, 780, 787, 791, 795, 799, 808, 814, 820, 826, 832, 838, 844, + 847, 854, 860, 865, 870, 877, 882, 889, 893, 897, 901, 906, 913, 920, 928, + 939, 950, 962, 973, 984, 995, 1006, 1016, 1026, 1035, 1046, 1057, 1067, 1077, + 1086, 1095, 1099, 1106, 1113, 1120, 1127, 1138, 1146, 1154, 1162, 1170, 1178, + 1186, 1195, 1200, 1205, 1211, 1220, 1233, 1244, 1257, 1270, 1282, 1286, 1291, + 1298, 1305, 1313, 1322, 1333, 1342, 1354, 1366, 1376, 1388, 1398, 1408, 1418, + 1427, 1440, 1449, 1457, 1469, 1479, 1483, 1490, 1500, 1506, 1510, 1514, 1518, + 1526, 1530, 1537, 1542, 1546, 1549 }; // @EnumStringEnd@ @@ -333,14 +358,7 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatRegister( VirtReg* vReg = cc->virtRegById(rId); ASMJIT_ASSERT(vReg != nullptr); - const char* name = vReg->name(); - if (name && name[0] != '\0') { - ASMJIT_PROPAGATE(sb.append(name)); - } - else { - ASMJIT_PROPAGATE(sb.appendFormat("%%%u", unsigned(Operand::virtIdToIndex(rId)))); - } - + ASMJIT_PROPAGATE(Formatter::formatVirtRegName(sb, vReg)); virtRegFormatted = true; } } @@ -352,18 +370,18 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatRegister( if (!virtRegFormatted) { char letter = '\0'; switch (regType) { - case RegType::kARM_VecB: - case RegType::kARM_VecH: - case RegType::kARM_VecS: - case RegType::kARM_VecD: - case RegType::kARM_VecV: - letter = bhsdq[uint32_t(regType) - uint32_t(RegType::kARM_VecB)]; + case RegType::kVec8: + case RegType::kVec16: + case RegType::kVec32: + case RegType::kVec64: + case RegType::kVec128: + letter = bhsdq[uint32_t(regType) - uint32_t(RegType::kVec8)]; if (elementType) { letter = 'v'; } break; - case RegType::kARM_GpW: + case RegType::kGp32: if (Environment::is64Bit(arch)) { letter = 'w'; @@ -380,7 +398,7 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatRegister( } break; - case RegType::kARM_GpX: + case RegType::kGp64: if (Environment::is64Bit(arch)) { if (rId == a64::Gp::kIdZr) { return sb.append("xzr", 3); @@ -413,7 +431,7 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatRegister( FormatElementData elementData = formatElementDataTable[elementType]; uint32_t elementCount = elementData.elementCount; - if (regType == RegType::kARM_VecD) { + if (regType == RegType::kVec64) { elementCount /= 2u; } @@ -481,20 +499,20 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatOperand( const Operand_& op) noexcept { if (op.isReg()) { - const BaseReg& reg = op.as(); + const Reg& reg = op.as(); - uint32_t elementType = op._signature.getField(); - uint32_t elementIndex = op.as().elementIndex(); + uint32_t elementType = op._signature.getField(); + uint32_t elementIndex = op.as().elementIndex(); - if (!op.as().hasElementIndex()) { + if (!op.as().hasElementIndex()) { elementIndex = 0xFFFFFFFFu; } - return formatRegister(sb, flags, emitter, arch, reg.type(), reg.id(), elementType, elementIndex); + return formatRegister(sb, flags, emitter, arch, reg.regType(), reg.id(), elementType, elementIndex); } if (op.isMem()) { - const Mem& m = op.as(); + const a64::Mem& m = op.as(); ASMJIT_PROPAGATE(sb.append('[')); if (m.hasBase()) { @@ -591,7 +609,7 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatOperand( if (op.isRegList()) { const BaseRegList& regList = op.as(); - return formatRegisterList(sb, flags, emitter, arch, regList.type(), regList.list()); + return formatRegisterList(sb, flags, emitter, arch, regList.regType(), regList.list()); } return sb.append(""); diff --git a/src/asmjit/arm/armformatter_p.h b/src/asmjit/arm/armformatter_p.h index 20b4812..d75fd91 100644 --- a/src/asmjit/arm/armformatter_p.h +++ b/src/asmjit/arm/armformatter_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_ARMFORMATTER_P_H_INCLUDED diff --git a/src/asmjit/arm/armglobals.h b/src/asmjit/arm/armglobals.h index 851f670..e05ee74 100644 --- a/src/asmjit/arm/armglobals.h +++ b/src/asmjit/arm/armglobals.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_ARMGLOBALS_H_INCLUDED diff --git a/src/asmjit/arm/armoperand.h b/src/asmjit/arm/armoperand.h deleted file mode 100644 index d18ac00..0000000 --- a/src/asmjit/arm/armoperand.h +++ /dev/null @@ -1,399 +0,0 @@ -// This file is part of AsmJit project -// -// See asmjit.h or LICENSE.md for license and copyright information -// SPDX-License-Identifier: Zlib - -#ifndef ASMJIT_ARM_ARMOPERAND_H_INCLUDED -#define ASMJIT_ARM_ARMOPERAND_H_INCLUDED - -#include "../core/archtraits.h" -#include "../core/operand.h" -#include "../core/type.h" -#include "../arm/armglobals.h" - -ASMJIT_BEGIN_SUB_NAMESPACE(arm) - -//! \addtogroup asmjit_arm -//! \{ - -class Reg; -class Mem; - -//! Register traits (AArch32/AArch64). -//! -//! Register traits contains information about a particular register type. It's used by asmjit to setup register -//! information on-the-fly and to populate tables that contain register information (this way it's possible to -//! change register types and groups without having to reorder these tables). -template -struct RegTraits : public BaseRegTraits {}; - -//! \cond -// <--------------------+------------------------+------------------------+---+------------------+ -// | Reg-Type | Reg-Group |Sz | TypeId | -// <--------------------+------------------------+------------------------+---+------------------+ -ASMJIT_DEFINE_REG_TRAITS(RegType::kARM_GpW , RegGroup::kGp , 4 , TypeId::kInt32 ); // AArch32 & AArch64 -ASMJIT_DEFINE_REG_TRAITS(RegType::kARM_GpX , RegGroup::kGp , 8 , TypeId::kInt64 ); // AArch64 -ASMJIT_DEFINE_REG_TRAITS(RegType::kARM_VecB , RegGroup::kVec , 1 , TypeId::kVoid ); // AArch64 -ASMJIT_DEFINE_REG_TRAITS(RegType::kARM_VecH , RegGroup::kVec , 2 , TypeId::kVoid ); // AArch64 -ASMJIT_DEFINE_REG_TRAITS(RegType::kARM_VecS , RegGroup::kVec , 4 , TypeId::kInt32x1 ); // AArch32 & AArch64 -ASMJIT_DEFINE_REG_TRAITS(RegType::kARM_VecD , RegGroup::kVec , 8 , TypeId::kInt32x2 ); // AArch32 & AArch64 -ASMJIT_DEFINE_REG_TRAITS(RegType::kARM_VecQ , RegGroup::kVec , 16, TypeId::kInt32x4 ); // AArch32 & AArch64 -ASMJIT_DEFINE_REG_TRAITS(RegType::kARM_PC , RegGroup::kPC , 8 , TypeId::kInt64 ); // AArch64 -//! \endcond - -//! Register operand that can represent AArch32 and AArch64 registers. -class Reg : public BaseReg { -public: - ASMJIT_DEFINE_ABSTRACT_REG(Reg, BaseReg) - - //! Gets whether the register is either `R` or `W` register (32-bit). - ASMJIT_INLINE_CONSTEXPR bool isGpR() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is either `R` or `W` register (32-bit). - ASMJIT_INLINE_CONSTEXPR bool isGpW() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is an `X` register (64-bit). - ASMJIT_INLINE_CONSTEXPR bool isGpX() const noexcept { return baseSignature() == RegTraits::kSignature; } - - //! Gets whether the register is a VEC-B register (8-bit). - ASMJIT_INLINE_CONSTEXPR bool isVecB() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is a VEC-H register (16-bit). - ASMJIT_INLINE_CONSTEXPR bool isVecH() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is a VEC-S register (32-bit). - ASMJIT_INLINE_CONSTEXPR bool isVecS() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is a VEC-D register (64-bit). - ASMJIT_INLINE_CONSTEXPR bool isVecD() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is a VEC-Q register (128-bit). - ASMJIT_INLINE_CONSTEXPR bool isVecQ() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is either VEC-D (64-bit) or VEC-Q (128-bit). - ASMJIT_INLINE_CONSTEXPR bool isVecDOrQ() const noexcept { return uint32_t(type()) - uint32_t(RegType::kARM_VecD) <= 1u; } - //! Gets whether the register is a VEC-V register (128-bit). - ASMJIT_INLINE_CONSTEXPR bool isVecV() const noexcept { return baseSignature() == RegTraits::kSignature; } - - //! Gets whether the register is an 8-bit vector register or view, alias if \ref isVecB(). - ASMJIT_INLINE_CONSTEXPR bool isVec8() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is a 16-bit vector register or view, alias if \ref isVecH(). - ASMJIT_INLINE_CONSTEXPR bool isVec16() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is a 32-bit vector register or view, alias if \ref isVecS(). - ASMJIT_INLINE_CONSTEXPR bool isVec32() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is a 64-bit vector register or view, alias if \ref isVecD(). - ASMJIT_INLINE_CONSTEXPR bool isVec64() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is a 128-bit vector register or view, alias if \ref isVecQ(). - ASMJIT_INLINE_CONSTEXPR bool isVec128() const noexcept { return baseSignature() == RegTraits::kSignature; } - - template - ASMJIT_INLINE_CONSTEXPR void setRegT(uint32_t id) noexcept { - setSignature(RegTraits::kSignature); - setId(id); - } - - ASMJIT_INLINE_NODEBUG void setTypeAndId(RegType type, uint32_t id) noexcept { - setSignature(signatureOf(type)); - setId(id); - } - - static ASMJIT_INLINE_NODEBUG RegGroup groupOf(RegType type) noexcept { return ArchTraits::byArch(Arch::kAArch64).regTypeToGroup(type); } - static ASMJIT_INLINE_NODEBUG TypeId typeIdOf(RegType type) noexcept { return ArchTraits::byArch(Arch::kAArch64).regTypeToTypeId(type); } - static ASMJIT_INLINE_NODEBUG OperandSignature signatureOf(RegType type) noexcept { return ArchTraits::byArch(Arch::kAArch64).regTypeToSignature(type); } - - template - static ASMJIT_INLINE_CONSTEXPR RegGroup groupOfT() noexcept { return RegTraits::kGroup; } - - template - static ASMJIT_INLINE_CONSTEXPR TypeId typeIdOfT() noexcept { return RegTraits::kTypeId; } - - template - static ASMJIT_INLINE_CONSTEXPR OperandSignature signatureOfT() noexcept { return OperandSignature{RegTraits::kSignature}; } - - static ASMJIT_INLINE_NODEBUG bool isGpW(const Operand_& op) noexcept { return op.as().isGpW(); } - static ASMJIT_INLINE_NODEBUG bool isGpX(const Operand_& op) noexcept { return op.as().isGpX(); } - static ASMJIT_INLINE_NODEBUG bool isVecB(const Operand_& op) noexcept { return op.as().isVecB(); } - static ASMJIT_INLINE_NODEBUG bool isVecH(const Operand_& op) noexcept { return op.as().isVecH(); } - static ASMJIT_INLINE_NODEBUG bool isVecS(const Operand_& op) noexcept { return op.as().isVecS(); } - static ASMJIT_INLINE_NODEBUG bool isVecD(const Operand_& op) noexcept { return op.as().isVecD(); } - static ASMJIT_INLINE_NODEBUG bool isVecQ(const Operand_& op) noexcept { return op.as().isVecQ(); } - static ASMJIT_INLINE_NODEBUG bool isVecV(const Operand_& op) noexcept { return op.as().isVecV(); } - - static ASMJIT_INLINE_NODEBUG bool isGpW(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isGpW(op)) & unsigned(op.id() == id)); } - static ASMJIT_INLINE_NODEBUG bool isGpX(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isGpX(op)) & unsigned(op.id() == id)); } - static ASMJIT_INLINE_NODEBUG bool isVecB(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isVecB(op)) & unsigned(op.id() == id)); } - static ASMJIT_INLINE_NODEBUG bool isVecH(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isVecH(op)) & unsigned(op.id() == id)); } - static ASMJIT_INLINE_NODEBUG bool isVecS(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isVecS(op)) & unsigned(op.id() == id)); } - static ASMJIT_INLINE_NODEBUG bool isVecD(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isVecD(op)) & unsigned(op.id() == id)); } - static ASMJIT_INLINE_NODEBUG bool isVecQ(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isVecQ(op)) & unsigned(op.id() == id)); } - static ASMJIT_INLINE_NODEBUG bool isVecV(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isVecV(op)) & unsigned(op.id() == id)); } -}; - -//! Vector register base - a common base for both AArch32 & AArch64 vector register. -class BaseVec : public Reg { -public: - ASMJIT_DEFINE_ABSTRACT_REG(BaseVec, Reg) - - //! Additional signature bits used by a vector register. - enum AdditionalBits : uint32_t { - // Register element type (3 bits). - // |........|........|.XXX....|........| - kSignatureRegElementTypeShift = 12, - kSignatureRegElementTypeMask = 0x07 << kSignatureRegElementTypeShift, - - // Register has element index (1 bit). - // |........|........|X.......|........| - kSignatureRegElementFlagShift = 15, - kSignatureRegElementFlagMask = 0x01 << kSignatureRegElementFlagShift, - - // Register element index (4 bits). - // |........|....XXXX|........|........| - kSignatureRegElementIndexShift = 16, - kSignatureRegElementIndexMask = 0x0F << kSignatureRegElementIndexShift - }; - - //! Returns whether the register has element index (it's an element index access). - ASMJIT_INLINE_CONSTEXPR bool hasElementIndex() const noexcept { return _signature.hasField(); } - //! Returns element index of the register. - ASMJIT_INLINE_CONSTEXPR uint32_t elementIndex() const noexcept { return _signature.getField(); } - //! Sets element index of the register to `elementType`. - ASMJIT_INLINE_CONSTEXPR void setElementIndex(uint32_t elementIndex) noexcept { - _signature |= kSignatureRegElementFlagMask; - _signature.setField(elementIndex); - } - //! Resets element index of the register. - ASMJIT_INLINE_CONSTEXPR void resetElementIndex() noexcept { - _signature &= ~(kSignatureRegElementFlagMask | kSignatureRegElementIndexMask); - } -}; - -//! Memory operand (ARM). -class Mem : public BaseMem { -public: - //! \cond INTERNAL - //! Additional bits of operand's signature used by `arm::Mem`. - enum AdditionalBits : uint32_t { - // Index shift value (5 bits). - // |........|.....XXX|XX......|........| - kSignatureMemShiftValueShift = 14, - kSignatureMemShiftValueMask = 0x1Fu << kSignatureMemShiftValueShift, - - // Index shift operation (4 bits). - // |........|XXXX....|........|........| - kSignatureMemShiftOpShift = 20, - kSignatureMemShiftOpMask = 0x0Fu << kSignatureMemShiftOpShift, - - // Offset mode type (2 bits). - // |......XX|........|........|........| - kSignatureMemOffsetModeShift = 24, - kSignatureMemOffsetModeMask = 0x03u << kSignatureMemOffsetModeShift - }; - //! \endcond - - //! \name Construction & Destruction - //! \{ - - //! Construct a default `Mem` operand, that points to [0]. - ASMJIT_INLINE_CONSTEXPR Mem() noexcept - : BaseMem() {} - - ASMJIT_INLINE_CONSTEXPR Mem(const Mem& other) noexcept - : BaseMem(other) {} - - ASMJIT_INLINE_NODEBUG explicit Mem(Globals::NoInit_) noexcept - : BaseMem(Globals::NoInit) {} - - ASMJIT_INLINE_CONSTEXPR Mem(const Signature& signature, uint32_t baseId, uint32_t indexId, int32_t offset) noexcept - : BaseMem(signature, baseId, indexId, offset) {} - - ASMJIT_INLINE_CONSTEXPR explicit Mem(const Label& base, int32_t off = 0, Signature signature = Signature{0}) noexcept - : BaseMem(Signature::fromOpType(OperandType::kMem) | - Signature::fromMemBaseType(RegType::kLabelTag) | - signature, base.id(), 0, off) {} - - ASMJIT_INLINE_CONSTEXPR explicit Mem(const BaseReg& base, int32_t off = 0, Signature signature = Signature{0}) noexcept - : BaseMem(Signature::fromOpType(OperandType::kMem) | - Signature::fromMemBaseType(base.type()) | - signature, base.id(), 0, off) {} - - ASMJIT_INLINE_CONSTEXPR Mem(const BaseReg& base, const BaseReg& index, Signature signature = Signature{0}) noexcept - : BaseMem(Signature::fromOpType(OperandType::kMem) | - Signature::fromMemBaseType(base.type()) | - Signature::fromMemIndexType(index.type()) | - signature, base.id(), index.id(), 0) {} - - ASMJIT_INLINE_CONSTEXPR Mem(const BaseReg& base, const BaseReg& index, const Shift& shift, Signature signature = Signature{0}) noexcept - : BaseMem(Signature::fromOpType(OperandType::kMem) | - Signature::fromMemBaseType(base.type()) | - Signature::fromMemIndexType(index.type()) | - Signature::fromValue(uint32_t(shift.op())) | - Signature::fromValue(shift.value()) | - signature, base.id(), index.id(), 0) {} - - ASMJIT_INLINE_CONSTEXPR explicit Mem(uint64_t base, Signature signature = Signature{0}) noexcept - : BaseMem(Signature::fromOpType(OperandType::kMem) | - signature, uint32_t(base >> 32), 0, int32_t(uint32_t(base & 0xFFFFFFFFu))) {} - - //! \} - - //! \name Overloaded Operators - //! \{ - - ASMJIT_INLINE_CONSTEXPR Mem& operator=(const Mem& other) noexcept { - copyFrom(other); - return *this; - } - - //! \} - - //! \name Clone - //! \{ - - //! Clones the memory operand. - ASMJIT_INLINE_CONSTEXPR Mem clone() const noexcept { return Mem(*this); } - - //! Gets new memory operand adjusted by `off`. - ASMJIT_INLINE_CONSTEXPR Mem cloneAdjusted(int64_t off) const noexcept { - Mem result(*this); - result.addOffset(off); - return result; - } - - //! Clones the memory operand and makes it pre-index. - ASMJIT_INLINE_CONSTEXPR Mem pre() const noexcept { - Mem result(*this); - result.setOffsetMode(OffsetMode::kPreIndex); - return result; - } - - //! Clones the memory operand, applies a given offset `off` and makes it pre-index. - ASMJIT_INLINE_CONSTEXPR Mem pre(int64_t off) const noexcept { - Mem result(*this); - result.setOffsetMode(OffsetMode::kPreIndex); - result.addOffset(off); - return result; - } - - //! Clones the memory operand and makes it post-index. - ASMJIT_INLINE_CONSTEXPR Mem post() const noexcept { - Mem result(*this); - result.setOffsetMode(OffsetMode::kPostIndex); - return result; - } - - //! Clones the memory operand, applies a given offset `off` and makes it post-index. - ASMJIT_INLINE_CONSTEXPR Mem post(int64_t off) const noexcept { - Mem result(*this); - result.setOffsetMode(OffsetMode::kPostIndex); - result.addOffset(off); - return result; - } - - //! \} - - //! \name Base & Index - //! \{ - - //! Converts memory `baseType` and `baseId` to `arm::Reg` instance. - //! - //! The memory must have a valid base register otherwise the result will be wrong. - ASMJIT_INLINE_NODEBUG Reg baseReg() const noexcept { return Reg::fromTypeAndId(baseType(), baseId()); } - - //! Converts memory `indexType` and `indexId` to `arm::Reg` instance. - //! - //! The memory must have a valid index register otherwise the result will be wrong. - ASMJIT_INLINE_NODEBUG Reg indexReg() const noexcept { return Reg::fromTypeAndId(indexType(), indexId()); } - - using BaseMem::setIndex; - - ASMJIT_INLINE_CONSTEXPR void setIndex(const BaseReg& index, uint32_t shift) noexcept { - setIndex(index); - setShift(shift); - } - - ASMJIT_INLINE_CONSTEXPR void setIndex(const BaseReg& index, Shift shift) noexcept { - setIndex(index); - setShift(shift); - } - - //! \} - - //! \name ARM Specific Features - //! \{ - - //! Gets offset mode. - ASMJIT_INLINE_CONSTEXPR OffsetMode offsetMode() const noexcept { return OffsetMode(_signature.getField()); } - //! Sets offset mode to `mode`. - ASMJIT_INLINE_CONSTEXPR void setOffsetMode(OffsetMode mode) noexcept { _signature.setField(uint32_t(mode)); } - //! Resets offset mode to default (fixed offset, without write-back). - ASMJIT_INLINE_CONSTEXPR void resetOffsetMode() noexcept { _signature.setField(uint32_t(OffsetMode::kFixed)); } - - //! Tests whether the current memory offset mode is fixed (see \ref OffsetMode::kFixed). - ASMJIT_INLINE_CONSTEXPR bool isFixedOffset() const noexcept { return offsetMode() == OffsetMode::kFixed; } - //! Tests whether the current memory offset mode is either pre-index or post-index (write-back is used). - ASMJIT_INLINE_CONSTEXPR bool isPreOrPost() const noexcept { return offsetMode() != OffsetMode::kFixed; } - //! Tests whether the current memory offset mode is pre-index (write-back is used). - ASMJIT_INLINE_CONSTEXPR bool isPreIndex() const noexcept { return offsetMode() == OffsetMode::kPreIndex; } - //! Tests whether the current memory offset mode is post-index (write-back is used). - ASMJIT_INLINE_CONSTEXPR bool isPostIndex() const noexcept { return offsetMode() == OffsetMode::kPostIndex; } - - //! Sets offset mode of this memory operand to pre-index (write-back is used). - ASMJIT_INLINE_CONSTEXPR void makePreIndex() noexcept { setOffsetMode(OffsetMode::kPreIndex); } - //! Sets offset mode of this memory operand to post-index (write-back is used). - ASMJIT_INLINE_CONSTEXPR void makePostIndex() noexcept { setOffsetMode(OffsetMode::kPostIndex); } - - //! Gets shift operation that is used by index register. - ASMJIT_INLINE_CONSTEXPR ShiftOp shiftOp() const noexcept { return ShiftOp(_signature.getField()); } - //! Sets shift operation that is used by index register. - ASMJIT_INLINE_CONSTEXPR void setShiftOp(ShiftOp sop) noexcept { _signature.setField(uint32_t(sop)); } - //! Resets shift operation that is used by index register to LSL (default value). - ASMJIT_INLINE_CONSTEXPR void resetShiftOp() noexcept { _signature.setField(uint32_t(ShiftOp::kLSL)); } - - //! Gets whether the memory operand has shift (aka scale) constant. - ASMJIT_INLINE_CONSTEXPR bool hasShift() const noexcept { return _signature.hasField(); } - //! Gets the memory operand's shift (aka scale) constant. - ASMJIT_INLINE_CONSTEXPR uint32_t shift() const noexcept { return _signature.getField(); } - //! Sets the memory operand's shift (aka scale) constant. - ASMJIT_INLINE_CONSTEXPR void setShift(uint32_t shift) noexcept { _signature.setField(shift); } - - //! Sets the memory operand's shift and shift operation. - ASMJIT_INLINE_CONSTEXPR void setShift(Shift shift) noexcept { - _signature.setField(uint32_t(shift.op())); - _signature.setField(shift.value()); - } - - //! Resets the memory operand's shift (aka scale) constant to zero. - ASMJIT_INLINE_CONSTEXPR void resetShift() noexcept { _signature.setField(0); } - - //! \} -}; - -//! \name Shift Operation Construction -//! \{ - -//! Constructs a `LSL #value` shift (logical shift left). -static ASMJIT_INLINE_CONSTEXPR Shift lsl(uint32_t value) noexcept { return Shift(ShiftOp::kLSL, value); } -//! Constructs a `LSR #value` shift (logical shift right). -static ASMJIT_INLINE_CONSTEXPR Shift lsr(uint32_t value) noexcept { return Shift(ShiftOp::kLSR, value); } -//! Constructs a `ASR #value` shift (arithmetic shift right). -static ASMJIT_INLINE_CONSTEXPR Shift asr(uint32_t value) noexcept { return Shift(ShiftOp::kASR, value); } -//! Constructs a `ROR #value` shift (rotate right). -static ASMJIT_INLINE_CONSTEXPR Shift ror(uint32_t value) noexcept { return Shift(ShiftOp::kROR, value); } -//! Constructs a `RRX` shift (rotate with carry by 1). -static ASMJIT_INLINE_CONSTEXPR Shift rrx() noexcept { return Shift(ShiftOp::kRRX, 0); } -//! Constructs a `MSL #value` shift (logical shift left filling ones). -static ASMJIT_INLINE_CONSTEXPR Shift msl(uint32_t value) noexcept { return Shift(ShiftOp::kMSL, value); } - -//! \} - -//! \name Memory Operand Construction -//! \{ - -//! Creates `[base]` absolute memory operand (AArch32 or AArch64). -//! -//! \note The concept of absolute memory operands doesn't exist on ARM, the ISA only provides PC relative addressing. -//! Absolute memory operands can only be used if it's known that the PC relative offset is encodable and that it -//! would be within the limits. Absolute address is also often output from disassemblers, so AsmJit supports it to -//! make it possible to assemble such output back. -static ASMJIT_INLINE_CONSTEXPR Mem ptr(uint64_t base) noexcept { return Mem(base); } - -//! \} - -//! \} - -ASMJIT_END_SUB_NAMESPACE - -#endif // ASMJIT_ARM_ARMOPERAND_H_INCLUDED diff --git a/src/asmjit/arm/armutils.h b/src/asmjit/arm/armutils.h index ac3441e..b81f72f 100644 --- a/src/asmjit/arm/armutils.h +++ b/src/asmjit/arm/armutils.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_ARMUTILS_H_INCLUDED diff --git a/src/asmjit/asmjit-scope-begin.h b/src/asmjit/asmjit-scope-begin.h index 93397b5..78ba411 100644 --- a/src/asmjit/asmjit-scope-begin.h +++ b/src/asmjit/asmjit-scope-begin.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifdef _WIN32 diff --git a/src/asmjit/asmjit-scope-end.h b/src/asmjit/asmjit-scope-end.h index 702cef4..b11ca22 100644 --- a/src/asmjit/asmjit-scope-end.h +++ b/src/asmjit/asmjit-scope-end.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifdef _WIN32 diff --git a/src/asmjit/asmjit.h b/src/asmjit/asmjit.h index 5d22d67..b95350a 100644 --- a/src/asmjit/asmjit.h +++ b/src/asmjit/asmjit.h @@ -24,7 +24,7 @@ #ifndef ASMJIT_ASMJIT_H_INCLUDED #define ASMJIT_ASMJIT_H_INCLUDED -#pragma message("asmjit/asmjit.h is deprecated! Please use asmjit/core.h, asmjit/x86.h, or asmjit/a64.h") +#pragma message("asmjit/asmjit.h is deprecated! Please use asmjit/[core|x86|a64|host].h instead.") #include "./core.h" diff --git a/src/asmjit/core.h b/src/asmjit/core.h index d643104..6beedd4 100644 --- a/src/asmjit/core.h +++ b/src/asmjit/core.h @@ -1,7 +1,26 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib +// Official GitHub Repository: https://github.com/asmjit/asmjit +// +// Copyright (c) 2008-2025 The AsmJit Authors +// +// This software is provided 'as-is', without any express or implied +// warranty. In no event will the authors be held liable for any damages +// arising from the use of this software. +// +// Permission is granted to anyone to use this software for any purpose, +// including commercial applications, and to alter it and redistribute it +// freely, subject to the following restrictions: +// +// 1. The origin of this software must not be misrepresented; you must not +// claim that you wrote the original software. If you use this software +// in a product, an acknowledgment in the product documentation would be +// appreciated but is not required. +// 2. Altered source versions must be plainly marked as such, and must not be +// misrepresented as being the original software. +// 3. This notice may not be removed or altered from any source distribution. + #ifndef ASMJIT_CORE_H_INCLUDED #define ASMJIT_CORE_H_INCLUDED @@ -265,16 +284,83 @@ namespace asmjit { //! //! Useful tips before you start: //! -//! - Visit our [Public Gitter Chat](https://app.gitter.im/#/room/#asmjit:gitter.im) if you need a quick help. +//! - Visit our [Public Chat](https://app.gitter.im/#/room/#asmjit:gitter.im) if you need a quick help. //! //! - Build AsmJit with `ASMJIT_NO_DEPRECATED` macro defined to make sure that you are not using deprecated //! functionality at all. Deprecated functions are decorated with `[[deprecated]]` attribute, but sometimes //! it's not possible to decorate everything like classes, which are used by deprecated functions as well, //! because some compilers would warn about that. If your project compiles fine with `ASMJIT_NO_DEPRECATED` -//! it's not using anything, which was deprecated. +//! it's not using anything, which will be definitely removed in the future. //! //! \section api_changes API Changes //! +//! ### Changes committed at 2025-06-15 +//! +//! Core changes: +//! +//! - No more architecture specific \ref RegTraits - removed `BaseRegTraits` and kept just \ref RegTraits: +//! +//! - `BaseRegTraits` -> `RegTraits` +//! - `arm::RegTraits` -> `RegTraits` +//! - `x86::RegTraits` -> `RegTraits` +//! +//! - Removed register signature and helper functions from ArchTraits. This functionality is now available +//! via asmjit::RegTraits and asmjit::RegUtils and doesn't require a valid architecture traits instance. +//! +//! - No more architecture specific Gp/Vec/Mask register types in \ref RegType and \ref RegGroup: +//! +//! - `RegGroup::kX86_Rip` -> `RegGroup::kPC` +//! - `RegGroup::kX86_KReg` -> `RegGroup::kMask` +//! - `RegType::kX86_GpbLo` -> `RegType::kGp8Lo` +//! - `RegType::kX86_GpbLo` -> `RegType::kGp8Lo` +//! - `RegType::kX86_GpbHi` -> `RegType::kGp8Hi` +//! - `RegType::kX86_Gpw` -> `RegType::kGp16` +//! - `RegType::kX86_Gpd` -> `RegType::kGp32` +//! - `RegType::kX86_Gpq` -> `RegType::kGp64` +//! - `RegType::kX86_Xmm` -> `RegType::kVec128` +//! - `RegType::kX86_Ymm` -> `RegType::kVec256` +//! - `RegType::kX86_Zmm` -> `RegType::kVec512` +//! - `RegType::kX86_KReg` -> `RegType::kMask` +//! - `RegType::kARM_PC` -> `RegType::kPC` +//! - `RegType::kARM_GpW` -> `RegType::kGp32` +//! - `RegType::kARM_GpX` -> `RegType::kGp64` +//! - `RegType::kARM_VecB` -> `RegType::kVec8` +//! - `RegType::kARM_VecH` -> `RegType::kVec16` +//! - `RegType::kARM_VecS` -> `RegType::kVec32` +//! - `RegType::kARM_VecD` -> `RegType::kVec64` +//! - `RegType::kARM_VecQ` -> `RegType::kVec128` +//! - `RegType::kARM_VecV` -> `RegType::kVec128` +//! +//! - Renamed `asmjit::BaseReg` to asmjit::Reg, added `asmjit::UniGp` and `asmjit::UniVec` which are now base +//! classes for platform specific `[x86|a64]::Gp` and `[x86|a64]::Vec` +//! +//! - Renamed some member functions in Operand and Reg: +//! +//! - `isType(regId)` -> `isReg(regId)` +//! - `isGroup(regGroup)` -> `isReg(regGroup)` +//! - `regOp.type()` -> `regOp.regType()` +//! - `regOp.group()` -> `regOp.regGroup()` +//! +//! - Removed some static functions from \ref Operand, \reg Reg, etc... in favor of member functions. Most +//! of the operand functionality is now provided by \ref Operand_: +//! +//! - `Operand::isGp(op)` -> op.isGp(); +//! - `x86::Reg::isGp(op, id)` -> op.isGp(id); +//! +//! - Removed sub-registers `x86::Gpb`, `x86::GpbLo`, `x86::GpbHi`, `x86::Gpw`, `x86::Gpd`, `x86::Gpq`, `x86::Xmm`, +//! `x86::Ymm`, `x86::Zmm` - use just `x86::Gp` and `x86::Vec`, which represent all removed X86 sub-registers. +//! +//! - Removed sub-registers `a64::GpW` and `a64::GpX`, `a64::VecB`, `a64::VecH`, `a64::VecS`, `a64::VecD`, +//! `a64::VecV`, which represent all removed AArch64 sub-registers. +//! +//! - Renamed some id getters - `Section::id()` -> `Section::sectionId()`, etc... +//! +//! Builder changes: +//! +//! - Removed BaseBuilder::deletePass() - this function was most likely never used by user code and it was also +//! never used by AsmJit. Passes should be only added and not removed, which simplifies some planned future +//! changes. +//! //! ### Changes committed at 2025-05-24 //! //! Core changes: @@ -534,12 +620,13 @@ namespace asmjit { //! //! - \ref Section - stores information about a code or data section. //! - \ref CodeBuffer - stores actual code or data, part of \ref Section. -//! - \ref LabelEntry - stores information about a label - its name, offset, section where it belongs to, and -//! other bits. -//! - \ref LabelLink - stores information about yet unbound label, which was already used by the assembler. +//! - \ref LabelEntry - stores information about a \ref Label - its name, offset, section where it belongs to, +//! and other bits. +//! - \ref Fixup - stores information about positions in code that needs to be fixed up later, for example +//! when referencing a \ref Label, which was not bound to a position in code yet. //! - \ref RelocEntry - stores information about a relocation. -//! - \ref AddressTableEntry - stores information about an address, which was used in a jump or call. Such -//! address may need relocation. +//! - \ref AddressTableEntry - stores information about an absolute address, which was used in a jump or call. +//! Code referencing an absolute address may need relocation or a record in address table. //! //! To generate code you would need to instantiate at least the following classes: //! @@ -692,21 +779,21 @@ namespace asmjit { //! required if you call external functions from the generated code that cannot be encoded by using a 32-bit //! displacement (64-bit displacements are not provided by aby supported architecture). //! -//! There is also a concept called \ref LabelLink - label link is a lightweight data structure that doesn't have any -//! identifier and is stored in \ref LabelEntry as a single-linked list. Label link represents either unbound yet used -//! label and cross-sections links (only relevant to code that uses multiple sections). Since crossing sections is -//! something that cannot be resolved immediately these links persist until offsets of these sections are assigned and -//! until \ref CodeHolder::resolveUnresolvedLinks() is called. It's an error if you end up with code that has -//! unresolved label links after flattening. You can verify it by calling \ref CodeHolder::hasUnresolvedLinks(), which -//! inspects the value returned by \ref CodeHolder::unresolvedLinkCount(). +//! There is also a concept called \ref Fixup - it's a lightweight data structure that doesn't have any identifier and +//! is stored in \ref LabelEntry and \ref CodeHolder as a single-linked list. Fixup represents either a reference to an +//! unbound label and cross-sections references (only relevant to code that uses multiple sections). Since crossing +//! sections is something that cannot be resolved immediately these fixups persist until offsets of these sections are +//! assigned and until \ref CodeHolder::resolveCrossSectionFixups() is called. It's an error if you end up with code that +//! still has fixups after flattening. You can verify it by calling \ref CodeHolder::hasUnresolvedFixups(), which inspects +//! the value returned by \ref CodeHolder::unresolvedFixupCount(). //! //! AsmJit can flatten code that uses multiple sections by assigning each section an incrementing offset that respects //! its alignment. Use \ref CodeHolder::flatten() to do that. After the sections are flattened their offsets and //! virtual sizes are adjusted to respect each section's buffer size and alignment. The \ref -//! CodeHolder::resolveUnresolvedLinks() function must be called before relocating the code held by \ref CodeHolder. +//! CodeHolder::resolveCrossSectionFixups() function must be called before relocating the code held by \ref CodeHolder. //! You can also flatten your code manually by iterating over all sections and calculating their offsets (relative to //! base) by your own algorithm. In that case \ref CodeHolder::flatten() should not be called, however, -//! \ref CodeHolder::resolveUnresolvedLinks() should be. +//! \ref CodeHolder::resolveCrossSectionFixups() should be. //! //! The example below shows how to use a built-in virtual memory allocator \ref JitAllocator instead of using \ref //! JitRuntime (just in case you want to use your own memory management) and how to relocate the generated code @@ -775,7 +862,7 @@ namespace asmjit { //! // such relocations. You can use `CodeHolder::hasAddressTable()` to verify //! // whether the address table section does exist. //! code.flatten(); -//! code.resolveUnresolvedLinks(); +//! code.resolveCrossSectionFixups(); //! //! // After the code was generated it can be relocated manually to any memory //! // location, however, we need to know it's size before we perform memory @@ -870,9 +957,9 @@ namespace asmjit { //! //! \section labels Label Offsets and Links //! -//! When a label that is not yet bound is used by the Assembler, it creates a \ref LabelLink, which is then added to -//! a \ref LabelEntry. These links are also created if a label is used in a different section than in which it was -//! bound. Let's examine some functions that can be used to check whether there are any unresolved links. +//! When a label that is not yet bound is used by the Assembler, it creates a \ref Fixup, which is then referenced +//! by \ref LabelEntry. Fixups are also created if a label is referenced in a different section than in which it was +//! bound. Let's examine some functions that can be used to check whether there are any unresolved fixups. //! //! ``` //! #include @@ -886,11 +973,11 @@ namespace asmjit { //! printf("Label %u is %s\n", label.id(), isBound ? "bound" : "not bound"); //! //! // Returns true if the code contains either referenced, but unbound -//! // labels, or cross-section label links that are not resolved yet. -//! bool hasUnresolved = code.hasUnresolvedLinks(); // Boolean answer. -//! size_t nUnresolved = code.unresolvedLinkCount(); // Count of unresolved links. +//! // labels, or cross-section fixups that are not resolved yet. +//! bool hasUnresolved = code.hasUnresolvedFixups(); // Boolean answer. +//! size_t nUnresolved = code.unresolvedFixupCount(); // Count of unresolved fixups. //! -//! printf("Number of unresolved links: %zu\n", nUnresolved); +//! printf("Number of unresolved fixups: %zu\n", nUnresolved); //! } //! ``` //! @@ -963,17 +1050,16 @@ namespace asmjit { //! a.section(text); // Switches to the end of .text section. //! a.add(x86::ebx, x86::eax); // Emits in .text section. //! -//! // References a label in .text section, which was bound in .data section. -//! // This would create a LabelLink even when the L_Data is already bound, -//! // because the reference crosses sections. See below... +//! // References a label in .text section, which was bound in .data section. This would create a +//! // fixup even when the L_Data is already bound, because the reference crosses sections. See below... //! a.lea(x86::rsi, x86::ptr(L_Data)); //! } //! ``` //! -//! The last line in the example above shows that a LabelLink would be created even for bound labels that cross -//! sections. In this case a referenced label was bound in another section, which means that the link couldn't be -//! resolved at that moment. If your code uses sections, but you wish AsmJit to flatten these sections (you don't -//! plan to flatten them manually) then there is an API for that. +//! The last line in the example above shows that a \ref Fixup would be created even for bound labels that cross +//! sections. In this case a referenced label was bound in another section, which means that the reference couldn't +//! be resolved at that moment. If your code uses sections, but you wish AsmJit to flatten these sections (you don't +//! plan to flatten them manually) then there is a ready API for that. //! //! ``` //! #include @@ -998,18 +1084,18 @@ namespace asmjit { //! // guaranteed that the offset cannot be greater than `2^32 - 1`. //! printf("Data section offset %zu", size_t(data->offset())); //! -//! // The flattening doesn't resolve unresolved label links, this +//! // The flattening doesn't resolve unresolved fixups, this //! // has to be done manually as flattening can be done separately. -//! err = code.resolveUnresolvedLinks(); +//! err = code.resolveCrossSectionFixups(); //! if (err) { //! // This is the kind of error that should always be handled... -//! printf("Failed to resolve label links: %s\n", DebugUtils::errorAsString(err)); +//! printf("Failed to resolve fixups: %s\n", DebugUtils::errorAsString(err)); //! exit(1); //! } //! -//! if (code.hasUnresolvedLinks()) { +//! if (code.hasUnresolvedFixups()) { //! // This would mean either unbound label or some other issue. -//! printf("The code has %zu unbound labels\n", code.unresolvedLinkCount()); +//! printf("The code has %zu unbound labels\n", code.unresolvedFixupCount()); //! exit(1); //! } //! } @@ -1108,7 +1194,7 @@ namespace asmjit { //! // Constructs [src + idx] memory address - referencing [rax + r10]. //! x86::Mem m = x86::ptr(src, idx); //! -//! // Examine `m`: Returns `RegType::kX86_Gpq`. +//! // Examine `m`: Returns `RegType::kGp64`. //! m.indexType(); //! // Examine `m`: Returns 10 (`r10`). //! m.indexId(); @@ -1882,9 +1968,6 @@ namespace asmjit { //! using namespace asmjit; //! //! void example(CodeHolder& code) { -//! // Contains all emitters attached to CodeHolder. -//! const ZoneVector& emitters = code.emitters(); -//! //! // Contains all section entries managed by CodeHolder. //! const ZoneVector& sections = code.sections(); //! @@ -2005,7 +2088,6 @@ namespace asmjit { //! \defgroup asmjit_a64 AArch64 Backend //! \brief AArch64 backend. - //! \cond INTERNAL //! \defgroup asmjit_ra RA //! \brief Register allocator internals. @@ -2017,6 +2099,7 @@ namespace asmjit { #include "core/archtraits.h" #include "core/assembler.h" #include "core/builder.h" +#include "core/codebuffer.h" #include "core/codeholder.h" #include "core/compiler.h" #include "core/constpool.h" @@ -2024,6 +2107,7 @@ namespace asmjit { #include "core/emitter.h" #include "core/environment.h" #include "core/errorhandler.h" +#include "core/fixup.h" #include "core/formatter.h" #include "core/func.h" #include "core/globals.h" diff --git a/src/asmjit/core/api-build_p.h b/src/asmjit/core/api-build_p.h index 3ddc9e7..bb65fc8 100644 --- a/src/asmjit/core/api-build_p.h +++ b/src/asmjit/core/api-build_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_API_BUILD_P_H_INCLUDED diff --git a/src/asmjit/core/api-config.h b/src/asmjit/core/api-config.h index 762692e..a97ea17 100644 --- a/src/asmjit/core/api-config.h +++ b/src/asmjit/core/api-config.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_API_CONFIG_H_INCLUDED @@ -16,7 +16,7 @@ #define ASMJIT_LIBRARY_MAKE_VERSION(major, minor, patch) ((major << 16) | (minor << 8) | (patch)) //! AsmJit library version, see \ref ASMJIT_LIBRARY_MAKE_VERSION for a version format reference. -#define ASMJIT_LIBRARY_VERSION ASMJIT_LIBRARY_MAKE_VERSION(1, 16, 0) +#define ASMJIT_LIBRARY_VERSION ASMJIT_LIBRARY_MAKE_VERSION(1, 17, 0) //! \def ASMJIT_ABI_NAMESPACE //! @@ -27,7 +27,7 @@ //! AsmJit default, which makes it possible to use multiple AsmJit libraries within a single project, totally //! controlled by users. This is useful especially in cases in which some of such library comes from third party. #if !defined(ASMJIT_ABI_NAMESPACE) - #define ASMJIT_ABI_NAMESPACE v1_16 + #define ASMJIT_ABI_NAMESPACE v1_17 #endif // !ASMJIT_ABI_NAMESPACE //! \} @@ -78,23 +78,20 @@ namespace asmjit { //! \note Can be defined explicitly to bypass auto-detection. #define ASMJIT_BUILD_RELEASE +//! Disables deprecated API at compile time (deprecated API won't be available). +#define ASMJIT_NO_DEPRECATED + +//! Disables the use of an inline ABI namespace within asmjit namespace (the inline namespace is used as an ABI tag). +#define ASMJIT_NO_ABI_NAMESPACE + //! Disables X86/X64 backends. #define ASMJIT_NO_X86 //! Disables AArch64 backend. #define ASMJIT_NO_AARCH64 -//! Disables non-host backends entirely (useful for JIT compilers to minimize the library size). -#define ASMJIT_NO_FOREIGN - -//! Disables deprecated API at compile time (deprecated API won't be available). -#define ASMJIT_NO_DEPRECATED - -//! Disables \ref asmjit_builder functionality completely. -#define ASMJIT_NO_BUILDER - -//! Disables \ref asmjit_compiler functionality completely. -#define ASMJIT_NO_COMPILER +//! Disables the use of `shm_open` on all targets even when it's supported. +#define ASMJIT_NO_SHM_OPEN //! Disables JIT memory management and \ref asmjit::JitRuntime. #define ASMJIT_NO_JIT @@ -111,21 +108,36 @@ namespace asmjit { //! Disables instruction introspection API. #define ASMJIT_NO_INTROSPECTION +//! Disables non-host backends entirely (useful for JIT compilers to minimize the library size). +#define ASMJIT_NO_FOREIGN + +//! Disables \ref asmjit_builder functionality completely. +#define ASMJIT_NO_BUILDER + +//! Disables \ref asmjit_compiler functionality completely. +#define ASMJIT_NO_COMPILER + // Avoid doxygen preprocessor using feature-selection definitions. #undef ASMJIT_BUILD_EMBED #undef ASMJIT_BUILD_STATIC #undef ASMJIT_BUILD_DEBUG #undef ASMJIT_BUILD_RELEASE -#undef ASMJIT_NO_X86 -#undef ASMJIT_NO_FOREIGN + // (keep ASMJIT_NO_DEPRECATED defined, we don't document deprecated APIs). -#undef ASMJIT_NO_BUILDER -#undef ASMJIT_NO_COMPILER +#undef ASMJIT_NO_ABI_NAMESPACE + +#undef ASMJIT_NO_X86 +#undef ASMJIT_NO_AARCH64 +#undef ASMJIT_NO_FOREIGN + #undef ASMJIT_NO_JIT #undef ASMJIT_NO_LOGGING #undef ASMJIT_NO_TEXT #undef ASMJIT_NO_VALIDATION #undef ASMJIT_NO_INTROSPECTION +#undef ASMJIT_NO_BUILDER +#undef ASMJIT_NO_COMPILER +#undef ASMJIT_NO_UJIT //! \} @@ -137,6 +149,11 @@ namespace asmjit { #define ASMJIT_NO_COMPILER #endif +// ASMJIT_NO_COMPILER implies ASMJIT_NO_UJIT. +#if defined(ASMJIT_NO_COMPILER) && !defined(ASMJIT_NO_UJIT) + #define ASMJIT_NO_UJIT +#endif + // Prevent compile-time errors caused by misconfiguration. #if defined(ASMJIT_NO_TEXT) && !defined(ASMJIT_NO_LOGGING) #pragma message("'ASMJIT_NO_TEXT' can only be defined when 'ASMJIT_NO_LOGGING' is defined.") @@ -194,6 +211,13 @@ namespace asmjit { //! //! Defined to 1 if the target architecture is big endian. +//! \def ASMJIT_HAS_HOST_BACKEND +//! +//! Defined when AsmJit is built with the target architecture backend. +//! +//! For example if AsmJit is building for x86 or x86_64 architectures and `ASMJIT_NO_X86` is not defined, +//! it would define `ASMJIT_HAS_HOST_BACKEND` when `` or ``` is included. + //! \} //! \cond NONE @@ -259,6 +283,24 @@ namespace asmjit { #endif #endif +#if ASMJIT_ARCH_X86 != 0 && !defined(ASMJIT_NO_X86) + #define ASMJIT_HAS_HOST_BACKEND +#endif + +#if ASMJIT_ARCH_ARM == 64 && !defined(ASMJIT_NO_AARCH64) + #define ASMJIT_HAS_HOST_BACKEND +#endif + +#if !defined(ASMJIT_NO_UJIT) + #if !defined(ASMJIT_NO_X86) && ASMJIT_ARCH_X86 != 0 + #define ASMJIT_UJIT_X86 + #elif !defined(ASMJIT_NO_AARCH64) && ASMJIT_ARCH_ARM == 64 + #define ASMJIT_UJIT_AARCH64 + #else + #define ASMJIT_NO_UJIT + #endif +#endif + //! \endcond // C++ Compiler and Features Detection @@ -424,11 +466,11 @@ namespace asmjit { // Type alignment (not allowed by C++17 'alignas' keyword). #if defined(__GNUC__) - #define ASMJIT_ALIGN_TYPE(TYPE, N) __attribute__((__aligned__(N))) TYPE + #define ASMJIT_ALIGN_TYPE(N, ...) __attribute__((__aligned__(N))) __VA_ARGS__ #elif defined(_MSC_VER) - #define ASMJIT_ALIGN_TYPE(TYPE, N) __declspec(align(N)) TYPE + #define ASMJIT_ALIGN_TYPE(N, ...) __declspec(align(N)) __VA_ARGS__ #else - #define ASMJIT_ALIGN_TYPE(TYPE, N) TYPE + #define ASMJIT_ALIGN_TYPE(N, ...) __VA_ARGS__ #endif //! \def ASMJIT_MAY_ALIAS @@ -600,4 +642,56 @@ namespace asmjit { } #endif +//! Defines a strong type `C` that wraps a value of `T`. +#define ASMJIT_DEFINE_STRONG_TYPE(C, T) \ +struct C { \ + T v; \ + \ + ASMJIT_INLINE_NODEBUG C() = default; \ + ASMJIT_INLINE_CONSTEXPR explicit C(T x) noexcept : v(x) {} \ + ASMJIT_INLINE_CONSTEXPR C(const C& other) noexcept = default; \ + \ + ASMJIT_INLINE_CONSTEXPR T value() const noexcept { return v; } \ + \ + ASMJIT_INLINE_CONSTEXPR T* valuePtr() noexcept { return &v; } \ + ASMJIT_INLINE_CONSTEXPR const T* valuePtr() const noexcept { return &v; } \ + \ + ASMJIT_INLINE_CONSTEXPR C& operator=(T x) noexcept { v = x; return *this; }; \ + ASMJIT_INLINE_CONSTEXPR C& operator=(const C& x) noexcept { v = x.v; return *this; } \ + \ + ASMJIT_INLINE_CONSTEXPR C operator+(T x) const noexcept { return C(v + x); } \ + ASMJIT_INLINE_CONSTEXPR C operator-(T x) const noexcept { return C(v - x); } \ + ASMJIT_INLINE_CONSTEXPR C operator*(T x) const noexcept { return C(v * x); } \ + ASMJIT_INLINE_CONSTEXPR C operator/(T x) const noexcept { return C(v / x); } \ + \ + ASMJIT_INLINE_CONSTEXPR C operator+(const C& x) const noexcept { return C(v + x.v); } \ + ASMJIT_INLINE_CONSTEXPR C operator-(const C& x) const noexcept { return C(v - x.v); } \ + ASMJIT_INLINE_CONSTEXPR C operator*(const C& x) const noexcept { return C(v * x.v); } \ + ASMJIT_INLINE_CONSTEXPR C operator/(const C& x) const noexcept { return C(v / x.v); } \ + \ + ASMJIT_INLINE_CONSTEXPR C& operator+=(T x) noexcept { v += x; return *this; } \ + ASMJIT_INLINE_CONSTEXPR C& operator-=(T x) noexcept { v -= x; return *this; } \ + ASMJIT_INLINE_CONSTEXPR C& operator*=(T x) noexcept { v *= x; return *this; } \ + ASMJIT_INLINE_CONSTEXPR C& operator/=(T x) noexcept { v /= x; return *this; } \ + \ + ASMJIT_INLINE_CONSTEXPR C& operator+=(const C& x) noexcept { v += x.v; return *this; } \ + ASMJIT_INLINE_CONSTEXPR C& operator-=(const C& x) noexcept { v -= x.v; return *this; } \ + ASMJIT_INLINE_CONSTEXPR C& operator*=(const C& x) noexcept { v *= x.v; return *this; } \ + ASMJIT_INLINE_CONSTEXPR C& operator/=(const C& x) noexcept { v /= x.v; return *this; } \ + \ + ASMJIT_INLINE_CONSTEXPR bool operator==(T x) const noexcept { return v == x; } \ + ASMJIT_INLINE_CONSTEXPR bool operator!=(T x) const noexcept { return v != x; } \ + ASMJIT_INLINE_CONSTEXPR bool operator> (T x) const noexcept { return v > x; } \ + ASMJIT_INLINE_CONSTEXPR bool operator>=(T x) const noexcept { return v >= x; } \ + ASMJIT_INLINE_CONSTEXPR bool operator< (T x) const noexcept { return v < x; } \ + ASMJIT_INLINE_CONSTEXPR bool operator<=(T x) const noexcept { return v <= x; } \ + \ + ASMJIT_INLINE_CONSTEXPR bool operator==(const C& x) const noexcept { return v == x.v; } \ + ASMJIT_INLINE_CONSTEXPR bool operator!=(const C& x) const noexcept { return v != x.v; } \ + ASMJIT_INLINE_CONSTEXPR bool operator> (const C& x) const noexcept { return v > x.v; } \ + ASMJIT_INLINE_CONSTEXPR bool operator>=(const C& x) const noexcept { return v >= x.v; } \ + ASMJIT_INLINE_CONSTEXPR bool operator< (const C& x) const noexcept { return v < x.v; } \ + ASMJIT_INLINE_CONSTEXPR bool operator<=(const C& x) const noexcept { return v <= x.v; } \ +}; + #endif // ASMJIT_CORE_API_CONFIG_H_INCLUDED diff --git a/src/asmjit/core/archcommons.h b/src/asmjit/core/archcommons.h index 41c96fc..16865b9 100644 --- a/src/asmjit/core/archcommons.h +++ b/src/asmjit/core/archcommons.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ARCHCOMMONS_H_INCLUDED diff --git a/src/asmjit/core/archtraits.cpp b/src/asmjit/core/archtraits.cpp index 095c165..b6f56bb 100644 --- a/src/asmjit/core/archtraits.cpp +++ b/src/asmjit/core/archtraits.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -20,18 +20,21 @@ ASMJIT_BEGIN_NAMESPACE static const constexpr ArchTraits noArchTraits = { // SP/FP/LR/PC. - 0xFF, 0xFF, 0xFF, 0xFF, + 0xFFu, 0xFFu, 0xFFu, 0xFFu, // Reserved, - { 0, 0, 0 }, + { 0u, 0u, 0u }, // HW stack alignment. - 0, + 0u, // Min/Max stack offset. 0, 0, - // ISA features [Gp, Vec, Other0, Other1]. + // Supported register types. + 0u, + + // ISA features [Gp, Vec, Mask, Extra]. {{ InstHints::kNoHints, InstHints::kNoHints, @@ -39,16 +42,6 @@ static const constexpr ArchTraits noArchTraits = { InstHints::kNoHints }}, - // RegTypeToSignature. - #define V(index) OperandSignature{0} - {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, - #undef V - - // RegTypeToTypeId. - #define V(index) TypeId::kVoid - {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, - #undef V - // TypeIdToRegType. #define V(index) RegType::kNone {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, @@ -107,7 +100,7 @@ ASMJIT_FAVOR_SIZE Error ArchUtils::typeIdToRegSignature(Arch arch, TypeId typeId // TODO: Remove this, should never be used like this. // Passed RegType instead of TypeId? if (uint32_t(typeId) <= uint32_t(RegType::kMaxValue)) { - typeId = archTraits.regTypeToTypeId(RegType(uint32_t(typeId))); + typeId = RegUtils::typeIdOf(RegType(uint32_t(typeId))); } if (ASMJIT_UNLIKELY(!TypeUtils::isValid(typeId))) { @@ -149,16 +142,16 @@ ASMJIT_FAVOR_SIZE Error ArchUtils::typeIdToRegSignature(Arch arch, TypeId typeId } } else { - if (size <= 8 && archTraits._regSignature[RegType::kVec64].isValid()) { + if (size <= 8 && archTraits.hasRegType(RegType::kVec64)) { regType = RegType::kVec64; } - else if (size <= 16 && archTraits._regSignature[RegType::kVec128].isValid()) { + else if (size <= 16 && archTraits.hasRegType(RegType::kVec128)) { regType = RegType::kVec128; } - else if (size == 32 && archTraits._regSignature[RegType::kVec256].isValid()) { + else if (size == 32 && archTraits.hasRegType(RegType::kVec256)) { regType = RegType::kVec256; } - else if (archTraits._regSignature[RegType::kVec512].isValid()) { + else if (archTraits.hasRegType(RegType::kVec512)) { regType = RegType::kVec512; } else { @@ -167,7 +160,7 @@ ASMJIT_FAVOR_SIZE Error ArchUtils::typeIdToRegSignature(Arch arch, TypeId typeId } *typeIdOut = typeId; - *regSignatureOut = archTraits.regTypeToSignature(regType); + *regSignatureOut = RegUtils::signatureOf(regType); return kErrorOk; } diff --git a/src/asmjit/core/archtraits.h b/src/asmjit/core/archtraits.h index 2ec29ba..dacfbeb 100644 --- a/src/asmjit/core/archtraits.h +++ b/src/asmjit/core/archtraits.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ARCHTRAITS_H_INCLUDED @@ -110,13 +110,13 @@ enum class SubArch : uint8_t { //! Identifier used to represent names of different data types across architectures. enum class ArchTypeNameId : uint8_t { - //! Describes 'db' (X86/X86_64 convention, always 8-bit quantity). + //! Describes 'db' (X86|X86_64 convention, always 8-bit quantity). kDB = 0, - //! Describes 'dw' (X86/X86_64 convention, always 16-bit word). + //! Describes 'dw' (X86|X86_64 convention, always 16-bit word). kDW, - //! Describes 'dd' (X86/X86_64 convention, always 32-bit word). + //! Describes 'dd' (X86|X86_64 convention, always 32-bit word). kDD, - //! Describes 'dq' (X86/X86_64 convention, always 64-bit word). + //! Describes 'dq' (X86|X86_64 convention, always 64-bit word). kDQ, //! Describes 'byte' (always 8-bit quantity). kByte, @@ -174,7 +174,7 @@ struct ArchTraits { //! Link register id. uint8_t _linkRegId; //! Instruction pointer (or program counter) register id, if accessible. - uint8_t _ipRegId; + uint8_t _pcRegId; // Reserved. uint8_t _reserved[3]; @@ -186,13 +186,12 @@ struct ArchTraits { //! Maximum addressable offset on stack depending on specific instruction. uint32_t _maxStackOffset; + //! Bit-mask indexed by \ref RegType that describes, which register types are supported by the ISA. + uint32_t _supportedRegTypes; + //! Flags for each virtual register group. Support::Array _instHints; - //! Maps register type into a signature, that provides group, size and can be used to construct register operands. - Support::Array _regSignature; - //! Maps a register to type-id, see \ref TypeId. - Support::Array _regTypeToTypeId; //! Maps scalar TypeId values (from TypeId::_kIdBaseStart) to register types, see \ref TypeId. Support::Array _typeIdToRegType; @@ -204,21 +203,21 @@ struct ArchTraits { //! \name Accessors //! \{ - //! Returns stack pointer register id. + //! Returns stack pointer register id (always GP register). [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t spRegId() const noexcept { return _spRegId; } - //! Returns stack frame register id. + //! Returns stack frame register id (always GP register). [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t fpRegId() const noexcept { return _fpRegId; } - //! Returns link register id, if the architecture provides it. + //! Returns link register id, if the architecture provides it (always GP register). [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t linkRegId() const noexcept { return _linkRegId; } - //! Returns instruction pointer register id, if the architecture provides it. + //! Returns program counter register id, if the architecture exposes it (always GP register). [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint32_t ipRegId() const noexcept { return _ipRegId; } + ASMJIT_INLINE_NODEBUG uint32_t pcRegId() const noexcept { return _pcRegId; } //! Returns a hardware stack alignment requirement. //! @@ -230,7 +229,7 @@ struct ArchTraits { //! Tests whether the architecture provides link register, which is used across function calls. If the link //! register is not provided then a function call pushes the return address on stack (X86/X64). [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool hasLinkReg() const noexcept { return _linkRegId != BaseReg::kIdBad; } + ASMJIT_INLINE_NODEBUG bool hasLinkReg() const noexcept { return _linkRegId != Reg::kIdBad; } //! Returns minimum addressable offset on stack guaranteed for all instructions. [[nodiscard]] @@ -258,25 +257,12 @@ struct ArchTraits { [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasRegType(RegType type) const noexcept { - return type <= RegType::kMaxValue && _regSignature[type].isValid(); + if (ASMJIT_UNLIKELY(type > RegType::kMaxValue)) { + type = RegType::kNone; + } + return Support::bitTest(_supportedRegTypes, uint32_t(type)); } - //! Returns an operand signature from the given register `type` of this architecture. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG OperandSignature regTypeToSignature(RegType type) const noexcept { return _regSignature[type]; } - - //! Returns a register from the given register `type` of this architecture. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG RegGroup regTypeToGroup(RegType type) const noexcept { return _regSignature[type].regGroup(); } - - //! Returns a register size the given register `type` of this architecture. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint32_t regTypeToSize(RegType type) const noexcept { return _regSignature[type].size(); } - - //! Returns a corresponding `TypeId` from the given register `type` of this architecture. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG TypeId regTypeToTypeId(RegType type) const noexcept { return _regTypeToTypeId[type]; } - //! Returns a table of ISA word names that appear in formatted text. Word names are ISA dependent. //! //! The index of this table is log2 of the size: diff --git a/src/asmjit/core/assembler.cpp b/src/asmjit/core/assembler.cpp index a2a2806..c5c7361 100644 --- a/src/asmjit/core/assembler.cpp +++ b/src/asmjit/core/assembler.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -42,13 +42,15 @@ Error BaseAssembler::setOffset(size_t offset) { // BaseAssembler - Section Management // ================================== -static void BaseAssembler_initSection(BaseAssembler* self, Section* section) noexcept { +static ASMJIT_INLINE Error BaseAssembler_initSection(BaseAssembler* self, Section* section) noexcept { uint8_t* p = section->_buffer._data; self->_section = section; self->_bufferData = p; self->_bufferPtr = p + section->_buffer._size; self->_bufferEnd = p + section->_buffer._capacity; + + return kErrorOk; } Error BaseAssembler::section(Section* section) { @@ -56,51 +58,50 @@ Error BaseAssembler::section(Section* section) { return reportError(DebugUtils::errored(kErrorNotInitialized)); } - if (!_code->isSectionValid(section->id()) || _code->_sections[section->id()] != section) { + if (!_code->isSectionValid(section->sectionId()) || _code->_sections[section->sectionId()] != section) { return reportError(DebugUtils::errored(kErrorInvalidSection)); } #ifndef ASMJIT_NO_LOGGING if (_logger) { - _logger->logf(".section %s {#%u}\n", section->name(), section->id()); + _logger->logf(".section %s {#%u}\n", section->name(), section->sectionId()); } #endif - BaseAssembler_initSection(this, section); - return kErrorOk; + return BaseAssembler_initSection(this, section); } // BaseAssembler - Label Management // ================================ Label BaseAssembler::newLabel() { - uint32_t labelId = Globals::kInvalidId; + Label label; + if (ASMJIT_LIKELY(_code)) { - LabelEntry* le; - Error err = _code->newLabelEntry(&le); + Error err = _code->newLabelId(&label._baseId); if (ASMJIT_UNLIKELY(err)) { reportError(err); } - else { - labelId = le->id(); - } } - return Label(labelId); + + return label; } Label BaseAssembler::newNamedLabel(const char* name, size_t nameSize, LabelType type, uint32_t parentId) { - uint32_t labelId = Globals::kInvalidId; + Label label; + if (ASMJIT_LIKELY(_code)) { - LabelEntry* le; - Error err = _code->newNamedLabelEntry(&le, name, nameSize, type, parentId); + uint32_t labelId; + Error err = _code->newNamedLabelId(&labelId, name, nameSize, type, parentId); if (ASMJIT_UNLIKELY(err)) { reportError(err); } else { - labelId = le->id(); + label.setId(labelId); } } - return Label(labelId); + + return label; } Error BaseAssembler::bind(const Label& label) { @@ -108,7 +109,7 @@ Error BaseAssembler::bind(const Label& label) { return reportError(DebugUtils::errored(kErrorNotInitialized)); } - Error err = _code->bindLabel(label, _section->id(), offset()); + Error err = _code->bindLabel(label, _section->sectionId(), offset()); #ifndef ASMJIT_NO_LOGGING if (_logger) { @@ -258,19 +259,18 @@ Error BaseAssembler::embedLabel(const Label& label, size_t dataSize) { return reportError(DebugUtils::errored(kErrorNotInitialized)); } - ASMJIT_ASSERT(_code != nullptr); - RelocEntry* re; - LabelEntry* le = _code->labelEntry(label); - - if (ASMJIT_UNLIKELY(!le)) { + if (ASMJIT_UNLIKELY(isLabelValid(label))) { return reportError(DebugUtils::errored(kErrorInvalidLabel)); } + RelocEntry* re; + LabelEntry& le = _code->labelEntry(label); + if (dataSize == 0) { dataSize = registerSize(); } - if (ASMJIT_UNLIKELY(!Support::isPowerOf2(dataSize) || dataSize > 8)) { + if (ASMJIT_UNLIKELY(!Support::isPowerOf2UpTo(dataSize, 8u))) { return reportError(DebugUtils::errored(kErrorInvalidOperandSize)); } @@ -294,24 +294,24 @@ Error BaseAssembler::embedLabel(const Label& label, size_t dataSize) { return reportError(err); } - re->_sourceSectionId = _section->id(); + re->_sourceSectionId = _section->sectionId(); re->_sourceOffset = offset(); re->_format.resetToSimpleValue(OffsetType::kUnsignedOffset, dataSize); - if (le->isBound()) { - re->_targetSectionId = le->section()->id(); - re->_payload = le->offset(); + if (le.isBound()) { + re->_targetSectionId = le.sectionId(); + re->_payload = le.offset(); } else { OffsetFormat of; of.resetToSimpleValue(OffsetType::kUnsignedOffset, dataSize); - LabelLink* link = _code->newLabelLink(le, _section->id(), offset(), 0, of); - if (ASMJIT_UNLIKELY(!link)) { + Fixup* fixup = _code->newFixup(&le, _section->sectionId(), offset(), 0, of); + if (ASMJIT_UNLIKELY(!fixup)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); } - link->relocId = re->id(); + fixup->labelOrRelocId = re->id(); } // Emit dummy DWORD/QWORD depending on the data size. @@ -326,18 +326,18 @@ Error BaseAssembler::embedLabelDelta(const Label& label, const Label& base, size return reportError(DebugUtils::errored(kErrorNotInitialized)); } - LabelEntry* labelEntry = _code->labelEntry(label); - LabelEntry* baseEntry = _code->labelEntry(base); - - if (ASMJIT_UNLIKELY(!labelEntry || !baseEntry)) { + if (ASMJIT_UNLIKELY(!Support::bool_and(_code->isLabelValid(label), _code->isLabelValid(base)))) { return reportError(DebugUtils::errored(kErrorInvalidLabel)); } + LabelEntry& labelEntry = _code->labelEntry(label); + LabelEntry& baseEntry = _code->labelEntry(base); + if (dataSize == 0) { dataSize = registerSize(); } - if (ASMJIT_UNLIKELY(!Support::isPowerOf2(dataSize) || dataSize > 8)) { + if (ASMJIT_UNLIKELY(!Support::isPowerOf2UpTo(dataSize, 8u))) { return reportError(DebugUtils::errored(kErrorInvalidOperandSize)); } @@ -359,8 +359,8 @@ Error BaseAssembler::embedLabelDelta(const Label& label, const Label& base, size #endif // If both labels are bound within the same section it means the delta can be calculated now. - if (labelEntry->isBound() && baseEntry->isBound() && labelEntry->section() == baseEntry->section()) { - uint64_t delta = labelEntry->offset() - baseEntry->offset(); + if (labelEntry.isBound() && baseEntry.isBound() && labelEntry.sectionId() == baseEntry.sectionId()) { + uint64_t delta = labelEntry.offset() - baseEntry.offset(); writer.emitValueLE(delta, dataSize); } else { @@ -377,11 +377,11 @@ Error BaseAssembler::embedLabelDelta(const Label& label, const Label& base, size exp->reset(); exp->opType = ExpressionOpType::kSub; - exp->setValueAsLabel(0, labelEntry); - exp->setValueAsLabel(1, baseEntry); + exp->setValueAsLabelId(0, label.id()); + exp->setValueAsLabelId(1, base.id()); re->_format.resetToSimpleValue(OffsetType::kSignedOffset, dataSize); - re->_sourceSectionId = _section->id(); + re->_sourceSectionId = _section->sectionId(); re->_sourceOffset = offset(); re->_payload = (uint64_t)(uintptr_t)exp; @@ -419,16 +419,14 @@ Error BaseAssembler::comment(const char* data, size_t size) { // BaseAssembler - Events // ====================== -Error BaseAssembler::onAttach(CodeHolder* code) noexcept { +Error BaseAssembler::onAttach(CodeHolder& code) noexcept { ASMJIT_PROPAGATE(Base::onAttach(code)); // Attach to the end of the .text section. - BaseAssembler_initSection(this, code->_sections[0]); - - return kErrorOk; + return BaseAssembler_initSection(this, code._sections[0]); } -Error BaseAssembler::onDetach(CodeHolder* code) noexcept { +Error BaseAssembler::onDetach(CodeHolder& code) noexcept { _section = nullptr; _bufferData = nullptr; _bufferEnd = nullptr; @@ -436,4 +434,11 @@ Error BaseAssembler::onDetach(CodeHolder* code) noexcept { return Base::onDetach(code); } +Error BaseAssembler::onReinit(CodeHolder& code) noexcept { + // BaseEmitter::onReinit() never fails. + (void)Base::onReinit(code); + + return BaseAssembler_initSection(this, code._sections[0]); +} + ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/assembler.h b/src/asmjit/core/assembler.h index 2efc95e..2ca7c6c 100644 --- a/src/asmjit/core/assembler.h +++ b/src/asmjit/core/assembler.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ASSEMBLER_H_INCLUDED @@ -125,8 +125,9 @@ public: //! \name Events //! \{ - ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; - ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + ASMJIT_API Error onAttach(CodeHolder& code) noexcept override; + ASMJIT_API Error onDetach(CodeHolder& code) noexcept override; + ASMJIT_API Error onReinit(CodeHolder& code) noexcept override; //! \} }; diff --git a/src/asmjit/core/builder.cpp b/src/asmjit/core/builder.cpp index ba771be..891ea91 100644 --- a/src/asmjit/core/builder.cpp +++ b/src/asmjit/core/builder.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -46,8 +46,7 @@ static void BaseBuilder_deletePasses(BaseBuilder* self) noexcept { BaseBuilder::BaseBuilder() noexcept : BaseEmitter(EmitterType::kBuilder), - _codeZone(32u * 1024u), - _dataZone(16u * 1024u), + _codeZone(64u * 1024u), _passZone(64u * 1024u), _allocator(&_codeZone) {} @@ -62,16 +61,15 @@ Error BaseBuilder::newInstNode(InstNode** out, InstId instId, InstOptions instOp uint32_t opCapacity = InstNode::capacityOfOpCount(opCount); ASMJIT_ASSERT(opCapacity >= InstNode::kBaseOpCapacity); - InstNode* node = _allocator.allocT(InstNode::nodeSizeOfOpCapacity(opCapacity)); - if (ASMJIT_UNLIKELY(!node)) { + void* ptr = _codeZone.alloc(InstNode::nodeSizeOfOpCapacity(opCapacity)); + if (ASMJIT_UNLIKELY(!ptr)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); } - *out = new(Support::PlacementNew{node}) InstNode(this, instId, instOptions, opCount, opCapacity); + *out = new(Support::PlacementNew{ptr}) InstNode(instId, instOptions, opCount, opCapacity); return kErrorOk; } - Error BaseBuilder::newLabelNode(LabelNode** out) { *out = nullptr; @@ -97,30 +95,19 @@ Error BaseBuilder::newEmbedDataNode(EmbedDataNode** out, TypeId typeId, const vo uint32_t typeSize = TypeUtils::sizeOf(finalTypeId); Support::FastUInt8 of = 0; - size_t dataSize = Support::mulOverflow(itemCount, size_t(typeSize), &of); + size_t nodeSize = Support::maddOverflow(itemCount, size_t(typeSize), sizeof(EmbedDataNode), &of); if (ASMJIT_UNLIKELY(of)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); } - EmbedDataNode* node; - ASMJIT_PROPAGATE(_newNodeT(&node)); - - node->_embed._typeId = typeId; - node->_embed._typeSize = uint8_t(typeSize); - node->_itemCount = itemCount; - node->_repeatCount = repeatCount; - - uint8_t* dstData = node->_inlineData; - if (dataSize > EmbedDataNode::kInlineBufferSize) { - dstData = static_cast(_dataZone.alloc(dataSize, 8)); - if (ASMJIT_UNLIKELY(!dstData)) { - return reportError(DebugUtils::errored(kErrorOutOfMemory)); - } - node->_externalData = dstData; - } + EmbedDataNode* node = nullptr; + ASMJIT_PROPAGATE(_newNodeTWithSize( + &node, Support::alignUp(nodeSize, Globals::kZoneAlignment), + typeId, uint8_t(typeSize), itemCount, repeatCount + )); if (data) { - memcpy(dstData, data, dataSize); + memcpy(node->data(), data, node->dataSize()); } *out = node; @@ -130,7 +117,7 @@ Error BaseBuilder::newEmbedDataNode(EmbedDataNode** out, TypeId typeId, const vo Error BaseBuilder::newConstPoolNode(ConstPoolNode** out) { *out = nullptr; - ASMJIT_PROPAGATE(_newNodeT(out)); + ASMJIT_PROPAGATE(_newNodeT(out, &_codeZone)); return registerLabelNode(*out); } @@ -143,7 +130,7 @@ Error BaseBuilder::newCommentNode(CommentNode** out, const char* data, size_t si } if (size > 0) { - data = static_cast(_dataZone.dup(data, size, true)); + data = static_cast(_codeZone.dup(data, size, true)); if (ASMJIT_UNLIKELY(!data)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); } @@ -184,7 +171,7 @@ BaseNode* BaseBuilder::addNode(BaseNode* node) noexcept { } } - node->addFlags(NodeFlags::kIsActive); + node->_addFlags(NodeFlags::kIsActive); if (node->isSection()) { _dirtySectionLinks = true; } @@ -203,7 +190,7 @@ BaseNode* BaseBuilder::addAfter(BaseNode* node, BaseNode* ref) noexcept { node->_prev = prev; node->_next = next; - node->addFlags(NodeFlags::kIsActive); + node->_addFlags(NodeFlags::kIsActive); if (node->isSection()) { _dirtySectionLinks = true; } @@ -231,7 +218,7 @@ BaseNode* BaseBuilder::addBefore(BaseNode* node, BaseNode* ref) noexcept { node->_prev = prev; node->_next = next; - node->addFlags(NodeFlags::kIsActive); + node->_addFlags(NodeFlags::kIsActive); if (node->isSection()) { _dirtySectionLinks = true; } @@ -271,7 +258,7 @@ BaseNode* BaseBuilder::removeNode(BaseNode* node) noexcept { node->_prev = nullptr; node->_next = nullptr; - node->clearFlags(NodeFlags::kIsActive); + node->_clearFlags(NodeFlags::kIsActive); if (node->isSection()) { _dirtySectionLinks = true; @@ -320,7 +307,7 @@ void BaseBuilder::removeNodes(BaseNode* first, BaseNode* last) noexcept { node->_prev = nullptr; node->_next = nullptr; - node->clearFlags(NodeFlags::kIsActive); + node->_clearFlags(NodeFlags::kIsActive); didRemoveSection |= uint32_t(node->isSection()); if (_cursor == node) { @@ -388,7 +375,7 @@ Error BaseBuilder::sectionNodeOf(SectionNode** out, uint32_t sectionId) { Error BaseBuilder::section(Section* section) { SectionNode* node; - ASMJIT_PROPAGATE(sectionNodeOf(&node, section->id())); + ASMJIT_PROPAGATE(sectionNodeOf(&node, section->sectionId())); ASMJIT_ASSUME(node != nullptr); if (!node->isActive()) { @@ -397,9 +384,8 @@ Error BaseBuilder::section(Section* section) { _cursor = node; } else { - // This is a bit tricky. We cache section links to make sure that - // switching sections doesn't involve traversal in linked-list unless - // the position of the section has changed. + // This is a bit tricky. We cache section links to make sure that switching sections doesn't involve + // traversal in linked-list unless the position of the section has changed. if (hasDirtySectionLinks()) { updateSectionLinks(); } @@ -474,9 +460,8 @@ Error BaseBuilder::registerLabelNode(LabelNode* node) { return DebugUtils::errored(kErrorNotInitialized); } - LabelEntry* le; - ASMJIT_PROPAGATE(_code->newLabelEntry(&le)); - uint32_t labelId = le->id(); + uint32_t labelId; + ASMJIT_PROPAGATE(_code->newLabelId(&labelId)); // We just added one label so it must be true. ASMJIT_ASSERT(_labelNodes.size() < labelId + 1); @@ -498,7 +483,7 @@ static Error BaseBuilder_newLabelInternal(BaseBuilder* self, uint32_t labelId) { return self->reportError(err); } - LabelNode* node; + LabelNode* node = nullptr; ASMJIT_PROPAGATE(self->_newNodeT(&node, labelId)); // SAFETY: No need to check for error condition as we have already reserved enough space. @@ -509,29 +494,43 @@ static Error BaseBuilder_newLabelInternal(BaseBuilder* self, uint32_t labelId) { } Label BaseBuilder::newLabel() { - uint32_t labelId = Globals::kInvalidId; - LabelEntry* le; + Label label; - if (_code && - _code->newLabelEntry(&le) == kErrorOk && - BaseBuilder_newLabelInternal(this, le->id()) == kErrorOk) { - labelId = le->id(); + if (ASMJIT_LIKELY(_code)) { + uint32_t labelId; + Error err = _code->newLabelId(&labelId); + + if (ASMJIT_UNLIKELY(err)) { + reportError(err); + } + else { + if (ASMJIT_LIKELY(BaseBuilder_newLabelInternal(this, labelId)) == kErrorOk) { + label.setId(labelId); + } + } } - return Label(labelId); + return label; } Label BaseBuilder::newNamedLabel(const char* name, size_t nameSize, LabelType type, uint32_t parentId) { - uint32_t labelId = Globals::kInvalidId; - LabelEntry* le; + Label label; - if (_code && - _code->newNamedLabelEntry(&le, name, nameSize, type, parentId) == kErrorOk && - BaseBuilder_newLabelInternal(this, le->id()) == kErrorOk) { - labelId = le->id(); + if (ASMJIT_LIKELY(_code)) { + uint32_t labelId; + Error err = _code->newNamedLabelId(&labelId, name, nameSize, type, parentId); + + if (ASMJIT_UNLIKELY(err)) { + reportError(err); + } + else { + if (ASMJIT_LIKELY(BaseBuilder_newLabelInternal(this, labelId) == kErrorOk)) { + label.setId(labelId); + } + } } - return Label(labelId); + return label; } Error BaseBuilder::bind(const Label& label) { @@ -577,31 +576,6 @@ ASMJIT_FAVOR_SIZE Error BaseBuilder::addPass(Pass* pass) noexcept { return kErrorOk; } -ASMJIT_FAVOR_SIZE Error BaseBuilder::deletePass(Pass* pass) noexcept { - if (ASMJIT_UNLIKELY(!_code)) { - return DebugUtils::errored(kErrorNotInitialized); - } - - if (ASMJIT_UNLIKELY(pass == nullptr)) { - return DebugUtils::errored(kErrorInvalidArgument); - } - - if (pass->_cb != nullptr) { - if (pass->_cb != this) { - return DebugUtils::errored(kErrorInvalidState); - } - - uint32_t index = _passes.indexOf(pass); - ASMJIT_ASSERT(index != Globals::kNotFound); - - pass->_cb = nullptr; - _passes.removeAt(index); - } - - pass->~Pass(); - return kErrorOk; -} - Error BaseBuilder::runPasses() { if (ASMJIT_UNLIKELY(!_code)) { return DebugUtils::errored(kErrorNotInitialized); @@ -673,18 +647,18 @@ Error BaseBuilder::_emit(InstId instId, const Operand_& o0, const Operand_& o1, uint32_t opCapacity = InstNode::capacityOfOpCount(opCount); ASMJIT_ASSERT(opCapacity >= InstNode::kBaseOpCapacity); - InstNode* node = _allocator.allocT(InstNode::nodeSizeOfOpCapacity(opCapacity)); + void* ptr = _codeZone.alloc(InstNode::nodeSizeOfOpCapacity(opCapacity)); const char* comment = inlineComment(); resetInstOptions(); resetInlineComment(); - if (ASMJIT_UNLIKELY(!node)) { + if (ASMJIT_UNLIKELY(!ptr)) { resetExtraReg(); return reportError(DebugUtils::errored(kErrorOutOfMemory)); } - node = new(Support::PlacementNew{node}) InstNode(this, instId, options, opCount, opCapacity); + InstNode* node = new(Support::PlacementNew{ptr}) InstNode(instId, options, opCount, opCapacity); node->setExtraReg(extraReg()); node->setOp(0, o0); node->setOp(1, o1); @@ -695,7 +669,7 @@ Error BaseBuilder::_emit(InstId instId, const Operand_& o0, const Operand_& o1, node->resetOpRange(opCount, opCapacity); if (comment) { - node->setInlineComment(static_cast(_dataZone.dup(comment, strlen(comment), true))); + node->setInlineComment(static_cast(_codeZone.dup(comment, strlen(comment), true))); } addNode(node); @@ -771,24 +745,13 @@ Error BaseBuilder::embedConstPool(const Label& label, const ConstPool& pool) { // BaseBuilder - EmbedLabel & EmbedLabelDelta // ========================================== -// -// If dataSize is zero it means that the size is the same as target register width, however, -// if it's provided we really want to validate whether it's within the possible range. - -static inline bool BaseBuilder_checkDataSize(size_t dataSize) noexcept { - return !dataSize || (Support::isPowerOf2(dataSize) && dataSize <= 8); -} Error BaseBuilder::embedLabel(const Label& label, size_t dataSize) { - if (ASMJIT_UNLIKELY(!_code)) { - return DebugUtils::errored(kErrorNotInitialized); + if (ASMJIT_UNLIKELY(!Support::bool_and(_code, Support::isZeroOrPowerOf2UpTo(dataSize, 8u)))) { + return reportError(DebugUtils::errored(!_code ? kErrorNotInitialized : kErrorInvalidArgument)); } - if (!BaseBuilder_checkDataSize(dataSize)) { - return reportError(DebugUtils::errored(kErrorInvalidArgument)); - } - - EmbedLabelNode* node; + EmbedLabelNode* node = nullptr; ASMJIT_PROPAGATE(_newNodeT(&node, label.id(), uint32_t(dataSize))); addNode(node); @@ -796,15 +759,11 @@ Error BaseBuilder::embedLabel(const Label& label, size_t dataSize) { } Error BaseBuilder::embedLabelDelta(const Label& label, const Label& base, size_t dataSize) { - if (ASMJIT_UNLIKELY(!_code)) { - return DebugUtils::errored(kErrorNotInitialized); + if (ASMJIT_UNLIKELY(!Support::bool_and(_code, Support::isZeroOrPowerOf2UpTo(dataSize, 8u)))) { + return reportError(DebugUtils::errored(!_code ? kErrorNotInitialized : kErrorInvalidArgument)); } - if (!BaseBuilder_checkDataSize(dataSize)) { - return reportError(DebugUtils::errored(kErrorInvalidArgument)); - } - - EmbedLabelDeltaNode* node; + EmbedLabelDeltaNode* node = nullptr; ASMJIT_PROPAGATE(_newNodeT(&node, label.id(), base.id(), uint32_t(dataSize))); addNode(node); @@ -895,7 +854,7 @@ Error BaseBuilder::serializeTo(BaseEmitter* dst) { } else if (node_->isSection()) { SectionNode* node = node_->as(); - err = dst->section(_code->sectionById(node->id())); + err = dst->section(_code->sectionById(node->sectionId())); } else if (node_->isComment()) { CommentNode* node = node_->as(); @@ -914,46 +873,58 @@ Error BaseBuilder::serializeTo(BaseEmitter* dst) { // BaseBuilder - Events // ==================== -Error BaseBuilder::onAttach(CodeHolder* code) noexcept { - ASMJIT_PROPAGATE(Base::onAttach(code)); +static ASMJIT_INLINE void BaseBuilder_clearAll(BaseBuilder* self) noexcept { + self->_sectionNodes.reset(); + self->_labelNodes.reset(); + self->_allocator.reset(&self->_codeZone); + self->_codeZone.reset(); + self->_passZone.reset(); + + self->_cursor = nullptr; + self->_nodeList.reset(); +} + +static ASMJIT_INLINE Error BaseBuilder_initSection(BaseBuilder* self) noexcept { SectionNode* initialSection; - Error err = sectionNodeOf(&initialSection, 0); - if (!err) { - err = _passes.willGrow(&_allocator, 8); - } - - if (ASMJIT_UNLIKELY(err)) { - onDetach(code); - return err; - } + ASMJIT_PROPAGATE(self->sectionNodeOf(&initialSection, 0)); + ASMJIT_PROPAGATE(self->_passes.willGrow(&self->_allocator, 4)); ASMJIT_ASSUME(initialSection != nullptr); - _cursor = initialSection; - _nodeList.reset(initialSection, initialSection); - initialSection->setFlags(NodeFlags::kIsActive); + self->_cursor = initialSection; + self->_nodeList.reset(initialSection, initialSection); + initialSection->_setFlags(NodeFlags::kIsActive); return kErrorOk; } -Error BaseBuilder::onDetach(CodeHolder* code) noexcept { +Error BaseBuilder::onAttach(CodeHolder& code) noexcept { + ASMJIT_PROPAGATE(Base::onAttach(code)); + + Error err = BaseBuilder_initSection(this); + if (ASMJIT_UNLIKELY(err)) { + onDetach(code); + } + return err; +} + +Error BaseBuilder::onDetach(CodeHolder& code) noexcept { BaseBuilder_deletePasses(this); - _sectionNodes.reset(); - _labelNodes.reset(); - - _allocator.reset(&_codeZone); - _codeZone.reset(); - _dataZone.reset(); - _passZone.reset(); - - _nodeFlags = NodeFlags::kNone; - _cursor = nullptr; - _nodeList.reset(); + BaseBuilder_clearAll(this); return Base::onDetach(code); } +Error BaseBuilder::onReinit(CodeHolder& code) noexcept { + // BaseEmitter::onReinit() never fails. + (void)Base::onReinit(code); + + BaseBuilder_deletePasses(this); + BaseBuilder_clearAll(this); + return BaseBuilder_initSection(this); +} + // Pass - Construction & Destruction // ================================= diff --git a/src/asmjit/core/builder.h b/src/asmjit/core/builder.h index 5e2f3e9..35e661b 100644 --- a/src/asmjit/core/builder.h +++ b/src/asmjit/core/builder.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_BUILDER_H_INCLUDED @@ -196,8 +196,6 @@ public: //! Base zone used to allocate nodes and passes. Zone _codeZone; - //! Data zone used to allocate data and names. - Zone _dataZone; //! Pass zone, passed to `Pass::run()`. Zone _passZone; //! Allocator that uses `_codeZone`. @@ -215,8 +213,6 @@ public: //! First and last nodes. NodeList _nodeList; - //! Flags assigned to each new node. - NodeFlags _nodeFlags = NodeFlags::kNone; //! The sections links are dirty (used internally). bool _dirtySectionLinks = false; @@ -246,6 +242,20 @@ public: [[nodiscard]] ASMJIT_INLINE_NODEBUG BaseNode* lastNode() const noexcept { return _nodeList.last(); } + //! Allocates data required for a node. + template + ASMJIT_INLINE Error _newNodeTWithSize(T** ASMJIT_NONNULL(out), size_t size, Args&&... args) { + ASMJIT_ASSERT(Support::isAligned(size, Globals::kZoneAlignment)); + + void* ptr =_codeZone.alloc(size); + if (ASMJIT_UNLIKELY(!ptr)) { + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } + + *out = new(Support::PlacementNew{ptr}) T(std::forward(args)...); + return kErrorOk; + } + //! Allocates and instantiates a new node of type `T` and returns its instance. If the allocation fails `nullptr` //! is returned. //! @@ -254,10 +264,14 @@ public: //! \remarks The pointer returned (if non-null) is owned by the Builder or Compiler. When the Builder/Compiler //! is destroyed it destroys all nodes it created so no manual memory management is required. template - inline Error _newNodeT(T** ASMJIT_NONNULL(out), Args&&... args) { - *out = _allocator.newT(this, std::forward(args)...); - if (ASMJIT_UNLIKELY(!*out)) + ASMJIT_INLINE Error _newNodeT(T** ASMJIT_NONNULL(out), Args&&... args) { + void* ptr = _codeZone.alloc(Zone::alignedSizeOf()); + + if (ASMJIT_UNLIKELY(!ptr)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } + + *out = new(Support::PlacementNew{ptr}) T(std::forward(args)...); return kErrorOk; } @@ -377,6 +391,7 @@ public: //! //! This function is used internally to register a newly created `LabelNode` with this instance of Builder/Compiler. //! Use \ref labelNodeOf() functions to get back \ref LabelNode from a label or its identifier. + [[nodiscard]] ASMJIT_API Error registerLabelNode(LabelNode* ASMJIT_NONNULL(node)); [[nodiscard]] @@ -427,9 +442,6 @@ public: //! Adds `pass` to the list of passes. ASMJIT_API Error addPass(Pass* pass) noexcept; - //! Removes `pass` from the list of passes and delete it. - ASMJIT_API Error deletePass(Pass* pass) noexcept; - //! Runs all passes in order. ASMJIT_API Error runPasses(); @@ -482,8 +494,9 @@ public: //! \name Events //! \{ - ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; - ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + ASMJIT_API Error onAttach(CodeHolder& code) noexcept override; + ASMJIT_API Error onDetach(CodeHolder& code) noexcept override; + ASMJIT_API Error onReinit(CodeHolder& code) noexcept override; //! \} }; @@ -508,16 +521,17 @@ public: //! Next node. BaseNode* _next; }; - //! Links (an alternative view to previous and next nodes). + //! Links (an alternative view of previous and next nodes). BaseNode* _links[2]; }; + //! Node type. + NodeType _nodeType; + //! Node flags. + NodeFlags _nodeFlags; + //! Data shared between all types of nodes. struct AnyData { - //! Node type. - NodeType _nodeType; - //! Node flags. - NodeFlags _nodeFlags; //! Not used by BaseNode. uint8_t _reserved0; //! Not used by BaseNode. @@ -526,10 +540,6 @@ public: //! Data used by \ref AlignNode. struct AlignData { - //! Node type. - NodeType _nodeType; - //! Node flags. - NodeFlags _nodeFlags; //! Align mode. AlignMode _alignMode; //! Not used by AlignNode. @@ -538,10 +548,6 @@ public: //! Data used by \ref InstNode. struct InstData { - //! Node type. - NodeType _nodeType; - //! Node flags. - NodeFlags _nodeFlags; //! Instruction operands count (used). uint8_t _opCount; //! Instruction operands capacity (allocated). @@ -550,10 +556,6 @@ public: //! Data used by \ref EmbedDataNode. struct EmbedData { - //! Node type. - NodeType _nodeType; - //! Node flags. - NodeFlags _nodeFlags; //! Type id. TypeId _typeId; //! Size of `_typeId`. @@ -562,10 +564,6 @@ public: //! Data used by \ref SentinelNode. struct SentinelData { - //! Node type. - NodeType _nodeType; - //! Node flags. - NodeFlags _nodeFlags; //! Sentinel type. SentinelType _sentinelType; //! Not used by BaseNode. @@ -609,11 +607,11 @@ public: //! \{ //! Creates a new `BaseNode` - always use `BaseBuilder` to allocate nodes. - ASMJIT_INLINE_NODEBUG BaseNode(BaseBuilder* cb, NodeType nodeType, NodeFlags nodeFlags = NodeFlags::kNone) noexcept { + ASMJIT_INLINE_NODEBUG explicit BaseNode(NodeType nodeType, NodeFlags nodeFlags = NodeFlags::kNone) noexcept { _prev = nullptr; _next = nullptr; - _any._nodeType = nodeType; - _any._nodeFlags = nodeFlags | cb->_nodeFlags; + _nodeType = nodeType; + _nodeFlags = nodeFlags; _any._reserved0 = 0; _any._reserved1 = 0; _position = 0; @@ -647,13 +645,13 @@ public: //! Returns the type of the node, see \ref NodeType. [[nodiscard]] - ASMJIT_INLINE_NODEBUG NodeType type() const noexcept { return _any._nodeType; } + ASMJIT_INLINE_NODEBUG NodeType type() const noexcept { return _nodeType; } //! Sets the type of the node, see `NodeType` (internal). //! //! \remarks You should never set a type of a node to anything else than the initial value. This function is only //! provided for users that use custom nodes and need to change the type either during construction or later. - ASMJIT_INLINE_NODEBUG void setType(NodeType type) noexcept { _any._nodeType = type; } + ASMJIT_INLINE_NODEBUG void _setType(NodeType type) noexcept { _nodeType = type; } //! Tests whether this node is either `InstNode` or extends it. [[nodiscard]] @@ -709,18 +707,20 @@ public: //! Returns the node flags. [[nodiscard]] - ASMJIT_INLINE_NODEBUG NodeFlags flags() const noexcept { return _any._nodeFlags; } + ASMJIT_INLINE_NODEBUG NodeFlags flags() const noexcept { return _nodeFlags; } //! Tests whether the node has the given `flag` set. [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool hasFlag(NodeFlags flag) const noexcept { return Support::test(_any._nodeFlags, flag); } + ASMJIT_INLINE_NODEBUG bool hasFlag(NodeFlags flag) const noexcept { return Support::test(_nodeFlags, flag); } //! Replaces node flags with `flags`. - ASMJIT_INLINE_NODEBUG void setFlags(NodeFlags flags) noexcept { _any._nodeFlags = flags; } + ASMJIT_INLINE_NODEBUG void _setFlags(NodeFlags flags) noexcept { _nodeFlags = flags; } + //! Adds the given `flags` to node flags. - ASMJIT_INLINE_NODEBUG void addFlags(NodeFlags flags) noexcept { _any._nodeFlags |= flags; } + ASMJIT_INLINE_NODEBUG void _addFlags(NodeFlags flags) noexcept { _nodeFlags |= flags; } + //! Clears the given `flags` from node flags. - ASMJIT_INLINE_NODEBUG void clearFlags(NodeFlags flags) noexcept { _any._nodeFlags &= ~flags; } + ASMJIT_INLINE_NODEBUG void _clearFlags(NodeFlags flags) noexcept { _nodeFlags &= ~flags; } //! Tests whether the node is code that can be executed. [[nodiscard]] @@ -840,13 +840,38 @@ public: //! embed 5. The rest (up to 6 operands) is considered extended. //! //! The number of operands InstNode holds is decided when \ref InstNode is created. - static inline constexpr uint32_t kBaseOpCapacity = uint32_t((128 - sizeof(BaseNode) - sizeof(BaseInst)) / sizeof(Operand_)); + static inline constexpr uint32_t kBaseOpCapacity = uint32_t((128u - sizeof(BaseNode) - sizeof(BaseInst)) / sizeof(Operand_)); //! Count of maximum number of operands \ref InstNode can hold. static inline constexpr uint32_t kFullOpCapacity = Globals::kMaxOpCount; //! \} + //! \cond INTERNAL + //! \name Operand Capacity Utilities + //! \{ + + //! Returns the capacity required for the given operands count `opCount`. + //! + //! There are only two capacities used - \ref kBaseOpCapacity and \ref kFullOpCapacity, so this function + //! is used to decide between these two. The general rule is that instructions that can be represented with + //! \ref kBaseOpCapacity would use this value, and all others would take \ref kFullOpCapacity. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR uint32_t capacityOfOpCount(uint32_t opCount) noexcept { + return opCount <= kBaseOpCapacity ? kBaseOpCapacity : kFullOpCapacity; + } + + //! Calculates the size of \ref InstNode required to hold at most `opCapacity` operands. + //! + //! This function is used internally to allocate \ref InstNode. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR size_t nodeSizeOfOpCapacity(uint32_t opCapacity) noexcept { + return Support::alignUp(sizeof(InstNode) + opCapacity * sizeof(Operand), Globals::kZoneAlignment); + } + + //! \} + //! \endcond + //! \name Members //! \{ @@ -859,8 +884,8 @@ public: //! \{ //! Creates a new `InstNode` instance. - ASMJIT_INLINE_NODEBUG InstNode(BaseBuilder* cb, InstId instId, InstOptions options, uint32_t opCount, uint32_t opCapacity = kBaseOpCapacity) noexcept - : BaseNode(cb, NodeType::kInst, NodeFlags::kIsCode | NodeFlags::kIsRemovable | NodeFlags::kActsAsInst), + ASMJIT_INLINE_NODEBUG InstNode(InstId instId, InstOptions options, uint32_t opCount, uint32_t opCapacity = kBaseOpCapacity) noexcept + : BaseNode(NodeType::kInst, NodeFlags::kIsCode | NodeFlags::kIsRemovable | NodeFlags::kActsAsInst), _baseInst(instId, options) { _inst._opCapacity = uint8_t(opCapacity); _inst._opCount = uint8_t(opCount); @@ -941,7 +966,7 @@ public: ASMJIT_INLINE_NODEBUG const RegOnly& extraReg() const noexcept { return _baseInst.extraReg(); } //! Sets extra register operand to `reg`. - ASMJIT_INLINE_NODEBUG void setExtraReg(const BaseReg& reg) noexcept { _baseInst.setExtraReg(reg); } + ASMJIT_INLINE_NODEBUG void setExtraReg(const Reg& reg) noexcept { _baseInst.setExtraReg(reg); } //! Sets extra register operand to `reg`. ASMJIT_INLINE_NODEBUG void setExtraReg(const RegOnly& reg) noexcept { _baseInst.setExtraReg(reg); } //! Resets extra register operand. @@ -965,15 +990,11 @@ public: //! Returns operands array. [[nodiscard]] - ASMJIT_INLINE_NODEBUG Operand* operands() noexcept { - return reinterpret_cast(reinterpret_cast(this) + sizeof(InstNode)); - } + ASMJIT_INLINE_NODEBUG Operand* operands() noexcept { return Support::offsetPtr(this, sizeof(InstNode)); } //! Returns operands array (const). [[nodiscard]] - ASMJIT_INLINE_NODEBUG const Operand* operands() const noexcept { - return reinterpret_cast(reinterpret_cast(this) + sizeof(InstNode)); - } + ASMJIT_INLINE_NODEBUG const Operand* operands() const noexcept { return Support::offsetPtr(this, sizeof(InstNode)); } //! Returns operand at the given `index`. [[nodiscard]] @@ -1025,9 +1046,11 @@ public: [[nodiscard]] inline bool hasOpType(OperandType opType) const noexcept { const Operand* ops = operands(); - for (uint32_t i = 0, count = opCount(); i < count; i++) - if (ops[i].opType() == opType) + for (uint32_t i = 0, count = opCount(); i < count; i++) { + if (ops[i].opType() == opType) { return true; + } + } return false; } @@ -1080,11 +1103,10 @@ public: //! \} + //! \cond INTERNAL //! \name Rewriting //! \{ - //! \cond INTERNAL - //! Returns uint32_t[] view that represents BaseInst::RegOnly and instruction operands. [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t* _getRewriteArray() noexcept { return &_baseInst._extraReg._id; } @@ -1098,11 +1120,11 @@ public: //! Returns a rewrite index of the given pointer to `id`. //! - //! This function returns a value that can be then passed to `\ref rewriteIdAtIndex() function. It can address + //! This function returns a value that can be then passed to `\ref _rewriteIdAtIndex() function. It can address //! any id from any operand that is used by the instruction in addition to \ref BaseInst::regOnly field, which //! can also be used by the register allocator. [[nodiscard]] - inline uint32_t getRewriteIndex(const uint32_t* id) const noexcept { + inline uint32_t _getRewriteIndex(const uint32_t* id) const noexcept { const uint32_t* array = _getRewriteArray(); ASMJIT_ASSERT(array <= id); @@ -1119,41 +1141,15 @@ public: //! and the given `index` describes a position in this array. For example a single \ref Operand would be //! decomposed to 4 uint32_t values, where the first at index 0 would be operand signature, next would be //! base id, etc... This is a comfortable way of patching operands without having to check for their types. - inline void rewriteIdAtIndex(uint32_t index, uint32_t id) noexcept { + inline void _rewriteIdAtIndex(uint32_t index, uint32_t id) noexcept { ASMJIT_ASSERT(index <= kMaxRewriteId); uint32_t* array = _getRewriteArray(); array[index] = id; } - //! \endcond //! \} - - //! \name Static Functions - //! \{ - - //! \cond INTERNAL - - //! Returns the capacity required for the given operands count `opCount`. - //! - //! There are only two capacities used - \ref kBaseOpCapacity and \ref kFullOpCapacity, so this function - //! is used to decide between these two. The general rule is that instructions that can be represented with - //! \ref kBaseOpCapacity would use this value, and all others would take \ref kFullOpCapacity. - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR uint32_t capacityOfOpCount(uint32_t opCount) noexcept { - return opCount <= kBaseOpCapacity ? kBaseOpCapacity : kFullOpCapacity; - } - - //! Calculates the size of \ref InstNode required to hold at most `opCapacity` operands. - //! - //! This function is used internally to allocate \ref InstNode. - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR size_t nodeSizeOfOpCapacity(uint32_t opCapacity) noexcept { - return sizeof(InstNode) + opCapacity * sizeof(Operand); - } //! \endcond - - //! \} }; //! Instruction node with embedded operands following \ref InstNode layout. @@ -1167,8 +1163,8 @@ public: Operand_ _operands[kN]; //! Creates a new `InstNodeWithOperands` instance. - ASMJIT_INLINE_NODEBUG InstNodeWithOperands(BaseBuilder* cb, InstId instId, InstOptions options, uint32_t opCount) noexcept - : InstNode(cb, instId, options, opCount, kN) {} + ASMJIT_INLINE_NODEBUG InstNodeWithOperands(InstId instId, InstOptions options, uint32_t opCount) noexcept + : InstNode(instId, options, opCount, kN) {} }; //! Section node. @@ -1180,7 +1176,7 @@ public: //! \{ //! Section id. - uint32_t _id; + uint32_t _sectionId; //! Next section node that follows this section. //! @@ -1195,9 +1191,9 @@ public: //! \{ //! Creates a new `SectionNode` instance. - ASMJIT_INLINE_NODEBUG SectionNode(BaseBuilder* cb, uint32_t sectionId = 0) noexcept - : BaseNode(cb, NodeType::kSection, NodeFlags::kHasNoEffect), - _id(sectionId), + ASMJIT_INLINE_NODEBUG explicit SectionNode(uint32_t sectionId = 0) noexcept + : BaseNode(NodeType::kSection, NodeFlags::kHasNoEffect), + _sectionId(sectionId), _nextSection(nullptr) {} //! \} @@ -1206,7 +1202,7 @@ public: //! \{ //! Returns the section id. - ASMJIT_INLINE_NODEBUG uint32_t id() const noexcept { return _id; } + ASMJIT_INLINE_NODEBUG uint32_t sectionId() const noexcept { return _sectionId; } //! \} }; @@ -1228,8 +1224,8 @@ public: //! \{ //! Creates a new `LabelNode` instance. - ASMJIT_INLINE_NODEBUG LabelNode(BaseBuilder* cb, uint32_t labelId = 0) noexcept - : BaseNode(cb, NodeType::kLabel, NodeFlags::kHasNoEffect | NodeFlags::kActsAsLabel), + ASMJIT_INLINE_NODEBUG explicit LabelNode(uint32_t labelId = Globals::kInvalidId) noexcept + : BaseNode(NodeType::kLabel, NodeFlags::kHasNoEffect | NodeFlags::kActsAsLabel), _labelId(labelId) {} //! \} @@ -1267,8 +1263,8 @@ public: //! \{ //! Creates a new `AlignNode` instance. - ASMJIT_INLINE_NODEBUG AlignNode(BaseBuilder* cb, AlignMode alignMode, uint32_t alignment) noexcept - : BaseNode(cb, NodeType::kAlign, NodeFlags::kIsCode | NodeFlags::kHasNoEffect) { + ASMJIT_INLINE_NODEBUG AlignNode(AlignMode alignMode, uint32_t alignment) noexcept + : BaseNode(NodeType::kAlign, NodeFlags::kIsCode | NodeFlags::kHasNoEffect) { _alignData._alignMode = alignMode; _alignment = alignment; @@ -1304,34 +1300,24 @@ class EmbedDataNode : public BaseNode { public: ASMJIT_NONCOPYABLE(EmbedDataNode) - //! \cond INTERNAL - static inline constexpr uint32_t kInlineBufferSize = 128 - (sizeof(BaseNode) + sizeof(size_t) * 2); - //! \endcond - //! \name Members //! \{ size_t _itemCount; size_t _repeatCount; - union { - uint8_t* _externalData; - uint8_t _inlineData[kInlineBufferSize]; - }; - //! \} //! \name Construction & Destruction //! \{ //! Creates a new `EmbedDataNode` instance. - ASMJIT_INLINE_NODEBUG EmbedDataNode(BaseBuilder* cb) noexcept - : BaseNode(cb, NodeType::kEmbedData, NodeFlags::kIsData), - _itemCount(0), - _repeatCount(0) { - _embed._typeId = TypeId::kUInt8; - _embed._typeSize = uint8_t(1); - memset(_inlineData, 0, kInlineBufferSize); + ASMJIT_INLINE_NODEBUG EmbedDataNode(TypeId typeId, uint8_t typeSize, size_t itemCount, size_t repeatCount) noexcept + : BaseNode(NodeType::kEmbedData, NodeFlags::kIsData), + _itemCount(itemCount), + _repeatCount(repeatCount) { + _embed._typeId = typeId; + _embed._typeSize = typeSize; } //! \} @@ -1347,16 +1333,15 @@ public: [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t typeSize() const noexcept { return _embed._typeSize; } - //! Returns a pointer to the data casted to `uint8_t`. + //! Returns a pointer to the data casted to `T*` - `uint8_t*` by default. + template [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint8_t* data() const noexcept { - return dataSize() <= kInlineBufferSize ? const_cast(_inlineData) : _externalData; - } + ASMJIT_INLINE_NODEBUG uint8_t* data() noexcept { return Support::offsetPtr(this, sizeof(EmbedDataNode)); } - //! Returns a pointer to the data casted to `T`. - template + //! Returns a pointer to the data casted to `T*` - `const uint8_t*` by default (const). + template [[nodiscard]] - ASMJIT_INLINE_NODEBUG T* dataAs() const noexcept { return reinterpret_cast(data()); } + ASMJIT_INLINE_NODEBUG const uint8_t* data() const noexcept { return Support::offsetPtr(this, sizeof(EmbedDataNode)); } //! Returns the number of (typed) items in the array. [[nodiscard]] @@ -1372,7 +1357,7 @@ public: //! //! \note The returned value is the same as `typeSize() * itemCount()`. [[nodiscard]] - ASMJIT_INLINE_NODEBUG size_t dataSize() const noexcept { return typeSize() * _itemCount; } + ASMJIT_INLINE_NODEBUG size_t dataSize() const noexcept { return size_t(typeSize()) * _itemCount; } //! \} }; @@ -1394,8 +1379,8 @@ public: //! \{ //! Creates a new `EmbedLabelNode` instance. - ASMJIT_INLINE_NODEBUG EmbedLabelNode(BaseBuilder* cb, uint32_t labelId = 0, uint32_t dataSize = 0) noexcept - : BaseNode(cb, NodeType::kEmbedLabel, NodeFlags::kIsData), + ASMJIT_INLINE_NODEBUG EmbedLabelNode(uint32_t labelId = 0, uint32_t dataSize = 0) noexcept + : BaseNode(NodeType::kEmbedLabel, NodeFlags::kIsData), _labelId(labelId), _dataSize(dataSize) {} @@ -1446,8 +1431,8 @@ public: //! \{ //! Creates a new `EmbedLabelDeltaNode` instance. - ASMJIT_INLINE_NODEBUG EmbedLabelDeltaNode(BaseBuilder* cb, uint32_t labelId = 0, uint32_t baseLabelId = 0, uint32_t dataSize = 0) noexcept - : BaseNode(cb, NodeType::kEmbedLabelDelta, NodeFlags::kIsData), + ASMJIT_INLINE_NODEBUG EmbedLabelDeltaNode(uint32_t labelId = 0, uint32_t baseLabelId = 0, uint32_t dataSize = 0) noexcept + : BaseNode(NodeType::kEmbedLabelDelta, NodeFlags::kIsData), _labelId(labelId), _baseLabelId(baseLabelId), _dataSize(dataSize) {} @@ -1510,13 +1495,13 @@ public: //! \{ //! Creates a new `ConstPoolNode` instance. - ASMJIT_INLINE_NODEBUG ConstPoolNode(BaseBuilder* cb, uint32_t id = 0) noexcept - : LabelNode(cb, id), - _constPool(&cb->_codeZone) { + ASMJIT_INLINE_NODEBUG ConstPoolNode(Zone* zone, uint32_t id = 0) noexcept + : LabelNode(id), + _constPool(zone) { - setType(NodeType::kConstPool); - addFlags(NodeFlags::kIsData); - clearFlags(NodeFlags::kIsCode | NodeFlags::kHasNoEffect); + _setType(NodeType::kConstPool); + _addFlags(NodeFlags::kIsData); + _clearFlags(NodeFlags::kIsCode | NodeFlags::kHasNoEffect); } //! \} @@ -1566,8 +1551,8 @@ public: //! \{ //! Creates a new `CommentNode` instance. - ASMJIT_INLINE_NODEBUG CommentNode(BaseBuilder* cb, const char* comment) noexcept - : BaseNode(cb, NodeType::kComment, NodeFlags::kIsInformative | NodeFlags::kHasNoEffect | NodeFlags::kIsRemovable) { + ASMJIT_INLINE_NODEBUG CommentNode(const char* comment) noexcept + : BaseNode(NodeType::kComment, NodeFlags::kIsInformative | NodeFlags::kHasNoEffect | NodeFlags::kIsRemovable) { _inlineComment = comment; } @@ -1586,8 +1571,8 @@ public: //! \{ //! Creates a new `SentinelNode` instance. - ASMJIT_INLINE_NODEBUG SentinelNode(BaseBuilder* cb, SentinelType sentinelType = SentinelType::kUnknown) noexcept - : BaseNode(cb, NodeType::kSentinel, NodeFlags::kIsInformative | NodeFlags::kHasNoEffect) { + ASMJIT_INLINE_NODEBUG SentinelNode(SentinelType sentinelType = SentinelType::kUnknown) noexcept + : BaseNode(NodeType::kSentinel, NodeFlags::kIsInformative | NodeFlags::kHasNoEffect) { _sentinel._sentinelType = sentinelType; } diff --git a/src/asmjit/core/builder_p.h b/src/asmjit/core/builder_p.h index 98790fd..887e05b 100644 --- a/src/asmjit/core/builder_p.h +++ b/src/asmjit/core/builder_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_BUILDER_P_H_INCLUDED @@ -18,8 +18,9 @@ ASMJIT_BEGIN_NAMESPACE //! \{ static inline void BaseBuilder_assignInlineComment(BaseBuilder* self, BaseNode* node, const char* comment) noexcept { - if (comment) - node->setInlineComment(static_cast(self->_dataZone.dup(comment, strlen(comment), true))); + if (comment) { + node->setInlineComment(static_cast(self->_codeZone.dup(comment, strlen(comment), true))); + } } static inline void BaseBuilder_assignInstState(BaseBuilder* self, InstNode* node, const BaseEmitter::State& state) noexcept { diff --git a/src/asmjit/core/codebuffer.h b/src/asmjit/core/codebuffer.h index 32bb03c..ed77cdd 100644 --- a/src/asmjit/core/codebuffer.h +++ b/src/asmjit/core/codebuffer.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_CODEBUFFER_H_INCLUDED diff --git a/src/asmjit/core/codeholder.cpp b/src/asmjit/core/codeholder.cpp index 9f18081..fd29409 100644 --- a/src/asmjit/core/codeholder.cpp +++ b/src/asmjit/core/codeholder.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -14,93 +14,181 @@ ASMJIT_BEGIN_NAMESPACE -// Globals -// ======= +// CodeHolder - X86 Utilities +// ========================== -static const char CodeHolder_addrTabName[] = ".addrtab"; - -//! Encode MOD byte. +//! Encodes a MOD byte. static inline uint32_t x86EncodeMod(uint32_t m, uint32_t o, uint32_t rm) noexcept { return (m << 6) | (o << 3) | rm; } -// LabelLinkIterator -// ================= +// CodeHolder - LabelEntry Globals & Utilities +// =========================================== -class LabelLinkIterator { +static constexpr LabelEntry::ExtraData _makeSharedLabelExtraData() noexcept { + LabelEntry::ExtraData extraData {}; + extraData._sectionId = Globals::kInvalidId; + extraData._parentId = Globals::kInvalidId; + return extraData; +} + +static constexpr LabelEntry::ExtraData CodeHolder_sharedLabelExtraData = _makeSharedLabelExtraData(); + +class ResolveFixupIterator { public: - inline LabelLinkIterator(LabelEntry* le) noexcept { reset(le); } + Fixup* _fixup {}; + Fixup** _pPrev {}; + size_t _resolvedCount {}; + size_t _unresolvedCount {}; - inline explicit operator bool() const noexcept { return isValid(); } - inline bool isValid() const noexcept { return _link != nullptr; } + ASMJIT_INLINE_NODEBUG explicit ResolveFixupIterator(Fixup** ppFixup) noexcept { reset(ppFixup); } + ASMJIT_INLINE_NODEBUG bool isValid() const noexcept { return _fixup != nullptr; } + ASMJIT_INLINE_NODEBUG Fixup* fixup() const noexcept { return _fixup; } - inline LabelLink* link() const noexcept { return _link; } - inline LabelLink* operator->() const noexcept { return _link; } - - inline void reset(LabelEntry* le) noexcept { - _pPrev = &le->_links; - _link = *_pPrev; + ASMJIT_INLINE void reset(Fixup** ppFixup) noexcept { + _pPrev = ppFixup; + _fixup = *_pPrev; } - inline void next() noexcept { - _pPrev = &_link->next; - _link = *_pPrev; + ASMJIT_INLINE void next() noexcept { + _pPrev = &_fixup->next; + _fixup = *_pPrev; + _unresolvedCount++; } - inline void resolveAndNext(CodeHolder* code) noexcept { - LabelLink* linkToDelete = _link; + ASMJIT_INLINE void resolveAndNext(CodeHolder* code) noexcept { + Fixup* fixupToDelete = _fixup; - _link = _link->next; - *_pPrev = _link; + _fixup = _fixup->next; + *_pPrev = _fixup; - code->_unresolvedLinkCount--; - code->_allocator.release(linkToDelete, sizeof(LabelLink)); + _resolvedCount++; + code->_fixupDataPool.release(fixupToDelete); } - LabelLink** _pPrev; - LabelLink* _link; + ASMJIT_INLINE_NODEBUG size_t resolvedCount() const noexcept { return _resolvedCount; } + ASMJIT_INLINE_NODEBUG size_t unresolvedCount() const noexcept { return _unresolvedCount; } }; +// CodeHolder - Section Globals & Utilities +// ======================================== + +static const char CodeHolder_addrTabName[] = ".addrtab"; + +static ASMJIT_INLINE void Section_initName( + Section* section, + char c0 = 0, char c1 = 0, char c2 = 0, char c3 = 0, + char c4 = 0, char c5 = 0, char c6 = 0, char c7 = 0) noexcept { + + section->_name.u32[0] = Support::bytepack32_4x8(uint8_t(c0), uint8_t(c1), uint8_t(c2), uint8_t(c3)); + section->_name.u32[1] = Support::bytepack32_4x8(uint8_t(c4), uint8_t(c5), uint8_t(c6), uint8_t(c7)); + section->_name.u32[2] = 0u; + section->_name.u32[3] = 0u; +} + +static ASMJIT_INLINE void Section_initData(Section* section, uint32_t sectionId, SectionFlags flags, uint32_t alignment, int order) noexcept { + section->_sectionId = sectionId; + + // These two fields are not used by sections (see \ref LabelEntry for more details about why). + section->_internalLabelType = LabelType::kAnonymous; + section->_internalLabelFlags = LabelFlags::kNone; + + section->assignFlags(flags); + section->_alignment = alignment; + section->_order = order; + section->_offset = 0; + section->_virtualSize = 0; +} + +static ASMJIT_INLINE void Section_initBuffer(Section* section) noexcept { + section->_buffer = CodeBuffer{}; +} + +static ASMJIT_INLINE void Section_releaseBuffer(Section* section) noexcept { + if (Support::bool_and(section->_buffer.data() != nullptr, !section->_buffer.isExternal())) { + ::free(section->_buffer._data); + } +} + // CodeHolder - Utilities // ====================== -static void CodeHolder_resetInternal(CodeHolder* self, ResetPolicy resetPolicy) noexcept { - uint32_t i; - const ZoneVector& emitters = self->emitters(); +static ASMJIT_INLINE Error CodeHolder_initSectionStorage(CodeHolder* self) noexcept { + Error err1 = self->_sections.willGrow(&self->_allocator); + Error err2 = self->_sectionsByOrder.willGrow(&self->_allocator); - i = emitters.size(); - while (i) - self->detach(emitters[--i]); + return err1 | err2; +} - // Reset everything into its construction state. +static ASMJIT_INLINE void CodeHolder_addTextSection(CodeHolder* self) noexcept { + Section* textSection = &self->_textSection; + + Section_initData(textSection, 0u, SectionFlags::kExecutable | SectionFlags::kReadOnly | SectionFlags::kBuiltIn, 0u, 0); + Section_initName(textSection, '.', 't', 'e', 'x', 't'); + + self->_sections.appendUnsafe(textSection); + self->_sectionsByOrder.appendUnsafe(textSection); +} + +static ASMJIT_NOINLINE void CodeHolder_detachEmitters(CodeHolder* self) noexcept { + BaseEmitter* emitter = self->_attachedFirst; + + while (emitter) { + BaseEmitter* next = emitter->_attachedNext; + + emitter->_attachedPrev = nullptr; + (void)emitter->onDetach(*self); + emitter->_attachedNext = nullptr; + emitter->_code = nullptr; + + emitter = next; + self->_attachedFirst = next; + } + + self->_attachedLast = nullptr; +} + +static ASMJIT_INLINE void CodeHolder_resetEnvAndAttachedLogAndEH(CodeHolder* self) noexcept { self->_environment.reset(); self->_cpuFeatures.reset(); self->_baseAddress = Globals::kNoBaseAddress; self->_logger = nullptr; self->_errorHandler = nullptr; +} - // Reset all sections. - uint32_t numSections = self->_sections.size(); - for (i = 0; i < numSections; i++) { +// Reset zone allocator and all containers using it. +static ASMJIT_INLINE void CodeHolder_resetSections(CodeHolder* self, ResetPolicy resetPolicy) noexcept { + // Reset all sections except the first one (.text section). + uint32_t fromSection = resetPolicy == ResetPolicy::kHard ? 0u : 1u; + uint32_t sectionCount = self->_sections.size(); + + for (uint32_t i = fromSection; i < sectionCount; i++) { Section* section = self->_sections[i]; - if (section->_buffer.data() && !section->_buffer.isExternal()) { - ::free(section->_buffer._data); - } + + Section_releaseBuffer(section); section->_buffer._data = nullptr; section->_buffer._capacity = 0; } +} + +// Reset zone allocator and all containers using it. +static ASMJIT_INLINE void CodeHolder_resetContainers(CodeHolder* self, ResetPolicy resetPolicy) noexcept { + // Soft reset won't wipe out the .text section, so set its size to 0 for future reuse. + self->_textSection._buffer._size = 0; - // Reset zone allocator and all containers using it. ZoneAllocator* allocator = self->allocator(); - self->_emitters.reset(); self->_namedLabels.reset(); self->_relocations.reset(); self->_labelEntries.reset(); + + self->_fixups = nullptr; + self->_fixupDataPool.reset(); + self->_unresolvedFixupCount = 0; + self->_sections.reset(); self->_sectionsByOrder.reset(); - self->_unresolvedLinkCount = 0; self->_addressTableSection = nullptr; self->_addressTableEntries.reset(); @@ -108,10 +196,18 @@ static void CodeHolder_resetInternal(CodeHolder* self, ResetPolicy resetPolicy) self->_zone.reset(resetPolicy); } -static void CodeHolder_onSettingsUpdated(CodeHolder* self) noexcept { +// Reset zone allocator and all containers using it. +static ASMJIT_NOINLINE void CodeHolder_resetSectionsAndContainers(CodeHolder* self, ResetPolicy resetPolicy) noexcept { + CodeHolder_resetSections(self, resetPolicy); + CodeHolder_resetContainers(self, resetPolicy); +} + +static ASMJIT_INLINE void CodeHolder_onSettingsUpdated(CodeHolder* self) noexcept { // Notify all attached emitters about a settings update. - for (BaseEmitter* emitter : self->emitters()) { + BaseEmitter* emitter = self->_attachedFirst; + while (emitter) { emitter->onSettingsUpdated(); + emitter = emitter->_attachedNext; } } @@ -124,70 +220,85 @@ CodeHolder::CodeHolder(const Support::Temporary* temporary) noexcept _baseAddress(Globals::kNoBaseAddress), _logger(nullptr), _errorHandler(nullptr), - _zone(16u * 1024u, 1, temporary), + _zone(16u * 1024u, temporary), _allocator(&_zone), - _unresolvedLinkCount(0), + _attachedFirst(nullptr), + _attachedLast(nullptr), + _fixups(nullptr), + _unresolvedFixupCount(0), + _textSection{}, _addressTableSection(nullptr) {} CodeHolder::~CodeHolder() noexcept { - CodeHolder_resetInternal(this, ResetPolicy::kHard); + if (isInitialized()) { + CodeHolder_detachEmitters(this); + CodeHolder_resetSections(this, ResetPolicy::kHard); + } + else { + Section_releaseBuffer(&_textSection); + } } // CodeHolder - Initialization & Reset // =================================== -inline void CodeHolder_setSectionDefaultName( - Section* section, - char c0 = 0, char c1 = 0, char c2 = 0, char c3 = 0, - char c4 = 0, char c5 = 0, char c6 = 0, char c7 = 0) noexcept { - - section->_name.u32[0] = Support::bytepack32_4x8(uint8_t(c0), uint8_t(c1), uint8_t(c2), uint8_t(c3)); - section->_name.u32[1] = Support::bytepack32_4x8(uint8_t(c4), uint8_t(c5), uint8_t(c6), uint8_t(c7)); -} - Error CodeHolder::init(const Environment& environment, uint64_t baseAddress) noexcept { return init(environment, CpuFeatures{}, baseAddress); } Error CodeHolder::init(const Environment& environment, const CpuFeatures& cpuFeatures, uint64_t baseAddress) noexcept { - // Cannot reinitialize if it's locked or there is one or more emitter attached. - if (isInitialized()) { - return DebugUtils::errored(kErrorAlreadyInitialized); + // Cannot initialize if it's already initialized or the environment passed is invalid. + if (ASMJIT_UNLIKELY(Support::bool_or(isInitialized(), !environment.isInitialized()))) { + Error err = isInitialized() ? kErrorAlreadyInitialized : kErrorInvalidArgument; + return DebugUtils::errored(err); } // If we are just initializing there should be no emitters attached. - ASMJIT_ASSERT(_emitters.empty()); + ASMJIT_ASSERT(_attachedFirst == nullptr); + ASMJIT_ASSERT(_attachedLast == nullptr); // Create a default section and insert it to the `_sections` array. - Error err = _sections.willGrow(&_allocator) | - _sectionsByOrder.willGrow(&_allocator); - if (err == kErrorOk) { - Section* section = _allocator.allocZeroedT
(); - if (ASMJIT_LIKELY(section)) { - section->_flags = SectionFlags::kExecutable | SectionFlags::kReadOnly; - CodeHolder_setSectionDefaultName(section, '.', 't', 'e', 'x', 't'); - _sections.appendUnsafe(section); - _sectionsByOrder.appendUnsafe(section); - } - else { - err = DebugUtils::errored(kErrorOutOfMemory); - } - } - + Error err = CodeHolder_initSectionStorage(this); if (ASMJIT_UNLIKELY(err)) { _zone.reset(); - return err; + return DebugUtils::errored(kErrorOutOfMemory); } - else { - _environment = environment; - _cpuFeatures = cpuFeatures; - _baseAddress = baseAddress; - return kErrorOk; + + _environment = environment; + _cpuFeatures = cpuFeatures; + _baseAddress = baseAddress; + + CodeHolder_addTextSection(this); + return kErrorOk; +} + +Error CodeHolder::reinit() noexcept { + // Cannot reinitialize if it's not initialized. + if (ASMJIT_UNLIKELY(!isInitialized())) { + return DebugUtils::errored(kErrorNotInitialized); } + + CodeHolder_resetSectionsAndContainers(this, ResetPolicy::kSoft); + + // Create a default section and insert it to the `_sections` array. + (void)CodeHolder_initSectionStorage(this); + CodeHolder_addTextSection(this); + + BaseEmitter* emitter = _attachedFirst; + while (emitter) { + emitter->onReinit(*this); + emitter = emitter->_attachedNext; + } + + return kErrorOk; } void CodeHolder::reset(ResetPolicy resetPolicy) noexcept { - CodeHolder_resetInternal(this, resetPolicy); + if (isInitialized()) { + CodeHolder_detachEmitters(this); + CodeHolder_resetEnvAndAttachedLogAndEH(this); + CodeHolder_resetSectionsAndContainers(this, resetPolicy); + } } // CodeHolder - Attach / Detach @@ -220,12 +331,25 @@ Error CodeHolder::attach(BaseEmitter* emitter) noexcept { } // Reserve the space now as we cannot fail after `onAttach()` succeeded. - ASMJIT_PROPAGATE(_emitters.willGrow(&_allocator, 1)); - ASMJIT_PROPAGATE(emitter->onAttach(this)); + ASMJIT_PROPAGATE(emitter->onAttach(*this)); - // Connect CodeHolder <-> BaseEmitter. + // Make sure CodeHolder <-> BaseEmitter are connected. ASMJIT_ASSERT(emitter->_code == this); - _emitters.appendUnsafe(emitter); + + // Add `emitter` to a double linked-list. + { + BaseEmitter* last = _attachedLast; + + emitter->_attachedPrev = last; + _attachedLast = emitter; + + if (last) { + last->_attachedNext = emitter; + } + else { + _attachedFirst = emitter; + } + } return kErrorOk; } @@ -244,15 +368,21 @@ Error CodeHolder::detach(BaseEmitter* emitter) noexcept { // be detached. Error err = kErrorOk; if (!emitter->isDestroyed()) { - err = emitter->onDetach(this); + err = emitter->onDetach(*this); } - // Disconnect CodeHolder <-> BaseEmitter. - uint32_t index = _emitters.indexOf(emitter); - ASMJIT_ASSERT(index != Globals::kNotFound); + // Remove `emitter` from a double linked-list. + { + BaseEmitter* prev = emitter->_attachedPrev; + BaseEmitter* next = emitter->_attachedNext; - _emitters.removeAt(index); - emitter->_code = nullptr; + if (prev) { prev->_attachedNext = next; } else { _attachedFirst = next; } + if (next) { next->_attachedPrev = prev; } else { _attachedLast = prev; } + + emitter->_code = nullptr; + emitter->_attachedPrev = nullptr; + emitter->_attachedNext = nullptr; + } return err; } @@ -299,7 +429,8 @@ static Error CodeHolder_reserveInternal(CodeHolder* self, CodeBuffer* cb, size_t cb->_capacity = n; // Update pointers used by assemblers, if attached. - for (BaseEmitter* emitter : self->emitters()) { + BaseEmitter* emitter = self->_attachedFirst; + while (emitter) { if (emitter->isAssembler()) { BaseAssembler* a = static_cast(emitter); if (&a->_section->_buffer == cb) { @@ -310,6 +441,7 @@ static Error CodeHolder_reserveInternal(CodeHolder* self, CodeBuffer* cb, size_t a->_bufferPtr = newData + offset; } } + emitter = emitter->_attachedNext; } return kErrorOk; @@ -335,7 +467,7 @@ Error CodeHolder::growBuffer(CodeBuffer* cb, size_t n) noexcept { return DebugUtils::errored(kErrorTooLarge); } - size_t kInitialCapacity = 8096; + size_t kInitialCapacity = 8192u - Globals::kAllocOverhead; if (capacity < kInitialCapacity) { capacity = kInitialCapacity; } @@ -345,12 +477,9 @@ Error CodeHolder::growBuffer(CodeBuffer* cb, size_t n) noexcept { do { size_t old = capacity; - if (capacity < Globals::kGrowThreshold) { - capacity *= 2; - } - else { - capacity += Globals::kGrowThreshold; - } + size_t capacityIncrease = capacity < Globals::kGrowThreshold ? capacity : Globals::kGrowThreshold; + + capacity += capacityIncrease; // Overflow. if (ASMJIT_UNLIKELY(old > capacity)) { @@ -381,18 +510,15 @@ Error CodeHolder::reserveBuffer(CodeBuffer* cb, size_t n) noexcept { Error CodeHolder::newSection(Section** sectionOut, const char* name, size_t nameSize, SectionFlags flags, uint32_t alignment, int32_t order) noexcept { *sectionOut = nullptr; + + if (ASMJIT_UNLIKELY(!Support::isZeroOrPowerOf2(alignment))) { + return DebugUtils::errored(kErrorInvalidArgument); + } + if (nameSize == SIZE_MAX) { nameSize = strlen(name); } - if (alignment == 0) { - alignment = 1; - } - - if (ASMJIT_UNLIKELY(!Support::isPowerOf2(alignment))) { - return DebugUtils::errored(kErrorInvalidArgument); - } - if (ASMJIT_UNLIKELY(nameSize > Globals::kMaxSectionNameSize)) { return DebugUtils::errored(kErrorInvalidSectionName); } @@ -405,19 +531,21 @@ Error CodeHolder::newSection(Section** sectionOut, const char* name, size_t name ASMJIT_PROPAGATE(_sections.willGrow(&_allocator)); ASMJIT_PROPAGATE(_sectionsByOrder.willGrow(&_allocator)); - Section* section = _allocator.allocZeroedT
(); + Section* section = _zone.alloc
(); if (ASMJIT_UNLIKELY(!section)) { return DebugUtils::errored(kErrorOutOfMemory); } - section->_id = sectionId; - section->_flags = flags; - section->_alignment = alignment; - section->_order = order; + if (alignment == 0u) { + alignment = 1u; + } + + Section_initData(section, sectionId, flags, alignment, order); + Section_initBuffer(section); memcpy(section->_name.str, name, nameSize); Section** insertPosition = std::lower_bound(_sectionsByOrder.begin(), _sectionsByOrder.end(), section, [](const Section* a, const Section* b) { - return std::make_tuple(a->order(), a->id()) < std::make_tuple(b->order(), b->id()); + return std::make_tuple(a->order(), a->sectionId()) < std::make_tuple(b->order(), b->sectionId()); }); _sections.appendUnsafe(section); @@ -432,9 +560,8 @@ Section* CodeHolder::sectionByName(const char* name, size_t nameSize) const noex nameSize = strlen(name); } - // This could be also put in a hash-table similarly like we do with labels, - // however it's questionable as the number of sections should be pretty low - // in general. Create an issue if this becomes a problem. + // This could be also put in a hash-table similarly like we do with labels, however it's questionable as + // the number of sections should be pretty low in general. Create an issue if this becomes a problem. if (nameSize <= Globals::kMaxSectionNameSize) { for (Section* section : _sections) { if (memcmp(section->_name.str, name, nameSize) == 0 && section->_name.str[nameSize] == '\0') { @@ -503,10 +630,10 @@ public: inline uint32_t hashCode() const noexcept { return _hashCode; } [[nodiscard]] - inline bool matches(const LabelEntry* entry) const noexcept { - return entry->nameSize() == _keySize && - entry->parentId() == _parentId && - ::memcmp(entry->name(), _key, _keySize) == 0; + inline bool matches(const CodeHolder::NamedLabelExtraData* node) const noexcept { + return Support::bool_and(node->extraData._nameSize == _keySize, + node->extraData._parentId == _parentId) && + ::memcmp(node->extraData.name(), _key, _keySize) == 0; } }; @@ -537,64 +664,65 @@ static uint32_t CodeHolder_hashNameAndGetSize(const char* name, size_t& nameSize return hashCode; } -LabelLink* CodeHolder::newLabelLink(LabelEntry* le, uint32_t sectionId, size_t offset, intptr_t rel, const OffsetFormat& format) noexcept { - LabelLink* link = _allocator.allocT(); +Fixup* CodeHolder::newFixup(LabelEntry* le, uint32_t sectionId, size_t offset, intptr_t rel, const OffsetFormat& format) noexcept { + // Cannot be bound if we are creating a link. + ASMJIT_ASSERT(!le->isBound()); + + Fixup* link = _fixupDataPool.alloc(_zone); if (ASMJIT_UNLIKELY(!link)) { return nullptr; } - link->next = le->_links; - le->_links = link; - + link->next = le->_getFixups(); link->sectionId = sectionId; - link->relocId = Globals::kInvalidId; + link->labelOrRelocId = Globals::kInvalidId; link->offset = offset; link->rel = rel; link->format = format; - _unresolvedLinkCount++; + le->_setFixups(link); + _unresolvedFixupCount++; + return link; } -Error CodeHolder::newLabelEntry(LabelEntry** entryOut) noexcept { - *entryOut = nullptr; - +Error CodeHolder::newLabelId(uint32_t* labelIdOut) noexcept { uint32_t labelId = _labelEntries.size(); - if (ASMJIT_UNLIKELY(labelId == Globals::kInvalidId)) { - return DebugUtils::errored(kErrorTooManyLabels); + Error err = _labelEntries.willGrow(&_allocator); + + if (ASMJIT_UNLIKELY(err != kErrorOk)) { + *labelIdOut = Globals::kInvalidId; + return err; } - - ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator)); - LabelEntry* le = _allocator.allocZeroedT(); - - if (ASMJIT_UNLIKELY(!le)) { - return DebugUtils::errored(kErrorOutOfMemory); + else { + *labelIdOut = labelId; + _labelEntries.appendUnsafe(LabelEntry{const_cast(&CodeHolder_sharedLabelExtraData), uint64_t(0)}); + return kErrorOk; } - - le->_setId(labelId); - le->_parentId = Globals::kInvalidId; - le->_offset = 0; - _labelEntries.appendUnsafe(le); - - *entryOut = le; - return kErrorOk; } -Error CodeHolder::newNamedLabelEntry(LabelEntry** entryOut, const char* name, size_t nameSize, LabelType type, uint32_t parentId) noexcept { - *entryOut = nullptr; +Error CodeHolder::newNamedLabelId(uint32_t* labelIdOut, const char* name, size_t nameSize, LabelType type, uint32_t parentId) noexcept { + uint32_t labelId = _labelEntries.size(); uint32_t hashCode = CodeHolder_hashNameAndGetSize(name, nameSize); - if (ASMJIT_UNLIKELY(nameSize == 0)) { - if (type == LabelType::kAnonymous) { - return newLabelEntry(entryOut); - } - else { + *labelIdOut = Globals::kInvalidId; + ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator)); + + if (nameSize == 0) { + if (type != LabelType::kAnonymous) { return DebugUtils::errored(kErrorInvalidLabelName); } + + *labelIdOut = labelId; + _labelEntries.appendUnsafe(LabelEntry{const_cast(&CodeHolder_sharedLabelExtraData), uint64_t(0)}); + return kErrorOk; } - if (ASMJIT_UNLIKELY(nameSize > Globals::kMaxLabelNameSize)) + if (ASMJIT_UNLIKELY(nameSize > Globals::kMaxLabelNameSize)) { return DebugUtils::errored(kErrorLabelNameTooLong); + } + + size_t extraDataSize = sizeof(LabelEntry::ExtraData) + nameSize + 1u; switch (type) { case LabelType::kAnonymous: { @@ -603,27 +731,23 @@ Error CodeHolder::newNamedLabelEntry(LabelEntry** entryOut, const char* name, si return DebugUtils::errored(kErrorInvalidParentLabel); } - uint32_t labelId = _labelEntries.size(); - if (ASMJIT_UNLIKELY(labelId == Globals::kInvalidId)) { - return DebugUtils::errored(kErrorTooManyLabels); - } - - ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator)); - LabelEntry* le = _allocator.allocZeroedT(); - - if (ASMJIT_UNLIKELY(!le)) { + LabelEntry::ExtraData* extraData = _zone.alloc(Support::alignUp(extraDataSize, Globals::kZoneAlignment)); + if (ASMJIT_UNLIKELY(!extraData)) { return DebugUtils::errored(kErrorOutOfMemory); } - // NOTE: This LabelEntry has a name, but we leave its hashCode as zero as it's anonymous. - le->_setId(labelId); - le->_parentId = Globals::kInvalidId; - le->_offset = 0; - ASMJIT_PROPAGATE(le->_name.setData(&_zone, name, nameSize)); + char* namePtr = reinterpret_cast(extraData) + sizeof(LabelEntry::ExtraData); + extraData->_sectionId = Globals::kInvalidId; + extraData->_internalLabelType = type; + extraData->_internalLabelFlags = LabelFlags::kHasOwnExtraData | LabelFlags::kHasName; + extraData->_internalUInt16Data = 0; + extraData->_parentId = Globals::kInvalidId; + extraData->_nameSize = uint32_t(nameSize); + memcpy(namePtr, name, nameSize); + namePtr[nameSize] = '\0'; - _labelEntries.appendUnsafe(le); - - *entryOut = le; + *labelIdOut = labelId; + _labelEntries.appendUnsafe(LabelEntry{extraData, uint64_t(0)}); return kErrorOk; } @@ -649,40 +773,44 @@ Error CodeHolder::newNamedLabelEntry(LabelEntry** entryOut, const char* name, si } } - // Don't allow to insert duplicates. Local labels allow duplicates that have - // different id, this is already accomplished by having a different hashes - // between the same label names having different parent labels. - LabelEntry* le = _namedLabels.get(LabelByName(name, nameSize, hashCode, parentId)); - if (ASMJIT_UNLIKELY(le)) { + extraDataSize += sizeof(ZoneHashNode); + + // Don't allow to insert duplicates. Local labels allow duplicates that have different ids, however, this is + // already accomplished by having a different hashes between the same label names having different parent labels. + NamedLabelExtraData* namedNode = _namedLabels.get(LabelByName(name, nameSize, hashCode, parentId)); + if (ASMJIT_UNLIKELY(namedNode)) { return DebugUtils::errored(kErrorLabelAlreadyDefined); } - Error err = kErrorOk; - uint32_t labelId = _labelEntries.size(); - - if (ASMJIT_UNLIKELY(labelId == Globals::kInvalidId)) { - return DebugUtils::errored(kErrorTooManyLabels); - } - - ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator)); - le = _allocator.allocZeroedT(); - - if (ASMJIT_UNLIKELY(!le)) { + namedNode = _zone.alloc(Support::alignUp(extraDataSize, Globals::kZoneAlignment)); + if (ASMJIT_UNLIKELY(!namedNode)) { return DebugUtils::errored(kErrorOutOfMemory); } - le->_hashCode = hashCode; - le->_setId(labelId); - le->_type = type; - le->_parentId = parentId; - le->_offset = 0; - ASMJIT_PROPAGATE(le->_name.setData(&_zone, name, nameSize)); + LabelFlags labelFlags = + (parentId == Globals::kInvalidId) + ? LabelFlags::kHasOwnExtraData | LabelFlags::kHasName + : LabelFlags::kHasOwnExtraData | LabelFlags::kHasName | LabelFlags::kHasParent; - _labelEntries.appendUnsafe(le); - _namedLabels.insert(allocator(), le); + namedNode->_hashNext = nullptr; + namedNode->_hashCode = hashCode; + namedNode->_customData = labelId; + namedNode->extraData._sectionId = Globals::kInvalidId; + namedNode->extraData._internalLabelType = type; + namedNode->extraData._internalLabelFlags = labelFlags; + namedNode->extraData._internalUInt16Data = 0; + namedNode->extraData._parentId = parentId; + namedNode->extraData._nameSize = uint32_t(nameSize); - *entryOut = le; - return err; + char* namePtr = reinterpret_cast(&namedNode->extraData) + sizeof(LabelEntry::ExtraData); + memcpy(namePtr, name, nameSize); + namePtr[nameSize] = '\0'; + + *labelIdOut = labelId; + _labelEntries.appendUnsafe(LabelEntry{&namedNode->extraData, uint64_t(0)}); + _namedLabels.insert(allocator(), namedNode); + + return kErrorOk; } uint32_t CodeHolder::labelIdByName(const char* name, size_t nameSize, uint32_t parentId) noexcept { @@ -695,125 +823,148 @@ uint32_t CodeHolder::labelIdByName(const char* name, size_t nameSize, uint32_t p hashCode ^= parentId; } - LabelEntry* le = _namedLabels.get(LabelByName(name, nameSize, hashCode, parentId)); - return le ? le->id() : uint32_t(Globals::kInvalidId); + NamedLabelExtraData* namedNode = _namedLabels.get(LabelByName(name, nameSize, hashCode, parentId)); + return namedNode ? namedNode->labelId() : uint32_t(Globals::kInvalidId); } -ASMJIT_API Error CodeHolder::resolveUnresolvedLinks() noexcept { - if (!hasUnresolvedLinks()) { + +ASMJIT_API Error CodeHolder::resolveCrossSectionFixups() noexcept { + if (!hasUnresolvedFixups()) { return kErrorOk; } Error err = kErrorOk; - for (LabelEntry* le : labelEntries()) { - if (!le->isBound()) { - continue; + ResolveFixupIterator it(&_fixups); + + while (it.isValid()) { + Fixup* fixup = it.fixup(); + LabelEntry& le = labelEntry(fixup->labelOrRelocId); + + Support::FastUInt8 of{}; + Section* toSection = _sections[le.sectionId()]; + uint64_t toOffset = Support::addOverflow(toSection->offset(), le.offset(), &of); + + Section* fromSection = sectionById(fixup->sectionId); + size_t fixupOffset = fixup->offset; + + CodeBuffer& buf = fromSection->buffer(); + ASMJIT_ASSERT(fixupOffset < buf.size()); + + // Calculate the offset relative to the start of the virtual base. + uint64_t fromOffset = Support::addOverflow(fromSection->offset(), fixupOffset, &of); + int64_t displacement = int64_t(toOffset - fromOffset + uint64_t(int64_t(fixup->rel))); + + if (ASMJIT_UNLIKELY(of)) { + err = DebugUtils::errored(kErrorInvalidDisplacement); + } + else { + ASMJIT_ASSERT(size_t(fixupOffset) < buf.size()); + ASMJIT_ASSERT(buf.size() - size_t(fixupOffset) >= fixup->format.valueSize()); + + // Overwrite a real displacement in the CodeBuffer. + if (CodeWriterUtils::writeOffset(buf._data + fixupOffset, displacement, fixup->format)) { + it.resolveAndNext(this); + continue; + } } - LabelLinkIterator link(le); - if (link) { - Support::FastUInt8 of = 0; - Section* toSection = le->section(); - uint64_t toOffset = Support::addOverflow(toSection->offset(), le->offset(), &of); - - do { - uint32_t linkSectionId = link->sectionId; - if (link->relocId == Globals::kInvalidId) { - Section* fromSection = sectionById(linkSectionId); - size_t linkOffset = link->offset; - - CodeBuffer& buf = _sections[linkSectionId]->buffer(); - ASMJIT_ASSERT(linkOffset < buf.size()); - - // Calculate the offset relative to the start of the virtual base. - Support::FastUInt8 localOF = of; - uint64_t fromOffset = Support::addOverflow(fromSection->offset(), linkOffset, &localOF); - int64_t displacement = int64_t(toOffset - fromOffset + uint64_t(int64_t(link->rel))); - - if (!localOF) { - ASMJIT_ASSERT(size_t(linkOffset) < buf.size()); - ASMJIT_ASSERT(buf.size() - size_t(linkOffset) >= link->format.valueSize()); - - // Overwrite a real displacement in the CodeBuffer. - if (CodeWriterUtils::writeOffset(buf._data + linkOffset, displacement, link->format)) { - link.resolveAndNext(this); - continue; - } - } - - err = DebugUtils::errored(kErrorInvalidDisplacement); - // Falls through to `link.next()`. - } - - link.next(); - } while (link); - } + it.next(); } + _unresolvedFixupCount -= it.resolvedCount(); return err; } ASMJIT_API Error CodeHolder::bindLabel(const Label& label, uint32_t toSectionId, uint64_t toOffset) noexcept { - LabelEntry* le = labelEntry(label); - if (ASMJIT_UNLIKELY(!le)) { + uint32_t labelId = label.id(); + + if (ASMJIT_UNLIKELY(labelId >= _labelEntries.size())) { return DebugUtils::errored(kErrorInvalidLabel); } - if (ASMJIT_UNLIKELY(toSectionId > _sections.size())) { + if (ASMJIT_UNLIKELY(toSectionId >= _sections.size())) { return DebugUtils::errored(kErrorInvalidSection); } + LabelEntry& le = _labelEntries[labelId]; + // Label can be bound only once. - if (ASMJIT_UNLIKELY(le->isBound())) { + if (ASMJIT_UNLIKELY(le.isBound())) { return DebugUtils::errored(kErrorLabelAlreadyBound); } - // Bind the label. Section* section = _sections[toSectionId]; - le->_section = section; - le->_offset = toOffset; - - Error err = kErrorOk; CodeBuffer& buf = section->buffer(); - // Fix all links to this label we have collected so far if they are within - // the same section. We ignore any inter-section links as these have to be - // fixed later. - LabelLinkIterator link(le); - while (link) { - uint32_t linkSectionId = link->sectionId; - size_t linkOffset = link->offset; + // Bind the label - this either assigns a section to LabelEntry's `_objectData` or `_sectionId` in own `ExtraData`. + // This is basically how this works - when the ExtraData is shared, we replace it by section as the section header + // is compatible with ExtraData header, and when the LabelEntry has its own ExtraData, the section identifier must + // be assigned. + if (le._hasOwnExtraData()) { + le._ownExtraData()->_sectionId = toSectionId; + } + else { + le._objectData = section; + } + + // It must be in this order as _offsetsOrFixups as basically a union. + Fixup* labelFixups = le._getFixups(); + le._offsetOrFixups = toOffset; + + if (!labelFixups) { + return kErrorOk; + } + + // Fix all fixups of this label we have collected so far if they are within the same + // section. We ignore any cross-section fixups as these have to be fixed later. + Error err = kErrorOk; + + ResolveFixupIterator it(&labelFixups); + ASMJIT_ASSERT(it.isValid()); + + do { + Fixup* fixup = it.fixup(); + + uint32_t relocId = fixup->labelOrRelocId; + uint32_t fromSectionId = fixup->sectionId; + size_t fromOffset = fixup->offset; - uint32_t relocId = link->relocId; if (relocId != Globals::kInvalidId) { - // Adjust relocation data only. + // Adjust the relocation payload. RelocEntry* re = _relocations[relocId]; re->_payload += toOffset; re->_targetSectionId = toSectionId; } + else if (fromSectionId != toSectionId) { + fixup->labelOrRelocId = labelId; + it.next(); + continue; + } else { - if (linkSectionId != toSectionId) { - link.next(); - continue; - } + ASMJIT_ASSERT(fromOffset < buf.size()); + int64_t displacement = int64_t(toOffset - uint64_t(fromOffset) + uint64_t(int64_t(fixup->rel))); - ASMJIT_ASSERT(linkOffset < buf.size()); - int64_t displacement = int64_t(toOffset - uint64_t(linkOffset) + uint64_t(int64_t(link->rel))); - - // Size of the value we are going to patch. Only BYTE/DWORD is allowed. - ASMJIT_ASSERT(buf.size() - size_t(linkOffset) >= link->format.regionSize()); + // Size of the value we are going to patch. + ASMJIT_ASSERT(buf.size() - size_t(fromOffset) >= fixup->format.regionSize()); // Overwrite a real displacement in the CodeBuffer. - if (!CodeWriterUtils::writeOffset(buf._data + linkOffset, displacement, link->format)) { + if (!CodeWriterUtils::writeOffset(buf._data + fromOffset, displacement, fixup->format)) { err = DebugUtils::errored(kErrorInvalidDisplacement); - link.next(); + fixup->labelOrRelocId = labelId; + it.next(); continue; } } - link.resolveAndNext(this); + it.resolveAndNext(this); + } while (it.isValid()); + + if (it.unresolvedCount()) { + *it._pPrev = _fixups; + _fixups = labelFixups; } + _unresolvedFixupCount -= it.resolvedCount(); return err; } @@ -828,15 +979,18 @@ Error CodeHolder::newRelocEntry(RelocEntry** dst, RelocType relocType) noexcept return DebugUtils::errored(kErrorTooManyRelocations); } - RelocEntry* re = _allocator.allocZeroedT(); + RelocEntry* re = _zone.alloc(); if (ASMJIT_UNLIKELY(!re)) { return DebugUtils::errored(kErrorOutOfMemory); } re->_id = relocId; re->_relocType = relocType; + re->_format = OffsetFormat{}; re->_sourceSectionId = Globals::kInvalidId; re->_targetSectionId = Globals::kInvalidId; + re->_sourceOffset = 0; + re->_payload = 0; _relocations.appendUnsafe(re); *dst = re; @@ -862,11 +1016,17 @@ static Error CodeHolder_evaluateExpression(CodeHolder* self, Expression* exp, ui } case ExpressionValueType::kLabel: { - LabelEntry* le = exp->value[i].label; - if (!le->isBound()) { + uint32_t labelId = exp->value[i].labelId; + if (ASMJIT_UNLIKELY(labelId >= self->labelCount())) { + return DebugUtils::errored(kErrorInvalidLabel); + } + + LabelEntry& le = self->_labelEntries[labelId]; + if (!le.isBound()) { return DebugUtils::errored(kErrorExpressionLabelNotBound); } - v = le->section()->offset() + le->offset(); + + v = self->_sections[le.sectionId()]->offset() + le.offset(); break; } @@ -985,7 +1145,16 @@ size_t CodeHolder::codeSize() const noexcept { return size_t(offset); } -Error CodeHolder::relocateToBase(uint64_t baseAddress) noexcept { +Error CodeHolder::relocateToBase(uint64_t baseAddress, RelocationSummary* summaryOut) noexcept { + // Make sure `summaryOut` pointer is always valid as we want to fill it. + RelocationSummary summaryTmp; + if (summaryOut == nullptr) { + summaryOut = &summaryTmp; + } + + // Fill `summaryOut` defaults. + summaryOut->codeSizeReduction = 0u; + // Base address must be provided. if (ASMJIT_UNLIKELY(baseAddress == Globals::kNoBaseAddress)) { return DebugUtils::errored(kErrorInvalidArgument); @@ -1118,7 +1287,7 @@ Error CodeHolder::relocateToBase(uint64_t baseAddress) noexcept { buffer[valueOffset - 2] = uint8_t(byte0); buffer[valueOffset - 1] = uint8_t(byte1); - Support::writeU64uLE(addressTableEntryData + atEntryIndex, re->payload()); + Support::storeu_u64_le(addressTableEntryData + atEntryIndex, re->payload()); } break; } @@ -1136,9 +1305,16 @@ Error CodeHolder::relocateToBase(uint64_t baseAddress) noexcept { if (_sectionsByOrder.last() == addressTableSection) { ASMJIT_ASSERT(addressTableSection != nullptr); + size_t reservedSize = size_t(addressTableSection->_virtualSize); size_t addressTableSize = addressTableEntryCount * addressSize; + addressTableSection->_buffer._size = addressTableSize; addressTableSection->_virtualSize = addressTableSize; + + ASMJIT_ASSERT(reservedSize >= addressTableSize); + size_t codeSizeReduction = reservedSize - addressTableSize; + + summaryOut->codeSizeReduction = codeSizeReduction; } return kErrorOk; @@ -1214,10 +1390,30 @@ UNIT(code_holder) { EXPECT_EQ(code.arch(), Arch::kX86); INFO("Verifying named labels"); - LabelEntry* le; - EXPECT_EQ(code.newNamedLabelEntry(&le, "NamedLabel", SIZE_MAX, LabelType::kGlobal), kErrorOk); - EXPECT_EQ(strcmp(le->name(), "NamedLabel"), 0); - EXPECT_EQ(code.labelIdByName("NamedLabel"), le->id()); + uint32_t dummyId; + uint32_t labelId1; + uint32_t labelId2; + + // Anonymous labels can have no-name (this is basically like calling `code.newLabelId()`). + EXPECT_EQ(code.newNamedLabelId(&dummyId, "", SIZE_MAX, LabelType::kAnonymous), kErrorOk); + + // Global labels must have a name - not providing one is an error. + EXPECT_EQ(code.newNamedLabelId(&dummyId, "", SIZE_MAX, LabelType::kGlobal), kErrorInvalidLabelName); + + // A name of a global label cannot repeat. + EXPECT_EQ(code.newNamedLabelId(&labelId1, "NamedLabel1", SIZE_MAX, LabelType::kGlobal), kErrorOk); + EXPECT_EQ(code.newNamedLabelId(&dummyId, "NamedLabel1", SIZE_MAX, LabelType::kGlobal), kErrorLabelAlreadyDefined); + EXPECT_TRUE(code.isLabelValid(labelId1)); + EXPECT_EQ(code.labelEntry(labelId1).nameSize(), 11u); + EXPECT_EQ(strcmp(code.labelEntry(labelId1).name(), "NamedLabel1"), 0); + EXPECT_EQ(code.labelIdByName("NamedLabel1"), labelId1); + + EXPECT_EQ(code.newNamedLabelId(&labelId2, "NamedLabel2", SIZE_MAX, LabelType::kGlobal), kErrorOk); + EXPECT_EQ(code.newNamedLabelId(&dummyId, "NamedLabel2", SIZE_MAX, LabelType::kGlobal), kErrorLabelAlreadyDefined); + EXPECT_TRUE(code.isLabelValid(labelId2)); + EXPECT_EQ(code.labelEntry(labelId2).nameSize(), 11u); + EXPECT_EQ(strcmp(code.labelEntry(labelId2).name(), "NamedLabel2"), 0); + EXPECT_EQ(code.labelIdByName("NamedLabel2"), labelId2); INFO("Verifying section ordering"); Section* section1; diff --git a/src/asmjit/core/codeholder.h b/src/asmjit/core/codeholder.h index 33247d7..28cbb1f 100644 --- a/src/asmjit/core/codeholder.h +++ b/src/asmjit/core/codeholder.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_CODEHOLDER_H_INCLUDED @@ -9,6 +9,7 @@ #include "../core/archtraits.h" #include "../core/codebuffer.h" #include "../core/errorhandler.h" +#include "../core/fixup.h" #include "../core/operand.h" #include "../core/string.h" #include "../core/support.h" @@ -65,8 +66,8 @@ struct Expression { uint64_t constant; //! Pointer to another expression. Expression* expression; - //! Pointer to \ref LabelEntry. - LabelEntry* label; + //! Label identifier + uint32_t labelId; }; //! \name Members @@ -98,9 +99,9 @@ struct Expression { } //! Sets the value type at `index` to \ref ExpressionValueType::kLabel and its content to `labelEntry`. - ASMJIT_INLINE_NODEBUG void setValueAsLabel(size_t index, LabelEntry* labelEntry) noexcept { + ASMJIT_INLINE_NODEBUG void setValueAsLabelId(size_t index, uint32_t labelId) noexcept { valueType[index] = ExpressionValueType::kLabel; - value[index].label = labelEntry; + value[index].labelId = labelId; } //! Sets the value type at `index` to \ref ExpressionValueType::kExpression and its content to `expression`. @@ -112,20 +113,64 @@ struct Expression { //! \} }; +//! Relocation type. +enum class RelocType : uint32_t { + //! None/deleted (no relocation). + kNone = 0, + //! Expression evaluation, `_payload` is pointer to `Expression`. + kExpression = 1, + //! Relative relocation from one section to another. + kSectionRelative = 2, + //! Relocate absolute to absolute. + kAbsToAbs = 3, + //! Relocate relative to absolute. + kRelToAbs = 4, + //! Relocate absolute to relative. + kAbsToRel = 5, + //! Relocate absolute to relative or use trampoline. + kX64AddressEntry = 6 +}; + +//! Type of the \ref Label. +enum class LabelType : uint8_t { + //! Anonymous label that can optionally have a name, which is only used for debugging purposes. + kAnonymous = 0u, + //! Local label (always has parentId). + kLocal = 1u, + //! Global label (never has parentId). + kGlobal = 2u, + //! External label (references an external symbol). + kExternal = 3u, + + //! Maximum value of `LabelType`. + kMaxValue = kExternal +}; + +//! Label flags describe some details about labels, mostly for AsmJit's own use. +enum class LabelFlags : uint8_t { + kNone = 0x00u, + kHasOwnExtraData = 0x01u, + kHasName = 0x02u, + kHasParent = 0x04u +}; +ASMJIT_DEFINE_ENUM_FLAGS(LabelFlags) + //! Section flags, used by \ref Section. enum class SectionFlags : uint32_t { //! No flags. kNone = 0, //! Executable (.text sections). - kExecutable = 0x00000001u, + kExecutable = 0x0001u, //! Read-only (.text and .data sections). - kReadOnly = 0x00000002u, + kReadOnly = 0x0002u, //! Zero initialized by the loader (BSS). - kZeroInitialized = 0x00000004u, + kZeroInitialized = 0x0004u, //! Info / comment flag. - kComment = 0x00000008u, + kComment = 0x0008u, + //! Section is built in and created by default (.text section). + kBuiltIn = 0x4000u, //! Section created implicitly, can be deleted by \ref Target. - kImplicit = 0x80000000u + kImplicit = 0x8000u }; ASMJIT_DEFINE_ENUM_FLAGS(SectionFlags) @@ -146,16 +191,36 @@ enum class CopySectionFlags : uint32_t { }; ASMJIT_DEFINE_ENUM_FLAGS(CopySectionFlags) -//! Section entry. -class Section { +//! Base class for both \ref Section and \ref LabelEntry::ExtraData. +class SectionOrLabelEntryExtraHeader { +public: + //! \name Members + //! \{ + + //! Section id - describes either a section where a \ref Label is bound or it's a real section id of \ref Section. + uint32_t _sectionId; + + //! Internal label type is only used by \ref LabelEntry::ExtraData. \ref Section always leaves this field zero, + //! which describes an anonymous label. Anonymous labels are default and always used when there is no + //! \ref LabelEntry::ExtraData + LabelType _internalLabelType; + + //! Internal label flags, used by \ref LabelEntry::ExtraData. \ref Section doesn't use these flags and sets them + //! to zero. + LabelFlags _internalLabelFlags; + + //! Internal data used freely by \ref Section and \ref LabelEntry::ExtraData. + uint16_t _internalUInt16Data; + + //! \} +}; + +//! Section entry. +class Section : public SectionOrLabelEntryExtraHeader { public: //! \name Members //! \{ - //! Section id. - uint32_t _id; - //! Section flags. - SectionFlags _flags; //! Section alignment requirements (0 if no requirements). uint32_t _alignment; //! Order (lower value means higher priority). @@ -176,7 +241,7 @@ public: //! Returns the section id. [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint32_t id() const noexcept { return _id; } + ASMJIT_INLINE_NODEBUG uint32_t sectionId() const noexcept { return _sectionId; } //! Returns the section name, as a null terminated string. [[nodiscard]] @@ -192,17 +257,20 @@ public: //! Returns the section flags. [[nodiscard]] - ASMJIT_INLINE_NODEBUG SectionFlags flags() const noexcept { return _flags; } + ASMJIT_INLINE_NODEBUG SectionFlags flags() const noexcept { return SectionFlags(_internalUInt16Data); } //! Tests whether the section has the given `flag`. [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool hasFlag(SectionFlags flag) const noexcept { return Support::test(_flags, flag); } + ASMJIT_INLINE_NODEBUG bool hasFlag(SectionFlags flag) const noexcept { return Support::test(_internalUInt16Data, uint32_t(flag)); } + + //! Assigns `flags` to the section (replaces all existing flags). + ASMJIT_INLINE_NODEBUG void assignFlags(SectionFlags flags) noexcept { _internalUInt16Data = uint16_t(flags); } //! Adds `flags` to the section flags. - ASMJIT_INLINE_NODEBUG void addFlags(SectionFlags flags) noexcept { _flags |= flags; } + ASMJIT_INLINE_NODEBUG void addFlags(SectionFlags flags) noexcept { _internalUInt16Data = uint16_t(_internalUInt16Data | uint32_t(flags)); } //! Removes `flags` from the section flags. - ASMJIT_INLINE_NODEBUG void clearFlags(SectionFlags flags) noexcept { _flags &= ~flags; } + ASMJIT_INLINE_NODEBUG void clearFlags(SectionFlags flags) noexcept { _internalUInt16Data = uint16_t(_internalUInt16Data | ~uint32_t(flags)); } //! Returns the minimum section alignment [[nodiscard]] @@ -304,260 +372,6 @@ public: //! \} }; -//! Offset format type, used by \ref OffsetFormat. -enum class OffsetType : uint8_t { - // Common Offset Formats - // --------------------- - - //! A value having `_immBitCount` bits and shifted by `_immBitShift`. - //! - //! This offset type is sufficient for many targets that store offset as a continuous set bits within an - //! instruction word / sequence of bytes. - kSignedOffset, - - //! An unsigned value having `_immBitCount` bits and shifted by `_immBitShift`. - kUnsignedOffset, - - // AArch64 Specific Offset Formats - // ------------------------------- - - //! AArch64 ADR format of `[.|immlo:2|.....|immhi:19|.....]`. - kAArch64_ADR, - - //! AArch64 ADRP format of `[.|immlo:2|.....|immhi:19|.....]` (4kB pages). - kAArch64_ADRP, - - // AArch32 Specific Offset Formats (T16 & T32) - // ------------------------------------------- - - //! AArch32 THUMBv2 immediate encoding of 'ADR' instruction (12-bit payload and sign bit): - //! - //! `|.....|imm:1|..N.N|......|imm:3|....|imm:8|` - //! - //! Where `N` is one if the offset is negative. The immediate is encoded as absolute value of the offset if negative. - kThumb32_ADR, - - //! AArch32 THUMBv2 immediate encoding of 'BLX' instruction (23-bit immediate payload, multiplied by 4): - //! - //! `|.....|imm[22]|imm[19:10]|..|ja|1|jb|imm[9:0]|0` - //! - //! Where: - //! - //! - `ja` is calculated as imm[22] ^ imm[21] ^ 1. - //! - `jb` is calculated as imm[22] ^ imm[20] ^ 1. - kThumb32_BLX, - - //! AArch32 THUMBv2 immediate encoding of 'B' instruction without `` (24-bit immediate payload, multiplied by 2): - //! - //! `|.....|imm[23]|imm[20:11]|..|ja|1|jb|imm[10:0]` - //! - //! Where: - //! - //! - `ja` is calculated as imm[23] ^ imm[22] ^ 1. - //! - `jb` is calculated as imm[23] ^ imm[21] ^ 1. - kThumb32_B, - - //! AArch32 THUMBv2 immediate encoding of 'B' instruction with `` (20-bit immediate payload, multiplied by 2). - //! - //! `|.....|imm[19]|....|imm[16:11]|..|ja|1|jb|imm[10:0]` - //! - //! Where: - //! - //! - `ja` is calculated as imm[19] ^ imm[18] ^ 1. - //! - `jb` is calculated as imm[19] ^ imm[17] ^ 1. - kThumb32_BCond, - - // AArch32 Specific Offset Formats (A32) - // ------------------------------------- - - //! AArch32 ADR instruction, which uses a standard 12-bit immediate encoding that is used by other ARM instructions. - kAArch32_ADR, - - //! AArch32 signed offset that is similar to `kSignedOffset`, however it uses absolute value of the offset and its - //! sign is encoded in 23rd bit of the opcode. - //! - //! `|........|U.......|........|........|` - //! - kAArch32_U23_SignedOffset, - - //! AArch32 offset format that encodes 8-bit offset as: - //! - //! `|........|U.......|....|imm[7:4]|....|imm[3:0]|` - //! - //! in a 32-bit word, where U is a sign of the displacement and the displacement itself is encoded as its absolute - //! value. - kAArch32_U23_0To3At0_4To7At8, - - //! AArch32 offset format that encodes a signed 25-bit offset as: - //! - //! `|.......|imm[0]|imm[24:1]|` - //! - //! in a 32-bit word. - kAArch32_1To24At0_0At24, - - //! Maximum value of `OffsetFormatType`. - kMaxValue = kAArch32_1To24At0_0At24 -}; - -//! Provides information about formatting offsets, absolute addresses, or their parts. Offset format is used by both -//! \ref RelocEntry and \ref LabelLink. The illustration below describes the relation of region size and offset size. -//! Region size is the size of the whole unit whereas offset size is the size of the unit that will be patched. -//! -//! ``` -//! +-> Code buffer | The subject of the relocation (region) | -//! | | (Word-Offset) (Word-Size) | -//! |xxxxxxxxxxxxxxx|................|*PATCHED*|................|xxxxxxxxxxxx-> -//! | | -//! [Word Offset points here]----+ +--- [WordOffset + WordSize] -//! ``` -//! -//! Once the offset word has been located it can be patched like this: -//! -//! ``` -//! |ImmDiscardLSB (discard LSB bits). -//! |.. -//! [0000000000000iiiiiiiiiiiiiiiiiDD] - Offset value (32-bit) -//! [000000000000000iiiiiiiiiiiiiiiii] - Offset value after discard LSB. -//! [00000000000iiiiiiiiiiiiiiiii0000] - Offset value shifted by ImmBitShift. -//! [xxxxxxxxxxxiiiiiiiiiiiiiiiiixxxx] - Patched word (32-bit) -//! |...............| -//! (ImmBitCount) +- ImmBitShift -//! ``` -struct OffsetFormat { - //! \name Members - //! \{ - - //! Type of the offset. - OffsetType _type; - //! Encoding flags. - uint8_t _flags; - //! Size of the region (in bytes) containing the offset value, if the offset value is part of an instruction, - //! otherwise it would be the same as `_valueSize`. - uint8_t _regionSize; - //! Size of the offset value, in bytes (1, 2, 4, or 8). - uint8_t _valueSize; - //! Offset of the offset value, in bytes, relative to the start of the region or data. Value offset would be - //! zero if both region size and value size are equal. - uint8_t _valueOffset; - //! Size of the offset immediate value in bits. - uint8_t _immBitCount; - //! Shift of the offset immediate value in bits in the target word. - uint8_t _immBitShift; - //! Number of least significant bits to discard before writing the immediate to the destination. All discarded - //! bits must be zero otherwise the value is invalid. - uint8_t _immDiscardLsb; - - //! \} - - //! \name Accessors - //! \{ - - //! Returns the type of the offset. - ASMJIT_INLINE_NODEBUG OffsetType type() const noexcept { return _type; } - - //! Returns whether the offset is encoded as an absolute value of the offset with additional field(s) that represent - //! the sign (AArch32 U/N fields in the opcode). - //! - //! If true, the offset itself is always positive and a separate U/N field is used to indicate the sign of the offset - //! (usually `U==1` means ADD, but sometimes `N==1` means negative offset, which implies SUB). - ASMJIT_INLINE_NODEBUG bool hasSignBit() const noexcept { - return _type == OffsetType::kThumb32_ADR || - _type == OffsetType::kAArch32_ADR || - _type == OffsetType::kAArch32_U23_SignedOffset || - _type == OffsetType::kAArch32_U23_0To3At0_4To7At8; - } - - //! Returns flags. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint32_t flags() const noexcept { return _flags; } - - //! Returns the size of the region/instruction where the offset is encoded. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint32_t regionSize() const noexcept { return _regionSize; } - - //! Returns the offset of the word relative to the start of the region where the offset is. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint32_t valueOffset() const noexcept { return _valueOffset; } - - //! Returns the size of the data-type (word) that contains the offset, in bytes. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint32_t valueSize() const noexcept { return _valueSize; } - - //! Returns the count of bits of the offset value in the data it's stored in. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint32_t immBitCount() const noexcept { return _immBitCount; } - - //! Returns the bit-shift of the offset value in the data it's stored in. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint32_t immBitShift() const noexcept { return _immBitShift; } - - //! Returns the number of least significant bits of the offset value, that must be zero and that are not part of - //! the encoded data. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint32_t immDiscardLsb() const noexcept { return _immDiscardLsb; } - - //! Resets this offset format to a simple data value of `dataSize` bytes. - //! - //! The region will be the same size as data and immediate bits would correspond to `dataSize * 8`. There will be - //! no immediate bit shift or discarded bits. - inline void resetToSimpleValue(OffsetType type, size_t valueSize) noexcept { - ASMJIT_ASSERT(valueSize <= 8u); - - _type = type; - _flags = uint8_t(0); - _regionSize = uint8_t(valueSize); - _valueSize = uint8_t(valueSize); - _valueOffset = uint8_t(0); - _immBitCount = uint8_t(valueSize * 8u); - _immBitShift = uint8_t(0); - _immDiscardLsb = uint8_t(0); - } - - inline void resetToImmValue(OffsetType type, size_t valueSize, uint32_t immBitShift, uint32_t immBitCount, uint32_t immDiscardLsb) noexcept { - ASMJIT_ASSERT(valueSize <= 8u); - ASMJIT_ASSERT(immBitShift < valueSize * 8u); - ASMJIT_ASSERT(immBitCount <= 64u); - ASMJIT_ASSERT(immDiscardLsb <= 64u); - - _type = type; - _flags = uint8_t(0); - _regionSize = uint8_t(valueSize); - _valueSize = uint8_t(valueSize); - _valueOffset = uint8_t(0); - _immBitCount = uint8_t(immBitCount); - _immBitShift = uint8_t(immBitShift); - _immDiscardLsb = uint8_t(immDiscardLsb); - } - - inline void setRegion(size_t regionSize, size_t valueOffset) noexcept { - _regionSize = uint8_t(regionSize); - _valueOffset = uint8_t(valueOffset); - } - - inline void setLeadingAndTrailingSize(size_t leadingSize, size_t trailingSize) noexcept { - _regionSize = uint8_t(leadingSize + trailingSize + _valueSize); - _valueOffset = uint8_t(leadingSize); - } - - //! \} -}; - -//! Relocation type. -enum class RelocType : uint32_t { - //! None/deleted (no relocation). - kNone = 0, - //! Expression evaluation, `_payload` is pointer to `Expression`. - kExpression = 1, - //! Relocate absolute to absolute. - kAbsToAbs = 2, - //! Relocate relative to absolute. - kRelToAbs = 3, - //! Relocate absolute to relative. - kAbsToRel = 4, - //! Relocate absolute to relative or use trampoline. - kX64AddressEntry = 5 -}; - //! Relocation entry. struct RelocEntry { //! \name Members @@ -612,156 +426,158 @@ struct RelocEntry { //! \} }; -//! Type of the \ref Label. -enum class LabelType : uint8_t { - //! Anonymous label that can optionally have a name, which is only used for debugging purposes. - kAnonymous = 0, - //! Local label (always has parentId). - kLocal = 1, - //! Global label (never has parentId). - kGlobal = 2, - //! External label (references an external symbol). - kExternal = 3, - - //! Maximum value of `LabelType`. - kMaxValue = kExternal -}; - -//! Data structure used to link either unbound labels or cross-section links. -struct LabelLink { - //! Next link (single-linked list). - LabelLink* next; - //! Section id where the label is bound. - uint32_t sectionId; - //! Relocation id or Globals::kInvalidId. - uint32_t relocId; - //! Label offset relative to the start of the section. - size_t offset; - //! Inlined rel8/rel32. - intptr_t rel; - //! Offset format information. - OffsetFormat format; -}; - -//! Label entry. +//! Label entry provides data stored by \ref CodeHolder for each \ref Label. //! -//! Contains the following properties: -//! - Label id - This is the only thing that is set to the `Label` operand. -//! - Label name - Optional, used mostly to create executables and libraries. -//! - Label type - Type of the label, default `LabelType::kAnonymous`. -//! - Label parent id - Derived from many assemblers that allow to define a local label that falls under a global -//! label. This allows to define many labels of the same name that have different parent (global) label. -//! - Offset - offset of the label bound by `Assembler`. -//! - Links - single-linked list that contains locations of code that has to be patched when the label gets bound. -//! Every use of unbound label adds one link to `_links` list. -//! - HVal - Hash value of label's name and optionally parentId. -//! - HashNext - Hash-table implementation detail. -class LabelEntry : public ZoneHashNode { +//! Label entry is used mostly internall by AsmJit, but it's possibly to use it to query various information about +//! a label. For example to get its type, flags, name, and fixups (if the label is not bound) or offset (if the label +//! is bound). +//! +//! To make the entry small, it's currently split into two data structures - \ref LabelEntry, which is stored in an +//! array as a value, and \ref LabelEntry::ExtraData, which can be pointed to via \ref LabelEntry::_objectData. Extra +//! data of unnamed anonymous labels is shared (and immutable), thus all unnamed anonymous labels would only use +//! \ref LabelEntry (16 bytes per label). +class LabelEntry { public: - //! \name Constants - //! \{ + //! Contains extra data that is only created when the label is not anonymous or has a name. + struct ExtraData : public SectionOrLabelEntryExtraHeader { + //! Label parent id or zero. + uint32_t _parentId; + //! Label name length. + uint32_t _nameSize; - //! SSO size of \ref _name. - //! - //! \cond INTERNAL - //! Let's round the size of `LabelEntry` to 64 bytes (as `ZoneAllocator` has granularity of 32 bytes anyway). This - //! gives `_name` the remaining space, which is should be 16 bytes on 64-bit and 28 bytes on 32-bit architectures. - //! \endcond - static inline constexpr uint32_t kStaticNameSize = - 64 - (sizeof(ZoneHashNode) + 8 + sizeof(Section*) + sizeof(size_t) + sizeof(LabelLink*)); - - //! \} + //! Returns a name associated with this extra data - a valid pointer is only returned when the label has a name, which + //! is marked by \ref LabelFlags::kHasName flag. + ASMJIT_INLINE_NODEBUG const char* name() const noexcept { return Support::offsetPtr(this, sizeof(ExtraData)); } + }; //! \name Members //! \{ - //! Type of the label. - LabelType _type; - //! Must be zero. - uint8_t _reserved[3]; - //! Label parent id or zero. - uint32_t _parentId; - //! Label offset relative to the start of the `_section`. - uint64_t _offset; - //! Section where the label was bound. - Section* _section; - //! Label links. - LabelLink* _links; - //! Label name. - ZoneString _name; + // Either references a \ref Section where the label is bound or \ref ExtraData. + SectionOrLabelEntryExtraHeader* _objectData; + + //! Label entry payload. + //! + //! When a Label is bound, `_offsetOrFixups` is the relative offset from the start of the section where + //! the \ref Label has been bound, otherwise `_offsetOrFixups` is a pointer to the first \ref Fixup. + uint64_t _offsetOrFixups; //! \} //! \name Accessors //! \{ - // NOTE: Label id is stored in `_customData`, which is provided by ZoneHashNode to fill a padding that a C++ - // compiler targeting 64-bit CPU will add to align the structure to 64-bits. - - //! Returns label id. + //! Returns the type of the label. + //! + //! The type of the label depends on how it was created. Most JIT code uses unnamed anonymous labels created by + //! emitters, for example \ref BaseEmitter::newLabel() returns a \ref Label instance having id that was created + //! by \ref CodeHolder::newLabelId. [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint32_t id() const noexcept { return _customData; } + ASMJIT_INLINE_NODEBUG LabelType labelType() const noexcept { return _objectData->_internalLabelType; } - //! Sets label id (internal, used only by `CodeHolder`). - ASMJIT_INLINE_NODEBUG void _setId(uint32_t id) noexcept { _customData = id; } - - //! Returns label type. + //! Returns label flags. + //! + //! \note Label flags are mostly for internal use, there is probably no reason to use them in user code. [[nodiscard]] - ASMJIT_INLINE_NODEBUG LabelType type() const noexcept { return _type; } + ASMJIT_INLINE_NODEBUG LabelFlags labelFlags() const noexcept { return _objectData->_internalLabelFlags; } - //! Tests whether the label has a parent label. + //! Tests whether the label has the given `flag` set. + //! + //! \note Using other getters instead is advised, for example using \ref hasName() and \ref hasParent() is better + //! (and shorter) than checking label flags. [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool hasParent() const noexcept { return _parentId != Globals::kInvalidId; } + ASMJIT_INLINE_NODEBUG bool hasLabelFlag(LabelFlags flag) const noexcept { return Support::test(_objectData->_internalLabelFlags, flag); } - //! Returns label's parent id. + //! Tests whether the LabelEntry has own extra data (see \ref LabelEntry::ExtraData). + //! + //! \note This should only be used by AsmJit for internal purposes. Own extra data means that the LabelEntry has + //! a mutable extra data separately allocated. This information should not be necessary to users as LabelEntry + //! getters should encapsulate label introspection. [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint32_t parentId() const noexcept { return _parentId; } + ASMJIT_INLINE_NODEBUG bool _hasOwnExtraData() const noexcept { return hasLabelFlag(LabelFlags::kHasOwnExtraData); } + + //! Tests whether the Label represented by this LabelEntry has a name. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasName() const noexcept { return hasLabelFlag(LabelFlags::kHasName); } + + //! Tests whether the Label represented by this LabelEntry has a parent label. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasParent() const noexcept { return hasLabelFlag(LabelFlags::kHasParent); } + + //! Tests whether the label represented by this LabelEntry is bound. + //! + //! Bound label means that it has an associated \ref Section and a position in such section. Labels are bound by + //! calling \ref BaseEmitter::bind() method with \ref Label operand. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool isBound() const noexcept { return _objectData->_sectionId != Globals::kInvalidId; } + + //! Tests whether the label is bound to a the given `section`. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool isBoundTo(const Section* section) const noexcept { return _objectData->_sectionId == section->sectionId(); } + + //! Tests whether the label is bound to a the given `sectionId`. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool isBoundTo(uint32_t sectionId) const noexcept { return _objectData->_sectionId == sectionId; } //! Returns the section where the label was bound. //! //! If the label was not yet bound the return value is `nullptr`. [[nodiscard]] - ASMJIT_INLINE_NODEBUG Section* section() const noexcept { return _section; } + ASMJIT_INLINE_NODEBUG uint32_t sectionId() const noexcept { return _objectData->_sectionId; } - //! Tests whether the label has name. [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool hasName() const noexcept { return !_name.empty(); } + ASMJIT_INLINE ExtraData* _ownExtraData() const noexcept { + ASMJIT_ASSERT(_hasOwnExtraData()); + return static_cast(_objectData); + } + + //! Returns label's parent id or \ref Globals::kInvalidId if the label has no parent. + [[nodiscard]] + ASMJIT_INLINE uint32_t parentId() const noexcept { + return _hasOwnExtraData() ? _ownExtraData()->_parentId : Globals::kInvalidId; + } //! Returns the label's name. //! //! \note Local labels will return their local name without their parent part, for example ".L1". [[nodiscard]] - ASMJIT_INLINE_NODEBUG const char* name() const noexcept { return _name.data(); } + ASMJIT_INLINE_NODEBUG const char* name() const noexcept { + return hasName() ? _ownExtraData()->name() : nullptr; + } //! Returns size of label's name. //! //! \note Label name is always null terminated, so you can use `strlen()` to get it, however, it's also cached in //! `LabelEntry` itself, so if you want to know the size the fastest way is to call `LabelEntry::nameSize()`. [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint32_t nameSize() const noexcept { return _name.size(); } + ASMJIT_INLINE_NODEBUG uint32_t nameSize() const noexcept { + return hasName() ? _ownExtraData()->_nameSize : uint32_t(0); + } - //! Returns links associated with this label. + //! Returns unresolved fixups associated with this label. [[nodiscard]] - ASMJIT_INLINE_NODEBUG LabelLink* links() const noexcept { return _links; } + ASMJIT_INLINE_NODEBUG bool hasFixups() const noexcept { + return Support::bool_and(!isBound(), _offsetOrFixups != 0u); + } - //! Tests whether the label is bound. [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool isBound() const noexcept { return _section != nullptr; } + ASMJIT_INLINE_NODEBUG Fixup* _getFixups() const noexcept { return reinterpret_cast(uintptr_t(_offsetOrFixups)); } - //! Tests whether the label is bound to a the given `sectionId`. + ASMJIT_INLINE_NODEBUG void _setFixups(Fixup* first) noexcept { _offsetOrFixups = reinterpret_cast(first); } + + //! Returns unresolved fixups associated with this label. [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool isBoundTo(Section* section) const noexcept { return _section == section; } + ASMJIT_INLINE_NODEBUG Fixup* unresolvedLinks() const noexcept { return !isBound() ? _getFixups() : nullptr; } - //! Returns the label offset (only useful if the label is bound). - [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint64_t offset() const noexcept { return _offset; } - - //! Returns the hash-value of label's name and its parent label (if any). + //! Returns the label offset (can only be used after the label is bound). //! - //! Label hash is calculated as `HASH(Name) ^ ParentId`. The hash function is implemented in `Support::hashString()` - //! and `Support::hashRound()`. + //! \note This would trigger an assertion failure in debug builds when called on an unbound label. When accessing + //! offsets, always check whether the label is bound. Unbound labels don't have offsets. [[nodiscard]] - ASMJIT_INLINE_NODEBUG uint32_t hashCode() const noexcept { return _hashCode; } + ASMJIT_INLINE uint64_t offset() const noexcept { + ASMJIT_ASSERT(isBound()); + return _offsetOrFixups; + } //! \} }; @@ -782,12 +598,84 @@ public: //! emitters. After the end of code generation it can be used to query physical locations of labels and to relocate //! the assembled code into the right address. //! +//! CodeHolder Reusability +//! ---------------------- +//! +//! If you intend to generate a lot of code, or tiny code, it's advised to reuse CodeHolder and emitter instances. +//! There are currently two ways of reusing CodeHolder and emitters - one is using \ref CodeHolder::init() followed +//! by \ref CodeHolder::reset(), and another is initializing once by \ref CodeHolder::init() and then reinitializing +//! by \ref CodeHolder::reinit(). The first strategy is shown below: +//! +//! ``` +//! // All of them will be reused for code generation by using an 'init()/reset()' strategy. +//! Environment env = ...; // Environment to use, for example from JitRuntime. +//! CodeHolder code; // CodeHolder to reuse (all allocated memory will be held by it until it's destroyed). +//! x86::Compiler cc; // Emitter to reuse (for example x86::Compiler). +//! +//! for (size_t i = 0; i < ...; i++) { +//! // Initialize the CodeHolder first. +//! code.init(env); +//! code.attach(&emitter); +//! +//! [[code generation as usual]] +//! +//! code.reset(); +//! } +//! ``` +//! +//! While this approach is good for many use-cases, there is even a faster strategy called reinitialization, which is +//! provided by \ref CodeHolder::reinit(). The idea of reinit is to reinitialize the CodeHolder into a state, which +//! was achieved by initializing it by \ref CodeHolder::init(), by optionally attaching \ref Logger, \ref ErrorHandler, +//! and emitters of any kind. See an example below: +//! +//! ``` +//! // All of them will be reused for code generation by using a 'reinit()' strategy. +//! Environment env = ...; // Environment to use, for example from JitRuntime. +//! CodeHolder code; // CodeHolder to reuse (all allocated memory will be held by it until it's destroyed). +//! x86::Compiler cc; // Emitter to reuse (for example x86::Compiler). +//! +//! // Initialize the CodeHolder and attach emitters to it (attaching ErrorHandler is advised!) +//! code.init(env); +//! code.attach(&emitter); +//! +//! for (size_t i = 0; i < ...; i++) { +//! [[code generation as usual]] +//! +//! // Optionally you can start the loop with 'code.reinit()', but this is cleaner as it wipes out all intermediate +//! // states of CodeHolder and the attached emitters. It won't detach Logger, ErrorHandler, nor attached emitters. +//! code.reinit(); +//! } +//! ``` +//! //! \note \ref CodeHolder has an ability to attach an \ref ErrorHandler, however, the error handler is not triggered //! by \ref CodeHolder itself, it's instead propagated to all emitters that attach to it. class CodeHolder { public: ASMJIT_NONCOPYABLE(CodeHolder) + //! \name Types + //! \{ + + //! \cond INTERNAL + struct NamedLabelExtraData : public ZoneHashNode { + LabelEntry::ExtraData extraData; + + ASMJIT_INLINE_NODEBUG uint32_t labelId() const noexcept { return _customData; } + }; + //! \endcond + + //! An informative data structure that is filled with some details that happened during \ref relocateToBase(). + struct RelocationSummary { + //! The number of bytes the final code has been reduced by. + //! + //! At the moment this is the same as the number of bytes that the address table was shrunk, because it was + //! possible to avoid certain entries during relocation - the functions that would be otherwise present were + //! close enough to avoid them in the .addrtab section. + size_t codeSizeReduction; + }; + + //! \} + //! \name Members //! \{ @@ -808,21 +696,32 @@ public: //! Zone allocator, used to manage internal containers. ZoneAllocator _allocator; - //! Attached emitters. - ZoneVector _emitters; + //! First emitter attached to this CodeHolder (double-linked list). + BaseEmitter* _attachedFirst; + //! Last emitter attached to this CodeHolder (double-linked list). + BaseEmitter* _attachedLast; + //! Section entries. ZoneVector _sections; //! Section entries sorted by section order and then section id. ZoneVector _sectionsByOrder; + //! Label entries. - ZoneVector _labelEntries; + ZoneVector _labelEntries; //! Relocation entries. ZoneVector _relocations; - //! Label name -> LabelEntry (only named labels). - ZoneHash _namedLabels; + //! Label name -> LabelEntry::ExtraData (only used by labels that have a name and are not anonymous). + ZoneHash _namedLabels; + //! Unresolved fixups that are most likely references across sections. + Fixup* _fixups; + //! Pool containing \ref Fixup instances for quickly recycling them. + ZonePool _fixupDataPool; + //! Count of unresolved fixups of unbound labels (at the end of assembling this should be zero). + size_t _unresolvedFixupCount; + + //! Text section - always one part of a CodeHolder itself. + Section _textSection; - //! Count of label links, which are not resolved. - size_t _unresolvedLinkCount; //! Pointer to an address table section (or null if this section doesn't exist). Section* _addressTableSection; //! Address table entries. @@ -856,6 +755,22 @@ public: ASMJIT_API Error init(const Environment& environment, uint64_t baseAddress = Globals::kNoBaseAddress) noexcept; //! Initializes CodeHolder to hold code described by the given `environment`, `cpuFeatures`, and `baseAddress`. ASMJIT_API Error init(const Environment& environment, const CpuFeatures& cpuFeatures, uint64_t baseAddress = Globals::kNoBaseAddress) noexcept; + + //! Reinitializes CodeHolder with the same environment, cpu features, and base address as it had, and notifies + //! all attached emitters of reinitialization. If the \ref CodeHolder was not initialized, \ref kErrorNotInitialized + //! is returned. + //! + //! Reinitialization is designed to be a faster alternative compared to \ref reset() followed by \ref init() chain. + //! The purpose of reinitialization is a very quick reuse of \ref CodeHolder and all attached emitters (most likely + //! Assembler or Compiler) without paying the cost of complete initialization and then assignment of all the loggers, + //! error handlers, and emitters. + //! + //! \note Semantically reinit() is the same as using \ref reset(ResetPolicy::kSoft), followed by \ref init(), and + //! then by attaching loggers, error handlers, and emitters that were attached previously. This means that after + //! reinitialization you will get a clean and ready for use \ref CodeHolder, which was initialized the same way as + //! before. + ASMJIT_API Error reinit() noexcept; + //! Detaches all code-generators attached and resets the `CodeHolder`. ASMJIT_API void reset(ResetPolicy resetPolicy = ResetPolicy::kSoft) noexcept; @@ -871,7 +786,7 @@ public: //! \} - //! \name Allocators + //! \name Memory Allocators //! \{ //! Returns the allocator that the `CodeHolder` uses. @@ -913,12 +828,21 @@ public: //! \} - //! \name Emitters + //! \name Attached Emitters //! \{ //! Returns a vector of attached emitters. [[nodiscard]] - ASMJIT_INLINE_NODEBUG const ZoneVector& emitters() const noexcept { return _emitters; } + ASMJIT_INLINE_NODEBUG BaseEmitter* attachedFirst() noexcept { return _attachedFirst; } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG BaseEmitter* attachedLast() noexcept { return _attachedLast; } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const BaseEmitter* attachedFirst() const noexcept { return _attachedFirst; } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const BaseEmitter* attachedLast() const noexcept { return _attachedLast; } //! \} @@ -928,8 +852,10 @@ public: //! Returns the attached logger. [[nodiscard]] ASMJIT_INLINE_NODEBUG Logger* logger() const noexcept { return _logger; } + //! Attaches a `logger` to CodeHolder and propagates it to all attached emitters. ASMJIT_API void setLogger(Logger* logger) noexcept; + //! Resets the logger to none. ASMJIT_INLINE_NODEBUG void resetLogger() noexcept { setLogger(nullptr); } @@ -939,11 +865,14 @@ public: //! Tests whether the CodeHolder has an attached error handler, see \ref ErrorHandler. [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasErrorHandler() const noexcept { return _errorHandler != nullptr; } + //! Returns the attached error handler. [[nodiscard]] ASMJIT_INLINE_NODEBUG ErrorHandler* errorHandler() const noexcept { return _errorHandler; } + //! Attach an error handler to this `CodeHolder`. ASMJIT_API void setErrorHandler(ErrorHandler* errorHandler) noexcept; + //! Resets the error handler to none. ASMJIT_INLINE_NODEBUG void resetErrorHandler() noexcept { setErrorHandler(nullptr); } @@ -1037,30 +966,32 @@ public: //! \name Labels & Symbols //! \{ - //! Returns array of `LabelEntry*` records. + //! Returns array of `LabelEntry` records. [[nodiscard]] - ASMJIT_INLINE_NODEBUG const ZoneVector& labelEntries() const noexcept { return _labelEntries; } + ASMJIT_INLINE_NODEBUG const ZoneVector& labelEntries() const noexcept { return _labelEntries; } //! Returns number of labels created. [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t labelCount() const noexcept { return _labelEntries.size(); } - //! Tests whether the label having `id` is valid (i.e. created by `newLabelEntry()`). + //! Tests whether the label having `labelId` is valid (i.e. created by `newLabelId()`). [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isLabelValid(uint32_t labelId) const noexcept { return labelId < _labelEntries.size(); } - //! Tests whether the `label` is valid (i.e. created by `newLabelEntry()`). + //! Tests whether the `label` is valid (i.e. created by `newLabelId()`). [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isLabelValid(const Label& label) const noexcept { - return label.id() < _labelEntries.size(); + return isLabelValid(label.id()); } - //! \overload + //! Tests whether a label having `labelId` is already bound. + //! + //! Returns `false` if the `labelId` is not valid. [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isLabelBound(uint32_t labelId) const noexcept { - return isLabelValid(labelId) && _labelEntries[labelId]->isBound(); + return isLabelValid(labelId) && _labelEntries[labelId].isBound(); } //! Tests whether the `label` is already bound. @@ -1071,26 +1002,46 @@ public: return isLabelBound(label.id()); } - //! Returns LabelEntry of the given label `id`. + //! Returns LabelEntry of the given label identifier `labelId` (or `label` if you are using overloads). + //! + //! \attention The passed `labelId` must be valid as it's used as an index to `_labelEntries[]` array. In debug + //! builds the array access uses an assertion, but such assertion is not present in release builds. To get whether + //! a label is valid, check out \ref CodeHolder::isLabelValid() function. [[nodiscard]] - ASMJIT_INLINE_NODEBUG LabelEntry* labelEntry(uint32_t labelId) const noexcept { - return isLabelValid(labelId) ? _labelEntries[labelId] : static_cast(nullptr); + ASMJIT_INLINE_NODEBUG LabelEntry& labelEntry(uint32_t labelId) noexcept { + return _labelEntries[labelId]; } - //! Returns LabelEntry of the given `label`. + //! \overload [[nodiscard]] - ASMJIT_INLINE_NODEBUG LabelEntry* labelEntry(const Label& label) const noexcept { + ASMJIT_INLINE_NODEBUG const LabelEntry& labelEntry(uint32_t labelId) const noexcept { + return _labelEntries[labelId]; + } + + //! \overload + [[nodiscard]] + ASMJIT_INLINE_NODEBUG LabelEntry& labelEntry(const Label& label) noexcept { + return labelEntry(label.id()); + } + + //! \overload + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const LabelEntry& labelEntry(const Label& label) const noexcept { return labelEntry(label.id()); } //! Returns offset of a `Label` by its `labelId`. //! - //! The offset returned is relative to the start of the section. Zero offset is returned for unbound labels, - //! which is their initial offset value. + //! The offset returned is relative to the start of the section where the label is bound. Zero offset is returned + //! for unbound labels, which is their initial offset value. + //! + //! \attention The passed `labelId` must be valid as it's used as an index to `_labelEntries[]` array. In debug + //! builds the array access uses an assertion, but such assertion is not present in release builds. To get whether + //! a label is valid, check out \ref CodeHolder::isLabelValid() function. [[nodiscard]] ASMJIT_INLINE_NODEBUG uint64_t labelOffset(uint32_t labelId) const noexcept { ASMJIT_ASSERT(isLabelValid(labelId)); - return _labelEntries[labelId]->offset(); + return _labelEntries[labelId].offset(); } //! \overload @@ -1101,13 +1052,19 @@ public: //! Returns offset of a label by it's `labelId` relative to the base offset. //! - //! \remarks The offset of the section where the label is bound must be valid in order to use this function, - //! otherwise the value returned will not be reliable. + //! \attention The passed `labelId` must be valid as it's used as an index to `_labelEntries[]` array. In debug + //! builds the array access uses an assertion, but such assertion is not present in release builds. To get whether + //! a label is valid, check out \ref CodeHolder::isLabelValid() function. + //! + //! \note The offset of the section where the label is bound must be valid in order to use this function, otherwise + //! the value returned will not be reliable. Typically, sections have offsets when they are flattened, see \ref + //! CodeHolder::flatten() function for more details. [[nodiscard]] inline uint64_t labelOffsetFromBase(uint32_t labelId) const noexcept { ASMJIT_ASSERT(isLabelValid(labelId)); - const LabelEntry* le = _labelEntries[labelId]; - return (le->isBound() ? le->section()->offset() : uint64_t(0)) + le->offset(); + + const LabelEntry& le = _labelEntries[labelId]; + return (le.isBound() ? _sections[le.sectionId()]->offset() : uint64_t(0)) + le.offset(); } //! \overload @@ -1116,10 +1073,11 @@ public: return labelOffsetFromBase(label.id()); } - //! Creates a new anonymous label and return its id in `idOut`. + //! Creates a new anonymous label and return its id in `labelIdOut`. //! //! Returns `Error`, does not report error to `ErrorHandler`. - ASMJIT_API Error newLabelEntry(LabelEntry** entryOut) noexcept; + [[nodiscard]] + ASMJIT_API Error newLabelId(uint32_t* labelIdOut) noexcept; //! Creates a new named \ref LabelEntry of the given label `type`. //! @@ -1136,7 +1094,8 @@ public: //! addition, AsmJit supports named anonymous labels, which are useful only for debugging purposes as the //! anonymous name will have a name, which will be formatted, but the label itself cannot be queried by such //! name. - ASMJIT_API Error newNamedLabelEntry(LabelEntry** entryOut, const char* name, size_t nameSize, LabelType type, uint32_t parentId = Globals::kInvalidId) noexcept; + [[nodiscard]] + ASMJIT_API Error newNamedLabelId(uint32_t* labelIdOut, const char* name, size_t nameSize, LabelType type, uint32_t parentId = Globals::kInvalidId) noexcept; //! Returns a label by name. //! @@ -1153,24 +1112,24 @@ public: [[nodiscard]] ASMJIT_API uint32_t labelIdByName(const char* name, size_t nameSize = SIZE_MAX, uint32_t parentId = Globals::kInvalidId) noexcept; - //! Tests whether there are any unresolved label links. + //! Tests whether there are any unresolved fixups related to unbound labels. [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool hasUnresolvedLinks() const noexcept { return _unresolvedLinkCount != 0; } + ASMJIT_INLINE_NODEBUG bool hasUnresolvedFixups() const noexcept { return _unresolvedFixupCount != 0u; } - //! Returns the number of label links, which are unresolved. + //! Returns the number of unresolved fixups. [[nodiscard]] - ASMJIT_INLINE_NODEBUG size_t unresolvedLinkCount() const noexcept { return _unresolvedLinkCount; } + ASMJIT_INLINE_NODEBUG size_t unresolvedFixupCount() const noexcept { return _unresolvedFixupCount; } //! Creates a new label-link used to store information about yet unbound labels. //! //! Returns `null` if the allocation failed. [[nodiscard]] - ASMJIT_API LabelLink* newLabelLink(LabelEntry* le, uint32_t sectionId, size_t offset, intptr_t rel, const OffsetFormat& format) noexcept; + ASMJIT_API Fixup* newFixup(LabelEntry* le, uint32_t sectionId, size_t offset, intptr_t rel, const OffsetFormat& format) noexcept; - //! Resolves cross-section links (`LabelLink`) associated with each label that was used as a destination in code - //! of a different section. It's only useful to people that use multiple sections as it will do nothing if the code - //! only contains a single section in which cross-section links are not possible. - ASMJIT_API Error resolveUnresolvedLinks() noexcept; + //! Resolves cross-section fixups associated with each label that was used as a destination in code of a different + //! section. It's only useful to people that use multiple sections as it will do nothing if the code only contains + //! a single section in which cross-section fixups are not possible. + ASMJIT_API Error resolveCrossSectionFixups() noexcept; //! Binds a label to a given `sectionId` and `offset` (relative to start of the section). //! @@ -1197,6 +1156,7 @@ public: //! Creates a new relocation entry of type `relocType`. //! //! Additional fields can be set after the relocation entry was created. + [[nodiscard]] ASMJIT_API Error newRelocEntry(RelocEntry** dst, RelocType relocType) noexcept; //! \} @@ -1224,7 +1184,7 @@ public: //! relocations. //! //! \note This should never be called more than once. - ASMJIT_API Error relocateToBase(uint64_t baseAddress) noexcept; + ASMJIT_API Error relocateToBase(uint64_t baseAddress, RelocationSummary* summaryOut = nullptr) noexcept; //! Copies a single section into `dst`. ASMJIT_API Error copySectionData(void* dst, size_t dstSize, uint32_t sectionId, CopySectionFlags copyFlags = CopySectionFlags::kNone) noexcept; diff --git a/src/asmjit/core/codewriter.cpp b/src/asmjit/core/codewriter.cpp index 31871a4..b9edeea 100644 --- a/src/asmjit/core/codewriter.cpp +++ b/src/asmjit/core/codewriter.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -258,7 +258,7 @@ bool CodeWriterUtils::writeOffset(void* dst, int64_t offset64, const OffsetForma return false; } - Support::writeU8(dst, uint8_t(Support::readU8(dst) | mask)); + Support::store_u8(dst, uint8_t(Support::load_u8(dst) | mask)); return true; } @@ -268,7 +268,7 @@ bool CodeWriterUtils::writeOffset(void* dst, int64_t offset64, const OffsetForma return false; } - Support::writeU16uLE(dst, uint16_t(Support::readU16uLE(dst) | mask)); + Support::storeu_u16_le(dst, uint16_t(Support::loadu_u16_le(dst) | mask)); return true; } @@ -278,7 +278,7 @@ bool CodeWriterUtils::writeOffset(void* dst, int64_t offset64, const OffsetForma return false; } - Support::writeU32uLE(dst, Support::readU32uLE(dst) | mask); + Support::storeu_u32_le(dst, Support::loadu_u32_le(dst) | mask); return true; } @@ -288,7 +288,7 @@ bool CodeWriterUtils::writeOffset(void* dst, int64_t offset64, const OffsetForma return false; } - Support::writeU64uLE(dst, Support::readU64uLE(dst) | mask); + Support::storeu_u64_le(dst, Support::loadu_u64_le(dst) | mask); return true; } diff --git a/src/asmjit/core/codewriter_p.h b/src/asmjit/core/codewriter_p.h index 2fc8415..efb6667 100644 --- a/src/asmjit/core/codewriter_p.h +++ b/src/asmjit/core/codewriter_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_CODEBUFFERWRITER_P_H_INCLUDED @@ -70,28 +70,28 @@ public: template ASMJIT_INLINE void emit16uLE(T val) noexcept { using U = std::make_unsigned_t; - Support::writeU16uLE(_cursor, uint16_t(U(val) & 0xFFFFu)); + Support::storeu_u16_le(_cursor, uint16_t(U(val) & 0xFFFFu)); _cursor += 2; } template ASMJIT_INLINE void emit16uBE(T val) noexcept { using U = std::make_unsigned_t; - Support::writeU16uBE(_cursor, uint16_t(U(val) & 0xFFFFu)); + Support::storeu_u16_be(_cursor, uint16_t(U(val) & 0xFFFFu)); _cursor += 2; } template ASMJIT_INLINE void emit32uLE(T val) noexcept { using U = std::make_unsigned_t; - Support::writeU32uLE(_cursor, uint32_t(U(val) & 0xFFFFFFFFu)); + Support::storeu_u32_le(_cursor, uint32_t(U(val) & 0xFFFFFFFFu)); _cursor += 4; } template ASMJIT_INLINE void emit32uBE(T val) noexcept { using U = std::make_unsigned_t; - Support::writeU32uBE(_cursor, uint32_t(U(val) & 0xFFFFFFFFu)); + Support::storeu_u32_be(_cursor, uint32_t(U(val) & 0xFFFFFFFFu)); _cursor += 4; } diff --git a/src/asmjit/core/compiler.cpp b/src/asmjit/core/compiler.cpp index f140471..82dc579 100644 --- a/src/asmjit/core/compiler.cpp +++ b/src/asmjit/core/compiler.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -51,7 +51,6 @@ public: BaseCompiler::BaseCompiler() noexcept : BaseBuilder(), _func(nullptr), - _vRegZone(4u * 1024u), _vRegArray(), _constPools { nullptr, nullptr } { _emitterType = EmitterType::kCompiler; @@ -66,7 +65,7 @@ Error BaseCompiler::newFuncNode(FuncNode** out, const FuncSignature& signature) *out = nullptr; // Create FuncNode together with all the required surrounding nodes. - FuncNode* funcNode; + FuncNode* funcNode = nullptr; ASMJIT_PROPAGATE(_newNodeT(&funcNode)); ASMJIT_PROPAGATE(newLabelNode(&funcNode->_exitNode)); ASMJIT_PROPAGATE(_newNodeT(&funcNode->_end, SentinelType::kFuncEnd)); @@ -94,7 +93,7 @@ Error BaseCompiler::newFuncNode(FuncNode** out, const FuncSignature& signature) // Allocate space for function arguments. funcNode->_args = nullptr; if (funcNode->argCount() != 0) { - funcNode->_args = _allocator.allocT(funcNode->argCount() * sizeof(FuncNode::ArgPack)); + funcNode->_args = _codeZone.alloc(funcNode->argCount() * sizeof(FuncNode::ArgPack)); if (ASMJIT_UNLIKELY(!funcNode->_args)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); } @@ -121,7 +120,7 @@ Error BaseCompiler::addFuncNode(FuncNode** out, const FuncSignature& signature) Error BaseCompiler::newFuncRetNode(FuncRetNode** out, const Operand_& o0, const Operand_& o1) { uint32_t opCount = !o1.isNone() ? 2u : !o0.isNone() ? 1u : 0u; - FuncRetNode* node; + FuncRetNode* node = nullptr; ASMJIT_PROPAGATE(_newNodeT(&node)); ASMJIT_ASSUME(node != nullptr); @@ -150,10 +149,10 @@ Error BaseCompiler::addFuncRetNode(FuncRetNode** out, const Operand_& o0, const FuncNode* BaseCompiler::addFunc(FuncNode* func) { _func = func; - addNode(func); // Function node. - BaseNode* prev = cursor(); // {CURSOR}. - addNode(func->exitNode()); // Function exit label. - addNode(func->endNode()); // Function end sentinel. + addNode(func); // Function node. + BaseNode* prev = cursor(); // {CURSOR}. + addNode(func->exitNode()); // Function exit label. + addNode(func->endNode()); // Function end sentinel. _setCursor(prev); return func; @@ -188,7 +187,7 @@ Error BaseCompiler::endFunc() { // ================================== Error BaseCompiler::newInvokeNode(InvokeNode** out, InstId instId, const Operand_& o0, const FuncSignature& signature) { - InvokeNode* node; + InvokeNode* node = nullptr; ASMJIT_PROPAGATE(_newNodeT(&node, instId, InstOptions::kNone)); node->setOpCount(1); @@ -203,7 +202,7 @@ Error BaseCompiler::newInvokeNode(InvokeNode** out, InstId instId, const Operand // Skip the allocation if there are no arguments. uint32_t argCount = signature.argCount(); if (argCount) { - node->_args = static_cast(_allocator.alloc(argCount * sizeof(InvokeNode::OperandPack))); + node->_args = _codeZone.alloc(argCount * sizeof(InvokeNode::OperandPack)); if (!node->_args) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); } @@ -228,16 +227,6 @@ Error BaseCompiler::addInvokeNode(InvokeNode** out, InstId instId, const Operand // BaseCompiler - Virtual Registers // ================================ -static void BaseCompiler_assignGenericName(BaseCompiler* self, VirtReg* vReg) { - uint32_t index = unsigned(Operand::virtIdToIndex(vReg->_id)); - - char buf[64]; - int size = snprintf(buf, ASMJIT_ARRAY_SIZE(buf), "%%%u", unsigned(index)); - - ASMJIT_ASSERT(size > 0 && size < int(ASMJIT_ARRAY_SIZE(buf))); - vReg->_name.setData(&self->_dataZone, buf, unsigned(size)); -} - Error BaseCompiler::newVirtReg(VirtReg** out, TypeId typeId, OperandSignature signature, const char* name) { *out = nullptr; uint32_t index = _vRegArray.size(); @@ -250,22 +239,18 @@ Error BaseCompiler::newVirtReg(VirtReg** out, TypeId typeId, OperandSignature si return reportError(DebugUtils::errored(kErrorOutOfMemory)); } - VirtReg* vReg = _vRegZone.allocZeroedT(); - if (ASMJIT_UNLIKELY(!vReg)) { + void* vRegPtr = _codeZone.alloc(Zone::alignedSizeOf()); + if (ASMJIT_UNLIKELY(!vRegPtr)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); } uint32_t size = TypeUtils::sizeOf(typeId); uint32_t alignment = Support::min(size, 64); - - vReg = new(Support::PlacementNew{vReg}) VirtReg(signature, Operand::indexToVirtId(index), size, alignment, typeId); + VirtReg* vReg = new(Support::PlacementNew{vRegPtr}) VirtReg(signature, Operand::indexToVirtId(index), size, alignment, typeId); #ifndef ASMJIT_NO_LOGGING if (name && name[0] != '\0') { - vReg->_name.setData(&_dataZone, name, SIZE_MAX); - } - else { - BaseCompiler_assignGenericName(this, vReg); + vReg->_name.setData(&_codeZone, name, SIZE_MAX); } #else DebugUtils::unused(name); @@ -277,7 +262,7 @@ Error BaseCompiler::newVirtReg(VirtReg** out, TypeId typeId, OperandSignature si return kErrorOk; } -Error BaseCompiler::_newReg(BaseReg* out, TypeId typeId, const char* name) { +Error BaseCompiler::_newReg(Reg* out, TypeId typeId, const char* name) { OperandSignature regSignature; out->reset(); @@ -294,7 +279,7 @@ Error BaseCompiler::_newReg(BaseReg* out, TypeId typeId, const char* name) { return kErrorOk; } -Error BaseCompiler::_newRegFmt(BaseReg* out, TypeId typeId, const char* fmt, ...) { +Error BaseCompiler::_newRegFmt(Reg* out, TypeId typeId, const char* fmt, ...) { va_list ap; StringTmp<256> sb; @@ -305,7 +290,7 @@ Error BaseCompiler::_newRegFmt(BaseReg* out, TypeId typeId, const char* fmt, ... return _newReg(out, typeId, sb.data()); } -Error BaseCompiler::_newReg(BaseReg* out, const BaseReg& ref, const char* name) { +Error BaseCompiler::_newReg(Reg* out, const Reg& ref, const char* name) { out->reset(); OperandSignature regSignature; @@ -364,7 +349,7 @@ Error BaseCompiler::_newReg(BaseReg* out, const BaseReg& ref, const char* name) } } else { - typeId = ArchTraits::byArch(arch()).regTypeToTypeId(ref.type()); + typeId = RegUtils::typeIdOf(ref.regType()); } Error err = ArchUtils::typeIdToRegSignature(arch(), typeId, &typeId, ®Signature); @@ -380,7 +365,7 @@ Error BaseCompiler::_newReg(BaseReg* out, const BaseReg& ref, const char* name) return kErrorOk; } -Error BaseCompiler::_newRegFmt(BaseReg* out, const BaseReg& ref, const char* fmt, ...) { +Error BaseCompiler::_newRegFmt(Reg* out, const Reg& ref, const char* fmt, ...) { va_list ap; StringTmp<256> sb; @@ -394,20 +379,16 @@ Error BaseCompiler::_newRegFmt(BaseReg* out, const BaseReg& ref, const char* fmt Error BaseCompiler::_newStack(BaseMem* out, uint32_t size, uint32_t alignment, const char* name) { out->reset(); - if (size == 0) { + if (ASMJIT_UNLIKELY(Support::bool_or(size == 0, !Support::isZeroOrPowerOf2(alignment)))) { return reportError(DebugUtils::errored(kErrorInvalidArgument)); } - if (alignment == 0) { - alignment = 1; + if (alignment == 0u) { + alignment = 1u; } - if (!Support::isPowerOf2(alignment)) { - return reportError(DebugUtils::errored(kErrorInvalidArgument)); - } - - if (alignment > 64) { - alignment = 64; + if (alignment > 64u) { + alignment = 64u; } VirtReg* vReg; @@ -431,12 +412,12 @@ Error BaseCompiler::setStackSize(uint32_t virtId, uint32_t newSize, uint32_t new return DebugUtils::errored(kErrorInvalidVirtId); } - if (newAlignment && !Support::isPowerOf2(newAlignment)) { + if (!Support::isZeroOrPowerOf2(newAlignment)) { return reportError(DebugUtils::errored(kErrorInvalidArgument)); } - if (newAlignment > 64) { - newAlignment = 64; + if (newAlignment > 64u) { + newAlignment = 64u; } VirtReg* vReg = virtRegById(virtId); @@ -485,7 +466,7 @@ Error BaseCompiler::_newConst(BaseMem* out, ConstPoolScope scope, const void* da return kErrorOk; } -void BaseCompiler::rename(const BaseReg& reg, const char* fmt, ...) { +void BaseCompiler::rename(const Reg& reg, const char* fmt, ...) { if (!reg.isVirtReg()) return; VirtReg* vReg = virtRegById(reg.id()); @@ -501,10 +482,7 @@ void BaseCompiler::rename(const BaseReg& reg, const char* fmt, ...) { vsnprintf(buf, ASMJIT_ARRAY_SIZE(buf), fmt, ap); va_end(ap); - vReg->_name.setData(&_dataZone, buf, SIZE_MAX); - } - else { - BaseCompiler_assignGenericName(this, vReg); + vReg->_name.setData(&_codeZone, buf, SIZE_MAX); } } @@ -512,15 +490,15 @@ void BaseCompiler::rename(const BaseReg& reg, const char* fmt, ...) { // =============================== Error BaseCompiler::newJumpNode(JumpNode** out, InstId instId, InstOptions instOptions, const Operand_& o0, JumpAnnotation* annotation) { - JumpNode* node = _allocator.allocT(); - uint32_t opCount = 1; + JumpNode* node = _codeZone.alloc(); *out = node; if (ASMJIT_UNLIKELY(!node)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); } - node = new(Support::PlacementNew{node}) JumpNode(this, instId, instOptions, opCount, annotation); + uint32_t opCount = 1; + node = new(Support::PlacementNew{node}) JumpNode(instId, instOptions, opCount, annotation); node->setOp(0, o0); node->resetOpRange(opCount, JumpNode::kBaseOpCapacity); @@ -547,7 +525,7 @@ JumpAnnotation* BaseCompiler::newJumpAnnotation() { } uint32_t id = _jumpAnnotations.size(); - JumpAnnotation* jumpAnnotation = _allocator.newT(this, id); + JumpAnnotation* jumpAnnotation = _codeZone.newT(this, id); if (!jumpAnnotation) { reportError(DebugUtils::errored(kErrorOutOfMemory)); @@ -561,33 +539,49 @@ JumpAnnotation* BaseCompiler::newJumpAnnotation() { // BaseCompiler - Events // ===================== -Error BaseCompiler::onAttach(CodeHolder* code) noexcept { +static ASMJIT_INLINE void BaseCompiler_clear(BaseCompiler* self) noexcept { + self->_func = nullptr; + self->_constPools[uint32_t(ConstPoolScope::kLocal)] = nullptr; + self->_constPools[uint32_t(ConstPoolScope::kGlobal)] = nullptr; + self->_vRegArray.reset(); +} + +static ASMJIT_INLINE Error BaseCompiler_initDefaultPasses(BaseCompiler* self) noexcept { + return self->addPassT(); +} + + +Error BaseCompiler::onAttach(CodeHolder& code) noexcept { ASMJIT_PROPAGATE(Base::onAttach(code)); - const ArchTraits& archTraits = ArchTraits::byArch(code->arch()); - RegType nativeRegType = Environment::is32Bit(code->arch()) ? RegType::kGp32 : RegType::kGp64; - _gpSignature = archTraits.regTypeToSignature(nativeRegType); - - Error err = addPassT(); + Error err = BaseCompiler_initDefaultPasses(this); if (ASMJIT_UNLIKELY(err)) { onDetach(code); return err; } - return kErrorOk; } -Error BaseCompiler::onDetach(CodeHolder* code) noexcept { - _func = nullptr; - _constPools[uint32_t(ConstPoolScope::kLocal)] = nullptr; - _constPools[uint32_t(ConstPoolScope::kGlobal)] = nullptr; - - _vRegArray.reset(); - _vRegZone.reset(); - +Error BaseCompiler::onDetach(CodeHolder& code) noexcept { + BaseCompiler_clear(this); return Base::onDetach(code); } +Error BaseCompiler::onReinit(CodeHolder& code) noexcept { + BaseCompiler_clear(this); + Error err = Base::onReinit(code); + + if (ASMJIT_LIKELY(err == kErrorOk)) { + err = BaseCompiler_initDefaultPasses(this); + if (ASMJIT_UNLIKELY(err)) { + onDetach(code); + return err; + } + } + + return err; +} + // FuncPass - Construction & Destruction // ===================================== @@ -599,9 +593,8 @@ FuncPass::FuncPass(const char* name) noexcept Error FuncPass::run(Zone* zone, Logger* logger) { BaseNode* node = cb()->firstNode(); - if (!node) return kErrorOk; - do { + while (node) { if (node->type() == NodeType::kFunc) { FuncNode* func = node->as(); node = func->endNode(); @@ -612,7 +605,7 @@ Error FuncPass::run(Zone* zone, Logger* logger) { do { node = node->next(); } while (node && node->type() != NodeType::kFunc); - } while (node); + } return kErrorOk; } diff --git a/src/asmjit/core/compiler.h b/src/asmjit/core/compiler.h index 60d9569..ed3388e 100644 --- a/src/asmjit/core/compiler.h +++ b/src/asmjit/core/compiler.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_COMPILER_H_INCLUDED @@ -58,8 +58,6 @@ public: //! Current function. FuncNode* _func; - //! Allocates `VirtReg` objects. - Zone _vRegZone; //! Stores array of `VirtReg` pointers. ZoneVector _vRegArray; //! Stores jump annotations. @@ -146,22 +144,22 @@ public: ASMJIT_API Error newVirtReg(VirtReg** ASMJIT_NONNULL(out), TypeId typeId, OperandSignature signature, const char* name); //! Creates a new virtual register of the given `typeId` and stores it to `out` operand. - ASMJIT_API Error _newReg(BaseReg* ASMJIT_NONNULL(out), TypeId typeId, const char* name = nullptr); + ASMJIT_API Error _newReg(Reg* ASMJIT_NONNULL(out), TypeId typeId, const char* name = nullptr); //! Creates a new virtual register of the given `typeId` and stores it to `out` operand. //! //! \note This version accepts a snprintf() format `fmt` followed by a variadic arguments. - ASMJIT_API Error _newRegFmt(BaseReg* ASMJIT_NONNULL(out), TypeId typeId, const char* fmt, ...); + ASMJIT_API Error _newRegFmt(Reg* ASMJIT_NONNULL(out), TypeId typeId, const char* fmt, ...); //! \overload - inline Error _newRegFmt(BaseReg* ASMJIT_NONNULL(out), TypeId typeId) { return _newRegFmt(out, typeId, nullptr); } + inline Error _newRegFmt(Reg* ASMJIT_NONNULL(out), TypeId typeId) { return _newRegFmt(out, typeId, nullptr); } //! Creates a new virtual register compatible with the provided reference register `ref`. - ASMJIT_API Error _newReg(BaseReg* ASMJIT_NONNULL(out), const BaseReg& ref, const char* name = nullptr); + ASMJIT_API Error _newReg(Reg* ASMJIT_NONNULL(out), const Reg& ref, const char* name = nullptr); //! Creates a new virtual register compatible with the provided reference register `ref`. //! //! \note This version accepts a snprintf() format `fmt` followed by a variadic arguments. - ASMJIT_API Error _newRegFmt(BaseReg* ASMJIT_NONNULL(out), const BaseReg& ref, const char* fmt, ...); + ASMJIT_API Error _newRegFmt(Reg* ASMJIT_NONNULL(out), const Reg& ref, const char* fmt, ...); //! Tests whether the given `id` is a valid virtual register id. [[nodiscard]] @@ -172,7 +170,7 @@ public: //! Tests whether the given `reg` is a virtual register having a valid id. [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool isVirtRegValid(const BaseReg& reg) const noexcept { + ASMJIT_INLINE_NODEBUG bool isVirtRegValid(const Reg& reg) const noexcept { return isVirtIdValid(reg.id()); } @@ -185,7 +183,7 @@ public: //! Returns \ref VirtReg associated with the given `reg`. [[nodiscard]] - ASMJIT_INLINE_NODEBUG VirtReg* virtRegByReg(const BaseReg& reg) const noexcept { return virtRegById(reg.id()); } + ASMJIT_INLINE_NODEBUG VirtReg* virtRegByReg(const Reg& reg) const noexcept { return virtRegById(reg.id()); } //! Returns \ref VirtReg associated with the given virtual register `index`. //! @@ -231,7 +229,7 @@ public: //! \{ //! Rename the given virtual register `reg` to a formatted string `fmt`. - ASMJIT_API void rename(const BaseReg& reg, const char* fmt, ...); + ASMJIT_API void rename(const Reg& reg, const char* fmt, ...); //! \} @@ -256,8 +254,9 @@ public: //! \name Events //! \{ - ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; - ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + ASMJIT_API Error onAttach(CodeHolder& code) noexcept override; + ASMJIT_API Error onDetach(CodeHolder& code) noexcept override; + ASMJIT_API Error onReinit(CodeHolder& code) noexcept override; //! \} }; @@ -348,10 +347,10 @@ public: //! \name Construction & Destruction //! \{ - inline JumpNode(BaseCompiler* ASMJIT_NONNULL(cc), InstId instId, InstOptions options, uint32_t opCount, JumpAnnotation* annotation) noexcept - : InstNodeWithOperands(cc, instId, options, opCount), + inline JumpNode(InstId instId, InstOptions options, uint32_t opCount, JumpAnnotation* annotation) noexcept + : InstNodeWithOperands(instId, options, opCount), _annotation(annotation) { - setType(NodeType::kJump); + _setType(NodeType::kJump); } //! \} @@ -452,14 +451,14 @@ public: //! Creates a new `FuncNode` instance. //! //! Always use `BaseCompiler::addFunc()` to create a new `FuncNode`. - inline FuncNode(BaseBuilder* ASMJIT_NONNULL(cb)) noexcept - : LabelNode(cb), + inline explicit FuncNode(uint32_t labelId = Globals::kInvalidId) noexcept + : LabelNode(labelId), _funcDetail(), _frame(), _exitNode(nullptr), _end(nullptr), _args(nullptr) { - setType(NodeType::kFunc); + _setType(NodeType::kFunc); } //! \} @@ -522,7 +521,7 @@ public: } //! Sets argument at `argIndex`. - inline void setArg(size_t argIndex, const BaseReg& vReg) noexcept { + inline void setArg(size_t argIndex, const Reg& vReg) noexcept { ASMJIT_ASSERT(argIndex < argCount()); _args[argIndex][0].init(vReg); } @@ -534,7 +533,7 @@ public: } //! Sets argument at `argIndex` and `valueIndex`. - inline void setArg(size_t argIndex, size_t valueIndex, const BaseReg& vReg) noexcept { + inline void setArg(size_t argIndex, size_t valueIndex, const Reg& vReg) noexcept { ASMJIT_ASSERT(argIndex < argCount()); _args[argIndex][valueIndex].init(vReg); } @@ -569,9 +568,9 @@ public: //! \{ //! Creates a new `FuncRetNode` instance. - inline FuncRetNode(BaseBuilder* ASMJIT_NONNULL(cb)) noexcept - : InstNodeWithOperands(cb, BaseInst::kIdAbstract, InstOptions::kNone, 0) { - _any._nodeType = NodeType::kFuncRet; + inline FuncRetNode() noexcept + : InstNodeWithOperands(BaseInst::kIdAbstract, InstOptions::kNone, 0) { + _nodeType = NodeType::kFuncRet; } //! \} @@ -626,14 +625,14 @@ public: //! \{ //! Creates a new `InvokeNode` instance. - inline InvokeNode(BaseBuilder* ASMJIT_NONNULL(cb), InstId instId, InstOptions options) noexcept - : InstNodeWithOperands(cb, instId, options, 0), + inline InvokeNode(InstId instId, InstOptions options) noexcept + : InstNodeWithOperands(instId, options, 0), _funcDetail(), _args(nullptr) { - setType(NodeType::kInvoke); + _setType(NodeType::kInvoke); _resetOps(); _rets.reset(); - addFlags(NodeFlags::kIsRemovable); + _addFlags(NodeFlags::kIsRemovable); } //! \} @@ -724,15 +723,15 @@ public: } //! Sets the function return value at `valueIndex` to `reg`. - ASMJIT_INLINE_NODEBUG void setRet(size_t valueIndex, const BaseReg& reg) noexcept { _setRet(valueIndex, reg); } + ASMJIT_INLINE_NODEBUG void setRet(size_t valueIndex, const Reg& reg) noexcept { _setRet(valueIndex, reg); } //! Sets the first function argument in a value-pack at `argIndex` to `reg`. - ASMJIT_INLINE_NODEBUG void setArg(size_t argIndex, const BaseReg& reg) noexcept { _setArg(argIndex, 0, reg); } + ASMJIT_INLINE_NODEBUG void setArg(size_t argIndex, const Reg& reg) noexcept { _setArg(argIndex, 0, reg); } //! Sets the first function argument in a value-pack at `argIndex` to `imm`. ASMJIT_INLINE_NODEBUG void setArg(size_t argIndex, const Imm& imm) noexcept { _setArg(argIndex, 0, imm); } //! Sets the function argument at `argIndex` and `valueIndex` to `reg`. - ASMJIT_INLINE_NODEBUG void setArg(size_t argIndex, size_t valueIndex, const BaseReg& reg) noexcept { _setArg(argIndex, valueIndex, reg); } + ASMJIT_INLINE_NODEBUG void setArg(size_t argIndex, size_t valueIndex, const Reg& reg) noexcept { _setArg(argIndex, valueIndex, reg); } //! Sets the function argument at `argIndex` and `valueIndex` to `imm`. ASMJIT_INLINE_NODEBUG void setArg(size_t argIndex, size_t valueIndex, const Imm& imm) noexcept { _setArg(argIndex, valueIndex, imm); } diff --git a/src/asmjit/core/compilerdefs.h b/src/asmjit/core/compilerdefs.h index 1922560..14b7346 100644 --- a/src/asmjit/core/compilerdefs.h +++ b/src/asmjit/core/compilerdefs.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_COMPILERDEFS_H_INCLUDED @@ -57,7 +57,7 @@ public: uint8_t _hasStackSlot : 1; uint8_t _reservedBits : 5; //! Home register hint for the register allocator (initially unassigned). - uint8_t _homeIdHint = BaseReg::kIdBad; + uint8_t _homeIdHint = Reg::kIdBad; //! Stack offset assigned by the register allocator relative to stack pointer (can be negative as well). int32_t _stackOffset = 0; @@ -65,7 +65,7 @@ public: //! Reserved for future use (padding). uint32_t _reservedU32 = 0; - //! Virtual register name (user provided or automatically generated). + //! Virtual register name (either empty or user provided). ZoneString<16> _name {}; // The following members are used exclusively by RAPass. They are initialized when the VirtReg is created to @@ -178,7 +178,7 @@ public: //! Tests whether this virtual register has assigned a physical register as a hint to the register allocator. [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool hasHomeIdHint() const noexcept { return _homeIdHint != BaseReg::kIdBad; } + ASMJIT_INLINE_NODEBUG bool hasHomeIdHint() const noexcept { return _homeIdHint != Reg::kIdBad; } //! Returns a physical register hint, which will be used by the register allocator. [[nodiscard]] @@ -187,7 +187,7 @@ public: //! Assigns a physical register hint, which will be used by the register allocator. ASMJIT_INLINE_NODEBUG void setHomeIdHint(uint32_t homeId) noexcept { _homeIdHint = uint8_t(homeId); } //! Resets a physical register hint. - ASMJIT_INLINE_NODEBUG void resetHomeIdHint() noexcept { _homeIdHint = BaseReg::kIdBad; } + ASMJIT_INLINE_NODEBUG void resetHomeIdHint() noexcept { _homeIdHint = Reg::kIdBad; } //! Returns a stack offset associated with a virtual register or explicit stack allocation. //! diff --git a/src/asmjit/core/constpool.cpp b/src/asmjit/core/constpool.cpp index 9e368ec..8531e11 100644 --- a/src/asmjit/core/constpool.cpp +++ b/src/asmjit/core/constpool.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -40,8 +40,9 @@ void ConstPool::reset(Zone* zone) noexcept { static inline ConstPool::Gap* ConstPool_allocGap(ConstPool* self) noexcept { ConstPool::Gap* gap = self->_gapPool; + if (!gap) { - return self->_zone->allocT(); + return self->_zone->alloc(); } self->_gapPool = gap->_next; diff --git a/src/asmjit/core/constpool.h b/src/asmjit/core/constpool.h index b545611..9cd838b 100644 --- a/src/asmjit/core/constpool.h +++ b/src/asmjit/core/constpool.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_CONSTPOOL_H_INCLUDED @@ -74,9 +74,10 @@ public: _offset(uint32_t(offset)) {} [[nodiscard]] - ASMJIT_INLINE_NODEBUG void* data() const noexcept { - return static_cast(const_cast(this) + 1); - } + ASMJIT_INLINE_NODEBUG void* data() noexcept { return Support::offsetPtr(this, sizeof(*this)); } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const void* data() const noexcept { return Support::offsetPtr(this, sizeof(*this)); } }; //! Data comparer used internally. @@ -175,8 +176,12 @@ public: [[nodiscard]] static inline Node* _newNode(Zone* zone, const void* data, size_t size, size_t offset, bool shared) noexcept { - Node* node = zone->allocT(Support::alignUp(sizeof(Node) + size, alignof(Node))); - if (ASMJIT_UNLIKELY(!node)) return nullptr; + size_t nodeSize = Support::alignUp(sizeof(Node) + size, Globals::kZoneAlignment); + Node* node = zone->alloc(nodeSize); + + if (ASMJIT_UNLIKELY(!node)) { + return nullptr; + } node = new(Support::PlacementNew{node}) Node(offset, shared); memcpy(node->data(), data, size); diff --git a/src/asmjit/core/cpuinfo.cpp b/src/asmjit/core/cpuinfo.cpp index a06bd16..ba5f319 100644 --- a/src/asmjit/core/cpuinfo.cpp +++ b/src/asmjit/core/cpuinfo.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -66,7 +66,7 @@ ASMJIT_BEGIN_NAMESPACE // CPU features detection is a minefield on non-X86 platforms. The following list describes which // operating systems and architectures are supported and the status of the implementation: // -// * X86, X86_64: +// * X86|X86_64: // - All OSes supported // - Detection is based on using a CPUID instruction, which is a user-space instruction, so there // is no need to use any OS specific APIs or syscalls to detect all features provided by the CPU. @@ -710,7 +710,7 @@ static inline void populateBaseARMFeatures(CpuInfo& cpu) noexcept { // Populates mandatory ARMv8.[v]A features. [[maybe_unused]] -static ASMJIT_FAVOR_SIZE void populateARMv8AFeatures(CpuFeatures::ARM& features, uint32_t v) noexcept { +static ASMJIT_NOINLINE void populateARMv8AFeatures(CpuFeatures::ARM& features, uint32_t v) noexcept { switch (v) { default: [[fallthrough]]; @@ -982,6 +982,9 @@ static inline void detectAArch64FeaturesViaCPUID_AA64ISAR2(CpuInfo& cpu, uint64_ MERGE_FEATURE_4B("RPRFM bits [51:48]" , isar2, 48, Ext::kRPRFM); MERGE_FEATURE_4B("CSSC bits [55:52]" , isar2, 52, Ext::kCSSC); MERGE_FEATURE_4B("LUT bits [59:56]" , isar2, 56, Ext::kLUT); + /* + MERGE_FEATURE_4B("ATS1A bits [63:60]" , isar2, 60, ...); + */ } // TODO: This register is not accessed at the moment. @@ -992,9 +995,15 @@ static inline void detectAArch64FeaturesViaCPUID_AA64ISAR3(CpuInfo& cpu, uint64_ // ID_AA64ISAR3_EL1 // ================ - MERGE_FEATURE_4B("CPA bits [3:0]" , isar3, 0, Ext::kCPA, Ext::kCPA2); - MERGE_FEATURE_4B("FAMINMAX bits [7:4]" , isar3, 4, Ext::kFAMINMAX); - MERGE_FEATURE_4B("TLBIW bits [11:8]" , isar3, 8, Ext::kTLBIW); + MERGE_FEATURE_4B("CPA bits [3:0]" , isar3, 0, Ext::kCPA, Ext::kCPA2); + MERGE_FEATURE_4B("FAMINMAX bits [7:4]" , isar3, 4, Ext::kFAMINMAX); + MERGE_FEATURE_4B("TLBIW bits [11:8]" , isar3, 8, Ext::kTLBIW); + /* + MERGE_FEATURE_4B("PACM bits [15:12]" , isar3, 12, ...); + */ + MERGE_FEATURE_4B("LSFE bits [19:16]" , isar3, 16, Ext::kLSFE); + MERGE_FEATURE_4B("OCCMO bits [23:20]" , isar3, 20, Ext::kOCCMO); + MERGE_FEATURE_4B("LSUI bits [27:24]" , isar3, 24, Ext::kLSUI); } #endif @@ -1080,11 +1089,12 @@ static inline void detectAArch64FeaturesViaCPUID_AA64MMFR2(CpuInfo& cpu, uint64_ */ } -// Detects features based on the content of ID_AA64ZFR0_EL1 register. +// Detects features based on the content of ID_AA64ZFR0_EL1 (SVE Feature ID register 0). [[maybe_unused]] static inline void detectAArch64FeaturesViaCPUID_AA64ZFR0(CpuInfo& cpu, uint64_t zfr0) noexcept { - MERGE_FEATURE_4B("SVEver bits [3:0]" , zfr0, 0, Ext::kSVE2, Ext::kSVE2_1); + MERGE_FEATURE_4B("SVEver bits [3:0]" , zfr0, 0, Ext::kSVE2, Ext::kSVE2_1, Ext::kSVE2_2); MERGE_FEATURE_4B("AES bits [7:4]" , zfr0, 4, Ext::kSVE_AES, Ext::kSVE_PMULL128); + MERGE_FEATURE_4B("EltPerm bits [15:12]" , zfr0, 12, Ext::kSVE_ELTPERM); MERGE_FEATURE_4B("BitPerm bits [19:16]" , zfr0, 16, Ext::kSVE_BITPERM); MERGE_FEATURE_4B("BF16 bits [23:20]" , zfr0, 20, Ext::kSVE_BF16, Ext::kSVE_EBF16); MERGE_FEATURE_4B("B16B16 bits [27:24]" , zfr0, 24, Ext::kSVE_B16B16); @@ -1097,7 +1107,12 @@ static inline void detectAArch64FeaturesViaCPUID_AA64ZFR0(CpuInfo& cpu, uint64_t [[maybe_unused]] static inline void detectAArch64FeaturesViaCPUID_AA64SMFR0(CpuInfo& cpu, uint64_t smfr0) noexcept { - MERGE_FEATURE_1B("SF8DP2 bit [28]" , smfr0, 29, Ext::kSSVE_FP8DOT2); + MERGE_FEATURE_1B("SMOP4 bit [0]" , smfr0, 0, Ext::kSME_MOP4); + MERGE_FEATURE_1B("STMOP bit [16]" , smfr0, 16, Ext::kSME_TMOP); + MERGE_FEATURE_1B("SFEXPA bit [23]" , smfr0, 23, Ext::kSSVE_FEXPA); + MERGE_FEATURE_1B("AES bit [24]" , smfr0, 24, Ext::kSSVE_AES); + MERGE_FEATURE_1B("SBitPerm bit [25]" , smfr0, 25, Ext::kSSVE_BITPERM); + MERGE_FEATURE_1B("SF8DP2 bit [28]" , smfr0, 28, Ext::kSSVE_FP8DOT2); MERGE_FEATURE_1B("SF8DP4 bit [29]" , smfr0, 29, Ext::kSSVE_FP8DOT4); MERGE_FEATURE_1B("SF8FMA bit [30]" , smfr0, 30, Ext::kSSVE_FP8FMA); MERGE_FEATURE_1B("F32F32 bit [32]" , smfr0, 32, Ext::kSME_F32F32); @@ -1112,7 +1127,7 @@ static inline void detectAArch64FeaturesViaCPUID_AA64SMFR0(CpuInfo& cpu, uint64_ MERGE_FEATURE_4S("I16I32 bits [47:44]" , smfr0, 44, 0x5, Ext::kSME_I16I32); MERGE_FEATURE_1B("F64F64 bit [48]" , smfr0, 48, Ext::kSME_F64F64); MERGE_FEATURE_4S("I16I64 bits [55:52]" , smfr0, 52, 0xF, Ext::kSME_I16I64); - MERGE_FEATURE_4B("SMEver bits [59:56]" , smfr0, 56, Ext::kSME2, Ext::kSME2_1); + MERGE_FEATURE_4B("SMEver bits [59:56]" , smfr0, 56, Ext::kSME2, Ext::kSME2_1, Ext::kSME2_2); MERGE_FEATURE_1B("LUTv2 bit [60]" , smfr0, 60, Ext::kSME_LUTv2); MERGE_FEATURE_1B("FA64 bit [63]" , smfr0, 63, Ext::kSME_FA64); } @@ -1129,17 +1144,25 @@ static inline void detectAArch64FeaturesViaCPUID_AA64SMFR0(CpuInfo& cpu, uint64_ // CPU features detection based on Apple family ID. enum class AppleFamilyId : uint32_t { // Apple design. - kSWIFT = 0x1E2D6381u, // Apple A6/A6X (ARMv7s). - kCYCLONE = 0x37A09642u, // Apple A7 (ARMv8.0-A). - kTYPHOON = 0x2C91A47Eu, // Apple A8 (ARMv8.0-A). - kTWISTER = 0x92FB37C8u, // Apple A9 (ARMv8.0-A). - kHURRICANE = 0x67CEEE93u, // Apple A10 (ARMv8.1-A). - kMONSOON_MISTRAL = 0xE81E7EF6u, // Apple A11 (ARMv8.2-A). - kVORTEX_TEMPEST = 0x07D34B9Fu, // Apple A12 (ARMv8.3-A). - kLIGHTNING_THUNDER = 0x462504D2u, // Apple A13 (ARMv8.4-A). - kFIRESTORM_ICESTORM = 0x1B588BB3u, // Apple A14/M1 (ARMv8.5-A). - kAVALANCHE_BLIZZARD = 0XDA33D83Du, // Apple A15/M2. - kEVEREST_SAWTOOTH = 0X8765EDEAu // Apple A16. + kSWIFT = 0x1E2D6381u, // Apple A6/A6X (ARMv7s). + kCYCLONE = 0x37A09642u, // Apple A7 (ARMv8.0-A). + kTYPHOON = 0x2C91A47Eu, // Apple A8 (ARMv8.0-A). + kTWISTER = 0x92FB37C8u, // Apple A9 (ARMv8.0-A). + kHURRICANE = 0x67CEEE93u, // Apple A10 (ARMv8.1-A). + kMONSOON_MISTRAL = 0xE81E7EF6u, // Apple A11 (ARMv8.2-A). + kVORTEX_TEMPEST = 0x07D34B9Fu, // Apple A12 (ARMv8.3-A). + kLIGHTNING_THUNDER = 0x462504D2u, // Apple A13 (ARMv8.4-A). + kFIRESTORM_ICESTORM = 0x1B588BB3u, // Apple A14/M1 (ARMv8.5-A). + kAVALANCHE_BLIZZARD = 0XDA33D83Du, // Apple A15/M2 (ARMv8.6-A). + kEVEREST_SAWTOOTH = 0X8765EDEAu, // Apple A16 (ARMv8.6-A). + kIBIZA = 0xFA33415Eu, // Apple M3 (ARMv8.6-A). + kPALMA = 0x72015832u, // Apple M3 Max (ARMv8.6-A). + kLOBOS = 0x5F4DEA93u, // Apple M3 Pro (ARMv8.6-A). + kCOLL = 0x2876F5B5u, // Apple A17 Pro (ARMv8.7-A). + kDONAN = 0x6F5129ACu, // Apple M4 (ARMv8.7-A). + kBRAVA = 0x17D5B93Au, // Apple M4 Max (ARMv8.7-A). + kTUPAI = 0x204526D0u, // Apple A18 . + kTAHITI = 0x75D4ACB9u // Apple A18 Pro . }; [[maybe_unused]] @@ -1153,55 +1176,93 @@ static ASMJIT_FAVOR_SIZE bool detectARMFeaturesViaAppleFamilyId(CpuInfo& cpu) no case uint32_t(Id::kTYPHOON): case uint32_t(Id::kTWISTER): populateARMv8AFeatures(features, 0); - features.add(Ext::kAES, Ext::kPMU, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256); + features.add( + Ext::kPMU, + Ext::kAES, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256 + ); return true; // Apple A10 (ARMv8.0-A). case uint32_t(Id::kHURRICANE): populateARMv8AFeatures(features, 0); - features.add(Ext::kAES, Ext::kCRC32, Ext::kLOR, Ext::kPAN, Ext::kPMU, Ext::kPMULL, Ext::kRDM, Ext::kSHA1, - Ext::kSHA256, Ext::kVHE); + features.add( + Ext::kLOR, Ext::kPAN, Ext::kPMU, Ext::kVHE, + Ext::kAES, Ext::kCRC32, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256, Ext::kRDM + ); return true; // Apple A11 (ARMv8.2-A). case uint32_t(Id::kMONSOON_MISTRAL): populateARMv8AFeatures(features, 2); - features.add(Ext::kAES, Ext::kFP16, Ext::kFP16CONV, Ext::kPMU, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256); + features.add( + Ext::kPMU, + Ext::kAES, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256, + Ext::kFP16, Ext::kFP16CONV + ); return true; // Apple A12 (ARMv8.3-A). case uint32_t(Id::kVORTEX_TEMPEST): populateARMv8AFeatures(features, 3); - features.add(Ext::kAES, Ext::kFP16, Ext::kFP16CONV, Ext::kPMU, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256); + features.add( + Ext::kPMU, + Ext::kAES, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256, + Ext::kFP16, Ext::kFP16CONV + ); return true; // Apple A13 (ARMv8.4-A). case uint32_t(Id::kLIGHTNING_THUNDER): populateARMv8AFeatures(features, 4); - features.add(Ext::kAES, Ext::kFHM, Ext::kFP16, Ext::kFP16CONV, Ext::kPMU, Ext::kPMULL, Ext::kSHA1, - Ext::kSHA256, Ext::kSHA3, Ext::kSHA512); + features.add( + Ext::kPMU, + Ext::kFP16, Ext::kFP16CONV, Ext::kFHM, + Ext::kAES, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256, Ext::kSHA3, Ext::kSHA512 + ); return true; // Apple A14/M1 (ARMv8.5-A). case uint32_t(Id::kFIRESTORM_ICESTORM): populateARMv8AFeatures(features, 4); - features.add(Ext::kAES, Ext::kCSV2, Ext::kCSV3, Ext::kDPB2, Ext::kECV, Ext::kFHM, Ext::kFLAGM2, - Ext::kFP16, Ext::kFP16CONV, Ext::kFRINTTS, Ext::kPMU, Ext::kPMULL, Ext::kSB, - Ext::kSHA1, Ext::kSHA256, Ext::kSHA3, Ext::kSHA512, Ext::kSSBS); + features.add( + Ext::kCSV2, Ext::kCSV3, Ext::kDPB2, Ext::kECV, Ext::kFLAGM2, Ext::kPMU, Ext::kSB, Ext::kSSBS, + Ext::kFP16, Ext::kFP16CONV, Ext::kFHM, + Ext::kFRINTTS, + Ext::kAES, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256, Ext::kSHA3, Ext::kSHA512); return true; // Apple A15/M2. case uint32_t(Id::kAVALANCHE_BLIZZARD): populateARMv8AFeatures(features, 6); - features.add(Ext::kAES, Ext::kFHM, Ext::kFP16, Ext::kFP16CONV, Ext::kPMU, Ext::kPMULL, Ext::kSHA1, - Ext::kSHA256, Ext::kSHA3, Ext::kSHA512); + features.add( + Ext::kPMU, + Ext::kFP16, Ext::kFP16CONV, Ext::kFHM, + Ext::kAES, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256, Ext::kSHA3, Ext::kSHA512); return true; - // Apple A16. + // Apple A16/M3. case uint32_t(Id::kEVEREST_SAWTOOTH): + case uint32_t(Id::kIBIZA): + case uint32_t(Id::kPALMA): + case uint32_t(Id::kLOBOS): populateARMv8AFeatures(features, 6); - features.add(Ext::kAES, Ext::kFHM, Ext::kFP16, Ext::kFP16CONV, Ext::kHCX, Ext::kPMU, Ext::kPMULL, - Ext::kSHA1, Ext::kSHA256, Ext::kSHA3, Ext::kSHA512); + features.add( + Ext::kHCX, Ext::kPMU, + Ext::kFP16, Ext::kFP16CONV, Ext::kFHM, + Ext::kAES, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256, Ext::kSHA3, Ext::kSHA512 + ); + return true; + + // Apple A17/M4. + case uint32_t(Id::kCOLL): + case uint32_t(Id::kDONAN): + case uint32_t(Id::kBRAVA): + populateARMv8AFeatures(features, 7); + features.add( + Ext::kPMU, + Ext::kFP16, Ext::kFP16CONV, Ext::kFHM, + Ext::kAES, Ext::kSHA1, Ext::kSHA3, Ext::kSHA256, Ext::kSHA512, + Ext::kSME, Ext::kSME2, Ext::kSME_F64F64, Ext::kSME_I16I64); return true; default: @@ -1625,48 +1686,96 @@ static void getAuxValues(unsigned long* vals, const unsigned long* tags, size_t #error "[asmjit] getAuxValues() - Unsupported OS." #endif -struct HWCapMapping { - uint8_t featureId; - uint8_t hwCapBit; -}; - static const unsigned long hwCapTags[2] = { AT_HWCAP, AT_HWCAP2 }; -static ASMJIT_FAVOR_SIZE void mergeHWCaps(CpuInfo& cpu, unsigned long mask, const HWCapMapping* mapping, size_t size) noexcept { - for (size_t i = 0; i < size; i++) { - cpu.features().addIf(Support::bitTest(mask, mapping[i].hwCapBit), mapping[i].featureId); +struct HWCapMapping32 { uint8_t featureId[32]; }; +struct HWCapMapping64 { uint8_t featureId[64]; }; + +template +static ASMJIT_FAVOR_SIZE void mergeHWCaps(CpuInfo& cpu, const Mask& mask, const Map& map) noexcept { + static_assert(sizeof(Mask) * 8u == sizeof(Map)); + + Support::BitWordIterator it(mask); + while (it.hasNext()) { + uint32_t featureId = map.featureId[it.next()]; + cpu.features().add(featureId); } + cpu.features().remove(0xFFu); } #if ASMJIT_ARCH_ARM == 32 // Reference: // - https://github.com/torvalds/linux/blob/master/arch/arm/include/uapi/asm/hwcap.h -static const HWCapMapping hwCapMapping[] = { - { uint8_t(Ext::kEDSP) , 7 }, // HWCAP_EDSP - { uint8_t(Ext::kASIMD) , 12 }, // HWCAP_NEON - { uint8_t(Ext::kFP) , 13 }, // HWCAP_VFPv3 - { uint8_t(Ext::kFMAC) , 16 }, // HWCAP_VFPv4 - { uint8_t(Ext::kIDIVA) , 17 }, // HWCAP_IDIVA - { uint8_t(Ext::kIDIVT) , 18 }, // HWCAP_IDIVT - { uint8_t(Ext::kVFP_D32) , 19 }, // HWCAP_VFPD32 - { uint8_t(Ext::kFP16CONV) , 22 }, // HWCAP_FPHP - { uint8_t(Ext::kFP16) , 23 }, // HWCAP_ASIMDHP - { uint8_t(Ext::kDOTPROD) , 24 }, // HWCAP_ASIMDDP - { uint8_t(Ext::kFHM) , 25 }, // HWCAP_ASIMDFHM - { uint8_t(Ext::kBF16) , 26 }, // HWCAP_ASIMDBF16 - { uint8_t(Ext::kI8MM) , 27 } // HWCAP_I8MM -}; +static constexpr HWCapMapping32 hwCap1Mapping = {{ + uint8_t(0xFF) , // [ 0] + uint8_t(0xFF) , // [ 1] + uint8_t(0xFF) , // [ 2] + uint8_t(0xFF) , // [ 3] + uint8_t(0xFF) , // [ 4] + uint8_t(0xFF) , // [ 5] + uint8_t(0xFF) , // [ 6] + uint8_t(Ext::kEDSP) , // [ 7] HWCAP_EDSP + uint8_t(0xFF) , // [ 8] + uint8_t(0xFF) , // [ 9] + uint8_t(0xFF) , // [10] + uint8_t(0xFF) , // [11] + uint8_t(Ext::kASIMD) , // [12] HWCAP_NEON + uint8_t(Ext::kFP) , // [13] HWCAP_VFPv3 + uint8_t(0xFF) , // [14] + uint8_t(0xFF) , // [15] + uint8_t(Ext::kFMAC) , // [16] HWCAP_VFPv4 + uint8_t(Ext::kIDIVA) , // [17] HWCAP_IDIVA + uint8_t(Ext::kIDIVT) , // [18] HWCAP_IDIVT + uint8_t(Ext::kVFP_D32) , // [19] HWCAP_VFPD32 + uint8_t(0xFF) , // [20] + uint8_t(0xFF) , // [21] + uint8_t(Ext::kFP16CONV) , // [22] HWCAP_FPHP + uint8_t(Ext::kFP16) , // [23] HWCAP_ASIMDHP + uint8_t(Ext::kDOTPROD) , // [24] HWCAP_ASIMDDP + uint8_t(Ext::kFHM) , // [25] HWCAP_ASIMDFHM + uint8_t(Ext::kBF16) , // [26] HWCAP_ASIMDBF16 + uint8_t(Ext::kI8MM) , // [27] HWCAP_I8MM + uint8_t(0xFF) , // [28] + uint8_t(0xFF) , // [29] + uint8_t(0xFF) , // [30] + uint8_t(0xFF) // [31] +}}; -static const HWCapMapping hwCap2Mapping[] = { - { uint8_t(Ext::kAES) , 0 }, // HWCAP2_AES - { uint8_t(Ext::kPMULL) , 1 }, // HWCAP2_PMULL - { uint8_t(Ext::kSHA1) , 2 }, // HWCAP2_SHA1 - { uint8_t(Ext::kSHA256) , 3 }, // HWCAP2_SHA2 - { uint8_t(Ext::kCRC32) , 4 }, // HWCAP2_CRC32 - { uint8_t(Ext::kSB) , 5 }, // HWCAP2_SB - { uint8_t(Ext::kSSBS) , 6 } // HWCAP2_SSBS -}; +static constexpr HWCapMapping32 hwCap2Mapping = {{ + uint8_t(Ext::kAES) , // [ 0] HWCAP2_AES + uint8_t(Ext::kPMULL) , // [ 1] HWCAP2_PMULL + uint8_t(Ext::kSHA1) , // [ 2] HWCAP2_SHA1 + uint8_t(Ext::kSHA256) , // [ 3] HWCAP2_SHA2 + uint8_t(Ext::kCRC32) , // [ 4] HWCAP2_CRC32 + uint8_t(Ext::kSB) , // [ 5] HWCAP2_SB + uint8_t(Ext::kSSBS) , // [ 6] HWCAP2_SSBS + uint8_t(0xFF) , // [ 7] + uint8_t(0xFF) , // [ 8] + uint8_t(0xFF) , // [ 9] + uint8_t(0xFF) , // [10] + uint8_t(0xFF) , // [11] + uint8_t(0xFF) , // [12] + uint8_t(0xFF) , // [13] + uint8_t(0xFF) , // [14] + uint8_t(0xFF) , // [15] + uint8_t(0xFF) , // [16] + uint8_t(0xFF) , // [17] + uint8_t(0xFF) , // [18] + uint8_t(0xFF) , // [19] + uint8_t(0xFF) , // [20] + uint8_t(0xFF) , // [21] + uint8_t(0xFF) , // [22] + uint8_t(0xFF) , // [23] + uint8_t(0xFF) , // [24] + uint8_t(0xFF) , // [25] + uint8_t(0xFF) , // [26] + uint8_t(0xFF) , // [27] + uint8_t(0xFF) , // [28] + uint8_t(0xFF) , // [29] + uint8_t(0xFF) , // [30] + uint8_t(0xFF) // [31] +}}; static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { cpu._wasDetected = true; @@ -1675,18 +1784,20 @@ static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { unsigned long hwCapMasks[2] {}; getAuxValues(hwCapMasks, hwCapTags, 2u); - mergeHWCaps(cpu, hwCapMasks[0], hwCapMapping, ASMJIT_ARRAY_SIZE(hwCapMapping)); - mergeHWCaps(cpu, hwCapMasks[1], hwCap2Mapping, ASMJIT_ARRAY_SIZE(hwCap2Mapping)); + mergeHWCaps(cpu, hwCapMasks[0], hwCap1Mapping); + mergeHWCaps(cpu, hwCapMasks[1], hwCap2Mapping); CpuFeatures::ARM& features = cpu.features().arm(); // ARMv7 provides FP|ASIMD. - if (features.hasFP() || features.hasASIMD()) + if (features.hasFP() || features.hasASIMD()) { features.add(CpuFeatures::ARM::kARMv7); + } // ARMv8 provives AES, CRC32, PMULL, SHA1, and SHA256. - if (features.hasAES() || features.hasCRC32() || features.hasPMULL() || features.hasSHA1() || features.hasSHA256()) + if (features.hasAES() || features.hasCRC32() || features.hasPMULL() || features.hasSHA1() || features.hasSHA256()) { features.add(CpuFeatures::ARM::kARMv8a); + } postProcessARMCpuInfo(cpu); } @@ -1696,95 +1807,139 @@ static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { // Reference: // - https://docs.kernel.org/arch/arm64/elf_hwcaps.html // - https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/hwcap.h -static const HWCapMapping hwCapMapping[] = { - { uint8_t(Ext::kFP) , 0 }, // HWCAP_FP - { uint8_t(Ext::kASIMD) , 1 }, // HWCAP_ASIMD - /* - { uint8_t(Ext::k) , 2 }, // HWCAP_EVTSTRM - */ - { uint8_t(Ext::kAES) , 3 }, // HWCAP_AES - { uint8_t(Ext::kPMULL) , 4 }, // HWCAP_PMULL - { uint8_t(Ext::kSHA1) , 5 }, // HWCAP_SHA1 - { uint8_t(Ext::kSHA256) , 6 }, // HWCAP_SHA2 - { uint8_t(Ext::kCRC32) , 7 }, // HWCAP_CRC32 - { uint8_t(Ext::kLSE) , 8 }, // HWCAP_ATOMICS - { uint8_t(Ext::kFP16CONV) , 9 }, // HWCAP_FPHP - { uint8_t(Ext::kFP16) , 10 }, // HWCAP_ASIMDHP - { uint8_t(Ext::kCPUID) , 11 }, // HWCAP_CPUID - { uint8_t(Ext::kRDM) , 12 }, // HWCAP_ASIMDRDM - { uint8_t(Ext::kJSCVT) , 13 }, // HWCAP_JSCVT - { uint8_t(Ext::kFCMA) , 14 }, // HWCAP_FCMA - { uint8_t(Ext::kLRCPC) , 15 }, // HWCAP_LRCPC - { uint8_t(Ext::kDPB) , 16 }, // HWCAP_DCPOP - { uint8_t(Ext::kSHA3) , 17 }, // HWCAP_SHA3 - { uint8_t(Ext::kSM3) , 18 }, // HWCAP_SM3 - { uint8_t(Ext::kSM4) , 19 }, // HWCAP_SM4 - { uint8_t(Ext::kDOTPROD) , 20 }, // HWCAP_ASIMDDP - { uint8_t(Ext::kSHA512) , 21 }, // HWCAP_SHA512 - { uint8_t(Ext::kSVE) , 22 }, // HWCAP_SVE - { uint8_t(Ext::kFHM) , 23 }, // HWCAP_ASIMDFHM - { uint8_t(Ext::kDIT) , 24 }, // HWCAP_DIT - { uint8_t(Ext::kLSE2) , 25 }, // HWCAP_USCAT - { uint8_t(Ext::kLRCPC2) , 26 }, // HWCAP_ILRCPC - { uint8_t(Ext::kFLAGM) , 27 }, // HWCAP_FLAGM - { uint8_t(Ext::kSSBS) , 28 }, // HWCAP_SSBS - { uint8_t(Ext::kSB) , 29 } // HWCAP_SB - /* - { uint8_t(Ext::k) , 30 }, // HWCAP_PACA - { uint8_t(Ext::k) , 31 } // HWCAP_PACG - */ -}; +static constexpr HWCapMapping64 hwCap1Mapping = {{ + uint8_t(Ext::kFP) , // [ 0] HWCAP_FP + uint8_t(Ext::kASIMD) , // [ 1] HWCAP_ASIMD + uint8_t(0xFF) , // [ 2] HWCAP_EVTSTRM + uint8_t(Ext::kAES) , // [ 3] HWCAP_AES + uint8_t(Ext::kPMULL) , // [ 4] HWCAP_PMULL + uint8_t(Ext::kSHA1) , // [ 5] HWCAP_SHA1 + uint8_t(Ext::kSHA256) , // [ 6] HWCAP_SHA2 + uint8_t(Ext::kCRC32) , // [ 7] HWCAP_CRC32 + uint8_t(Ext::kLSE) , // [ 8] HWCAP_ATOMICS + uint8_t(Ext::kFP16CONV) , // [ 9] HWCAP_FPHP + uint8_t(Ext::kFP16) , // [10] HWCAP_ASIMDHP + uint8_t(Ext::kCPUID) , // [11] HWCAP_CPUID + uint8_t(Ext::kRDM) , // [12] HWCAP_ASIMDRDM + uint8_t(Ext::kJSCVT) , // [13] HWCAP_JSCVT + uint8_t(Ext::kFCMA) , // [14] HWCAP_FCMA + uint8_t(Ext::kLRCPC) , // [15] HWCAP_LRCPC + uint8_t(Ext::kDPB) , // [16] HWCAP_DCPOP + uint8_t(Ext::kSHA3) , // [17] HWCAP_SHA3 + uint8_t(Ext::kSM3) , // [18] HWCAP_SM3 + uint8_t(Ext::kSM4) , // [19] HWCAP_SM4 + uint8_t(Ext::kDOTPROD) , // [20] HWCAP_ASIMDDP + uint8_t(Ext::kSHA512) , // [21] HWCAP_SHA512 + uint8_t(Ext::kSVE) , // [22] HWCAP_SVE + uint8_t(Ext::kFHM) , // [23] HWCAP_ASIMDFHM + uint8_t(Ext::kDIT) , // [24] HWCAP_DIT + uint8_t(Ext::kLSE2) , // [25] HWCAP_USCAT + uint8_t(Ext::kLRCPC2) , // [26] HWCAP_ILRCPC + uint8_t(Ext::kFLAGM) , // [27] HWCAP_FLAGM + uint8_t(Ext::kSSBS) , // [28] HWCAP_SSBS + uint8_t(Ext::kSB) , // [29] HWCAP_SB + uint8_t(0xFF) , // [30] HWCAP_PACA + uint8_t(0xFF) , // [31] HWCAP_PACG + uint8_t(Ext::kGCS) , // [32] HWCAP_GCS + uint8_t(Ext::kCMPBR) , // [33] HWCAP_CMPBR + uint8_t(Ext::kFPRCVT) , // [34] HWCAP_FPRCVT + uint8_t(Ext::kF8F32MM) , // [35] HWCAP_F8MM8 + uint8_t(Ext::kF8F16MM) , // [36] HWCAP_F8MM4 + uint8_t(Ext::kSVE_F16MM) , // [37] HWCAP_SVE_F16MM + uint8_t(Ext::kSVE_ELTPERM) , // [38] HWCAP_SVE_ELTPERM + uint8_t(Ext::kSVE_AES2) , // [39] HWCAP_SVE_AES2 + uint8_t(Ext::kSVE_BFSCALE) , // [40] HWCAP_SVE_BFSCALE + uint8_t(Ext::kSVE2_2) , // [41] HWCAP_SVE2P2 + uint8_t(Ext::kSME2_2) , // [42] HWCAP_SME2P2 + uint8_t(Ext::kSSVE_BITPERM) , // [43] HWCAP_SME_SBITPERM + uint8_t(Ext::kSME_AES) , // [44] HWCAP_SME_AES + uint8_t(Ext::kSSVE_FEXPA) , // [45] HWCAP_SME_SFEXPA + uint8_t(Ext::kSME_TMOP) , // [46] HWCAP_SME_STMOP + uint8_t(Ext::kSME_MOP4) , // [47] HWCAP_SME_SMOP4 + uint8_t(0xFF) , // [48] + uint8_t(0xFF) , // [49] + uint8_t(0xFF) , // [50] + uint8_t(0xFF) , // [51] + uint8_t(0xFF) , // [52] + uint8_t(0xFF) , // [53] + uint8_t(0xFF) , // [54] + uint8_t(0xFF) , // [55] + uint8_t(0xFF) , // [56] + uint8_t(0xFF) , // [57] + uint8_t(0xFF) , // [58] + uint8_t(0xFF) , // [59] + uint8_t(0xFF) , // [60] + uint8_t(0xFF) , // [61] + uint8_t(0xFF) , // [62] + uint8_t(0xFF) // [63] +}}; -static const HWCapMapping hwCap2Mapping[] = { - { uint8_t(Ext::kDPB2) , 0 }, // HWCAP2_DCPODP - { uint8_t(Ext::kSVE2) , 1 }, // HWCAP2_SVE2 - { uint8_t(Ext::kSVE_AES) , 2 }, // HWCAP2_SVEAES - { uint8_t(Ext::kSVE_PMULL128) , 3 }, // HWCAP2_SVEPMULL - { uint8_t(Ext::kSVE_BITPERM) , 4 }, // HWCAP2_SVEBITPERM - { uint8_t(Ext::kSVE_SHA3) , 5 }, // HWCAP2_SVESHA3 - { uint8_t(Ext::kSVE_SM4) , 6 }, // HWCAP2_SVESM4 - { uint8_t(Ext::kFLAGM2) , 7 }, // HWCAP2_FLAGM2 - { uint8_t(Ext::kFRINTTS) , 8 }, // HWCAP2_FRINT - { uint8_t(Ext::kSVE_I8MM) , 9 }, // HWCAP2_SVEI8MM - { uint8_t(Ext::kSVE_F32MM) , 10 }, // HWCAP2_SVEF32MM - { uint8_t(Ext::kSVE_F64MM) , 11 }, // HWCAP2_SVEF64MM - { uint8_t(Ext::kSVE_BF16) , 12 }, // HWCAP2_SVEBF16 - { uint8_t(Ext::kI8MM) , 13 }, // HWCAP2_I8MM - { uint8_t(Ext::kBF16) , 14 }, // HWCAP2_BF16 - { uint8_t(Ext::kDGH) , 15 }, // HWCAP2_DGH - { uint8_t(Ext::kRNG) , 16 }, // HWCAP2_RNG - { uint8_t(Ext::kBTI) , 17 }, // HWCAP2_BTI - { uint8_t(Ext::kMTE) , 18 }, // HWCAP2_MTE - { uint8_t(Ext::kECV) , 19 }, // HWCAP2_ECV - { uint8_t(Ext::kAFP) , 20 }, // HWCAP2_AFP - { uint8_t(Ext::kRPRES) , 21 }, // HWCAP2_RPRES - { uint8_t(Ext::kMTE3) , 22 }, // HWCAP2_MTE3 - { uint8_t(Ext::kSME) , 23 }, // HWCAP2_SME - { uint8_t(Ext::kSME_I16I64) , 24 }, // HWCAP2_SME_I16I64 - { uint8_t(Ext::kSME_F64F64) , 25 }, // HWCAP2_SME_F64F64 - { uint8_t(Ext::kSME_I8I32) , 26 }, // HWCAP2_SME_I8I32 - { uint8_t(Ext::kSME_F16F32) , 27 }, // HWCAP2_SME_F16F32 - { uint8_t(Ext::kSME_B16F32) , 28 }, // HWCAP2_SME_B16F32 - { uint8_t(Ext::kSME_F32F32) , 29 }, // HWCAP2_SME_F32F32 - { uint8_t(Ext::kSME_FA64) , 30 }, // HWCAP2_SME_FA64 - { uint8_t(Ext::kWFXT) , 31 }, // HWCAP2_WFXT - { uint8_t(Ext::kEBF16) , 32 }, // HWCAP2_EBF16 - { uint8_t(Ext::kSVE_EBF16) , 33 }, // HWCAP2_SVE_EBF16 - { uint8_t(Ext::kCSSC) , 34 }, // HWCAP2_CSSC - { uint8_t(Ext::kRPRFM) , 35 }, // HWCAP2_RPRFM - { uint8_t(Ext::kSVE2_1) , 36 }, // HWCAP2_SVE2P1 - { uint8_t(Ext::kSME2) , 37 }, // HWCAP2_SME2 - { uint8_t(Ext::kSME2_1) , 38 }, // HWCAP2_SME2P1 - { uint8_t(Ext::kSME_I16I32) , 39 }, // HWCAP2_SME_I16I32 - { uint8_t(Ext::kSME_BI32I32) , 40 }, // HWCAP2_SME_BI32I32 - { uint8_t(Ext::kSME_B16B16) , 41 }, // HWCAP2_SME_B16B16 - { uint8_t(Ext::kSME_F16F16) , 42 }, // HWCAP2_SME_F16F16 - { uint8_t(Ext::kMOPS) , 43 }, // HWCAP2_MOPS - { uint8_t(Ext::kHBC) , 44 }, // HWCAP2_HBC - { uint8_t(Ext::kSVE_B16B16) , 45 }, // HWCAP2_SVE_B16B16 - { uint8_t(Ext::kLRCPC3) , 46 }, // HWCAP2_LRCPC3 - { uint8_t(Ext::kLSE128) , 47 }, // HWCAP2_LSE128 -}; +static constexpr HWCapMapping64 hwCap2Mapping = {{ + uint8_t(Ext::kDPB2) , // [ 0] HWCAP2_DCPODP + uint8_t(Ext::kSVE2) , // [ 1] HWCAP2_SVE2 + uint8_t(Ext::kSVE_AES) , // [ 2] HWCAP2_SVEAES + uint8_t(Ext::kSVE_PMULL128) , // [ 3] HWCAP2_SVEPMULL + uint8_t(Ext::kSVE_BITPERM) , // [ 4] HWCAP2_SVEBITPERM + uint8_t(Ext::kSVE_SHA3) , // [ 5] HWCAP2_SVESHA3 + uint8_t(Ext::kSVE_SM4) , // [ 6] HWCAP2_SVESM4 + uint8_t(Ext::kFLAGM2) , // [ 7] HWCAP2_FLAGM2 + uint8_t(Ext::kFRINTTS) , // [ 8] HWCAP2_FRINT + uint8_t(Ext::kSVE_I8MM) , // [ 9] HWCAP2_SVEI8MM + uint8_t(Ext::kSVE_F32MM) , // [10] HWCAP2_SVEF32MM + uint8_t(Ext::kSVE_F64MM) , // [11] HWCAP2_SVEF64MM + uint8_t(Ext::kSVE_BF16) , // [12] HWCAP2_SVEBF16 + uint8_t(Ext::kI8MM) , // [13] HWCAP2_I8MM + uint8_t(Ext::kBF16) , // [14] HWCAP2_BF16 + uint8_t(Ext::kDGH) , // [15] HWCAP2_DGH + uint8_t(Ext::kRNG) , // [16] HWCAP2_RNG + uint8_t(Ext::kBTI) , // [17] HWCAP2_BTI + uint8_t(Ext::kMTE) , // [18] HWCAP2_MTE + uint8_t(Ext::kECV) , // [19] HWCAP2_ECV + uint8_t(Ext::kAFP) , // [20] HWCAP2_AFP + uint8_t(Ext::kRPRES) , // [21] HWCAP2_RPRES + uint8_t(Ext::kMTE3) , // [22] HWCAP2_MTE3 + uint8_t(Ext::kSME) , // [23] HWCAP2_SME + uint8_t(Ext::kSME_I16I64) , // [24] HWCAP2_SME_I16I64 + uint8_t(Ext::kSME_F64F64) , // [25] HWCAP2_SME_F64F64 + uint8_t(Ext::kSME_I8I32) , // [26] HWCAP2_SME_I8I32 + uint8_t(Ext::kSME_F16F32) , // [27] HWCAP2_SME_F16F32 + uint8_t(Ext::kSME_B16F32) , // [28] HWCAP2_SME_B16F32 + uint8_t(Ext::kSME_F32F32) , // [29] HWCAP2_SME_F32F32 + uint8_t(Ext::kSME_FA64) , // [30] HWCAP2_SME_FA64 + uint8_t(Ext::kWFXT) , // [31] HWCAP2_WFXT + uint8_t(Ext::kEBF16) , // [32] HWCAP2_EBF16 + uint8_t(Ext::kSVE_EBF16) , // [33] HWCAP2_SVE_EBF16 + uint8_t(Ext::kCSSC) , // [34] HWCAP2_CSSC + uint8_t(Ext::kRPRFM) , // [35] HWCAP2_RPRFM + uint8_t(Ext::kSVE2_1) , // [36] HWCAP2_SVE2P1 + uint8_t(Ext::kSME2) , // [37] HWCAP2_SME2 + uint8_t(Ext::kSME2_1) , // [38] HWCAP2_SME2P1 + uint8_t(Ext::kSME_I16I32) , // [39] HWCAP2_SME_I16I32 + uint8_t(Ext::kSME_BI32I32) , // [40] HWCAP2_SME_BI32I32 + uint8_t(Ext::kSME_B16B16) , // [41] HWCAP2_SME_B16B16 + uint8_t(Ext::kSME_F16F16) , // [42] HWCAP2_SME_F16F16 + uint8_t(Ext::kMOPS) , // [43] HWCAP2_MOPS + uint8_t(Ext::kHBC) , // [44] HWCAP2_HBC + uint8_t(Ext::kSVE_B16B16) , // [45] HWCAP2_SVE_B16B16 + uint8_t(Ext::kLRCPC3) , // [46] HWCAP2_LRCPC3 + uint8_t(Ext::kLSE128) , // [47] HWCAP2_LSE128 + uint8_t(Ext::kFPMR) , // [48] HWCAP2_FPMR + uint8_t(Ext::kLUT) , // [49] HWCAP2_LUT + uint8_t(Ext::kFAMINMAX) , // [50] HWCAP2_FAMINMAX + uint8_t(Ext::kFP8) , // [51] HWCAP2_F8CVT + uint8_t(Ext::kFP8FMA) , // [52] HWCAP2_F8FMA + uint8_t(Ext::kFP8DOT4) , // [53] HWCAP2_F8DP4 + uint8_t(Ext::kFP8DOT2) , // [54] HWCAP2_F8DP2 + uint8_t(Ext::kF8E4M3) , // [55] HWCAP2_F8E4M3 + uint8_t(Ext::kF8E5M2) , // [56] HWCAP2_F8E5M2 + uint8_t(Ext::kSME_LUTv2) , // [57] HWCAP2_SME_LUTV2 + uint8_t(Ext::kSME_F8F16) , // [58] HWCAP2_SME_F8F16 + uint8_t(Ext::kSME_F8F32) , // [59] HWCAP2_SME_F8F32 + uint8_t(Ext::kSSVE_FP8FMA) , // [60] HWCAP2_SME_SF8FMA + uint8_t(Ext::kSSVE_FP8DOT4) , // [61] HWCAP2_SME_SF8DP4 + uint8_t(Ext::kSSVE_FP8DOT2) , // [62] HWCAP2_SME_SF8DP2 + uint8_t(0xFF) // [63] HWCAP2_POE +}}; static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { cpu._wasDetected = true; @@ -1793,8 +1948,8 @@ static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { unsigned long hwCapMasks[2] {}; getAuxValues(hwCapMasks, hwCapTags, 2u); - mergeHWCaps(cpu, hwCapMasks[0], hwCapMapping, ASMJIT_ARRAY_SIZE(hwCapMapping)); - mergeHWCaps(cpu, hwCapMasks[1], hwCap2Mapping, ASMJIT_ARRAY_SIZE(hwCap2Mapping)); + mergeHWCaps(cpu, hwCapMasks[0], hwCap1Mapping); + mergeHWCaps(cpu, hwCapMasks[1], hwCap2Mapping); #if defined(ASMJIT_ARM_DETECT_VIA_CPUID) if (cpu.features().arm().hasCPUID()) { diff --git a/src/asmjit/core/cpuinfo.h b/src/asmjit/core/cpuinfo.h index a0eabaf..0ad1411 100644 --- a/src/asmjit/core/cpuinfo.h +++ b/src/asmjit/core/cpuinfo.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_CPUINFO_H_INCLUDED @@ -560,6 +560,7 @@ public: kCHK, //!< CPU has CHK (check feature status - CHKFEAT instruction) {A64}. kCLRBHB, //!< CPU has CLRBHB (clear BHB instruction). kCMOW, //!< CPU has CMOW (control for cache maintenance permission) {A64}. + kCMPBR, //!< CPU has CMPBR (Compare and branch instructions) {A64}. kCONSTPACFIELD, //!< CPU has CONSTPACFIELD (PAC algorithm enhancement) {A64}. kCPA, //!< CPU has CPA (instruction-only Checked Pointer Arithmetic) {A64}. kCPA2, //!< CPU has CPA2 (checked Pointer Arithmetic) {A64}. @@ -581,6 +582,10 @@ public: kECV, //!< CPU has ECV (enhanced counter virtualization). kEDHSR, //!< CPU has EDHSR (support for EDHSR) {A64}. kEDSP, //!< CPU has EDSP (ARM/THUMB only). + kF8E4M3, //!< CPU has F8E4M3 {A64}. + kF8E5M2, //!< CPU has F8E5M2 {A64}. + kF8F16MM, //!< CPU has F8F16MM (8-bit floating-point matrix multiply-accumulate to half-precision) {A64} + kF8F32MM, //!< CPU has F8F32MM (8-bit floating-point matrix multiply-accumulate to single-precision) {A64} kFAMINMAX, //!< CPU has FAMINMAX (floating-point maximum and minimum absolute value instructions) {A64}. kFCMA, //!< CPU has FCMA (FCADD/FCMLA). kFGT, //!< CPU has FGT (fine-grained traps). @@ -597,6 +602,7 @@ public: kFP8DOT4, //!< CPU has FP8DOT4 (FP8 4-way dot product to single-precision instructions) {A64}. kFP8FMA, //!< CPU has FP8FMA (FP8 multiply-accumulate to half-precision and single-precision instructions) {A64}. kFPMR, //!< CPU has FPMR (floating-point Mode Register) {A64}. + kFPRCVT, //!< CPU has FPRCVT (floating-point to/from integer in scalar FP register) {A64}. kFRINTTS, //!< CPU has FRINTTS (FRINT[32|64][X|Z] instructions) {A64}. kGCS, //!< CPU has GCS (guarded control stack extension) {A64}. kHACDBS, //!< CPU has HACDBS (hardware accelerator for cleaning Dirty state) {A64}. @@ -619,9 +625,12 @@ public: kLS64, //!< CPU has LS64 (64 byte loads/stores without return) {A64}. kLS64_ACCDATA, //!< CPU has LS64_ACCDATA (64-byte EL0 stores with return) {A64}. kLS64_V, //!< CPU has LS64_V (64-byte stores with return) {A64}. + kLS64WB, //!< CPU has LS64WB (LS64 for Write-back cacheable memory) {A64} kLSE, //!< CPU has LSE (large system extensions) {A64}. kLSE128, //!< CPU has LSE128 (128-bit atomics) {A64}. kLSE2, //!< CPU has LSE2 (large system extensions v2) {A64}. + kLSFE, //!< CPU has LSFE (large system float extension) {A64}. + kLSUI, //!< CPU has LSUI (unprivileged load store) {A64}. kLUT, //!< CPU has LUT (lookup table instructions with 2-bit and 4-bit indices) {A64}. kLVA, //!< CPU has LVA (large VA support) {A64}. kLVA3, //!< CPU has LVA3 (56-bit VA) {A64}. @@ -643,6 +652,7 @@ public: kNMI, //!< CPU has NMI (non-maskable Interrupt) {A64}. kNV, //!< CPU has NV (nested virtualization enchancement) {A64}. kNV2, //!< CPU has NV2 (enhanced support for nested virtualization) {A64}. + kOCCMO, //!< CPU has OCCMO (outer cacheable cache maintenance operation) {A64}. kPAN, //!< CPU has PAN (privileged access-never extension) {A64}. kPAN2, //!< CPU has PAN2 (PAN s1e1R and s1e1W variants) {A64}. kPAN3, //!< CPU has PAN3 (support for SCTLR_ELx.EPAN) {A64}. @@ -678,6 +688,8 @@ public: kSME, //!< CPU has SME (SME v1 - scalable matrix extension) {A64}. kSME2, //!< CPU has SME2 (SME v2) {A64}. kSME2_1, //!< CPU has SME2p1 (SME v2.1) {A64}. + kSME2_2, //!< CPU has SME2p1 (SME v2.2) {A64}. + kSME_AES, //!< CPU has SME_AES {A64}. kSME_B16B16, //!< CPU has SME_B16B16 (SME non-widening BFloat16 to BFloat16 arithmetic) {A64}. kSME_B16F32, //!< CPU has SME_B16F32 (BFMOPA and BFMOPS instructions that accumulate BFloat16 outer products into single-precision tiles) {A64}. kSME_BI32I32, //!< CPU has SME_BI32I32 (BMOPA and BMOPS instructions that accumulate 1-bit binary outer products into 32-bit integer tiles) {A64}. @@ -692,6 +704,8 @@ public: kSME_I16I64, //!< CPU has SME_I16I64 {A64}. kSME_I8I32, //!< CPU has SME_I8I32 {A64}. kSME_LUTv2, //!< CPU has SME_LUTv2 (lookup table instructions with 4-bit indices and 8-bit elements) {A64}. + kSME_MOP4, //!< CPU has SME_MOP4 (quarter-tile outer product instructions) {A64}. + kSME_TMOP, //!< CPU has SME_TMOP {A64}. kSPE, //!< CPU has SPE (statistical profiling extension) {A64}. kSPE1_1, //!< CPU has SPEv1p1 (statistical profiling extensions version 1.1) {A64}. kSPE1_2, //!< CPU has SPEv1p2 (statistical profiling extensions version 1.2) {A64}. @@ -708,17 +722,25 @@ public: kSPMU, //!< CPU has SPMU (system performance monitors extension) {A64}. kSSBS, //!< CPU has SSBS (speculative store bypass safe instruction). kSSBS2, //!< CPU has SSBS2 (MRS and MSR instructions for SSBS). + kSSVE_AES, //!< CPU has SSVE_AES {A64}. + kSSVE_BITPERM, //!< CPU has SSVE_BITPERM {A64}. + kSSVE_FEXPA, //!< CPU has SSVE_FEXPA {A64}. kSSVE_FP8DOT2, //!< CPU has SSVE_FP8DOT2 (SVE2 FP8 2-way dot product to half-precision instructions in Streaming SVE mode) {A64}. kSSVE_FP8DOT4, //!< CPU has SSVE_FP8DOT4 (SVE2 FP8 4-way dot product to single-precision instructions in Streaming SVE mode) {A64}. kSSVE_FP8FMA, //!< CPU has SSVE_FP8FMA (SVE2 FP8 multiply-accumulate to half-precision and single-precision instructions in Streaming SVE mode) {A64}. kSVE, //!< CPU has SVE (SVE v1 - scalable vector extension) {A64}. kSVE2, //!< CPU has SVE2 (SVE v2) {A64}. kSVE2_1, //!< CPU has SVE2p1 (SVE v2.1) {A64}. + kSVE2_2, //!< CPU has SVE2p2 (SVE v2.2) {A64}. kSVE_AES, //!< CPU has SVE_AES (SVE AES instructions) {A64}. + kSVE_AES2, //!< CPU has SVE_AES2 {A64}. kSVE_B16B16, //!< CPU has SVE_B16B16 (SVE non-widening BFloat16 to BFloat16 arithmetic) {A64}. kSVE_BF16, //!< CPU has SVE_BF16 (SVE BF16 instructions) {A64}. + kSVE_BFSCALE, //!< CPU has SVE_BFSCALE {A64}. kSVE_BITPERM, //!< CPU has SVE_BITPERM (SVE bit permute) {A64}. kSVE_EBF16, //!< CPU has SVE_EBF16 (SVE extended BFloat16 mode) {A64}. + kSVE_ELTPERM, //!< CPU has SVE_ELTPERM {A64}. + kSVE_F16MM, //!< CPU has SVE_F16MM (SVE half-precision floating-point matrix multiply instruction) {A64}. kSVE_F32MM, //!< CPU has SVE_F32MM (SVE single-precision floating-point matrix multiply instruction) {A64}. kSVE_F64MM, //!< CPU has SVE_F64MM (SVE double-precision floating-point matrix multiply instruction) {A64}. kSVE_I8MM, //!< CPU has SVE_I8MM (SVE int8 matrix multiplication) {A64}. @@ -773,6 +795,7 @@ public: ASMJIT_ARM_FEATURE(CHK) ASMJIT_ARM_FEATURE(CLRBHB) ASMJIT_ARM_FEATURE(CMOW) + ASMJIT_ARM_FEATURE(CMPBR) ASMJIT_ARM_FEATURE(CONSTPACFIELD) ASMJIT_ARM_FEATURE(CPA) ASMJIT_ARM_FEATURE(CPA2) @@ -794,6 +817,10 @@ public: ASMJIT_ARM_FEATURE(ECV) ASMJIT_ARM_FEATURE(EDHSR) ASMJIT_ARM_FEATURE(EDSP) + ASMJIT_ARM_FEATURE(F8E4M3) + ASMJIT_ARM_FEATURE(F8E5M2) + ASMJIT_ARM_FEATURE(F8F16MM) + ASMJIT_ARM_FEATURE(F8F32MM) ASMJIT_ARM_FEATURE(FAMINMAX) ASMJIT_ARM_FEATURE(FCMA) ASMJIT_ARM_FEATURE(FGT) @@ -810,6 +837,7 @@ public: ASMJIT_ARM_FEATURE(FP8DOT4) ASMJIT_ARM_FEATURE(FP8FMA) ASMJIT_ARM_FEATURE(FPMR) + ASMJIT_ARM_FEATURE(FPRCVT) ASMJIT_ARM_FEATURE(FRINTTS) ASMJIT_ARM_FEATURE(GCS) ASMJIT_ARM_FEATURE(HACDBS) @@ -832,9 +860,12 @@ public: ASMJIT_ARM_FEATURE(LS64) ASMJIT_ARM_FEATURE(LS64_ACCDATA) ASMJIT_ARM_FEATURE(LS64_V) + ASMJIT_ARM_FEATURE(LS64WB) ASMJIT_ARM_FEATURE(LSE) ASMJIT_ARM_FEATURE(LSE128) ASMJIT_ARM_FEATURE(LSE2) + ASMJIT_ARM_FEATURE(LSFE) + ASMJIT_ARM_FEATURE(LSUI) ASMJIT_ARM_FEATURE(LUT) ASMJIT_ARM_FEATURE(LVA) ASMJIT_ARM_FEATURE(LVA3) @@ -856,6 +887,7 @@ public: ASMJIT_ARM_FEATURE(NMI) ASMJIT_ARM_FEATURE(NV) ASMJIT_ARM_FEATURE(NV2) + ASMJIT_ARM_FEATURE(OCCMO) ASMJIT_ARM_FEATURE(PAN) ASMJIT_ARM_FEATURE(PAN2) ASMJIT_ARM_FEATURE(PAN3) @@ -891,6 +923,8 @@ public: ASMJIT_ARM_FEATURE(SME) ASMJIT_ARM_FEATURE(SME2) ASMJIT_ARM_FEATURE(SME2_1) + ASMJIT_ARM_FEATURE(SME2_2) + ASMJIT_ARM_FEATURE(SME_AES) ASMJIT_ARM_FEATURE(SME_B16B16) ASMJIT_ARM_FEATURE(SME_B16F32) ASMJIT_ARM_FEATURE(SME_BI32I32) @@ -905,6 +939,8 @@ public: ASMJIT_ARM_FEATURE(SME_I16I64) ASMJIT_ARM_FEATURE(SME_I8I32) ASMJIT_ARM_FEATURE(SME_LUTv2) + ASMJIT_ARM_FEATURE(SME_MOP4) + ASMJIT_ARM_FEATURE(SME_TMOP) ASMJIT_ARM_FEATURE(SPE) ASMJIT_ARM_FEATURE(SPE1_1) ASMJIT_ARM_FEATURE(SPE1_2) @@ -921,17 +957,25 @@ public: ASMJIT_ARM_FEATURE(SPMU) ASMJIT_ARM_FEATURE(SSBS) ASMJIT_ARM_FEATURE(SSBS2) + ASMJIT_ARM_FEATURE(SSVE_AES) + ASMJIT_ARM_FEATURE(SSVE_BITPERM) + ASMJIT_ARM_FEATURE(SSVE_FEXPA) ASMJIT_ARM_FEATURE(SSVE_FP8DOT2) ASMJIT_ARM_FEATURE(SSVE_FP8DOT4) ASMJIT_ARM_FEATURE(SSVE_FP8FMA) ASMJIT_ARM_FEATURE(SVE) ASMJIT_ARM_FEATURE(SVE2) ASMJIT_ARM_FEATURE(SVE2_1) + ASMJIT_ARM_FEATURE(SVE2_2) ASMJIT_ARM_FEATURE(SVE_AES) + ASMJIT_ARM_FEATURE(SVE_AES2) ASMJIT_ARM_FEATURE(SVE_B16B16) ASMJIT_ARM_FEATURE(SVE_BF16) + ASMJIT_ARM_FEATURE(SVE_BFSCALE) ASMJIT_ARM_FEATURE(SVE_BITPERM) ASMJIT_ARM_FEATURE(SVE_EBF16) + ASMJIT_ARM_FEATURE(SVE_ELTPERM) + ASMJIT_ARM_FEATURE(SVE_F16MM) ASMJIT_ARM_FEATURE(SVE_F32MM) ASMJIT_ARM_FEATURE(SVE_F64MM) ASMJIT_ARM_FEATURE(SVE_I8MM) diff --git a/src/asmjit/core/emithelper.cpp b/src/asmjit/core/emithelper.cpp index f534e33..51a1e6c 100644 --- a/src/asmjit/core/emithelper.cpp +++ b/src/asmjit/core/emithelper.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -75,12 +75,12 @@ Error BaseEmitHelper::emitRegMove(const Operand_& dst_, const Operand_& src_, Ty return DebugUtils::errored(kErrorInvalidState); } -Error BaseEmitHelper::emitRegSwap(const BaseReg& a, const BaseReg& b, const char* comment) { +Error BaseEmitHelper::emitRegSwap(const Reg& a, const Reg& b, const char* comment) { DebugUtils::unused(a, b, comment); return DebugUtils::errored(kErrorInvalidState); } -Error BaseEmitHelper::emitArgMove(const BaseReg& dst_, TypeId dstTypeId, const Operand_& src_, TypeId srcTypeId, const char* comment) { +Error BaseEmitHelper::emitArgMove(const Reg& dst_, TypeId dstTypeId, const Operand_& src_, TypeId srcTypeId, const char* comment) { DebugUtils::unused(dst_, dstTypeId, src_, srcTypeId, comment); return DebugUtils::errored(kErrorInvalidState); } @@ -120,8 +120,8 @@ ASMJIT_FAVOR_SIZE Error BaseEmitHelper::emitArgsAssignment(const FuncFrame& fram uint32_t varCount = ctx._varCount; uint32_t saVarId = ctx._saVarId; - BaseReg sp = BaseReg(_emitter->_gpSignature, archTraits.spRegId()); - BaseReg sa = sp; + Reg sp = Reg(_emitter->_gpSignature, archTraits.spRegId()); + Reg sa = sp; if (frame.hasDynamicAlignment()) { if (frame.hasPreservedFP()) { @@ -151,7 +151,7 @@ ASMJIT_FAVOR_SIZE Error BaseEmitHelper::emitArgsAssignment(const FuncFrame& fram FuncValue& out = var.out; ASMJIT_ASSERT(cur.isReg() || cur.isStack()); - BaseReg reg; + Reg reg; BaseMem dstStackPtr = baseStackPtr.cloneAdjusted(out.stackOffset()); BaseMem srcStackPtr = baseArgPtr.cloneAdjusted(cur.stackOffset()); @@ -167,10 +167,10 @@ ASMJIT_FAVOR_SIZE Error BaseEmitHelper::emitArgsAssignment(const FuncFrame& fram } if (cur.isReg() && !cur.isIndirect()) { - WorkData& wd = workData[archTraits.regTypeToGroup(cur.regType())]; + WorkData& wd = workData[RegUtils::groupOf(cur.regType())]; uint32_t regId = cur.regId(); - reg.setSignatureAndId(archTraits.regTypeToSignature(cur.regType()), regId); + reg.setSignatureAndId(RegUtils::signatureOf(cur.regType()), regId); wd.unassign(varId, regId); } else { @@ -217,8 +217,8 @@ ASMJIT_FAVOR_SIZE Error BaseEmitHelper::emitArgsAssignment(const FuncFrame& fram FuncValue& cur = var.cur; FuncValue& out = var.out; - RegGroup curGroup = archTraits.regTypeToGroup(cur.regType()); - RegGroup outGroup = archTraits.regTypeToGroup(out.regType()); + RegGroup curGroup = RegUtils::groupOf(cur.regType()); + RegGroup outGroup = RegUtils::groupOf(out.regType()); uint32_t curId = cur.regId(); uint32_t outId = out.regId(); @@ -233,8 +233,8 @@ ASMJIT_FAVOR_SIZE Error BaseEmitHelper::emitArgsAssignment(const FuncFrame& fram EmitMove: ASMJIT_PROPAGATE( emitArgMove( - BaseReg(archTraits.regTypeToSignature(out.regType()), outId), out.typeId(), - BaseReg(archTraits.regTypeToSignature(cur.regType()), curId), cur.typeId())); + Reg(RegUtils::signatureOf(out.regType()), outId), out.typeId(), + Reg(RegUtils::signatureOf(cur.regType()), curId), cur.typeId())); // Only reassign if this is not a sign/zero extension that happens on the same in/out register. if (curId != outId) { @@ -260,8 +260,8 @@ EmitMove: highestType = RegType::kGp32; } - OperandSignature signature = archTraits.regTypeToSignature(highestType); - ASMJIT_PROPAGATE(emitRegSwap(BaseReg(signature, outId), BaseReg(signature, curId))); + OperandSignature signature = RegUtils::signatureOf(highestType); + ASMJIT_PROPAGATE(emitRegSwap(Reg(signature, outId), Reg(signature, curId))); wd.swap(varId, curId, altId, outId); cur.setRegId(outId); @@ -333,7 +333,7 @@ EmitMove: uint32_t outId = var.out.regId(); RegType outType = var.out.regType(); - RegGroup group = archTraits.regTypeToGroup(outType); + RegGroup group = RegUtils::groupOf(outType); WorkData& wd = workData[group]; if (outId == sa.id() && group == RegGroup::kGp) { @@ -345,7 +345,7 @@ EmitMove: wd.unassign(wd._physToVarId[outId], outId); } - BaseReg dstReg = BaseReg(archTraits.regTypeToSignature(outType), outId); + Reg dstReg = Reg(RegUtils::signatureOf(outType), outId); BaseMem srcMem = baseArgPtr.cloneAdjusted(var.cur.stackOffset()); ASMJIT_PROPAGATE(emitArgMove( diff --git a/src/asmjit/core/emithelper_p.h b/src/asmjit/core/emithelper_p.h index 9d75550..32add43 100644 --- a/src/asmjit/core/emithelper_p.h +++ b/src/asmjit/core/emithelper_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_EMITHELPER_P_H_INCLUDED @@ -39,8 +39,8 @@ public: //! Emits swap between two registers. virtual Error emitRegSwap( - const BaseReg& a, - const BaseReg& b, const char* comment = nullptr); + const Reg& a, + const Reg& b, const char* comment = nullptr); //! Emits move from a function argument (either register or stack) to a register. //! @@ -48,7 +48,7 @@ public: //! to another, if it's possible. Any attempt of conversion that requires third register of a different group //! (for example conversion from K to MMX on X86/X64) will fail. virtual Error emitArgMove( - const BaseReg& dst_, TypeId dstTypeId, + const Reg& dst_, TypeId dstTypeId, const Operand_& src_, TypeId srcTypeId, const char* comment = nullptr); Error emitArgsAssignment(const FuncFrame& frame, const FuncArgsAssignment& args); diff --git a/src/asmjit/core/emitter.cpp b/src/asmjit/core/emitter.cpp index 01843b7..3f76507 100644 --- a/src/asmjit/core/emitter.cpp +++ b/src/asmjit/core/emitter.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -121,7 +121,9 @@ void BaseEmitter::setErrorHandler(ErrorHandler* errorHandler) noexcept { } } -Error BaseEmitter::reportError(Error err, const char* message) { +Error BaseEmitter::_reportError(Error err, const char* message) { + ASMJIT_ASSERT(err != kErrorOk); + ErrorHandler* eh = _errorHandler; if (eh) { if (!message) { @@ -129,6 +131,7 @@ Error BaseEmitter::reportError(Error err, const char* message) { } eh->handleError(err, message, this); } + return err; } @@ -370,20 +373,22 @@ Error BaseEmitter::commentv(const char* fmt, va_list ap) { // BaseEmitter - Events // ==================== -Error BaseEmitter::onAttach(CodeHolder* code) noexcept { - _code = code; - _environment = code->environment(); +Error BaseEmitter::onAttach(CodeHolder& code) noexcept { + _code = &code; + _environment = code.environment(); _addEmitterFlags(EmitterFlags::kAttached); - const ArchTraits& archTraits = ArchTraits::byArch(code->arch()); - RegType nativeRegType = Environment::is32Bit(code->arch()) ? RegType::kGp32 : RegType::kGp64; - _gpSignature = archTraits.regTypeToSignature(nativeRegType); + _gpSignature.setBits( + Environment::is32Bit(code.arch()) + ? RegTraits::kSignature + : RegTraits::kSignature + ); onSettingsUpdated(); return kErrorOk; } -Error BaseEmitter::onDetach(CodeHolder* code) noexcept { +Error BaseEmitter::onDetach(CodeHolder& code) noexcept { DebugUtils::unused(code); if (!hasOwnLogger()) { @@ -405,7 +410,17 @@ Error BaseEmitter::onDetach(CodeHolder* code) noexcept { _instOptions = InstOptions::kNone; _extraReg.reset(); _inlineComment = nullptr; - _funcs.reset(); + + return kErrorOk; +} + +Error BaseEmitter::onReinit(CodeHolder& code) noexcept { + ASMJIT_ASSERT(_code == &code); + DebugUtils::unused(code); + + _instOptions = InstOptions::kNone; + _extraReg.reset(); + _inlineComment = nullptr; return kErrorOk; } diff --git a/src/asmjit/core/emitter.h b/src/asmjit/core/emitter.h index f811e5f..93f6369 100644 --- a/src/asmjit/core/emitter.h +++ b/src/asmjit/core/emitter.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_EMITTER_H_INCLUDED @@ -186,7 +186,7 @@ enum class DiagnosticOptions : uint32_t { kRADebugUnreachable = 0x00000800u, //! Enable all debug options (Compiler/RA). - kRADebugAll = 0x0000FF00u, + kRADebugAll = 0x0000FF00u }; ASMJIT_DEFINE_ENUM_FLAGS(DiagnosticOptions) @@ -196,49 +196,9 @@ public: ASMJIT_BASE_CLASS(BaseEmitter) ASMJIT_NONCOPYABLE(BaseEmitter) - //! \name Members + //! \name Types //! \{ - //! See \ref EmitterType. - EmitterType _emitterType = EmitterType::kNone; - //! See \ref EmitterFlags. - EmitterFlags _emitterFlags = EmitterFlags::kNone; - //! Instruction alignment. - uint8_t _instructionAlignment = 0u; - //! \cond - uint8_t _reservedBaseEmitter = 0u; - //! \endcond - //! Validation flags in case validation is used. - //! - //! \note Validation flags are specific to the emitter and they are setup at construction time and then never - //! changed. - ValidationFlags _validationFlags = ValidationFlags::kNone; - //! Validation options. - DiagnosticOptions _diagnosticOptions = DiagnosticOptions::kNone; - - //! All supported architectures in a bit-mask, where LSB is the bit with a zero index. - uint64_t _archMask = 0; - - //! Encoding options. - EncodingOptions _encodingOptions = EncodingOptions::kNone; - - //! Forced instruction options, combined with \ref _instOptions by \ref emit(). - InstOptions _forcedInstOptions = InstOptions::kReserved; - //! Internal private data used freely by any emitter. - uint32_t _privateData = 0; - - //! CodeHolder the emitter is attached to. - CodeHolder* _code = nullptr; - //! Attached \ref Logger. - Logger* _logger = nullptr; - //! Attached \ref ErrorHandler. - ErrorHandler* _errorHandler = nullptr; - - //! Describes the target environment, matches \ref CodeHolder::environment(). - Environment _environment {}; - //! Native GP register signature and signature related information. - OperandSignature _gpSignature {}; - //! Emitter state that can be used to specify options and inline comment of a next node or instruction. struct State { InstOptions options; @@ -246,14 +206,7 @@ public: const char* comment; }; - //! Next instruction options (affects the next instruction). - InstOptions _instOptions = InstOptions::kNone; - //! Extra register (op-mask {k} on AVX-512) (affects the next instruction). - RegOnly _extraReg {}; - //! Inline comment of the next instruction (affects the next instruction). - const char* _inlineComment = nullptr; - - //! Function callbacks used by emitter implementation. + //! Functions used by backend-specific emitter implementation. //! //! These are typically shared between Assembler/Builder/Compiler of a single backend. struct Funcs { @@ -282,16 +235,73 @@ public: ValidateFunc validate; //! Resets all functions to nullptr. - ASMJIT_INLINE_NODEBUG void reset() noexcept { - emitProlog = nullptr; - emitEpilog = nullptr; - emitArgsAssignment = nullptr; - validate = nullptr; - } + ASMJIT_INLINE_NODEBUG void reset() noexcept { *this = Funcs{}; } }; + //! \} + + //! \name Members + //! \{ + + //! See \ref EmitterType. + EmitterType _emitterType = EmitterType::kNone; + + //! See \ref EmitterFlags. + EmitterFlags _emitterFlags = EmitterFlags::kNone; + + //! Instruction alignment. + uint8_t _instructionAlignment = 0u; + + //! Validation flags in case validation is used. + //! + //! \note Validation flags are specific to the emitter and they are setup at construction time and then never + //! changed. + ValidationFlags _validationFlags = ValidationFlags::kNone; + + //! Validation options. + DiagnosticOptions _diagnosticOptions = DiagnosticOptions::kNone; + + //! Encoding options. + EncodingOptions _encodingOptions = EncodingOptions::kNone; + + //! Forced instruction options, combined with \ref _instOptions by \ref emit(). + InstOptions _forcedInstOptions = InstOptions::kReserved; + + //! All supported architectures in a bit-mask, where LSB is the bit with a zero index. + uint64_t _archMask = 0; + + //! CodeHolder the emitter is attached to. + CodeHolder* _code = nullptr; + + //! Attached \ref Logger. + Logger* _logger = nullptr; + + //! Attached \ref ErrorHandler. + ErrorHandler* _errorHandler = nullptr; + + //! Describes the target environment, matches \ref CodeHolder::environment(). + Environment _environment {}; + + //! Native GP register signature (either a 32-bit or 64-bit GP register signature). + OperandSignature _gpSignature {}; + //! Internal private data used freely by any emitter. + uint32_t _privateData = 0; + + //! Next instruction options (affects the next instruction). + InstOptions _instOptions = InstOptions::kNone; + //! Extra register (op-mask {k} on AVX-512) (affects the next instruction). + RegOnly _extraReg {}; + //! Inline comment of the next instruction (affects the next instruction). + const char* _inlineComment = nullptr; + + //! Pointer to functions used by backend-specific emitter implementation. Funcs _funcs {}; + //! Emitter attached before this emitter in \ref CodeHolder, otherwise nullptr if there is no emitter before. + BaseEmitter* _attachedPrev = nullptr; + //! Emitter attached after this emitter in \ref CodeHolder, otherwise nullptr if there is no emitter after. + BaseEmitter* _attachedNext = nullptr; + //! \} //! \name Construction & Destruction @@ -493,11 +503,23 @@ public: //! Resets the error handler. ASMJIT_INLINE_NODEBUG void resetErrorHandler() noexcept { setErrorHandler(nullptr); } + //! \cond INTERNAL + ASMJIT_API Error _reportError(Error err, const char* message = nullptr); + //! \endcond + //! Handles the given error in the following way: //! 1. If the emitter has \ref ErrorHandler attached, it calls its \ref ErrorHandler::handleError() member function //! first, and then returns the error. The `handleError()` function may throw. //! 2. if the emitter doesn't have \ref ErrorHandler, the error is simply returned. - ASMJIT_API Error reportError(Error err, const char* message = nullptr); + ASMJIT_INLINE Error reportError(Error err, const char* message = nullptr) { + Error e = _reportError(err, message); + + // Static analysis is not working properly without these assumptions. + ASMJIT_ASSUME(e == err); + ASMJIT_ASSUME(e != kErrorOk); + + return e; + } //! \} @@ -590,7 +612,7 @@ public: ASMJIT_INLINE_NODEBUG const RegOnly& extraReg() const noexcept { return _extraReg; } //! Sets an extra operand that will be used by the next instruction (architecture specific). - ASMJIT_INLINE_NODEBUG void setExtraReg(const BaseReg& reg) noexcept { _extraReg.init(reg); } + ASMJIT_INLINE_NODEBUG void setExtraReg(const Reg& reg) noexcept { _extraReg.init(reg); } //! Sets an extra operand that will be used by the next instruction (architecture specific). ASMJIT_INLINE_NODEBUG void setExtraReg(const RegOnly& reg) noexcept { _extraReg.init(reg); } @@ -856,10 +878,13 @@ public: //! \{ //! Called after the emitter was attached to `CodeHolder`. - ASMJIT_API virtual Error onAttach(CodeHolder* ASMJIT_NONNULL(code)) noexcept; + ASMJIT_API virtual Error onAttach(CodeHolder& code) noexcept; //! Called after the emitter was detached from `CodeHolder`. - ASMJIT_API virtual Error onDetach(CodeHolder* ASMJIT_NONNULL(code)) noexcept; + ASMJIT_API virtual Error onDetach(CodeHolder& code) noexcept; + + //! Called when CodeHolder is reinitialized when the emitter is attached. + ASMJIT_API virtual Error onReinit(CodeHolder& code) noexcept; //! Called when \ref CodeHolder has updated an important setting, which involves the following: //! diff --git a/src/asmjit/core/emitterutils.cpp b/src/asmjit/core/emitterutils.cpp index 888d079..c6fc891 100644 --- a/src/asmjit/core/emitterutils.cpp +++ b/src/asmjit/core/emitterutils.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/emitterutils_p.h b/src/asmjit/core/emitterutils_p.h index b4faae3..77e93df 100644 --- a/src/asmjit/core/emitterutils_p.h +++ b/src/asmjit/core/emitterutils_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_EMITTERUTILS_P_H_INCLUDED diff --git a/src/asmjit/core/environment.cpp b/src/asmjit/core/environment.cpp index fa0b1c9..f0e1f52 100644 --- a/src/asmjit/core/environment.cpp +++ b/src/asmjit/core/environment.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/environment.h b/src/asmjit/core/environment.h index a66586f..9d1992f 100644 --- a/src/asmjit/core/environment.h +++ b/src/asmjit/core/environment.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ENVIRONMENT_H_INCLUDED diff --git a/src/asmjit/core/errorhandler.cpp b/src/asmjit/core/errorhandler.cpp index ce2ad01..d0efa9d 100644 --- a/src/asmjit/core/errorhandler.cpp +++ b/src/asmjit/core/errorhandler.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/errorhandler.h b/src/asmjit/core/errorhandler.h index a1a2dd2..9b5c8c2 100644 --- a/src/asmjit/core/errorhandler.h +++ b/src/asmjit/core/errorhandler.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ERRORHANDLER_H_INCLUDED diff --git a/src/asmjit/core/fixup.h b/src/asmjit/core/fixup.h new file mode 100644 index 0000000..7abb529 --- /dev/null +++ b/src/asmjit/core/fixup.h @@ -0,0 +1,282 @@ +// This file is part of AsmJit project +// +// See or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_FIXUP_H_INCLUDED +#define ASMJIT_CORE_FIXUP_H_INCLUDED + +#include "../core/globals.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_core +//! \{ + +//! Offset format type, used by \ref OffsetFormat. +enum class OffsetType : uint8_t { + // Common Offset Formats + // --------------------- + + //! A value having `_immBitCount` bits and shifted by `_immBitShift`. + //! + //! This offset type is sufficient for many targets that store offset as a continuous set bits within an + //! instruction word / sequence of bytes. + kSignedOffset, + + //! An unsigned value having `_immBitCount` bits and shifted by `_immBitShift`. + kUnsignedOffset, + + // AArch64 Specific Offset Formats + // ------------------------------- + + //! AArch64 ADR format of `[.|immlo:2|.....|immhi:19|.....]`. + kAArch64_ADR, + + //! AArch64 ADRP format of `[.|immlo:2|.....|immhi:19|.....]` (4kB pages). + kAArch64_ADRP, + + // AArch32 Specific Offset Formats (T16 & T32) + // ------------------------------------------- + + //! AArch32 THUMBv2 immediate encoding of 'ADR' instruction (12-bit payload and sign bit): + //! + //! `|.....|imm:1|..N.N|......|imm:3|....|imm:8|` + //! + //! Where `N` is one if the offset is negative. The immediate is encoded as absolute value of the offset if negative. + kThumb32_ADR, + + //! AArch32 THUMBv2 immediate encoding of 'BLX' instruction (23-bit immediate payload, multiplied by 4): + //! + //! `|.....|imm[22]|imm[19:10]|..|ja|1|jb|imm[9:0]|0` + //! + //! Where: + //! + //! - `ja` is calculated as imm[22] ^ imm[21] ^ 1. + //! - `jb` is calculated as imm[22] ^ imm[20] ^ 1. + kThumb32_BLX, + + //! AArch32 THUMBv2 immediate encoding of 'B' instruction without `` (24-bit immediate payload, multiplied by 2): + //! + //! `|.....|imm[23]|imm[20:11]|..|ja|1|jb|imm[10:0]` + //! + //! Where: + //! + //! - `ja` is calculated as imm[23] ^ imm[22] ^ 1. + //! - `jb` is calculated as imm[23] ^ imm[21] ^ 1. + kThumb32_B, + + //! AArch32 THUMBv2 immediate encoding of 'B' instruction with `` (20-bit immediate payload, multiplied by 2). + //! + //! `|.....|imm[19]|....|imm[16:11]|..|ja|1|jb|imm[10:0]` + //! + //! Where: + //! + //! - `ja` is calculated as imm[19] ^ imm[18] ^ 1. + //! - `jb` is calculated as imm[19] ^ imm[17] ^ 1. + kThumb32_BCond, + + // AArch32 Specific Offset Formats (A32) + // ------------------------------------- + + //! AArch32 ADR instruction, which uses a standard 12-bit immediate encoding that is used by other ARM instructions. + kAArch32_ADR, + + //! AArch32 signed offset that is similar to `kSignedOffset`, however it uses absolute value of the offset and its + //! sign is encoded in 23rd bit of the opcode. + //! + //! `|........|U.......|........|........|` + //! + kAArch32_U23_SignedOffset, + + //! AArch32 offset format that encodes 8-bit offset as: + //! + //! `|........|U.......|....|imm[7:4]|....|imm[3:0]|` + //! + //! in a 32-bit word, where U is a sign of the displacement and the displacement itself is encoded as its absolute + //! value. + kAArch32_U23_0To3At0_4To7At8, + + //! AArch32 offset format that encodes a signed 25-bit offset as: + //! + //! `|.......|imm[0]|imm[24:1]|` + //! + //! in a 32-bit word. + kAArch32_1To24At0_0At24, + + //! Maximum value of `OffsetFormatType`. + kMaxValue = kAArch32_1To24At0_0At24 +}; + +//! Provides information about formatting offsets, absolute addresses, or their parts. Offset format is used by both +//! \ref RelocEntry and \ref Fixup. The illustration below describes the relation of region size and offset size. +//! Region size is the size of the whole unit whereas offset size is the size of the unit that will be patched. +//! +//! ``` +//! +-> Code buffer | The subject of the relocation (region) | +//! | | (Word-Offset) (Word-Size) | +//! |xxxxxxxxxxxxxxx|................|*PATCHED*|................|xxxxxxxxxxxx-> +//! | | +//! [Word Offset points here]----+ +--- [WordOffset + WordSize] +//! ``` +//! +//! Once the offset word has been located it can be patched like this: +//! +//! ``` +//! |ImmDiscardLSB (discard LSB bits). +//! |.. +//! [0000000000000iiiiiiiiiiiiiiiiiDD] - Offset value (32-bit) +//! [000000000000000iiiiiiiiiiiiiiiii] - Offset value after discard LSB. +//! [00000000000iiiiiiiiiiiiiiiii0000] - Offset value shifted by ImmBitShift. +//! [xxxxxxxxxxxiiiiiiiiiiiiiiiiixxxx] - Patched word (32-bit) +//! |...............| +//! (ImmBitCount) +- ImmBitShift +//! ``` +struct OffsetFormat { + //! \name Members + //! \{ + + //! Type of the offset. + OffsetType _type; + //! Encoding flags. + uint8_t _flags; + //! Size of the region (in bytes) containing the offset value, if the offset value is part of an instruction, + //! otherwise it would be the same as `_valueSize`. + uint8_t _regionSize; + //! Size of the offset value, in bytes (1, 2, 4, or 8). + uint8_t _valueSize; + //! Offset of the offset value, in bytes, relative to the start of the region or data. Value offset would be + //! zero if both region size and value size are equal. + uint8_t _valueOffset; + //! Size of the offset immediate value in bits. + uint8_t _immBitCount; + //! Shift of the offset immediate value in bits in the target word. + uint8_t _immBitShift; + //! Number of least significant bits to discard before writing the immediate to the destination. All discarded + //! bits must be zero otherwise the value is invalid. + uint8_t _immDiscardLsb; + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the type of the offset. + ASMJIT_INLINE_NODEBUG OffsetType type() const noexcept { return _type; } + + //! Returns whether the offset is encoded as an absolute value of the offset with additional field(s) that represent + //! the sign (AArch32 U/N fields in the opcode). + //! + //! If true, the offset itself is always positive and a separate U/N field is used to indicate the sign of the offset + //! (usually `U==1` means ADD, but sometimes `N==1` means negative offset, which implies SUB). + ASMJIT_INLINE_NODEBUG bool hasSignBit() const noexcept { + return _type == OffsetType::kThumb32_ADR || + _type == OffsetType::kAArch32_ADR || + _type == OffsetType::kAArch32_U23_SignedOffset || + _type == OffsetType::kAArch32_U23_0To3At0_4To7At8; + } + + //! Returns flags. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t flags() const noexcept { return _flags; } + + //! Returns the size of the region/instruction where the offset is encoded. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t regionSize() const noexcept { return _regionSize; } + + //! Returns the offset of the word relative to the start of the region where the offset is. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t valueOffset() const noexcept { return _valueOffset; } + + //! Returns the size of the data-type (word) that contains the offset, in bytes. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t valueSize() const noexcept { return _valueSize; } + + //! Returns the count of bits of the offset value in the data it's stored in. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t immBitCount() const noexcept { return _immBitCount; } + + //! Returns the bit-shift of the offset value in the data it's stored in. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t immBitShift() const noexcept { return _immBitShift; } + + //! Returns the number of least significant bits of the offset value, that must be zero and that are not part of + //! the encoded data. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t immDiscardLsb() const noexcept { return _immDiscardLsb; } + + //! Resets this offset format to a simple data value of `dataSize` bytes. + //! + //! The region will be the same size as data and immediate bits would correspond to `dataSize * 8`. There will be + //! no immediate bit shift or discarded bits. + inline void resetToSimpleValue(OffsetType type, size_t valueSize) noexcept { + ASMJIT_ASSERT(valueSize <= 8u); + + _type = type; + _flags = uint8_t(0); + _regionSize = uint8_t(valueSize); + _valueSize = uint8_t(valueSize); + _valueOffset = uint8_t(0); + _immBitCount = uint8_t(valueSize * 8u); + _immBitShift = uint8_t(0); + _immDiscardLsb = uint8_t(0); + } + + inline void resetToImmValue(OffsetType type, size_t valueSize, uint32_t immBitShift, uint32_t immBitCount, uint32_t immDiscardLsb) noexcept { + ASMJIT_ASSERT(valueSize <= 8u); + ASMJIT_ASSERT(immBitShift < valueSize * 8u); + ASMJIT_ASSERT(immBitCount <= 64u); + ASMJIT_ASSERT(immDiscardLsb <= 64u); + + _type = type; + _flags = uint8_t(0); + _regionSize = uint8_t(valueSize); + _valueSize = uint8_t(valueSize); + _valueOffset = uint8_t(0); + _immBitCount = uint8_t(immBitCount); + _immBitShift = uint8_t(immBitShift); + _immDiscardLsb = uint8_t(immDiscardLsb); + } + + inline void setRegion(size_t regionSize, size_t valueOffset) noexcept { + _regionSize = uint8_t(regionSize); + _valueOffset = uint8_t(valueOffset); + } + + inline void setLeadingAndTrailingSize(size_t leadingSize, size_t trailingSize) noexcept { + _regionSize = uint8_t(leadingSize + trailingSize + _valueSize); + _valueOffset = uint8_t(leadingSize); + } + + //! \} +}; + +//! Data structure used to mark where a fixup in code or data is necessary. +//! +//! Fixups are generally resolved during machine code generation. For example if a branch instruction is used to +//! jump to a label, which hasn't been bound yet, a fixup is created. However, when such label is bound, the fixup +//! is processed and removed from a list of fixups. +struct Fixup { + //! Next fixup in a single-linked list. + Fixup* next; + //! Section where the fixup comes from. + uint32_t sectionId; + //! Label id, relocation id, or \ref Globals::kInvalidId. + //! + //! \note Fixup that is used with a LabelEntry always uses relocation id here, however, when a fixup is turned + //! into unresolved and generally detached from LabelEntry, this field becomes a label identifier as unresolved + //! fixups won't reference a relocation. This is just a space optimization. + uint32_t labelOrRelocId; + //! Label offset relative to the start of the section where the unresolved link comes from. + size_t offset; + //! Inlined rel8/rel32. + intptr_t rel; + //! Offset format information. + OffsetFormat format; +}; + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_FIXUP_H_INCLUDED diff --git a/src/asmjit/core/formatter.cpp b/src/asmjit/core/formatter.cpp index 79caae1..10660fa 100644 --- a/src/asmjit/core/formatter.cpp +++ b/src/asmjit/core/formatter.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -32,6 +32,26 @@ class VirtReg; namespace Formatter { +Error formatVirtRegName(String& sb, const VirtReg* vReg) noexcept { + if (vReg->nameSize()) { + return sb.append(vReg->name(), vReg->nameSize()); + } + else { + return sb.appendFormat("%%%u", unsigned(Operand::virtIdToIndex(vReg->id()))); + } +} + +Error formatVirtRegNameWithPrefix(String& sb, const char* prefix, size_t prefixSize, const VirtReg* vReg) noexcept { + ASMJIT_PROPAGATE(sb.append(prefix, prefixSize)); + + if (vReg->nameSize()) { + return sb.append(vReg->name(), vReg->nameSize()); + } + else { + return sb.appendFormat("%%%u", unsigned(Operand::virtIdToIndex(vReg->id()))); + } +} + static const char wordNameTable[][8] = { "db", "dw", @@ -128,33 +148,31 @@ Error formatLabel( DebugUtils::unused(formatFlags); if (emitter && emitter->code()) { - const LabelEntry* le = emitter->code()->labelEntry(labelId); - if (ASMJIT_UNLIKELY(!le)) { + CodeHolder* code = emitter->code(); + if (ASMJIT_UNLIKELY(!code->isLabelValid(labelId))) { return sb.appendFormat("", labelId); } - if (le->hasName()) { - if (le->hasParent()) { - uint32_t parentId = le->parentId(); - const LabelEntry* pe = emitter->code()->labelEntry(parentId); + const LabelEntry& le = code->labelEntry(labelId); + if (le.hasName()) { + if (le.hasParent()) { + uint32_t parentId = le.parentId(); + const LabelEntry& pe = code->labelEntry(parentId); - if (ASMJIT_UNLIKELY(!pe)) { - ASMJIT_PROPAGATE(sb.appendFormat("", labelId)); - } - else if (ASMJIT_UNLIKELY(!pe->hasName())) { - ASMJIT_PROPAGATE(sb.appendFormat("L%u", parentId)); + if (pe.hasName()) { + ASMJIT_PROPAGATE(sb.append(pe.name())); } else { - ASMJIT_PROPAGATE(sb.append(pe->name())); + ASMJIT_PROPAGATE(sb.appendFormat("L%u", parentId)); } ASMJIT_PROPAGATE(sb.append('.')); } - if (le->type() == LabelType::kAnonymous) { + if (le.labelType() == LabelType::kAnonymous) { ASMJIT_PROPAGATE(sb.appendFormat("L%u@", labelId)); } - return sb.append(le->name()); + return sb.append(le.name()); } } @@ -241,9 +259,9 @@ static Error formatDataHelper(String& sb, const char* typeName, uint32_t typeSiz switch (typeSize) { case 1: v = data[0]; break; - case 2: v = Support::readU16u(data); break; - case 4: v = Support::readU32u(data); break; - case 8: v = Support::readU64u(data); break; + case 2: v = Support::loadu_u16(data); break; + case 4: v = Support::loadu_u32(data); break; + case 8: v = Support::loadu_u64(data); break; } ASMJIT_PROPAGATE(sb.appendUInt(v, 16, typeSize * 2, StringFormatFlags::kAlternate)); @@ -381,7 +399,14 @@ static Error formatFuncValuePack( virtReg = cc->virtRegById(vRegs[valueIndex].id()); } - ASMJIT_PROPAGATE(sb.appendFormat(" %s", virtReg ? virtReg->name() : nullReg)); + ASMJIT_PROPAGATE(sb.append(' ')); + + if (virtReg) { + ASMJIT_PROPAGATE(Formatter::formatVirtRegName(sb, virtReg)); + } + else { + ASMJIT_PROPAGATE(sb.append(nullReg, sizeof(nullReg) - 1)); + } } } @@ -448,8 +473,8 @@ Error formatNode( case NodeType::kSection: { const SectionNode* sectionNode = node->as(); - if (builder->_code->isSectionValid(sectionNode->id())) { - const Section* section = builder->_code->sectionById(sectionNode->id()); + if (builder->_code->isSectionValid(sectionNode->sectionId())) { + const Section* section = builder->_code->sectionById(sectionNode->sectionId()); ASMJIT_PROPAGATE(sb.appendFormat(".section %s", section->name())); } break; diff --git a/src/asmjit/core/formatter.h b/src/asmjit/core/formatter.h index 780af0c..840d0c1 100644 --- a/src/asmjit/core/formatter.h +++ b/src/asmjit/core/formatter.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_FORMATTER_H_INCLUDED diff --git a/src/asmjit/core/formatter_p.h b/src/asmjit/core/formatter_p.h index 941e62e..7cb0d45 100644 --- a/src/asmjit/core/formatter_p.h +++ b/src/asmjit/core/formatter_p.h @@ -1,12 +1,14 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_FORMATTER_P_H_INCLUDED #define ASMJIT_CORE_FORMATTER_P_H_INCLUDED +#include "../core/compilerdefs.h" #include "../core/formatter.h" +#include "../core/operand.h" ASMJIT_BEGIN_NAMESPACE @@ -16,6 +18,7 @@ ASMJIT_BEGIN_NAMESPACE namespace Formatter { +[[maybe_unused]] static ASMJIT_INLINE size_t paddingFromOptions(const FormatOptions& formatOptions, FormatPaddingGroup group) noexcept { static constexpr uint16_t _defaultPaddingTable[uint32_t(FormatPaddingGroup::kMaxValue) + 1] = { 44, 26 }; static_assert(uint32_t(FormatPaddingGroup::kMaxValue) + 1 == 2, "If a new group is defined it must be added here"); @@ -24,6 +27,9 @@ static ASMJIT_INLINE size_t paddingFromOptions(const FormatOptions& formatOption return padding ? padding : size_t(_defaultPaddingTable[uint32_t(group)]); } +Error formatVirtRegName(String& sb, const VirtReg* vReg) noexcept; +Error formatVirtRegNameWithPrefix(String& sb, const char* prefix, size_t prefixSize, const VirtReg* vReg) noexcept; + } // {Formatter} //! \} diff --git a/src/asmjit/core/func.cpp b/src/asmjit/core/func.cpp index b273b92..843d228 100644 --- a/src/asmjit/core/func.cpp +++ b/src/asmjit/core/func.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -106,7 +106,7 @@ ASMJIT_FAVOR_SIZE Error FuncFrame::init(const FuncDetail& func) noexcept { _arch = arch; _spRegId = uint8_t(archTraits.spRegId()); - _saRegId = uint8_t(BaseReg::kIdBad); + _saRegId = uint8_t(Reg::kIdBad); uint32_t naturalStackAlignment = func.callConv().naturalStackAlignment(); uint32_t minDynamicAlignment = Support::max(naturalStackAlignment, 16); @@ -172,14 +172,14 @@ ASMJIT_FAVOR_SIZE Error FuncFrame::finalize() noexcept { // Currently required by ARM, if this works differently across architectures we would have to generalize most // likely in CallConv. - if (kLr != BaseReg::kIdBad) { + if (kLr != Reg::kIdBad) { _dirtyRegs[RegGroup::kGp] |= Support::bitMask(kLr); } } // These two are identical if the function doesn't align its stack dynamically. uint32_t saRegId = _saRegId; - if (saRegId == BaseReg::kIdBad) { + if (saRegId == Reg::kIdBad) { saRegId = kSp; } diff --git a/src/asmjit/core/func.h b/src/asmjit/core/func.h index 60919af..da7844c 100644 --- a/src/asmjit/core/func.h +++ b/src/asmjit/core/func.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_FUNC_H_INCLUDED @@ -49,7 +49,7 @@ enum class CallConvId : uint8_t { //! this calling convention it will be replaced by \ref CallConvId::kCDecl. kFastCall = 2, - //! `__vectorcall` on targets that support this calling convention (X86/X64). + //! `__vectorcall` on targets that support this calling convention (X86|X86_64). //! //! \note This calling convention is only supported on 32-bit and 64-bit X86 architecture on Windows platform. //! If used on environment that doesn't support this calling it will be replaced by \ref CallConvId::kCDecl. @@ -796,10 +796,10 @@ public: } //! Assigns a register at the given `index` to `reg` and an optional `typeId`. - inline void assignReg(size_t index, const BaseReg& reg, TypeId typeId = TypeId::kVoid) noexcept { + inline void assignReg(size_t index, const Reg& reg, TypeId typeId = TypeId::kVoid) noexcept { ASMJIT_ASSERT(index < Globals::kMaxValuePack); ASMJIT_ASSERT(reg.isPhysReg()); - _values[index].initReg(reg.type(), reg.id(), typeId); + _values[index].initReg(reg.regType(), reg.id(), typeId); } //! Assigns a register at the given `index` to `regType`, `regId`, and an optional `typeId`. @@ -849,7 +849,7 @@ enum class FuncAttributes : uint32_t { kAlignedVecSR = 0x00000040u, //! Function must begin with an instruction that marks a start of a branch or function. //! - //! * `ENDBR32/ENDBR64` instruction is inserted at the beginning of the function (X86, X86_64). + //! * `ENDBR32/ENDBR64` instruction is inserted at the beginning of the function (X86|X86_64). //! * `BTI` instruction is inserted at the beginning of the function (AArch64) kIndirectBranchProtection = 0x00000080u, //! FuncFrame is finalized and can be used by prolog/epilog inserter (PEI). @@ -858,22 +858,22 @@ enum class FuncAttributes : uint32_t { // X86 Specific Attributes // ----------------------- - //! Enables the use of AVX within the function's body, prolog, and epilog (X86). + //! Enables the use of AVX within the function's body, prolog, and epilog (X86|X86_64). //! //! This flag instructs prolog and epilog emitter to use AVX instead of SSE for manipulating XMM registers. kX86_AVXEnabled = 0x00010000u, - //! Enables the use of AVX-512 within the function's body, prolog, and epilog (X86). + //! Enables the use of AVX-512 within the function's body, prolog, and epilog (X86|X86_64). //! //! This flag instructs Compiler register allocator to use additional 16 registers introduced by AVX-512. //! Additionally, if the functions saves full width of ZMM registers (custom calling conventions only) then //! the prolog/epilog inserter would use AVX-512 move instructions to emit the save and restore sequence. kX86_AVX512Enabled = 0x00020000u, - //! This flag instructs the epilog writer to emit EMMS instruction before RET (X86). + //! This flag instructs the epilog writer to emit EMMS instruction before RET (X86|X86_64). kX86_MMXCleanup = 0x00040000u, - //! This flag instructs the epilog writer to emit VZEROUPPER instruction before RET (X86). + //! This flag instructs the epilog writer to emit VZEROUPPER instruction before RET (X86|X86_64). kX86_AVXCleanup = 0x00080000u }; ASMJIT_DEFINE_ENUM_FLAGS(FuncAttributes) @@ -1148,9 +1148,9 @@ public: //! Target architecture. Arch _arch {}; //! SP register ID (to access call stack and local stack). - uint8_t _spRegId = uint8_t(BaseReg::kIdBad); + uint8_t _spRegId = uint8_t(Reg::kIdBad); //! SA register ID (to access stack arguments). - uint8_t _saRegId = uint8_t(BaseReg::kIdBad); + uint8_t _saRegId = uint8_t(Reg::kIdBad); //! Red zone size (copied from CallConv). uint8_t _redZoneSize = 0; @@ -1514,14 +1514,14 @@ public: } //! \overload - inline void addDirtyRegs(const BaseReg& reg) noexcept { + inline void addDirtyRegs(const Reg& reg) noexcept { ASMJIT_ASSERT(reg.id() < Globals::kMaxPhysRegs); - addDirtyRegs(reg.group(), Support::bitMask(reg.id())); + addDirtyRegs(reg.regGroup(), Support::bitMask(reg.id())); } //! \overload template - inline void addDirtyRegs(const BaseReg& reg, Args&&... args) noexcept { + inline void addDirtyRegs(const Reg& reg, Args&&... args) noexcept { addDirtyRegs(reg); addDirtyRegs(std::forward(args)...); } @@ -1583,14 +1583,14 @@ public: } //! Adds a single register to the unavailable set. - inline void addUnavailableRegs(const BaseReg& reg) noexcept { + inline void addUnavailableRegs(const Reg& reg) noexcept { ASMJIT_ASSERT(reg.id() < Globals::kMaxPhysRegs); - addUnavailableRegs(reg.group(), Support::bitMask(reg.id())); + addUnavailableRegs(reg.regGroup(), Support::bitMask(reg.id())); } //! Adds multiple registers to the unavailable set. template - inline void addUnavailableRegs(const BaseReg& reg, Args&&... args) noexcept { + inline void addUnavailableRegs(const Reg& reg, Args&&... args) noexcept { addUnavailableRegs(reg); addUnavailableRegs(std::forward(args)...); } @@ -1635,14 +1635,14 @@ public: } [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool hasSARegId() const noexcept { return _saRegId != BaseReg::kIdBad; } + ASMJIT_INLINE_NODEBUG bool hasSARegId() const noexcept { return _saRegId != Reg::kIdBad; } [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t saRegId() const noexcept { return _saRegId; } ASMJIT_INLINE_NODEBUG void setSARegId(uint32_t regId) { _saRegId = uint8_t(regId); } - ASMJIT_INLINE_NODEBUG void resetSARegId() { setSARegId(BaseReg::kIdBad); } + ASMJIT_INLINE_NODEBUG void resetSARegId() { setSARegId(Reg::kIdBad); } //! Returns stack size required to save/restore registers via push/pop. [[nodiscard]] @@ -1694,7 +1694,7 @@ public: //! Function detail. const FuncDetail* _funcDetail {}; //! Register that can be used to access arguments passed by stack. - uint8_t _saRegId = uint8_t(BaseReg::kIdBad); + uint8_t _saRegId = uint8_t(Reg::kIdBad); //! Reserved for future use. uint8_t _reserved[3] {}; //! Mapping of each function argument. @@ -1715,7 +1715,7 @@ public: //! if non-null. inline void reset(const FuncDetail* fd = nullptr) noexcept { _funcDetail = fd; - _saRegId = uint8_t(BaseReg::kIdBad); + _saRegId = uint8_t(Reg::kIdBad); memset(_reserved, 0, sizeof(_reserved)); memset(_argPacks, 0, sizeof(_argPacks)); } @@ -1741,14 +1741,14 @@ public: ASMJIT_INLINE_NODEBUG void setFuncDetail(const FuncDetail* fd) noexcept { _funcDetail = fd; } [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool hasSARegId() const noexcept { return _saRegId != BaseReg::kIdBad; } + ASMJIT_INLINE_NODEBUG bool hasSARegId() const noexcept { return _saRegId != Reg::kIdBad; } [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t saRegId() const noexcept { return _saRegId; } ASMJIT_INLINE_NODEBUG void setSARegId(uint32_t regId) { _saRegId = uint8_t(regId); } - ASMJIT_INLINE_NODEBUG void resetSARegId() { _saRegId = uint8_t(BaseReg::kIdBad); } + ASMJIT_INLINE_NODEBUG void resetSARegId() { _saRegId = uint8_t(Reg::kIdBad); } //! Returns assigned argument at `argIndex` and `valueIndex`. //! @@ -1775,10 +1775,10 @@ public: } //! Assigns register at `argIndex` and value index of 0 to `reg` and an optional `typeId`. - inline void assignReg(size_t argIndex, const BaseReg& reg, TypeId typeId = TypeId::kVoid) noexcept { + inline void assignReg(size_t argIndex, const Reg& reg, TypeId typeId = TypeId::kVoid) noexcept { ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); ASMJIT_ASSERT(reg.isPhysReg()); - _argPacks[argIndex][0].initReg(reg.type(), reg.id(), typeId); + _argPacks[argIndex][0].initReg(reg.regType(), reg.id(), typeId); } //! Assigns register at `argIndex` and value index of 0 to `regType`, `regId`, and an optional `typeId`. @@ -1794,10 +1794,10 @@ public: } //! Assigns register at `argIndex` and `valueIndex` to `reg` and an optional `typeId`. - inline void assignRegInPack(size_t argIndex, size_t valueIndex, const BaseReg& reg, TypeId typeId = TypeId::kVoid) noexcept { + inline void assignRegInPack(size_t argIndex, size_t valueIndex, const Reg& reg, TypeId typeId = TypeId::kVoid) noexcept { ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); ASMJIT_ASSERT(reg.isPhysReg()); - _argPacks[argIndex][valueIndex].initReg(reg.type(), reg.id(), typeId); + _argPacks[argIndex][valueIndex].initReg(reg.regType(), reg.id(), typeId); } //! Assigns register at `argIndex` and `valueIndex` to `regType`, `regId`, and an optional `typeId`. @@ -1815,12 +1815,12 @@ public: // NOTE: All `assignAll()` methods are shortcuts to assign all arguments at once, however, since registers are // passed all at once these initializers don't provide any way to pass TypeId and/or to keep any argument between // the arguments passed unassigned. - inline void _assignAllInternal(size_t argIndex, const BaseReg& reg) noexcept { + inline void _assignAllInternal(size_t argIndex, const Reg& reg) noexcept { assignReg(argIndex, reg); } template - inline void _assignAllInternal(size_t argIndex, const BaseReg& reg, Args&&... args) noexcept { + inline void _assignAllInternal(size_t argIndex, const Reg& reg, Args&&... args) noexcept { assignReg(argIndex, reg); _assignAllInternal(argIndex + 1, std::forward(args)...); } diff --git a/src/asmjit/core/funcargscontext.cpp b/src/asmjit/core/funcargscontext.cpp index 2580a0c..f01e685 100644 --- a/src/asmjit/core/funcargscontext.cpp +++ b/src/asmjit/core/funcargscontext.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -58,7 +58,7 @@ ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, co FuncValue& dst = var.out; RegGroup dstGroup = RegGroup::kMaxValue; - uint32_t dstId = BaseReg::kIdBad; + uint32_t dstId = Reg::kIdBad; WorkData* dstWd = nullptr; // Not supported. @@ -75,10 +75,10 @@ ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, co // Copy TypeId from source if the destination doesn't have it. The RA used by BaseCompiler would never // leave TypeId undefined, but users of FuncAPI can just assign phys regs without specifying their types. if (!dst.hasTypeId()) { - dst.setTypeId(archTraits().regTypeToTypeId(dst.regType())); + dst.setTypeId(RegUtils::typeIdOf(dst.regType())); } - dstGroup = archTraits().regTypeToGroup(dstType); + dstGroup = RegUtils::groupOf(dstType); if (ASMJIT_UNLIKELY(dstGroup > RegGroup::kMaxVirt)) { return DebugUtils::errored(kErrorInvalidRegGroup); } @@ -112,7 +112,7 @@ ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, co if (src.isReg()) { uint32_t srcId = src.regId(); - RegGroup srcGroup = archTraits().regTypeToGroup(src.regType()); + RegGroup srcGroup = RegUtils::groupOf(src.regType()); if (dstGroup == srcGroup) { ASMJIT_ASSERT(dstWd != nullptr); @@ -173,14 +173,14 @@ ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, co uint32_t saCurRegId = frame.saRegId(); uint32_t saOutRegId = args.saRegId(); - if (saCurRegId != BaseReg::kIdBad) { + if (saCurRegId != Reg::kIdBad) { // Check if the provided `SARegId` doesn't collide with input registers. if (ASMJIT_UNLIKELY(gpRegs.isAssigned(saCurRegId))) { return DebugUtils::errored(kErrorOverlappedRegs); } } - if (saOutRegId != BaseReg::kIdBad) { + if (saOutRegId != Reg::kIdBad) { // Check if the provided `SARegId` doesn't collide with argument assignments. if (ASMJIT_UNLIKELY(Support::bitTest(gpRegs.dstRegs(), saOutRegId))) { return DebugUtils::errored(kErrorOverlappedRegs); @@ -198,8 +198,8 @@ ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, co Var& var = _vars[varId]; var.reset(); - if (saCurRegId == BaseReg::kIdBad) { - if (saOutRegId != BaseReg::kIdBad && !gpRegs.isAssigned(saOutRegId)) { + if (saCurRegId == Reg::kIdBad) { + if (saOutRegId != Reg::kIdBad && !gpRegs.isAssigned(saOutRegId)) { saCurRegId = saOutRegId; } else { @@ -220,7 +220,7 @@ ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, co gpRegs.assign(varId, saCurRegId); gpRegs._workRegs |= Support::bitMask(saCurRegId); - if (saOutRegId != BaseReg::kIdBad) { + if (saOutRegId != Reg::kIdBad) { var.out.initReg(ptrRegType, saOutRegId, ptrTypeId); gpRegs._dstRegs |= Support::bitMask(saOutRegId); gpRegs._workRegs |= Support::bitMask(saOutRegId); @@ -241,15 +241,15 @@ ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, co uint32_t srcId = var.cur.regId(); uint32_t dstId = var.out.regId(); - RegGroup group = archTraits().regTypeToGroup(var.cur.regType()); - if (group != archTraits().regTypeToGroup(var.out.regType())) { + RegGroup group = RegUtils::groupOf(var.cur.regType()); + if (group != RegUtils::groupOf(var.out.regType())) { continue; } WorkData& wd = _workData[group]; if (wd.isAssigned(dstId)) { Var& other = _vars[wd._physToVarId[dstId]]; - if (archTraits().regTypeToGroup(other.out.regType()) == group && other.out.regId() == srcId) { + if (RegUtils::groupOf(other.out.regType()) == group && other.out.regId() == srcId) { wd._numSwaps++; _regSwapsMask = uint8_t(_regSwapsMask | Support::bitMask(group)); } diff --git a/src/asmjit/core/funcargscontext_p.h b/src/asmjit/core/funcargscontext_p.h index 4124305..bcfc784 100644 --- a/src/asmjit/core/funcargscontext_p.h +++ b/src/asmjit/core/funcargscontext_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_FUNCARGSCONTEXT_P_H_INCLUDED @@ -22,30 +22,30 @@ ASMJIT_BEGIN_NAMESPACE static inline OperandSignature getSuitableRegForMemToMemMove(Arch arch, TypeId dstTypeId, TypeId srcTypeId) noexcept { const ArchTraits& archTraits = ArchTraits::byArch(arch); + uint32_t signature = 0u; uint32_t dstSize = TypeUtils::sizeOf(dstTypeId); uint32_t srcSize = TypeUtils::sizeOf(srcTypeId); uint32_t maxSize = Support::max(dstSize, srcSize); uint32_t regSize = Environment::registerSizeFromArch(arch); - OperandSignature signature{0}; if (maxSize <= regSize || (TypeUtils::isInt(dstTypeId) && TypeUtils::isInt(srcTypeId))) { - signature = maxSize <= 4 ? archTraits.regTypeToSignature(RegType::kGp32) - : archTraits.regTypeToSignature(RegType::kGp64); + signature = maxSize <= 4 ? RegTraits::kSignature + : RegTraits::kSignature; } else if (maxSize <= 8 && archTraits.hasRegType(RegType::kVec64)) { - signature = archTraits.regTypeToSignature(RegType::kVec64); + signature = RegTraits::kSignature; } else if (maxSize <= 16 && archTraits.hasRegType(RegType::kVec128)) { - signature = archTraits.regTypeToSignature(RegType::kVec128); + signature = RegTraits::kSignature; } else if (maxSize <= 32 && archTraits.hasRegType(RegType::kVec256)) { - signature = archTraits.regTypeToSignature(RegType::kVec256); + signature = RegTraits::kSignature; } else if (maxSize <= 64 && archTraits.hasRegType(RegType::kVec512)) { - signature = archTraits.regTypeToSignature(RegType::kVec512); + signature = RegTraits::kSignature; } - return signature; + return OperandSignature{signature}; } class FuncArgsContext { diff --git a/src/asmjit/core/globals.cpp b/src/asmjit/core/globals.cpp index 4a98431..e344882 100644 --- a/src/asmjit/core/globals.cpp +++ b/src/asmjit/core/globals.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/globals.h b/src/asmjit/core/globals.h index 1d35de8..f5e46ef 100644 --- a/src/asmjit/core/globals.h +++ b/src/asmjit/core/globals.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_GLOBALS_H_INCLUDED @@ -30,13 +30,13 @@ namespace Support { ASMJIT_INLINE void operatorDelete(void* p) noexcept { if (p) free(p); } } // {Support} -#define ASMJIT_BASE_CLASS(TYPE) \ +#define ASMJIT_BASE_CLASS(TYPE) \ ASMJIT_INLINE void* operator new(size_t n) noexcept { return Support::operatorNew(n); } \ ASMJIT_INLINE void operator delete(void* ptr) noexcept { Support::operatorDelete(ptr); } \ - \ + \ ASMJIT_INLINE void* operator new(size_t, void* ptr) noexcept { return ptr; } \ ASMJIT_INLINE void operator delete(void*, void*) noexcept {} \ - \ + \ ASMJIT_INLINE void* operator new(size_t, Support::PlacementNew ptr) noexcept { return ptr.ptr; } \ ASMJIT_INLINE void operator delete(void*, Support::PlacementNew) noexcept {} #else @@ -73,13 +73,16 @@ enum class ResetPolicy : uint32_t { namespace Globals { //! Host memory allocator overhead. -static constexpr uint32_t kAllocOverhead = uint32_t(sizeof(intptr_t) * 4); +static constexpr uint32_t kAllocOverhead = uint32_t(sizeof(intptr_t) * 4u); //! Host memory allocator alignment. -static constexpr uint32_t kAllocAlignment = 8; +static constexpr uint32_t kAllocAlignment = 8u; //! Aggressive growing strategy threshold. -static constexpr uint32_t kGrowThreshold = 1024 * 1024 * 16; +static constexpr uint32_t kGrowThreshold = 1024u * 1024u * 16u; + +//! Default alignment of allocation requests to use when using Zone. +static constexpr uint32_t kZoneAlignment = 8u; //! Maximum depth of RB-Tree is: //! @@ -212,8 +215,8 @@ enum ErrorCode : uint32_t { kErrorLabelNameTooLong, //! Label must always be local if it's anonymous (without a name). kErrorInvalidLabelName, - //! Parent id passed to \ref CodeHolder::newNamedLabelEntry() was either invalid or parent is not supported - //! by the requested `LabelType`. + //! Parent id passed to \ref CodeHolder::newNamedLabelId() was either invalid or parent is not supported by + //! the requested `LabelType`. kErrorInvalidParentLabel, //! Invalid section. @@ -278,7 +281,7 @@ enum ErrorCode : uint32_t { kErrorInvalidAddress64BitZeroExtension, //! Invalid displacement (not encodable). kErrorInvalidDisplacement, - //! Invalid segment (X86). + //! Invalid segment (X86|X86_64). kErrorInvalidSegment, //! Invalid immediate (out of bounds on X86 and invalid pattern on ARM). @@ -386,14 +389,32 @@ ASMJIT_API void assertionFailed(const char* file, int line, const char* msg) noe //! //! AsmJit's own assert macro used in AsmJit code-base. #if defined(ASMJIT_BUILD_DEBUG) -#define ASMJIT_ASSERT(...) \ + #define ASMJIT_ASSERT(...) \ + do { \ + if (ASMJIT_UNLIKELY(!(__VA_ARGS__))) { \ + ::asmjit::DebugUtils::assertionFailed(__FILE__, __LINE__, #__VA_ARGS__); \ + } \ + } while (0) +#else + #define ASMJIT_ASSERT(...) ((void)0) +#endif + +#define ASMJIT_RUNTIME_ASSERT(...) \ do { \ if (ASMJIT_UNLIKELY(!(__VA_ARGS__))) { \ ::asmjit::DebugUtils::assertionFailed(__FILE__, __LINE__, #__VA_ARGS__); \ } \ } while (0) + +//! \def ASMJIT_NOT_REACHED() +//! +//! Run-time assertion used in code that should never be reached. +#if defined(ASMJIT_BUILD_DEBUG) + #define ASMJIT_NOT_REACHED() ::asmjit::DebugUtils::assertionFailed(__FILE__, __LINE__, "ASMJIT_NOT_REACHED()") +#elif defined(__GNUC__) + #define ASMJIT_NOT_REACHED() __builtin_unreachable() #else -#define ASMJIT_ASSERT(...) ((void)0) + #define ASMJIT_NOT_REACHED() ASMJIT_ASSUME(0) #endif //! \def ASMJIT_PROPAGATE(...) diff --git a/src/asmjit/core/inst.cpp b/src/asmjit/core/inst.cpp index da8a81e..7acb4f2 100644 --- a/src/asmjit/core/inst.cpp +++ b/src/asmjit/core/inst.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/inst.h b/src/asmjit/core/inst.h index 110e144..7c11238 100644 --- a/src/asmjit/core/inst.h +++ b/src/asmjit/core/inst.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_INST_H_INCLUDED @@ -256,7 +256,7 @@ public: _options(options), _extraReg(extraReg) {} - ASMJIT_INLINE_NODEBUG BaseInst(InstId instId, InstOptions options, const BaseReg& extraReg) noexcept + ASMJIT_INLINE_NODEBUG BaseInst(InstId instId, InstOptions options, const Reg& extraReg) noexcept : _id(instId), _options(options), _extraReg { extraReg.signature(), extraReg.id() } {} @@ -334,7 +334,7 @@ public: [[nodiscard]] ASMJIT_INLINE_NODEBUG const RegOnly& extraReg() const noexcept { return _extraReg; } - ASMJIT_INLINE_NODEBUG void setExtraReg(const BaseReg& reg) noexcept { _extraReg.init(reg); } + ASMJIT_INLINE_NODEBUG void setExtraReg(const Reg& reg) noexcept { _extraReg.init(reg); } ASMJIT_INLINE_NODEBUG void setExtraReg(const RegOnly& reg) noexcept { _extraReg.init(reg); } @@ -414,34 +414,34 @@ enum class CpuRWFlags : uint32_t { // X86 Specific RW Flags // ---------------------------------- - //! Carry flag (X86, X86_64). + //! Carry flag (X86|X86_64). kX86_CF = kCF, - //! Overflow flag (X86, X86_64). + //! Overflow flag (X86|X86_64). kX86_OF = kOF, - //! Sign flag (X86, X86_64). + //! Sign flag (X86|X86_64). kX86_SF = kSF, - //! Zero flag (X86, X86_64). + //! Zero flag (X86|X86_64). kX86_ZF = kZF, - //! Adjust flag (X86, X86_64). + //! Adjust flag (X86|X86_64). kX86_AF = 0x00000100u, - //! Parity flag (X86, X86_64). + //! Parity flag (X86|X86_64). kX86_PF = 0x00000200u, - //! Direction flag (X86, X86_64). + //! Direction flag (X86|X86_64). kX86_DF = 0x00000400u, - //! Interrupt enable flag (X86, X86_64). + //! Interrupt enable flag (X86|X86_64). kX86_IF = 0x00000800u, - //! Alignment check flag (X86, X86_64). + //! Alignment check flag (X86|X86_64). kX86_AC = 0x00001000u, - //! FPU C0 status flag (X86, X86_64). + //! FPU C0 status flag (X86|X86_64). kX86_C0 = 0x00010000u, - //! FPU C1 status flag (X86, X86_64). + //! FPU C1 status flag (X86|X86_64). kX86_C1 = 0x00020000u, - //! FPU C2 status flag (X86, X86_64). + //! FPU C2 status flag (X86|X86_64). kX86_C2 = 0x00040000u, - //! FPU C3 status flag (X86, X86_64). + //! FPU C3 status flag (X86|X86_64). kX86_C3 = 0x00080000u, // ARM Specific RW Flags @@ -477,7 +477,7 @@ enum class OpRWFlags : uint32_t { //! //! This flag is used by all architectures to describe instructions that use consecutive registers, where only the //! first one is encoded in the instruction, and the others are just a sequence that starts with the first one. On - //! X86/X86_64 architecture this is used by instructions such as VP2INTERSECTD and VP2INTERSECTQ. On ARM/AArch64 + //! X86|X86_64 architecture this is used by instructions such as VP2INTERSECTD and VP2INTERSECTQ. On ARM/AArch64 //! this is used by vector load and store instructions that can load or store multiple registers at once. kConsecutive = 0x00000008u, @@ -559,7 +559,7 @@ struct OpRWInfo { //! Resets this operand info (resets all members) and set common information //! to the given `opFlags`, `regSize`, and possibly `physId`. - inline void reset(OpRWFlags opFlags, uint32_t regSize, uint32_t physId = BaseReg::kIdBad) noexcept { + inline void reset(OpRWFlags opFlags, uint32_t regSize, uint32_t physId = Reg::kIdBad) noexcept { _opFlags = opFlags; _physId = uint8_t(physId); _rmSize = Support::test(opFlags, OpRWFlags::kRegMem) ? uint8_t(regSize) : uint8_t(0); @@ -708,13 +708,13 @@ struct OpRWInfo { //! Returns a physical id of the register that is fixed for this operand. //! - //! Returns \ref BaseReg::kIdBad if any register can be used. + //! Returns \ref Reg::kIdBad if any register can be used. [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t physId() const noexcept { return _physId; } //! Tests whether \ref physId() would return a valid physical register id. [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool hasPhysId() const noexcept { return _physId != BaseReg::kIdBad; } + ASMJIT_INLINE_NODEBUG bool hasPhysId() const noexcept { return _physId != Reg::kIdBad; } //! Sets physical register id, which would be fixed for this operand. ASMJIT_INLINE_NODEBUG void setPhysId(uint32_t physId) noexcept { _physId = uint8_t(physId); } @@ -878,7 +878,7 @@ struct InstRWInfo { }; //! Validation flags that can be used with \ref InstAPI::validate(). -enum class ValidationFlags : uint32_t { +enum class ValidationFlags : uint8_t { //! No flags. kNone = 0, //! Allow virtual registers in the instruction. diff --git a/src/asmjit/core/instdb.cpp b/src/asmjit/core/instdb.cpp index a7cfcea..deacb22 100644 --- a/src/asmjit/core/instdb.cpp +++ b/src/asmjit/core/instdb.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/instdb_p.h b/src/asmjit/core/instdb_p.h index 9360565..24f0d92 100644 --- a/src/asmjit/core/instdb_p.h +++ b/src/asmjit/core/instdb_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_INSTDB_P_H_INCLUDED diff --git a/src/asmjit/core/jitallocator.cpp b/src/asmjit/core/jitallocator.cpp index 7a107b4..2ed14ca 100644 --- a/src/asmjit/core/jitallocator.cpp +++ b/src/asmjit/core/jitallocator.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -15,6 +15,10 @@ #include "../core/zonelist.h" #include "../core/zonetree.h" +#if defined(ASMJIT_TEST) +#include "../../../test/asmjit_test_random.h" +#endif // ASMJIT_TEST + ASMJIT_BEGIN_NAMESPACE // JitAllocator - Constants @@ -147,6 +151,9 @@ class JitAllocatorPool { public: ASMJIT_NONCOPYABLE(JitAllocatorPool) + //! \name Members + //! \{ + //! Double linked list of blocks. ZoneList blocks; //! Where to start looking first. @@ -168,12 +175,14 @@ public: //! Overhead of all blocks (in bytes). size_t totalOverheadBytes = 0; - inline JitAllocatorPool(uint32_t granularity) noexcept + //! \} + + ASMJIT_INLINE JitAllocatorPool(uint32_t granularity) noexcept : blocks(), granularity(uint16_t(granularity)), granularityLog2(uint8_t(Support::ctz(granularity))) {} - inline void reset() noexcept { + ASMJIT_INLINE void reset() noexcept { blocks.reset(); cursor = nullptr; blockCount = 0u; @@ -184,12 +193,11 @@ public: totalOverheadBytes = 0u; } - inline size_t byteSizeFromAreaSize(uint32_t areaSize) const noexcept { return size_t(areaSize) * granularity; } - inline uint32_t areaSizeFromByteSize(size_t size) const noexcept { return uint32_t((size + granularity - 1) >> granularityLog2); } + ASMJIT_INLINE_NODEBUG size_t byteSizeFromAreaSize(uint32_t areaSize) const noexcept { return size_t(areaSize) * granularity; } + ASMJIT_INLINE_NODEBUG uint32_t areaSizeFromByteSize(size_t size) const noexcept { return uint32_t((size + granularity - 1) >> granularityLog2); } - inline size_t bitWordCountFromAreaSize(uint32_t areaSize) const noexcept { - using namespace Support; - return alignUp(areaSize, kBitWordSizeInBits) / kBitWordSizeInBits; + ASMJIT_INLINE_NODEBUG size_t bitWordCountFromAreaSize(uint32_t areaSize) const noexcept { + return Support::alignUp(areaSize, Support::kBitWordSizeInBits) / Support::kBitWordSizeInBits; } }; @@ -206,12 +214,14 @@ public: kFlagInitialPadding = 0x00000001u, //! Block is empty. kFlagEmpty = 0x00000002u, - //! Block is dirty (largestUnusedArea, searchStart, searchEnd). + //! Block is dirty (dirty largestUnusedArea, searchStart, searchEnd, ...). kFlagDirty = 0x00000004u, + //! Block is in incremental mode, which means that there is no memory used after searchStart. + kFlagIncremental = 0x00000008u, //! Block represents memory that is using large pages. - kFlagLargePages = 0x00000008u, + kFlagLargePages = 0x00000010u, //! Block represents memory that is dual-mapped. - kFlagDualMapped = 0x00000010u + kFlagDualMapped = 0x00000020u }; static_assert(kFlagInitialPadding == 1, "JitAllocatorBlock::kFlagInitialPadding must be equal to 1"); @@ -244,7 +254,7 @@ public: //! Stop bit-vector (0 = don't care, 1 = stop). Support::BitWord* _stopBitVector {}; - inline JitAllocatorBlock( + ASMJIT_INLINE JitAllocatorBlock( JitAllocatorPool* pool, VirtMem::DualMapping mapping, size_t blockSize, @@ -270,58 +280,61 @@ public: } [[nodiscard]] - inline JitAllocatorPool* pool() const noexcept { return _pool; } + ASMJIT_INLINE_NODEBUG JitAllocatorPool* pool() const noexcept { return _pool; } [[nodiscard]] - inline uint8_t* rxPtr() const noexcept { return static_cast(_mapping.rx); } + ASMJIT_INLINE_NODEBUG uint8_t* rxPtr() const noexcept { return static_cast(_mapping.rx); } [[nodiscard]] - inline uint8_t* rwPtr() const noexcept { return static_cast(_mapping.rw); } + ASMJIT_INLINE_NODEBUG uint8_t* rwPtr() const noexcept { return static_cast(_mapping.rw); } [[nodiscard]] - inline bool hasFlag(uint32_t f) const noexcept { return (_flags & f) != 0; } + ASMJIT_INLINE_NODEBUG bool hasFlag(uint32_t f) const noexcept { return (_flags & f) != 0; } - inline void addFlags(uint32_t f) noexcept { _flags |= f; } - inline void clearFlags(uint32_t f) noexcept { _flags &= ~f; } + ASMJIT_INLINE_NODEBUG void addFlags(uint32_t f) noexcept { _flags |= f; } + ASMJIT_INLINE_NODEBUG void clearFlags(uint32_t f) noexcept { _flags &= ~f; } [[nodiscard]] - inline bool empty() const noexcept { return hasFlag(kFlagEmpty); } + ASMJIT_INLINE_NODEBUG bool isEmpty() const noexcept { return hasFlag(kFlagEmpty); } [[nodiscard]] - inline bool isDirty() const noexcept { return hasFlag(kFlagDirty); } + ASMJIT_INLINE_NODEBUG bool isDirty() const noexcept { return hasFlag(kFlagDirty); } - inline void makeDirty() noexcept { addFlags(kFlagDirty); } + ASMJIT_INLINE_NODEBUG void makeDirty() noexcept { addFlags(kFlagDirty); } [[nodiscard]] - inline bool hasLargePages() const noexcept { return hasFlag(kFlagLargePages); } + ASMJIT_INLINE_NODEBUG bool isIncremental() const noexcept { return hasFlag(kFlagIncremental); } [[nodiscard]] - inline bool hasInitialPadding() const noexcept { return hasFlag(kFlagInitialPadding); } + ASMJIT_INLINE_NODEBUG bool hasLargePages() const noexcept { return hasFlag(kFlagLargePages); } [[nodiscard]] - inline uint32_t initialAreaStart() const noexcept { return initialAreaStartByFlags(_flags); } + ASMJIT_INLINE_NODEBUG bool hasInitialPadding() const noexcept { return hasFlag(kFlagInitialPadding); } [[nodiscard]] - inline size_t blockSize() const noexcept { return _blockSize; } + ASMJIT_INLINE_NODEBUG uint32_t initialAreaStart() const noexcept { return initialAreaStartByFlags(_flags); } [[nodiscard]] - inline uint32_t areaSize() const noexcept { return _areaSize; } + ASMJIT_INLINE_NODEBUG size_t blockSize() const noexcept { return _blockSize; } [[nodiscard]] - inline uint32_t areaUsed() const noexcept { return _areaUsed; } + ASMJIT_INLINE_NODEBUG uint32_t areaSize() const noexcept { return _areaSize; } [[nodiscard]] - inline uint32_t areaAvailable() const noexcept { return _areaSize - _areaUsed; } + ASMJIT_INLINE_NODEBUG uint32_t areaUsed() const noexcept { return _areaUsed; } [[nodiscard]] - inline uint32_t largestUnusedArea() const noexcept { return _largestUnusedArea; } + ASMJIT_INLINE_NODEBUG uint32_t areaAvailable() const noexcept { return _areaSize - _areaUsed; } - inline void decreaseUsedArea(uint32_t value) noexcept { + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t largestUnusedArea() const noexcept { return _largestUnusedArea; } + + ASMJIT_INLINE void decreaseUsedArea(uint32_t value) noexcept { _areaUsed -= value; _pool->totalAreaUsed[size_t(hasLargePages())] -= value; } - inline void clearBlock() noexcept { + ASMJIT_INLINE void clearBlock() noexcept { bool bit = hasInitialPadding(); size_t numBitWords = _pool->bitWordCountFromAreaSize(_areaSize); @@ -337,11 +350,11 @@ public: _searchStart = start; _searchEnd = _areaSize; - addFlags(JitAllocatorBlock::kFlagEmpty); + addFlags(JitAllocatorBlock::kFlagEmpty | JitAllocatorBlock::kFlagIncremental); clearFlags(JitAllocatorBlock::kFlagDirty); } - inline void markAllocatedArea(uint32_t allocatedAreaStart, uint32_t allocatedAreaEnd) noexcept { + ASMJIT_INLINE void markAllocatedArea(uint32_t allocatedAreaStart, uint32_t allocatedAreaEnd) noexcept { uint32_t allocatedAreaSize = allocatedAreaEnd - allocatedAreaStart; // Mark the newly allocated space as occupied and also the sentinel. @@ -373,32 +386,44 @@ public: } } - inline void markReleasedArea(uint32_t releasedAreaStart, uint32_t releasedAreaEnd) noexcept { + ASMJIT_INLINE void markReleasedArea(uint32_t releasedAreaStart, uint32_t releasedAreaEnd) noexcept { uint32_t releasedAreaSize = releasedAreaEnd - releasedAreaStart; // Update the search region and statistics. _pool->totalAreaUsed[size_t(hasLargePages())] -= releasedAreaSize; _areaUsed -= releasedAreaSize; - _searchStart = Support::min(_searchStart, releasedAreaStart); - _searchEnd = Support::max(_searchEnd, releasedAreaEnd); // Unmark occupied bits and also the sentinel. Support::bitVectorClear(_usedBitVector, releasedAreaStart, releasedAreaSize); Support::bitVectorSetBit(_stopBitVector, releasedAreaEnd - 1, false); - if (areaUsed() == initialAreaStart()) { - _searchStart = initialAreaStart(); - _searchEnd = _areaSize; - _largestUnusedArea = _areaSize - initialAreaStart(); - addFlags(kFlagEmpty); - clearFlags(kFlagDirty); + if (Support::bool_and(isIncremental(), _searchStart == releasedAreaEnd)) { + // Incremental mode: If the area released is at the end of the fully used area, we would like to + // keep the incremental mode of the block. In order to do that, we are going to use a different + // approach - decrement `_searchStart` and increment `_largestUnusedArea`, which are essential + // for incremental mode blocks. + ASMJIT_ASSERT(_searchStart >= releasedAreaSize); + _searchStart -= releasedAreaSize; + _largestUnusedArea += releasedAreaSize; } else { - addFlags(kFlagDirty); + _searchStart = Support::min(_searchStart, releasedAreaStart); + _searchEnd = Support::max(_searchEnd, releasedAreaEnd); + clearFlags(kFlagDirty | kFlagIncremental); + + if (areaUsed() == initialAreaStart()) { + _searchStart = initialAreaStart(); + _searchEnd = _areaSize; + _largestUnusedArea = _areaSize - initialAreaStart(); + addFlags(kFlagEmpty); + } + else { + addFlags(kFlagDirty); + } } } - inline void markShrunkArea(uint32_t shrunkAreaStart, uint32_t shrunkAreaEnd) noexcept { + ASMJIT_INLINE void markShrunkArea(uint32_t shrunkAreaStart, uint32_t shrunkAreaEnd) noexcept { uint32_t shrunkAreaSize = shrunkAreaEnd - shrunkAreaStart; // Shrunk area cannot start at zero as it would mean that we have shrunk the first @@ -409,24 +434,32 @@ public: // Update the search region and statistics. _pool->totalAreaUsed[size_t(hasLargePages())] -= shrunkAreaSize; _areaUsed -= shrunkAreaSize; - _searchStart = Support::min(_searchStart, shrunkAreaStart); - _searchEnd = Support::max(_searchEnd, shrunkAreaEnd); + + if (Support::bool_and(isIncremental(), _searchStart == shrunkAreaEnd)) { + _searchStart -= shrunkAreaSize; + _largestUnusedArea += shrunkAreaSize; + } + else { + _searchStart = Support::min(_searchStart, shrunkAreaStart); + _searchEnd = Support::max(_searchEnd, shrunkAreaEnd); + + clearFlags(kFlagIncremental); + addFlags(kFlagDirty); + } // Unmark the released space and move the sentinel. Support::bitVectorClear(_usedBitVector, shrunkAreaStart, shrunkAreaSize); Support::bitVectorSetBit(_stopBitVector, shrunkAreaEnd - 1, false); Support::bitVectorSetBit(_stopBitVector, shrunkAreaStart - 1, true); - - addFlags(kFlagDirty); } // RBTree default CMP uses '<' and '>' operators. - inline bool operator<(const JitAllocatorBlock& other) const noexcept { return rxPtr() < other.rxPtr(); } - inline bool operator>(const JitAllocatorBlock& other) const noexcept { return rxPtr() > other.rxPtr(); } + ASMJIT_INLINE_NODEBUG bool operator<(const JitAllocatorBlock& other) const noexcept { return rxPtr() < other.rxPtr(); } + ASMJIT_INLINE_NODEBUG bool operator>(const JitAllocatorBlock& other) const noexcept { return rxPtr() > other.rxPtr(); } // Special implementation for querying blocks by `key`, which must be in `[BlockPtr, BlockPtr + BlockSize)` range. - inline bool operator<(const uint8_t* key) const noexcept { return rxPtr() + _blockSize <= key; } - inline bool operator>(const uint8_t* key) const noexcept { return rxPtr() > key; } + ASMJIT_INLINE_NODEBUG bool operator<(const uint8_t* key) const noexcept { return rxPtr() + _blockSize <= key; } + ASMJIT_INLINE_NODEBUG bool operator>(const uint8_t* key) const noexcept { return rxPtr() > key; } }; // JitAllocator - PrivateImpl @@ -434,6 +467,9 @@ public: class JitAllocatorPrivateImpl : public JitAllocator::Impl { public: + //! \name Members + //! \{ + //! Lock for thread safety. mutable Lock lock; //! System page size (also a minimum block size). @@ -448,13 +484,15 @@ public: //! Number of allocator pools. size_t poolCount; - inline JitAllocatorPrivateImpl(JitAllocatorPool* pools, size_t poolCount) noexcept + //! \} + + ASMJIT_INLINE JitAllocatorPrivateImpl(JitAllocatorPool* pools, size_t poolCount) noexcept : JitAllocator::Impl {}, pageSize(0), allocationCount(0), pools(pools), poolCount(poolCount) {} - inline ~JitAllocatorPrivateImpl() noexcept {} + ASMJIT_INLINE ~JitAllocatorPrivateImpl() noexcept {} }; static const JitAllocator::Impl JitAllocatorImpl_none {}; @@ -466,8 +504,9 @@ static const JitAllocator::CreateParams JitAllocatorParams_none {}; static inline JitAllocatorPrivateImpl* JitAllocatorImpl_new(const JitAllocator::CreateParams* params) noexcept { VirtMem::Info vmInfo = VirtMem::info(); - if (!params) + if (!params) { params = &JitAllocatorParams_none; + } JitAllocatorOptions options = params->options; uint32_t blockSize = params->blockSize; @@ -528,12 +567,12 @@ static inline JitAllocatorPrivateImpl* JitAllocatorImpl_new(const JitAllocator:: return impl; } -static inline void JitAllocatorImpl_destroy(JitAllocatorPrivateImpl* impl) noexcept { +static ASMJIT_INLINE void JitAllocatorImpl_destroy(JitAllocatorPrivateImpl* impl) noexcept { impl->~JitAllocatorPrivateImpl(); ::free(impl); } -static inline size_t JitAllocatorImpl_sizeToPoolId(const JitAllocatorPrivateImpl* impl, size_t size) noexcept { +static ASMJIT_INLINE size_t JitAllocatorImpl_sizeToPoolId(const JitAllocatorPrivateImpl* impl, size_t size) noexcept { size_t poolId = impl->poolCount - 1; size_t granularity = size_t(impl->granularity) << poolId; @@ -548,12 +587,12 @@ static inline size_t JitAllocatorImpl_sizeToPoolId(const JitAllocatorPrivateImpl return poolId; } -static inline size_t JitAllocatorImpl_bitVectorSizeToByteSize(uint32_t areaSize) noexcept { +static ASMJIT_INLINE size_t JitAllocatorImpl_bitVectorSizeToByteSize(uint32_t areaSize) noexcept { using Support::kBitWordSizeInBits; return ((areaSize + kBitWordSizeInBits - 1u) / kBitWordSizeInBits) * sizeof(Support::BitWord); } -static inline size_t JitAllocatorImpl_calculateIdealBlockSize(JitAllocatorPrivateImpl* impl, JitAllocatorPool* pool, size_t allocationSize) noexcept { +static ASMJIT_INLINE size_t JitAllocatorImpl_calculateIdealBlockSize(JitAllocatorPrivateImpl* impl, JitAllocatorPool* pool, size_t allocationSize) noexcept { JitAllocatorBlock* last = pool->blocks.last(); size_t blockSize = last ? last->blockSize() : size_t(impl->blockSize); @@ -617,8 +656,9 @@ static Error JitAllocatorImpl_newBlock(JitAllocatorPrivateImpl* impl, JitAllocat using Support::kBitWordSizeInBits; uint32_t blockFlags = 0; - if (!Support::test(impl->options, JitAllocatorOptions::kDisableInitialPadding)) + if (!Support::test(impl->options, JitAllocatorOptions::kDisableInitialPadding)) { blockFlags |= JitAllocatorBlock::kFlagInitialPadding; + } VirtMem::DualMapping virtMem {}; VirtMem::MemoryFlags memFlags = VirtMem::MemoryFlags::kAccessRWX; @@ -852,23 +892,19 @@ JitAllocator::Statistics JitAllocator::statistics() const noexcept { // ============================== Error JitAllocator::alloc(Span& out, size_t size) noexcept { - out = Span{}; - - if (ASMJIT_UNLIKELY(_impl == &JitAllocatorImpl_none)) { - return DebugUtils::errored(kErrorNotInitialized); - } + constexpr uint32_t kNoIndex = std::numeric_limits::max(); + constexpr size_t kMaxRequestSize = std::numeric_limits::max() / 2u; JitAllocatorPrivateImpl* impl = static_cast(_impl); - constexpr uint32_t kNoIndex = std::numeric_limits::max(); + bool notInitialized = _impl == &JitAllocatorImpl_none; // Align to the minimum granularity by default. size = Support::alignUp(size, impl->granularity); - if (ASMJIT_UNLIKELY(size == 0)) { - return DebugUtils::errored(kErrorInvalidArgument); - } + out = Span{}; - if (ASMJIT_UNLIKELY(size > std::numeric_limits::max() / 2)) { - return DebugUtils::errored(kErrorTooLarge); + if (ASMJIT_UNLIKELY(Support::bool_or(notInitialized, size - 1u >= kMaxRequestSize))) { + return DebugUtils::errored(notInitialized ? kErrorNotInitialized : + size == 0u ? kErrorInvalidArgument : kErrorTooLarge); } LockGuard guard(impl->lock); @@ -878,13 +914,27 @@ Error JitAllocator::alloc(Span& out, size_t size) noexcept { uint32_t areaSize = uint32_t(pool->areaSizeFromByteSize(size)); // Try to find the requested memory area in existing blocks. - JitAllocatorBlock* block = pool->blocks.first(); + JitAllocatorBlock* block = pool->cursor; + if (block) { JitAllocatorBlock* initial = block; + do { - JitAllocatorBlock* next = block->hasNext() ? block->next() : pool->blocks.first(); - if (block->areaAvailable() >= areaSize) { - if (block->isDirty() || block->largestUnusedArea() >= areaSize) { + uint32_t largestUnusedArea = block->largestUnusedArea(); + + if (Support::bool_and(block->isIncremental(), largestUnusedArea >= areaSize)) { + // Fast path: If the block is in incremental mode, which means that it's guaranteed it's full before + // `searchStart` and completely empty after it, we can just quickly increment `searchStart` and be + // done with the allocation. This is a little bit faster than constructing a BitVectorRangeIterator + // and searching for zero bit clusters. When a block is in incremental mode its `largestUnusedArea` + // is basically the free space after `searchStart`, so that's the only thing to check. + areaIndex = block->_searchStart; + block->_largestUnusedArea -= areaSize; + break; + } + else if (block->areaAvailable() >= areaSize) { + // Regular path: Search for a cluster of bits that would mark an empty area we want to allocate. + if (Support::bool_or(block->isDirty(), largestUnusedArea >= areaSize)) { BitVectorRangeIterator it(block->_usedBitVector, pool->bitWordCountFromAreaSize(block->areaSize()), block->_searchStart, block->_searchEnd); size_t rangeStart = 0; @@ -921,7 +971,9 @@ Error JitAllocator::alloc(Span& out, size_t size) noexcept { } } - block = next; + // The block cursor doesn't have to start with the first block and we want to + // iterate all before concluding that there is no free space in any block. + block = block->hasNext() ? block->next() : pool->blocks.first(); } while (block != initial); } @@ -961,12 +1013,10 @@ Error JitAllocator::alloc(Span& out, size_t size) noexcept { } Error JitAllocator::release(void* rx) noexcept { - if (ASMJIT_UNLIKELY(_impl == &JitAllocatorImpl_none)) { - return DebugUtils::errored(kErrorNotInitialized); - } + bool notInitialized = _impl == &JitAllocatorImpl_none; - if (ASMJIT_UNLIKELY(!rx)) { - return DebugUtils::errored(kErrorInvalidArgument); + if (ASMJIT_UNLIKELY(Support::bool_or(notInitialized, !rx))) { + return DebugUtils::errored(notInitialized ? kErrorNotInitialized : kErrorInvalidArgument); } JitAllocatorPrivateImpl* impl = static_cast(_impl); @@ -999,7 +1049,7 @@ Error JitAllocator::release(void* rx) noexcept { } // Release the whole block if it became empty. - if (block->empty()) { + if (block->isEmpty()) { if (pool->emptyBlockCount || Support::test(impl->options, JitAllocatorOptions::kImmediateRelease)) { JitAllocatorImpl_removeBlock(impl, block); JitAllocatorImpl_deleteBlock(impl, block); @@ -1066,12 +1116,10 @@ static Error JitAllocatorImpl_shrink(JitAllocatorPrivateImpl* impl, JitAllocator } Error JitAllocator::shrink(Span& span, size_t newSize) noexcept { - if (ASMJIT_UNLIKELY(_impl == &JitAllocatorImpl_none)) { - return DebugUtils::errored(kErrorNotInitialized); - } + bool notInitialized = _impl == &JitAllocatorImpl_none; - if (ASMJIT_UNLIKELY(!span.rx())) { - return DebugUtils::errored(kErrorInvalidArgument); + if (ASMJIT_UNLIKELY(Support::bool_or(notInitialized, !span.rx()))) { + return DebugUtils::errored(notInitialized ? kErrorNotInitialized : kErrorInvalidArgument); } if (ASMJIT_UNLIKELY(newSize == 0)) { @@ -1226,56 +1274,6 @@ Error JitAllocator::scopedWrite(WriteScopeData& scope, Span& span, WriteFunc wri // ==================== #if defined(ASMJIT_TEST) -// A pseudo random number generator based on a paper by Sebastiano Vigna: -// http://vigna.di.unimi.it/ftp/papers/xorshiftplus.pdf -class Random { -public: - // Constants suggested as `23/18/5`. - enum Steps : uint32_t { - kStep1_SHL = 23, - kStep2_SHR = 18, - kStep3_SHR = 5 - }; - - inline explicit Random(uint64_t seed = 0) noexcept { reset(seed); } - inline Random(const Random& other) noexcept = default; - - inline void reset(uint64_t seed = 0) noexcept { - // The number is arbitrary, it means nothing. - constexpr uint64_t kZeroSeed = 0x1F0A2BE71D163FA0u; - - // Generate the state data by using splitmix64. - for (uint32_t i = 0; i < 2; i++) { - seed += 0x9E3779B97F4A7C15u; - uint64_t x = seed; - x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9u; - x = (x ^ (x >> 27)) * 0x94D049BB133111EBu; - x = (x ^ (x >> 31)); - _state[i] = x != 0 ? x : kZeroSeed; - } - } - - inline uint32_t nextUInt32() noexcept { - return uint32_t(nextUInt64() >> 32); - } - - inline uint64_t nextUInt64() noexcept { - uint64_t x = _state[0]; - uint64_t y = _state[1]; - - x ^= x << kStep1_SHL; - y ^= y >> kStep3_SHR; - x ^= x >> kStep2_SHR; - x ^= y; - - _state[0] = y; - _state[1] = x; - return x + y; - } - - uint64_t _state[2]; -}; - namespace JitAllocatorUtils { static void fillPattern64(void* p_, uint64_t pattern, size_t sizeInBytes) noexcept { uint64_t* p = static_cast(p_); @@ -1343,20 +1341,19 @@ public: }; Zone _zone; - ZoneAllocator _heap; + ZonePool _recordPool; ZoneTree _records; JitAllocator _allocator; - Random _rng; + TestUtils::Random _rng; explicit JitAllocatorWrapper(const JitAllocator::CreateParams* params) noexcept : _zone(1024u * 1024u), - _heap(&_zone), _allocator(params), _rng(0x123456789u) {} void _insert(void* pRX, void* pRW, size_t size) noexcept { uint8_t* p = static_cast(pRX); - uint8_t* pEnd = p + size - 1; + uint8_t* pEnd = p + size - 1u; Record* record; @@ -1369,7 +1366,7 @@ public: .message("Address [%p:%p] collides with a newly allocated [%p:%p]\n", record->addr, record->addr + record->size, p, p + size); uint64_t pattern = _rng.nextUInt64(); - record = _heap.newT(pRX, pRW, size, pattern); + record = new(Support::PlacementNew{_recordPool.alloc(_zone)}) Record(pRX, pRW, size, pattern); EXPECT_NOT_NULL(record); { @@ -1391,7 +1388,7 @@ public: EXPECT_TRUE(JitAllocatorUtils::verifyPattern64(record->rw(), record->pattern, record->size)); _records.remove(record); - _heap.release(record, sizeof(Record)); + _recordPool.release(record); } void* alloc(size_t size) noexcept { @@ -1428,7 +1425,7 @@ public: } }; -static void JitAllocatorTest_shuffle(void** ptrArray, size_t count, Random& prng) noexcept { +static void JitAllocatorTest_shuffle(void** ptrArray, size_t count, TestUtils::Random& prng) noexcept { for (size_t i = 0; i < count; ++i) std::swap(ptrArray[i], ptrArray[size_t(prng.nextUInt32() % count)]); } @@ -1442,7 +1439,7 @@ static void JitAllocatorTest_usage(JitAllocator& allocator) noexcept { } template -static void BitVectorRangeIterator_testRandom(Random& rnd, size_t count) noexcept { +static void BitVectorRangeIterator_testRandom(TestUtils::Random& rnd, size_t count) noexcept { for (size_t i = 0; i < count; i++) { T in[kPatternSize]; T out[kPatternSize]; @@ -1508,13 +1505,13 @@ static void test_jit_allocator_alloc_release() noexcept { INFO("BitVectorRangeIterator"); { - Random rnd; + TestUtils::Random rnd; BitVectorRangeIterator_testRandom(rnd, kCount); } INFO("BitVectorRangeIterator"); { - Random rnd; + TestUtils::Random rnd; BitVectorRangeIterator_testRandom(rnd, kCount); } @@ -1535,7 +1532,7 @@ static void test_jit_allocator_alloc_release() noexcept { size_t fixedBlockSize = 256; JitAllocatorWrapper wrapper(¶ms); - Random prng(100); + TestUtils::Random prng(100); size_t i; diff --git a/src/asmjit/core/jitallocator.h b/src/asmjit/core/jitallocator.h index 28ec4d5..4e4c49b 100644 --- a/src/asmjit/core/jitallocator.h +++ b/src/asmjit/core/jitallocator.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_JITALLOCATOR_H_INCLUDED diff --git a/src/asmjit/core/jitruntime.cpp b/src/asmjit/core/jitruntime.cpp index c117c75..cfb4c03 100644 --- a/src/asmjit/core/jitruntime.cpp +++ b/src/asmjit/core/jitruntime.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -24,7 +24,7 @@ Error JitRuntime::_add(void** dst, CodeHolder* code) noexcept { *dst = nullptr; ASMJIT_PROPAGATE(code->flatten()); - ASMJIT_PROPAGATE(code->resolveUnresolvedLinks()); + ASMJIT_PROPAGATE(code->resolveCrossSectionFixups()); size_t estimatedCodeSize = code->codeSize(); if (ASMJIT_UNLIKELY(estimatedCodeSize == 0)) { @@ -35,7 +35,8 @@ Error JitRuntime::_add(void** dst, CodeHolder* code) noexcept { ASMJIT_PROPAGATE(_allocator.alloc(span, estimatedCodeSize)); // Relocate the code. - Error err = code->relocateToBase(uintptr_t(span.rx())); + CodeHolder::RelocationSummary relocationSummary; + Error err = code->relocateToBase(uintptr_t(span.rx()), &relocationSummary); if (ASMJIT_UNLIKELY(err)) { _allocator.release(span.rx()); return err; @@ -43,8 +44,10 @@ Error JitRuntime::_add(void** dst, CodeHolder* code) noexcept { // Recalculate the final code size and shrink the memory we allocated for it // in case that some relocations didn't require records in an address table. - size_t codeSize = code->codeSize(); - ASMJIT_ASSERT(codeSize <= estimatedCodeSize); + size_t codeSize = estimatedCodeSize - relocationSummary.codeSizeReduction; + + // If not true it means that `relocateToBase()` filled wrong information in `relocationSummary`. + ASMJIT_ASSERT(codeSize == code->codeSize()); _allocator.write(span, [&](JitAllocator::Span& span) noexcept -> Error { uint8_t* rw = static_cast(span.rw()); diff --git a/src/asmjit/core/jitruntime.h b/src/asmjit/core/jitruntime.h index acea4a3..4b726ea 100644 --- a/src/asmjit/core/jitruntime.h +++ b/src/asmjit/core/jitruntime.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_JITRUNTIME_H_INCLUDED @@ -36,6 +36,11 @@ public: //! Creates a `JitRuntime` instance. ASMJIT_API explicit JitRuntime(const JitAllocator::CreateParams* params = nullptr) noexcept; + + //! Creates a `JitRuntime` instance. + ASMJIT_INLINE explicit JitRuntime(const JitAllocator::CreateParams& params) noexcept + : JitRuntime(¶ms) {} + //! Destroys the `JitRuntime` instance. ASMJIT_API ~JitRuntime() noexcept override; diff --git a/src/asmjit/core/logger.cpp b/src/asmjit/core/logger.cpp index 2b9dd9b..dfe4489 100644 --- a/src/asmjit/core/logger.cpp +++ b/src/asmjit/core/logger.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/logger.h b/src/asmjit/core/logger.h index 1d8337e..b91e951 100644 --- a/src/asmjit/core/logger.h +++ b/src/asmjit/core/logger.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_LOGGING_H_INCLUDED diff --git a/src/asmjit/core/misc_p.h b/src/asmjit/core/misc_p.h index 5cd934e..284b9eb 100644 --- a/src/asmjit/core/misc_p.h +++ b/src/asmjit/core/misc_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_MISC_P_H_INCLUDED diff --git a/src/asmjit/core/operand.cpp b/src/asmjit/core/operand.cpp index b78dc54..252d378 100644 --- a/src/asmjit/core/operand.cpp +++ b/src/asmjit/core/operand.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -20,7 +20,7 @@ enum class StrongEnumForImmTests : uint32_t { UNIT(operand) { INFO("Checking operand sizes"); EXPECT_EQ(sizeof(Operand), 16u); - EXPECT_EQ(sizeof(BaseReg), 16u); + EXPECT_EQ(sizeof(Reg), 16u); EXPECT_EQ(sizeof(BaseMem), 16u); EXPECT_EQ(sizeof(Imm), 16u); EXPECT_EQ(sizeof(Label), 16u); @@ -43,19 +43,19 @@ UNIT(operand) { EXPECT_FALSE(label.isValid()); EXPECT_EQ(label.id(), Globals::kInvalidId); - INFO("Checking basic functionality of BaseReg"); - EXPECT_TRUE(BaseReg().isReg()); - EXPECT_FALSE(BaseReg().isValid()); - EXPECT_EQ(BaseReg()._data[0], 0u); - EXPECT_EQ(BaseReg()._data[1], 0u); - EXPECT_FALSE(dummy.as().isValid()); + INFO("Checking basic functionality of Reg"); + EXPECT_TRUE(Reg().isReg()); + EXPECT_FALSE(Reg().isValid()); + EXPECT_EQ(Reg()._data[0], 0u); + EXPECT_EQ(Reg()._data[1], 0u); + EXPECT_FALSE(dummy.as().isValid()); // Create some register (not specific to any architecture). OperandSignature rSig = OperandSignature::fromOpType(OperandType::kReg) | OperandSignature::fromRegType(RegType::kVec128) | OperandSignature::fromRegGroup(RegGroup::kVec) | OperandSignature::fromSize(8); - BaseReg r1(rSig, 5); + Reg r1(rSig, 5); EXPECT_TRUE(r1.isValid()); EXPECT_TRUE(r1.isReg()); @@ -63,8 +63,8 @@ UNIT(operand) { EXPECT_TRUE(r1.isPhysReg()); EXPECT_FALSE(r1.isVirtReg()); EXPECT_EQ(r1.signature(), rSig); - EXPECT_EQ(r1.type(), RegType::kVec128); - EXPECT_EQ(r1.group(), RegGroup::kVec); + EXPECT_EQ(r1.regType(), RegType::kVec128); + EXPECT_EQ(r1.regGroup(), RegGroup::kVec); EXPECT_EQ(r1.size(), 8u); EXPECT_EQ(r1.id(), 5u); EXPECT_TRUE(r1.isReg(RegType::kVec128, 5)); // RegType and Id. @@ -72,15 +72,15 @@ UNIT(operand) { EXPECT_EQ(r1._data[1], 0u); // The same type of register having different id. - BaseReg r2(r1, 6); + Reg r2(r1, 6); EXPECT_TRUE(r2.isValid()); EXPECT_TRUE(r2.isReg()); EXPECT_TRUE(r2.isReg(RegType::kVec128)); EXPECT_TRUE(r2.isPhysReg()); EXPECT_FALSE(r2.isVirtReg()); EXPECT_EQ(r2.signature(), rSig); - EXPECT_EQ(r2.type(), r1.type()); - EXPECT_EQ(r2.group(), r1.group()); + EXPECT_EQ(r2.regType(), r1.regType()); + EXPECT_EQ(r2.regGroup(), r1.regGroup()); EXPECT_EQ(r2.size(), r1.size()); EXPECT_EQ(r2.id(), 6u); EXPECT_TRUE(r2.isReg(RegType::kVec128, 6)); diff --git a/src/asmjit/core/operand.h b/src/asmjit/core/operand.h index 8a90084..6ba988f 100644 --- a/src/asmjit/core/operand.h +++ b/src/asmjit/core/operand.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_OPERAND_H_INCLUDED @@ -35,7 +35,7 @@ enum class OperandType : uint32_t { }; static_assert(uint32_t(OperandType::kMem) == uint32_t(OperandType::kReg) + 1, - "AsmJit requires that `OperandType::kMem` equals to `OperandType::kReg + 1`"); + "AsmJit requires that `OperandType::kMem` equals `OperandType::kReg + 1`"); //! Register mask is a convenience typedef that describes a mask where each bit describes a physical register id //! in the same \ref RegGroup. At the moment 32 bits are enough as AsmJit doesn't support any architecture that @@ -56,106 +56,65 @@ enum class RegType : uint8_t { //! operand (register vs. label) by just assigning its type (register type or label-tag) and operand id. kLabelTag = 1, - //! Universal type describing program counter (PC) or instruction pointer (IP) register, if the target architecture - //! actually exposes it as a separate register type, which most modern architectures do. - kPC = 2, + //! 8-bit low general purpose register (X86|X86_64). + kGp8Lo = 2, + //! 8-bit high general purpose register (X86|X86_64). + kGp8Hi = 3, + //! 16-bit general purpose register (X86|X86_64). + kGp16 = 4, + //! 32-bit general purpose register (X86|X86_64|AArch32|AArch64). + kGp32 = 5, + //! 64-bit general purpose register (X86_64|AArch64). + kGp64 = 6, - //! 8-bit low general purpose register (X86). - kGp8Lo = 3, - //! 8-bit high general purpose register (X86). - kGp8Hi = 4, - //! 16-bit general purpose register (X86). - kGp16 = 5, - //! 32-bit general purpose register (X86|AArch32|AArch64). - kGp32 = 6, - //! 64-bit general purpose register (X86|AArch64). - kGp64 = 7, //! 8-bit view of a vector register (AArch64). - kVec8 = 8, + kVec8 = 7, //! 16-bit view of a vector register (AArch64). - kVec16 = 9, + kVec16 = 8, //! 32-bit view of a vector register (AArch32|AArch64). - kVec32 = 10, + kVec32 = 9, //! 64-bit view of a vector register (AArch32|AArch64). //! //! \note This is never used for MMX registers on X86, MMX registers have its own category. - kVec64 = 11, - //! 128-bit view of a vector register (X86|AArch32|AArch64). - kVec128 = 12, - //! 256-bit view of a vector register (X86). - kVec256 = 13, - //! 512-bit view of a vector register (X86). - kVec512 = 14, + kVec64 = 10, + //! 128-bit view of a vector register (X86|X86_64|AArch32|AArch64). + kVec128 = 11, + //! 256-bit view of a vector register (X86|X86_64). + kVec256 = 12, + //! 512-bit view of a vector register (X86|X86_64). + kVec512 = 13, //! 1024-bit view of a vector register (future). - kVec1024 = 15, + kVec1024 = 14, //! View of a vector register, which width is implementation specific (AArch64). - kVecNLen = 16, + kVecNLen = 15, - //! Mask register (X86). - kMask = 17, + //! Mask register (X86|X86_64|AArch64). + kMask = 16, + //! Tile register (X86_64: `TMM`). + kTile = 17, - //! Start of architecture dependent register types. - kExtra = 18, + //! Segment register (X86|X86_64: None, ES, CS, SS, DS, FS, GS). + kSegment = 25, + //! Control register (X86|X86_64: `CR`). + kControl = 26, + //! Debug register (X86|X86_64: `DR`). + kDebug = 27, - // X86 Specific Register Types - // --------------------------- + //! MMX register (X86|X86_64: `MM`). + kX86_Mm = 28, + //! FPU (x87) register (X86|X86_64: `ST`). + kX86_St = 29, + //! Bound register (X86|X86_64: `BND`). + kX86_Bnd = 30, + //! Universal type describing program counter (PC) or instruction pointer (EIP/RIP) register, if the target + //! architecture actually exposes it as a separate register type, which most modern architectures do. + //! + //! X86 Specific + //! ------------ + //! //! Instruction pointer (RIP), only addressable in \ref x86::Mem in 64-bit targets. - kX86_Rip = kPC, - //! Low GPB register (AL, BL, CL, DL, ...). - kX86_GpbLo = kGp8Lo, - //! High GPB register (AH, BH, CH, DH only). - kX86_GpbHi = kGp8Hi, - //! GPW register. - kX86_Gpw = kGp16, - //! GPD register. - kX86_Gpd = kGp32, - //! GPQ register (64-bit). - kX86_Gpq = kGp64, - //! XMM register (SSE+). - kX86_Xmm = kVec128, - //! YMM register (AVX+). - kX86_Ymm = kVec256, - //! ZMM register (AVX512+). - kX86_Zmm = kVec512, - //! K register (AVX512+). - kX86_KReg = kMask, - //! MMX register. - kX86_Mm = kExtra + 0, - //! Segment register (None, ES, CS, SS, DS, FS, GS). - kX86_SReg = kExtra + 1, - //! Control register (CR). - kX86_CReg = kExtra + 2, - //! Debug register (DR). - kX86_DReg = kExtra + 3, - //! FPU (x87) register. - kX86_St = kExtra + 4, - //! Bound register (BND). - kX86_Bnd = kExtra + 5, - //! TMM register (AMX_TILE) - kX86_Tmm = kExtra + 6, - - // ARM Specific Register Types - // --------------------------- - - //! Program pointer (PC) register (AArch64). - kARM_PC = kPC, - //! 32-bit general purpose register (R or W). - kARM_GpW = kGp32, - //! 64-bit general purpose register (X). - kARM_GpX = kGp64, - //! 8-bit view of VFP/ASIMD register (B). - kARM_VecB = kVec8, - //! 16-bit view of VFP/ASIMD register (H). - kARM_VecH = kVec16, - //! 32-bit view of VFP/ASIMD register (S). - kARM_VecS = kVec32, - //! 64-bit view of VFP/ASIMD register (D). - kARM_VecD = kVec64, - //! 128-bit view of VFP/ASIMD register (Q). - kARM_VecQ = kVec128, - //! 128-bit view of VFP/ASIMD register (V). - kARM_VecV = kVec128, + kPC = 31, //! Maximum value of `RegType`. kMaxValue = 31 @@ -168,51 +127,47 @@ ASMJIT_DEFINE_ENUM_COMPARE(RegType) enum class RegGroup : uint8_t { //! General purpose register group compatible with all backends. kGp = 0, + //! Vector register group compatible with all backends. //! - //! Describes X86 XMM|YMM|ZMM registers ARM/AArch64 V registers. + //! Describes `XMM|YMM|ZMM` registers on X86|X86_64 targets and `V|Q|D|S|H|B` registers on ARM/AArch64 targets. kVec = 1, //! Mask register group compatible with all backends that can use masking. + //! + //! Describes `K` registers on X86|X86_64 targets (AVX-512) and `P` registers on AArch64 targets (SVE/SVE2). kMask = 2, + //! Extra virtual group #3 that can be used by Compiler for register allocation. kExtraVirt3 = 3, - //! Program counter group. - kPC = 4, + //! TMM register group (X86|X86_64). + kTile = 4, - //! Extra non-virtual group that can be used by registers not managed by Compiler. - kExtraNonVirt = 5, + //! Segment register group (X86|X86_64). + kSegment = 10, - // X86 Specific Register Groups - // ---------------------------- + //! Control register group (X86|X86_64). + kControl = 11, - //! K register group (KReg) - maps to \ref RegGroup::kMask (X86, X86_64). - kX86_K = kMask, - //! MMX register group (MM) - maps to \ref RegGroup::kExtraVirt3 (X86, X86_64). + //! Debug register group (X86|X86_64). + kDebug = 12, + + //! MMX register group (MM) - maps to \ref RegGroup::kExtraVirt3 (X86|X86_64). kX86_MM = kExtraVirt3, + //! FPU register group (X86|X86_64). + kX86_St = 13, + //! BND register group (X86|X86_64). + kX86_Bnd = 14, - //! Instruction pointer (X86, X86_64). - kX86_Rip = kPC, - //! Segment register group (X86, X86_64). - kX86_SReg = kExtraNonVirt + 0, - //! CR register group (X86, X86_64). - kX86_CReg = kExtraNonVirt + 1, - //! DR register group (X86, X86_64). - kX86_DReg = kExtraNonVirt + 2, - //! FPU register group (X86, X86_64). - kX86_St = kExtraNonVirt + 3, - //! BND register group (X86, X86_64). - kX86_Bnd = kExtraNonVirt + 4, - //! TMM register group (X86, X86_64). - kX86_Tmm = kExtraNonVirt + 5, + //! Program counter group (represents also EIP/RIP on X86|X86_64 targets). + kPC = 15, - //! First group - only used in loops. - k0 = 0, - //! Last value of a virtual register that is managed by \ref BaseCompiler. - kMaxVirt = Globals::kNumVirtGroups - 1, //! Maximum value of `RegGroup`. - kMaxValue = 15 + kMaxValue = 15, + + //! Last value of a virtual register that is managed by \ref BaseCompiler. + kMaxVirt = Globals::kNumVirtGroups - 1 }; ASMJIT_DEFINE_ENUM_COMPARE(RegGroup) @@ -287,6 +242,72 @@ struct OperandSignature { //! \} + //! \name Static Constructors + //! \{ + + //! Constructs operand signature from the given `bits`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromBits(uint32_t bits) noexcept { + return OperandSignature{bits}; + } + + //! Constructs operand signature from the given `value`, use `kFieldMask` to describe where the value is in the signature. + template + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromValue(const T& value) noexcept { + return OperandSignature{uint32_t(value) << Support::ConstCTZ::value}; + } + + //! Constructs operand signature describing the given operand type `opType`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromOpType(OperandType opType) noexcept { + return OperandSignature{uint32_t(opType) << kOpTypeShift}; + } + + //! Constructs operand signature describing the given register type `regType`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromRegType(RegType regType) noexcept { + return OperandSignature{uint32_t(regType) << kRegTypeShift}; + } + + //! Constructs operand signature describing the given register group `regGroup`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromRegGroup(RegGroup regGroup) noexcept { + return OperandSignature{uint32_t(regGroup) << kRegGroupShift}; + } + + //! Constructs operand signature describing both register type `regType` and register group `regGroup`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromRegTypeAndGroup(RegType regType, RegGroup regGroup) noexcept { + return fromRegType(regType) | fromRegGroup(regGroup); + } + + //! Constructs operand signature describing a memory base type `baseType`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromMemBaseType(RegType baseType) noexcept { + return OperandSignature{uint32_t(baseType) << kMemBaseTypeShift}; + } + + //! Constructs operand signature describing a memory index type `indexType`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromMemIndexType(RegType indexType) noexcept { + return OperandSignature{uint32_t(indexType) << kMemIndexTypeShift}; + } + + //! Constructs operand signature describing a `predicate`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromPredicate(uint32_t predicate) noexcept { + return OperandSignature{predicate << kPredicateShift}; + } + + //! Constructs operand signature describing a `size`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromSize(uint32_t size) noexcept { + return OperandSignature{size << kSizeShift}; + } + + //! \} + //! \name Overloaded Operators //! //! Overloaded operators make `OperandSignature` behave like regular integer. @@ -400,6 +421,28 @@ struct OperandSignature { [[nodiscard]] ASMJIT_INLINE_CONSTEXPR OperandType opType() const noexcept { return (OperandType)getField(); } + //! Tests whether the operand type matches opType + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isOpType(OperandType opType) const noexcept { return getField() == uint32_t(opType); } + + //! Tests whether the operand signature represents a register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg() const noexcept { return isOpType(OperandType::kReg); } + + //! Tests whether the operand signature represents a register of the given register type `regType`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegType regType) const noexcept { + constexpr uint32_t kMask = kOpTypeMask | kRegTypeMask; + return subset(kMask) == (fromOpType(OperandType::kReg) | fromRegType(regType)); + } + + //! Tests whether the operand signature represents a register of the given register type `regType`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegGroup regGroup) const noexcept { + constexpr uint32_t kMask = kOpTypeMask | kRegGroupMask; + return subset(kMask) == (fromOpType(OperandType::kReg) | fromRegGroup(regGroup)); + } + [[nodiscard]] ASMJIT_INLINE_CONSTEXPR RegType regType() const noexcept { return (RegType)getField(); } @@ -429,72 +472,6 @@ struct OperandSignature { ASMJIT_INLINE_CONSTEXPR void setSize(uint32_t size) noexcept { setField(size); } //! \} - - //! \name Static Constructors - //! \{ - - //! Constructs operand signature from the given `bits`. - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR OperandSignature fromBits(uint32_t bits) noexcept { - return OperandSignature{bits}; - } - - //! Constructs operand signature from the given `value`, use `kFieldMask` to describe where the value is in the signature. - template - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR OperandSignature fromValue(const T& value) noexcept { - return OperandSignature{uint32_t(value) << Support::ConstCTZ::value}; - } - - //! Constructs operand signature describing the given operand type `opType`. - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR OperandSignature fromOpType(OperandType opType) noexcept { - return OperandSignature{uint32_t(opType) << kOpTypeShift}; - } - - //! Constructs operand signature describing the given register type `regType`. - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR OperandSignature fromRegType(RegType regType) noexcept { - return OperandSignature{uint32_t(regType) << kRegTypeShift}; - } - - //! Constructs operand signature describing the given register group `regGroup`. - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR OperandSignature fromRegGroup(RegGroup regGroup) noexcept { - return OperandSignature{uint32_t(regGroup) << kRegGroupShift}; - } - - //! Constructs operand signature describing both register type `regType` and register group `regGroup`. - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR OperandSignature fromRegTypeAndGroup(RegType regType, RegGroup regGroup) noexcept { - return fromRegType(regType) | fromRegGroup(regGroup); - } - - //! Constructs operand signature describing a memory base type `baseType`. - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR OperandSignature fromMemBaseType(RegType baseType) noexcept { - return OperandSignature{uint32_t(baseType) << kMemBaseTypeShift}; - } - - //! Constructs operand signature describing a memory index type `indexType`. - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR OperandSignature fromMemIndexType(RegType indexType) noexcept { - return OperandSignature{uint32_t(indexType) << kMemIndexTypeShift}; - } - - //! Constructs operand signature describing a `predicate`. - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR OperandSignature fromPredicate(uint32_t predicate) noexcept { - return OperandSignature{predicate << kPredicateShift}; - } - - //! Constructs operand signature describing a `size`. - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR OperandSignature fromSize(uint32_t size) noexcept { - return OperandSignature{size << kSizeShift}; - } - - //! \} }; //! Base class representing an operand in AsmJit (non-default constructed version). @@ -521,24 +498,22 @@ struct Operand_ { //! \name Constants //! \{ - // Indexes to `_data` array. - enum DataIndex : uint32_t { - kDataMemIndexId = 0, - kDataMemOffsetLo = 1, + //! Memory index offset in a `_data[2]` array. + static inline constexpr uint32_t kDataMemIndexId = 0; + //! Low 32-bit offset value a `_data[2]` array. + static inline constexpr uint32_t kDataMemOffsetLo = 1; - kDataImmValueLo = ASMJIT_ARCH_LE ? 0 : 1, - kDataImmValueHi = ASMJIT_ARCH_LE ? 1 : 0 - }; + //! Low 32-bit immediate value in a `_data[2]` array. + static inline constexpr uint32_t kDataImmValueLo = ASMJIT_ARCH_LE ? 0 : 1; + //! High 32-bit immediate value in a `_data[2]` array. + static inline constexpr uint32_t kDataImmValueHi = ASMJIT_ARCH_LE ? 1 : 0; - //! Constants useful for VirtId <-> Index translation. - enum VirtIdConstants : uint32_t { - //! Minimum valid packed-id. - kVirtIdMin = 256, - //! Maximum valid packed-id, excludes Globals::kInvalidId. - kVirtIdMax = Globals::kInvalidId - 1, - //! Count of valid packed-ids. - kVirtIdCount = uint32_t(kVirtIdMax - kVirtIdMin + 1) - }; + //! Minimum valid packed-id. + static inline constexpr uint32_t kVirtIdMin = 256; + //! Maximum valid packed-id, excludes Globals::kInvalidId. + static inline constexpr uint32_t kVirtIdMax = Globals::kInvalidId - 1; + //! Count of valid packed-ids. + static inline constexpr uint32_t kVirtIdCount = uint32_t(kVirtIdMax - kVirtIdMin + 1); //! \} @@ -577,7 +552,7 @@ struct Operand_ { //! \{ //! \cond INTERNAL - //! Initializes a `BaseReg` operand from `signature` and register `id`. + //! Initializes a `Reg` operand from `signature` and register `id`. ASMJIT_INLINE_CONSTEXPR void _initReg(const Signature& signature, uint32_t id) noexcept { _signature = signature; _baseId = id; @@ -635,7 +610,7 @@ struct Operand_ { //! Tests whether this operand is the same as `other`. [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool operator==(const Operand_& other) const noexcept { return equals(other); } + ASMJIT_INLINE_CONSTEXPR bool operator==(const Operand_& other) const noexcept { return equals(other); } //! Tests whether this operand is not the same as `other`. [[nodiscard]] @@ -667,25 +642,19 @@ struct Operand_ { //! different the operands are not equal. [[nodiscard]] ASMJIT_INLINE_CONSTEXPR bool equals(const Operand_& other) const noexcept { - return bool(unsigned(_signature == other._signature) & - unsigned(_baseId == other._baseId ) & - unsigned(_data[0] == other._data[0] ) & - unsigned(_data[1] == other._data[1] )); + return Support::bool_and( + _signature == other._signature, + _baseId == other._baseId, + _data[0] == other._data[0], + _data[1] == other._data[1] + ); } //! \} - //! \name Accessors + //! \name Generic Accessors //! \{ - //! Tests whether the operand's signature matches the signature of the `other` operand. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool hasSignature(const Operand_& other) const noexcept { return _signature == other._signature; } - - //! Tests whether the operand's signature matches the given signature `sign`. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool hasSignature(const Signature& other) const noexcept { return _signature == other; } - //! Returns operand signature as unsigned 32-bit integer. //! //! Signature is first 4 bytes of the operand data. It's used mostly for operand checking as it's @@ -693,6 +662,14 @@ struct Operand_ { [[nodiscard]] ASMJIT_INLINE_CONSTEXPR Signature signature() const noexcept { return _signature; } + //! Tests whether the operand's signature matches the signature of the `other` operand. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasSignature(const Operand_& other) const noexcept { return _signature == other._signature; } + + //! Tests whether the operand's signature matches the given signature `sign`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasSignature(Signature sign) const noexcept { return _signature == sign; } + //! Sets the operand signature, see `signature()`. //! //! \note Improper use of `setSignature()` can lead to hard-to-debug errors. @@ -705,125 +682,35 @@ struct Operand_ { [[nodiscard]] ASMJIT_INLINE_CONSTEXPR OperandType opType() const noexcept { return _signature.opType(); } + //! Tests whether the operand's type matches the given `type`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isOpType(OperandType opType) const noexcept { return _signature.isOpType(opType); } + //! Tests whether the operand is none (`OperandType::kNone`). [[nodiscard]] ASMJIT_INLINE_CONSTEXPR bool isNone() const noexcept { return _signature == Signature::fromBits(0); } //! Tests whether the operand is a register (`OperandType::kReg`). [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isReg() const noexcept { return opType() == OperandType::kReg; } + ASMJIT_INLINE_CONSTEXPR bool isReg() const noexcept { return isOpType(OperandType::kReg); } //! Tests whether the operand is a register-list. //! //! \note Register-list is currently only used by 32-bit ARM architecture. [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isRegList() const noexcept { return opType() == OperandType::kRegList; } + ASMJIT_INLINE_CONSTEXPR bool isRegList() const noexcept { return isOpType(OperandType::kRegList); } //! Tests whether the operand is a memory location (`OperandType::kMem`). [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isMem() const noexcept { return opType() == OperandType::kMem; } + ASMJIT_INLINE_CONSTEXPR bool isMem() const noexcept { return isOpType(OperandType::kMem); } //! Tests whether the operand is an immediate (`OperandType::kImm`). [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isImm() const noexcept { return opType() == OperandType::kImm; } + ASMJIT_INLINE_CONSTEXPR bool isImm() const noexcept { return isOpType(OperandType::kImm); } //! Tests whether the operand is a label (`OperandType::kLabel`). [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isLabel() const noexcept { return opType() == OperandType::kLabel; } - - //! Tests whether the operand is a physical register. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isPhysReg() const noexcept { return isReg() && _baseId < 0xFFu; } - - //! Tests whether the operand is a virtual register. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVirtReg() const noexcept { return isReg() && _baseId > 0xFFu; } - - //! Returns the operand id. - //! - //! The value returned should be interpreted accordingly to the operand type: - //! * None - Should be `0`. - //! * Reg - Physical or virtual register id. - //! * Mem - Multiple meanings - BASE address (register or label id), or high value of a 64-bit absolute address. - //! * Imm - Should be `0`. - //! * Label - Label id if it was created by using `newLabel()` or `Globals::kInvalidId` if the label is invalid or - //! not initialized. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR uint32_t id() const noexcept { return _baseId; } - - //! Tests whether the operand is a register matching the given register `type`. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isReg(RegType type) const noexcept { - return _signature.subset(Signature::kOpTypeMask | Signature::kRegTypeMask) == (Signature::fromOpType(OperandType::kReg) | Signature::fromRegType(type)); - } - - //! Tests whether the operand is a register of the provided register group `regGroup`. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isReg(RegGroup regGroup) const noexcept { - return _signature.subset(Signature::kOpTypeMask | Signature::kRegGroupMask) == (Signature::fromOpType(OperandType::kReg) | Signature::fromRegGroup(regGroup)); - } - - //! Tests whether the operand is register and of register type `regType` and `regId`. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isReg(RegType regType, uint32_t regId) const noexcept { return isReg(regType) && _baseId == regId; } - - //! Tests whether the operand is register and of register group `regGroup` and `regId`. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isReg(RegGroup regGroup, uint32_t regId) const noexcept { return isReg(regGroup) && _baseId == regId; } - - //! Tests whether the register is a general purpose register (any size). - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isGp() const noexcept { return isReg(RegGroup::kGp); } - - //! Tests whether the register is a 32-bit general purpose register. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isGp32() const noexcept { return isReg(RegType::kGp32); } - - //! Tests whether the register is a 64-bit general purpose register. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isGp64() const noexcept { return isReg(RegType::kGp64); } - - //! Tests whether the register is a vector register of any size. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVec() const noexcept { return isReg(RegGroup::kVec); } - - //! Tests whether the register is an 8-bit vector register or view (AArch64). - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVec8() const noexcept { return isReg(RegType::kVec8); } - - //! Tests whether the register is a 16-bit vector register or view (AArch64). - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVec16() const noexcept { return isReg(RegType::kVec16); } - - //! Tests whether the register is a 32-bit vector register or view (AArch32, AArch64). - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVec32() const noexcept { return isReg(RegType::kVec32); } - - //! Tests whether the register is a 64-bit vector register or view (AArch32, AArch64). - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVec64() const noexcept { return isReg(RegType::kVec64); } - - //! Tests whether the register is a 128-bit vector register or view (AArch32, AArch64, X86, X86_64). - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVec128() const noexcept { return isReg(RegType::kVec128); } - - //! Tests whether the register is a 256-bit vector register or view (X86, X86_64). - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVec256() const noexcept { return isReg(RegType::kVec256); } - - //! Tests whether the register is a 512-bit vector register or view (X86, X86_64). - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVec512() const noexcept { return isReg(RegType::kVec512); } - - //! Tests whether the register is a mask register of any size. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isMask() const noexcept { return isReg(RegGroup::kMask); } - - //! Tests whether the operand is a register matching the given register `type`. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isRegList(RegType type) const noexcept { - return _signature.subset(Signature::kOpTypeMask | Signature::kRegTypeMask) == (Signature::fromOpType(OperandType::kRegList) | Signature::fromRegType(type)); - } + ASMJIT_INLINE_CONSTEXPR bool isLabel() const noexcept { return isOpType(OperandType::kLabel); } //! Tests whether the operand is a register or memory. //! @@ -845,9 +732,267 @@ struct Operand_ { return Support::isBetween(uint32_t(opType()), uint32_t(OperandType::kReg), uint32_t(OperandType::kRegList)); } + //! Returns the operand id. + //! + //! The value returned should be interpreted accordingly to the operand type: + //! * None - Should be `0`. + //! * Reg - Physical or virtual register id. + //! * Mem - Multiple meanings - BASE address (register or label id), or high value of a 64-bit absolute address. + //! * Imm - Should be `0`. + //! * Label - Label id if it was created by using `newLabel()` or `Globals::kInvalidId` if the label is invalid or + //! not initialized. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t id() const noexcept { return _baseId; } + //! \} - //! \name Accessors (X86 Specific) + //! \name Register Accessors + //! \{ + + //! Tests whether the operand is a physical register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isPhysReg() const noexcept { return isReg() && _baseId < 0xFFu; } + + //! Tests whether the operand is a virtual register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVirtReg() const noexcept { return isReg() && _baseId > 0xFFu; } + + //! Tests whether the operand is a register matching the given register `type`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegType regType) const noexcept { return _signature.isReg(regType); } + + //! Tests whether the operand is register and of register type `regType` and `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegType regType, uint32_t regId) const noexcept { return Support::bool_and(isReg(regType), _baseId == regId); } + + //! Tests whether the operand is a register of the provided register group `regGroup`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegGroup regGroup) const noexcept { return _signature.isReg(regGroup); } + + //! Tests whether the operand is register and of register group `regGroup` and `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegGroup regGroup, uint32_t regId) const noexcept { return Support::bool_and(isReg(regGroup), _baseId == regId); } + + //! Tests whether the operand is a general purpose register of any type. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isPc() const noexcept { return isReg(RegType::kPC); } + + //! Tests whether the operand is a general purpose register of any type. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp() const noexcept { return isReg(RegGroup::kGp); } + + //! Tests whether the operand is a general purpose register of any type having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp(uint32_t regId) const noexcept { return isReg(RegGroup::kGp, regId); } + + //! Tests whether the register is an 8-bit low or high general purpose register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8() const noexcept { return Support::bool_or(isReg(RegType::kGp8Lo), isReg(RegType::kGp8Hi)); } + + //! Tests whether the register is an 8-bit low or high general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8(uint32_t regId) const noexcept { return Support::bool_and(isGp8(), id() == regId); } + + //! Tests whether the register is an 8-bit low general purpose register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Lo() const noexcept { return isReg(RegType::kGp8Lo); } + + //! Tests whether the register is an 8-bit low general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Lo(uint32_t regId) const noexcept { return isReg(RegType::kGp8Lo, regId); } + + //! Tests whether the register is an 8-bit high general purpose register (X86|X86_64 only - AH, BH, CH, DH). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Hi() const noexcept { return isReg(RegType::kGp8Hi); } + + //! Tests whether the register is an 8-bit high general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Hi(uint32_t regId) const noexcept { return isReg(RegType::kGp8Hi, regId); } + + //! Tests whether the register is a 16-bit general purpose register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp16() const noexcept { return isReg(RegType::kGp16); } + + //! Tests whether the register is a 16-bit general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp16(uint32_t regId) const noexcept { return isReg(RegType::kGp16, regId); } + + //! Tests whether the register is a 32-bit general purpose register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp32() const noexcept { return isReg(RegType::kGp32); } + + //! Tests whether the register is a 32-bit general purpose register having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp32(uint32_t regId) const noexcept { return isReg(RegType::kGp32, regId); } + + //! Tests whether the register is a 64-bit general purpose register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp64() const noexcept { return isReg(RegType::kGp64); } + + //! Tests whether the register is a 64-bit general purpose register having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp64(uint32_t regId) const noexcept { return isReg(RegType::kGp64, regId); } + + //! Tests whether the register is a vector register of any size. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec() const noexcept { return isReg(RegGroup::kVec); } + + //! Tests whether the register is a vector register of any size having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec(uint32_t regId) const noexcept { return isReg(RegGroup::kVec, regId); } + + //! Tests whether the register is an 8-bit vector register or view (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec8() const noexcept { return isReg(RegType::kVec8); } + + //! Tests whether the register is an 8-bit vector register or view having the given id `regId` (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec8(uint32_t regId) const noexcept { return isReg(RegType::kVec8, regId); } + + //! Tests whether the register is a 16-bit vector register or view (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec16() const noexcept { return isReg(RegType::kVec16); } + + //! Tests whether the register is a 16-bit vector register or view having the given id `regId` (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec16(uint32_t regId) const noexcept { return isReg(RegType::kVec16, regId); } + + //! Tests whether the register is a 32-bit vector register or view (AArch32, AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec32() const noexcept { return isReg(RegType::kVec32); } + + //! Tests whether the register is a 32-bit vector register or view having the given id `regId` (AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec32(uint32_t regId) const noexcept { return isReg(RegType::kVec32, regId); } + + //! Tests whether the register is a 64-bit vector register or view (AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec64() const noexcept { return isReg(RegType::kVec64); } + + //! Tests whether the register is a 64-bit vector register or view having the given id `regId` (AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec64(uint32_t regId) const noexcept { return isReg(RegType::kVec64, regId); } + + //! Tests whether the register is a 128-bit vector register or view (X86|X86_64/AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec128() const noexcept { return isReg(RegType::kVec128); } + + //! Tests whether the register is a 128-bit vector register or view having the given id `regId` (X86|X86_64/AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec128(uint32_t regId) const noexcept { return isReg(RegType::kVec128, regId); } + + //! Tests whether the register is a 256-bit vector register or view (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec256() const noexcept { return isReg(RegType::kVec256); } + + //! Tests whether the register is a 256-bit vector register or view having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec256(uint32_t regId) const noexcept { return isReg(RegType::kVec256, regId); } + + //! Tests whether the register is a 512-bit vector register or view (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec512() const noexcept { return isReg(RegType::kVec512); } + + //! Tests whether the register is a 512-bit vector register or view having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec512(uint32_t regId) const noexcept { return isReg(RegType::kVec512, regId); } + + //! Tests whether the register is a mask register of any size. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isMaskReg() const noexcept { return isReg(RegType::kMask); } + + //! Tests whether the register is a mask register of any size having the given id `regId` (X86|X86_64/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isMaskReg(uint32_t regId) const noexcept { return isReg(RegType::kMask, regId); } + + //! Tests whether the register is a mask register (`K` register on X86|X86_64) - alias of \ref isMask(). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isKReg() const noexcept { return isReg(RegType::kMask); } + + //! Tests whether the register is a mask register (`K` register on X86|X86_64) of any size having the given id + //! `regId` (X86|X86_64/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isKReg(uint32_t regId) const noexcept { return isReg(RegType::kMask, regId); } + + //! Tests whether the register is a tile register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isTileReg() const noexcept { return isReg(RegType::kTile); } + + //! Tests whether the register is a tile register of the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isTileReg(uint32_t regId) const noexcept { return isReg(RegType::kTile, regId); } + + //! Tests whether the register is a tile register (`TMM` register on X86_64) - alias of \ref isTileReg(). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isTmmReg() const noexcept { return isReg(RegType::kTile); } + + //! Tests whether the register is a tile register (`TMM` register on X86_64) of the given id `regId` - alias of + //! \ref isTileReg(). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isTmmReg(uint32_t regId) const noexcept { return isReg(RegType::kTile, regId); } + + //! Tests whether the register is a segment register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isSegmentReg() const noexcept { return isReg(RegType::kSegment); } + + //! Tests whether the register is a segment register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isSegmentReg(uint32_t regId) const noexcept { return isReg(RegType::kSegment, regId); } + + //! Tests whether the register is a control register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isControlReg() const noexcept { return isReg(RegType::kControl); } + + //! Tests whether the register is a control register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isControlReg(uint32_t regId) const noexcept { return isReg(RegType::kControl, regId); } + + //! Tests whether the register is a debug register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isDebugReg() const noexcept { return isReg(RegType::kDebug); } + + //! Tests whether the register is a debug register of the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isDebugReg(uint32_t regId) const noexcept { return isReg(RegType::kDebug, regId); } + + //! Tests whether the register is an MMX register (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isMmReg() const noexcept { return isReg(RegType::kX86_Mm); } + + //! Tests whether the register is an MMX register of the given id `regId` (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isMmReg(uint32_t regId) const noexcept { return isReg(RegType::kX86_Mm, regId); } + + //! Tests whether the register is an FPU register (`ST` register on X86|X86_64) (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isStReg() const noexcept { return isReg(RegType::kX86_St); } + + //! Tests whether the register is an FPU register (`ST` register on X86|X86_64) of the given id `regId` (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isStReg(uint32_t regId) const noexcept { return isReg(RegType::kX86_St, regId); } + + //! Tests whether the register is a BND register (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isBndReg() const noexcept { return isReg(RegType::kX86_Bnd); } + + //! Tests whether the register is a BND register of the given id `regId` (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isBndReg(uint32_t regId) const noexcept { return isReg(RegType::kX86_Bnd, regId); } + + //! \} + + //! \name Register-List Accessors + //! \{ + + //! Tests whether the operand is a register matching the given register `type`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isRegList(RegType type) const noexcept { + return _signature.subset(Signature::kOpTypeMask | Signature::kRegTypeMask) == (Signature::fromOpType(OperandType::kRegList) | Signature::fromRegType(type)); + } + + //! \} + + //! \name X86-Specific Accessors //! \{ //! Returns a size of a register or an X86 memory operand. @@ -994,28 +1139,136 @@ public: //! \} }; -//! \cond INTERNAL -//! Default register traits. -struct BaseRegTraits { +//! Register traits. +//! +//! Register traits contain metadata about a particular register type. The metadata is used by AsmJit to setup +//! register information on-the-fly and to populate tables that contain register information (this way it's possible +//! to change register types and groups without having to reorder these tables). +template +struct RegTraits { //! \ref TypeId representing this register type, could be \ref TypeId::kVoid if such type doesn't exist. - static inline constexpr uint32_t kTypeId = uint32_t(TypeId::kVoid); + static inline constexpr TypeId kTypeId = TypeId::kVoid; //! RegType is not valid by default. static inline constexpr uint32_t kValid = 0; //! Zero type by default (defaults to None). - static inline constexpr uint32_t kType = uint32_t(RegType::kNone); + static inline constexpr RegType kType = RegType::kNone; //! Zero group by default (defaults to GP). - static inline constexpr uint32_t kGroup = uint32_t(RegGroup::kGp); + static inline constexpr RegGroup kGroup = RegGroup::kGp; //! No size by default. static inline constexpr uint32_t kSize = 0u; //! Empty signature by default (not even having operand type set to register). static inline constexpr uint32_t kSignature = 0; }; + +//! \cond + +//! Adds a template specialization for `REG_TYPE` into the local `RegTraits`. +#define ASMJIT_DEFINE_REG_TRAITS(REG_TYPE, GROUP, SIZE, TYPE_ID) \ +template<> \ +struct RegTraits { \ + static inline constexpr uint32_t kValid = 1; \ + static inline constexpr RegType kType = REG_TYPE; \ + static inline constexpr RegGroup kGroup = GROUP; \ + static inline constexpr uint32_t kSize = SIZE; \ + static inline constexpr TypeId kTypeId = TYPE_ID; \ + \ + static inline constexpr uint32_t kSignature = \ + (OperandSignature::fromOpType(OperandType::kReg) | \ + OperandSignature::fromRegType(kType) | \ + OperandSignature::fromRegGroup(kGroup) | \ + OperandSignature::fromSize(kSize)).bits(); \ + \ +} + +// <--------------------+------------------------+------------------------+----+------------------+ +---+---+---+---+ +// | Reg-Type | Reg-Group |Size| TypeId | |X86|X64|A32|A64| +// <--------------------+------------------------+------------------------+----+------------------+ +---+---+---+---+ +ASMJIT_DEFINE_REG_TRAITS(RegType::kPC , RegGroup::kPC , 8 , TypeId::kInt64 ); // | x | x | | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kGp8Lo , RegGroup::kGp , 1 , TypeId::kInt8 ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kGp8Hi , RegGroup::kGp , 1 , TypeId::kInt8 ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kGp16 , RegGroup::kGp , 2 , TypeId::kInt16 ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kGp32 , RegGroup::kGp , 4 , TypeId::kInt32 ); // | x | x | x | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kGp64 , RegGroup::kGp , 8 , TypeId::kInt64 ); // | x | x | x | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVec8 , RegGroup::kVec , 1 , TypeId::kVoid ); // | | | x | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVec16 , RegGroup::kVec , 2 , TypeId::kVoid ); // | | | x | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVec32 , RegGroup::kVec , 4 , TypeId::kInt32x1 ); // | | | x | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVec64 , RegGroup::kVec , 8 , TypeId::kInt32x2 ); // | | | x | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVec128 , RegGroup::kVec , 16 , TypeId::kInt32x4 ); // | x | x | x | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVec256 , RegGroup::kVec , 32 , TypeId::kInt32x8 ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVec512 , RegGroup::kVec , 64 , TypeId::kInt32x16); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVecNLen , RegGroup::kVec , 0 , TypeId::kVoid ); // | | | | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kMask , RegGroup::kMask , 0 , TypeId::kVoid ); // | x | x | | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kTile , RegGroup::kTile , 0 , TypeId::kVoid ); // | | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kSegment , RegGroup::kSegment , 2 , TypeId::kVoid ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kControl , RegGroup::kControl , 0 , TypeId::kVoid ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kDebug , RegGroup::kDebug , 0 , TypeId::kVoid ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kX86_Mm , RegGroup::kX86_MM , 8 , TypeId::kMmx64 ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kX86_St , RegGroup::kX86_St , 10 , TypeId::kFloat80 ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kX86_Bnd , RegGroup::kX86_Bnd , 16 , TypeId::kVoid ); // | x | x | | | + +#undef ASMJIT_DEFINE_REG_TRAITS + //! \endcond -//! Physical or virtual register operand (base). -class BaseReg : public Operand { +namespace RegUtils { + +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR OperandSignature signatureOf(RegType regType) noexcept { + constexpr uint32_t signatureTable[] = { + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature + }; + return OperandSignature{signatureTable[size_t(regType)]}; +} + +[[nodiscard]] +static ASMJIT_INLINE_NODEBUG OperandSignature signatureOfVecSize(uint32_t size) noexcept { + RegType regType = RegType(Support::ctz((size | 0x40u) & 0x0Fu) - 4u + uint32_t(RegType::kVec128)); + return signatureOf(regType); +} + +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR RegGroup groupOf(RegType regType) noexcept { + constexpr RegGroup groupTable[] = { + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup + }; + return groupTable[size_t(regType)]; +} + +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR TypeId typeIdOf(RegType regType) noexcept { + constexpr TypeId typeIdTable[] = { + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId + }; + return typeIdTable[size_t(regType)]; +} + +} // {RegUtils} + +//! Unified physical or virtual register operand. +class Reg : public Operand { public: //! \name Constants //! \{ @@ -1032,50 +1285,58 @@ public: static inline constexpr uint32_t kTypeNone = uint32_t(RegType::kNone); static inline constexpr uint32_t kSignature = Signature::fromOpType(OperandType::kReg).bits(); + template + static ASMJIT_INLINE_CONSTEXPR Signature _signatureOf() noexcept { return Signature{RegTraits::kSignature}; } + + static ASMJIT_INLINE_CONSTEXPR Signature signatureOf(RegType regType) noexcept { return RegUtils::signatureOf(regType); } + //! \} //! \name Construction & Destruction //! \{ //! Creates a dummy register operand. - ASMJIT_INLINE_CONSTEXPR BaseReg() noexcept - : Operand(Globals::Init, Signature::fromOpType(OperandType::kReg), kIdBad, 0, 0) {} + ASMJIT_INLINE_CONSTEXPR Reg() noexcept + : Operand(Globals::Init, Signature::fromOpType(OperandType::kReg), kIdBad, 0u, 0u) {} //! Creates a new register operand which is the same as `other` . - ASMJIT_INLINE_CONSTEXPR BaseReg(const BaseReg& other) noexcept + ASMJIT_INLINE_CONSTEXPR Reg(const Reg& other) noexcept : Operand(other) {} //! Creates a new register operand compatible with `other`, but with a different `id`. - ASMJIT_INLINE_CONSTEXPR BaseReg(const BaseReg& other, uint32_t id) noexcept - : Operand(Globals::Init, other._signature, id, 0, 0) {} + ASMJIT_INLINE_CONSTEXPR Reg(const Reg& other, uint32_t id) noexcept + : Operand(Globals::Init, other._signature, id, 0u, 0u) {} //! Creates a register initialized to the given `signature` and `id`. - ASMJIT_INLINE_CONSTEXPR BaseReg(const Signature& signature, uint32_t id) noexcept - : Operand(Globals::Init, signature, id, 0, 0) {} + ASMJIT_INLINE_CONSTEXPR Reg(const Signature& signature, uint32_t id) noexcept + : Operand(Globals::Init, signature, id, 0u, 0u) {} - ASMJIT_INLINE_NODEBUG explicit BaseReg(Globals::NoInit_) noexcept + ASMJIT_INLINE_NODEBUG explicit Reg(Globals::NoInit_) noexcept : Operand(Globals::NoInit) {} + //! Creates a new register from register type and id. + static ASMJIT_INLINE_CONSTEXPR Reg fromTypeAndId(RegType type, uint32_t id) noexcept { return Reg(signatureOf(type), id); } + //! \} //! \name Overloaded Operators //! \{ - ASMJIT_INLINE_CONSTEXPR BaseReg& operator=(const BaseReg& other) noexcept { + ASMJIT_INLINE_CONSTEXPR Reg& operator=(const Reg& other) noexcept { copyFrom(other); return *this; } //! \} - //! \name Accessors + //! \name Signature Accessors //! \{ //! Returns base signature of the register associated with each register type. //! //! Base signature only contains the operand type, register type, register group, and register size. It doesn't //! contain element type, predicate, or other architecture-specific data. Base signature is a signature that is - //! provided by architecture-specific `RegTraits`, like \ref x86::RegTraits. + //! provided by \ref RegTraits. [[nodiscard]] ASMJIT_INLINE_CONSTEXPR OperandSignature baseSignature() const noexcept { return _signature & kBaseSignatureMask; } @@ -1089,7 +1350,20 @@ public: //! Tests whether the operand's base signature matches the base signature of the `other` operand. [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool hasBaseSignature(const BaseReg& other) const noexcept { return baseSignature() == other.baseSignature(); } + ASMJIT_INLINE_CONSTEXPR bool hasBaseSignature(const Reg& other) const noexcept { return baseSignature() == other.baseSignature(); } + + //! \} + + //! \name Register Accessors + //! \{ + + //! Returns the type of the register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegType regType() const noexcept { return _signature.regType(); } + + //! Returns the group this register belongs to. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegGroup regGroup() const noexcept { return _signature.regGroup(); } //! Tests whether this register is the same as `other`. //! @@ -1099,13 +1373,11 @@ public: //! or other metadata in the upper 8 bytes then \ref isSame() may return `true` in cases in which \ref equals() //! returns false. [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isSame(const BaseReg& other) const noexcept { - return (_signature == other._signature) & (_baseId == other._baseId); - } + ASMJIT_INLINE_CONSTEXPR bool isSame(const Reg& other) const noexcept { return (_signature == other._signature) & (_baseId == other._baseId); } //! Tests whether the register is valid (either virtual or physical). [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isValid() const noexcept { return bool(unsigned(_signature != 0) & unsigned(_baseId != kIdBad)); } + ASMJIT_INLINE_CONSTEXPR bool isValid() const noexcept { return Support::bool_and(_signature != 0u, _baseId != kIdBad); } //! Tests whether this is a physical register. [[nodiscard]] @@ -1115,43 +1387,245 @@ public: [[nodiscard]] ASMJIT_INLINE_CONSTEXPR bool isVirtReg() const noexcept { return _baseId > kIdBad; } - //! Tests whether the register type matches `type` - same as `isReg(type)`, provided for convenience. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isType(RegType type) const noexcept { return _signature.subset(Signature::kRegTypeMask) == Signature::fromRegType(type); } + // NOTE: A careful reader may wonder why the member functions here are repeated and basically do the same as in + // `Operand_`, however, they don't always do the same. In `Operand_` case each test function must check + // whether the operand is actually a register, whereas here we don't have to. However, in some cases it's + // actually beneficial to do that (if the check is using a constant expression on the input). Since C++ + // doesn't support mixins the implementation is basically duplicating some of the `Operand_` checks here. + using Operand_::isReg; - //! Tests whether the register group matches `group`. + //! Tests whether the register is of the given register type `regType`. [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isGroup(RegGroup group) const noexcept { return _signature.subset(Signature::kRegGroupMask) == Signature::fromRegGroup(group); } + ASMJIT_INLINE_CONSTEXPR bool isReg(RegType regType) const noexcept { +#if defined(__GNUC__) + // At the moment operand type is 3 bits and register type is 5 bits, which form a value that is stored in + // the 8 least significant bits of the operand signature. Checking ALL of these 8 bits is much better than + // extracting register type at least on X86 and X86_64 targets. + if (__builtin_constant_p(regType)) { + return _signature.isReg(regType); + } +#endif - //! Tests whether the register is a general purpose register (any size). + return _signature.subset(Signature::kRegTypeMask) == Signature::fromRegType(regType); + } + + //! Tests whether the register is of the given register type `regType` having the given id `regId`. [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isGp() const noexcept { return isGroup(RegGroup::kGp); } + ASMJIT_INLINE_CONSTEXPR bool isReg(RegType regType, uint32_t regId) const noexcept { return Support::bool_and(isReg(regType), _baseId == regId); } + + //! Tests whether the register is of the given register group `regGroup`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegGroup regGroup) const noexcept { return _signature.subset(Signature::kRegGroupMask) == Signature::fromRegGroup(regGroup); } + + //! Tests whether the register is of the given register group `regGroup` having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegGroup regGroup, uint32_t regId) const noexcept { return Support::bool_and(isReg(regGroup), _baseId == regId); } + + //! Tests whether the operand is a general purpose register of any type. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isPc() const noexcept { return isReg(RegType::kPC); } + + //! Tests whether the register is a general purpose register of any type. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp() const noexcept { return isReg(RegGroup::kGp); } + + //! Tests whether the register is a general purpose register of any type having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp(uint32_t regId) const noexcept { return isReg(RegGroup::kGp, regId); } + + //! Tests whether the register is an 8-bit low or high general purpose register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8() const noexcept { return Support::bool_or(isReg(RegType::kGp8Lo), isReg(RegType::kGp8Hi)); } + + //! Tests whether the register is an 8-bit low or high general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8(uint32_t regId) const noexcept { return Support::bool_and(isGp8(), id() == regId); } + + //! Tests whether the register is an 8-bit low general purpose register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Lo() const noexcept { return isReg(RegType::kGp8Lo); } + + //! Tests whether the register is an 8-bit low general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Lo(uint32_t regId) const noexcept { return isReg(RegType::kGp8Lo, regId); } + + //! Tests whether the register is an 8-bit high general purpose register (X86|X86_64 only - AH, BH, CH, DH). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Hi() const noexcept { return isReg(RegType::kGp8Hi); } + + //! Tests whether the register is an 8-bit high general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Hi(uint32_t regId) const noexcept { return isReg(RegType::kGp8Hi, regId); } + + //! Tests whether the register is a 16-bit general purpose register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp16() const noexcept { return isReg(RegType::kGp16); } + + //! Tests whether the register is a 16-bit general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp16(uint32_t regId) const noexcept { return isReg(RegType::kGp16, regId); } + + //! Tests whether the register is a 32-bit general purpose register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp32() const noexcept { return isReg(RegType::kGp32); } + + //! Tests whether the register is a 32-bit general purpose register having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp32(uint32_t regId) const noexcept { return isReg(RegType::kGp32, regId); } + + //! Tests whether the register is a 64-bit general purpose register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp64() const noexcept { return isReg(RegType::kGp64); } + + //! Tests whether the register is a 64-bit general purpose register having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp64(uint32_t regId) const noexcept { return isReg(RegType::kGp64, regId); } //! Tests whether the register is a vector register of any size. [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVec() const noexcept { return isGroup(RegGroup::kVec); } + ASMJIT_INLINE_CONSTEXPR bool isVec() const noexcept { return isReg(RegGroup::kVec); } + + //! Tests whether the register is a vector register of any size having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec(uint32_t regId) const noexcept { return isReg(RegGroup::kVec, regId); } + + //! Tests whether the register is an 8-bit vector register or view (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec8() const noexcept { return isReg(RegType::kVec8); } + + //! Tests whether the register is an 8-bit vector register or view having the given id `regId` (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec8(uint32_t regId) const noexcept { return isReg(RegType::kVec8, regId); } + + //! Tests whether the register is a 16-bit vector register or view (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec16() const noexcept { return isReg(RegType::kVec16); } + + //! Tests whether the register is a 16-bit vector register or view having the given id `regId` (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec16(uint32_t regId) const noexcept { return isReg(RegType::kVec16, regId); } + + //! Tests whether the register is a 32-bit vector register or view (AArch32, AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec32() const noexcept { return isReg(RegType::kVec32); } + + //! Tests whether the register is a 32-bit vector register or view having the given id `regId` (AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec32(uint32_t regId) const noexcept { return isReg(RegType::kVec32, regId); } + + //! Tests whether the register is a 64-bit vector register or view (AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec64() const noexcept { return isReg(RegType::kVec64); } + + //! Tests whether the register is a 64-bit vector register or view having the given id `regId` (AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec64(uint32_t regId) const noexcept { return isReg(RegType::kVec64, regId); } + + //! Tests whether the register is a 128-bit vector register or view (X86|X86_64/AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec128() const noexcept { return isReg(RegType::kVec128); } + + //! Tests whether the register is a 128-bit vector register or view having the given id `regId` (X86|X86_64/AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec128(uint32_t regId) const noexcept { return isReg(RegType::kVec128, regId); } + + //! Tests whether the register is a 256-bit vector register or view (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec256() const noexcept { return isReg(RegType::kVec256); } + + //! Tests whether the register is a 256-bit vector register or view having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec256(uint32_t regId) const noexcept { return isReg(RegType::kVec256, regId); } + + //! Tests whether the register is a 512-bit vector register or view (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec512() const noexcept { return isReg(RegType::kVec512); } + + //! Tests whether the register is a 512-bit vector register or view having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec512(uint32_t regId) const noexcept { return isReg(RegType::kVec512, regId); } //! Tests whether the register is a mask register of any size. [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isMask() const noexcept { return isGroup(RegGroup::kMask); } + ASMJIT_INLINE_CONSTEXPR bool isMaskReg() const noexcept { return isReg(RegType::kMask); } - using Operand_::isReg; - - //! Same as `isType()`, provided for convenience. + //! Tests whether the register is a mask register of any size having the given id `regId` (X86|X86_64/AArch64). [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isReg(RegType rType) const noexcept { return isType(rType); } + ASMJIT_INLINE_CONSTEXPR bool isMaskReg(uint32_t regId) const noexcept { return isReg(RegType::kMask, regId); } - //! Tests whether the register type matches `type` and register id matches `id`. + //! Tests whether the register is a mask register (`K` register on X86|X86_64) - alias of \ref isMask(). [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isReg(RegType rType, uint32_t id) const noexcept { return isType(rType) && this->id() == id; } + ASMJIT_INLINE_CONSTEXPR bool isKReg() const noexcept { return isReg(RegType::kMask); } - //! Returns the register type. + //! Tests whether the register is a mask register (`K` register on X86|X86_64) of any size having the given id + //! `regId` (X86|X86_64/AArch64). [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR RegType type() const noexcept { return _signature.regType(); } + ASMJIT_INLINE_CONSTEXPR bool isKReg(uint32_t regId) const noexcept { return isReg(RegType::kMask, regId); } - //! Returns the register group. + //! Tests whether the register is a tile register. [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR RegGroup group() const noexcept { return _signature.regGroup(); } + ASMJIT_INLINE_CONSTEXPR bool isTileReg() const noexcept { return isReg(RegType::kTile); } + + //! Tests whether the register is a tile register of the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isTileReg(uint32_t regId) const noexcept { return isReg(RegType::kTile, regId); } + + //! Tests whether the register is a tile register (`TMM` register on X86_64) - alias of \ref isTileReg(). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isTmmReg() const noexcept { return isReg(RegType::kTile); } + + //! Tests whether the register is a tile register (`TMM` register on X86_64) of the given id `regId` - alias of + //! \ref isTileReg(). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isTmmReg(uint32_t regId) const noexcept { return isReg(RegType::kTile, regId); } + + //! Tests whether the register is a segment register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isSegmentReg() const noexcept { return isReg(RegType::kSegment); } + + //! Tests whether the register is a segment register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isSegmentReg(uint32_t regId) const noexcept { return isReg(RegType::kSegment, regId); } + + //! Tests whether the register is a control register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isControlReg() const noexcept { return isReg(RegType::kControl); } + + //! Tests whether the register is a control register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isControlReg(uint32_t regId) const noexcept { return isReg(RegType::kControl, regId); } + + //! Tests whether the register is a debug register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isDebugReg() const noexcept { return isReg(RegType::kDebug); } + + //! Tests whether the register is a debug register of the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isDebugReg(uint32_t regId) const noexcept { return isReg(RegType::kDebug, regId); } + + //! Tests whether the register is an MMX register (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isMmReg() const noexcept { return isReg(RegType::kX86_Mm); } + + //! Tests whether the register is an MMX register of the given id `regId` (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isMmReg(uint32_t regId) const noexcept { return isReg(RegType::kX86_Mm, regId); } + + //! Tests whether the register is an FPU register (`ST` register on X86|X86_64) (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isStReg() const noexcept { return isReg(RegType::kX86_St); } + + //! Tests whether the register is an FPU register (`ST` register on X86|X86_64) of the given id `regId` (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isStReg(uint32_t regId) const noexcept { return isReg(RegType::kX86_St, regId); } + + //! Tests whether the register is a BND register (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isBndReg() const noexcept { return isReg(RegType::kX86_Bnd); } + + //! Tests whether the register is a BND register of the given id `regId` (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isBndReg(uint32_t regId) const noexcept { return isReg(RegType::kX86_Bnd, regId); } //! Tests whether the register specifies a size (i.e. the size is not zero). [[nodiscard]] @@ -1184,7 +1658,7 @@ public: //! Clones the register operand. [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR BaseReg clone() const noexcept { return BaseReg(*this); } + ASMJIT_INLINE_CONSTEXPR Reg clone() const noexcept { return Reg(*this); } //! Casts this register to `RegT` by also changing its signature. //! @@ -1200,6 +1674,12 @@ public: [[nodiscard]] ASMJIT_INLINE_CONSTEXPR RegT cloneAs(const RegT& other) const noexcept { return RegT(other.signature(), id()); } + template + ASMJIT_INLINE_CONSTEXPR void setRegT(uint32_t id) noexcept { + setSignature(RegTraits::kSignature); + setId(id); + } + //! Sets the register id to `id`. ASMJIT_INLINE_CONSTEXPR void setId(uint32_t id) noexcept { _baseId = id; } @@ -1215,139 +1695,22 @@ public: //! \} - //! \name Static Functions - //! \{ - - //! Tests whether the `op` operand is a general purpose register. - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR bool isGp(const Operand_& op) noexcept { - // Check operand type and register group. Not interested in register type and size. - return op.signature().subset(Signature::kOpTypeMask | Signature::kRegGroupMask) == (Signature::fromOpType(OperandType::kReg) | Signature::fromRegGroup(RegGroup::kGp)); - } - - //! Tests whether the `op` operand is a vector register. - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR bool isVec(const Operand_& op) noexcept { - // Check operand type and register group. Not interested in register type and size. - return op.signature().subset(Signature::kOpTypeMask | Signature::kRegGroupMask) == (Signature::fromOpType(OperandType::kReg) | Signature::fromRegGroup(RegGroup::kVec)); - } - - //! Tests whether the `op` is a general purpose register of the given `id`. - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR bool isGp(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isGp(op)) & unsigned(op.id() == id)); } - - //! Tests whether the `op` is a vector register of the given `id`. - [[nodiscard]] - static ASMJIT_INLINE_CONSTEXPR bool isVec(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isVec(op)) & unsigned(op.id() == id)); } - - //! \} -}; - -//! RegOnly is 8-byte version of `BaseReg` that allows to store either register or nothing. -//! -//! It's designed to decrease the space consumed by an extra "operand" in \ref BaseEmitter and \ref InstNode. -struct RegOnly { - //! \name Types - //! \{ - - using Signature = OperandSignature; - - //! \} - - //! Operand signature - only \ref OperandType::kNone and \ref OperandType::kReg are supported. - Signature _signature; - //! Physical or virtual register id. - uint32_t _id; - - //! \name Construction & Destruction - //! \{ - - //! Initializes the `RegOnly` instance to hold register `signature` and `id`. - ASMJIT_INLINE_CONSTEXPR void init(const OperandSignature& signature, uint32_t id) noexcept { - _signature = signature; - _id = id; - } - - ASMJIT_INLINE_CONSTEXPR void init(const BaseReg& reg) noexcept { init(reg.signature(), reg.id()); } - ASMJIT_INLINE_CONSTEXPR void init(const RegOnly& reg) noexcept { init(reg.signature(), reg.id()); } - - //! Resets the `RegOnly` members to zeros (none). - ASMJIT_INLINE_CONSTEXPR void reset() noexcept { init(Signature::fromBits(0), 0); } - - //! \} - - //! \name Accessors - //! \{ - - //! Tests whether this ExtraReg is none (same as calling `Operand_::isNone()`). - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isNone() const noexcept { return _signature == 0; } - - //! Tests whether the register is valid (either virtual or physical). - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isReg() const noexcept { return _signature != 0; } - - //! Tests whether this is a physical register. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isPhysReg() const noexcept { return _id < BaseReg::kIdBad; } - - //! Tests whether this is a virtual register (used by `BaseCompiler`). - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR bool isVirtReg() const noexcept { return _id > BaseReg::kIdBad; } - - //! Returns the register signature or 0 if no register is assigned. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR OperandSignature signature() const noexcept { return _signature; } - - //! Returns the register id. - //! - //! \note Always check whether the register is assigned before using the returned identifier as - //! non-assigned `RegOnly` instance would return zero id, which is still a valid register id. - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR uint32_t id() const noexcept { return _id; } - - //! Sets the register id. - ASMJIT_INLINE_CONSTEXPR void setId(uint32_t id) noexcept { _id = id; } - - //! Returns the register type. +#if !defined(ASMJIT_NO_DEPRECATED) + [[deprecated("Use regType() instead")]] [[nodiscard]] ASMJIT_INLINE_CONSTEXPR RegType type() const noexcept { return _signature.regType(); } - //! Returns the register group. + [[deprecated("Use regGroup() instead")]] [[nodiscard]] ASMJIT_INLINE_CONSTEXPR RegGroup group() const noexcept { return _signature.regGroup(); } - - //! \} - - //! \name Utilities - //! \{ - - //! Converts this ExtraReg to a real `RegT` operand. - template - [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR RegT toReg() const noexcept { return RegT(_signature, _id); } - - //! \} +#endif // !ASMJIT_NO_DEPRECATED }; -//! \cond INTERNAL -//! Adds a template specialization for `REG_TYPE` into the local `RegTraits`. -#define ASMJIT_DEFINE_REG_TRAITS(REG_TYPE, GROUP, SIZE, TYPE_ID) \ -template<> \ -struct RegTraits { \ - static inline constexpr uint32_t kValid = 1; \ - static inline constexpr RegType kType = REG_TYPE; \ - static inline constexpr RegGroup kGroup = GROUP; \ - static inline constexpr uint32_t kSize = SIZE; \ - static inline constexpr TypeId kTypeId = TYPE_ID; \ - \ - static inline constexpr uint32_t kSignature = \ - (OperandSignature::fromOpType(OperandType::kReg) | \ - OperandSignature::fromRegType(kType) | \ - OperandSignature::fromRegGroup(kGroup) | \ - OperandSignature::fromSize(kSize)).bits(); \ - \ -} +#if !defined(ASMJIT_NO_DEPRECATED) +using BaseReg [[deprecated("Use asmjit::Reg instead of asmjit::BaseReg")]] = Reg; +#endif // !ASMJIT_NO_DEPRECATED + +//! \cond //! Adds constructors and member functions to a class that implements abstract register. Abstract register is register //! that doesn't have type or signature yet, it's a base class like `x86::Reg` or `arm::Reg`. @@ -1362,7 +1725,7 @@ public: : BASE(other) {} \ \ /*! Makes a copy of the `other` register having id set to `id` */ \ - ASMJIT_INLINE_CONSTEXPR REG(const BaseReg& other, uint32_t id) noexcept \ + ASMJIT_INLINE_CONSTEXPR REG(const Reg& other, uint32_t id) noexcept \ : BASE(other, id) {} \ \ /*! Creates a register based on `signature` and `id`. */ \ @@ -1401,8 +1764,175 @@ public: /*! Creates a register operand having its id set to `id`. */ \ ASMJIT_INLINE_CONSTEXPR explicit REG(uint32_t id) noexcept \ : BASE(Signature{kSignature}, id) {} + //! \endcond +//! Unified general purpose register (also acts as a base class for architecture specific GP registers). +class UniGp : public Reg { +public: + ASMJIT_DEFINE_ABSTRACT_REG(UniGp, Reg) + + //! \name Static Constructors + //! \{ + + //! Creates a new 32-bit GP register having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR UniGp make_r32(uint32_t regId) noexcept { return UniGp(_signatureOf(), regId); } + + //! Creates a new 64-bit GP register having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR UniGp make_r64(uint32_t regId) noexcept { return UniGp(_signatureOf(), regId); } + + //! \} + + //! \name Unified Accessors + //! \{ + + //! Clones and casts this register to a 32-bit GP register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR UniGp r32() const noexcept { return UniGp(_signatureOf(), id()); } + + //! Clones and casts this register to a 64-bit GP register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR UniGp r64() const noexcept { return UniGp(_signatureOf(), id()); } + + //! \} +}; + +//! Unified vector register (also acts as a base class for architecture specific vector registers). +class UniVec : public Reg { +public: + ASMJIT_DEFINE_ABSTRACT_REG(UniVec, Reg) + + //! \name Static Constructors + //! \{ + + //! Creates a new 128-bit vector register having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR UniVec make_v128(uint32_t regId) noexcept { return UniVec(_signatureOf(), regId); } + + //! Creates a new 256-bit vector register having the given register id `regId`. + //! + //! \note 256-bit vector registers are only supported by X86|X86_64. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR UniVec make_v256(uint32_t regId) noexcept { return UniVec(_signatureOf(), regId); } + + //! Creates a new 512-bit vector register having the given register id `regId`. + //! + //! \note 512-bit vector registers are only supported by X86|X86_64. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR UniVec make_v512(uint32_t regId) noexcept { return UniVec(_signatureOf(), regId); } + + //! \} + + //! \name Unified Accessors + //! \{ + + //! Clones and casts this register to a 128-bit vector register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR UniVec v128() const noexcept { return UniVec(_signatureOf(), id()); } + + //! Clones and casts this register to a 256-bit vector register. + //! + //! \note 256-bit vector registers are only supported by X86|X86_64. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR UniVec v256() const noexcept { return UniVec(_signatureOf(), id()); } + + //! Clones and casts this register to a 512-bit vector register. + //! + //! \note 512-bit vector registers are only supported by X86|X86_64. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR UniVec v512() const noexcept { return UniVec(_signatureOf(), id()); } + + //! \} +}; +//! RegOnly is 8-byte version of `Reg` that allows to store either register or nothing. +//! +//! It's designed to decrease the space consumed by an extra "operand" in \ref BaseEmitter and \ref InstNode. +struct RegOnly { + //! \name Types + //! \{ + + using Signature = OperandSignature; + + //! \} + + //! Operand signature - only \ref OperandType::kNone and \ref OperandType::kReg are supported. + Signature _signature; + //! Physical or virtual register id. + uint32_t _id; + + //! \name Construction & Destruction + //! \{ + + //! Initializes the `RegOnly` instance to hold register `signature` and `id`. + ASMJIT_INLINE_CONSTEXPR void init(const OperandSignature& signature, uint32_t id) noexcept { + _signature = signature; + _id = id; + } + + ASMJIT_INLINE_CONSTEXPR void init(const Reg& reg) noexcept { init(reg.signature(), reg.id()); } + ASMJIT_INLINE_CONSTEXPR void init(const RegOnly& reg) noexcept { init(reg.signature(), reg.id()); } + + //! Resets the `RegOnly` members to zeros (none). + ASMJIT_INLINE_CONSTEXPR void reset() noexcept { init(Signature::fromBits(0), 0); } + + //! \} + + //! \name Accessors + //! \{ + + //! Tests whether this ExtraReg is none (same as calling `Operand_::isNone()`). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isNone() const noexcept { return _signature == 0; } + + //! Tests whether the register is valid (either virtual or physical). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg() const noexcept { return _signature != 0; } + + //! Tests whether this is a physical register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isPhysReg() const noexcept { return _id < Reg::kIdBad; } + + //! Tests whether this is a virtual register (used by `BaseCompiler`). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVirtReg() const noexcept { return _id > Reg::kIdBad; } + + //! Returns the register signature or 0 if no register is assigned. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR OperandSignature signature() const noexcept { return _signature; } + + //! Returns the register id. + //! + //! \note Always check whether the register is assigned before using the returned identifier as + //! non-assigned `RegOnly` instance would return zero id, which is still a valid register id. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t id() const noexcept { return _id; } + + //! Sets the register id. + ASMJIT_INLINE_CONSTEXPR void setId(uint32_t id) noexcept { _id = id; } + + //! Returns the register type. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegType type() const noexcept { return _signature.regType(); } + + //! Returns the register group. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegGroup group() const noexcept { return _signature.regGroup(); } + + //! \} + + //! \name Utilities + //! \{ + + //! Converts this ExtraReg to a real `RegT` operand. + template + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegT toReg() const noexcept { return RegT(_signature, _id); } + + //! \} +}; + //! List of physical registers (base). //! //! \note List of registers is only used by some ARM instructions at the moment. @@ -1474,11 +2004,11 @@ public: //! Returns the register type. [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR RegType type() const noexcept { return _signature.regType(); } + ASMJIT_INLINE_CONSTEXPR RegType regType() const noexcept { return _signature.regType(); } //! Returns the register group. [[nodiscard]] - ASMJIT_INLINE_CONSTEXPR RegGroup group() const noexcept { return _signature.regGroup(); } + ASMJIT_INLINE_CONSTEXPR RegGroup regGroup() const noexcept { return _signature.regGroup(); } //! Returns the size of a single register in this register-list or 0 if unspecified. [[nodiscard]] @@ -1664,9 +2194,9 @@ public: //! //! \note This is an architecture independent constructor that can be used to create an architecture //! independent memory operand to be used in portable code that can handle multiple architectures. - ASMJIT_INLINE_CONSTEXPR explicit BaseMem(const BaseReg& baseReg, int32_t offset = 0) noexcept + ASMJIT_INLINE_CONSTEXPR explicit BaseMem(const Reg& baseReg, int32_t offset = 0) noexcept : Operand(Globals::Init, - Signature::fromOpType(OperandType::kMem) | Signature::fromMemBaseType(baseReg.type()), + Signature::fromOpType(OperandType::kMem) | Signature::fromMemBaseType(baseReg.regType()), baseReg.id(), 0, uint32_t(offset)) {} @@ -1807,10 +2337,10 @@ public: ASMJIT_INLINE_CONSTEXPR void setIndexType(RegType regType) noexcept { _signature.setMemIndexType(regType); } //! Sets the base register to type and id of the given `base` operand. - ASMJIT_INLINE_CONSTEXPR void setBase(const BaseReg& base) noexcept { return _setBase(base.type(), base.id()); } + ASMJIT_INLINE_CONSTEXPR void setBase(const Reg& base) noexcept { return _setBase(base.regType(), base.id()); } //! Sets the index register to type and id of the given `index` operand. - ASMJIT_INLINE_CONSTEXPR void setIndex(const BaseReg& index) noexcept { return _setIndex(index.type(), index.id()); } + ASMJIT_INLINE_CONSTEXPR void setIndex(const Reg& index) noexcept { return _setIndex(index.regType(), index.id()); } //! \cond INTERNAL ASMJIT_INLINE_CONSTEXPR void _setBase(RegType type, uint32_t id) noexcept { diff --git a/src/asmjit/core/osutils.cpp b/src/asmjit/core/osutils.cpp index 63b7497..f716d27 100644 --- a/src/asmjit/core/osutils.cpp +++ b/src/asmjit/core/osutils.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/osutils.h b/src/asmjit/core/osutils.h index 2a35dac..8e4fdeb 100644 --- a/src/asmjit/core/osutils.h +++ b/src/asmjit/core/osutils.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_OSUTILS_H_INCLUDED diff --git a/src/asmjit/core/osutils_p.h b/src/asmjit/core/osutils_p.h index 0d9d369..0969fe2 100644 --- a/src/asmjit/core/osutils_p.h +++ b/src/asmjit/core/osutils_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_OSUTILS_P_H_INCLUDED diff --git a/src/asmjit/core/raassignment_p.h b/src/asmjit/core/raassignment_p.h index 198b78b..fd357b8 100644 --- a/src/asmjit/core/raassignment_p.h +++ b/src/asmjit/core/raassignment_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_RAASSIGNMENT_P_H_INCLUDED @@ -68,10 +68,10 @@ public: [[nodiscard]] static ASMJIT_INLINE_NODEBUG size_t sizeOf(size_t count) noexcept { - return sizeof(PhysToWorkMap) - sizeof(uint32_t) + count * sizeof(uint32_t); + return Support::alignUp(sizeof(PhysToWorkMap) - sizeof(uint32_t) + count * sizeof(uint32_t), Globals::kZoneAlignment); } - inline void reset(size_t count) noexcept { + ASMJIT_INLINE void reset(size_t count) noexcept { assigned.reset(); dirty.reset(); @@ -80,12 +80,12 @@ public: } } - inline void copyFrom(const PhysToWorkMap* other, size_t count) noexcept { + ASMJIT_INLINE void copyFrom(const PhysToWorkMap* other, size_t count) noexcept { size_t size = sizeOf(count); memcpy(this, other, size); } - inline void unassign(RegGroup group, uint32_t physId, uint32_t indexInWorkIds) noexcept { + ASMJIT_INLINE void unassign(RegGroup group, uint32_t physId, uint32_t indexInWorkIds) noexcept { assigned.clear(group, Support::bitMask(physId)); dirty.clear(group, Support::bitMask(physId)); workIds[indexInWorkIds] = kWorkNone; @@ -97,17 +97,17 @@ public: uint8_t physIds[1 /* ... */]; [[nodiscard]] - static inline size_t sizeOf(size_t count) noexcept { - return size_t(count) * sizeof(uint8_t); + static ASMJIT_INLINE_NODEBUG size_t sizeOf(size_t count) noexcept { + return Support::alignUp(size_t(count) * sizeof(uint8_t), Globals::kZoneAlignment); } - inline void reset(size_t count) noexcept { + ASMJIT_INLINE void reset(size_t count) noexcept { for (size_t i = 0; i < count; i++) { physIds[i] = kPhysNone; } } - inline void copyFrom(const WorkToPhysMap* other, size_t count) noexcept { + ASMJIT_INLINE void copyFrom(const WorkToPhysMap* other, size_t count) noexcept { size_t size = sizeOf(count); if (ASMJIT_LIKELY(size)) { memcpy(this, other, size); @@ -334,7 +334,7 @@ public: } inline void assignWorkIdsFromPhysIds() noexcept { - memset(_workToPhysMap, uint8_t(BaseReg::kIdBad), WorkToPhysMap::sizeOf(_layout.workCount)); + memset(_workToPhysMap, uint8_t(Reg::kIdBad), WorkToPhysMap::sizeOf(_layout.workCount)); for (RegGroup group : RegGroupVirtValues{}) { uint32_t physBaseIndex = _layout.physIndex[group]; diff --git a/src/asmjit/core/rabuilders_p.h b/src/asmjit/core/rabuilders_p.h index 93dde9e..e4f520e 100644 --- a/src/asmjit/core/rabuilders_p.h +++ b/src/asmjit/core/rabuilders_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_RABUILDERS_P_H_INCLUDED @@ -177,7 +177,7 @@ public: RAWorkReg* workReg = _pass->workRegById(tiedReg.workId()); if (workReg->group() == RegGroup::kGp) { uint32_t useId = tiedReg.useId(); - if (useId == BaseReg::kIdBad) { + if (useId == Reg::kIdBad) { useId = _pass->_scratchRegIndexes[fixedRegCount++]; tiedReg.setUseId(useId); } diff --git a/src/asmjit/core/radefs_p.h b/src/asmjit/core/radefs_p.h index effcfb8..370e503 100644 --- a/src/asmjit/core/radefs_p.h +++ b/src/asmjit/core/radefs_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_RADEFS_P_H_INCLUDED @@ -758,7 +758,7 @@ public: struct LiveRegData { uint32_t id; - ASMJIT_INLINE_NODEBUG explicit LiveRegData(uint32_t id = BaseReg::kIdBad) noexcept : id(id) {} + ASMJIT_INLINE_NODEBUG explicit LiveRegData(uint32_t id = Reg::kIdBad) noexcept : id(id) {} ASMJIT_INLINE_NODEBUG LiveRegData(const LiveRegData& other) noexcept = default; ASMJIT_INLINE_NODEBUG void init(const LiveRegData& other) noexcept { id = other.id; } @@ -1085,11 +1085,11 @@ struct RATiedReg { //! Tests whether the register must be allocated to a fixed physical register before it's used. [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool hasUseId() const noexcept { return _useId != BaseReg::kIdBad; } + ASMJIT_INLINE_NODEBUG bool hasUseId() const noexcept { return _useId != Reg::kIdBad; } //! Tests whether the register must be allocated to a fixed physical register before it's written. [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool hasOutId() const noexcept { return _outId != BaseReg::kIdBad; } + ASMJIT_INLINE_NODEBUG bool hasOutId() const noexcept { return _outId != Reg::kIdBad; } //! Returns a physical register id used for 'use' operation. [[nodiscard]] @@ -1227,9 +1227,9 @@ public: //! Argument value index in the pack (0 by default). uint8_t _argValueIndex = 0; //! Global home register ID (if any, assigned by RA). - uint8_t _homeRegId = BaseReg::kIdBad; + uint8_t _homeRegId = Reg::kIdBad; //! Global hint register ID (provided by RA or user). - uint8_t _hintRegId = BaseReg::kIdBad; + uint8_t _hintRegId = Reg::kIdBad; //! Live spans of the `VirtReg`. LiveRegSpans _liveSpans {}; @@ -1392,7 +1392,7 @@ public: } [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool hasHomeRegId() const noexcept { return _homeRegId != BaseReg::kIdBad; } + ASMJIT_INLINE_NODEBUG bool hasHomeRegId() const noexcept { return _homeRegId != Reg::kIdBad; } [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t homeRegId() const noexcept { return _homeRegId; } @@ -1400,7 +1400,7 @@ public: ASMJIT_INLINE_NODEBUG void setHomeRegId(uint32_t physId) noexcept { _homeRegId = uint8_t(physId); } [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool hasHintRegId() const noexcept { return _hintRegId != BaseReg::kIdBad; } + ASMJIT_INLINE_NODEBUG bool hasHintRegId() const noexcept { return _hintRegId != Reg::kIdBad; } [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t hintRegId() const noexcept { return _hintRegId; } @@ -1414,7 +1414,7 @@ public: ASMJIT_INLINE_NODEBUG bool hasUseIdMask() const noexcept { return _useIdMask != 0u; } [[nodiscard]] - ASMJIT_INLINE_NODEBUG bool hasMultipleUseIds() const noexcept { return _useIdMask != 0u && !Support::isPowerOf2(_useIdMask); } + ASMJIT_INLINE_NODEBUG bool hasMultipleUseIds() const noexcept { return Support::hasAtLeast2BitsSet(_useIdMask); } ASMJIT_INLINE_NODEBUG void addUseIdMask(RegMask mask) noexcept { _useIdMask |= mask; } diff --git a/src/asmjit/core/ralocal.cpp b/src/asmjit/core/ralocal.cpp index 78f5c02..f44ea09 100644 --- a/src/asmjit/core/ralocal.cpp +++ b/src/asmjit/core/ralocal.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/ralocal_p.h b/src/asmjit/core/ralocal_p.h index 02beabe..6abad83 100644 --- a/src/asmjit/core/ralocal_p.h +++ b/src/asmjit/core/ralocal_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_RALOCAL_P_H_INCLUDED diff --git a/src/asmjit/core/rapass.cpp b/src/asmjit/core/rapass.cpp index 794eeb6..87bca95 100644 --- a/src/asmjit/core/rapass.cpp +++ b/src/asmjit/core/rapass.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -58,8 +58,6 @@ BaseRAPass::~BaseRAPass() noexcept {} // ========================== static void BaseRAPass_reset(BaseRAPass* self, FuncDetail* funcDetail) noexcept { - ZoneAllocator* allocator = self->allocator(); - self->_blocks.reset(); self->_exits.reset(); self->_pov.reset(); @@ -74,7 +72,7 @@ static void BaseRAPass_reset(BaseRAPass* self, FuncDetail* funcDetail) noexcept self->_physRegIndex.reset(); self->_physRegCount.reset(); self->_physRegTotal = 0; - self->_scratchRegIndexes.fill(BaseReg::kIdBad); + self->_scratchRegIndexes.fill(Reg::kIdBad); self->_availableRegs.reset(); self->_clobberedRegs.reset(); @@ -86,7 +84,7 @@ static void BaseRAPass_reset(BaseRAPass* self, FuncDetail* funcDetail) noexcept self->_globalMaxLiveCount.reset(); self->_temporaryMem.reset(); - self->_stackAllocator.reset(allocator); + self->_stackAllocator.reset(self->_allocator.zone(), &self->_allocator); self->_argsAssignment.reset(funcDetail); self->_numStackArgsToStackSlots = 0; self->_maxWorkRegNameSize = 0; @@ -716,7 +714,7 @@ RAAssignment::WorkToPhysMap* BaseRAPass::newWorkToPhysMap() noexcept { return const_cast(&nullMap); } - WorkToPhysMap* map = zone()->allocT(size); + WorkToPhysMap* map = zone()->alloc(size); if (ASMJIT_UNLIKELY(!map)) { return nullptr; } @@ -729,7 +727,7 @@ RAAssignment::PhysToWorkMap* BaseRAPass::newPhysToWorkMap() noexcept { uint32_t count = physRegTotal(); size_t size = PhysToWorkMap::sizeOf(count); - PhysToWorkMap* map = zone()->allocT(size); + PhysToWorkMap* map = zone()->alloc(size); if (ASMJIT_UNLIKELY(!map)) { return nullptr; } @@ -1188,7 +1186,7 @@ Error BaseRAPass::assignArgIndexToWorkRegs() noexcept { workReg->setArgIndex(argIndex, valueIndex); const FuncValue& arg = func()->detail().arg(argIndex, valueIndex); - if (arg.isReg() && _archTraits->regTypeToGroup(arg.regType()) == workReg->group()) { + if (arg.isReg() && RegUtils::groupOf(arg.regType()) == workReg->group()) { workReg->setHintRegId(arg.regId()); } } @@ -1232,7 +1230,7 @@ ASMJIT_FAVOR_SPEED Error BaseRAPass::initGlobalLiveSpans() noexcept { LiveRegSpans* liveSpans = nullptr; if (physCount) { - liveSpans = allocator()->allocT(physCount * sizeof(LiveRegSpans)); + liveSpans = allocator()->zone()->alloc(physCount * sizeof(LiveRegSpans)); if (ASMJIT_UNLIKELY(!liveSpans)) { return DebugUtils::errored(kErrorOutOfMemory); } @@ -2072,7 +2070,7 @@ ASMJIT_FAVOR_SIZE Error BaseRAPass::annotateCode() noexcept { } } - node->setInlineComment(static_cast(cc()->_dataZone.dup(sb.data(), sb.size(), true))); + node->setInlineComment(static_cast(cc()->_codeZone.dup(sb.data(), sb.size(), true))); if (node == last) { break; } diff --git a/src/asmjit/core/rapass_p.h b/src/asmjit/core/rapass_p.h index 1cf19f9..eef59b5 100644 --- a/src/asmjit/core/rapass_p.h +++ b/src/asmjit/core/rapass_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_RAPASS_P_H_INCLUDED @@ -615,13 +615,13 @@ public: RegGroup group = workReg->group(); RATiedReg* tiedReg = workReg->tiedReg(); - if (useId != BaseReg::kIdBad) { + if (useId != Reg::kIdBad) { _stats.makeFixed(group); _used[group] |= Support::bitMask(useId); flags |= RATiedFlags::kUseFixed; } - if (outId != BaseReg::kIdBad) { + if (outId != Reg::kIdBad) { _clobbered[group] |= Support::bitMask(outId); flags |= RATiedFlags::kOutFixed; } @@ -650,14 +650,14 @@ public: tiedReg->_consecutiveParent = consecutiveParent; } - if (useId != BaseReg::kIdBad) { + if (useId != Reg::kIdBad) { if (ASMJIT_UNLIKELY(tiedReg->hasUseId())) { return DebugUtils::errored(kErrorOverlappedRegs); } tiedReg->setUseId(useId); } - if (outId != BaseReg::kIdBad) { + if (outId != Reg::kIdBad) { if (ASMJIT_UNLIKELY(tiedReg->hasOutId())) { return DebugUtils::errored(kErrorOverlappedRegs); } @@ -677,7 +677,7 @@ public: [[nodiscard]] Error addCallArg(RAWorkReg* workReg, uint32_t useId) noexcept { - ASMJIT_ASSERT(useId != BaseReg::kIdBad); + ASMJIT_ASSERT(useId != Reg::kIdBad); RATiedFlags flags = RATiedFlags::kUse | RATiedFlags::kRead | RATiedFlags::kUseFixed; RegGroup group = workReg->group(); @@ -694,7 +694,7 @@ public: ASMJIT_ASSERT(tiedRegCount() < ASMJIT_ARRAY_SIZE(_tiedRegs)); tiedReg = _cur++; - tiedReg->init(workReg->workId(), flags, allocable, useId, 0, allocable, BaseReg::kIdBad, 0); + tiedReg->init(workReg->workId(), flags, allocable, useId, 0, allocable, Reg::kIdBad, 0); workReg->setTiedReg(tiedReg); workReg->assignBasicBlock(_basicBlockId); @@ -720,7 +720,7 @@ public: [[nodiscard]] Error addCallRet(RAWorkReg* workReg, uint32_t outId) noexcept { - ASMJIT_ASSERT(outId != BaseReg::kIdBad); + ASMJIT_ASSERT(outId != Reg::kIdBad); RATiedFlags flags = RATiedFlags::kOut | RATiedFlags::kWrite | RATiedFlags::kOutFixed; RegGroup group = workReg->group(); @@ -737,7 +737,7 @@ public: ASMJIT_ASSERT(tiedRegCount() < ASMJIT_ARRAY_SIZE(_tiedRegs)); tiedReg = _cur++; - tiedReg->init(workReg->workId(), flags, Support::allOnes(), BaseReg::kIdBad, 0, outRegs, outId, 0); + tiedReg->init(workReg->workId(), flags, Support::allOnes(), Reg::kIdBad, 0, outRegs, outId, 0); workReg->setTiedReg(tiedReg); workReg->assignBasicBlock(_basicBlockId); @@ -901,9 +901,9 @@ public: Operand _temporaryMem = Operand(); //! Stack pointer. - BaseReg _sp = BaseReg(); + Reg _sp = Reg(); //! Frame pointer. - BaseReg _fp = BaseReg(); + Reg _fp = Reg(); //! Stack manager. RAStackAllocator _stackAllocator {}; //! Function arguments assignment. @@ -1302,7 +1302,7 @@ public: inline BaseMem workRegAsMem(RAWorkReg* workReg) noexcept { (void)getOrCreateStackSlot(workReg); return BaseMem(OperandSignature::fromOpType(OperandType::kMem) | - OperandSignature::fromMemBaseType(_sp.type()) | + OperandSignature::fromMemBaseType(_sp.regType()) | OperandSignature::fromBits(OperandSignature::kMemRegHomeFlag), workReg->virtId(), 0, 0); } @@ -1315,8 +1315,7 @@ public: [[nodiscard]] inline PhysToWorkMap* clonePhysToWorkMap(const PhysToWorkMap* map) noexcept { - size_t size = PhysToWorkMap::sizeOf(_physRegTotal); - return static_cast(zone()->dupAligned(map, size, sizeof(uint32_t))); + return static_cast(zone()->dup(map, PhysToWorkMap::sizeOf(_physRegTotal))); } //! \name Liveness Analysis & Statistics diff --git a/src/asmjit/core/rastack.cpp b/src/asmjit/core/rastack.cpp index bb8b06e..4ee11bc 100644 --- a/src/asmjit/core/rastack.cpp +++ b/src/asmjit/core/rastack.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -19,7 +19,7 @@ RAStackSlot* RAStackAllocator::newSlot(uint32_t baseRegId, uint32_t size, uint32 return nullptr; } - RAStackSlot* slot = allocator()->allocT(); + RAStackSlot* slot = zone()->alloc(); if (ASMJIT_UNLIKELY(!slot)) { return nullptr; } @@ -27,9 +27,9 @@ RAStackSlot* RAStackAllocator::newSlot(uint32_t baseRegId, uint32_t size, uint32 slot->_baseRegId = uint8_t(baseRegId); slot->_alignment = uint8_t(Support::max(alignment, 1)); slot->_flags = uint16_t(flags); - slot->_useCount = 0; slot->_size = size; + slot->_useCount = 0; slot->_weight = 0; slot->_offset = 0; diff --git a/src/asmjit/core/rastack_p.h b/src/asmjit/core/rastack_p.h index 1aff61b..e64d3a0 100644 --- a/src/asmjit/core/rastack_p.h +++ b/src/asmjit/core/rastack_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_RASTACK_P_H_INCLUDED @@ -27,8 +27,8 @@ struct RAStackSlot { kFlagRegHome = 0x0001u, //! Stack slot position matches argument passed via stack. kFlagStackArg = 0x0002u - }; + }; enum ArgIndex : uint32_t { kNoArgIndex = 0xFF }; @@ -121,8 +121,11 @@ public: //! \name Members //! \{ - //! Allocator used to allocate internal data. + //! Zone used to allocate internal data. + Zone* _zone {}; + //! Container allocator used to allocate internal data. ZoneAllocator* _allocator {}; + //! Count of bytes used by all slots. uint32_t _bytesUsed {}; //! Calculated stack size (can be a bit greater than `_bytesUsed`). @@ -139,7 +142,8 @@ public: ASMJIT_INLINE_NODEBUG RAStackAllocator() noexcept {} - ASMJIT_INLINE_NODEBUG void reset(ZoneAllocator* allocator) noexcept { + ASMJIT_INLINE_NODEBUG void reset(Zone* zone, ZoneAllocator* allocator) noexcept { + _zone = zone; _allocator = allocator; _bytesUsed = 0; _stackSize = 0; @@ -152,6 +156,9 @@ public: //! \name Accessors //! \{ + [[nodiscard]] + ASMJIT_INLINE_NODEBUG Zone* zone() const noexcept { return _zone; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG ZoneAllocator* allocator() const noexcept { return _allocator; } diff --git a/src/asmjit/core/string.cpp b/src/asmjit/core/string.cpp index 0aa349b..12e6fb3 100644 --- a/src/asmjit/core/string.cpp +++ b/src/asmjit/core/string.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/string.h b/src/asmjit/core/string.h index dccaf25..56bccfe 100644 --- a/src/asmjit/core/string.h +++ b/src/asmjit/core/string.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_STRING_H_INCLUDED @@ -35,7 +35,6 @@ union FixedString { //! \name Constants //! \{ - // This cannot be constexpr as GCC 4.8 refuses constexpr members of unions. static inline constexpr uint32_t kNumUInt32Words = uint32_t((N + sizeof(uint32_t) - 1) / sizeof(uint32_t)); //! \} diff --git a/src/asmjit/core/support.cpp b/src/asmjit/core/support.cpp index bebabbc..7e994ea 100644 --- a/src/asmjit/core/support.cpp +++ b/src/asmjit/core/support.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -113,10 +113,92 @@ static void testBitUtils() noexcept { EXPECT_EQ(Support::popcnt(0x7FFFFFFF), 31u); INFO("Support::isPowerOf2()"); + EXPECT_FALSE(Support::isPowerOf2(uint8_t(0))); + EXPECT_FALSE(Support::isPowerOf2(uint16_t(0))); + EXPECT_FALSE(Support::isPowerOf2(uint32_t(0))); + EXPECT_FALSE(Support::isPowerOf2(uint64_t(0))); + + EXPECT_FALSE(Support::isPowerOf2(uint8_t(0xFFu))); + EXPECT_FALSE(Support::isPowerOf2(uint16_t(0xFFFFu))); + EXPECT_FALSE(Support::isPowerOf2(uint32_t(0xFFFFFFFFu))); + EXPECT_FALSE(Support::isPowerOf2(uint64_t(0xFFFFFFFFFFFFFFFFu))); + + for (i = 0; i < 32; i++) { + EXPECT_TRUE(Support::isPowerOf2(uint32_t(1) << i)); + EXPECT_FALSE(Support::isPowerOf2((uint32_t(1) << i) ^ 0x001101)); + } + for (i = 0; i < 64; i++) { EXPECT_TRUE(Support::isPowerOf2(uint64_t(1) << i)); EXPECT_FALSE(Support::isPowerOf2((uint64_t(1) << i) ^ 0x001101)); } + + INFO("Support::isPowerOf2UpTo()"); + EXPECT_FALSE(Support::isPowerOf2UpTo(uint8_t(0), 8)); + EXPECT_FALSE(Support::isPowerOf2UpTo(uint16_t(0), 8)); + EXPECT_FALSE(Support::isPowerOf2UpTo(uint32_t(0), 8)); + EXPECT_FALSE(Support::isPowerOf2UpTo(uint64_t(0), 8)); + + EXPECT_FALSE(Support::isPowerOf2UpTo(uint8_t(0xFFu), 8)); + EXPECT_FALSE(Support::isPowerOf2UpTo(uint16_t(0xFFFFu), 8)); + EXPECT_FALSE(Support::isPowerOf2UpTo(uint32_t(0xFFFFFFFFu), 8)); + EXPECT_FALSE(Support::isPowerOf2UpTo(uint64_t(0xFFFFFFFFFFFFFFFFu), 8)); + + EXPECT_TRUE(Support::isPowerOf2UpTo(uint32_t(1), 8)); + EXPECT_TRUE(Support::isPowerOf2UpTo(uint32_t(2), 8)); + EXPECT_FALSE(Support::isPowerOf2UpTo(uint32_t(3), 8)); + EXPECT_TRUE(Support::isPowerOf2UpTo(uint32_t(4), 8)); + EXPECT_TRUE(Support::isPowerOf2UpTo(uint32_t(8), 8)); + EXPECT_FALSE(Support::isPowerOf2UpTo(uint32_t(9), 8)); + EXPECT_FALSE(Support::isPowerOf2UpTo(uint32_t(16), 8)); + EXPECT_FALSE(Support::isPowerOf2UpTo(uint32_t(0xFFFFFFFFu), 8)); + + EXPECT_TRUE(Support::isPowerOf2UpTo(uint32_t(16), 16)); + EXPECT_FALSE(Support::isPowerOf2UpTo(uint32_t(32), 16)); + + INFO("Support::isZeroOrPowerOf2()"); + EXPECT_TRUE(Support::isZeroOrPowerOf2(uint8_t(0))); + EXPECT_TRUE(Support::isZeroOrPowerOf2(uint16_t(0))); + EXPECT_TRUE(Support::isZeroOrPowerOf2(uint32_t(0))); + EXPECT_TRUE(Support::isZeroOrPowerOf2(uint64_t(0))); + + EXPECT_FALSE(Support::isZeroOrPowerOf2(uint8_t(0xFFu))); + EXPECT_FALSE(Support::isZeroOrPowerOf2(uint16_t(0xFFFFu))); + EXPECT_FALSE(Support::isZeroOrPowerOf2(uint32_t(0xFFFFFFFFu))); + EXPECT_FALSE(Support::isZeroOrPowerOf2(uint64_t(0xFFFFFFFFFFFFFFFFu))); + + for (i = 0; i < 32; i++) { + EXPECT_TRUE(Support::isZeroOrPowerOf2(uint32_t(1) << i)); + EXPECT_FALSE(Support::isZeroOrPowerOf2((uint32_t(1) << i) ^ 0x001101)); + } + + for (i = 0; i < 64; i++) { + EXPECT_TRUE(Support::isZeroOrPowerOf2(uint64_t(1) << i)); + EXPECT_FALSE(Support::isZeroOrPowerOf2((uint64_t(1) << i) ^ 0x001101)); + } + + INFO("Support::isZeroOrPowerOf2UpTo()"); + EXPECT_TRUE(Support::isZeroOrPowerOf2UpTo(uint8_t(0), 8)); + EXPECT_TRUE(Support::isZeroOrPowerOf2UpTo(uint16_t(0), 8)); + EXPECT_TRUE(Support::isZeroOrPowerOf2UpTo(uint32_t(0), 8)); + EXPECT_TRUE(Support::isZeroOrPowerOf2UpTo(uint64_t(0), 8)); + + EXPECT_FALSE(Support::isZeroOrPowerOf2UpTo(uint8_t(0xFFu), 8)); + EXPECT_FALSE(Support::isZeroOrPowerOf2UpTo(uint16_t(0xFFFFu), 8)); + EXPECT_FALSE(Support::isZeroOrPowerOf2UpTo(uint32_t(0xFFFFFFFFu), 8)); + EXPECT_FALSE(Support::isZeroOrPowerOf2UpTo(uint64_t(0xFFFFFFFFFFFFFFFFu), 8)); + + EXPECT_TRUE(Support::isZeroOrPowerOf2UpTo(uint32_t(1), 8)); + EXPECT_TRUE(Support::isZeroOrPowerOf2UpTo(uint32_t(2), 8)); + EXPECT_FALSE(Support::isZeroOrPowerOf2UpTo(uint32_t(3), 8)); + EXPECT_TRUE(Support::isZeroOrPowerOf2UpTo(uint32_t(4), 8)); + EXPECT_TRUE(Support::isZeroOrPowerOf2UpTo(uint32_t(8), 8)); + EXPECT_FALSE(Support::isZeroOrPowerOf2UpTo(uint32_t(9), 8)); + EXPECT_FALSE(Support::isZeroOrPowerOf2UpTo(uint32_t(16), 8)); + EXPECT_FALSE(Support::isZeroOrPowerOf2UpTo(uint32_t(0xFFFFFFFFu), 8)); + + EXPECT_TRUE(Support::isZeroOrPowerOf2UpTo(uint32_t(16), 16)); + EXPECT_FALSE(Support::isZeroOrPowerOf2UpTo(uint32_t(32), 16)); } static void testIntUtils() noexcept { @@ -194,19 +276,19 @@ static void testReadWrite() noexcept { uint8_t arr[32] = { 0 }; - Support::writeU16uBE(arr + 1, 0x0102u); - Support::writeU16uBE(arr + 3, 0x0304u); - EXPECT_EQ(Support::readU32uBE(arr + 1), 0x01020304u); - EXPECT_EQ(Support::readU32uLE(arr + 1), 0x04030201u); - EXPECT_EQ(Support::readU32uBE(arr + 2), 0x02030400u); - EXPECT_EQ(Support::readU32uLE(arr + 2), 0x00040302u); + Support::storeu_u16_be(arr + 1, 0x0102u); + Support::storeu_u16_be(arr + 3, 0x0304u); + EXPECT_EQ(Support::loadu_u32_be(arr + 1), 0x01020304u); + EXPECT_EQ(Support::loadu_u32_le(arr + 1), 0x04030201u); + EXPECT_EQ(Support::loadu_u32_be(arr + 2), 0x02030400u); + EXPECT_EQ(Support::loadu_u32_le(arr + 2), 0x00040302u); - Support::writeU32uLE(arr + 5, 0x05060708u); - EXPECT_EQ(Support::readU64uBE(arr + 1), 0x0102030408070605u); - EXPECT_EQ(Support::readU64uLE(arr + 1), 0x0506070804030201u); + Support::storeu_u32_le(arr + 5, 0x05060708u); + EXPECT_EQ(Support::loadu_u64_be(arr + 1), 0x0102030408070605u); + EXPECT_EQ(Support::loadu_u64_le(arr + 1), 0x0506070804030201u); - Support::writeU64uLE(arr + 7, 0x1122334455667788u); - EXPECT_EQ(Support::readU32uBE(arr + 8), 0x77665544u); + Support::storeu_u64_le(arr + 7, 0x1122334455667788u); + EXPECT_EQ(Support::loadu_u32_be(arr + 8), 0x77665544u); } static void testBitVector() noexcept { diff --git a/src/asmjit/core/support.h b/src/asmjit/core/support.h index bf25f2b..e4ad29b 100644 --- a/src/asmjit/core/support.h +++ b/src/asmjit/core/support.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_SUPPORT_H_INCLUDED @@ -35,16 +35,17 @@ namespace Internal { template struct AliasedUInt {}; - template<> struct AliasedUInt { typedef uint16_t ASMJIT_MAY_ALIAS T; }; - template<> struct AliasedUInt { typedef uint32_t ASMJIT_MAY_ALIAS T; }; - template<> struct AliasedUInt { typedef uint64_t ASMJIT_MAY_ALIAS T; }; + template<> struct AliasedUInt { typedef uint8_t Type; }; + template<> struct AliasedUInt { typedef uint16_t ASMJIT_MAY_ALIAS Type; }; + template<> struct AliasedUInt { typedef uint32_t ASMJIT_MAY_ALIAS Type; }; + template<> struct AliasedUInt { typedef uint64_t ASMJIT_MAY_ALIAS Type; }; - template<> struct AliasedUInt { typedef uint16_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 1); }; - template<> struct AliasedUInt { typedef uint32_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 1); }; - template<> struct AliasedUInt { typedef uint32_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 2); }; - template<> struct AliasedUInt { typedef uint64_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 1); }; - template<> struct AliasedUInt { typedef uint64_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 2); }; - template<> struct AliasedUInt { typedef uint64_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 4); }; + template<> struct AliasedUInt { typedef uint16_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(1, Type); }; + template<> struct AliasedUInt { typedef uint32_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(1, Type); }; + template<> struct AliasedUInt { typedef uint32_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(2, Type); }; + template<> struct AliasedUInt { typedef uint64_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(1, Type); }; + template<> struct AliasedUInt { typedef uint64_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(2, Type); }; + template<> struct AliasedUInt { typedef uint64_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(4, Type); }; // StdInt - Make an int-type by size (signed or unsigned) that is the // same as types defined by . @@ -121,6 +122,42 @@ struct EnumValues { ASMJIT_INLINE_NODEBUG Iterator end() const noexcept { return Iterator{ValueType(to) + 1}; } }; +// Support - Pointer Operations +// ============================ + +template +static ASMJIT_INLINE_NODEBUG Dst* offsetPtr(Src* ptr, const Offset& n) noexcept { + return static_cast( + static_cast(static_cast(static_cast(ptr)) + n) + ); +} + +template +static ASMJIT_INLINE_NODEBUG const Dst* offsetPtr(const Src* ptr, const Offset& n) noexcept { + return static_cast( + static_cast(static_cast(static_cast(ptr)) + n) + ); +} + +// Support - Boolean Operations +// ============================ + +namespace Internal { + static ASMJIT_INLINE_CONSTEXPR unsigned unsigned_from_bool(bool b) noexcept { + return unsigned(b); + } +} + +template +static ASMJIT_INLINE_CONSTEXPR bool bool_and(Args&&... args) noexcept { + return bool( (... & Internal::unsigned_from_bool(args)) ); +} + +template +static ASMJIT_INLINE_CONSTEXPR bool bool_or(Args&&... args) noexcept { + return bool( (... | Internal::unsigned_from_bool(args)) ); +} + // Support - BitCast // ================= @@ -139,7 +176,16 @@ namespace Internal { //! //! Useful to bit-cast between integers and floating points. template -static ASMJIT_INLINE_NODEBUG Dst bitCast(const Src& x) noexcept { return Internal::BitCastUnion(x).dst; } +static ASMJIT_INLINE_NODEBUG Dst bitCast(const Src& x) noexcept { + static_assert(sizeof(Dst) == sizeof(Src), "bitCast can only be used to cast types of same size"); + + if constexpr (std::is_integral_v && std::is_integral_v) { + return Dst(x); + } + else { + return Internal::BitCastUnion(x).dst; + } +} // Support - BitOps // ================ @@ -571,6 +617,17 @@ template [[nodiscard]] static ASMJIT_INLINE_NODEBUG uint32_t constPopcnt(T x) noexcept { return Internal::constPopcntImpl(asUInt(x)); } +// Support - HasAtLeast2BitsSet +// ============================ + +//! Tests whether `x` has at least 2 bits set. +template +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR bool hasAtLeast2BitsSet(T x) noexcept { + using U = std::make_unsigned_t; + return !(U(x) & U(U(x) - U(1))); +} + // Support - Min/Max // ================= @@ -637,7 +694,7 @@ namespace Internal { inline T subOverflowFallback(T x, T y, FastUInt8* of) noexcept { using U = std::make_unsigned_t; - U result = U(x) - U(y); + U result = U(U(x) - U(y)); *of = FastUInt8(*of | FastUInt8(std::is_unsigned_v ? result > U(x) : T((U(x) ^ U(y)) & (U(x) ^ result)) < 0)); return T(result); } @@ -731,6 +788,49 @@ static inline T subOverflow(const T& x, const T& y, FastUInt8* of) noexcept { re template static inline T mulOverflow(const T& x, const T& y, FastUInt8* of) noexcept { return T(Internal::mulOverflowImpl(asStdInt(x), asStdInt(y), of)); } +template +static inline T maddOverflow(const T& x, const T& y, const T& addend, FastUInt8* of) noexcept { + T v = T(Internal::mulOverflowImpl(asStdInt(x), asStdInt(y), of)); + return T(Internal::addOverflowImpl(asStdInt(v), asStdInt(addend), of)); +} + +// Support - IsPowerOf2 +// ==================== + +//! Tests whether `x` is a power of two (only one bit is set). +template +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR bool isPowerOf2(T x) noexcept { + using U = std::make_unsigned_t; + U x_minus_1 = U(U(x) - U(1)); + return U(U(x) ^ x_minus_1) > x_minus_1; +} + +//! Tests whether `x` is a power of two up to `n`. +template +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR bool isPowerOf2UpTo(T x, N n) noexcept { + using U = std::make_unsigned_t; + U x_minus_1 = U(U(x) - U(1)); + return bool_and(x_minus_1 < U(n), !(U(x) & x_minus_1)); +} + +//! Tests whether `x` is either zero or a power of two (only one bit is set). +template +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR bool isZeroOrPowerOf2(T x) noexcept { + using U = std::make_unsigned_t; + return !(U(x) & (U(x) - U(1))); +} + +//! Tests whether `x` is either zero or a power of two up to `n`. +template +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR bool isZeroOrPowerOf2UpTo(T x, N n) noexcept { + using U = std::make_unsigned_t; + return bool_and(U(x) <= U(n), !(U(x) & (U(x) - U(1)))); +} + // Support - Alignment // =================== @@ -741,14 +841,6 @@ static ASMJIT_INLINE_CONSTEXPR bool isAligned(X base, Y alignment) noexcept { return ((U)base % (U)alignment) == 0; } -//! Tests whether the `x` is a power of two (only one bit is set). -template -[[nodiscard]] -static ASMJIT_INLINE_CONSTEXPR bool isPowerOf2(T x) noexcept { - using U = std::make_unsigned_t; - return x && !(U(x) & (U(x) - U(1))); -} - template [[nodiscard]] static ASMJIT_INLINE_CONSTEXPR X alignUp(X x, Y alignment) noexcept { @@ -971,6 +1063,25 @@ static ASMJIT_INLINE_NODEBUG uint64_t byteswap64(uint64_t x) noexcept { #endif } +template +[[nodiscard]] +static ASMJIT_INLINE_NODEBUG T byteswap(T x) noexcept { + static_assert(std::is_integral_v, "byteswap() expects the given type to be integral"); + if constexpr (sizeof(T) == 8) { + return T(byteswap64(uint64_t(x))); + } + else if constexpr (sizeof(T) == 4) { + return T(byteswap32(uint32_t(x))); + } + else if constexpr (sizeof(T) == 2) { + return T(byteswap16(uint16_t(x))); + } + else { + static_assert(sizeof(T) == 1, "byteswap() can be used with a type of size 1, 2, 4, or 8"); + return x; + } +} + // Support - BytePack & Unpack // =========================== @@ -1056,321 +1167,179 @@ static ASMJIT_INLINE int compareStringViews(const char* aData, size_t aSize, con return int(aSize) - int(bSize); } -// Support - Memory Read Access - 8 Bits -// ===================================== +// Support - Aligned / Unaligned Memory Read Access +// ================================================ +template [[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint8_t readU8(const void* p) noexcept { return static_cast(p)[0]; } +static ASMJIT_INLINE_NODEBUG T loada(const void* p) noexcept { + static_assert(std::is_integral_v, "loada() expects the data-type to be integral"); -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int8_t readI8(const void* p) noexcept { return static_cast(p)[0]; } - -// Support - Memory Read Access - 16 Bits -// ====================================== - -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint16_t readU16x(const void* p) noexcept { - using U16AlignedToN = typename Internal::AliasedUInt::T; - uint16_t x = static_cast(p)[0]; - return BO == ByteOrder::kNative ? x : byteswap16(x); + return *static_cast(p); } -template +template [[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint16_t readU16u(const void* p) noexcept { return readU16x(p); } +static ASMJIT_INLINE_NODEBUG T loadu(const void* p) noexcept { + static_assert(std::is_integral_v, "loadu() expects the data-type to be integral"); -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint16_t readU16uLE(const void* p) noexcept { return readU16x(p); } + using UnsignedType = typename Internal::StdInt::Type; + using UnalignedType = typename Internal::AliasedUInt::Type; -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint16_t readU16uBE(const void* p) noexcept { return readU16x(p); } - -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint16_t readU16a(const void* p) noexcept { return readU16x(p); } - -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint16_t readU16aLE(const void* p) noexcept { return readU16x(p); } - -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint16_t readU16aBE(const void* p) noexcept { return readU16x(p); } - -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int16_t readI16x(const void* p) noexcept { return int16_t(readU16x(p)); } - -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int16_t readI16u(const void* p) noexcept { return int16_t(readU16x(p)); } - -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int16_t readI16uLE(const void* p) noexcept { return int16_t(readU16x(p)); } - -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int16_t readI16uBE(const void* p) noexcept { return int16_t(readU16x(p)); } - -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int16_t readI16a(const void* p) noexcept { return int16_t(readU16x(p)); } - -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int16_t readI16aLE(const void* p) noexcept { return int16_t(readU16x(p)); } - -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int16_t readI16aBE(const void* p) noexcept { return int16_t(readU16x(p)); } - -// Support - Memory Read Access - 24 Bits -// ====================================== - -template -[[nodiscard]] -static inline uint32_t readU24u(const void* p) noexcept { - uint32_t b0 = readU8(static_cast(p) + (BO == ByteOrder::kLE ? 2u : 0u)); - uint32_t b1 = readU8(static_cast(p) + 1u); - uint32_t b2 = readU8(static_cast(p) + (BO == ByteOrder::kLE ? 0u : 2u)); - return (b0 << 16) | (b1 << 8) | b2; + return T(*static_cast(p)); } +template [[nodiscard]] -static inline uint32_t readU24uLE(const void* p) noexcept { return readU24u(p); } - -[[nodiscard]] -static inline uint32_t readU24uBE(const void* p) noexcept { return readU24u(p); } - -// Support - Memory Read Access - 32 Bits -// ====================================== - -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint32_t readU32x(const void* p) noexcept { - using U32AlignedToN = typename Internal::AliasedUInt::T; - uint32_t x = static_cast(p)[0]; - return BO == ByteOrder::kNative ? x : byteswap32(x); +static ASMJIT_INLINE_NODEBUG T loada(const void* p) noexcept { + T v = loada(p); + if constexpr (BO != ByteOrder::kNative) { + v = byteswap(v); + } + return v; } -template +template [[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint32_t readU32u(const void* p) noexcept { return readU32x(p); } - -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint32_t readU32uLE(const void* p) noexcept { return readU32x(p); } - -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint32_t readU32uBE(const void* p) noexcept { return readU32x(p); } - -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint32_t readU32a(const void* p) noexcept { return readU32x(p); } - -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint32_t readU32aLE(const void* p) noexcept { return readU32x(p); } - -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint32_t readU32aBE(const void* p) noexcept { return readU32x(p); } - -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint32_t readI32x(const void* p) noexcept { return int32_t(readU32x(p)); } - -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int32_t readI32u(const void* p) noexcept { return int32_t(readU32x(p)); } - -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int32_t readI32uLE(const void* p) noexcept { return int32_t(readU32x(p)); } - -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int32_t readI32uBE(const void* p) noexcept { return int32_t(readU32x(p)); } - -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int32_t readI32a(const void* p) noexcept { return int32_t(readU32x(p)); } - -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int32_t readI32aLE(const void* p) noexcept { return int32_t(readU32x(p)); } - -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int32_t readI32aBE(const void* p) noexcept { return int32_t(readU32x(p)); } - -// Support - Memory Read Access - 64 Bits -// ====================================== - -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint64_t readU64x(const void* p) noexcept { - using U64AlignedToN = typename Internal::AliasedUInt::T; - uint64_t x = static_cast(p)[0]; - return BO == ByteOrder::kNative ? x : byteswap64(x); +static ASMJIT_INLINE_NODEBUG T loadu(const void* p) noexcept { + T v = loadu(p); + if constexpr (BO != ByteOrder::kNative) { + v = byteswap(v); + } + return v; } -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint64_t readU64u(const void* p) noexcept { return readU64x(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int8_t load_i8(const void* p) noexcept { return *static_cast(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint8_t load_u8(const void* p) noexcept { return *static_cast(p); } -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint64_t readU64uLE(const void* p) noexcept { return readU64x(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int16_t loada_i16(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int16_t loadu_i16(const void* p) noexcept { return loadu(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint16_t loada_u16(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint16_t loadu_u16(const void* p) noexcept { return loadu(p); } -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint64_t readU64uBE(const void* p) noexcept { return readU64x(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int16_t loada_i16_le(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int16_t loadu_i16_le(const void* p) noexcept { return loadu(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint16_t loada_u16_le(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint16_t loadu_u16_le(const void* p) noexcept { return loadu(p); } -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint64_t readU64a(const void* p) noexcept { return readU64x(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int16_t loada_i16_be(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int16_t loadu_i16_be(const void* p) noexcept { return loadu(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint16_t loada_u16_be(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint16_t loadu_u16_be(const void* p) noexcept { return loadu(p); } -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint64_t readU64aLE(const void* p) noexcept { return readU64x(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int32_t loada_i32(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int32_t loadu_i32(const void* p) noexcept { return loadu(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint32_t loada_u32(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint32_t loadu_u32(const void* p) noexcept { return loadu(p); } -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG uint64_t readU64aBE(const void* p) noexcept { return readU64x(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int32_t loada_i32_le(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int32_t loadu_i32_le(const void* p) noexcept { return loadu(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint32_t loada_u32_le(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint32_t loadu_u32_le(const void* p) noexcept { return loadu(p); } -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int64_t readI64x(const void* p) noexcept { return int64_t(readU64x(p)); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int32_t loada_i32_be(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int32_t loadu_i32_be(const void* p) noexcept { return loadu(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint32_t loada_u32_be(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint32_t loadu_u32_be(const void* p) noexcept { return loadu(p); } -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int64_t readI64u(const void* p) noexcept { return int64_t(readU64x(p)); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int64_t loada_i64(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int64_t loadu_i64(const void* p) noexcept { return loadu(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint64_t loada_u64(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint64_t loadu_u64(const void* p) noexcept { return loadu(p); } -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int64_t readI64uLE(const void* p) noexcept { return int64_t(readU64x(p)); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int64_t loada_i64_le(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int64_t loadu_i64_le(const void* p) noexcept { return loadu(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint64_t loada_u64_le(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint64_t loadu_u64_le(const void* p) noexcept { return loadu(p); } -template -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int64_t readI64uBE(const void* p) noexcept { return int64_t(readU64x(p)); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int64_t loada_i64_be(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG int64_t loadu_i64_be(const void* p) noexcept { return loadu(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint64_t loada_u64_be(const void* p) noexcept { return loada(p); } +[[nodiscard]] static ASMJIT_INLINE_NODEBUG uint64_t loadu_u64_be(const void* p) noexcept { return loadu(p); } -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int64_t readI64a(const void* p) noexcept { return int64_t(readU64x(p)); } +// Support - Aligned / Unaligned Memory Write Access +// ================================================= -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int64_t readI64aLE(const void* p) noexcept { return int64_t(readU64x(p)); } +template +static ASMJIT_INLINE_NODEBUG void storea(void* p, T x) noexcept { + static_assert(std::is_integral_v, "storea() expects its data-type to be integral"); -[[nodiscard]] -static ASMJIT_INLINE_NODEBUG int64_t readI64aBE(const void* p) noexcept { return int64_t(readU64x(p)); } - -// Support - Memory Write Access - 8 Bits -// ====================================== - -static ASMJIT_INLINE_NODEBUG void writeU8(void* p, uint8_t x) noexcept { static_cast(p)[0] = x; } -static ASMJIT_INLINE_NODEBUG void writeI8(void* p, int8_t x) noexcept { static_cast(p)[0] = x; } - -// Support - Memory Write Access - 16 Bits -// ======================================= - -template -static ASMJIT_INLINE_NODEBUG void writeU16x(void* p, uint16_t x) noexcept { - using U16AlignedToN = typename Internal::AliasedUInt::T; - static_cast(p)[0] = BO == ByteOrder::kNative ? x : byteswap16(x); + *static_cast(p) = x; } -template -static ASMJIT_INLINE_NODEBUG void writeU16uLE(void* p, uint16_t x) noexcept { writeU16x(p, x); } -template -static ASMJIT_INLINE_NODEBUG void writeU16uBE(void* p, uint16_t x) noexcept { writeU16x(p, x); } +template +static ASMJIT_INLINE_NODEBUG void storeu(void* p, T x) noexcept { + static_assert(std::is_integral_v, "storeu() expects its data-type to be integral"); -static ASMJIT_INLINE_NODEBUG void writeU16a(void* p, uint16_t x) noexcept { writeU16x(p, x); } -static ASMJIT_INLINE_NODEBUG void writeU16aLE(void* p, uint16_t x) noexcept { writeU16x(p, x); } -static ASMJIT_INLINE_NODEBUG void writeU16aBE(void* p, uint16_t x) noexcept { writeU16x(p, x); } + using UnsignedType = typename Internal::StdInt::Type; + using UnalignedType = typename Internal::AliasedUInt::Type; - -template -static ASMJIT_INLINE_NODEBUG void writeI16x(void* p, int16_t x) noexcept { writeU16x(p, uint16_t(x)); } - -template -static ASMJIT_INLINE_NODEBUG void writeI16uLE(void* p, int16_t x) noexcept { writeU16x(p, uint16_t(x)); } -template -static ASMJIT_INLINE_NODEBUG void writeI16uBE(void* p, int16_t x) noexcept { writeU16x(p, uint16_t(x)); } - -static ASMJIT_INLINE_NODEBUG void writeI16a(void* p, int16_t x) noexcept { writeU16x(p, uint16_t(x)); } -static ASMJIT_INLINE_NODEBUG void writeI16aLE(void* p, int16_t x) noexcept { writeU16x(p, uint16_t(x)); } -static ASMJIT_INLINE_NODEBUG void writeI16aBE(void* p, int16_t x) noexcept { writeU16x(p, uint16_t(x)); } - -// Support - Memory Write Access - 24 Bits -// ======================================= - -template -static inline void writeU24u(void* p, uint32_t v) noexcept { - static_cast(p)[0] = uint8_t((v >> (BO == ByteOrder::kLE ? 0 : 16)) & 0xFFu); - static_cast(p)[1] = uint8_t((v >> 8) & 0xFFu); - static_cast(p)[2] = uint8_t((v >> (BO == ByteOrder::kLE ? 16 : 0)) & 0xFFu); + *static_cast(p) = UnsignedType(x); } -static inline void writeU24uLE(void* p, uint32_t v) noexcept { writeU24u(p, v); } -static inline void writeU24uBE(void* p, uint32_t v) noexcept { writeU24u(p, v); } - -// Support - Memory Write Access - 32 Bits -// ======================================= - -template -static ASMJIT_INLINE_NODEBUG void writeU32x(void* p, uint32_t x) noexcept { - using U32AlignedToN = typename Internal::AliasedUInt::T; - static_cast(p)[0] = (BO == ByteOrder::kNative) ? x : Support::byteswap32(x); +template +static ASMJIT_INLINE_NODEBUG void storea(void* p, T x) noexcept { + if constexpr (BO != ByteOrder::kNative) { + x = byteswap(x); + } + storea(p, x); } -template -static ASMJIT_INLINE_NODEBUG void writeU32u(void* p, uint32_t x) noexcept { writeU32x(p, x); } -template -static ASMJIT_INLINE_NODEBUG void writeU32uLE(void* p, uint32_t x) noexcept { writeU32x(p, x); } -template -static ASMJIT_INLINE_NODEBUG void writeU32uBE(void* p, uint32_t x) noexcept { writeU32x(p, x); } - -static ASMJIT_INLINE_NODEBUG void writeU32a(void* p, uint32_t x) noexcept { writeU32x(p, x); } -static ASMJIT_INLINE_NODEBUG void writeU32aLE(void* p, uint32_t x) noexcept { writeU32x(p, x); } -static ASMJIT_INLINE_NODEBUG void writeU32aBE(void* p, uint32_t x) noexcept { writeU32x(p, x); } - -template -static ASMJIT_INLINE_NODEBUG void writeI32x(void* p, int32_t x) noexcept { writeU32x(p, uint32_t(x)); } - -template -static ASMJIT_INLINE_NODEBUG void writeI32u(void* p, int32_t x) noexcept { writeU32x(p, uint32_t(x)); } -template -static ASMJIT_INLINE_NODEBUG void writeI32uLE(void* p, int32_t x) noexcept { writeU32x(p, uint32_t(x)); } -template -static ASMJIT_INLINE_NODEBUG void writeI32uBE(void* p, int32_t x) noexcept { writeU32x(p, uint32_t(x)); } - -static ASMJIT_INLINE_NODEBUG void writeI32a(void* p, int32_t x) noexcept { writeU32x(p, uint32_t(x)); } -static ASMJIT_INLINE_NODEBUG void writeI32aLE(void* p, int32_t x) noexcept { writeU32x(p, uint32_t(x)); } -static ASMJIT_INLINE_NODEBUG void writeI32aBE(void* p, int32_t x) noexcept { writeU32x(p, uint32_t(x)); } - -// Support - Memory Write Access - 64 Bits -// ======================================= - -template -static ASMJIT_INLINE_NODEBUG void writeU64x(void* p, uint64_t x) noexcept { - using U64AlignedToN = typename Internal::AliasedUInt::T; - static_cast(p)[0] = BO == ByteOrder::kNative ? x : byteswap64(x); +template +static ASMJIT_INLINE_NODEBUG void storeu(void* p, T x) noexcept { + if constexpr (BO != ByteOrder::kNative) { + x = byteswap(x); + } + storeu(p, x); } -template -static ASMJIT_INLINE_NODEBUG void writeU64u(void* p, uint64_t x) noexcept { writeU64x(p, x); } -template -static ASMJIT_INLINE_NODEBUG void writeU64uLE(void* p, uint64_t x) noexcept { writeU64x(p, x); } -template -static ASMJIT_INLINE_NODEBUG void writeU64uBE(void* p, uint64_t x) noexcept { writeU64x(p, x); } +static ASMJIT_INLINE_NODEBUG void store_i8(void* p, int8_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void store_u8(void* p, uint8_t x) noexcept { storea(p, x); } -static ASMJIT_INLINE_NODEBUG void writeU64a(void* p, uint64_t x) noexcept { writeU64x(p, x); } -static ASMJIT_INLINE_NODEBUG void writeU64aLE(void* p, uint64_t x) noexcept { writeU64x(p, x); } -static ASMJIT_INLINE_NODEBUG void writeU64aBE(void* p, uint64_t x) noexcept { writeU64x(p, x); } +static ASMJIT_INLINE_NODEBUG void storea_i16(void* p, int16_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_i16(void* p, int16_t x) noexcept { storeu(p, x); } +static ASMJIT_INLINE_NODEBUG void storea_u16(void* p, uint16_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_u16(void* p, uint16_t x) noexcept { storeu(p, x); } -template -static ASMJIT_INLINE_NODEBUG void writeI64x(void* p, int64_t x) noexcept { writeU64x(p, uint64_t(x)); } +static ASMJIT_INLINE_NODEBUG void storea_i16_le(void* p, int16_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_i16_le(void* p, int16_t x) noexcept { storeu(p, x); } +static ASMJIT_INLINE_NODEBUG void storea_u16_le(void* p, uint16_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_u16_le(void* p, uint16_t x) noexcept { storeu(p, x); } -template -static ASMJIT_INLINE_NODEBUG void writeI64u(void* p, int64_t x) noexcept { writeU64x(p, uint64_t(x)); } -template -static ASMJIT_INLINE_NODEBUG void writeI64uLE(void* p, int64_t x) noexcept { writeU64x(p, uint64_t(x)); } -template -static ASMJIT_INLINE_NODEBUG void writeI64uBE(void* p, int64_t x) noexcept { writeU64x(p, uint64_t(x)); } +static ASMJIT_INLINE_NODEBUG void storea_i16_be(void* p, int16_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_i16_be(void* p, int16_t x) noexcept { storeu(p, x); } +static ASMJIT_INLINE_NODEBUG void storea_u16_be(void* p, uint16_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_u16_be(void* p, uint16_t x) noexcept { storeu(p, x); } -static ASMJIT_INLINE_NODEBUG void writeI64a(void* p, int64_t x) noexcept { writeU64x(p, uint64_t(x)); } -static ASMJIT_INLINE_NODEBUG void writeI64aLE(void* p, int64_t x) noexcept { writeU64x(p, uint64_t(x)); } -static ASMJIT_INLINE_NODEBUG void writeI64aBE(void* p, int64_t x) noexcept { writeU64x(p, uint64_t(x)); } +static ASMJIT_INLINE_NODEBUG void storea_i32(void* p, int32_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_i32(void* p, int32_t x) noexcept { storeu(p, x); } +static ASMJIT_INLINE_NODEBUG void storea_u32(void* p, uint32_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_u32(void* p, uint32_t x) noexcept { storeu(p, x); } + +static ASMJIT_INLINE_NODEBUG void storea_i32_le(void* p, int32_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_i32_le(void* p, int32_t x) noexcept { storeu(p, x); } +static ASMJIT_INLINE_NODEBUG void storea_u32_le(void* p, uint32_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_u32_le(void* p, uint32_t x) noexcept { storeu(p, x); } + +static ASMJIT_INLINE_NODEBUG void storea_i32_be(void* p, int32_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_i32_be(void* p, int32_t x) noexcept { storeu(p, x); } +static ASMJIT_INLINE_NODEBUG void storea_u32_be(void* p, uint32_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_u32_be(void* p, uint32_t x) noexcept { storeu(p, x); } + +static ASMJIT_INLINE_NODEBUG void storea_i64(void* p, int64_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_i64(void* p, int64_t x) noexcept { storeu(p, x); } +static ASMJIT_INLINE_NODEBUG void storea_u64(void* p, uint64_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_u64(void* p, uint64_t x) noexcept { storeu(p, x); } + +static ASMJIT_INLINE_NODEBUG void storea_i64_le(void* p, int64_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_i64_le(void* p, int64_t x) noexcept { storeu(p, x); } +static ASMJIT_INLINE_NODEBUG void storea_u64_le(void* p, uint64_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_u64_le(void* p, uint64_t x) noexcept { storeu(p, x); } + +static ASMJIT_INLINE_NODEBUG void storea_i64_be(void* p, int64_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_i64_be(void* p, int64_t x) noexcept { storeu(p, x); } +static ASMJIT_INLINE_NODEBUG void storea_u64_be(void* p, uint64_t x) noexcept { storea(p, x); } +static ASMJIT_INLINE_NODEBUG void storeu_u64_be(void* p, uint64_t x) noexcept { storeu(p, x); } // Support - Operators // =================== diff --git a/src/asmjit/core/support_p.h b/src/asmjit/core/support_p.h index 1caec93..f9f7bb7 100644 --- a/src/asmjit/core/support_p.h +++ b/src/asmjit/core/support_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_SUPPORT_P_H_INCLUDED diff --git a/src/asmjit/core/target.cpp b/src/asmjit/core/target.cpp index cbc6ab5..d2bce87 100644 --- a/src/asmjit/core/target.cpp +++ b/src/asmjit/core/target.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/target.h b/src/asmjit/core/target.h index 85df976..7e187dd 100644 --- a/src/asmjit/core/target.h +++ b/src/asmjit/core/target.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_TARGET_H_INCLUDED diff --git a/src/asmjit/core/type.cpp b/src/asmjit/core/type.cpp index 837cecf..ea55002 100644 --- a/src/asmjit/core/type.cpp +++ b/src/asmjit/core/type.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/type.h b/src/asmjit/core/type.h index 7b84d48..d1f6133 100644 --- a/src/asmjit/core/type.h +++ b/src/asmjit/core/type.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_TYPE_H_INCLUDED @@ -466,6 +466,13 @@ struct Float32 {}; //! double as C++ type-name. struct Float64 {}; +//! 128-bit vector register as C++ type-name. +struct Vec128 {}; +//! 256-bit vector register as C++ type-name. +struct Vec256 {}; +//! 512-bit vector register as C++ type-name. +struct Vec512 {}; + } // {Type} //! \cond @@ -491,6 +498,11 @@ ASMJIT_DEFINE_TYPE_ID(Type::IntPtr , TypeId::kIntPtr); ASMJIT_DEFINE_TYPE_ID(Type::UIntPtr, TypeId::kUIntPtr); ASMJIT_DEFINE_TYPE_ID(Type::Float32, TypeId::kFloat32); ASMJIT_DEFINE_TYPE_ID(Type::Float64, TypeId::kFloat64); +ASMJIT_DEFINE_TYPE_ID(Type::Vec128 , TypeId::kInt32x4); +ASMJIT_DEFINE_TYPE_ID(Type::Vec256 , TypeId::kInt32x8); +ASMJIT_DEFINE_TYPE_ID(Type::Vec512 , TypeId::kInt32x16); + +#undef ASMJIT_DEFINE_TYPE_ID //! \endcond //! \} diff --git a/src/asmjit/core/virtmem.cpp b/src/asmjit/core/virtmem.cpp index c1b9523..77d0661 100644 --- a/src/asmjit/core/virtmem.cpp +++ b/src/asmjit/core/virtmem.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -1153,7 +1153,7 @@ Error releaseDualMapping(DualMapping* dm, size_t size) noexcept { void flushInstructionCache(void* p, size_t size) noexcept { #if ASMJIT_ARCH_X86 || defined(__EMSCRIPTEN__) - // X86/X86_64 architecture doesn't require to do anything to flush instruction cache. + // X86|X86_64 architecture doesn't require to do anything to flush instruction cache. DebugUtils::unused(p, size); #elif defined(__APPLE__) sys_icache_invalidate(p, size); diff --git a/src/asmjit/core/virtmem.h b/src/asmjit/core/virtmem.h index 3520b19..a6fe695 100644 --- a/src/asmjit/core/virtmem.h +++ b/src/asmjit/core/virtmem.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_VIRTMEM_H_INCLUDED diff --git a/src/asmjit/core/zone.cpp b/src/asmjit/core/zone.cpp index ef48335..94e96c5 100644 --- a/src/asmjit/core/zone.cpp +++ b/src/asmjit/core/zone.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -24,8 +24,7 @@ static inline void Zone_assignZeroBlock(Zone* zone) noexcept { } static inline void Zone_assignBlock(Zone* zone, Zone::Block* block) noexcept { - size_t alignment = zone->blockAlignment(); - zone->_ptr = Support::alignUp(block->data(), alignment); + zone->_ptr = Support::alignUp(block->data(), Globals::kZoneAlignment); zone->_end = block->data() + block->size; zone->_block = block; } @@ -33,22 +32,19 @@ static inline void Zone_assignBlock(Zone* zone, Zone::Block* block) noexcept { // Zone - Initialization & Reset // ============================= -void Zone::_init(size_t blockSize, size_t blockAlignment, const Support::Temporary* temporary) noexcept { +void Zone::_init(size_t blockSize, const Support::Temporary* temporary) noexcept { ASMJIT_ASSERT(blockSize >= kMinBlockSize); ASMJIT_ASSERT(blockSize <= kMaxBlockSize); - ASMJIT_ASSERT(blockAlignment <= 64); Zone_assignZeroBlock(this); size_t blockSizeShift = Support::bitSizeOf() - Support::clz(blockSize); - size_t blockAlignmentShift = Support::bitSizeOf() - Support::clz(blockAlignment | (size_t(1) << 3)); - _blockAlignmentShift = uint8_t(blockAlignmentShift); + _currentBlockSizeShift = uint8_t(blockSizeShift); _minimumBlockSizeShift = uint8_t(blockSizeShift); _maximumBlockSizeShift = uint8_t(25); // (1 << 25) Equals 32 MiB blocks (should be enough for all cases) _hasStaticBlock = uint8_t(temporary != nullptr); _reserved = uint8_t(0u); - _blockCount = size_t(temporary != nullptr); // Setup the first [temporary] block, if necessary. if (temporary) { @@ -60,7 +56,6 @@ void Zone::_init(size_t blockSize, size_t blockAlignment, const Support::Tempora block->size = temporary->size() - kBlockSize; Zone_assignBlock(this, block); - _blockCount = 1u; } } @@ -79,7 +74,7 @@ void Zone::reset(ResetPolicy resetPolicy) noexcept { _ptr = initial->data(); _end = initial->data(); _block = initial; - _blockCount = size_t(hasStatic); + _currentBlockSizeShift = _minimumBlockSizeShift; // Since cur can be in the middle of the double-linked list, we have to traverse both directions (`prev` and // `next`) separately to visit all. @@ -117,18 +112,27 @@ void Zone::reset(ResetPolicy resetPolicy) noexcept { // Zone - Alloc // ============ -void* Zone::_alloc(size_t size, size_t alignment) noexcept { +void* Zone::_alloc(size_t size) noexcept { + ASMJIT_ASSERT(Support::isAligned(size, Globals::kZoneAlignment)); + + // Overhead of block alignment (we want to achieve at least Globals::kZoneAlignment). + constexpr size_t kAlignmentOverhead = + (Globals::kZoneAlignment <= Globals::kAllocAlignment) + ? size_t(0) + : Globals::kZoneAlignment - Globals::kAllocAlignment; + + // Total overhead per a block allocated with malloc - we want to decrease the size of each block by this value to + // make sure that malloc is not mmapping() additional page just to hold metadata. + constexpr size_t kBlockSizeOverhead = kBlockSize + Globals::kAllocOverhead + kAlignmentOverhead; + Block* curBlock = _block; Block* next = curBlock->next; - size_t defaultBlockAlignment = blockAlignment(); - size_t requiredBlockAlignment = Support::max(alignment, defaultBlockAlignment); - - // If the `Zone` has been cleared the current block doesn't have to be the last one. Check if there is a block + // If the `Zone` has been soft-reset the current block doesn't have to be the last one. Check if there is a block // that can be used instead of allocating a new one. If there is a `next` block it's completely unused, we don't // have to check for remaining bytes in that case. if (next) { - uint8_t* ptr = Support::alignUp(next->data(), requiredBlockAlignment); + uint8_t* ptr = Support::alignUp(next->data(), Globals::kZoneAlignment); uint8_t* end = next->data() + next->size; if (size <= (size_t)(end - ptr)) { @@ -139,44 +143,40 @@ void* Zone::_alloc(size_t size, size_t alignment) noexcept { } } - // Calculates the "default" size of a next block - in most cases this would be enough for the allocation. In + // Calculates the initial size of a next block - in most cases this would be enough for the allocation. In // general we want to gradually increase block size when more and more blocks are allocated until the maximum // block size. Since we use shifts (aka log2(size) sizes) we just need block count and minumum/maximum block // size shift to calculate the final size. - size_t defaultBlockSizeShift = Support::min(_blockCount + _minimumBlockSizeShift, _maximumBlockSizeShift); - size_t defaultBlockSize = size_t(1) << defaultBlockSizeShift; + uint32_t blockSizeShift = uint32_t(_currentBlockSizeShift); + size_t blockSize = size_t(1) << blockSizeShift; - // Allocate a new block. We have to accommodate all possible overheads so after the memory is allocated and then - // properly aligned there will be size for the requested memory. In 99.9999% cases this is never a problem, but - // we must be sure that even rare border cases would allocate properly. - size_t alignmentOverhead = requiredBlockAlignment - Support::min(requiredBlockAlignment, Globals::kAllocAlignment); - size_t blockSizeOverhead = kBlockSize + Globals::kAllocOverhead + alignmentOverhead; + // Allocate a new block. We have to accommodate all possible overheads so after the memory is allocated and + // then properly aligned there will be size for the requested memory. In 99.9999% cases this is never a problem, + // but we must be sure that even rare border cases would allocate properly. - // If the requested size is larger than a default calculated block size -> increase block size so the allocation - // would be enough to fit the requested size. - size_t finalBlockSize = defaultBlockSize; - - if (ASMJIT_UNLIKELY(size > defaultBlockSize - blockSizeOverhead)) { - if (ASMJIT_UNLIKELY(size > SIZE_MAX - blockSizeOverhead)) { + if (ASMJIT_UNLIKELY(size > blockSize - kBlockSizeOverhead)) { + // If the requested size is larger than a default calculated block size -> increase block size so the + // allocation would be enough to fit the requested size. + if (ASMJIT_UNLIKELY(size > SIZE_MAX - kBlockSizeOverhead)) { // This would probably never happen in practice - however, it needs to be done to stop malicious cases like // `alloc(SIZE_MAX)`. return nullptr; } - finalBlockSize = size + alignmentOverhead + kBlockSize; + blockSize = size + kAlignmentOverhead + kBlockSize; } else { - finalBlockSize -= Globals::kAllocOverhead; + blockSize -= Globals::kAllocOverhead; } // Allocate new block. - Block* newBlock = static_cast(::malloc(finalBlockSize)); + Block* newBlock = static_cast(::malloc(blockSize)); if (ASMJIT_UNLIKELY(!newBlock)) { return nullptr; } - // finalBlockSize includes the struct size, which must be avoided when assigning the size to a newly allocated block. - size_t realBlockSize = finalBlockSize - kBlockSize; + // blockSize includes the struct size, which must be avoided when assigning the size to a newly allocated block. + size_t realBlockSize = blockSize - kBlockSize; // Align the pointer to `minimumAlignment` and adjust the size of this block accordingly. It's the same as using // `minimumAlignment - Support::alignUpDiff()`, just written differently. @@ -196,23 +196,26 @@ void* Zone::_alloc(size_t size, size_t alignment) noexcept { } } - uint8_t* ptr = Support::alignUp(newBlock->data(), requiredBlockAlignment); + uint8_t* ptr = Support::alignUp(newBlock->data(), Globals::kZoneAlignment); uint8_t* end = newBlock->data() + realBlockSize; _ptr = ptr + size; _end = end; _block = newBlock; - _blockCount++; + _currentBlockSizeShift = uint8_t(Support::min(uint32_t(blockSizeShift) + 1u, _maximumBlockSizeShift)); ASMJIT_ASSERT(_ptr <= _end); return static_cast(ptr); } -void* Zone::allocZeroed(size_t size, size_t alignment) noexcept { - void* p = alloc(size, alignment); +void* Zone::allocZeroed(size_t size) noexcept { + ASMJIT_ASSERT(Support::isAligned(size, Globals::kZoneAlignment)); + + void* p = alloc(size); if (ASMJIT_UNLIKELY(!p)) { return p; } + return memset(p, 0, size); } @@ -223,16 +226,18 @@ void* Zone::dup(const void* data, size_t size, bool nullTerminate) noexcept { ASMJIT_ASSERT(size != SIZE_MAX); - uint8_t* m = allocT(size + nullTerminate); + size_t allocSize = Support::alignUp(size + size_t(nullTerminate), Globals::kZoneAlignment); + uint8_t* m = alloc(allocSize); + if (ASMJIT_UNLIKELY(!m)) { return nullptr; } - memcpy(m, data, size); - if (nullTerminate) { - m[size] = '\0'; - } + // Clear the last 8 bytes, which clears potential padding and null terminates at the same time. + static_assert(Globals::kZoneAlignment == 8u, "the code below must be fixed if zone alignment was changed"); + Support::storeu(m + allocSize - sizeof(uint64_t), 0u); + memcpy(m, data, size); return static_cast(m); } @@ -286,14 +291,14 @@ void ZoneAllocator::reset(Zone* zone) noexcept { _dynamicBlocks = nullptr; } -// asmjit::ZoneAllocator - Alloc & Release -// ======================================= +// ZoneAllocator - Alloc & Release +// =============================== void* ZoneAllocator::_alloc(size_t size, size_t& allocatedSize) noexcept { ASMJIT_ASSERT(isInitialized()); // Use the memory pool only if the requested block has a reasonable size. - uint32_t slot; + size_t slot; if (_getSlotIndex(size, slot, allocatedSize)) { // Slot reuse. uint8_t* p = reinterpret_cast(_slots[slot]); @@ -329,7 +334,7 @@ void* ZoneAllocator::_alloc(size_t size, size_t& allocatedSize) noexcept { _zone->setPtr(p); } - p = static_cast(_zone->_alloc(size, kBlockAlignment)); + p = static_cast(_zone->_alloc(size)); if (ASMJIT_UNLIKELY(!p)) { allocatedSize = 0; return nullptr; @@ -411,4 +416,32 @@ void ZoneAllocator::_releaseDynamic(void* p, size_t size) noexcept { ::free(block); } +// Zone - Tests +// ============ + +#if defined(ASMJIT_TEST) +UNIT(zone_allocator_slots) { + constexpr size_t kLoMaxSize = ZoneAllocator::kLoCount * ZoneAllocator::kLoGranularity; + constexpr size_t kHiMaxSize = ZoneAllocator::kHiCount * ZoneAllocator::kHiGranularity + kLoMaxSize; + + for (size_t size = 1; size <= kLoMaxSize; size++) { + size_t acquired_slot; + size_t expected_slot = (size - 1) / ZoneAllocator::kLoGranularity; + + EXPECT_TRUE(ZoneAllocator::_getSlotIndex(size, acquired_slot)); + EXPECT_EQ(acquired_slot, expected_slot); + EXPECT_LT(acquired_slot, ZoneAllocator::kLoCount); + } + + for (size_t size = kLoMaxSize + 1; size <= kHiMaxSize; size++) { + size_t acquired_slot; + size_t expected_slot = (size - kLoMaxSize - 1) / ZoneAllocator::kHiGranularity + ZoneAllocator::kLoCount; + + EXPECT_TRUE(ZoneAllocator::_getSlotIndex(size, acquired_slot)); + EXPECT_EQ(acquired_slot, expected_slot); + EXPECT_LT(acquired_slot, ZoneAllocator::kLoCount + ZoneAllocator::kHiCount); + } +} +#endif // ASMJIT_TEST + ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/zone.h b/src/asmjit/core/zone.h index 03b651a..05eb8fe 100644 --- a/src/asmjit/core/zone.h +++ b/src/asmjit/core/zone.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ZONE_H_INCLUDED @@ -41,17 +41,29 @@ public: size_t size; }; + //! A state that can be saved and restored. + struct State { + //! Pointer in the current block. + uint8_t* ptr; + //! End of the current block. + uint8_t* end; + //! Current block. + Block* block; + }; + static inline constexpr size_t kMinBlockSize = 256; // The number is ridiculously small, but still possible. static inline constexpr size_t kMaxBlockSize = size_t(1) << (sizeof(size_t) * 8 - 1); - static inline constexpr size_t kMinAlignment = 1; - static inline constexpr size_t kMaxAlignment = 64; - static inline constexpr size_t kBlockSize = sizeof(Block); static inline constexpr size_t kBlockOverhead = kBlockSize + Globals::kAllocOverhead; static ASMJIT_API const Block _zeroBlock; + template + static ASMJIT_INLINE_CONSTEXPR size_t alignedSizeOf() noexcept { + return Support::alignUp(sizeof(T), Globals::kZoneAlignment); + } + //! \endcond //! \name Members @@ -64,8 +76,8 @@ public: //! Current block. Block* _block; - //! Block alignment shift - uint8_t _blockAlignmentShift; + //! Current block size shift - reverted to _minimumBlockSizeShift every time the Zone is `reset(ResetPolicy::kHard)`. + uint8_t _currentBlockSizeShift; //! Minimum log2(blockSize) to allocate. uint8_t _minimumBlockSizeShift; //! Maximum log2(blockSize) to allocate. @@ -74,8 +86,6 @@ public: uint8_t _hasStaticBlock; //! Reserved for future use, must be zero. uint32_t _reserved; - //! Count of allocated blocks. - size_t _blockCount; //! \} @@ -90,18 +100,18 @@ public: //! //! It's not required, but it's good practice to set `blockSize` to a reasonable value that depends on the usage //! of `Zone`. Greater block sizes are generally safer and perform better than unreasonably low block sizes. - ASMJIT_INLINE_NODEBUG explicit Zone(size_t minimumBlockSize, size_t blockAlignment = 1) noexcept { - _init(minimumBlockSize, blockAlignment, nullptr); + ASMJIT_INLINE_NODEBUG explicit Zone(size_t minimumBlockSize) noexcept { + _init(minimumBlockSize, nullptr); } //! Creates a new Zone with a first block pointing to a `temporary` memory. - ASMJIT_INLINE_NODEBUG Zone(size_t minimumBlockSize, size_t blockAlignment, const Support::Temporary& temporary) noexcept { - _init(minimumBlockSize, blockAlignment, &temporary); + ASMJIT_INLINE_NODEBUG Zone(size_t minimumBlockSize, const Support::Temporary& temporary) noexcept { + _init(minimumBlockSize, &temporary); } //! \overload - ASMJIT_INLINE_NODEBUG Zone(size_t minimumBlockSize, size_t blockAlignment, const Support::Temporary* temporary) noexcept { - _init(minimumBlockSize, blockAlignment, temporary); + ASMJIT_INLINE_NODEBUG Zone(size_t minimumBlockSize, const Support::Temporary* temporary) noexcept { + _init(minimumBlockSize, temporary); } //! Moves an existing `Zone`. @@ -112,17 +122,15 @@ public: : _ptr(other._ptr), _end(other._end), _block(other._block), - _blockAlignmentShift(other._blockAlignmentShift), + _currentBlockSizeShift(other._currentBlockSizeShift), _minimumBlockSizeShift(other._minimumBlockSizeShift), _maximumBlockSizeShift(other._maximumBlockSizeShift), _hasStaticBlock(other._hasStaticBlock), - _reserved(other._reserved), - _blockCount(other._blockCount) { + _reserved(other._reserved) { ASMJIT_ASSERT(!other.hasStaticBlock()); other._block = const_cast(&_zeroBlock); other._ptr = other._block->data(); other._end = other._block->data(); - other._blockCount = 0u; } //! Destroys the `Zone` instance. @@ -131,7 +139,7 @@ public: //! `reset(ResetPolicy::kHard)`. ASMJIT_INLINE_NODEBUG ~Zone() noexcept { reset(ResetPolicy::kHard); } - ASMJIT_API void _init(size_t blockSize, size_t blockAlignment, const Support::Temporary* temporary) noexcept; + ASMJIT_API void _init(size_t blockSize, const Support::Temporary* temporary) noexcept; //! Resets the `Zone` invalidating all blocks allocated. //! @@ -143,10 +151,6 @@ public: //! \name Accessors //! \{ - //! Returns the default block alignment. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG size_t blockAlignment() const noexcept { return size_t(1) << _blockAlignmentShift; } - //! Returns a minimum block size. [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t minimumBlockSize() const noexcept { return size_t(1) << _minimumBlockSizeShift; } @@ -177,7 +181,7 @@ public: //! Sets the current zone pointer to `ptr` (must be within the current block). template - inline void setPtr(T* ptr) noexcept { + ASMJIT_INLINE void setPtr(T* ptr) noexcept { uint8_t* p = reinterpret_cast(ptr); ASMJIT_ASSERT(p >= _ptr && p <= _end); _ptr = p; @@ -185,7 +189,7 @@ public: //! Sets the end zone pointer to `end` (must be within the current block). template - inline void setEnd(T* end) noexcept { + ASMJIT_INLINE void setEnd(T* end) noexcept { uint8_t* p = reinterpret_cast(end); ASMJIT_ASSERT(p >= _ptr && p <= _end); _end = p; @@ -196,7 +200,7 @@ public: //! \name Utilities //! \{ - inline void swap(Zone& other) noexcept { + ASMJIT_INLINE void swap(Zone& other) noexcept { // This could lead to a disaster. ASMJIT_ASSERT(!this->hasStaticBlock()); ASMJIT_ASSERT(!other.hasStaticBlock()); @@ -204,13 +208,11 @@ public: std::swap(_ptr, other._ptr); std::swap(_end, other._end); std::swap(_block, other._block); - - std::swap(_blockAlignmentShift, other._blockAlignmentShift); + std::swap(_currentBlockSizeShift, other._currentBlockSizeShift); std::swap(_minimumBlockSizeShift, other._minimumBlockSizeShift); std::swap(_maximumBlockSizeShift, other._maximumBlockSizeShift); std::swap(_hasStaticBlock, other._hasStaticBlock); std::swap(_reserved, other._reserved); - std::swap(_blockCount, other._blockCount); } //! Aligns the current pointer to `alignment`. @@ -218,17 +220,23 @@ public: _ptr = Support::min(Support::alignUp(_ptr, alignment), _end); } - //! Ensures the remaining size is at least equal or greater than `size`. - //! - //! \note This function doesn't respect any alignment. If you need to ensure there is enough room for an aligned - //! allocation you need to call `align()` before calling `ensure()`. - [[nodiscard]] - ASMJIT_INLINE_NODEBUG Error ensure(size_t size) noexcept { - if (ASMJIT_LIKELY(size <= remainingSize())) { - return kErrorOk; + //! \} + + //! \name State Save & Restore + //! \{ + + ASMJIT_INLINE State _saveState() const noexcept { + return State{_ptr, _end, _block}; + } + + ASMJIT_INLINE void _restoreState(const State& state) noexcept { + if (ASMJIT_LIKELY(state.block != nullptr)) { + _ptr = state.ptr; + _end = state.end; + _block = state.block; } else { - return _alloc(0, 1) ? kErrorOk : DebugUtils::errored(kErrorOutOfMemory); + reset(ResetPolicy::kSoft); } } @@ -237,7 +245,15 @@ public: //! \name Allocation //! \{ - //! Allocates the requested memory specified by `size`. + //! \cond INTERNAL + + //! Internal alloc function used by inline wrappers. + [[nodiscard]] + ASMJIT_API void* _alloc(size_t size) noexcept; + + //! \endcond + + //! Allocates the requested memory specified by `size` and optionally casts the returned value to `T*`. //! //! Pointer returned is valid until the `Zone` instance is destroyed or reset by calling `reset()`. If you plan to //! make an instance of C++ from the given pointer use placement `new` and `delete` operators: @@ -268,131 +284,76 @@ public: //! // Reset or destroy `Zone`. //! zone.reset(); //! ``` + template [[nodiscard]] - inline void* alloc(size_t size) noexcept { - if (ASMJIT_UNLIKELY(size > remainingSize())) - return _alloc(size, 1); + ASMJIT_INLINE T* alloc(size_t size) noexcept { + ASMJIT_ASSERT(Support::isAligned(size, Globals::kZoneAlignment)); + +#if defined(__GNUC__) + // We can optimize this function a little bit if we know that `size` is relatively small - which would mean + // that we cannot possibly overflow `_ptr`. Since most of the time `alloc()` is used for known types (which + // implies their size is known as well) this optimization is worth it as it may save us 1 or 2 instructions. + if (__builtin_constant_p(size) && size <= 1024u) { + uint8_t* after = _ptr + size; + + if (ASMJIT_UNLIKELY(after > _end)) { + return static_cast(_alloc(size)); + } + + uint8_t* ptr = _ptr; + _ptr = after; + return static_cast(static_cast(ptr)); + } +#endif + + if (ASMJIT_UNLIKELY(size > remainingSize())) { + return static_cast(_alloc(size)); + } uint8_t* ptr = _ptr; _ptr += size; - return static_cast(ptr); + return static_cast(static_cast(ptr)); } - //! Allocates the requested memory specified by `size` and `alignment`. + template [[nodiscard]] - inline void* alloc(size_t size, size_t alignment) noexcept { - ASMJIT_ASSERT(Support::isPowerOf2(alignment)); - uint8_t* ptr = Support::alignUp(_ptr, alignment); - - if (ptr >= _end || size > (size_t)(_end - ptr)) - return _alloc(size, alignment); - - _ptr = ptr + size; - return static_cast(ptr); - } - - //! Allocates the requested memory specified by `size` without doing any checks. - //! - //! Can only be called if `remainingSize()` returns size at least equal to `size`. - [[nodiscard]] - inline void* allocNoCheck(size_t size) noexcept { - ASMJIT_ASSERT(remainingSize() >= size); - - uint8_t* ptr = _ptr; - _ptr += size; - return static_cast(ptr); - } - - //! Allocates the requested memory specified by `size` and `alignment` without doing any checks. - //! - //! Performs the same operation as `Zone::allocNoCheck(size)` with `alignment` applied. - [[nodiscard]] - inline void* allocNoCheck(size_t size, size_t alignment) noexcept { - ASMJIT_ASSERT(Support::isPowerOf2(alignment)); - - uint8_t* ptr = Support::alignUp(_ptr, alignment); - ASMJIT_ASSERT(size <= (size_t)(_end - ptr)); - - _ptr = ptr + size; - return static_cast(ptr); + ASMJIT_INLINE T* alloc() noexcept { + return alloc(alignedSizeOf()); } //! Allocates `size` bytes of zeroed memory. See `alloc()` for more details. [[nodiscard]] - ASMJIT_API void* allocZeroed(size_t size, size_t alignment = 1) noexcept; - - //! Like `alloc()`, but the return pointer is casted to `T*`. - template - [[nodiscard]] - inline T* allocT(size_t size = sizeof(T), size_t alignment = alignof(T)) noexcept { - return static_cast(alloc(size, alignment)); - } - - //! Like `allocNoCheck()`, but the return pointer is casted to `T*`. - template - [[nodiscard]] - inline T* allocNoCheckT(size_t size = sizeof(T), size_t alignment = alignof(T)) noexcept { - return static_cast(allocNoCheck(size, alignment)); - } - - //! Like `allocZeroed()`, but the return pointer is casted to `T*`. - template - [[nodiscard]] - inline T* allocZeroedT(size_t size = sizeof(T), size_t alignment = alignof(T)) noexcept { - return static_cast(allocZeroed(size, alignment)); - } + ASMJIT_API void* allocZeroed(size_t size) noexcept; //! Like `new(std::nothrow) T(...)`, but allocated by `Zone`. template [[nodiscard]] - inline T* newT() noexcept { - void* p = alloc(sizeof(T), alignof(T)); - if (ASMJIT_UNLIKELY(!p)) + ASMJIT_INLINE T* newT() noexcept { + void* ptr = alloc(alignedSizeOf()); + if (ASMJIT_UNLIKELY(!ptr)) return nullptr; - return new(Support::PlacementNew{p}) T(); + return new(Support::PlacementNew{ptr}) T(); } //! Like `new(std::nothrow) T(...)`, but allocated by `Zone`. template [[nodiscard]] - inline T* newT(Args&&... args) noexcept { - void* p = alloc(sizeof(T), alignof(T)); - if (ASMJIT_UNLIKELY(!p)) + ASMJIT_INLINE T* newT(Args&&... args) noexcept { + void* ptr = alloc(alignedSizeOf()); + if (ASMJIT_UNLIKELY(!ptr)) return nullptr; - return new(Support::PlacementNew{p}) T(std::forward(args)...); + return new(Support::PlacementNew{ptr}) T(std::forward(args)...); } - //! \cond INTERNAL - //! - //! Internal alloc function used by other inlines. - [[nodiscard]] - ASMJIT_API void* _alloc(size_t size, size_t alignment) noexcept; - //! \endcond - //! Helper to duplicate data. [[nodiscard]] ASMJIT_API void* dup(const void* data, size_t size, bool nullTerminate = false) noexcept; - //! Helper to duplicate data. - [[nodiscard]] - inline void* dupAligned(const void* data, size_t size, size_t alignment, bool nullTerminate = false) noexcept { - align(alignment); - return dup(data, size, nullTerminate); - } - //! Helper to duplicate a formatted string, maximum size is 256 bytes. [[nodiscard]] ASMJIT_API char* sformat(const char* str, ...) noexcept; //! \} - -#if !defined(ASMJIT_NO_DEPRECATED) - [[deprecated("Use Zone::minimumBlockSize() instead of Zone::blockSize()")]] - ASMJIT_INLINE_NODEBUG size_t blockSize() const noexcept { return minimumBlockSize(); } - - [[deprecated("Use Zone::hasStaticBlock() instead of Zone::isTemporary()")]] - ASMJIT_INLINE_NODEBUG bool isTemporary() const noexcept { return hasStaticBlock() != 0u; } -#endif }; //! \ref Zone with `N` bytes of a static storage, used for the initial block. @@ -410,8 +371,8 @@ public: } _storage; //! Creates a temporary zone. Dynamic block size is specified by `blockSize`. - inline explicit ZoneTmp(size_t blockSize, size_t blockAlignment = 1) noexcept - : Zone(blockSize, blockAlignment, Support::Temporary(_storage.data, N)) {} + inline explicit ZoneTmp(size_t blockSize) noexcept + : Zone(blockSize, Support::Temporary(_storage.data, N)) {} }; //! Zone-based memory allocator that uses an existing `Zone` and provides a `release()` functionality on top of it. @@ -446,6 +407,9 @@ public: //! Maximum size of a block that can be allocated in a high granularity pool. static inline constexpr uint32_t kHiMaxSize = kLoMaxSize + kHiGranularity * kHiCount; + //! Number of slots. + static inline constexpr uint32_t kSlotCount = kLoCount + kHiCount; + //! Alignment of every pointer returned by `alloc()`. static inline constexpr uint32_t kBlockAlignment = kLoGranularity; @@ -462,6 +426,30 @@ public: DynamicBlock* next; }; + //! Returns the slot index to be used for `size`. Returns `true` if a valid slot has been written to `slot` and + //! `allocatedSize` has been filled with slot exact size (`allocatedSize` can be equal or slightly greater than + //! `size`). + [[nodiscard]] + static ASMJIT_INLINE bool _getSlotIndex(size_t size, size_t& slot) noexcept { + size_t slot_lo = (size - 1u) / kLoGranularity; + size_t slot_hi = (size - kLoCount * kLoGranularity + kLoCount * kHiGranularity - 1u) / kHiGranularity; + + slot = Support::min(slot_lo, slot_hi); + return slot < kSlotCount; + } + + //! \overload + [[nodiscard]] + static ASMJIT_INLINE bool _getSlotIndex(size_t size, size_t& slot, size_t& allocatedSize) noexcept { + size_t slot_lo = (size - 1u) / kLoGranularity; + size_t slot_hi = (size - kLoCount * kLoGranularity + kLoCount * kHiGranularity - 1u) / kHiGranularity; + + slot = Support::min(slot_lo, slot_hi); + allocatedSize = Support::alignUp(size, slot < kLoCount ? kLoGranularity : kHiGranularity); + + return slot < kSlotCount; + } + //! \endcond //! \name Members @@ -470,7 +458,7 @@ public: //! Zone used to allocate memory that fits into slots. Zone* _zone {}; //! Indexed slots containing released memory. - Slot* _slots[kLoCount + kHiCount] {}; + Slot* _slots[kSlotCount] {}; //! Dynamic blocks for larger allocations (no slots). DynamicBlock* _dynamicBlocks {}; @@ -515,53 +503,6 @@ public: //! \} - //! \cond - //! \name Internals - //! \{ - - //! Returns the slot index to be used for `size`. Returns `true` if a valid slot has been written to `slot` and - //! `allocatedSize` has been filled with slot exact size (`allocatedSize` can be equal or slightly greater than - //! `size`). - [[nodiscard]] - static inline bool _getSlotIndex(size_t size, uint32_t& slot) noexcept { - ASMJIT_ASSERT(size > 0); - if (size > kHiMaxSize) { - return false; - } - - if (size <= kLoMaxSize) { - slot = uint32_t((size - 1) / kLoGranularity); - } - else { - slot = uint32_t((size - kLoMaxSize - 1) / kHiGranularity) + kLoCount; - } - - return true; - } - - //! \overload - [[nodiscard]] - static inline bool _getSlotIndex(size_t size, uint32_t& slot, size_t& allocatedSize) noexcept { - ASMJIT_ASSERT(size > 0); - if (size > kHiMaxSize) { - return false; - } - - if (size <= kLoMaxSize) { - slot = uint32_t((size - 1) / kLoGranularity); - allocatedSize = Support::alignUp(size, kLoGranularity); - } - else { - slot = uint32_t((size - kLoMaxSize - 1) / kHiGranularity) + kLoCount; - allocatedSize = Support::alignUp(size, kHiGranularity); - } - - return true; - } - - //! \} - //! \endcond - //! \name Allocation //! \{ @@ -578,69 +519,38 @@ public: //! Allocates `size` bytes of memory, ideally from an available pool. //! //! \note `size` can't be zero, it will assert in debug mode in such case. + template [[nodiscard]] - inline void* alloc(size_t size) noexcept { + inline T* alloc(size_t size) noexcept { ASMJIT_ASSERT(isInitialized()); size_t allocatedSize; - return _alloc(size, allocatedSize); + return static_cast(_alloc(size, allocatedSize)); } //! Like `alloc(size)`, but provides a second argument `allocatedSize` that provides a way to know how big //! the block returned actually is. This is useful for containers to prevent growing too early. + template [[nodiscard]] - inline void* alloc(size_t size, size_t& allocatedSize) noexcept { + inline T* alloc(size_t size, size_t& allocatedSize) noexcept { ASMJIT_ASSERT(isInitialized()); - return _alloc(size, allocatedSize); - } - - //! Like `alloc()`, but the return pointer is casted to `T*`. - template - [[nodiscard]] - inline T* allocT(size_t size = sizeof(T)) noexcept { - return static_cast(alloc(size)); + return static_cast(_alloc(size, allocatedSize)); } //! Like `alloc(size)`, but returns zeroed memory. + template [[nodiscard]] inline void* allocZeroed(size_t size) noexcept { ASMJIT_ASSERT(isInitialized()); size_t allocatedSize; - return _allocZeroed(size, allocatedSize); + return static_cast(_allocZeroed(size, allocatedSize)); } //! Like `alloc(size, allocatedSize)`, but returns zeroed memory. + template [[nodiscard]] - inline void* allocZeroed(size_t size, size_t& allocatedSize) noexcept { + inline T* allocZeroed(size_t size, size_t& allocatedSize) noexcept { ASMJIT_ASSERT(isInitialized()); - return _allocZeroed(size, allocatedSize); - } - - //! Like `allocZeroed()`, but the return pointer is casted to `T*`. - template - [[nodiscard]] - inline T* allocZeroedT(size_t size = sizeof(T)) noexcept { - return static_cast(allocZeroed(size)); - } - - //! Like `new(std::nothrow) T(...)`, but allocated by `Zone`. - template - [[nodiscard]] - inline T* newT() noexcept { - void* p = allocT(); - if (ASMJIT_UNLIKELY(!p)) { - return nullptr; - } - return new(Support::PlacementNew{p}) T(); - } - //! Like `new(std::nothrow) T(...)`, but allocated by `Zone`. - template - [[nodiscard]] - inline T* newT(Args&&... args) noexcept { - void* p = allocT(); - if (ASMJIT_UNLIKELY(!p)) { - return nullptr; - } - return new(Support::PlacementNew{p}) T(std::forward(args)...); + return static_cast(_allocZeroed(size, allocatedSize)); } //! Releases the memory previously allocated by `alloc()`. The `size` argument has to be the same as used to call @@ -650,7 +560,7 @@ public: ASMJIT_ASSERT(p != nullptr); ASMJIT_ASSERT(size != 0); - uint32_t slot; + size_t slot; if (_getSlotIndex(size, slot)) { static_cast(p)->next = static_cast(_slots[slot]); _slots[slot] = static_cast(p); @@ -663,6 +573,43 @@ public: //! \} }; +//! Helper class for implementing pooling of arena-allocated objects. +template +class ZonePool { +public: + ASMJIT_NONCOPYABLE(ZonePool) + + struct Link { Link* next; }; + Link* _data {}; + + ASMJIT_INLINE_NODEBUG ZonePool() noexcept = default; + + //! Resets the arena pool. + //! + //! Reset must be called after the associated `ArenaAllocator` has been reset, otherwise the existing pool will + //! collide with possible allocations made on the `ArenaAllocator` object after the reset. + ASMJIT_INLINE_NODEBUG void reset() noexcept { _data = nullptr; } + + //! Allocates a memory (or reuses the existing allocation) of `SizeOfT` (in bytes). + [[nodiscard]] + ASMJIT_INLINE T* alloc(Zone& zone) noexcept { + Link* p = _data; + if (ASMJIT_UNLIKELY(p == nullptr)) { + return zone.alloc(Support::alignUp(SizeOfT, Globals::kZoneAlignment)); + } + _data = p->next; + return static_cast(static_cast(p)); + } + + //! Pools the previously allocated memory. + ASMJIT_INLINE void release(T* ptr) noexcept { + ASMJIT_ASSERT(ptr != nullptr); + Link* p = reinterpret_cast(ptr); + + p->next = _data; + _data = p; + } +}; //! \} ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/zonehash.cpp b/src/asmjit/core/zonehash.cpp index 40fc8ef..b916a70 100644 --- a/src/asmjit/core/zonehash.cpp +++ b/src/asmjit/core/zonehash.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/zonehash.h b/src/asmjit/core/zonehash.h index 0d85143..323934a 100644 --- a/src/asmjit/core/zonehash.h +++ b/src/asmjit/core/zonehash.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ZONEHASH_H_INCLUDED diff --git a/src/asmjit/core/zonelist.cpp b/src/asmjit/core/zonelist.cpp index ff2c2cf..a540b11 100644 --- a/src/asmjit/core/zonelist.cpp +++ b/src/asmjit/core/zonelist.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/zonelist.h b/src/asmjit/core/zonelist.h index 0ba3d97..5e60b8a 100644 --- a/src/asmjit/core/zonelist.h +++ b/src/asmjit/core/zonelist.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ZONELIST_H_INCLUDED @@ -38,10 +38,10 @@ public: //! \{ ASMJIT_INLINE_NODEBUG ZoneListNode() noexcept - : _listNodes { nullptr, nullptr } {} + : _listNodes{nullptr, nullptr} {} ASMJIT_INLINE_NODEBUG ZoneListNode(ZoneListNode&& other) noexcept - : _listNodes { other._listNodes[0], other._listNodes[1] } {} + : _listNodes{other._listNodes[0], other._listNodes[1]} {} //! \} @@ -164,8 +164,8 @@ public: NodeT* prev = node->prev(); NodeT* next = node->next(); - if (prev) { prev->_listNodes[1] = next; node->_listNodes[0] = nullptr; } else { _nodes[0] = next; } - if (next) { next->_listNodes[0] = prev; node->_listNodes[1] = nullptr; } else { _nodes[1] = prev; } + if (prev) { prev->_listNodes[1] = next; } else { _nodes[0] = next; } + if (next) { next->_listNodes[0] = prev; } else { _nodes[1] = prev; } node->_listNodes[0] = nullptr; node->_listNodes[1] = nullptr; diff --git a/src/asmjit/core/zonestack.cpp b/src/asmjit/core/zonestack.cpp index 1da3664..324300c 100644 --- a/src/asmjit/core/zonestack.cpp +++ b/src/asmjit/core/zonestack.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -29,7 +29,8 @@ Error ZoneStackBase::_init(ZoneAllocator* allocator, size_t middleIndex) noexcep } if (allocator) { - Block* block = static_cast(allocator->alloc(kBlockSize)); + size_t allocated; + Block* block = static_cast(allocator->alloc(kBlockSize, allocated)); if (ASMJIT_UNLIKELY(!block)) { return DebugUtils::errored(kErrorOutOfMemory); } @@ -56,11 +57,15 @@ Error ZoneStackBase::_prepareBlock(uint32_t side, size_t initialIndex) noexcept Block* prev = _block[side]; ASMJIT_ASSERT(!prev->empty()); - Block* block = _allocator->allocT(kBlockSize); + size_t allocated; + Block* block = static_cast(_allocator->alloc(kBlockSize, allocated)); + if (ASMJIT_UNLIKELY(!block)) { return DebugUtils::errored(kErrorOutOfMemory); } + ASMJIT_ASSERT(kBlockSize == allocated); + block->_link[ side] = nullptr; block->_link[!side] = prev; block->_start = (uint8_t*)block + initialIndex; diff --git a/src/asmjit/core/zonestack.h b/src/asmjit/core/zonestack.h index 39cf2a8..a246bf9 100644 --- a/src/asmjit/core/zonestack.h +++ b/src/asmjit/core/zonestack.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ZONESTACK_H_INCLUDED diff --git a/src/asmjit/core/zonestring.h b/src/asmjit/core/zonestring.h index 872fcc0..c91b4a7 100644 --- a/src/asmjit/core/zonestring.h +++ b/src/asmjit/core/zonestring.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ZONESTRING_H_INCLUDED diff --git a/src/asmjit/core/zonetree.cpp b/src/asmjit/core/zonetree.cpp index 3bcce88..41f22c8 100644 --- a/src/asmjit/core/zonetree.cpp +++ b/src/asmjit/core/zonetree.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/zonetree.h b/src/asmjit/core/zonetree.h index 2b047d1..a1818f7 100644 --- a/src/asmjit/core/zonetree.h +++ b/src/asmjit/core/zonetree.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ZONETREE_H_INCLUDED diff --git a/src/asmjit/core/zonevector.cpp b/src/asmjit/core/zonevector.cpp index 9087693..c9779c5 100644 --- a/src/asmjit/core/zonevector.cpp +++ b/src/asmjit/core/zonevector.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/src/asmjit/core/zonevector.h b/src/asmjit/core/zonevector.h index 5d7c1e9..318b451 100644 --- a/src/asmjit/core/zonevector.h +++ b/src/asmjit/core/zonevector.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ZONEVECTOR_H_INCLUDED @@ -269,9 +269,7 @@ public: //! Prepends `item` to the vector. ASMJIT_INLINE Error prepend(ZoneAllocator* allocator, const T& item) noexcept { - if (ASMJIT_UNLIKELY(_size == _capacity)) { - ASMJIT_PROPAGATE(grow(allocator, 1)); - } + ASMJIT_PROPAGATE(grow(allocator)); memmove(static_cast(static_cast(_data) + 1), static_cast(_data), @@ -288,10 +286,7 @@ public: //! Inserts an `item` at the specified `index`. ASMJIT_INLINE Error insert(ZoneAllocator* allocator, size_t index, const T& item) noexcept { ASMJIT_ASSERT(index <= _size); - - if (ASMJIT_UNLIKELY(_size == _capacity)) { - ASMJIT_PROPAGATE(grow(allocator, 1)); - } + ASMJIT_PROPAGATE(grow(allocator)); T* dst = static_cast(_data) + index; memmove(static_cast(dst + 1), @@ -308,9 +303,7 @@ public: //! Appends `item` to the vector. ASMJIT_INLINE Error append(ZoneAllocator* allocator, const T& item) noexcept { - if (ASMJIT_UNLIKELY(_size == _capacity)) { - ASMJIT_PROPAGATE(grow(allocator, 1)); - } + ASMJIT_PROPAGATE(grow(allocator)); memcpy(static_cast(static_cast(_data) + _size), static_cast(&item), @@ -490,6 +483,17 @@ public: _release(allocator, sizeof(T)); } + //! Called to grow the buffer to fit at least 1 element more. + [[nodiscard]] + inline Error grow(ZoneAllocator* allocator) noexcept { + if (ASMJIT_LIKELY(_size < _capacity)) { + return kErrorOk; + } + else { + return ZoneVectorBase::_grow(allocator, sizeof(T), 1u); + } + } + //! Called to grow the buffer to fit at least `n` elements more. [[nodiscard]] inline Error grow(ZoneAllocator* allocator, uint32_t n) noexcept { @@ -530,7 +534,11 @@ public: } } - inline Error willGrow(ZoneAllocator* allocator, uint32_t n = 1) noexcept { + inline Error willGrow(ZoneAllocator* allocator) noexcept { + return _capacity == _size ? grow(allocator, 1u) : Error(kErrorOk); + } + + inline Error willGrow(ZoneAllocator* allocator, uint32_t n) noexcept { return _capacity - _size < n ? grow(allocator, n) : Error(kErrorOk); } @@ -827,7 +835,7 @@ public: return; } - allocator->release(_data, _capacity / 8); + allocator->release(_data, _capacity / 8u); reset(); } diff --git a/src/asmjit/host.h b/src/asmjit/host.h new file mode 100644 index 0000000..87fd71a --- /dev/null +++ b/src/asmjit/host.h @@ -0,0 +1,33 @@ +// This file is part of AsmJit project +// +// See or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_HOST_H_INCLUDED +#define ASMJIT_HOST_H_INCLUDED + +#include "core.h" + +// Detect 'X86' or 'X86_64' host architectures. +#if ASMJIT_ARCH_X86 != 0 && !defined(ASMJIT_NO_X86) + +#include "x86.h" + +ASMJIT_BEGIN_NAMESPACE +namespace host { using namespace x86; } +ASMJIT_END_NAMESPACE + +#endif + +// Detect 'AArch64' host architecture. +#if ASMJIT_ARCH_ARM == 64 && !defined(ASMJIT_NO_AARCH64) + +#include "a64.h" + +ASMJIT_BEGIN_NAMESPACE +namespace host { using namespace a64; } +ASMJIT_END_NAMESPACE + +#endif + +#endif // ASMJIT_HOST_H_INCLUDED diff --git a/src/asmjit/x86.h b/src/asmjit/x86.h index 84bc84b..7fdff3a 100644 --- a/src/asmjit/x86.h +++ b/src/asmjit/x86.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_X86_H_INCLUDED @@ -28,30 +28,21 @@ //! with \ref x86::Compiler. //! //! - Instruction representation: -//! - \ref x86::Inst::Id - Provides instruction identifiers for both X86/X86_64 architectures. -//! - \ref InstOptions - Provides generic and X86/X86_64 specific options. +//! - \ref x86::Inst::Id - Provides instruction identifiers of X86|X86_64 architecture. +//! - \ref InstOptions - Provides generic and X86|X86_64 specific options. //! //! ### Register Operands //! -//! - \ref x86::Reg - Base class for any X86 register. -//! - \ref x86::Gp - General purpose register: -//! - \ref x86::GpbLo - 8-bit low register. -//! - \ref x86::GpbHi - 8-bit high register. -//! - \ref x86::Gpw - 16-bit register. -//! - \ref x86::Gpd - 32-bit register. -//! - \ref x86::Gpq - 64-bit register (X64 only). -//! - \ref x86::Vec - Vector (SIMD) register: -//! - \ref x86::Xmm - 128-bit SIMD register (SSE+). -//! - \ref x86::Ymm - 256-bit SIMD register (AVX+). -//! - \ref x86::Zmm - 512-bit SIMD register (AVX512+). -//! - \ref x86::Mm - 64-bit MMX register. -//! - \ref x86::St - 80-bit FPU register. -//! - \ref x86::KReg - opmask registers (AVX512+). -//! - \ref x86::SReg - segment register. -//! - \ref x86::CReg - control register. -//! - \ref x86::DReg - debug register. -//! - \ref x86::Bnd - bound register (discontinued). -//! - \ref x86::Rip - relative instruction pointer. +//! - \ref x86::Gp - General purpose register. +//! - \ref x86::Vec - Vector (SIMD) register. +//! - \ref x86::Mm - 64-bit MMX register. +//! - \ref x86::St - 80-bit FPU register. +//! - \ref x86::KReg - opmask registers (AVX512+). +//! - \ref x86::SReg - segment register. +//! - \ref x86::CReg - control register. +//! - \ref x86::DReg - debug register. +//! - \ref x86::Bnd - bound register (discontinued). +//! - \ref x86::Rip - relative instruction pointer. //! //! ### Memory Operands //! diff --git a/src/asmjit/x86/x86archtraits_p.h b/src/asmjit/x86/x86archtraits_p.h index f05c7a5..7544ad4 100644 --- a/src/asmjit/x86/x86archtraits_p.h +++ b/src/asmjit/x86/x86archtraits_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_X86_X86ARCHTRAITS_P_H_INCLUDED @@ -30,6 +30,24 @@ static const constexpr ArchTraits x86ArchTraits = { // Min/Max stack offset 0x7FFFFFFFu, 0x7FFFFFFFu, + // Supported register types. + 0u | (1u << uint32_t(RegType::kGp8Lo )) + | (1u << uint32_t(RegType::kGp8Hi )) + | (1u << uint32_t(RegType::kGp16 )) + | (1u << uint32_t(RegType::kGp32 )) + | (1u << uint32_t(RegType::kGp64 )) + | (1u << uint32_t(RegType::kVec128 )) + | (1u << uint32_t(RegType::kVec256 )) + | (1u << uint32_t(RegType::kVec512 )) + | (1u << uint32_t(RegType::kMask )) + | (1u << uint32_t(RegType::kSegment)) + | (1u << uint32_t(RegType::kControl)) + | (1u << uint32_t(RegType::kDebug )) + | (1u << uint32_t(RegType::kX86_Mm )) + | (1u << uint32_t(RegType::kX86_St )) + | (1u << uint32_t(RegType::kX86_Bnd)) + | (1u << uint32_t(RegType::kPC )), + // ISA features [Gp, Vec, Other0, Other1]. {{ InstHints::kRegSwap | InstHints::kPushPop, @@ -38,31 +56,21 @@ static const constexpr ArchTraits x86ArchTraits = { InstHints::kNoHints }}, - // Register signatures. - #define V(index) OperandSignature{x86::RegTraits::kSignature} - {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, - #undef V - - // RegTypeToTypeId. - #define V(index) TypeId(x86::RegTraits::kTypeId) - {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, - #undef V - // TypeIdToRegType. - #define V(index) (index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt8) ? RegType::kX86_GpbLo : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt8) ? RegType::kX86_GpbLo : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt16) ? RegType::kX86_Gpw : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt16) ? RegType::kX86_Gpw : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt32) ? RegType::kX86_Gpd : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt32) ? RegType::kX86_Gpd : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kIntPtr) ? RegType::kX86_Gpd : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUIntPtr) ? RegType::kX86_Gpd : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat32) ? RegType::kX86_Xmm : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat64) ? RegType::kX86_Xmm : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask8) ? RegType::kX86_KReg : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask16) ? RegType::kX86_KReg : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask32) ? RegType::kX86_KReg : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask64) ? RegType::kX86_KReg : \ + #define V(index) (index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt8) ? RegType::kGp8Lo : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt8) ? RegType::kGp8Lo : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt16) ? RegType::kGp16 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt16) ? RegType::kGp16 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt32) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt32) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kIntPtr) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUIntPtr) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat32) ? RegType::kVec128 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat64) ? RegType::kVec128 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask8) ? RegType::kMask : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask16) ? RegType::kMask : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask32) ? RegType::kMask : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask64) ? RegType::kMask : \ index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMmx32) ? RegType::kX86_Mm : \ index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMmx64) ? RegType::kX86_Mm : RegType::kNone) {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, @@ -91,7 +99,26 @@ static const constexpr ArchTraits x64ArchTraits = { // Min/Max stack offset 0x7FFFFFFFu, 0x7FFFFFFFu, - // ISA features [Gp, Vec, Other0, Other1]. + // Supported register types. + 0u | (1u << uint32_t(RegType::kGp8Lo )) + | (1u << uint32_t(RegType::kGp8Hi )) + | (1u << uint32_t(RegType::kGp16 )) + | (1u << uint32_t(RegType::kGp32 )) + | (1u << uint32_t(RegType::kGp64 )) + | (1u << uint32_t(RegType::kVec128 )) + | (1u << uint32_t(RegType::kVec256 )) + | (1u << uint32_t(RegType::kVec512 )) + | (1u << uint32_t(RegType::kMask )) + | (1u << uint32_t(RegType::kTile )) + | (1u << uint32_t(RegType::kSegment)) + | (1u << uint32_t(RegType::kControl)) + | (1u << uint32_t(RegType::kDebug )) + | (1u << uint32_t(RegType::kX86_Mm )) + | (1u << uint32_t(RegType::kX86_St )) + | (1u << uint32_t(RegType::kX86_Bnd)) + | (1u << uint32_t(RegType::kPC )), + + // ISA features [Gp, Vec, Mask, Extra]. {{ InstHints::kRegSwap | InstHints::kPushPop, InstHints::kNoHints, @@ -99,35 +126,25 @@ static const constexpr ArchTraits x64ArchTraits = { InstHints::kNoHints }}, - // Register signatures. - #define V(index) OperandSignature{x86::RegTraits::kSignature} - {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, - #undef V - - // RegTypeToTypeId. - #define V(index) TypeId(x86::RegTraits::kTypeId) - {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, - #undef V - // TypeIdToRegType. - #define V(index) (index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt8) ? RegType::kX86_GpbLo : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt8) ? RegType::kX86_GpbLo : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt16) ? RegType::kX86_Gpw : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt16) ? RegType::kX86_Gpw : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt32) ? RegType::kX86_Gpd : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt32) ? RegType::kX86_Gpd : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt64) ? RegType::kX86_Gpq : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt64) ? RegType::kX86_Gpq : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kIntPtr) ? RegType::kX86_Gpq : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUIntPtr) ? RegType::kX86_Gpq : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat32) ? RegType::kX86_Xmm : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat64) ? RegType::kX86_Xmm : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask8) ? RegType::kX86_KReg : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask16) ? RegType::kX86_KReg : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask32) ? RegType::kX86_KReg : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask64) ? RegType::kX86_KReg : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMmx32) ? RegType::kX86_Mm : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMmx64) ? RegType::kX86_Mm : RegType::kNone) + #define V(index) (index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt8) ? RegType::kGp8Lo : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt8) ? RegType::kGp8Lo : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt16) ? RegType::kGp16 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt16) ? RegType::kGp16 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt32) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt32) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt64) ? RegType::kGp64 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt64) ? RegType::kGp64 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kIntPtr) ? RegType::kGp64 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUIntPtr) ? RegType::kGp64 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat32) ? RegType::kVec128 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat64) ? RegType::kVec128 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask8) ? RegType::kMask : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask16) ? RegType::kMask : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask32) ? RegType::kMask : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMask64) ? RegType::kMask : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMmx32) ? RegType::kX86_Mm : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kMmx64) ? RegType::kX86_Mm : RegType::kNone) {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, #undef V diff --git a/src/asmjit/x86/x86assembler.cpp b/src/asmjit/x86/x86assembler.cpp index c7fd495..f9ee27c 100644 --- a/src/asmjit/x86/x86assembler.cpp +++ b/src/asmjit/x86/x86assembler.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -172,29 +172,29 @@ struct X86MemInfo_T { static inline constexpr uint32_t I = (X >> 5) & 0x1F; static inline constexpr uint32_t kBase = - (B >= uint32_t(RegType::kX86_Gpw) && B <= uint32_t(RegType::kX86_Gpq)) ? kX86MemInfo_BaseGp : - (B == uint32_t(RegType::kX86_Rip) ) ? kX86MemInfo_BaseRip : + (B >= uint32_t(RegType::kGp16) && B <= uint32_t(RegType::kGp64) ) ? kX86MemInfo_BaseGp : + (B == uint32_t(RegType::kPC) ) ? kX86MemInfo_BaseRip : (B == uint32_t(RegType::kLabelTag) ) ? kX86MemInfo_BaseLabel : 0; static inline constexpr uint32_t kIndex = - (I >= uint32_t(RegType::kX86_Gpw) && I <= uint32_t(RegType::kX86_Gpq)) ? kX86MemInfo_Index : - (I >= uint32_t(RegType::kX86_Xmm) && I <= uint32_t(RegType::kX86_Zmm)) ? kX86MemInfo_Index : 0; + (I >= uint32_t(RegType::kGp16) && I <= uint32_t(RegType::kGp64) ) ? kX86MemInfo_Index : + (I >= uint32_t(RegType::kVec128) && I <= uint32_t(RegType::kVec512) ) ? kX86MemInfo_Index : 0; static inline constexpr uint32_t k67H = - (B == uint32_t(RegType::kX86_Gpw) && I == uint32_t(RegType::kNone) ) ? kX86MemInfo_67H_X86 : - (B == uint32_t(RegType::kX86_Gpd) && I == uint32_t(RegType::kNone) ) ? kX86MemInfo_67H_X64 : - (B == uint32_t(RegType::kNone) && I == uint32_t(RegType::kX86_Gpw)) ? kX86MemInfo_67H_X86 : - (B == uint32_t(RegType::kNone) && I == uint32_t(RegType::kX86_Gpd)) ? kX86MemInfo_67H_X64 : - (B == uint32_t(RegType::kX86_Gpw) && I == uint32_t(RegType::kX86_Gpw)) ? kX86MemInfo_67H_X86 : - (B == uint32_t(RegType::kX86_Gpd) && I == uint32_t(RegType::kX86_Gpd)) ? kX86MemInfo_67H_X64 : - (B == uint32_t(RegType::kX86_Gpw) && I == uint32_t(RegType::kX86_Xmm)) ? kX86MemInfo_67H_X86 : - (B == uint32_t(RegType::kX86_Gpd) && I == uint32_t(RegType::kX86_Xmm)) ? kX86MemInfo_67H_X64 : - (B == uint32_t(RegType::kX86_Gpw) && I == uint32_t(RegType::kX86_Ymm)) ? kX86MemInfo_67H_X86 : - (B == uint32_t(RegType::kX86_Gpd) && I == uint32_t(RegType::kX86_Ymm)) ? kX86MemInfo_67H_X64 : - (B == uint32_t(RegType::kX86_Gpw) && I == uint32_t(RegType::kX86_Zmm)) ? kX86MemInfo_67H_X86 : - (B == uint32_t(RegType::kX86_Gpd) && I == uint32_t(RegType::kX86_Zmm)) ? kX86MemInfo_67H_X64 : - (B == uint32_t(RegType::kLabelTag) && I == uint32_t(RegType::kX86_Gpw)) ? kX86MemInfo_67H_X86 : - (B == uint32_t(RegType::kLabelTag) && I == uint32_t(RegType::kX86_Gpd)) ? kX86MemInfo_67H_X64 : 0; + (B == uint32_t(RegType::kGp16) && I == uint32_t(RegType::kNone) ) ? kX86MemInfo_67H_X86 : + (B == uint32_t(RegType::kGp32) && I == uint32_t(RegType::kNone) ) ? kX86MemInfo_67H_X64 : + (B == uint32_t(RegType::kNone) && I == uint32_t(RegType::kGp16) ) ? kX86MemInfo_67H_X86 : + (B == uint32_t(RegType::kNone) && I == uint32_t(RegType::kGp32) ) ? kX86MemInfo_67H_X64 : + (B == uint32_t(RegType::kGp16) && I == uint32_t(RegType::kGp16) ) ? kX86MemInfo_67H_X86 : + (B == uint32_t(RegType::kGp32) && I == uint32_t(RegType::kGp32) ) ? kX86MemInfo_67H_X64 : + (B == uint32_t(RegType::kGp16) && I == uint32_t(RegType::kVec128) ) ? kX86MemInfo_67H_X86 : + (B == uint32_t(RegType::kGp32) && I == uint32_t(RegType::kVec128) ) ? kX86MemInfo_67H_X64 : + (B == uint32_t(RegType::kGp16) && I == uint32_t(RegType::kVec256) ) ? kX86MemInfo_67H_X86 : + (B == uint32_t(RegType::kGp32) && I == uint32_t(RegType::kVec256) ) ? kX86MemInfo_67H_X64 : + (B == uint32_t(RegType::kGp16) && I == uint32_t(RegType::kVec512) ) ? kX86MemInfo_67H_X86 : + (B == uint32_t(RegType::kGp32) && I == uint32_t(RegType::kVec512) ) ? kX86MemInfo_67H_X64 : + (B == uint32_t(RegType::kLabelTag) && I == uint32_t(RegType::kGp16) ) ? kX86MemInfo_67H_X86 : + (B == uint32_t(RegType::kLabelTag) && I == uint32_t(RegType::kGp32) ) ? kX86MemInfo_67H_X64 : 0; static inline constexpr uint32_t kValue = kBase | kIndex | k67H | 0x04u | 0x08u; }; @@ -227,8 +227,8 @@ static const uint32_t x86LLBySizeDiv16[] = { ASMJIT_LOOKUP_TABLE_16(VALUE, 0) }; // Table that contains LL opcode field addressed by a register size / 16. It's used to propagate L.256 or L.512 when // YMM or ZMM registers are used, respectively. -#define VALUE(x) x == uint32_t(RegType::kX86_Zmm) ? Opcode::kLL_2 : \ - x == uint32_t(RegType::kX86_Ymm) ? Opcode::kLL_1 : Opcode::kLL_0 +#define VALUE(x) x == uint32_t(RegType::kVec512) ? Opcode::kLL_2 : \ + x == uint32_t(RegType::kVec256) ? Opcode::kLL_1 : Opcode::kLL_0 static const uint32_t x86LLByRegType[] = { ASMJIT_LOOKUP_TABLE_16(VALUE, 0) }; #undef VALUE @@ -345,7 +345,7 @@ static ASMJIT_INLINE uint32_t x86AltOpcodeOf(const InstDB::InstInfo* info) noexc } static ASMJIT_INLINE bool x86IsMmxOrXmm(const Reg& reg) noexcept { - return reg.type() == RegType::kX86_Mm || reg.type() == RegType::kX86_Xmm; + return reg.regType() == RegType::kX86_Mm || reg.regType() == RegType::kVec128; } // x86::Assembler - X86BufferWriter @@ -457,7 +457,7 @@ public: // - Disallow REX prefix. #define FIXUP_GPB(REG_OP, REG_ID) \ do { \ - if (!static_cast(REG_OP).isGpbHi()) { \ + if (!static_cast(REG_OP).isGp8Hi()) { \ options |= (REG_ID) >= 4 ? InstOptions::kX86_Rex \ : InstOptions::kNone; \ } \ @@ -540,6 +540,8 @@ static ASMJIT_INLINE bool x86ShouldUseMovabs(Assembler* self, X86BufferWriter& w Assembler::Assembler(CodeHolder* code) noexcept : BaseAssembler() { _archMask = (uint64_t(1) << uint32_t(Arch::kX86)) | (uint64_t(1) << uint32_t(Arch::kX64)) ; + initEmitterFuncs(this); + if (code) { code->attach(this); } @@ -711,7 +713,7 @@ ASMJIT_FAVOR_SPEED Error Assembler::_emit(InstId instId, const Operand_& o0, con if (ASMJIT_UNLIKELY(!o0.isReg())) goto InvalidInstruction; - rmInfo = x86MemInfo[size_t(o0.as().type())]; + rmInfo = x86MemInfo[size_t(o0.as().regType())]; writer.emitAddressOverride((rmInfo & _addressOverrideMask()) != 0); goto EmitX86Op; @@ -783,7 +785,7 @@ CaseX86M_GPB_MulDiv: if (isign3 > 0x7) { // [AX] <- [AX] div|mul r8. if (isign3 == ENC_OPS2(Reg, Reg)) { - if (ASMJIT_UNLIKELY(!Reg::isGpw(o0, Gp::kIdAx) || !Reg::isGpb(o1))) + if (ASMJIT_UNLIKELY(!o0.isGp16(Gp::kIdAx) || !o1.isGp8())) goto InvalidInstruction; rbReg = o1.id(); @@ -793,7 +795,7 @@ CaseX86M_GPB_MulDiv: // [AX] <- [AX] div|mul m8. if (isign3 == ENC_OPS2(Reg, Mem)) { - if (ASMJIT_UNLIKELY(!Reg::isGpw(o0, Gp::kIdAx))) + if (ASMJIT_UNLIKELY(!o0.isGp16(Gp::kIdAx))) goto InvalidInstruction; rmRel = &o1; @@ -848,7 +850,7 @@ CaseX86M_GPB_MulDiv: break; case InstDB::kEncodingX86M_Only_EDX_EAX: - if (isign3 == ENC_OPS3(Mem, Reg, Reg) && Reg::isGpd(o1, Gp::kIdDx) && Reg::isGpd(o2, Gp::kIdAx)) { + if (isign3 == ENC_OPS3(Mem, Reg, Reg) && o1.isGp32(Gp::kIdDx) && o2.isGp32(Gp::kIdAx)) { rmRel = &o0; goto EmitX86M; } @@ -907,7 +909,7 @@ CaseX86M_GPB_MulDiv: case InstDB::kEncodingX86R32_EDX_EAX: // Explicit form: R32, EDX, EAX. if (isign3 == ENC_OPS3(Reg, Reg, Reg)) { - if (!Reg::isGpd(o1, Gp::kIdDx) || !Reg::isGpd(o2, Gp::kIdAx)) + if (!o1.isGp32(Gp::kIdDx) || !o2.isGp32(Gp::kIdAx)) goto InvalidInstruction; rbReg = o0.id(); goto EmitX86R; @@ -915,7 +917,7 @@ CaseX86M_GPB_MulDiv: // Implicit form: R32. if (isign3 == ENC_OPS1(Reg)) { - if (!Reg::isGpd(o0)) + if (!o0.isGp32()) goto InvalidInstruction; rbReg = o0.id(); goto EmitX86R; @@ -1189,8 +1191,9 @@ CaseX86M_GPB_MulDiv: case InstDB::kEncodingX86Cmpxchg: { // Convert explicit to implicit. if (isign3 & (0x7 << 6)) { - if (!Reg::isGp(o2) || o2.id() != Gp::kIdAx) + if (!o2.isGp() || o2.id() != Gp::kIdAx) { goto InvalidInstruction; + } isign3 &= 0x3F; } @@ -1473,8 +1476,9 @@ CaseX86M_GPB_MulDiv: rmRel = &o0; // Explicit jecxz|loop [r|e]cx, dst if (o0.isReg()) { - if (ASMJIT_UNLIKELY(!Reg::isGp(o0, Gp::kIdCx))) + if (ASMJIT_UNLIKELY(!o0.isGp(Gp::kIdCx))) { goto InvalidInstruction; + } writer.emitAddressOverride((is32Bit() && o0.x86RmSize() == 2) || (is64Bit() && o0.x86RmSize() == 4)); rmRel = &o1; @@ -1553,12 +1557,12 @@ CaseX86M_GPB_MulDiv: // register. The segment register is always `opReg`, because the MOV instruction uses either RM or MR encoding. // GP <- ?? - if (Reg::isGp(o0)) { + if (o0.as().isGp()) { rbReg = o0.id(); opReg = o1.id(); // GP <- GP - if (Reg::isGp(o1)) { + if (o1.as().isGp()) { uint32_t opSize = o0.x86RmSize(); if (opSize != o1.x86RmSize()) goto InvalidInstruction; @@ -1589,7 +1593,7 @@ CaseX86M_GPB_MulDiv: } // GP <- SReg - if (Reg::isSReg(o1)) { + if (o1.isSegmentReg()) { opcode = 0x8C; opcode.addPrefixBySize(o0.x86RmSize()); opReg--; @@ -1597,7 +1601,7 @@ CaseX86M_GPB_MulDiv: } // GP <- CReg - if (Reg::isCReg(o1)) { + if (o1.isControlReg()) { opcode = Opcode::k000F00 | 0x20; // Use `LOCK MOV` in 32-bit mode if CR8+ register is accessed (AMD extension). @@ -1609,7 +1613,7 @@ CaseX86M_GPB_MulDiv: } // GP <- DReg - if (Reg::isDReg(o1)) { + if (o1.isDebugReg()) { opcode = Opcode::k000F00 | 0x21; goto EmitX86R; } @@ -1619,11 +1623,11 @@ CaseX86M_GPB_MulDiv: rbReg = o1.id(); // ?? <- GP - if (!Reg::isGp(o1)) + if (!o1.as().isGp()) goto InvalidInstruction; // SReg <- GP - if (Reg::isSReg(o0)) { + if (o0.isSegmentReg()) { opcode = 0x8E; opcode.addPrefixBySize(o1.x86RmSize()); opReg--; @@ -1631,7 +1635,7 @@ CaseX86M_GPB_MulDiv: } // CReg <- GP - if (Reg::isCReg(o0)) { + if (o0.isControlReg()) { opcode = Opcode::k000F00 | 0x22; // Use `LOCK MOV` in 32-bit mode if CR8+ register is accessed (AMD extension). @@ -1643,7 +1647,7 @@ CaseX86M_GPB_MulDiv: } // DReg <- GP - if (Reg::isDReg(o0)) { + if (o0.isDebugReg()) { opcode = Opcode::k000F00 | 0x23; goto EmitX86R; } @@ -1657,7 +1661,7 @@ CaseX86M_GPB_MulDiv: rmRel = &o1; // SReg <- Mem - if (Reg::isSReg(o0)) { + if (o0.isSegmentReg()) { opcode = 0x8E; opcode.addPrefixBySize(o1.x86RmSize()); opReg--; @@ -1690,7 +1694,7 @@ CaseX86M_GPB_MulDiv: rmRel = &o0; // Mem <- SReg - if (Reg::isSReg(o1)) { + if (o1.isSegmentReg()) { opcode = 0x8C; opcode.addPrefixBySize(o0.x86RmSize()); opReg--; @@ -1815,7 +1819,7 @@ CaseX86M_GPB_MulDiv: // Reg <- Imm. if (isign3 == ENC_OPS2(Reg, Imm)) { - if (ASMJIT_UNLIKELY(!o0.as().isGpq())) + if (ASMJIT_UNLIKELY(!o0.isGp64())) goto InvalidInstruction; opReg = o0.id(); @@ -1853,7 +1857,7 @@ CaseX86M_GPB_MulDiv: case InstDB::kEncodingX86MovntiMovdiri: if (isign3 == ENC_OPS2(Mem, Reg)) { - opcode.addWIf(Reg::isGpq(o1)); + opcode.addWIf(o1.isGp64()); opReg = o1.id(); rmRel = &o0; @@ -1928,7 +1932,7 @@ CaseX86M_GPB_MulDiv: case InstDB::kEncodingX86Push: if (isign3 == ENC_OPS1(Reg)) { - if (Reg::isSReg(o0)) { + if (o0.isSegmentReg()) { uint32_t segment = o0.id(); if (ASMJIT_UNLIKELY(segment >= SReg::kIdCount)) goto InvalidSegment; @@ -1955,7 +1959,7 @@ CaseX86M_GPB_MulDiv: case InstDB::kEncodingX86Pop: if (isign3 == ENC_OPS1(Reg)) { - if (Reg::isSReg(o0)) { + if (o0.isSegmentReg()) { uint32_t segment = o0.id(); if (ASMJIT_UNLIKELY(segment == SReg::kIdCs || segment >= SReg::kIdCount)) goto InvalidSegment; @@ -2129,7 +2133,7 @@ CaseX86PushPop_Gp: case InstDB::kEncodingX86StrRm: if (isign3 == ENC_OPS2(Reg, Mem)) { rmRel = &o1; - if (ASMJIT_UNLIKELY(rmRel->as().offsetLo32() || !Reg::isGp(o0.as(), Gp::kIdAx))) + if (ASMJIT_UNLIKELY(rmRel->as().offsetLo32() || !o0.as().isGp(Gp::kIdAx))) goto InvalidInstruction; uint32_t size = o0.x86RmSize(); @@ -2144,7 +2148,7 @@ CaseX86PushPop_Gp: case InstDB::kEncodingX86StrMr: if (isign3 == ENC_OPS2(Mem, Reg)) { rmRel = &o0; - if (ASMJIT_UNLIKELY(rmRel->as().offsetLo32() || !Reg::isGp(o1.as(), Gp::kIdAx))) + if (ASMJIT_UNLIKELY(rmRel->as().offsetLo32() || !o1.isGp(Gp::kIdAx))) goto InvalidInstruction; uint32_t size = o1.x86RmSize(); @@ -2496,7 +2500,7 @@ CaseFpuArith_Mem: case InstDB::kEncodingExtPextrw: if (isign3 == ENC_OPS3(Reg, Reg, Imm)) { - opcode.add66hIf(Reg::isXmm(o1)); + opcode.add66hIf(o1.isVec128()); immValue = o2.as().value(); immSize = 1; @@ -2509,7 +2513,7 @@ CaseFpuArith_Mem: if (isign3 == ENC_OPS3(Mem, Reg, Imm)) { // Secondary opcode of 'pextrw' instruction (SSE4.1). opcode = x86AltOpcodeOf(instInfo); - opcode.add66hIf(Reg::isXmm(o1)); + opcode.add66hIf(o1.isVec128()); immValue = o2.as().value(); immSize = 1; @@ -2522,7 +2526,7 @@ CaseFpuArith_Mem: case InstDB::kEncodingExtExtract: if (isign3 == ENC_OPS3(Reg, Reg, Imm)) { - opcode.add66hIf(Reg::isXmm(o1)); + opcode.add66hIf(o1.isVec128()); immValue = o2.as().value(); immSize = 1; @@ -2533,7 +2537,7 @@ CaseFpuArith_Mem: } if (isign3 == ENC_OPS3(Mem, Reg, Imm)) { - opcode.add66hIf(Reg::isXmm(o1)); + opcode.add66hIf(o1.isVec128()); immValue = o2.as().value(); immSize = 1; @@ -2605,10 +2609,10 @@ CaseFpuArith_Mem: CaseExtMovd: if (x86IsMmxOrXmm(o0.as())) { opReg = o0.id(); - opcode.add66hIf(Reg::isXmm(o0)); + opcode.add66hIf(o0.isVec128()); // MM/XMM <- Gp - if (isign3 == ENC_OPS2(Reg, Reg) && Reg::isGp(o1)) { + if (isign3 == ENC_OPS2(Reg, Reg) && o1.as().isGp()) { rbReg = o1.id(); goto EmitX86R; } @@ -2625,10 +2629,10 @@ CaseExtMovd: opcode &= Opcode::kW; opcode |= x86AltOpcodeOf(instInfo); opReg = o1.id(); - opcode.add66hIf(Reg::isXmm(o1)); + opcode.add66hIf(o1.isVec128()); // GP <- MM/XMM - if (isign3 == ENC_OPS2(Reg, Reg) && Reg::isGp(o0)) { + if (isign3 == ENC_OPS2(Reg, Reg) && o0.as().isGp()) { rbReg = o0.id(); goto EmitX86R; } @@ -2647,7 +2651,7 @@ CaseExtMovd: rbReg = o1.id(); // MM <- MM - if (Reg::isMm(o0) && Reg::isMm(o1)) { + if (o0.isMmReg() && o1.isMmReg()) { opcode = Opcode::k000F00 | 0x6F; if (!Support::test(options, InstOptions::kX86_ModMR)) @@ -2659,7 +2663,7 @@ CaseExtMovd: } // XMM <- XMM - if (Reg::isXmm(o0) && Reg::isXmm(o1)) { + if (o0.isVec128() && o1.isVec128()) { opcode = Opcode::kF30F00 | 0x7E; if (!Support::test(options, InstOptions::kX86_ModMR)) @@ -2676,13 +2680,13 @@ CaseExtMovd: rmRel = &o1; // MM <- Mem - if (Reg::isMm(o0)) { + if (o0.isMmReg()) { opcode = Opcode::k000F00 | 0x6F; goto EmitX86M; } // XMM <- Mem - if (Reg::isXmm(o0)) { + if (o0.isVec128()) { opcode = Opcode::kF30F00 | 0x7E; goto EmitX86M; } @@ -2693,13 +2697,13 @@ CaseExtMovd: rmRel = &o0; // Mem <- MM - if (Reg::isMm(o1)) { + if (o1.isMmReg()) { opcode = Opcode::k000F00 | 0x7F; goto EmitX86M; } // Mem <- XMM - if (Reg::isXmm(o1)) { + if (o1.isVec128()) { opcode = Opcode::k660F00 | 0xD6; goto EmitX86M; } @@ -2710,7 +2714,7 @@ CaseExtMovd: goto CaseExtMovd; case InstDB::kEncodingExtRm_XMM0: - if (ASMJIT_UNLIKELY(!o2.isNone() && !Reg::isXmm(o2, 0))) + if (ASMJIT_UNLIKELY(!o2.isNone() && !o2.isVec128(0))) goto InvalidInstruction; isign3 &= 0x3F; @@ -2728,7 +2732,7 @@ CaseExtMovd: [[fallthrough]]; case InstDB::kEncodingExtRm_Wx_GpqOnly: - opcode.addWIf(Reg::isGpq(o0)); + opcode.addWIf(o0.isGp64()); [[fallthrough]]; case InstDB::kEncodingExtRm: @@ -2748,7 +2752,7 @@ CaseExtRm: case InstDB::kEncodingExtRm_P: if (isign3 == ENC_OPS2(Reg, Reg)) { - opcode.add66hIf(unsigned(Reg::isXmm(o0)) | unsigned(Reg::isXmm(o1))); + opcode.add66hIf(Support::bool_or(o0.isVec128(), o1.isVec128())); opReg = o0.id(); rbReg = o1.id(); @@ -2756,7 +2760,7 @@ CaseExtRm: } if (isign3 == ENC_OPS2(Reg, Mem)) { - opcode.add66hIf(Reg::isXmm(o0)); + opcode.add66hIf(o0.isVec128()); opReg = o0.id(); rmRel = &o1; @@ -2792,7 +2796,7 @@ CaseExtRm: case InstDB::kEncodingExtRmRi_P: if (isign3 == ENC_OPS2(Reg, Reg)) { - opcode.add66hIf(unsigned(Reg::isXmm(o0)) | unsigned(Reg::isXmm(o1))); + opcode.add66hIf(Support::bool_or(o0.isVec128(), o1.isVec128())); opReg = o0.id(); rbReg = o1.id(); @@ -2800,7 +2804,7 @@ CaseExtRm: } if (isign3 == ENC_OPS2(Reg, Mem)) { - opcode.add66hIf(Reg::isXmm(o0)); + opcode.add66hIf(o0.isVec128()); opReg = o0.id(); rmRel = &o1; @@ -2812,7 +2816,7 @@ CaseExtRm: opReg = opcode.extractModO(); if (isign3 == ENC_OPS2(Reg, Imm)) { - opcode.add66hIf(Reg::isXmm(o0)); + opcode.add66hIf(o0.isVec128()); immValue = o1.as().value(); immSize = 1; @@ -2844,7 +2848,7 @@ CaseExtRm: immSize = 1; if (isign3 == ENC_OPS3(Reg, Reg, Imm)) { - opcode.add66hIf(unsigned(Reg::isXmm(o0)) | unsigned(Reg::isXmm(o1))); + opcode.add66hIf(Support::bool_or(o0.isVec128(), o1.isVec128())); opReg = o0.id(); rbReg = o1.id(); @@ -2852,7 +2856,7 @@ CaseExtRm: } if (isign3 == ENC_OPS3(Reg, Mem, Imm)) { - opcode.add66hIf(Reg::isXmm(o0)); + opcode.add66hIf(o0.isVec128()); opReg = o0.id(); rmRel = &o1; @@ -2944,13 +2948,13 @@ CaseExtRm: rbReg = o1.id(); // Form 'k, reg'. - if (Reg::isGp(o1)) { + if (o1.as().isGp()) { opcode = x86AltOpcodeOf(instInfo); goto EmitVexEvexR; } // Form 'reg, k'. - if (Reg::isGp(o0)) { + if (o0.as().isGp()) { opcode = x86AltOpcodeOf(instInfo) + 1; goto EmitVexEvexR; } @@ -2982,7 +2986,7 @@ CaseExtRm: case InstDB::kEncodingVexR_Wx: if (isign3 == ENC_OPS1(Reg)) { rbReg = o0.id(); - opcode.addWIf(o0.as().isGpq()); + opcode.addWIf(o0.isGp64()); goto EmitVexEvexR; } break; @@ -3037,7 +3041,7 @@ CaseExtRm: case InstDB::kEncodingVexMvr_Wx: if (isign3 == ENC_OPS3(Mem, Reg, Reg)) { - opcode.addWIf(unsigned(Reg::isGpq(o1))); + opcode.addWIf(o1.isGp64()); opReg = x86PackRegAndVvvvv(o1.id(), o2.id()); rmRel = &o0; goto EmitVexEvexM; @@ -3074,7 +3078,7 @@ CaseVexMri: goto CaseVexRm; case InstDB::kEncodingVexRm_Wx: - opcode.addWIf(unsigned(Reg::isGpq(o0)) | unsigned(Reg::isGpq(o1))); + opcode.addWIf(Support::bool_or(o0.isGp64(), o1.isGp64())); goto CaseVexRm; case InstDB::kEncodingVexRm_Lx_Narrow: @@ -3085,7 +3089,7 @@ CaseVexMri: goto CaseVexRm; case InstDB::kEncodingVexRm_Lx_Bcst: - if (isign3 == ENC_OPS2(Reg, Reg) && Reg::isGp(o1.as())) { + if (isign3 == ENC_OPS2(Reg, Reg) && o1.as().isGp()) { opcode = x86AltOpcodeOf(instInfo) | x86OpcodeLBySize(o0.x86RmSize() | o1.x86RmSize()); opReg = o0.id(); rbReg = o1.id(); @@ -3122,7 +3126,7 @@ CaseVexRm: break; case InstDB::kEncodingVexRmi_Wx: - opcode.addWIf(unsigned(Reg::isGpq(o0)) | unsigned(Reg::isGpq(o1))); + opcode.addWIf(Support::bool_or(o0.isGp64(), o1.isGp64())); goto CaseVexRmi; case InstDB::kEncodingVexRmi_Lx: @@ -3165,18 +3169,18 @@ CaseVexRvm_R: case InstDB::kEncodingVexRvm_ZDX_Wx: { const Operand_& o3 = opExt[EmitterUtils::kOp3]; - if (ASMJIT_UNLIKELY(!o3.isNone() && !Reg::isGp(o3, Gp::kIdDx))) + if (ASMJIT_UNLIKELY(!o3.isNone() && !o3.isGp(Gp::kIdDx))) goto InvalidInstruction; [[fallthrough]]; } case InstDB::kEncodingVexRvm_Wx: { - opcode.addWIf(unsigned(Reg::isGpq(o0)) | unsigned((o2.x86RmSize() == 8))); + opcode.addWIf(Support::bool_or(o0.isGp64(), o2.x86RmSize() == 8)); goto CaseVexRvm; } case InstDB::kEncodingVexRvm_Lx_KEvex: { - opcode.forceEvexIf(Reg::isKReg(o0)); + opcode.forceEvexIf(o0.isMaskReg()); [[fallthrough]]; } @@ -3238,11 +3242,11 @@ CaseVexRvm_R: } case InstDB::kEncodingVexRvmi_KEvex: - opcode.forceEvexIf(Reg::isKReg(o0)); + opcode.forceEvexIf(o0.isMaskReg()); goto VexRvmi; case InstDB::kEncodingVexRvmi_Lx_KEvex: - opcode.forceEvexIf(Reg::isKReg(o0)); + opcode.forceEvexIf(o0.isMaskReg()); [[fallthrough]]; case InstDB::kEncodingVexRvmi_Lx: @@ -3273,7 +3277,7 @@ VexRvmi: } case InstDB::kEncodingVexRmv_Wx: - opcode.addWIf(unsigned(Reg::isGpq(o0)) | unsigned(Reg::isGpq(o2))); + opcode.addWIf(Support::bool_or(o0.isGp64(), o2.isGp64())); [[fallthrough]]; case InstDB::kEncodingVexRmv: @@ -3335,7 +3339,7 @@ VexRvmi: case InstDB::kEncodingVexMovdMovq: if (isign3 == ENC_OPS2(Reg, Reg)) { - if (Reg::isGp(o0)) { + if (o0.as().isGp()) { opcode = x86AltOpcodeOf(instInfo); opcode.addWBySize(o0.x86RmSize()); opReg = o1.id(); @@ -3343,7 +3347,7 @@ VexRvmi: goto EmitVexEvexR; } - if (Reg::isGp(o1)) { + if (o1.as().isGp()) { opcode.addWBySize(o1.x86RmSize()); opReg = o0.id(); rbReg = o1.id(); @@ -3626,7 +3630,7 @@ VexRvmi: break; case InstDB::kEncodingVexVm_Wx: - opcode.addWIf(unsigned(Reg::isGpq(o0)) | unsigned(Reg::isGpq(o1))); + opcode.addWIf(Support::bool_or(o0.isGp64(), o1.isGp64())); [[fallthrough]]; case InstDB::kEncodingVexVm: @@ -3671,7 +3675,7 @@ CaseVexVmi_AfterImm: break; case InstDB::kEncodingVexVmi4_Wx: - opcode.addWIf(Reg::isGpq(o0) || o1.x86RmSize() == 8); + opcode.addWIf(Support::bool_or(o0.isGp64(), o1.x86RmSize() == 8)); immValue = o2.as().value(); immSize = 4; goto CaseVexVmi_AfterImm; @@ -4167,7 +4171,7 @@ EmitModSib: uint32_t kModRel32Size = 5; uint64_t virtualOffset = uint64_t(writer.offsetFrom(_bufferData)) + immSize + kModRel32Size; - if (baseAddress == Globals::kNoBaseAddress || _section->id() != 0) { + if (baseAddress == Globals::kNoBaseAddress || _section->sectionId() != 0) { // Create a new RelocEntry as we cannot calculate the offset right now. err = _code->newRelocEntry(&re, RelocType::kAbsToRel); if (ASMJIT_UNLIKELY(err)) @@ -4175,7 +4179,7 @@ EmitModSib: writer.emit8(x86EncodeMod(0, opReg, 5)); - re->_sourceSectionId = _section->id(); + re->_sourceSectionId = _section->sectionId(); re->_sourceOffset = offset(); re->_format.resetToSimpleValue(OffsetType::kSignedOffset, 4); re->_format.setLeadingAndTrailingSize(writer.offsetFrom(_bufferPtr), immSize); @@ -4264,15 +4268,18 @@ EmitModSib_LabelRip_X86: relOffset = rmRel->as().offsetLo32(); if (rmInfo & kX86MemInfo_BaseLabel) { // [LABEL->ABS]. - label = _code->labelEntry(rmRel->as().baseId()); - if (ASMJIT_UNLIKELY(!label)) - goto InvalidLabel; + uint32_t baseLabelId = rmRel->as().baseId(); + if (ASMJIT_UNLIKELY(!_code->isLabelValid(baseLabelId))) { + } + label = &_code->labelEntry(baseLabelId); err = _code->newRelocEntry(&re, RelocType::kRelToAbs); - if (ASMJIT_UNLIKELY(err)) - goto Failed; - re->_sourceSectionId = _section->id(); + if (ASMJIT_UNLIKELY(err)) { + goto Failed; + } + + re->_sourceSectionId = _section->sectionId(); re->_sourceOffset = offset(); re->_format.resetToSimpleValue(OffsetType::kUnsignedOffset, 4); re->_format.setLeadingAndTrailingSize(writer.offsetFrom(_bufferPtr), immSize); @@ -4281,7 +4288,7 @@ EmitModSib_LabelRip_X86: if (label->isBound()) { // Label bound to the current section. re->_payload += label->offset(); - re->_targetSectionId = label->section()->id(); + re->_targetSectionId = label->sectionId(); writer.emit32uLE(0); } else { @@ -4297,8 +4304,8 @@ EmitModSib_LabelRip_X86: if (ASMJIT_UNLIKELY(err)) goto Failed; - re->_sourceSectionId = _section->id(); - re->_targetSectionId = _section->id(); + re->_sourceSectionId = _section->sectionId(); + re->_targetSectionId = _section->sectionId(); re->_format.resetToSimpleValue(OffsetType::kUnsignedOffset, 4); re->_format.setLeadingAndTrailingSize(writer.offsetFrom(_bufferPtr), immSize); re->_sourceOffset = offset(); @@ -4311,11 +4318,14 @@ EmitModSib_LabelRip_X86: relOffset = rmRel->as().offsetLo32(); if (rmInfo & kX86MemInfo_BaseLabel) { // [RIP]. - label = _code->labelEntry(rmRel->as().baseId()); - if (ASMJIT_UNLIKELY(!label)) + uint32_t baseLabelId = rmRel->as().baseId(); + if (ASMJIT_UNLIKELY(!_code->isLabelValid(baseLabelId))) { goto InvalidLabel; + } + label = &_code->labelEntry(baseLabelId); relOffset -= (4 + immSize); + if (label->isBoundTo(_section)) { // Label bound to the current section. relOffset += int32_t(label->offset() - writer.offsetFrom(_bufferData)); @@ -4807,10 +4817,12 @@ EmitJmpCall: inst32Size += uint32_t((opcode & Opcode::kMM_Mask) == Opcode::kMM_0F); if (rmRel->isLabel()) { - label = _code->labelEntry(rmRel->as