MacroAssemblerARM64.cpp
1 /* 2 * Copyright (C) 2013-2020 Apple Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY 14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR 17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26 #include "config.h" 27 28 #if ENABLE(ASSEMBLER) && CPU(ARM64) 29 #include "MacroAssembler.h" 30 31 #include "JSCPtrTag.h" 32 #include "ProbeContext.h" 33 #include <wtf/InlineASM.h> 34 35 #if OS(LINUX) 36 #include <asm/hwcap.h> 37 #include <sys/auxv.h> 38 #endif 39 40 namespace JSC { 41 42 #if ENABLE(MASM_PROBE) 43 44 extern "C" JSC_DECLARE_JIT_OPERATION(ctiMasmProbeTrampoline, void, ()); 45 JSC_ANNOTATE_JIT_OPERATION(ctiMasmProbeTrampolineId, ctiMasmProbeTrampoline); 46 47 using namespace ARM64Registers; 48 49 #if COMPILER(GCC_COMPATIBLE) 50 51 // The following are offsets for Probe::State fields accessed 52 // by the ctiMasmProbeTrampoline stub. 53 #if CPU(ADDRESS64) 54 #define PTR_SIZE 8 55 #else 56 #define PTR_SIZE 4 57 #endif 58 59 #define PROBE_PROBE_FUNCTION_OFFSET (0 * PTR_SIZE) 60 #define PROBE_ARG_OFFSET (1 * PTR_SIZE) 61 #define PROBE_INIT_STACK_FUNCTION_OFFSET (2 * PTR_SIZE) 62 #define PROBE_INIT_STACK_ARG_OFFSET (3 * PTR_SIZE) 63 64 #define PROBE_FIRST_GPREG_OFFSET (4 * PTR_SIZE) 65 66 #define GPREG_SIZE 8 67 #define PROBE_CPU_X0_OFFSET (PROBE_FIRST_GPREG_OFFSET + (0 * GPREG_SIZE)) 68 #define PROBE_CPU_X1_OFFSET (PROBE_FIRST_GPREG_OFFSET + (1 * GPREG_SIZE)) 69 #define PROBE_CPU_X2_OFFSET (PROBE_FIRST_GPREG_OFFSET + (2 * GPREG_SIZE)) 70 #define PROBE_CPU_X3_OFFSET (PROBE_FIRST_GPREG_OFFSET + (3 * GPREG_SIZE)) 71 #define PROBE_CPU_X4_OFFSET (PROBE_FIRST_GPREG_OFFSET + (4 * GPREG_SIZE)) 72 #define PROBE_CPU_X5_OFFSET (PROBE_FIRST_GPREG_OFFSET + (5 * GPREG_SIZE)) 73 #define PROBE_CPU_X6_OFFSET (PROBE_FIRST_GPREG_OFFSET + (6 * GPREG_SIZE)) 74 #define PROBE_CPU_X7_OFFSET (PROBE_FIRST_GPREG_OFFSET + (7 * GPREG_SIZE)) 75 #define PROBE_CPU_X8_OFFSET (PROBE_FIRST_GPREG_OFFSET + (8 * GPREG_SIZE)) 76 #define PROBE_CPU_X9_OFFSET (PROBE_FIRST_GPREG_OFFSET + (9 * GPREG_SIZE)) 77 #define PROBE_CPU_X10_OFFSET (PROBE_FIRST_GPREG_OFFSET + (10 * GPREG_SIZE)) 78 #define PROBE_CPU_X11_OFFSET (PROBE_FIRST_GPREG_OFFSET + (11 * GPREG_SIZE)) 79 #define PROBE_CPU_X12_OFFSET (PROBE_FIRST_GPREG_OFFSET + (12 * GPREG_SIZE)) 80 #define PROBE_CPU_X13_OFFSET (PROBE_FIRST_GPREG_OFFSET + (13 * GPREG_SIZE)) 81 #define PROBE_CPU_X14_OFFSET (PROBE_FIRST_GPREG_OFFSET + (14 * GPREG_SIZE)) 82 #define PROBE_CPU_X15_OFFSET (PROBE_FIRST_GPREG_OFFSET + (15 * GPREG_SIZE)) 83 #define PROBE_CPU_X16_OFFSET (PROBE_FIRST_GPREG_OFFSET + (16 * GPREG_SIZE)) 84 #define PROBE_CPU_X17_OFFSET (PROBE_FIRST_GPREG_OFFSET + (17 * GPREG_SIZE)) 85 #define PROBE_CPU_X18_OFFSET (PROBE_FIRST_GPREG_OFFSET + (18 * GPREG_SIZE)) 86 #define PROBE_CPU_X19_OFFSET (PROBE_FIRST_GPREG_OFFSET + (19 * GPREG_SIZE)) 87 #define PROBE_CPU_X20_OFFSET (PROBE_FIRST_GPREG_OFFSET + (20 * GPREG_SIZE)) 88 #define PROBE_CPU_X21_OFFSET (PROBE_FIRST_GPREG_OFFSET + (21 * GPREG_SIZE)) 89 #define PROBE_CPU_X22_OFFSET (PROBE_FIRST_GPREG_OFFSET + (22 * GPREG_SIZE)) 90 #define PROBE_CPU_X23_OFFSET (PROBE_FIRST_GPREG_OFFSET + (23 * GPREG_SIZE)) 91 #define PROBE_CPU_X24_OFFSET (PROBE_FIRST_GPREG_OFFSET + (24 * GPREG_SIZE)) 92 #define PROBE_CPU_X25_OFFSET (PROBE_FIRST_GPREG_OFFSET + (25 * GPREG_SIZE)) 93 #define PROBE_CPU_X26_OFFSET (PROBE_FIRST_GPREG_OFFSET + (26 * GPREG_SIZE)) 94 #define PROBE_CPU_X27_OFFSET (PROBE_FIRST_GPREG_OFFSET + (27 * GPREG_SIZE)) 95 #define PROBE_CPU_X28_OFFSET (PROBE_FIRST_GPREG_OFFSET + (28 * GPREG_SIZE)) 96 #define PROBE_CPU_FP_OFFSET (PROBE_FIRST_GPREG_OFFSET + (29 * GPREG_SIZE)) 97 #define PROBE_CPU_LR_OFFSET (PROBE_FIRST_GPREG_OFFSET + (30 * GPREG_SIZE)) 98 #define PROBE_CPU_SP_OFFSET (PROBE_FIRST_GPREG_OFFSET + (31 * GPREG_SIZE)) 99 100 #define PROBE_CPU_PC_OFFSET (PROBE_FIRST_GPREG_OFFSET + (32 * GPREG_SIZE)) 101 #define PROBE_CPU_NZCV_OFFSET (PROBE_FIRST_GPREG_OFFSET + (33 * GPREG_SIZE)) 102 #define PROBE_CPU_FPSR_OFFSET (PROBE_FIRST_GPREG_OFFSET + (34 * GPREG_SIZE)) 103 104 #define PROBE_FIRST_FPREG_OFFSET (PROBE_FIRST_GPREG_OFFSET + (35 * GPREG_SIZE)) 105 106 #define FPREG_SIZE 8 107 #define PROBE_CPU_Q0_OFFSET (PROBE_FIRST_FPREG_OFFSET + (0 * FPREG_SIZE)) 108 #define PROBE_CPU_Q1_OFFSET (PROBE_FIRST_FPREG_OFFSET + (1 * FPREG_SIZE)) 109 #define PROBE_CPU_Q2_OFFSET (PROBE_FIRST_FPREG_OFFSET + (2 * FPREG_SIZE)) 110 #define PROBE_CPU_Q3_OFFSET (PROBE_FIRST_FPREG_OFFSET + (3 * FPREG_SIZE)) 111 #define PROBE_CPU_Q4_OFFSET (PROBE_FIRST_FPREG_OFFSET + (4 * FPREG_SIZE)) 112 #define PROBE_CPU_Q5_OFFSET (PROBE_FIRST_FPREG_OFFSET + (5 * FPREG_SIZE)) 113 #define PROBE_CPU_Q6_OFFSET (PROBE_FIRST_FPREG_OFFSET + (6 * FPREG_SIZE)) 114 #define PROBE_CPU_Q7_OFFSET (PROBE_FIRST_FPREG_OFFSET + (7 * FPREG_SIZE)) 115 #define PROBE_CPU_Q8_OFFSET (PROBE_FIRST_FPREG_OFFSET + (8 * FPREG_SIZE)) 116 #define PROBE_CPU_Q9_OFFSET (PROBE_FIRST_FPREG_OFFSET + (9 * FPREG_SIZE)) 117 #define PROBE_CPU_Q10_OFFSET (PROBE_FIRST_FPREG_OFFSET + (10 * FPREG_SIZE)) 118 #define PROBE_CPU_Q11_OFFSET (PROBE_FIRST_FPREG_OFFSET + (11 * FPREG_SIZE)) 119 #define PROBE_CPU_Q12_OFFSET (PROBE_FIRST_FPREG_OFFSET + (12 * FPREG_SIZE)) 120 #define PROBE_CPU_Q13_OFFSET (PROBE_FIRST_FPREG_OFFSET + (13 * FPREG_SIZE)) 121 #define PROBE_CPU_Q14_OFFSET (PROBE_FIRST_FPREG_OFFSET + (14 * FPREG_SIZE)) 122 #define PROBE_CPU_Q15_OFFSET (PROBE_FIRST_FPREG_OFFSET + (15 * FPREG_SIZE)) 123 #define PROBE_CPU_Q16_OFFSET (PROBE_FIRST_FPREG_OFFSET + (16 * FPREG_SIZE)) 124 #define PROBE_CPU_Q17_OFFSET (PROBE_FIRST_FPREG_OFFSET + (17 * FPREG_SIZE)) 125 #define PROBE_CPU_Q18_OFFSET (PROBE_FIRST_FPREG_OFFSET + (18 * FPREG_SIZE)) 126 #define PROBE_CPU_Q19_OFFSET (PROBE_FIRST_FPREG_OFFSET + (19 * FPREG_SIZE)) 127 #define PROBE_CPU_Q20_OFFSET (PROBE_FIRST_FPREG_OFFSET + (20 * FPREG_SIZE)) 128 #define PROBE_CPU_Q21_OFFSET (PROBE_FIRST_FPREG_OFFSET + (21 * FPREG_SIZE)) 129 #define PROBE_CPU_Q22_OFFSET (PROBE_FIRST_FPREG_OFFSET + (22 * FPREG_SIZE)) 130 #define PROBE_CPU_Q23_OFFSET (PROBE_FIRST_FPREG_OFFSET + (23 * FPREG_SIZE)) 131 #define PROBE_CPU_Q24_OFFSET (PROBE_FIRST_FPREG_OFFSET + (24 * FPREG_SIZE)) 132 #define PROBE_CPU_Q25_OFFSET (PROBE_FIRST_FPREG_OFFSET + (25 * FPREG_SIZE)) 133 #define PROBE_CPU_Q26_OFFSET (PROBE_FIRST_FPREG_OFFSET + (26 * FPREG_SIZE)) 134 #define PROBE_CPU_Q27_OFFSET (PROBE_FIRST_FPREG_OFFSET + (27 * FPREG_SIZE)) 135 #define PROBE_CPU_Q28_OFFSET (PROBE_FIRST_FPREG_OFFSET + (28 * FPREG_SIZE)) 136 #define PROBE_CPU_Q29_OFFSET (PROBE_FIRST_FPREG_OFFSET + (29 * FPREG_SIZE)) 137 #define PROBE_CPU_Q30_OFFSET (PROBE_FIRST_FPREG_OFFSET + (30 * FPREG_SIZE)) 138 #define PROBE_CPU_Q31_OFFSET (PROBE_FIRST_FPREG_OFFSET + (31 * FPREG_SIZE)) 139 #define PROBE_SIZE (PROBE_FIRST_FPREG_OFFSET + (32 * FPREG_SIZE)) 140 141 #define SAVED_PROBE_RETURN_PC_OFFSET (PROBE_SIZE + (0 * GPREG_SIZE)) 142 #define PROBE_SIZE_PLUS_EXTRAS (PROBE_SIZE + (3 * GPREG_SIZE)) 143 144 // These ASSERTs remind you that if you change the layout of Probe::State, 145 // you need to change ctiMasmProbeTrampoline offsets above to match. 146 #define PROBE_OFFSETOF(x) offsetof(struct Probe::State, x) 147 static_assert(PROBE_OFFSETOF(probeFunction) == PROBE_PROBE_FUNCTION_OFFSET, "Probe::State::probeFunction's offset matches ctiMasmProbeTrampoline"); 148 static_assert(PROBE_OFFSETOF(arg) == PROBE_ARG_OFFSET, "Probe::State::arg's offset matches ctiMasmProbeTrampoline"); 149 static_assert(PROBE_OFFSETOF(initializeStackFunction) == PROBE_INIT_STACK_FUNCTION_OFFSET, "Probe::State::initializeStackFunction's offset matches ctiMasmProbeTrampoline"); 150 static_assert(PROBE_OFFSETOF(initializeStackArg) == PROBE_INIT_STACK_ARG_OFFSET, "Probe::State::initializeStackArg's offset matches ctiMasmProbeTrampoline"); 151 152 static_assert(!(PROBE_CPU_X0_OFFSET & 0x7), "Probe::State::cpu.gprs[x0]'s offset should be 8 byte aligned"); 153 154 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x0]) == PROBE_CPU_X0_OFFSET, "Probe::State::cpu.gprs[x0]'s offset matches ctiMasmProbeTrampoline"); 155 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x1]) == PROBE_CPU_X1_OFFSET, "Probe::State::cpu.gprs[x1]'s offset matches ctiMasmProbeTrampoline"); 156 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x2]) == PROBE_CPU_X2_OFFSET, "Probe::State::cpu.gprs[x2]'s offset matches ctiMasmProbeTrampoline"); 157 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x3]) == PROBE_CPU_X3_OFFSET, "Probe::State::cpu.gprs[x3]'s offset matches ctiMasmProbeTrampoline"); 158 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x4]) == PROBE_CPU_X4_OFFSET, "Probe::State::cpu.gprs[x4]'s offset matches ctiMasmProbeTrampoline"); 159 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x5]) == PROBE_CPU_X5_OFFSET, "Probe::State::cpu.gprs[x5]'s offset matches ctiMasmProbeTrampoline"); 160 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x6]) == PROBE_CPU_X6_OFFSET, "Probe::State::cpu.gprs[x6]'s offset matches ctiMasmProbeTrampoline"); 161 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x7]) == PROBE_CPU_X7_OFFSET, "Probe::State::cpu.gprs[x7]'s offset matches ctiMasmProbeTrampoline"); 162 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x8]) == PROBE_CPU_X8_OFFSET, "Probe::State::cpu.gprs[x8]'s offset matches ctiMasmProbeTrampoline"); 163 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x9]) == PROBE_CPU_X9_OFFSET, "Probe::State::cpu.gprs[x9]'s offset matches ctiMasmProbeTrampoline"); 164 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x10]) == PROBE_CPU_X10_OFFSET, "Probe::State::cpu.gprs[x10]'s offset matches ctiMasmProbeTrampoline"); 165 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x11]) == PROBE_CPU_X11_OFFSET, "Probe::State::cpu.gprs[x11]'s offset matches ctiMasmProbeTrampoline"); 166 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x12]) == PROBE_CPU_X12_OFFSET, "Probe::State::cpu.gprs[x12]'s offset matches ctiMasmProbeTrampoline"); 167 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x13]) == PROBE_CPU_X13_OFFSET, "Probe::State::cpu.gprs[x13]'s offset matches ctiMasmProbeTrampoline"); 168 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x14]) == PROBE_CPU_X14_OFFSET, "Probe::State::cpu.gprs[x14]'s offset matches ctiMasmProbeTrampoline"); 169 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x15]) == PROBE_CPU_X15_OFFSET, "Probe::State::cpu.gprs[x15]'s offset matches ctiMasmProbeTrampoline"); 170 171 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x16]) == PROBE_CPU_X16_OFFSET, "Probe::State::cpu.gprs[x16]'s offset matches ctiMasmProbeTrampoline"); 172 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x17]) == PROBE_CPU_X17_OFFSET, "Probe::State::cpu.gprs[x17]'s offset matches ctiMasmProbeTrampoline"); 173 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x18]) == PROBE_CPU_X18_OFFSET, "Probe::State::cpu.gprs[x18]'s offset matches ctiMasmProbeTrampoline"); 174 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x19]) == PROBE_CPU_X19_OFFSET, "Probe::State::cpu.gprs[x19]'s offset matches ctiMasmProbeTrampoline"); 175 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x20]) == PROBE_CPU_X20_OFFSET, "Probe::State::cpu.gprs[x20]'s offset matches ctiMasmProbeTrampoline"); 176 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x21]) == PROBE_CPU_X21_OFFSET, "Probe::State::cpu.gprs[x21]'s offset matches ctiMasmProbeTrampoline"); 177 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x22]) == PROBE_CPU_X22_OFFSET, "Probe::State::cpu.gprs[x22]'s offset matches ctiMasmProbeTrampoline"); 178 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x23]) == PROBE_CPU_X23_OFFSET, "Probe::State::cpu.gprs[x23]'s offset matches ctiMasmProbeTrampoline"); 179 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x24]) == PROBE_CPU_X24_OFFSET, "Probe::State::cpu.gprs[x24]'s offset matches ctiMasmProbeTrampoline"); 180 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x25]) == PROBE_CPU_X25_OFFSET, "Probe::State::cpu.gprs[x25]'s offset matches ctiMasmProbeTrampoline"); 181 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x26]) == PROBE_CPU_X26_OFFSET, "Probe::State::cpu.gprs[x26]'s offset matches ctiMasmProbeTrampoline"); 182 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x27]) == PROBE_CPU_X27_OFFSET, "Probe::State::cpu.gprs[x27]'s offset matches ctiMasmProbeTrampoline"); 183 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::x28]) == PROBE_CPU_X28_OFFSET, "Probe::State::cpu.gprs[x28]'s offset matches ctiMasmProbeTrampoline"); 184 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::fp]) == PROBE_CPU_FP_OFFSET, "Probe::State::cpu.gprs[fp]'s offset matches ctiMasmProbeTrampoline"); 185 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::lr]) == PROBE_CPU_LR_OFFSET, "Probe::State::cpu.gprs[lr]'s offset matches ctiMasmProbeTrampoline"); 186 static_assert(PROBE_OFFSETOF(cpu.gprs[ARM64Registers::sp]) == PROBE_CPU_SP_OFFSET, "Probe::State::cpu.gprs[sp]'s offset matches ctiMasmProbeTrampoline"); 187 188 static_assert(PROBE_OFFSETOF(cpu.sprs[ARM64Registers::pc]) == PROBE_CPU_PC_OFFSET, "Probe::State::cpu.sprs[pc]'s offset matches ctiMasmProbeTrampoline"); 189 static_assert(PROBE_OFFSETOF(cpu.sprs[ARM64Registers::nzcv]) == PROBE_CPU_NZCV_OFFSET, "Probe::State::cpu.sprs[nzcv]'s offset matches ctiMasmProbeTrampoline"); 190 static_assert(PROBE_OFFSETOF(cpu.sprs[ARM64Registers::fpsr]) == PROBE_CPU_FPSR_OFFSET, "Probe::State::cpu.sprs[fpsr]'s offset matches ctiMasmProbeTrampoline"); 191 192 static_assert(!(PROBE_CPU_Q0_OFFSET & 0x7), "Probe::State::cpu.fprs[q0]'s offset should be 8 byte aligned"); 193 194 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q0]) == PROBE_CPU_Q0_OFFSET, "Probe::State::cpu.fprs[q0]'s offset matches ctiMasmProbeTrampoline"); 195 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q1]) == PROBE_CPU_Q1_OFFSET, "Probe::State::cpu.fprs[q1]'s offset matches ctiMasmProbeTrampoline"); 196 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q2]) == PROBE_CPU_Q2_OFFSET, "Probe::State::cpu.fprs[q2]'s offset matches ctiMasmProbeTrampoline"); 197 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q3]) == PROBE_CPU_Q3_OFFSET, "Probe::State::cpu.fprs[q3]'s offset matches ctiMasmProbeTrampoline"); 198 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q4]) == PROBE_CPU_Q4_OFFSET, "Probe::State::cpu.fprs[q4]'s offset matches ctiMasmProbeTrampoline"); 199 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q5]) == PROBE_CPU_Q5_OFFSET, "Probe::State::cpu.fprs[q5]'s offset matches ctiMasmProbeTrampoline"); 200 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q6]) == PROBE_CPU_Q6_OFFSET, "Probe::State::cpu.fprs[q6]'s offset matches ctiMasmProbeTrampoline"); 201 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q7]) == PROBE_CPU_Q7_OFFSET, "Probe::State::cpu.fprs[q7]'s offset matches ctiMasmProbeTrampoline"); 202 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q8]) == PROBE_CPU_Q8_OFFSET, "Probe::State::cpu.fprs[q8]'s offset matches ctiMasmProbeTrampoline"); 203 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q9]) == PROBE_CPU_Q9_OFFSET, "Probe::State::cpu.fprs[q9]'s offset matches ctiMasmProbeTrampoline"); 204 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q10]) == PROBE_CPU_Q10_OFFSET, "Probe::State::cpu.fprs[q10]'s offset matches ctiMasmProbeTrampoline"); 205 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q11]) == PROBE_CPU_Q11_OFFSET, "Probe::State::cpu.fprs[q11]'s offset matches ctiMasmProbeTrampoline"); 206 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q12]) == PROBE_CPU_Q12_OFFSET, "Probe::State::cpu.fprs[q12]'s offset matches ctiMasmProbeTrampoline"); 207 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q13]) == PROBE_CPU_Q13_OFFSET, "Probe::State::cpu.fprs[q13]'s offset matches ctiMasmProbeTrampoline"); 208 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q14]) == PROBE_CPU_Q14_OFFSET, "Probe::State::cpu.fprs[q14]'s offset matches ctiMasmProbeTrampoline"); 209 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q15]) == PROBE_CPU_Q15_OFFSET, "Probe::State::cpu.fprs[q15]'s offset matches ctiMasmProbeTrampoline"); 210 211 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q16]) == PROBE_CPU_Q16_OFFSET, "Probe::State::cpu.fprs[q16]'s offset matches ctiMasmProbeTrampoline"); 212 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q17]) == PROBE_CPU_Q17_OFFSET, "Probe::State::cpu.fprs[q17]'s offset matches ctiMasmProbeTrampoline"); 213 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q18]) == PROBE_CPU_Q18_OFFSET, "Probe::State::cpu.fprs[q18]'s offset matches ctiMasmProbeTrampoline"); 214 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q19]) == PROBE_CPU_Q19_OFFSET, "Probe::State::cpu.fprs[q19]'s offset matches ctiMasmProbeTrampoline"); 215 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q20]) == PROBE_CPU_Q20_OFFSET, "Probe::State::cpu.fprs[q20]'s offset matches ctiMasmProbeTrampoline"); 216 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q21]) == PROBE_CPU_Q21_OFFSET, "Probe::State::cpu.fprs[q21]'s offset matches ctiMasmProbeTrampoline"); 217 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q22]) == PROBE_CPU_Q22_OFFSET, "Probe::State::cpu.fprs[q22]'s offset matches ctiMasmProbeTrampoline"); 218 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q23]) == PROBE_CPU_Q23_OFFSET, "Probe::State::cpu.fprs[q23]'s offset matches ctiMasmProbeTrampoline"); 219 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q24]) == PROBE_CPU_Q24_OFFSET, "Probe::State::cpu.fprs[q24]'s offset matches ctiMasmProbeTrampoline"); 220 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q25]) == PROBE_CPU_Q25_OFFSET, "Probe::State::cpu.fprs[q25]'s offset matches ctiMasmProbeTrampoline"); 221 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q26]) == PROBE_CPU_Q26_OFFSET, "Probe::State::cpu.fprs[q26]'s offset matches ctiMasmProbeTrampoline"); 222 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q27]) == PROBE_CPU_Q27_OFFSET, "Probe::State::cpu.fprs[q27]'s offset matches ctiMasmProbeTrampoline"); 223 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q28]) == PROBE_CPU_Q28_OFFSET, "Probe::State::cpu.fprs[q28]'s offset matches ctiMasmProbeTrampoline"); 224 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q29]) == PROBE_CPU_Q29_OFFSET, "Probe::State::cpu.fprs[q29]'s offset matches ctiMasmProbeTrampoline"); 225 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q30]) == PROBE_CPU_Q30_OFFSET, "Probe::State::cpu.fprs[q30]'s offset matches ctiMasmProbeTrampoline"); 226 static_assert(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q31]) == PROBE_CPU_Q31_OFFSET, "Probe::State::cpu.fprs[q31]'s offset matches ctiMasmProbeTrampoline"); 227 228 static_assert(sizeof(Probe::State) == PROBE_SIZE, "Probe::State's size matches ctiMasmProbeTrampoline"); 229 230 // Conditions for using ldp and stp. 231 static_assert(PROBE_CPU_PC_OFFSET == PROBE_CPU_SP_OFFSET + GPREG_SIZE, "PROBE_CPU_SP_OFFSET and PROBE_CPU_PC_OFFSET must be adjacent"); 232 static_assert(!(PROBE_SIZE_PLUS_EXTRAS & 0xf), "PROBE_SIZE_PLUS_EXTRAS should be 16 byte aligned"); // the Probe::State copying code relies on this. 233 234 #undef PROBE_OFFSETOF 235 236 #define FPR_OFFSET(fpr) (PROBE_CPU_##fpr##_OFFSET - PROBE_CPU_Q0_OFFSET) 237 238 struct IncomingProbeRecord { 239 UCPURegister x24; 240 UCPURegister x25; 241 UCPURegister x26; 242 UCPURegister x27; 243 UCPURegister x28; 244 UCPURegister x30; // lr 245 }; 246 247 #define IN_X24_OFFSET (0 * GPREG_SIZE) 248 #define IN_X25_OFFSET (1 * GPREG_SIZE) 249 #define IN_X26_OFFSET (2 * GPREG_SIZE) 250 #define IN_X27_OFFSET (3 * GPREG_SIZE) 251 #define IN_X28_OFFSET (4 * GPREG_SIZE) 252 #define IN_X30_OFFSET (5 * GPREG_SIZE) 253 #define IN_SIZE (6 * GPREG_SIZE) 254 255 static_assert(IN_X24_OFFSET == offsetof(IncomingProbeRecord, x24), "IN_X24_OFFSET is incorrect"); 256 static_assert(IN_X25_OFFSET == offsetof(IncomingProbeRecord, x25), "IN_X25_OFFSET is incorrect"); 257 static_assert(IN_X26_OFFSET == offsetof(IncomingProbeRecord, x26), "IN_X26_OFFSET is incorrect"); 258 static_assert(IN_X27_OFFSET == offsetof(IncomingProbeRecord, x27), "IN_X27_OFFSET is incorrect"); 259 static_assert(IN_X28_OFFSET == offsetof(IncomingProbeRecord, x28), "IN_X22_OFFSET is incorrect"); 260 static_assert(IN_X30_OFFSET == offsetof(IncomingProbeRecord, x30), "IN_X23_OFFSET is incorrect"); 261 static_assert(IN_SIZE == sizeof(IncomingProbeRecord), "IN_SIZE is incorrect"); 262 static_assert(!(sizeof(IncomingProbeRecord) & 0xf), "IncomingProbeStack must be 16-byte aligned"); 263 264 struct OutgoingProbeRecord { 265 UCPURegister nzcv; 266 UCPURegister fpsr; 267 UCPURegister x27; 268 UCPURegister x28; 269 UCPURegister fp; 270 UCPURegister lr; 271 }; 272 273 #define OUT_NZCV_OFFSET (0 * GPREG_SIZE) 274 #define OUT_FPSR_OFFSET (1 * GPREG_SIZE) 275 #define OUT_X27_OFFSET (2 * GPREG_SIZE) 276 #define OUT_X28_OFFSET (3 * GPREG_SIZE) 277 #define OUT_FP_OFFSET (4 * GPREG_SIZE) 278 #define OUT_LR_OFFSET (5 * GPREG_SIZE) 279 #define OUT_SIZE (6 * GPREG_SIZE) 280 281 static_assert(OUT_NZCV_OFFSET == offsetof(OutgoingProbeRecord, nzcv), "OUT_NZCV_OFFSET is incorrect"); 282 static_assert(OUT_FPSR_OFFSET == offsetof(OutgoingProbeRecord, fpsr), "OUT_FPSR_OFFSET is incorrect"); 283 static_assert(OUT_X27_OFFSET == offsetof(OutgoingProbeRecord, x27), "OUT_X27_OFFSET is incorrect"); 284 static_assert(OUT_X28_OFFSET == offsetof(OutgoingProbeRecord, x28), "OUT_X28_OFFSET is incorrect"); 285 static_assert(OUT_FP_OFFSET == offsetof(OutgoingProbeRecord, fp), "OUT_FP_OFFSET is incorrect"); 286 static_assert(OUT_LR_OFFSET == offsetof(OutgoingProbeRecord, lr), "OUT_LR_OFFSET is incorrect"); 287 static_assert(OUT_SIZE == sizeof(OutgoingProbeRecord), "OUT_SIZE is incorrect"); 288 static_assert(!(sizeof(OutgoingProbeRecord) & 0xf), "OutgoingProbeStack must be 16-byte aligned"); 289 290 struct LRRestorationRecord { 291 UCPURegister lr; 292 UCPURegister unusedDummyToEnsureSizeIs16ByteAligned; 293 }; 294 295 #define LR_RESTORATION_LR_OFFSET (0 * GPREG_SIZE) 296 #define LR_RESTORATION_SIZE (2 * GPREG_SIZE) 297 298 static_assert(LR_RESTORATION_LR_OFFSET == offsetof(LRRestorationRecord, lr), "LR_RESTORATION_LR_OFFSET is incorrect"); 299 static_assert(LR_RESTORATION_SIZE == sizeof(LRRestorationRecord), "LR_RESTORATION_SIZE is incorrect"); 300 static_assert(!(sizeof(LRRestorationRecord) & 0xf), "LRRestorationRecord must be 16-byte aligned"); 301 302 #if CPU(ARM64E) 303 #define JIT_PROBE_PC_PTR_TAG 0xeeac 304 #define JIT_PROBE_EXECUTOR_PTR_TAG 0x28de 305 #define JIT_PROBE_STACK_INITIALIZATION_FUNCTION_PTR_TAG 0x315c 306 static_assert(JIT_PROBE_PC_PTR_TAG == JITProbePCPtrTag); 307 static_assert(JIT_PROBE_EXECUTOR_PTR_TAG == JITProbeExecutorPtrTag); 308 static_assert(JIT_PROBE_STACK_INITIALIZATION_FUNCTION_PTR_TAG == JITProbeStackInitializationFunctionPtrTag); 309 #endif 310 311 // We use x29 and x30 instead of fp and lr because GCC's inline assembler does not recognize fp and lr. 312 // See https://bugs.webkit.org/show_bug.cgi?id=175512 for details. 313 asm ( 314 ".text" "\n" 315 ".balign 16" "\n" 316 ".globl " SYMBOL_STRING(ctiMasmProbeTrampoline) "\n" 317 HIDE_SYMBOL(ctiMasmProbeTrampoline) "\n" 318 SYMBOL_STRING(ctiMasmProbeTrampoline) ":" "\n" 319 320 // MacroAssemblerARM64::probe() has already generated code to store some values in an 321 // IncomingProbeRecord. sp points to the IncomingProbeRecord. 322 // 323 // Incoming register values: 324 // x24: probe function 325 // x25: probe arg 326 // x26: scratch, was ctiMasmProbeTrampoline 327 // x27: scratch 328 // x28: Probe::executeProbe 329 // x30: return address 330 331 "mov x26, sp" "\n" 332 "mov x27, sp" "\n" 333 334 "sub x27, x27, #" STRINGIZE_VALUE_OF(PROBE_SIZE_PLUS_EXTRAS + OUT_SIZE) "\n" 335 "bic x27, x27, #0xf" "\n" // The ARM EABI specifies that the stack needs to be 16 byte aligned. 336 "mov sp, x27" "\n" // Set the sp to protect the Probe::State from interrupts before we initialize it. 337 338 "stp x24, x25, [sp, #" STRINGIZE_VALUE_OF(PROBE_PROBE_FUNCTION_OFFSET) "]" "\n" // Store the probe handler function and arg (preloaded into x24 and x25 339 340 "stp x0, x1, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X0_OFFSET) "]" "\n" 341 "mrs x0, nzcv" "\n" // Preload nzcv. 342 "stp x2, x3, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X2_OFFSET) "]" "\n" 343 "stp x4, x5, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X4_OFFSET) "]" "\n" 344 "mrs x1, fpsr" "\n" // Preload fpsr. 345 "stp x6, x7, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X6_OFFSET) "]" "\n" 346 "stp x8, x9, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X8_OFFSET) "]" "\n" 347 348 "ldp x2, x3, [x26, #" STRINGIZE_VALUE_OF(IN_X24_OFFSET) "]" "\n" // Preload saved x24 and x25. 349 "ldp x4, x5, [x26, #" STRINGIZE_VALUE_OF(IN_X26_OFFSET) "]" "\n" // Preload saved x26 and x27. 350 "ldp x6, x7, [x26, #" STRINGIZE_VALUE_OF(IN_X28_OFFSET) "]" "\n" // Preload saved x28 and lr. 351 "add x26, x26, #" STRINGIZE_VALUE_OF(IN_SIZE) "\n" // Compute the sp before the probe. 352 353 "stp x10, x11, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X10_OFFSET) "]" "\n" 354 "stp x12, x13, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X12_OFFSET) "]" "\n" 355 "stp x14, x15, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X14_OFFSET) "]" "\n" 356 "stp x16, x17, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X16_OFFSET) "]" "\n" 357 "stp x18, x19, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X18_OFFSET) "]" "\n" 358 "stp x20, x21, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X20_OFFSET) "]" "\n" 359 "stp x22, x23, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X22_OFFSET) "]" "\n" 360 "stp x2, x3, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X24_OFFSET) "]" "\n" // Store saved r24 and r25 (preloaded into x2 and x3 above). 361 "stp x4, x5, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X26_OFFSET) "]" "\n" // Store saved r26 and r27 (preloaded into x4 and x5 above). 362 "stp x6, x29, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X28_OFFSET) "]" "\n" 363 "stp x7, x26, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_LR_OFFSET) "]" "\n" // Save values lr and sp (original sp value computed into x26 above). 364 365 "add x30, x30, #" STRINGIZE_VALUE_OF(2 * GPREG_SIZE) "\n" // The PC after the probe is at 2 instructions past the return point. 366 #if CPU(ARM64E) 367 "movz x27, #" STRINGIZE_VALUE_OF(JIT_PROBE_PC_PTR_TAG) "\n" 368 "pacib x30, x27" "\n" 369 #endif 370 "str x30, [sp, #" STRINGIZE_VALUE_OF(SAVED_PROBE_RETURN_PC_OFFSET) "]" "\n" // Save a duplicate copy of return pc (in lr). 371 "str x30, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_PC_OFFSET) "]" "\n" 372 373 "stp x0, x1, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_NZCV_OFFSET) "]" "\n" // Store nzcv and fpsr (preloaded into x0 and x1 above). 374 375 "add x9, sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q0_OFFSET) "\n" 376 "stp d0, d1, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q0)) "]" "\n" 377 "stp d2, d3, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q2)) "]" "\n" 378 "stp d4, d5, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q4)) "]" "\n" 379 "stp d6, d7, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q6)) "]" "\n" 380 "stp d8, d9, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q8)) "]" "\n" 381 "stp d10, d11, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q10)) "]" "\n" 382 "stp d12, d13, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q12)) "]" "\n" 383 "stp d14, d15, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q14)) "]" "\n" 384 "stp d16, d17, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q16)) "]" "\n" 385 "stp d18, d19, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q18)) "]" "\n" 386 "stp d20, d21, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q20)) "]" "\n" 387 "stp d22, d23, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q22)) "]" "\n" 388 "stp d24, d25, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q24)) "]" "\n" 389 "stp d26, d27, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q26)) "]" "\n" 390 "stp d28, d29, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q28)) "]" "\n" 391 "stp d30, d31, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q30)) "]" "\n" 392 393 "mov x27, sp" "\n" // Save the Probe::State* in a callee saved register. 394 395 // Note: we haven't changed the value of fp. Hence, it is still pointing to the frame of 396 // the caller of the probe (which is what we want in order to play nice with debuggers e.g. lldb). 397 "mov x0, sp" "\n" // Set the Probe::State* arg. 398 #if CPU(ARM64E) 399 "movz lr, #" STRINGIZE_VALUE_OF(JIT_PROBE_EXECUTOR_PTR_TAG) "\n" 400 "blrab x28, lr" "\n" // Call the probe handler. 401 #else 402 "blr x28" "\n" // Call the probe handler. 403 #endif 404 405 // Make sure the Probe::State is entirely below the result stack pointer so 406 // that register values are still preserved when we call the initializeStack 407 // function. 408 "ldr x1, [x27, #" STRINGIZE_VALUE_OF(PROBE_CPU_SP_OFFSET) "]" "\n" // Result sp. 409 "add x2, x27, #" STRINGIZE_VALUE_OF(PROBE_SIZE_PLUS_EXTRAS + OUT_SIZE) "\n" // End of Probe::State + buffer. 410 "cmp x1, x2" "\n" 411 "bge " LOCAL_LABEL_STRING(ctiMasmProbeTrampolineProbeStateIsSafe) "\n" 412 413 // Allocate a safe place on the stack below the result stack pointer to stash the Probe::State. 414 "sub x1, x1, #" STRINGIZE_VALUE_OF(PROBE_SIZE_PLUS_EXTRAS + OUT_SIZE) "\n" 415 "bic x1, x1, #0xf" "\n" // The ARM EABI specifies that the stack needs to be 16 byte aligned. 416 "mov sp, x1" "\n" // Set the new sp to protect that memory from interrupts before we copy the Probe::State. 417 418 // Copy the Probe::State to the safe place. 419 // Note: we have to copy from low address to higher address because we're moving the 420 // Probe::State to a lower address. 421 "mov x5, x27" "\n" 422 "mov x6, x1" "\n" 423 "add x7, x27, #" STRINGIZE_VALUE_OF(PROBE_SIZE_PLUS_EXTRAS) "\n" 424 425 LOCAL_LABEL_STRING(ctiMasmProbeTrampolineCopyLoop) ":" "\n" 426 "ldp x3, x4, [x5], #16" "\n" 427 "stp x3, x4, [x6], #16" "\n" 428 "cmp x5, x7" "\n" 429 "blt " LOCAL_LABEL_STRING(ctiMasmProbeTrampolineCopyLoop) "\n" 430 431 "mov x27, x1" "\n" 432 433 // Call initializeStackFunction if present. 434 LOCAL_LABEL_STRING(ctiMasmProbeTrampolineProbeStateIsSafe) ":" "\n" 435 "ldr x2, [x27, #" STRINGIZE_VALUE_OF(PROBE_INIT_STACK_FUNCTION_OFFSET) "]" "\n" 436 "cbz x2, " LOCAL_LABEL_STRING(ctiMasmProbeTrampolineRestoreRegisters) "\n" 437 438 "mov x0, x27" "\n" // Set the Probe::State* arg. 439 #if CPU(ARM64E) 440 "movz lr, #" STRINGIZE_VALUE_OF(JIT_PROBE_STACK_INITIALIZATION_FUNCTION_PTR_TAG) "\n" 441 "blrab x2, lr" "\n" // Call the initializeStackFunction (loaded into x2 above).all the probe handler. 442 #else 443 "blr x2" "\n" // Call the initializeStackFunction (loaded into x2 above). 444 #endif 445 446 LOCAL_LABEL_STRING(ctiMasmProbeTrampolineRestoreRegisters) ":" "\n" 447 448 "mov sp, x27" "\n" 449 450 // To enable probes to modify register state, we copy all registers 451 // out of the Probe::State before returning. That is except for x18. 452 // x18 is "reserved for the platform. Conforming software should not make use of it." 453 // Hence, the JITs would not be using it, and the probe should also not be modifying it. 454 // See https://developer.apple.com/library/ios/documentation/Xcode/Conceptual/iPhoneOSABIReference/Articles/ARM64FunctionCallingConventions.html. 455 456 "add x9, sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q0_OFFSET) "\n" 457 "ldp d0, d1, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q0)) "]" "\n" 458 "ldp d2, d3, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q2)) "]" "\n" 459 "ldp d4, d5, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q4)) "]" "\n" 460 "ldp d6, d7, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q6)) "]" "\n" 461 "ldp d8, d9, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q8)) "]" "\n" 462 "ldp d10, d11, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q10)) "]" "\n" 463 "ldp d12, d13, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q12)) "]" "\n" 464 "ldp d14, d15, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q14)) "]" "\n" 465 "ldp d16, d17, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q16)) "]" "\n" 466 "ldp d18, d19, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q18)) "]" "\n" 467 "ldp d20, d21, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q20)) "]" "\n" 468 "ldp d22, d23, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q22)) "]" "\n" 469 "ldp d24, d25, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q24)) "]" "\n" 470 "ldp d26, d27, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q26)) "]" "\n" 471 "ldp d28, d29, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q28)) "]" "\n" 472 "ldp d30, d31, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q30)) "]" "\n" 473 474 "ldp x0, x1, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X0_OFFSET) "]" "\n" 475 "ldp x2, x3, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X2_OFFSET) "]" "\n" 476 "ldp x4, x5, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X4_OFFSET) "]" "\n" 477 "ldp x6, x7, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X6_OFFSET) "]" "\n" 478 "ldp x8, x9, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X8_OFFSET) "]" "\n" 479 "ldp x10, x11, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X10_OFFSET) "]" "\n" 480 "ldp x12, x13, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X12_OFFSET) "]" "\n" 481 "ldp x14, x15, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X14_OFFSET) "]" "\n" 482 "ldp x16, x17, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X16_OFFSET) "]" "\n" 483 // x18 should not be modified by the probe. See comment above for details. 484 "ldp x19, x20, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X19_OFFSET) "]" "\n" 485 "ldp x21, x22, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X21_OFFSET) "]" "\n" 486 "ldp x23, x24, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X23_OFFSET) "]" "\n" 487 "ldp x25, x26, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X25_OFFSET) "]" "\n" 488 489 // Remaining registers to restore are: fpsr, nzcv, x27, x28, fp, lr, sp, and pc. 490 491 // The only way to set the pc on ARM64 (from user space) is via an indirect branch 492 // or a ret, which means we'll need a free register to do so. For our purposes, lr 493 // happens to be available in applications of the probe where we may want to 494 // continue executing at a different location (i.e. change the pc) after the probe 495 // returns. So, the ARM64 probe implementation will allow the probe handler to 496 // either modify lr or pc, but not both in the same probe invocation. The probe 497 // mechanism ensures that we never try to modify both lr and pc with a RELEASE_ASSERT 498 // in Probe::executeProbe(). 499 500 // Determine if the probe handler changed the pc. 501 "ldr x30, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_SP_OFFSET) "]" "\n" // preload the target sp. 502 "ldr x27, [sp, #" STRINGIZE_VALUE_OF(SAVED_PROBE_RETURN_PC_OFFSET) "]" "\n" 503 "ldr x28, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_PC_OFFSET) "]" "\n" 504 "cmp x27, x28" "\n" 505 "bne " LOCAL_LABEL_STRING(ctiMasmProbeTrampolineEnd) "\n" 506 507 // We didn't change the PC. So, let's prepare for setting a potentially new lr value. 508 509 // 1. Make room for the LRRestorationRecord. The probe site will pop this off later. 510 "sub x30, x30, #" STRINGIZE_VALUE_OF(LR_RESTORATION_SIZE) "\n" 511 // 2. Store the lp value to restore at the probe return site. 512 "ldr x27, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_LR_OFFSET) "]" "\n" 513 "str x27, [x30, #" STRINGIZE_VALUE_OF(LR_RESTORATION_LR_OFFSET) "]" "\n" 514 // 3. Force the return ramp to return to the probe return site. 515 "ldr x27, [sp, #" STRINGIZE_VALUE_OF(SAVED_PROBE_RETURN_PC_OFFSET) "]" "\n" 516 #if CPU(ARM64E) 517 "movz x28, #" STRINGIZE_VALUE_OF(JIT_PROBE_PC_PTR_TAG) "\n" 518 "autib x27, x28" "\n" 519 "lsr x28, x27, #8" "\n" 520 "and x28, x28, #0xff000000000000" "\n" 521 "orr x28, x28, x27" "\n" 522 "ldrb w28, [x28]" "\n" 523 #endif 524 "sub x27, x27, #" STRINGIZE_VALUE_OF(2 * GPREG_SIZE) "\n" // The return point PC is at 2 instructions before the end of the probe. 525 #if CPU(ARM64E) 526 "movz x28, #" STRINGIZE_VALUE_OF(JIT_PROBE_PC_PTR_TAG) "\n" 527 "pacib x27, x28" "\n" 528 #endif 529 "str x27, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_PC_OFFSET) "]" "\n" 530 531 LOCAL_LABEL_STRING(ctiMasmProbeTrampolineEnd) ":" "\n" 532 533 // Fill in the OutgoingProbeRecord. 534 "sub x30, x30, #" STRINGIZE_VALUE_OF(OUT_SIZE) "\n" 535 536 "ldp x27, x28, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_NZCV_OFFSET) "]" "\n" 537 "stp x27, x28, [x30, #" STRINGIZE_VALUE_OF(OUT_NZCV_OFFSET) "]" "\n" 538 "ldp x27, x28, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X27_OFFSET) "]" "\n" 539 "stp x27, x28, [x30, #" STRINGIZE_VALUE_OF(OUT_X27_OFFSET) "]" "\n" 540 "ldr x28, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_PC_OFFSET) "]" "\n" // Set up the outgoing record so that we'll jump to the new PC. 541 #if CPU(ARM64E) 542 "movz x27, #" STRINGIZE_VALUE_OF(JIT_PROBE_PC_PTR_TAG) "\n" 543 "autib x28, x27" "\n" 544 "lsr x27, x28, #8" "\n" 545 "and x27, x27, #0xff000000000000" "\n" 546 "orr x27, x27, x28" "\n" 547 "ldrb w27, [x27]" "\n" 548 "add x27, x30, #48" "\n" // Compute sp at return point. 549 "pacib x28, x27" "\n" 550 #endif 551 "ldr x27, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_FP_OFFSET) "]" "\n" 552 "stp x27, x28, [x30, #" STRINGIZE_VALUE_OF(OUT_FP_OFFSET) "]" "\n" 553 "mov sp, x30" "\n" 554 555 // Restore the remaining registers and pop the OutgoingProbeRecord. 556 "ldp x27, x28, [sp], #" STRINGIZE_VALUE_OF(2 * GPREG_SIZE) "\n" 557 "msr nzcv, x27" "\n" 558 "msr fpsr, x28" "\n" 559 "ldp x27, x28, [sp], #" STRINGIZE_VALUE_OF(2 * GPREG_SIZE) "\n" 560 "ldp x29, x30, [sp], #" STRINGIZE_VALUE_OF(2 * GPREG_SIZE) "\n" 561 #if CPU(ARM64E) 562 "retab" "\n" 563 #else 564 "ret" "\n" 565 #endif 566 ); 567 #endif // COMPILER(GCC_COMPATIBLE) 568 569 void MacroAssembler::probe(Probe::Function function, void* arg) 570 { 571 sub64(TrustedImm32(sizeof(IncomingProbeRecord)), sp); 572 573 storePair64(x24, x25, sp, TrustedImm32(offsetof(IncomingProbeRecord, x24))); 574 storePair64(x26, x27, sp, TrustedImm32(offsetof(IncomingProbeRecord, x26))); 575 storePair64(x28, x30, sp, TrustedImm32(offsetof(IncomingProbeRecord, x28))); // Note: x30 is lr. 576 move(TrustedImmPtr(tagCFunction<OperationPtrTag>(ctiMasmProbeTrampoline)), x26); 577 move(TrustedImmPtr(tagCFunction<JITProbeExecutorPtrTag>(Probe::executeProbe)), x28); 578 #if CPU(ARM64E) 579 assertIsTaggedWith<JITProbePtrTag>(function); 580 #endif 581 move(TrustedImmPtr(reinterpret_cast<void*>(function)), x24); 582 move(TrustedImmPtr(arg), x25); 583 call(x26, OperationPtrTag); 584 585 // ctiMasmProbeTrampoline should have restored every register except for lr and the sp. 586 load64(Address(sp, offsetof(LRRestorationRecord, lr)), lr); 587 add64(TrustedImm32(sizeof(LRRestorationRecord)), sp); 588 } 589 590 #endif // ENABLE(MASM_PROBE) 591 592 void MacroAssemblerARM64::collectCPUFeatures() 593 { 594 #if OS(LINUX) 595 static std::once_flag onceKey; 596 std::call_once(onceKey, [] { 597 // A register for describing ARM64 CPU features are only accessible in kernel mode. 598 // Thus, some kernel support is necessary to collect CPU features. In Linux, the 599 // kernel passes CPU feature flags in AT_HWCAP auxiliary vector which is passed 600 // when the process starts. While this may pose a bit conservative information 601 // (for example, the Linux kernel may add a flag for a feature after the feature 602 // is shipped and implemented in some CPUs. In that case, even if the CPU has 603 // that feature, the kernel does not tell it to users.), it is a stable approach. 604 // https://www.kernel.org/doc/Documentation/arm64/elf_hwcaps.txt 605 uint64_t hwcaps = getauxval(AT_HWCAP); 606 607 #if !defined(HWCAP_JSCVT) 608 #define HWCAP_JSCVT (1 << 13) 609 #endif 610 611 s_jscvtCheckState = (hwcaps & HWCAP_JSCVT) ? CPUIDCheckState::Set : CPUIDCheckState::Clear; 612 }); 613 #elif HAVE(FJCVTZS_INSTRUCTION) 614 s_jscvtCheckState = CPUIDCheckState::Set; 615 #else 616 s_jscvtCheckState = CPUIDCheckState::Clear; 617 #endif 618 } 619 620 MacroAssemblerARM64::CPUIDCheckState MacroAssemblerARM64::s_jscvtCheckState = CPUIDCheckState::NotChecked; 621 622 } // namespace JSC 623 624 #endif // ENABLE(ASSEMBLER) && CPU(ARM64)