InstEmitMemoryHelper.cs
1 using ARMeilleure.Decoders; 2 using ARMeilleure.IntermediateRepresentation; 3 using ARMeilleure.Memory; 4 using ARMeilleure.Translation; 5 using ARMeilleure.Translation.PTC; 6 using System; 7 using System.Reflection; 8 using static ARMeilleure.Instructions.InstEmitHelper; 9 using static ARMeilleure.IntermediateRepresentation.Operand.Factory; 10 11 namespace ARMeilleure.Instructions 12 { 13 static class InstEmitMemoryHelper 14 { 15 private const int PageBits = 12; 16 private const int PageMask = (1 << PageBits) - 1; 17 18 private enum Extension 19 { 20 Zx, 21 Sx32, 22 Sx64, 23 } 24 25 public static void EmitLoadZx(ArmEmitterContext context, Operand address, int rt, int size) 26 { 27 EmitLoad(context, address, Extension.Zx, rt, size); 28 } 29 30 public static void EmitLoadSx32(ArmEmitterContext context, Operand address, int rt, int size) 31 { 32 EmitLoad(context, address, Extension.Sx32, rt, size); 33 } 34 35 public static void EmitLoadSx64(ArmEmitterContext context, Operand address, int rt, int size) 36 { 37 EmitLoad(context, address, Extension.Sx64, rt, size); 38 } 39 40 private static void EmitLoad(ArmEmitterContext context, Operand address, Extension ext, int rt, int size) 41 { 42 bool isSimd = IsSimd(context); 43 44 if ((uint)size > (isSimd ? 4 : 3)) 45 { 46 throw new ArgumentOutOfRangeException(nameof(size)); 47 } 48 49 if (isSimd) 50 { 51 EmitReadVector(context, address, context.VectorZero(), rt, 0, size); 52 } 53 else 54 { 55 EmitReadInt(context, address, rt, size); 56 } 57 58 if (!isSimd && !(context.CurrOp is OpCode32 && rt == State.RegisterAlias.Aarch32Pc)) 59 { 60 Operand value = GetInt(context, rt); 61 62 if (ext == Extension.Sx32 || ext == Extension.Sx64) 63 { 64 OperandType destType = ext == Extension.Sx64 ? OperandType.I64 : OperandType.I32; 65 66 switch (size) 67 { 68 case 0: 69 value = context.SignExtend8(destType, value); 70 break; 71 case 1: 72 value = context.SignExtend16(destType, value); 73 break; 74 case 2: 75 value = context.SignExtend32(destType, value); 76 break; 77 } 78 } 79 80 SetInt(context, rt, value); 81 } 82 } 83 84 public static void EmitLoadSimd( 85 ArmEmitterContext context, 86 Operand address, 87 Operand vector, 88 int rt, 89 int elem, 90 int size) 91 { 92 EmitReadVector(context, address, vector, rt, elem, size); 93 } 94 95 public static void EmitStore(ArmEmitterContext context, Operand address, int rt, int size) 96 { 97 bool isSimd = IsSimd(context); 98 99 if ((uint)size > (isSimd ? 4 : 3)) 100 { 101 throw new ArgumentOutOfRangeException(nameof(size)); 102 } 103 104 if (isSimd) 105 { 106 EmitWriteVector(context, address, rt, 0, size); 107 } 108 else 109 { 110 EmitWriteInt(context, address, rt, size); 111 } 112 } 113 114 public static void EmitStoreSimd( 115 ArmEmitterContext context, 116 Operand address, 117 int rt, 118 int elem, 119 int size) 120 { 121 EmitWriteVector(context, address, rt, elem, size); 122 } 123 124 private static bool IsSimd(ArmEmitterContext context) 125 { 126 return context.CurrOp is IOpCodeSimd && 127 !(context.CurrOp is OpCodeSimdMemMs || 128 context.CurrOp is OpCodeSimdMemSs); 129 } 130 131 public static Operand EmitReadInt(ArmEmitterContext context, Operand address, int size) 132 { 133 Operand temp = context.AllocateLocal(size == 3 ? OperandType.I64 : OperandType.I32); 134 135 Operand lblSlowPath = Label(); 136 Operand lblEnd = Label(); 137 138 Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: false, size); 139 140 Operand value = default; 141 142 switch (size) 143 { 144 case 0: 145 value = context.Load8(physAddr); 146 break; 147 case 1: 148 value = context.Load16(physAddr); 149 break; 150 case 2: 151 value = context.Load(OperandType.I32, physAddr); 152 break; 153 case 3: 154 value = context.Load(OperandType.I64, physAddr); 155 break; 156 } 157 158 context.Copy(temp, value); 159 160 if (!context.Memory.Type.IsHostMappedOrTracked()) 161 { 162 context.Branch(lblEnd); 163 164 context.MarkLabel(lblSlowPath, BasicBlockFrequency.Cold); 165 166 context.Copy(temp, EmitReadIntFallback(context, address, size)); 167 168 context.MarkLabel(lblEnd); 169 } 170 171 return temp; 172 } 173 174 private static void EmitReadInt(ArmEmitterContext context, Operand address, int rt, int size) 175 { 176 Operand lblSlowPath = Label(); 177 Operand lblEnd = Label(); 178 179 Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: false, size); 180 181 Operand value = default; 182 183 switch (size) 184 { 185 case 0: 186 value = context.Load8(physAddr); 187 break; 188 case 1: 189 value = context.Load16(physAddr); 190 break; 191 case 2: 192 value = context.Load(OperandType.I32, physAddr); 193 break; 194 case 3: 195 value = context.Load(OperandType.I64, physAddr); 196 break; 197 } 198 199 SetInt(context, rt, value); 200 201 if (!context.Memory.Type.IsHostMappedOrTracked()) 202 { 203 context.Branch(lblEnd); 204 205 context.MarkLabel(lblSlowPath, BasicBlockFrequency.Cold); 206 207 EmitReadIntFallback(context, address, rt, size); 208 209 context.MarkLabel(lblEnd); 210 } 211 } 212 213 public static Operand EmitReadIntAligned(ArmEmitterContext context, Operand address, int size) 214 { 215 if ((uint)size > 4) 216 { 217 throw new ArgumentOutOfRangeException(nameof(size)); 218 } 219 220 Operand physAddr = EmitPtPointerLoad(context, address, default, write: false, size); 221 222 return size switch 223 { 224 0 => context.Load8(physAddr), 225 1 => context.Load16(physAddr), 226 2 => context.Load(OperandType.I32, physAddr), 227 3 => context.Load(OperandType.I64, physAddr), 228 _ => context.Load(OperandType.V128, physAddr), 229 }; 230 } 231 232 private static void EmitReadVector( 233 ArmEmitterContext context, 234 Operand address, 235 Operand vector, 236 int rt, 237 int elem, 238 int size) 239 { 240 Operand lblSlowPath = Label(); 241 Operand lblEnd = Label(); 242 243 Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: false, size); 244 245 Operand value = default; 246 247 switch (size) 248 { 249 case 0: 250 value = context.VectorInsert8(vector, context.Load8(physAddr), elem); 251 break; 252 case 1: 253 value = context.VectorInsert16(vector, context.Load16(physAddr), elem); 254 break; 255 case 2: 256 value = context.VectorInsert(vector, context.Load(OperandType.I32, physAddr), elem); 257 break; 258 case 3: 259 value = context.VectorInsert(vector, context.Load(OperandType.I64, physAddr), elem); 260 break; 261 case 4: 262 value = context.Load(OperandType.V128, physAddr); 263 break; 264 } 265 266 context.Copy(GetVec(rt), value); 267 268 if (!context.Memory.Type.IsHostMappedOrTracked()) 269 { 270 context.Branch(lblEnd); 271 272 context.MarkLabel(lblSlowPath, BasicBlockFrequency.Cold); 273 274 EmitReadVectorFallback(context, address, vector, rt, elem, size); 275 276 context.MarkLabel(lblEnd); 277 } 278 } 279 280 private static Operand VectorCreate(ArmEmitterContext context, Operand value) 281 { 282 return context.VectorInsert(context.VectorZero(), value, 0); 283 } 284 285 private static void EmitWriteInt(ArmEmitterContext context, Operand address, int rt, int size) 286 { 287 Operand lblSlowPath = Label(); 288 Operand lblEnd = Label(); 289 290 Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: true, size); 291 292 Operand value = GetInt(context, rt); 293 294 if (size < 3 && value.Type == OperandType.I64) 295 { 296 value = context.ConvertI64ToI32(value); 297 } 298 299 switch (size) 300 { 301 case 0: 302 context.Store8(physAddr, value); 303 break; 304 case 1: 305 context.Store16(physAddr, value); 306 break; 307 case 2: 308 context.Store(physAddr, value); 309 break; 310 case 3: 311 context.Store(physAddr, value); 312 break; 313 } 314 315 if (!context.Memory.Type.IsHostMappedOrTracked()) 316 { 317 context.Branch(lblEnd); 318 319 context.MarkLabel(lblSlowPath, BasicBlockFrequency.Cold); 320 321 EmitWriteIntFallback(context, address, rt, size); 322 323 context.MarkLabel(lblEnd); 324 } 325 } 326 327 public static void EmitWriteIntAligned(ArmEmitterContext context, Operand address, Operand value, int size) 328 { 329 if ((uint)size > 4) 330 { 331 throw new ArgumentOutOfRangeException(nameof(size)); 332 } 333 334 Operand physAddr = EmitPtPointerLoad(context, address, default, write: true, size); 335 336 if (size < 3 && value.Type == OperandType.I64) 337 { 338 value = context.ConvertI64ToI32(value); 339 } 340 341 if (size == 0) 342 { 343 context.Store8(physAddr, value); 344 } 345 else if (size == 1) 346 { 347 context.Store16(physAddr, value); 348 } 349 else 350 { 351 context.Store(physAddr, value); 352 } 353 } 354 355 private static void EmitWriteVector( 356 ArmEmitterContext context, 357 Operand address, 358 int rt, 359 int elem, 360 int size) 361 { 362 Operand lblSlowPath = Label(); 363 Operand lblEnd = Label(); 364 365 Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: true, size); 366 367 Operand value = GetVec(rt); 368 369 switch (size) 370 { 371 case 0: 372 context.Store8(physAddr, context.VectorExtract8(value, elem)); 373 break; 374 case 1: 375 context.Store16(physAddr, context.VectorExtract16(value, elem)); 376 break; 377 case 2: 378 context.Store(physAddr, context.VectorExtract(OperandType.I32, value, elem)); 379 break; 380 case 3: 381 context.Store(physAddr, context.VectorExtract(OperandType.I64, value, elem)); 382 break; 383 case 4: 384 context.Store(physAddr, value); 385 break; 386 } 387 388 if (!context.Memory.Type.IsHostMappedOrTracked()) 389 { 390 context.Branch(lblEnd); 391 392 context.MarkLabel(lblSlowPath, BasicBlockFrequency.Cold); 393 394 EmitWriteVectorFallback(context, address, rt, elem, size); 395 396 context.MarkLabel(lblEnd); 397 } 398 } 399 400 public static Operand EmitPtPointerLoad(ArmEmitterContext context, Operand address, Operand lblSlowPath, bool write, int size) 401 { 402 if (context.Memory.Type.IsHostMapped()) 403 { 404 return EmitHostMappedPointer(context, address); 405 } 406 else if (context.Memory.Type.IsHostTracked()) 407 { 408 if (address.Type == OperandType.I32) 409 { 410 address = context.ZeroExtend32(OperandType.I64, address); 411 } 412 413 if (context.Memory.Type == MemoryManagerType.HostTracked) 414 { 415 Operand mask = Const(ulong.MaxValue >> (64 - context.Memory.AddressSpaceBits)); 416 address = context.BitwiseAnd(address, mask); 417 } 418 419 Operand ptBase = !context.HasPtc 420 ? Const(context.Memory.PageTablePointer.ToInt64()) 421 : Const(context.Memory.PageTablePointer.ToInt64(), Ptc.PageTableSymbol); 422 423 Operand ptOffset = context.ShiftRightUI(address, Const(PageBits)); 424 425 return context.Add(address, context.Load(OperandType.I64, context.Add(ptBase, context.ShiftLeft(ptOffset, Const(3))))); 426 } 427 428 int ptLevelBits = context.Memory.AddressSpaceBits - PageBits; 429 int ptLevelSize = 1 << ptLevelBits; 430 int ptLevelMask = ptLevelSize - 1; 431 432 Operand addrRotated = size != 0 ? context.RotateRight(address, Const(size)) : address; 433 Operand addrShifted = context.ShiftRightUI(addrRotated, Const(PageBits - size)); 434 435 Operand pte = !context.HasPtc 436 ? Const(context.Memory.PageTablePointer.ToInt64()) 437 : Const(context.Memory.PageTablePointer.ToInt64(), Ptc.PageTableSymbol); 438 439 Operand pteOffset = context.BitwiseAnd(addrShifted, Const(addrShifted.Type, ptLevelMask)); 440 441 if (pteOffset.Type == OperandType.I32) 442 { 443 pteOffset = context.ZeroExtend32(OperandType.I64, pteOffset); 444 } 445 446 pte = context.Load(OperandType.I64, context.Add(pte, context.ShiftLeft(pteOffset, Const(3)))); 447 448 if (addrShifted.Type == OperandType.I32) 449 { 450 addrShifted = context.ZeroExtend32(OperandType.I64, addrShifted); 451 } 452 453 // If the VA is out of range, or not aligned to the access size, force PTE to 0 by masking it. 454 pte = context.BitwiseAnd(pte, context.ShiftRightSI(context.Add(addrShifted, Const(-(long)ptLevelSize)), Const(63))); 455 456 if (lblSlowPath != default) 457 { 458 if (write) 459 { 460 context.BranchIf(lblSlowPath, pte, Const(0L), Comparison.LessOrEqual); 461 pte = context.BitwiseAnd(pte, Const(0xffffffffffffUL)); // Ignore any software protection bits. (they are still used by C# memory access) 462 } 463 else 464 { 465 pte = context.ShiftLeft(pte, Const(1)); 466 context.BranchIf(lblSlowPath, pte, Const(0L), Comparison.LessOrEqual); 467 pte = context.ShiftRightUI(pte, Const(1)); 468 } 469 } 470 else 471 { 472 // When no label is provided to jump to a slow path if the address is invalid, 473 // we do the validation ourselves, and throw if needed. 474 475 Operand lblNotWatched = Label(); 476 477 // Is the page currently being tracked for read/write? If so we need to call SignalMemoryTracking. 478 context.BranchIf(lblNotWatched, pte, Const(0L), Comparison.GreaterOrEqual, BasicBlockFrequency.Cold); 479 480 // Signal memory tracking. Size here doesn't matter as address is assumed to be size aligned here. 481 context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.SignalMemoryTracking)), address, Const(1UL), Const(write ? 1 : 0)); 482 context.MarkLabel(lblNotWatched); 483 484 pte = context.BitwiseAnd(pte, Const(0xffffffffffffUL)); // Ignore any software protection bits. (they are still used by C# memory access) 485 486 Operand lblNonNull = Label(); 487 488 // Skip exception if the PTE address is non-null (not zero). 489 context.BranchIfTrue(lblNonNull, pte, BasicBlockFrequency.Cold); 490 491 // The call is not expected to return (it should throw). 492 context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ThrowInvalidMemoryAccess)), address); 493 context.MarkLabel(lblNonNull); 494 } 495 496 Operand pageOffset = context.BitwiseAnd(address, Const(address.Type, PageMask)); 497 498 if (pageOffset.Type == OperandType.I32) 499 { 500 pageOffset = context.ZeroExtend32(OperandType.I64, pageOffset); 501 } 502 503 return context.Add(pte, pageOffset); 504 } 505 506 public static Operand EmitHostMappedPointer(ArmEmitterContext context, Operand address) 507 { 508 if (address.Type == OperandType.I32) 509 { 510 address = context.ZeroExtend32(OperandType.I64, address); 511 } 512 513 if (context.Memory.Type == MemoryManagerType.HostMapped) 514 { 515 Operand mask = Const(ulong.MaxValue >> (64 - context.Memory.AddressSpaceBits)); 516 address = context.BitwiseAnd(address, mask); 517 } 518 519 Operand baseAddr = !context.HasPtc 520 ? Const(context.Memory.PageTablePointer.ToInt64()) 521 : Const(context.Memory.PageTablePointer.ToInt64(), Ptc.PageTableSymbol); 522 523 return context.Add(baseAddr, address); 524 } 525 526 private static void EmitReadIntFallback(ArmEmitterContext context, Operand address, int rt, int size) 527 { 528 SetInt(context, rt, EmitReadIntFallback(context, address, size)); 529 } 530 531 private static Operand EmitReadIntFallback(ArmEmitterContext context, Operand address, int size) 532 { 533 MethodInfo info = null; 534 535 switch (size) 536 { 537 case 0: 538 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadByte)); 539 break; 540 case 1: 541 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt16)); 542 break; 543 case 2: 544 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt32)); 545 break; 546 case 3: 547 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt64)); 548 break; 549 } 550 551 return context.Call(info, address); 552 } 553 554 private static void EmitReadVectorFallback( 555 ArmEmitterContext context, 556 Operand address, 557 Operand vector, 558 int rt, 559 int elem, 560 int size) 561 { 562 MethodInfo info = null; 563 564 switch (size) 565 { 566 case 0: 567 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadByte)); 568 break; 569 case 1: 570 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt16)); 571 break; 572 case 2: 573 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt32)); 574 break; 575 case 3: 576 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt64)); 577 break; 578 case 4: 579 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadVector128)); 580 break; 581 } 582 583 Operand value = context.Call(info, address); 584 585 switch (size) 586 { 587 case 0: 588 value = context.VectorInsert8(vector, value, elem); 589 break; 590 case 1: 591 value = context.VectorInsert16(vector, value, elem); 592 break; 593 case 2: 594 value = context.VectorInsert(vector, value, elem); 595 break; 596 case 3: 597 value = context.VectorInsert(vector, value, elem); 598 break; 599 } 600 601 context.Copy(GetVec(rt), value); 602 } 603 604 private static void EmitWriteIntFallback(ArmEmitterContext context, Operand address, int rt, int size) 605 { 606 MethodInfo info = null; 607 608 switch (size) 609 { 610 case 0: 611 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteByte)); 612 break; 613 case 1: 614 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt16)); 615 break; 616 case 2: 617 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt32)); 618 break; 619 case 3: 620 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt64)); 621 break; 622 } 623 624 Operand value = GetInt(context, rt); 625 626 if (size < 3 && value.Type == OperandType.I64) 627 { 628 value = context.ConvertI64ToI32(value); 629 } 630 631 context.Call(info, address, value); 632 } 633 634 private static void EmitWriteVectorFallback( 635 ArmEmitterContext context, 636 Operand address, 637 int rt, 638 int elem, 639 int size) 640 { 641 MethodInfo info = null; 642 643 switch (size) 644 { 645 case 0: 646 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteByte)); 647 break; 648 case 1: 649 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt16)); 650 break; 651 case 2: 652 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt32)); 653 break; 654 case 3: 655 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt64)); 656 break; 657 case 4: 658 info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteVector128)); 659 break; 660 } 661 662 Operand value = default; 663 664 if (size < 4) 665 { 666 switch (size) 667 { 668 case 0: 669 value = context.VectorExtract8(GetVec(rt), elem); 670 break; 671 case 1: 672 value = context.VectorExtract16(GetVec(rt), elem); 673 break; 674 case 2: 675 value = context.VectorExtract(OperandType.I32, GetVec(rt), elem); 676 break; 677 case 3: 678 value = context.VectorExtract(OperandType.I64, GetVec(rt), elem); 679 break; 680 } 681 } 682 else 683 { 684 value = GetVec(rt); 685 } 686 687 context.Call(info, address, value); 688 } 689 690 private static Operand GetInt(ArmEmitterContext context, int rt) 691 { 692 return context.CurrOp is OpCode32 ? GetIntA32(context, rt) : GetIntOrZR(context, rt); 693 } 694 695 private static void SetInt(ArmEmitterContext context, int rt, Operand value) 696 { 697 if (context.CurrOp is OpCode32) 698 { 699 SetIntA32(context, rt, value); 700 } 701 else 702 { 703 SetIntOrZR(context, rt, value); 704 } 705 } 706 707 // ARM32 helpers. 708 public static Operand GetMemM(ArmEmitterContext context, bool setCarry = true) 709 { 710 return context.CurrOp switch 711 { 712 IOpCode32MemRsImm op => GetMShiftedByImmediate(context, op, setCarry), 713 IOpCode32MemReg op => GetIntA32(context, op.Rm), 714 IOpCode32Mem op => Const(op.Immediate), 715 OpCode32SimdMemImm op => Const(op.Immediate), 716 _ => throw InvalidOpCodeType(context.CurrOp), 717 }; 718 } 719 720 private static Exception InvalidOpCodeType(OpCode opCode) 721 { 722 return new InvalidOperationException($"Invalid OpCode type \"{opCode?.GetType().Name ?? "null"}\"."); 723 } 724 725 public static Operand GetMShiftedByImmediate(ArmEmitterContext context, IOpCode32MemRsImm op, bool setCarry) 726 { 727 Operand m = GetIntA32(context, op.Rm); 728 729 int shift = op.Immediate; 730 731 if (shift == 0) 732 { 733 switch (op.ShiftType) 734 { 735 case ShiftType.Lsr: 736 shift = 32; 737 break; 738 case ShiftType.Asr: 739 shift = 32; 740 break; 741 case ShiftType.Ror: 742 shift = 1; 743 break; 744 } 745 } 746 747 if (shift != 0) 748 { 749 setCarry &= false; 750 751 switch (op.ShiftType) 752 { 753 case ShiftType.Lsl: 754 m = InstEmitAluHelper.GetLslC(context, m, setCarry, shift); 755 break; 756 case ShiftType.Lsr: 757 m = InstEmitAluHelper.GetLsrC(context, m, setCarry, shift); 758 break; 759 case ShiftType.Asr: 760 m = InstEmitAluHelper.GetAsrC(context, m, setCarry, shift); 761 break; 762 case ShiftType.Ror: 763 if (op.Immediate != 0) 764 { 765 m = InstEmitAluHelper.GetRorC(context, m, setCarry, shift); 766 } 767 else 768 { 769 m = InstEmitAluHelper.GetRrxC(context, m, setCarry); 770 } 771 break; 772 } 773 } 774 775 return m; 776 } 777 } 778 }