InstEmitSurface.cs
1 using Ryujinx.Graphics.Shader.Decoders; 2 using Ryujinx.Graphics.Shader.IntermediateRepresentation; 3 using Ryujinx.Graphics.Shader.StructuredIr; 4 using Ryujinx.Graphics.Shader.Translation; 5 using System; 6 using System.Collections.Generic; 7 using System.Numerics; 8 using static Ryujinx.Graphics.Shader.Instructions.InstEmitHelper; 9 using static Ryujinx.Graphics.Shader.IntermediateRepresentation.OperandHelper; 10 11 namespace Ryujinx.Graphics.Shader.Instructions 12 { 13 static partial class InstEmit 14 { 15 public static void SuatomB(EmitterContext context) 16 { 17 InstSuatomB op = context.GetOp<InstSuatomB>(); 18 19 EmitSuatom( 20 context, 21 op.Dim, 22 op.Op, 23 op.Size, 24 0, 25 op.SrcA, 26 op.SrcB, 27 op.SrcC, 28 op.Dest, 29 op.Ba, 30 isBindless: true, 31 compareAndSwap: false); 32 } 33 34 public static void Suatom(EmitterContext context) 35 { 36 InstSuatom op = context.GetOp<InstSuatom>(); 37 38 EmitSuatom( 39 context, 40 op.Dim, 41 op.Op, 42 op.Size, 43 op.TidB, 44 op.SrcA, 45 op.SrcB, 46 0, 47 op.Dest, 48 op.Ba, 49 isBindless: false, 50 compareAndSwap: false); 51 } 52 53 public static void SuatomB2(EmitterContext context) 54 { 55 InstSuatomB2 op = context.GetOp<InstSuatomB2>(); 56 57 EmitSuatom( 58 context, 59 op.Dim, 60 op.Op, 61 op.Size, 62 0, 63 op.SrcA, 64 op.SrcB, 65 op.SrcC, 66 op.Dest, 67 op.Ba, 68 isBindless: true, 69 compareAndSwap: false); 70 } 71 72 public static void SuatomCasB(EmitterContext context) 73 { 74 InstSuatomCasB op = context.GetOp<InstSuatomCasB>(); 75 76 EmitSuatom( 77 context, 78 op.Dim, 79 0, 80 op.Size, 81 0, 82 op.SrcA, 83 op.SrcB, 84 op.SrcC, 85 op.Dest, 86 op.Ba, 87 isBindless: true, 88 compareAndSwap: true); 89 } 90 91 public static void SuatomCas(EmitterContext context) 92 { 93 InstSuatomCas op = context.GetOp<InstSuatomCas>(); 94 95 EmitSuatom( 96 context, 97 op.Dim, 98 0, 99 op.Size, 100 op.TidB, 101 op.SrcA, 102 op.SrcB, 103 0, 104 op.Dest, 105 op.Ba, 106 isBindless: false, 107 compareAndSwap: true); 108 } 109 110 public static void SuldDB(EmitterContext context) 111 { 112 InstSuldDB op = context.GetOp<InstSuldDB>(); 113 114 EmitSuld(context, op.CacheOp, op.Dim, op.Size, 0, 0, op.SrcA, op.Dest, op.SrcC, useComponents: false, op.Ba, isBindless: true); 115 } 116 117 public static void SuldD(EmitterContext context) 118 { 119 InstSuldD op = context.GetOp<InstSuldD>(); 120 121 EmitSuld(context, op.CacheOp, op.Dim, op.Size, op.TidB, 0, op.SrcA, op.Dest, 0, useComponents: false, op.Ba, isBindless: false); 122 } 123 124 public static void SuldB(EmitterContext context) 125 { 126 InstSuldB op = context.GetOp<InstSuldB>(); 127 128 EmitSuld(context, op.CacheOp, op.Dim, 0, 0, op.Rgba, op.SrcA, op.Dest, op.SrcC, useComponents: true, false, isBindless: true); 129 } 130 131 public static void Suld(EmitterContext context) 132 { 133 InstSuld op = context.GetOp<InstSuld>(); 134 135 EmitSuld(context, op.CacheOp, op.Dim, 0, op.TidB, op.Rgba, op.SrcA, op.Dest, 0, useComponents: true, false, isBindless: false); 136 } 137 138 public static void SuredB(EmitterContext context) 139 { 140 InstSuredB op = context.GetOp<InstSuredB>(); 141 142 EmitSured(context, op.Dim, op.Op, op.Size, 0, op.SrcA, op.Dest, op.SrcC, op.Ba, isBindless: true); 143 } 144 145 public static void Sured(EmitterContext context) 146 { 147 InstSured op = context.GetOp<InstSured>(); 148 149 EmitSured(context, op.Dim, op.Op, op.Size, op.TidB, op.SrcA, op.Dest, 0, op.Ba, isBindless: false); 150 } 151 152 public static void SustDB(EmitterContext context) 153 { 154 InstSustDB op = context.GetOp<InstSustDB>(); 155 156 EmitSust(context, op.CacheOp, op.Dim, op.Size, 0, 0, op.SrcA, op.Dest, op.SrcC, useComponents: false, op.Ba, isBindless: true); 157 } 158 159 public static void SustD(EmitterContext context) 160 { 161 InstSustD op = context.GetOp<InstSustD>(); 162 163 EmitSust(context, op.CacheOp, op.Dim, op.Size, op.TidB, 0, op.SrcA, op.Dest, 0, useComponents: false, op.Ba, isBindless: false); 164 } 165 166 public static void SustB(EmitterContext context) 167 { 168 InstSustB op = context.GetOp<InstSustB>(); 169 170 EmitSust(context, op.CacheOp, op.Dim, 0, 0, op.Rgba, op.SrcA, op.Dest, op.SrcC, useComponents: true, false, isBindless: true); 171 } 172 173 public static void Sust(EmitterContext context) 174 { 175 InstSust op = context.GetOp<InstSust>(); 176 177 EmitSust(context, op.CacheOp, op.Dim, 0, op.TidB, op.Rgba, op.SrcA, op.Dest, 0, useComponents: true, false, isBindless: false); 178 } 179 180 private static void EmitSuatom( 181 EmitterContext context, 182 SuDim dimensions, 183 SuatomOp atomicOp, 184 SuatomSize size, 185 int imm, 186 int srcA, 187 int srcB, 188 int srcC, 189 int dest, 190 bool byteAddress, 191 bool isBindless, 192 bool compareAndSwap) 193 { 194 SamplerType type = ConvertSamplerType(dimensions); 195 196 if (type == SamplerType.None) 197 { 198 context.TranslatorContext.GpuAccessor.Log("Invalid image atomic sampler type."); 199 return; 200 } 201 202 Operand Ra() 203 { 204 if (srcA > RegisterConsts.RegisterZeroIndex) 205 { 206 return Const(0); 207 } 208 209 return context.Copy(Register(srcA++, RegisterType.Gpr)); 210 } 211 212 Operand Rb() 213 { 214 if (srcB > RegisterConsts.RegisterZeroIndex) 215 { 216 return Const(0); 217 } 218 219 return context.Copy(Register(srcB++, RegisterType.Gpr)); 220 } 221 222 Operand d = Register(dest, RegisterType.Gpr); 223 224 List<Operand> sourcesList = new(); 225 226 if (isBindless) 227 { 228 sourcesList.Add(context.Copy(GetSrcReg(context, srcC))); 229 } 230 231 int coordsCount = type.GetDimensions(); 232 233 for (int index = 0; index < coordsCount; index++) 234 { 235 sourcesList.Add(Ra()); 236 } 237 238 if (Sample1DAs2D && (type & SamplerType.Mask) == SamplerType.Texture1D) 239 { 240 sourcesList.Add(Const(0)); 241 242 type &= ~SamplerType.Mask; 243 type |= SamplerType.Texture2D; 244 } 245 246 if (type.HasFlag(SamplerType.Array)) 247 { 248 sourcesList.Add(Ra()); 249 250 type |= SamplerType.Array; 251 } 252 253 if (byteAddress) 254 { 255 int xIndex = isBindless ? 1 : 0; 256 257 sourcesList[xIndex] = context.ShiftRightS32(sourcesList[xIndex], Const(GetComponentSizeInBytesLog2(size))); 258 } 259 260 // TODO: FP and 64-bit formats. 261 TextureFormat format = size == SuatomSize.Sd32 || size == SuatomSize.Sd64 262 ? (isBindless ? TextureFormat.Unknown : ShaderProperties.GetTextureFormatAtomic(context.TranslatorContext.GpuAccessor, imm)) 263 : GetTextureFormat(size); 264 265 if (compareAndSwap) 266 { 267 sourcesList.Add(Rb()); 268 } 269 270 sourcesList.Add(Rb()); 271 272 Operand[] sources = sourcesList.ToArray(); 273 274 TextureFlags flags = compareAndSwap ? TextureFlags.CAS : GetAtomicOpFlags(atomicOp); 275 276 if (isBindless) 277 { 278 flags |= TextureFlags.Bindless; 279 } 280 281 SetBindingPair setAndBinding = isBindless ? default : context.ResourceManager.GetTextureOrImageBinding( 282 Instruction.ImageAtomic, 283 type, 284 format, 285 flags, 286 TextureOperation.DefaultCbufSlot, 287 imm); 288 289 Operand res = context.ImageAtomic(type, format, flags, setAndBinding, sources); 290 291 context.Copy(d, res); 292 } 293 294 private static void EmitSuld( 295 EmitterContext context, 296 CacheOpLd cacheOp, 297 SuDim dimensions, 298 SuSize size, 299 int imm, 300 SuRgba componentMask, 301 int srcA, 302 int srcB, 303 int srcC, 304 bool useComponents, 305 bool byteAddress, 306 bool isBindless) 307 { 308 if (srcB == RegisterConsts.RegisterZeroIndex) 309 { 310 return; 311 } 312 313 SamplerType type = ConvertSamplerType(dimensions); 314 315 if (type == SamplerType.None) 316 { 317 context.TranslatorContext.GpuAccessor.Log("Invalid image store sampler type."); 318 return; 319 } 320 321 Operand Ra() 322 { 323 if (srcA > RegisterConsts.RegisterZeroIndex) 324 { 325 return Const(0); 326 } 327 328 return context.Copy(Register(srcA++, RegisterType.Gpr)); 329 } 330 331 List<Operand> sourcesList = new(); 332 333 if (isBindless) 334 { 335 sourcesList.Add(context.Copy(Register(srcC, RegisterType.Gpr))); 336 } 337 338 int coordsCount = type.GetDimensions(); 339 340 for (int index = 0; index < coordsCount; index++) 341 { 342 sourcesList.Add(Ra()); 343 } 344 345 if (Sample1DAs2D && (type & SamplerType.Mask) == SamplerType.Texture1D) 346 { 347 sourcesList.Add(Const(0)); 348 349 type &= ~SamplerType.Mask; 350 type |= SamplerType.Texture2D; 351 } 352 353 if (type.HasFlag(SamplerType.Array)) 354 { 355 sourcesList.Add(Ra()); 356 } 357 358 Operand[] sources = sourcesList.ToArray(); 359 360 int handle = imm; 361 362 TextureFlags flags = isBindless ? TextureFlags.Bindless : TextureFlags.None; 363 364 if (cacheOp == CacheOpLd.Cg) 365 { 366 flags |= TextureFlags.Coherent; 367 } 368 369 if (useComponents) 370 { 371 Operand[] dests = new Operand[BitOperations.PopCount((uint)componentMask)]; 372 373 int outputIndex = 0; 374 375 for (int i = 0; i < dests.Length; i++) 376 { 377 if (srcB + i >= RegisterConsts.RegisterZeroIndex) 378 { 379 break; 380 } 381 382 dests[outputIndex++] = Register(srcB + i, RegisterType.Gpr); 383 } 384 385 if (outputIndex != dests.Length) 386 { 387 Array.Resize(ref dests, outputIndex); 388 } 389 390 TextureFormat format = isBindless ? TextureFormat.Unknown : ShaderProperties.GetTextureFormat(context.TranslatorContext.GpuAccessor, handle); 391 392 SetBindingPair setAndBinding = isBindless ? default : context.ResourceManager.GetTextureOrImageBinding( 393 Instruction.ImageLoad, 394 type, 395 format, 396 flags, 397 TextureOperation.DefaultCbufSlot, 398 handle); 399 400 context.ImageLoad(type, format, flags, setAndBinding, (int)componentMask, dests, sources); 401 } 402 else 403 { 404 if (byteAddress) 405 { 406 int xIndex = isBindless ? 1 : 0; 407 408 sources[xIndex] = context.ShiftRightS32(sources[xIndex], Const(GetComponentSizeInBytesLog2(size))); 409 } 410 411 int components = GetComponents(size); 412 int compMask = (1 << components) - 1; 413 414 Operand[] dests = new Operand[components]; 415 416 int outputIndex = 0; 417 418 for (int i = 0; i < dests.Length; i++) 419 { 420 if (srcB + i >= RegisterConsts.RegisterZeroIndex) 421 { 422 break; 423 } 424 425 dests[outputIndex++] = Register(srcB + i, RegisterType.Gpr); 426 } 427 428 if (outputIndex != dests.Length) 429 { 430 Array.Resize(ref dests, outputIndex); 431 } 432 433 TextureFormat format = GetTextureFormat(size); 434 435 SetBindingPair setAndBinding = isBindless ? default : context.ResourceManager.GetTextureOrImageBinding( 436 Instruction.ImageLoad, 437 type, 438 format, 439 flags, 440 TextureOperation.DefaultCbufSlot, 441 handle); 442 443 context.ImageLoad(type, format, flags, setAndBinding, compMask, dests, sources); 444 445 switch (size) 446 { 447 case SuSize.U8: 448 context.Copy(dests[0], ZeroExtendTo32(context, dests[0], 8)); 449 break; 450 case SuSize.U16: 451 context.Copy(dests[0], ZeroExtendTo32(context, dests[0], 16)); 452 break; 453 case SuSize.S8: 454 context.Copy(dests[0], SignExtendTo32(context, dests[0], 8)); 455 break; 456 case SuSize.S16: 457 context.Copy(dests[0], SignExtendTo32(context, dests[0], 16)); 458 break; 459 } 460 } 461 } 462 463 private static void EmitSured( 464 EmitterContext context, 465 SuDim dimensions, 466 RedOp atomicOp, 467 SuatomSize size, 468 int imm, 469 int srcA, 470 int srcB, 471 int srcC, 472 bool byteAddress, 473 bool isBindless) 474 { 475 SamplerType type = ConvertSamplerType(dimensions); 476 477 if (type == SamplerType.None) 478 { 479 context.TranslatorContext.GpuAccessor.Log("Invalid image reduction sampler type."); 480 return; 481 } 482 483 Operand Ra() 484 { 485 if (srcA > RegisterConsts.RegisterZeroIndex) 486 { 487 return Const(0); 488 } 489 490 return context.Copy(Register(srcA++, RegisterType.Gpr)); 491 } 492 493 Operand Rb() 494 { 495 if (srcB > RegisterConsts.RegisterZeroIndex) 496 { 497 return Const(0); 498 } 499 500 return context.Copy(Register(srcB++, RegisterType.Gpr)); 501 } 502 503 List<Operand> sourcesList = new(); 504 505 if (isBindless) 506 { 507 sourcesList.Add(context.Copy(GetSrcReg(context, srcC))); 508 } 509 510 int coordsCount = type.GetDimensions(); 511 512 for (int index = 0; index < coordsCount; index++) 513 { 514 sourcesList.Add(Ra()); 515 } 516 517 if (Sample1DAs2D && (type & SamplerType.Mask) == SamplerType.Texture1D) 518 { 519 sourcesList.Add(Const(0)); 520 521 type &= ~SamplerType.Mask; 522 type |= SamplerType.Texture2D; 523 } 524 525 if (type.HasFlag(SamplerType.Array)) 526 { 527 sourcesList.Add(Ra()); 528 529 type |= SamplerType.Array; 530 } 531 532 if (byteAddress) 533 { 534 int xIndex = isBindless ? 1 : 0; 535 536 sourcesList[xIndex] = context.ShiftRightS32(sourcesList[xIndex], Const(GetComponentSizeInBytesLog2(size))); 537 } 538 539 // TODO: FP and 64-bit formats. 540 TextureFormat format = size == SuatomSize.Sd32 || size == SuatomSize.Sd64 541 ? (isBindless ? TextureFormat.Unknown : ShaderProperties.GetTextureFormatAtomic(context.TranslatorContext.GpuAccessor, imm)) 542 : GetTextureFormat(size); 543 544 sourcesList.Add(Rb()); 545 546 Operand[] sources = sourcesList.ToArray(); 547 548 TextureFlags flags = GetAtomicOpFlags((SuatomOp)atomicOp); 549 550 if (isBindless) 551 { 552 flags |= TextureFlags.Bindless; 553 } 554 555 SetBindingPair setAndBinding = isBindless ? default : context.ResourceManager.GetTextureOrImageBinding( 556 Instruction.ImageAtomic, 557 type, 558 format, 559 flags, 560 TextureOperation.DefaultCbufSlot, 561 imm); 562 563 context.ImageAtomic(type, format, flags, setAndBinding, sources); 564 } 565 566 private static void EmitSust( 567 EmitterContext context, 568 CacheOpSt cacheOp, 569 SuDim dimensions, 570 SuSize size, 571 int imm, 572 SuRgba componentMask, 573 int srcA, 574 int srcB, 575 int srcC, 576 bool useComponents, 577 bool byteAddress, 578 bool isBindless) 579 { 580 SamplerType type = ConvertSamplerType(dimensions); 581 582 if (type == SamplerType.None) 583 { 584 context.TranslatorContext.GpuAccessor.Log("Invalid image store sampler type."); 585 return; 586 } 587 588 Operand Ra() 589 { 590 if (srcA > RegisterConsts.RegisterZeroIndex) 591 { 592 return Const(0); 593 } 594 595 return context.Copy(Register(srcA++, RegisterType.Gpr)); 596 } 597 598 Operand Rb() 599 { 600 if (srcB > RegisterConsts.RegisterZeroIndex) 601 { 602 return Const(0); 603 } 604 605 return context.Copy(Register(srcB++, RegisterType.Gpr)); 606 } 607 608 List<Operand> sourcesList = new(); 609 610 if (isBindless) 611 { 612 sourcesList.Add(context.Copy(Register(srcC, RegisterType.Gpr))); 613 } 614 615 int coordsCount = type.GetDimensions(); 616 617 for (int index = 0; index < coordsCount; index++) 618 { 619 sourcesList.Add(Ra()); 620 } 621 622 if (Sample1DAs2D && (type & SamplerType.Mask) == SamplerType.Texture1D) 623 { 624 sourcesList.Add(Const(0)); 625 626 type &= ~SamplerType.Mask; 627 type |= SamplerType.Texture2D; 628 } 629 630 if (type.HasFlag(SamplerType.Array)) 631 { 632 sourcesList.Add(Ra()); 633 } 634 635 TextureFormat format = TextureFormat.Unknown; 636 637 if (useComponents) 638 { 639 for (int compMask = (int)componentMask, compIndex = 0; compMask != 0; compMask >>= 1, compIndex++) 640 { 641 if ((compMask & 1) != 0) 642 { 643 sourcesList.Add(Rb()); 644 } 645 } 646 647 if (!isBindless) 648 { 649 format = ShaderProperties.GetTextureFormat(context.TranslatorContext.GpuAccessor, imm); 650 } 651 } 652 else 653 { 654 if (byteAddress) 655 { 656 int xIndex = isBindless ? 1 : 0; 657 658 sourcesList[xIndex] = context.ShiftRightS32(sourcesList[xIndex], Const(GetComponentSizeInBytesLog2(size))); 659 } 660 661 int components = GetComponents(size); 662 663 for (int compIndex = 0; compIndex < components; compIndex++) 664 { 665 sourcesList.Add(Rb()); 666 } 667 668 format = GetTextureFormat(size); 669 } 670 671 Operand[] sources = sourcesList.ToArray(); 672 673 int handle = imm; 674 675 TextureFlags flags = isBindless ? TextureFlags.Bindless : TextureFlags.None; 676 677 if (cacheOp == CacheOpSt.Cg) 678 { 679 flags |= TextureFlags.Coherent; 680 } 681 682 SetBindingPair setAndBinding = isBindless ? default : context.ResourceManager.GetTextureOrImageBinding( 683 Instruction.ImageStore, 684 type, 685 format, 686 flags, 687 TextureOperation.DefaultCbufSlot, 688 handle); 689 690 context.ImageStore(type, format, flags, setAndBinding, sources); 691 } 692 693 private static int GetComponentSizeInBytesLog2(SuatomSize size) 694 { 695 return size switch 696 { 697 SuatomSize.U32 => 2, 698 SuatomSize.S32 => 2, 699 SuatomSize.U64 => 3, 700 SuatomSize.F32FtzRn => 2, 701 SuatomSize.F16x2FtzRn => 2, 702 SuatomSize.S64 => 3, 703 SuatomSize.Sd32 => 2, 704 SuatomSize.Sd64 => 3, 705 _ => 2, 706 }; 707 } 708 709 private static TextureFormat GetTextureFormat(SuatomSize size) 710 { 711 return size switch 712 { 713 SuatomSize.U32 => TextureFormat.R32Uint, 714 SuatomSize.S32 => TextureFormat.R32Sint, 715 SuatomSize.U64 => TextureFormat.R32G32Uint, 716 SuatomSize.F32FtzRn => TextureFormat.R32Float, 717 SuatomSize.F16x2FtzRn => TextureFormat.R16G16Float, 718 SuatomSize.S64 => TextureFormat.R32G32Uint, 719 SuatomSize.Sd32 => TextureFormat.R32Uint, 720 SuatomSize.Sd64 => TextureFormat.R32G32Uint, 721 _ => TextureFormat.R32Uint, 722 }; 723 } 724 725 private static TextureFlags GetAtomicOpFlags(SuatomOp op) 726 { 727 return op switch 728 { 729 SuatomOp.Add => TextureFlags.Add, 730 SuatomOp.Min => TextureFlags.Minimum, 731 SuatomOp.Max => TextureFlags.Maximum, 732 SuatomOp.Inc => TextureFlags.Increment, 733 SuatomOp.Dec => TextureFlags.Decrement, 734 SuatomOp.And => TextureFlags.BitwiseAnd, 735 SuatomOp.Or => TextureFlags.BitwiseOr, 736 SuatomOp.Xor => TextureFlags.BitwiseXor, 737 SuatomOp.Exch => TextureFlags.Swap, 738 _ => TextureFlags.Add, 739 }; 740 } 741 742 private static int GetComponents(SuSize size) 743 { 744 return size switch 745 { 746 SuSize.B64 => 2, 747 SuSize.B128 => 4, 748 SuSize.UB128 => 4, 749 _ => 1, 750 }; 751 } 752 753 private static int GetComponentSizeInBytesLog2(SuSize size) 754 { 755 return size switch 756 { 757 SuSize.U8 => 0, 758 SuSize.S8 => 0, 759 SuSize.U16 => 1, 760 SuSize.S16 => 1, 761 SuSize.B32 => 2, 762 SuSize.B64 => 3, 763 SuSize.B128 => 4, 764 SuSize.UB128 => 4, 765 _ => 2, 766 }; 767 } 768 769 private static TextureFormat GetTextureFormat(SuSize size) 770 { 771 return size switch 772 { 773 SuSize.U8 => TextureFormat.R8Uint, 774 SuSize.S8 => TextureFormat.R8Sint, 775 SuSize.U16 => TextureFormat.R16Uint, 776 SuSize.S16 => TextureFormat.R16Sint, 777 SuSize.B32 => TextureFormat.R32Uint, 778 SuSize.B64 => TextureFormat.R32G32Uint, 779 SuSize.B128 => TextureFormat.R32G32B32A32Uint, 780 SuSize.UB128 => TextureFormat.R32G32B32A32Uint, 781 _ => TextureFormat.R32Uint, 782 }; 783 } 784 785 private static SamplerType ConvertSamplerType(SuDim target) 786 { 787 return target switch 788 { 789 SuDim._1d => SamplerType.Texture1D, 790 SuDim._1dBuffer => SamplerType.TextureBuffer, 791 SuDim._1dArray => SamplerType.Texture1D | SamplerType.Array, 792 SuDim._2d => SamplerType.Texture2D, 793 SuDim._2dArray => SamplerType.Texture2D | SamplerType.Array, 794 SuDim._3d => SamplerType.Texture3D, 795 _ => SamplerType.None, 796 }; 797 } 798 } 799 }