1  using Ryujinx.Graphics.Shader.Decoders;
  2  using System;
  3  using System.Collections.Generic;
  4  using System.Runtime.CompilerServices;
  5  
  6  namespace Ryujinx.Graphics.Shader.Translation
  7  {
  8      static class FunctionMatch
  9      {
 10          private static readonly IPatternTreeNode[] _fsiGetAddressTree = PatternTrees.GetFsiGetAddress();
 11          private static readonly IPatternTreeNode[] _fsiGetAddressV2Tree = PatternTrees.GetFsiGetAddressV2();
 12          private static readonly IPatternTreeNode[] _fsiIsLastWarpThreadPatternTree = PatternTrees.GetFsiIsLastWarpThread();
 13          private static readonly IPatternTreeNode[] _fsiBeginPatternTree = PatternTrees.GetFsiBeginPattern();
 14          private static readonly IPatternTreeNode[] _fsiEndPatternTree = PatternTrees.GetFsiEndPattern();
 15  
 16          public static void RunPass(DecodedProgram program)
 17          {
 18              byte[] externalRegs = new byte[4];
 19              bool hasGetAddress = false;
 20  
 21              foreach (DecodedFunction function in program)
 22              {
 23                  if (function == program.MainFunction)
 24                  {
 25                      continue;
 26                  }
 27  
 28                  int externalReg4 = 0;
 29  
 30                  TreeNode[] functionTree = BuildTree(function.Blocks);
 31  
 32                  if (Matches(_fsiGetAddressTree, functionTree))
 33                  {
 34                      externalRegs[1] = functionTree[0].GetRd();
 35                      externalRegs[2] = functionTree[2].GetRd();
 36                      externalRegs[3] = functionTree[1].GetRd();
 37                      externalReg4 = functionTree[3].GetRd();
 38                  }
 39                  else if (Matches(_fsiGetAddressV2Tree, functionTree))
 40                  {
 41                      externalRegs[1] = functionTree[2].GetRd();
 42                      externalRegs[2] = functionTree[1].GetRd();
 43                      externalRegs[3] = functionTree[0].GetRd();
 44                      externalReg4 = functionTree[3].GetRd();
 45                  }
 46  
 47                  // Ensure the register allocation is valid.
 48                  // If so, then we have a match.
 49                  if (externalRegs[1] != externalRegs[2] &&
 50                      externalRegs[2] != externalRegs[3] &&
 51                      externalRegs[1] != externalRegs[3] &&
 52                      externalRegs[1] + 1 != externalRegs[2] &&
 53                      externalRegs[1] + 1 != externalRegs[3] &&
 54                      externalRegs[1] + 1 == externalReg4 &&
 55                      externalRegs[2] != RegisterConsts.RegisterZeroIndex &&
 56                      externalRegs[3] != RegisterConsts.RegisterZeroIndex &&
 57                      externalReg4 != RegisterConsts.RegisterZeroIndex)
 58                  {
 59                      hasGetAddress = true;
 60                      function.Type = FunctionType.Unused;
 61                      break;
 62                  }
 63              }
 64  
 65              foreach (DecodedFunction function in program)
 66              {
 67                  if (function.IsCompilerGenerated || function == program.MainFunction)
 68                  {
 69                      continue;
 70                  }
 71  
 72                  if (hasGetAddress)
 73                  {
 74                      TreeNode[] functionTree = BuildTree(function.Blocks);
 75  
 76                      if (MatchesFsi(_fsiBeginPatternTree, program, function, functionTree, externalRegs))
 77                      {
 78                          function.Type = FunctionType.BuiltInFSIBegin;
 79                          continue;
 80                      }
 81                      else if (MatchesFsi(_fsiEndPatternTree, program, function, functionTree, externalRegs))
 82                      {
 83                          function.Type = FunctionType.BuiltInFSIEnd;
 84                          continue;
 85                      }
 86                  }
 87              }
 88          }
 89  
 90          private readonly struct TreeNodeUse
 91          {
 92              public TreeNode Node { get; }
 93              public int Index { get; }
 94              public bool Inverted { get; }
 95  
 96              private TreeNodeUse(int index, bool inverted, TreeNode node)
 97              {
 98                  Index = index;
 99                  Inverted = inverted;
100                  Node = node;
101              }
102  
103              public TreeNodeUse(int index, TreeNode node) : this(index, false, node)
104              {
105              }
106  
107              public TreeNodeUse Flip()
108              {
109                  return new TreeNodeUse(Index, !Inverted, Node);
110              }
111          }
112  
113          private enum TreeNodeType : byte
114          {
115              Op,
116              Label,
117          }
118  
119          private class TreeNode
120          {
121              public readonly InstOp Op;
122              public readonly List<TreeNodeUse> Uses;
123              public TreeNodeType Type { get; }
124              public byte Order { get; }
125  
126              public TreeNode(byte order)
127              {
128                  Type = TreeNodeType.Label;
129                  Order = order;
130              }
131  
132              public TreeNode(InstOp op, byte order)
133              {
134                  Op = op;
135                  Uses = new List<TreeNodeUse>();
136                  Type = TreeNodeType.Op;
137                  Order = order;
138              }
139  
140              public byte GetPd()
141              {
142                  return (byte)((Op.RawOpCode >> 3) & 7);
143              }
144  
145              public byte GetRd()
146              {
147                  return (byte)Op.RawOpCode;
148              }
149          }
150  
151          private static TreeNode[] BuildTree(Block[] blocks)
152          {
153              List<TreeNode> nodes = new();
154  
155              Dictionary<ulong, TreeNode> labels = new();
156  
157              TreeNodeUse[] predDefs = new TreeNodeUse[RegisterConsts.PredsCount];
158              TreeNodeUse[] gprDefs = new TreeNodeUse[RegisterConsts.GprsCount];
159  
160              void DefPred(byte predIndex, int index, TreeNode node)
161              {
162                  if (predIndex != RegisterConsts.PredicateTrueIndex)
163                  {
164                      predDefs[predIndex] = new TreeNodeUse(index, node);
165                  }
166              }
167  
168              void DefGpr(byte regIndex, int index, TreeNode node)
169              {
170                  if (regIndex != RegisterConsts.RegisterZeroIndex)
171                  {
172                      gprDefs[regIndex] = new TreeNodeUse(index, node);
173                  }
174              }
175  
176              TreeNodeUse UsePred(byte predIndex, bool predInv)
177              {
178                  if (predIndex != RegisterConsts.PredicateTrueIndex)
179                  {
180                      TreeNodeUse use = predDefs[predIndex];
181  
182                      if (use.Node != null)
183                      {
184                          nodes.Remove(use.Node);
185                      }
186                      else
187                      {
188                          use = new TreeNodeUse(-(predIndex + 2), null);
189                      }
190  
191                      return predInv ? use.Flip() : use;
192                  }
193  
194                  return new TreeNodeUse(-1, null);
195              }
196  
197              TreeNodeUse UseGpr(byte regIndex)
198              {
199                  if (regIndex != RegisterConsts.RegisterZeroIndex)
200                  {
201                      TreeNodeUse use = gprDefs[regIndex];
202  
203                      if (use.Node != null)
204                      {
205                          nodes.Remove(use.Node);
206                      }
207                      else
208                      {
209                          use = new TreeNodeUse(-(regIndex + 2), null);
210                      }
211  
212                      return use;
213                  }
214  
215                  return new TreeNodeUse(-1, null);
216              }
217  
218              byte order = 0;
219  
220              for (int index = 0; index < blocks.Length; index++)
221              {
222                  Block block = blocks[index];
223  
224                  if (block.Predecessors.Count > 1)
225                  {
226                      TreeNode label = new(order++);
227                      nodes.Add(label);
228                      labels.Add(block.Address, label);
229                  }
230  
231                  for (int opIndex = 0; opIndex < block.OpCodes.Count; opIndex++)
232                  {
233                      InstOp op = block.OpCodes[opIndex];
234  
235                      TreeNode node = new(op, IsOrderDependant(op.Name) ? order : (byte)0);
236  
237                      // Add uses.
238  
239                      if (!op.Props.HasFlag(InstProps.NoPred))
240                      {
241                          byte predIndex = (byte)((op.RawOpCode >> 16) & 7);
242                          bool predInv = (op.RawOpCode & 0x80000) != 0;
243                          node.Uses.Add(UsePred(predIndex, predInv));
244                      }
245  
246                      if (op.Props.HasFlag(InstProps.Ps))
247                      {
248                          byte predIndex = (byte)((op.RawOpCode >> 39) & 7);
249                          bool predInv = (op.RawOpCode & 0x40000000000) != 0;
250                          node.Uses.Add(UsePred(predIndex, predInv));
251                      }
252  
253                      if (op.Props.HasFlag(InstProps.Ra))
254                      {
255                          byte ra = (byte)(op.RawOpCode >> 8);
256                          node.Uses.Add(UseGpr(ra));
257                      }
258  
259                      if ((op.Props & (InstProps.Rb | InstProps.Rb2)) != 0)
260                      {
261                          byte rb = op.Props.HasFlag(InstProps.Rb2) ? (byte)op.RawOpCode : (byte)(op.RawOpCode >> 20);
262                          node.Uses.Add(UseGpr(rb));
263                      }
264  
265                      if (op.Props.HasFlag(InstProps.Rc))
266                      {
267                          byte rc = (byte)(op.RawOpCode >> 39);
268                          node.Uses.Add(UseGpr(rc));
269                      }
270  
271                      if (op.Name == InstName.Bra && labels.TryGetValue(op.GetAbsoluteAddress(), out TreeNode label))
272                      {
273                          node.Uses.Add(new TreeNodeUse(0, label));
274                      }
275  
276                      // Make definitions.
277  
278                      int defIndex = 0;
279  
280                      InstProps pdType = op.Props & InstProps.PdMask;
281  
282                      if (pdType != 0)
283                      {
284                          int bit = pdType switch
285                          {
286                              InstProps.Pd => 3,
287                              InstProps.LPd => 48,
288                              InstProps.SPd => 30,
289                              InstProps.TPd => 51,
290                              InstProps.VPd => 45,
291                              _ => throw new InvalidOperationException($"Table has unknown predicate destination {pdType}."),
292                          };
293  
294                          byte predIndex = (byte)((op.RawOpCode >> bit) & 7);
295                          DefPred(predIndex, defIndex++, node);
296                      }
297  
298                      if (op.Props.HasFlag(InstProps.Rd))
299                      {
300                          byte rd = (byte)op.RawOpCode;
301                          DefGpr(rd, defIndex++, node);
302                      }
303  
304                      nodes.Add(node);
305                  }
306              }
307  
308              return nodes.ToArray();
309          }
310  
311          private static bool IsOrderDependant(InstName name)
312          {
313              switch (name)
314              {
315                  case InstName.Atom:
316                  case InstName.AtomCas:
317                  case InstName.Atoms:
318                  case InstName.AtomsCas:
319                  case InstName.Ld:
320                  case InstName.Ldg:
321                  case InstName.Ldl:
322                  case InstName.Lds:
323                  case InstName.Suatom:
324                  case InstName.SuatomB:
325                  case InstName.SuatomB2:
326                  case InstName.SuatomCas:
327                  case InstName.SuatomCasB:
328                  case InstName.Suld:
329                  case InstName.SuldB:
330                  case InstName.SuldD:
331                  case InstName.SuldDB:
332                      return true;
333              }
334  
335              return false;
336          }
337  
338          private interface IPatternTreeNode
339          {
340              List<PatternTreeNodeUse> Uses { get; }
341              InstName Name { get; }
342              TreeNodeType Type { get; }
343              byte Order { get; }
344              bool IsImm { get; }
345              bool Matches(in InstOp opInfo);
346          }
347  
348          private readonly struct PatternTreeNodeUse
349          {
350              public IPatternTreeNode Node { get; }
351              public int Index { get; }
352              public bool Inverted { get; }
353              public PatternTreeNodeUse Inv => new(Index, !Inverted, Node);
354  
355              private PatternTreeNodeUse(int index, bool inverted, IPatternTreeNode node)
356              {
357                  Index = index;
358                  Inverted = inverted;
359                  Node = node;
360              }
361  
362              public PatternTreeNodeUse(int index, IPatternTreeNode node) : this(index, false, node)
363              {
364              }
365          }
366  
367          private class PatternTreeNode<T> : IPatternTreeNode
368          {
369              public List<PatternTreeNodeUse> Uses { get; }
370              private readonly Func<T, bool> _match;
371  
372              public InstName Name { get; }
373              public TreeNodeType Type { get; }
374              public byte Order { get; }
375              public bool IsImm { get; }
376              public PatternTreeNodeUse Out => new(0, this);
377  
378              public PatternTreeNode(InstName name, Func<T, bool> match, TreeNodeType type = TreeNodeType.Op, byte order = 0, bool isImm = false)
379              {
380                  Name = name;
381                  _match = match;
382                  Type = type;
383                  Order = order;
384                  IsImm = isImm;
385                  Uses = new List<PatternTreeNodeUse>();
386              }
387  
388              public PatternTreeNode<T> Use(PatternTreeNodeUse use)
389              {
390                  Uses.Add(use);
391                  return this;
392              }
393  
394              public PatternTreeNodeUse OutAt(int index)
395              {
396                  return new PatternTreeNodeUse(index, this);
397              }
398  
399              public bool Matches(in InstOp opInfo)
400              {
401                  if (opInfo.Name != Name)
402                  {
403                      return false;
404                  }
405  
406                  ulong rawOp = opInfo.RawOpCode;
407                  T op = Unsafe.As<ulong, T>(ref rawOp);
408  
409                  if (!_match(op))
410                  {
411                      return false;
412                  }
413  
414                  return true;
415              }
416          }
417  
418          private static bool MatchesFsi(
419              IPatternTreeNode[] pattern,
420              DecodedProgram program,
421              DecodedFunction function,
422              TreeNode[] functionTree,
423              byte[] externalRegs)
424          {
425              if (function.Blocks.Length == 0)
426              {
427                  return false;
428              }
429  
430              InstOp callOp = function.Blocks[0].GetLastOp();
431  
432              if (callOp.Name != InstName.Cal)
433              {
434                  return false;
435              }
436  
437              DecodedFunction callTarget = program.GetFunctionByAddress(callOp.GetAbsoluteAddress());
438              TreeNode[] callTargetTree;
439  
440              if (callTarget == null || !Matches(_fsiIsLastWarpThreadPatternTree, callTargetTree = BuildTree(callTarget.Blocks)))
441              {
442                  return false;
443              }
444  
445              externalRegs[0] = callTargetTree[0].GetPd();
446  
447              if (Matches(pattern, functionTree, externalRegs))
448              {
449                  callTarget.RemoveCaller(function);
450                  return true;
451              }
452  
453              return false;
454          }
455  
456          private static bool Matches(IPatternTreeNode[] pTree, TreeNode[] cTree, byte[] externalRegs = null)
457          {
458              if (pTree.Length != cTree.Length)
459              {
460                  return false;
461              }
462  
463              for (int index = 0; index < pTree.Length; index++)
464              {
465                  if (!Matches(pTree[index], cTree[index], externalRegs))
466                  {
467                      return false;
468                  }
469              }
470  
471              return true;
472          }
473  
474          private static bool Matches(IPatternTreeNode pTreeNode, TreeNode cTreeNode, byte[] externalRegs)
475          {
476              if (!pTreeNode.Matches(in cTreeNode.Op) ||
477                  pTreeNode.Type != cTreeNode.Type ||
478                  pTreeNode.Order != cTreeNode.Order ||
479                  pTreeNode.IsImm != cTreeNode.Op.Props.HasFlag(InstProps.Ib))
480              {
481                  return false;
482              }
483  
484              if (pTreeNode.Type == TreeNodeType.Op)
485              {
486                  if (pTreeNode.Uses.Count != cTreeNode.Uses.Count)
487                  {
488                      return false;
489                  }
490  
491                  for (int index = 0; index < pTreeNode.Uses.Count; index++)
492                  {
493                      var pUse = pTreeNode.Uses[index];
494                      var cUse = cTreeNode.Uses[index];
495  
496                      if (pUse.Index <= -2)
497                      {
498                          if (externalRegs[-pUse.Index - 2] != (-cUse.Index - 2))
499                          {
500                              return false;
501                          }
502                      }
503                      else if (pUse.Index != cUse.Index)
504                      {
505                          return false;
506                      }
507  
508                      if (pUse.Inverted != cUse.Inverted || (pUse.Node == null) != (cUse.Node == null))
509                      {
510                          return false;
511                      }
512  
513                      if (pUse.Node != null && !Matches(pUse.Node, cUse.Node, externalRegs))
514                      {
515                          return false;
516                      }
517                  }
518              }
519  
520              return true;
521          }
522  
523          private static class PatternTrees
524          {
525              public static IPatternTreeNode[] GetFsiGetAddress()
526              {
527                  var affinityValue = S2r(SReg.Affinity).Use(PT).Out;
528                  var orderingTicketValue = S2r(SReg.OrderingTicket).Use(PT).Out;
529  
530                  return new IPatternTreeNode[]
531                  {
532                      Iscadd(cc: true, 2, 0, 404)
533                          .Use(PT)
534                          .Use(Iscadd(cc: false, 8)
535                              .Use(PT)
536                              .Use(Lop32i(LogicOp.And, 0xff)
537                                  .Use(PT)
538                                  .Use(affinityValue).Out)
539                              .Use(Lop32i(LogicOp.And, 0xff)
540                                  .Use(PT)
541                                  .Use(orderingTicketValue).Out).Out),
542                      ShrU32W(16)
543                          .Use(PT)
544                          .Use(orderingTicketValue),
545                      Iadd32i(0x200)
546                          .Use(PT)
547                          .Use(Lop32i(LogicOp.And, 0xfe00)
548                              .Use(PT)
549                              .Use(orderingTicketValue).Out),
550                      Iadd(x: true, 0, 405).Use(PT).Use(RZ),
551                      Ret().Use(PT),
552                  };
553              }
554  
555              public static IPatternTreeNode[] GetFsiGetAddressV2()
556              {
557                  var affinityValue = S2r(SReg.Affinity).Use(PT).Out;
558                  var orderingTicketValue = S2r(SReg.OrderingTicket).Use(PT).Out;
559  
560                  return new IPatternTreeNode[]
561                  {
562                      ShrU32W(16)
563                          .Use(PT)
564                          .Use(orderingTicketValue),
565                      Iadd32i(0x200)
566                          .Use(PT)
567                          .Use(Lop32i(LogicOp.And, 0xfe00)
568                              .Use(PT)
569                              .Use(orderingTicketValue).Out),
570                      Iscadd(cc: true, 2, 0, 404)
571                          .Use(PT)
572                          .Use(Bfi(0x808)
573                              .Use(PT)
574                              .Use(affinityValue)
575                              .Use(Lop32i(LogicOp.And, 0xff)
576                                  .Use(PT)
577                                  .Use(orderingTicketValue).Out).Out),
578                      Iadd(x: true, 0, 405).Use(PT).Use(RZ),
579                      Ret().Use(PT),
580                  };
581              }
582  
583              public static IPatternTreeNode[] GetFsiIsLastWarpThread()
584              {
585                  var threadKillValue = S2r(SReg.ThreadKill).Use(PT).Out;
586                  var laneIdValue = S2r(SReg.LaneId).Use(PT).Out;
587  
588                  return new IPatternTreeNode[]
589                  {
590                      IsetpU32(IComp.Eq)
591                          .Use(PT)
592                          .Use(PT)
593                          .Use(FloU32()
594                              .Use(PT)
595                              .Use(Vote(VoteMode.Any)
596                                  .Use(PT)
597                                  .Use(IsetpU32(IComp.Ne)
598                                      .Use(PT)
599                                      .Use(PT)
600                                      .Use(Lop(negB: true, LogicOp.PassB)
601                                          .Use(PT)
602                                          .Use(RZ)
603                                          .Use(threadKillValue).OutAt(1))
604                                      .Use(RZ).Out).OutAt(1)).Out)
605                          .Use(laneIdValue),
606                      Ret().Use(PT),
607                  };
608              }
609  
610              public static IPatternTreeNode[] GetFsiBeginPattern()
611              {
612                  var addressLowValue = CallArg(1);
613  
614                  static PatternTreeNodeUse HighU16Equals(PatternTreeNodeUse x)
615                  {
616                      var expectedValue = CallArg(3);
617  
618                      return IsetpU32(IComp.Eq)
619                          .Use(PT)
620                          .Use(PT)
621                          .Use(ShrU32W(16).Use(PT).Use(x).Out)
622                          .Use(expectedValue).Out;
623                  }
624  
625                  PatternTreeNode<byte> label;
626  
627                  return new IPatternTreeNode[]
628                  {
629                      Cal(),
630                      Ret().Use(CallArg(0).Inv),
631                      Ret()
632                          .Use(HighU16Equals(LdgE(CacheOpLd.Cg, LsSize.B32)
633                              .Use(PT)
634                              .Use(addressLowValue).Out)),
635                      label = Label(),
636                      Bra()
637                          .Use(HighU16Equals(LdgE(CacheOpLd.Cg, LsSize.B32, 1)
638                              .Use(PT)
639                              .Use(addressLowValue).Out).Inv)
640                          .Use(label.Out),
641                      Ret().Use(PT),
642                  };
643              }
644  
645              public static IPatternTreeNode[] GetFsiEndPattern()
646              {
647                  var voteResult = Vote(VoteMode.All).Use(PT).Use(PT).OutAt(1);
648                  var popcResult = Popc().Use(PT).Use(voteResult).Out;
649                  var threadKillValue = S2r(SReg.ThreadKill).Use(PT).Out;
650                  var laneIdValue = S2r(SReg.LaneId).Use(PT).Out;
651  
652                  var addressLowValue = CallArg(1);
653                  var incrementValue = CallArg(2);
654  
655                  return new IPatternTreeNode[]
656                  {
657                      Cal(),
658                      Ret().Use(CallArg(0).Inv),
659                      Membar(Decoders.Membar.Vc).Use(PT),
660                      Ret().Use(IsetpU32(IComp.Ne)
661                          .Use(PT)
662                          .Use(PT)
663                          .Use(threadKillValue)
664                          .Use(RZ).Out),
665                      RedE(RedOp.Add, AtomSize.U32)
666                          .Use(IsetpU32(IComp.Eq)
667                              .Use(PT)
668                              .Use(PT)
669                              .Use(FloU32()
670                                  .Use(PT)
671                                  .Use(voteResult).Out)
672                              .Use(laneIdValue).Out)
673                          .Use(addressLowValue)
674                          .Use(Xmad(XmadCop.Cbcc, psl: true, hiloA: true, hiloB: true)
675                              .Use(PT)
676                              .Use(incrementValue)
677                              .Use(Xmad(XmadCop.Cfull, mrg: true, hiloB: true)
678                                  .Use(PT)
679                                  .Use(incrementValue)
680                                  .Use(popcResult)
681                                  .Use(RZ).Out)
682                              .Use(Xmad(XmadCop.Cfull)
683                                  .Use(PT)
684                                  .Use(incrementValue)
685                                  .Use(popcResult)
686                                  .Use(RZ).Out).Out),
687                      Ret().Use(PT),
688                  };
689              }
690  
691              private static PatternTreeNode<InstBfiI> Bfi(int imm)
692              {
693                  return new(InstName.Bfi, (op) => !op.WriteCC && op.Imm20 == imm, isImm: true);
694              }
695  
696              private static PatternTreeNode<InstBra> Bra()
697              {
698                  return new(InstName.Bra, (op) => op.Ccc == Ccc.T && !op.Ca);
699              }
700  
701              private static PatternTreeNode<InstCal> Cal()
702              {
703                  return new(InstName.Cal, (op) => !op.Ca && op.Inc);
704              }
705  
706              private static PatternTreeNode<InstFloR> FloU32()
707              {
708                  return new(InstName.Flo, (op) => !op.Signed && !op.Sh && !op.NegB && !op.WriteCC);
709              }
710  
711              private static PatternTreeNode<InstIaddC> Iadd(bool x, int cbufSlot, int cbufOffset)
712              {
713                  return new(InstName.Iadd, (op) =>
714                      !op.Sat &&
715                      !op.WriteCC &&
716                      op.X == x &&
717                      op.AvgMode == AvgMode.NoNeg &&
718                      op.CbufSlot == cbufSlot &&
719                      op.CbufOffset == cbufOffset);
720              }
721  
722              private static PatternTreeNode<InstIadd32i> Iadd32i(int imm)
723              {
724                  return new(InstName.Iadd32i, (op) => !op.Sat && !op.WriteCC && !op.X && op.AvgMode == AvgMode.NoNeg && op.Imm32 == imm);
725              }
726  
727              private static PatternTreeNode<InstIscaddR> Iscadd(bool cc, int imm)
728              {
729                  return new(InstName.Iscadd, (op) => op.WriteCC == cc && op.AvgMode == AvgMode.NoNeg && op.Imm5 == imm);
730              }
731  
732              private static PatternTreeNode<InstIscaddC> Iscadd(bool cc, int imm, int cbufSlot, int cbufOffset)
733              {
734                  return new(InstName.Iscadd, (op) =>
735                      op.WriteCC == cc &&
736                      op.AvgMode == AvgMode.NoNeg &&
737                      op.Imm5 == imm &&
738                      op.CbufSlot == cbufSlot &&
739                      op.CbufOffset == cbufOffset);
740              }
741  
742              private static PatternTreeNode<InstIsetpR> IsetpU32(IComp comp)
743              {
744                  return new(InstName.Isetp, (op) => !op.Signed && op.IComp == comp && op.Bop == BoolOp.And);
745              }
746  
747              private static PatternTreeNode<byte> Label()
748              {
749                  return new(InstName.Invalid, (op) => true, type: TreeNodeType.Label);
750              }
751  
752              private static PatternTreeNode<InstLopR> Lop(bool negB, LogicOp logicOp)
753              {
754                  return new(InstName.Lop, (op) => !op.NegA && op.NegB == negB && !op.WriteCC && !op.X && op.Lop == logicOp && op.PredicateOp == PredicateOp.F);
755              }
756  
757              private static PatternTreeNode<InstLop32i> Lop32i(LogicOp logicOp, int imm)
758              {
759                  return new(InstName.Lop32i, (op) => !op.NegA && !op.NegB && !op.X && !op.WriteCC && op.LogicOp == logicOp && op.Imm32 == imm);
760              }
761  
762              private static PatternTreeNode<InstMembar> Membar(Membar membar)
763              {
764                  return new(InstName.Membar, (op) => op.Membar == membar);
765              }
766  
767              private static PatternTreeNode<InstPopcR> Popc()
768              {
769                  return new(InstName.Popc, (op) => !op.NegB);
770              }
771  
772              private static PatternTreeNode<InstRet> Ret()
773              {
774                  return new(InstName.Ret, (op) => op.Ccc == Ccc.T);
775              }
776  
777              private static PatternTreeNode<InstS2r> S2r(SReg reg)
778              {
779                  return new(InstName.S2r, (op) => op.SReg == reg);
780              }
781  
782              private static PatternTreeNode<InstShrI> ShrU32W(int imm)
783              {
784                  return new(InstName.Shr, (op) => !op.Signed && !op.Brev && op.M && op.XMode == 0 && op.Imm20 == imm, isImm: true);
785              }
786  
787              private static PatternTreeNode<InstLdg> LdgE(CacheOpLd cacheOp, LsSize size, byte order = 0)
788              {
789                  return new(InstName.Ldg, (op) => op.E && op.CacheOp == cacheOp && op.LsSize == size, order: order);
790              }
791  
792              private static PatternTreeNode<InstRed> RedE(RedOp redOp, AtomSize size, byte order = 0)
793              {
794                  return new(InstName.Red, (op) => op.E && op.RedOp == redOp && op.RedSize == size, order: order);
795              }
796  
797              private static PatternTreeNode<InstVote> Vote(VoteMode mode)
798              {
799                  return new(InstName.Vote, (op) => op.VoteMode == mode);
800              }
801  
802              private static PatternTreeNode<InstXmadR> Xmad(XmadCop cop, bool psl = false, bool mrg = false, bool hiloA = false, bool hiloB = false)
803              {
804                  return new(InstName.Xmad, (op) => op.XmadCop == cop && op.Psl == psl && op.Mrg == mrg && op.HiloA == hiloA && op.HiloB == hiloB);
805              }
806  
807              private static PatternTreeNodeUse PT => PTOrRZ();
808              private static PatternTreeNodeUse RZ => PTOrRZ();
809  
810              private static PatternTreeNodeUse CallArg(int index)
811              {
812                  return new PatternTreeNodeUse(-(index + 2), null);
813              }
814  
815              private static PatternTreeNodeUse PTOrRZ()
816              {
817                  return new PatternTreeNodeUse(-1, null);
818              }
819          }
820  
821          private static void PrintTreeNode(TreeNode node, string indentation)
822          {
823              Console.WriteLine($" {node.Op.Name}");
824  
825              for (int i = 0; i < node.Uses.Count; i++)
826              {
827                  TreeNodeUse use = node.Uses[i];
828                  bool last = i == node.Uses.Count - 1;
829                  char separator = last ? '`' : '|';
830  
831                  if (use.Node != null)
832                  {
833                      Console.Write($"{indentation} {separator}- ({(use.Inverted ? "INV " : "")}{use.Index})");
834                      PrintTreeNode(use.Node, indentation + (last ? "       " : " |     "));
835                  }
836                  else
837                  {
838                      Console.WriteLine($"{indentation} {separator}- ({(use.Inverted ? "INV " : "")}{use.Index}) NULL");
839                  }
840              }
841          }
842  
843          private static void PrintTreeNode(IPatternTreeNode node, string indentation)
844          {
845              Console.WriteLine($" {node.Name}");
846  
847              for (int i = 0; i < node.Uses.Count; i++)
848              {
849                  PatternTreeNodeUse use = node.Uses[i];
850                  bool last = i == node.Uses.Count - 1;
851                  char separator = last ? '`' : '|';
852  
853                  if (use.Node != null)
854                  {
855                      Console.Write($"{indentation} {separator}- ({(use.Inverted ? "INV " : "")}{use.Index})");
856                      PrintTreeNode(use.Node, indentation + (last ? "       " : " |     "));
857                  }
858                  else
859                  {
860                      Console.WriteLine($"{indentation} {separator}- ({(use.Inverted ? "INV " : "")}{use.Index}) NULL");
861                  }
862              }
863          }
864      }
865  }