1  using ARMeilleure.Decoders;
  2  using ARMeilleure.IntermediateRepresentation;
  3  using ARMeilleure.State;
  4  using ARMeilleure.Translation;
  5  using System;
  6  using System.Diagnostics;
  7  using static ARMeilleure.Instructions.InstEmitHelper;
  8  using static ARMeilleure.Instructions.InstEmitSimdHelper;
  9  using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 10  
 11  namespace ARMeilleure.Instructions
 12  {
 13      using Func1I = Func<Operand, Operand>;
 14      using Func2I = Func<Operand, Operand, Operand>;
 15      using Func3I = Func<Operand, Operand, Operand, Operand>;
 16  
 17      static class InstEmitSimdHelper32Arm64
 18      {
 19          // Intrinsic Helpers
 20  
 21          public static Operand EmitMoveDoubleWordToSide(ArmEmitterContext context, Operand input, int originalV, int targetV)
 22          {
 23              Debug.Assert(input.Type == OperandType.V128);
 24  
 25              int originalSide = originalV & 1;
 26              int targetSide = targetV & 1;
 27  
 28              if (originalSide == targetSide)
 29              {
 30                  return input;
 31              }
 32  
 33              Intrinsic vType = Intrinsic.Arm64VDWord | Intrinsic.Arm64V128;
 34  
 35              if (targetSide == 1)
 36              {
 37                  return context.AddIntrinsic(Intrinsic.Arm64DupVe | vType, input, Const(OperandType.I32, 0)); // Low to high.
 38              }
 39              else
 40              {
 41                  return context.AddIntrinsic(Intrinsic.Arm64DupVe | vType, input, Const(OperandType.I32, 1)); // High to low.
 42              }
 43          }
 44  
 45          public static Operand EmitDoubleWordInsert(ArmEmitterContext context, Operand target, Operand value, int targetV)
 46          {
 47              Debug.Assert(target.Type == OperandType.V128 && value.Type == OperandType.V128);
 48  
 49              int targetSide = targetV & 1;
 50              Operand idx = Const(targetSide);
 51  
 52              return context.AddIntrinsic(Intrinsic.Arm64InsVe | Intrinsic.Arm64VDWord, target, idx, value, idx);
 53          }
 54  
 55          public static Operand EmitScalarInsert(ArmEmitterContext context, Operand target, Operand value, int reg, bool doubleWidth)
 56          {
 57              Debug.Assert(target.Type == OperandType.V128 && value.Type == OperandType.V128);
 58  
 59              // Insert from index 0 in value to index in target.
 60              int index = reg & (doubleWidth ? 1 : 3);
 61  
 62              if (doubleWidth)
 63              {
 64                  return context.AddIntrinsic(Intrinsic.Arm64InsVe | Intrinsic.Arm64VDWord, target, Const(index), value, Const(0));
 65              }
 66              else
 67              {
 68                  return context.AddIntrinsic(Intrinsic.Arm64InsVe | Intrinsic.Arm64VWord, target, Const(index), value, Const(0));
 69              }
 70          }
 71  
 72          public static Operand EmitExtractScalar(ArmEmitterContext context, Operand target, int reg, bool doubleWidth)
 73          {
 74              int index = reg & (doubleWidth ? 1 : 3);
 75              if (index == 0)
 76              {
 77                  return target; // Element is already at index 0, so just return the vector directly.
 78              }
 79  
 80              if (doubleWidth)
 81              {
 82                  return context.AddIntrinsic(Intrinsic.Arm64DupSe | Intrinsic.Arm64VDWord, target, Const(1)); // Extract high (index 1).
 83              }
 84              else
 85              {
 86                  return context.AddIntrinsic(Intrinsic.Arm64DupSe | Intrinsic.Arm64VWord, target, Const(index)); // Extract element at index.
 87              }
 88          }
 89  
 90          // Vector Operand Templates
 91  
 92          public static void EmitVectorUnaryOpSimd32(ArmEmitterContext context, Func1I vectorFunc)
 93          {
 94              OpCode32Simd op = (OpCode32Simd)context.CurrOp;
 95  
 96              Operand m = GetVecA32(op.Qm);
 97              Operand d = GetVecA32(op.Qd);
 98  
 99              if (!op.Q) // Register swap: move relevant doubleword to destination side.
100              {
101                  m = EmitMoveDoubleWordToSide(context, m, op.Vm, op.Vd);
102              }
103  
104              Operand res = vectorFunc(m);
105  
106              if (!op.Q) // Register insert.
107              {
108                  res = EmitDoubleWordInsert(context, d, res, op.Vd);
109              }
110  
111              context.Copy(d, res);
112          }
113  
114          public static void EmitVectorUnaryOpF32(ArmEmitterContext context, Intrinsic inst)
115          {
116              OpCode32Simd op = (OpCode32Simd)context.CurrOp;
117  
118              inst |= ((op.Size & 1) != 0 ? Intrinsic.Arm64VDouble : Intrinsic.Arm64VFloat) | Intrinsic.Arm64V128;
119              EmitVectorUnaryOpSimd32(context, (m) => context.AddIntrinsic(inst, m));
120          }
121  
122          public static void EmitVectorBinaryOpSimd32(ArmEmitterContext context, Func2I vectorFunc, int side = -1)
123          {
124              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
125  
126              Operand n = GetVecA32(op.Qn);
127              Operand m = GetVecA32(op.Qm);
128              Operand d = GetVecA32(op.Qd);
129  
130              if (side == -1)
131              {
132                  side = op.Vd;
133              }
134  
135              if (!op.Q) // Register swap: move relevant doubleword to destination side.
136              {
137                  n = EmitMoveDoubleWordToSide(context, n, op.Vn, side);
138                  m = EmitMoveDoubleWordToSide(context, m, op.Vm, side);
139              }
140  
141              Operand res = vectorFunc(n, m);
142  
143              if (!op.Q) // Register insert.
144              {
145                  if (side != op.Vd)
146                  {
147                      res = EmitMoveDoubleWordToSide(context, res, side, op.Vd);
148                  }
149                  res = EmitDoubleWordInsert(context, d, res, op.Vd);
150              }
151  
152              context.Copy(d, res);
153          }
154  
155          public static void EmitVectorBinaryOpF32(ArmEmitterContext context, Intrinsic inst)
156          {
157              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
158  
159              inst |= ((op.Size & 1) != 0 ? Intrinsic.Arm64VDouble : Intrinsic.Arm64VFloat) | Intrinsic.Arm64V128;
160              EmitVectorBinaryOpSimd32(context, (n, m) => context.AddIntrinsic(inst, n, m));
161          }
162  
163          public static void EmitVectorTernaryOpSimd32(ArmEmitterContext context, Func3I vectorFunc)
164          {
165              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
166  
167              Operand n = GetVecA32(op.Qn);
168              Operand m = GetVecA32(op.Qm);
169              Operand d = GetVecA32(op.Qd);
170              Operand initialD = d;
171  
172              if (!op.Q) // Register swap: move relevant doubleword to destination side.
173              {
174                  n = EmitMoveDoubleWordToSide(context, n, op.Vn, op.Vd);
175                  m = EmitMoveDoubleWordToSide(context, m, op.Vm, op.Vd);
176              }
177  
178              Operand res = vectorFunc(d, n, m);
179  
180              if (!op.Q) // Register insert.
181              {
182                  res = EmitDoubleWordInsert(context, initialD, res, op.Vd);
183              }
184  
185              context.Copy(initialD, res);
186          }
187  
188          public static void EmitVectorTernaryOpF32(ArmEmitterContext context, Intrinsic inst)
189          {
190              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
191  
192              inst |= ((op.Size & 1) != 0 ? Intrinsic.Arm64VDouble : Intrinsic.Arm64VFloat) | Intrinsic.Arm64V128;
193              EmitVectorTernaryOpSimd32(context, (d, n, m) => context.AddIntrinsic(inst, d, n, m));
194          }
195  
196          public static void EmitScalarUnaryOpSimd32(ArmEmitterContext context, Func1I scalarFunc, bool doubleSize)
197          {
198              OpCode32SimdS op = (OpCode32SimdS)context.CurrOp;
199  
200              int shift = doubleSize ? 1 : 2;
201              Operand m = GetVecA32(op.Vm >> shift);
202              Operand d = GetVecA32(op.Vd >> shift);
203  
204              m = EmitExtractScalar(context, m, op.Vm, doubleSize);
205  
206              Operand res = scalarFunc(m);
207  
208              // Insert scalar into vector.
209              res = EmitScalarInsert(context, d, res, op.Vd, doubleSize);
210  
211              context.Copy(d, res);
212          }
213  
214          public static void EmitScalarUnaryOpF32(ArmEmitterContext context, Intrinsic inst)
215          {
216              OpCode32SimdS op = (OpCode32SimdS)context.CurrOp;
217  
218              EmitScalarUnaryOpF32(context, inst, (op.Size & 1) != 0);
219          }
220  
221          public static void EmitScalarUnaryOpF32(ArmEmitterContext context, Intrinsic inst, bool doubleSize)
222          {
223              inst |= (doubleSize ? Intrinsic.Arm64VDouble : Intrinsic.Arm64VFloat) | Intrinsic.Arm64V128;
224              EmitScalarUnaryOpSimd32(context, (m) => (inst == 0) ? m : context.AddIntrinsic(inst, m), doubleSize);
225          }
226  
227          public static void EmitScalarBinaryOpSimd32(ArmEmitterContext context, Func2I scalarFunc)
228          {
229              OpCode32SimdRegS op = (OpCode32SimdRegS)context.CurrOp;
230  
231              bool doubleSize = (op.Size & 1) != 0;
232              int shift = doubleSize ? 1 : 2;
233              Operand n = GetVecA32(op.Vn >> shift);
234              Operand m = GetVecA32(op.Vm >> shift);
235              Operand d = GetVecA32(op.Vd >> shift);
236  
237              n = EmitExtractScalar(context, n, op.Vn, doubleSize);
238              m = EmitExtractScalar(context, m, op.Vm, doubleSize);
239  
240              Operand res = scalarFunc(n, m);
241  
242              // Insert scalar into vector.
243              res = EmitScalarInsert(context, d, res, op.Vd, doubleSize);
244  
245              context.Copy(d, res);
246          }
247  
248          public static void EmitScalarBinaryOpF32(ArmEmitterContext context, Intrinsic inst)
249          {
250              OpCode32SimdRegS op = (OpCode32SimdRegS)context.CurrOp;
251  
252              inst |= ((op.Size & 1) != 0 ? Intrinsic.Arm64VDouble : Intrinsic.Arm64VFloat) | Intrinsic.Arm64V128;
253              EmitScalarBinaryOpSimd32(context, (n, m) => context.AddIntrinsic(inst, n, m));
254          }
255  
256          public static void EmitScalarTernaryOpSimd32(ArmEmitterContext context, Func3I scalarFunc)
257          {
258              OpCode32SimdRegS op = (OpCode32SimdRegS)context.CurrOp;
259  
260              bool doubleSize = (op.Size & 1) != 0;
261              int shift = doubleSize ? 1 : 2;
262              Operand n = GetVecA32(op.Vn >> shift);
263              Operand m = GetVecA32(op.Vm >> shift);
264              Operand d = GetVecA32(op.Vd >> shift);
265              Operand initialD = d;
266  
267              n = EmitExtractScalar(context, n, op.Vn, doubleSize);
268              m = EmitExtractScalar(context, m, op.Vm, doubleSize);
269              d = EmitExtractScalar(context, d, op.Vd, doubleSize);
270  
271              Operand res = scalarFunc(d, n, m);
272  
273              // Insert scalar into vector.
274              res = EmitScalarInsert(context, initialD, res, op.Vd, doubleSize);
275  
276              context.Copy(initialD, res);
277          }
278  
279          public static void EmitScalarTernaryOpF32(ArmEmitterContext context, Intrinsic inst)
280          {
281              OpCode32SimdRegS op = (OpCode32SimdRegS)context.CurrOp;
282  
283              inst |= ((op.Size & 1) != 0 ? Intrinsic.Arm64VDouble : Intrinsic.Arm64VFloat) | Intrinsic.Arm64V128;
284              EmitScalarTernaryOpSimd32(context, (d, n, m) => context.AddIntrinsic(inst, d, n, m));
285          }
286  
287          // Pairwise
288  
289          public static void EmitVectorPairwiseOpF32(ArmEmitterContext context, Intrinsic inst32)
290          {
291              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
292  
293              inst32 |= Intrinsic.Arm64V64 | Intrinsic.Arm64VFloat;
294              EmitVectorBinaryOpSimd32(context, (n, m) => context.AddIntrinsic(inst32, n, m), 0);
295          }
296  
297          public static void EmitVcmpOrVcmpe(ArmEmitterContext context, bool signalNaNs)
298          {
299              OpCode32SimdS op = (OpCode32SimdS)context.CurrOp;
300  
301              bool cmpWithZero = (op.Opc & 2) != 0;
302  
303              Intrinsic inst = signalNaNs ? Intrinsic.Arm64FcmpeS : Intrinsic.Arm64FcmpS;
304              inst |= ((op.Size & 1) != 0 ? Intrinsic.Arm64VDouble : Intrinsic.Arm64VFloat) | Intrinsic.Arm64V128;
305  
306              bool doubleSize = (op.Size & 1) != 0;
307              int shift = doubleSize ? 1 : 2;
308              Operand n = GetVecA32(op.Vd >> shift);
309              Operand m = GetVecA32(op.Vm >> shift);
310  
311              n = EmitExtractScalar(context, n, op.Vd, doubleSize);
312              m = cmpWithZero ? Const(0) : EmitExtractScalar(context, m, op.Vm, doubleSize);
313  
314              Operand nzcv = context.AddIntrinsicInt(inst, n, m);
315  
316              Operand one = Const(1);
317  
318              SetFpFlag(context, FPState.VFlag, context.BitwiseAnd(context.ShiftRightUI(nzcv, Const(28)), one));
319              SetFpFlag(context, FPState.CFlag, context.BitwiseAnd(context.ShiftRightUI(nzcv, Const(29)), one));
320              SetFpFlag(context, FPState.ZFlag, context.BitwiseAnd(context.ShiftRightUI(nzcv, Const(30)), one));
321              SetFpFlag(context, FPState.NFlag, context.BitwiseAnd(context.ShiftRightUI(nzcv, Const(31)), one));
322          }
323  
324          public static void EmitCmpOpF32(ArmEmitterContext context, CmpCondition cond, bool zero)
325          {
326              OpCode32Simd op = (OpCode32Simd)context.CurrOp;
327  
328              int sizeF = op.Size & 1;
329  
330              Intrinsic inst;
331              if (zero)
332              {
333                  inst = cond switch
334                  {
335                      CmpCondition.Equal => Intrinsic.Arm64FcmeqVz,
336                      CmpCondition.GreaterThan => Intrinsic.Arm64FcmgtVz,
337                      CmpCondition.GreaterThanOrEqual => Intrinsic.Arm64FcmgeVz,
338                      CmpCondition.LessThan => Intrinsic.Arm64FcmltVz,
339                      CmpCondition.LessThanOrEqual => Intrinsic.Arm64FcmleVz,
340                      _ => throw new InvalidOperationException(),
341                  };
342              }
343              else
344              {
345                  inst = cond switch
346                  {
347                      CmpCondition.Equal => Intrinsic.Arm64FcmeqV,
348                      CmpCondition.GreaterThan => Intrinsic.Arm64FcmgtV,
349                      CmpCondition.GreaterThanOrEqual => Intrinsic.Arm64FcmgeV,
350                      _ => throw new InvalidOperationException(),
351                  };
352              }
353  
354              inst |= (sizeF != 0 ? Intrinsic.Arm64VDouble : Intrinsic.Arm64VFloat) | Intrinsic.Arm64V128;
355  
356              if (zero)
357              {
358                  EmitVectorUnaryOpSimd32(context, (m) =>
359                  {
360                      return context.AddIntrinsic(inst, m);
361                  });
362              }
363              else
364              {
365                  EmitVectorBinaryOpSimd32(context, (n, m) =>
366                  {
367                      return context.AddIntrinsic(inst, n, m);
368                  });
369              }
370          }
371      }
372  }