1  using ARMeilleure.Decoders;
   2  using ARMeilleure.IntermediateRepresentation;
   3  using ARMeilleure.Translation;
   4  using System;
   5  using System.Diagnostics;
   6  using System.Reflection;
   7  using static ARMeilleure.Instructions.InstEmitHelper;
   8  using static ARMeilleure.Instructions.InstEmitSimdHelper;
   9  using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
  10  
  11  namespace ARMeilleure.Instructions
  12  {
  13      using Func1I = Func<Operand, Operand>;
  14      using Func2I = Func<Operand, Operand, Operand>;
  15      using Func3I = Func<Operand, Operand, Operand, Operand>;
  16  
  17      static class InstEmitSimdHelper32
  18      {
  19          public static (int, int) GetQuadwordAndSubindex(int index, RegisterSize size)
  20          {
  21              return size switch
  22              {
  23                  RegisterSize.Simd128 => (index >> 1, 0),
  24                  RegisterSize.Simd64 or RegisterSize.Int64 => (index >> 1, index & 1),
  25                  RegisterSize.Int32 => (index >> 2, index & 3),
  26                  _ => throw new ArgumentException("Unrecognized Vector Register Size."),
  27              };
  28          }
  29  
  30          public static Operand ExtractScalar(ArmEmitterContext context, OperandType type, int reg)
  31          {
  32              Debug.Assert(type != OperandType.V128);
  33  
  34              if (type == OperandType.FP64 || type == OperandType.I64)
  35              {
  36                  // From dreg.
  37                  return context.VectorExtract(type, GetVecA32(reg >> 1), reg & 1);
  38              }
  39              else
  40              {
  41                  // From sreg.
  42                  return context.VectorExtract(type, GetVecA32(reg >> 2), reg & 3);
  43              }
  44          }
  45  
  46          public static void InsertScalar(ArmEmitterContext context, int reg, Operand value)
  47          {
  48              Debug.Assert(value.Type != OperandType.V128);
  49  
  50              Operand vec, insert;
  51              if (value.Type == OperandType.FP64 || value.Type == OperandType.I64)
  52              {
  53                  // From dreg.
  54                  vec = GetVecA32(reg >> 1);
  55                  insert = context.VectorInsert(vec, value, reg & 1);
  56              }
  57              else
  58              {
  59                  // From sreg.
  60                  vec = GetVecA32(reg >> 2);
  61                  insert = context.VectorInsert(vec, value, reg & 3);
  62              }
  63  
  64              context.Copy(vec, insert);
  65          }
  66  
  67          public static Operand ExtractScalar16(ArmEmitterContext context, int reg, bool top)
  68          {
  69              return context.VectorExtract16(GetVecA32(reg >> 2), ((reg & 3) << 1) | (top ? 1 : 0));
  70          }
  71  
  72          public static void InsertScalar16(ArmEmitterContext context, int reg, bool top, Operand value)
  73          {
  74              Debug.Assert(value.Type == OperandType.FP32 || value.Type == OperandType.I32);
  75  
  76              Operand vec, insert;
  77              vec = GetVecA32(reg >> 2);
  78              insert = context.VectorInsert16(vec, value, ((reg & 3) << 1) | (top ? 1 : 0));
  79  
  80              context.Copy(vec, insert);
  81          }
  82  
  83          public static Operand ExtractElement(ArmEmitterContext context, int reg, int size, bool signed)
  84          {
  85              return EmitVectorExtract32(context, reg >> (4 - size), reg & ((16 >> size) - 1), size, signed);
  86          }
  87  
  88          public static void EmitVectorImmUnaryOp32(ArmEmitterContext context, Func1I emit)
  89          {
  90              IOpCode32SimdImm op = (IOpCode32SimdImm)context.CurrOp;
  91  
  92              Operand imm = Const(op.Immediate);
  93  
  94              int elems = op.Elems;
  95              (int index, int subIndex) = GetQuadwordAndSubindex(op.Vd, op.RegisterSize);
  96  
  97              Operand vec = GetVecA32(index);
  98              Operand res = vec;
  99  
 100              for (int item = 0; item < elems; item++)
 101              {
 102                  res = EmitVectorInsert(context, res, emit(imm), item + subIndex * elems, op.Size);
 103              }
 104  
 105              context.Copy(vec, res);
 106          }
 107  
 108          public static void EmitScalarUnaryOpF32(ArmEmitterContext context, Func1I emit)
 109          {
 110              OpCode32SimdS op = (OpCode32SimdS)context.CurrOp;
 111  
 112              OperandType type = (op.Size & 1) != 0 ? OperandType.FP64 : OperandType.FP32;
 113  
 114              Operand m = ExtractScalar(context, type, op.Vm);
 115  
 116              InsertScalar(context, op.Vd, emit(m));
 117          }
 118  
 119          public static void EmitScalarBinaryOpF32(ArmEmitterContext context, Func2I emit)
 120          {
 121              OpCode32SimdRegS op = (OpCode32SimdRegS)context.CurrOp;
 122  
 123              OperandType type = (op.Size & 1) != 0 ? OperandType.FP64 : OperandType.FP32;
 124  
 125              Operand n = ExtractScalar(context, type, op.Vn);
 126              Operand m = ExtractScalar(context, type, op.Vm);
 127  
 128              InsertScalar(context, op.Vd, emit(n, m));
 129          }
 130  
 131          public static void EmitScalarBinaryOpI32(ArmEmitterContext context, Func2I emit)
 132          {
 133              OpCode32SimdRegS op = (OpCode32SimdRegS)context.CurrOp;
 134  
 135              OperandType type = (op.Size & 1) != 0 ? OperandType.I64 : OperandType.I32;
 136  
 137              if (op.Size < 2)
 138              {
 139                  throw new NotSupportedException("Cannot perform a scalar SIMD operation on integers smaller than 32 bits.");
 140              }
 141  
 142              Operand n = ExtractScalar(context, type, op.Vn);
 143              Operand m = ExtractScalar(context, type, op.Vm);
 144  
 145              InsertScalar(context, op.Vd, emit(n, m));
 146          }
 147  
 148          public static void EmitScalarTernaryOpF32(ArmEmitterContext context, Func3I emit)
 149          {
 150              OpCode32SimdRegS op = (OpCode32SimdRegS)context.CurrOp;
 151  
 152              OperandType type = (op.Size & 1) != 0 ? OperandType.FP64 : OperandType.FP32;
 153  
 154              Operand a = ExtractScalar(context, type, op.Vd);
 155              Operand n = ExtractScalar(context, type, op.Vn);
 156              Operand m = ExtractScalar(context, type, op.Vm);
 157  
 158              InsertScalar(context, op.Vd, emit(a, n, m));
 159          }
 160  
 161          public static void EmitVectorUnaryOpF32(ArmEmitterContext context, Func1I emit)
 162          {
 163              OpCode32Simd op = (OpCode32Simd)context.CurrOp;
 164  
 165              int sizeF = op.Size & 1;
 166  
 167              OperandType type = sizeF != 0 ? OperandType.FP64 : OperandType.FP32;
 168  
 169              int elems = op.GetBytesCount() >> sizeF + 2;
 170  
 171              Operand res = GetVecA32(op.Qd);
 172  
 173              for (int index = 0; index < elems; index++)
 174              {
 175                  Operand me = context.VectorExtract(type, GetVecA32(op.Qm), op.Fm + index);
 176  
 177                  res = context.VectorInsert(res, emit(me), op.Fd + index);
 178              }
 179  
 180              context.Copy(GetVecA32(op.Qd), res);
 181          }
 182  
 183          public static void EmitVectorBinaryOpF32(ArmEmitterContext context, Func2I emit)
 184          {
 185              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
 186  
 187              int sizeF = op.Size & 1;
 188  
 189              OperandType type = sizeF != 0 ? OperandType.FP64 : OperandType.FP32;
 190  
 191              int elems = op.GetBytesCount() >> (sizeF + 2);
 192  
 193              Operand res = GetVecA32(op.Qd);
 194  
 195              for (int index = 0; index < elems; index++)
 196              {
 197                  Operand ne = context.VectorExtract(type, GetVecA32(op.Qn), op.Fn + index);
 198                  Operand me = context.VectorExtract(type, GetVecA32(op.Qm), op.Fm + index);
 199  
 200                  res = context.VectorInsert(res, emit(ne, me), op.Fd + index);
 201              }
 202  
 203              context.Copy(GetVecA32(op.Qd), res);
 204          }
 205  
 206          public static void EmitVectorTernaryOpF32(ArmEmitterContext context, Func3I emit)
 207          {
 208              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
 209  
 210              int sizeF = op.Size & 1;
 211  
 212              OperandType type = sizeF != 0 ? OperandType.FP64 : OperandType.FP32;
 213  
 214              int elems = op.GetBytesCount() >> sizeF + 2;
 215  
 216              Operand res = GetVecA32(op.Qd);
 217  
 218              for (int index = 0; index < elems; index++)
 219              {
 220                  Operand de = context.VectorExtract(type, GetVecA32(op.Qd), op.Fd + index);
 221                  Operand ne = context.VectorExtract(type, GetVecA32(op.Qn), op.Fn + index);
 222                  Operand me = context.VectorExtract(type, GetVecA32(op.Qm), op.Fm + index);
 223  
 224                  res = context.VectorInsert(res, emit(de, ne, me), op.Fd + index);
 225              }
 226  
 227              context.Copy(GetVecA32(op.Qd), res);
 228          }
 229  
 230          // Integer
 231  
 232          public static void EmitVectorUnaryAccumulateOpI32(ArmEmitterContext context, Func1I emit, bool signed)
 233          {
 234              OpCode32Simd op = (OpCode32Simd)context.CurrOp;
 235  
 236              Operand res = GetVecA32(op.Qd);
 237  
 238              int elems = op.GetBytesCount() >> op.Size;
 239  
 240              for (int index = 0; index < elems; index++)
 241              {
 242                  Operand de = EmitVectorExtract32(context, op.Qd, op.Id + index, op.Size, signed);
 243                  Operand me = EmitVectorExtract32(context, op.Qm, op.Im + index, op.Size, signed);
 244  
 245                  res = EmitVectorInsert(context, res, context.Add(de, emit(me)), op.Id + index, op.Size);
 246              }
 247  
 248              context.Copy(GetVecA32(op.Qd), res);
 249          }
 250  
 251          public static void EmitVectorUnaryOpI32(ArmEmitterContext context, Func1I emit, bool signed)
 252          {
 253              OpCode32Simd op = (OpCode32Simd)context.CurrOp;
 254  
 255              Operand res = GetVecA32(op.Qd);
 256  
 257              int elems = op.GetBytesCount() >> op.Size;
 258  
 259              for (int index = 0; index < elems; index++)
 260              {
 261                  Operand me = EmitVectorExtract32(context, op.Qm, op.Im + index, op.Size, signed);
 262  
 263                  res = EmitVectorInsert(context, res, emit(me), op.Id + index, op.Size);
 264              }
 265  
 266              context.Copy(GetVecA32(op.Qd), res);
 267          }
 268  
 269          public static void EmitVectorBinaryOpI32(ArmEmitterContext context, Func2I emit, bool signed)
 270          {
 271              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
 272  
 273              Operand res = GetVecA32(op.Qd);
 274  
 275              int elems = op.GetBytesCount() >> op.Size;
 276  
 277              for (int index = 0; index < elems; index++)
 278              {
 279                  Operand ne = EmitVectorExtract32(context, op.Qn, op.In + index, op.Size, signed);
 280                  Operand me = EmitVectorExtract32(context, op.Qm, op.Im + index, op.Size, signed);
 281  
 282                  res = EmitVectorInsert(context, res, emit(ne, me), op.Id + index, op.Size);
 283              }
 284  
 285              context.Copy(GetVecA32(op.Qd), res);
 286          }
 287  
 288          public static void EmitVectorBinaryLongOpI32(ArmEmitterContext context, Func2I emit, bool signed)
 289          {
 290              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
 291  
 292              Operand res = context.VectorZero();
 293  
 294              int elems = op.GetBytesCount() >> op.Size;
 295  
 296              for (int index = 0; index < elems; index++)
 297              {
 298                  Operand ne = EmitVectorExtract32(context, op.Qn, op.In + index, op.Size, signed);
 299                  Operand me = EmitVectorExtract32(context, op.Qm, op.Im + index, op.Size, signed);
 300  
 301                  if (op.Size == 2)
 302                  {
 303                      ne = signed ? context.SignExtend32(OperandType.I64, ne) : context.ZeroExtend32(OperandType.I64, ne);
 304                      me = signed ? context.SignExtend32(OperandType.I64, me) : context.ZeroExtend32(OperandType.I64, me);
 305                  }
 306  
 307                  res = EmitVectorInsert(context, res, emit(ne, me), index, op.Size + 1);
 308              }
 309  
 310              context.Copy(GetVecA32(op.Qd), res);
 311          }
 312  
 313          public static void EmitVectorBinaryWideOpI32(ArmEmitterContext context, Func2I emit, bool signed)
 314          {
 315              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
 316  
 317              Operand res = context.VectorZero();
 318  
 319              int elems = op.GetBytesCount() >> op.Size;
 320  
 321              for (int index = 0; index < elems; index++)
 322              {
 323                  Operand ne = EmitVectorExtract32(context, op.Qn, op.In + index, op.Size + 1, signed);
 324                  Operand me = EmitVectorExtract32(context, op.Qm, op.Im + index, op.Size, signed);
 325  
 326                  if (op.Size == 2)
 327                  {
 328                      me = signed ? context.SignExtend32(OperandType.I64, me) : context.ZeroExtend32(OperandType.I64, me);
 329                  }
 330  
 331                  res = EmitVectorInsert(context, res, emit(ne, me), index, op.Size + 1);
 332              }
 333  
 334              context.Copy(GetVecA32(op.Qd), res);
 335          }
 336  
 337          public static void EmitVectorImmBinaryQdQmOpZx32(ArmEmitterContext context, Func2I emit)
 338          {
 339              EmitVectorImmBinaryQdQmOpI32(context, emit, false);
 340          }
 341  
 342          public static void EmitVectorImmBinaryQdQmOpSx32(ArmEmitterContext context, Func2I emit)
 343          {
 344              EmitVectorImmBinaryQdQmOpI32(context, emit, true);
 345          }
 346  
 347          public static void EmitVectorImmBinaryQdQmOpI32(ArmEmitterContext context, Func2I emit, bool signed)
 348          {
 349              OpCode32SimdShImm op = (OpCode32SimdShImm)context.CurrOp;
 350  
 351              Operand res = GetVecA32(op.Qd);
 352  
 353              int elems = op.GetBytesCount() >> op.Size;
 354  
 355              for (int index = 0; index < elems; index++)
 356              {
 357                  Operand de = EmitVectorExtract32(context, op.Qd, op.Id + index, op.Size, signed);
 358                  Operand me = EmitVectorExtract32(context, op.Qm, op.Im + index, op.Size, signed);
 359  
 360                  res = EmitVectorInsert(context, res, emit(de, me), op.Id + index, op.Size);
 361              }
 362  
 363              context.Copy(GetVecA32(op.Qd), res);
 364          }
 365  
 366          public static void EmitVectorTernaryLongOpI32(ArmEmitterContext context, Func3I emit, bool signed)
 367          {
 368              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
 369  
 370              Operand res = context.VectorZero();
 371  
 372              int elems = op.GetBytesCount() >> op.Size;
 373  
 374              for (int index = 0; index < elems; index++)
 375              {
 376                  Operand de = EmitVectorExtract32(context, op.Qd, op.Id + index, op.Size + 1, signed);
 377                  Operand ne = EmitVectorExtract32(context, op.Qn, op.In + index, op.Size, signed);
 378                  Operand me = EmitVectorExtract32(context, op.Qm, op.Im + index, op.Size, signed);
 379  
 380                  if (op.Size == 2)
 381                  {
 382                      ne = signed ? context.SignExtend32(OperandType.I64, ne) : context.ZeroExtend32(OperandType.I64, ne);
 383                      me = signed ? context.SignExtend32(OperandType.I64, me) : context.ZeroExtend32(OperandType.I64, me);
 384                  }
 385  
 386                  res = EmitVectorInsert(context, res, emit(de, ne, me), index, op.Size + 1);
 387              }
 388  
 389              context.Copy(GetVecA32(op.Qd), res);
 390          }
 391  
 392          public static void EmitVectorTernaryOpI32(ArmEmitterContext context, Func3I emit, bool signed)
 393          {
 394              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
 395  
 396              Operand res = GetVecA32(op.Qd);
 397  
 398              int elems = op.GetBytesCount() >> op.Size;
 399  
 400              for (int index = 0; index < elems; index++)
 401              {
 402                  Operand de = EmitVectorExtract32(context, op.Qd, op.Id + index, op.Size, signed);
 403                  Operand ne = EmitVectorExtract32(context, op.Qn, op.In + index, op.Size, signed);
 404                  Operand me = EmitVectorExtract32(context, op.Qm, op.Im + index, op.Size, signed);
 405  
 406                  res = EmitVectorInsert(context, res, emit(de, ne, me), op.Id + index, op.Size);
 407              }
 408  
 409              context.Copy(GetVecA32(op.Qd), res);
 410          }
 411  
 412          public static void EmitVectorUnaryOpSx32(ArmEmitterContext context, Func1I emit)
 413          {
 414              EmitVectorUnaryOpI32(context, emit, true);
 415          }
 416  
 417          public static void EmitVectorUnaryOpSx32(ArmEmitterContext context, Func1I emit, bool accumulate)
 418          {
 419              if (accumulate)
 420              {
 421                  EmitVectorUnaryAccumulateOpI32(context, emit, true);
 422              }
 423              else
 424              {
 425                  EmitVectorUnaryOpI32(context, emit, true);
 426              }
 427          }
 428  
 429          public static void EmitVectorBinaryOpSx32(ArmEmitterContext context, Func2I emit)
 430          {
 431              EmitVectorBinaryOpI32(context, emit, true);
 432          }
 433  
 434          public static void EmitVectorTernaryOpSx32(ArmEmitterContext context, Func3I emit)
 435          {
 436              EmitVectorTernaryOpI32(context, emit, true);
 437          }
 438  
 439          public static void EmitVectorUnaryOpZx32(ArmEmitterContext context, Func1I emit)
 440          {
 441              EmitVectorUnaryOpI32(context, emit, false);
 442          }
 443  
 444          public static void EmitVectorUnaryOpZx32(ArmEmitterContext context, Func1I emit, bool accumulate)
 445          {
 446              if (accumulate)
 447              {
 448                  EmitVectorUnaryAccumulateOpI32(context, emit, false);
 449              }
 450              else
 451              {
 452                  EmitVectorUnaryOpI32(context, emit, false);
 453              }
 454          }
 455  
 456          public static void EmitVectorBinaryOpZx32(ArmEmitterContext context, Func2I emit)
 457          {
 458              EmitVectorBinaryOpI32(context, emit, false);
 459          }
 460  
 461          public static void EmitVectorTernaryOpZx32(ArmEmitterContext context, Func3I emit)
 462          {
 463              EmitVectorTernaryOpI32(context, emit, false);
 464          }
 465  
 466          // Vector by scalar
 467  
 468          public static void EmitVectorByScalarOpF32(ArmEmitterContext context, Func2I emit)
 469          {
 470              OpCode32SimdRegElem op = (OpCode32SimdRegElem)context.CurrOp;
 471  
 472              int sizeF = op.Size & 1;
 473  
 474              OperandType type = sizeF != 0 ? OperandType.FP64 : OperandType.FP32;
 475  
 476              int elems = op.GetBytesCount() >> sizeF + 2;
 477  
 478              Operand m = ExtractScalar(context, type, op.Vm);
 479  
 480              Operand res = GetVecA32(op.Qd);
 481  
 482              for (int index = 0; index < elems; index++)
 483              {
 484                  Operand ne = context.VectorExtract(type, GetVecA32(op.Qn), op.Fn + index);
 485  
 486                  res = context.VectorInsert(res, emit(ne, m), op.Fd + index);
 487              }
 488  
 489              context.Copy(GetVecA32(op.Qd), res);
 490          }
 491  
 492          public static void EmitVectorByScalarOpI32(ArmEmitterContext context, Func2I emit, bool signed)
 493          {
 494              OpCode32SimdRegElem op = (OpCode32SimdRegElem)context.CurrOp;
 495  
 496              Operand m = ExtractElement(context, op.Vm, op.Size, signed);
 497  
 498              Operand res = GetVecA32(op.Qd);
 499  
 500              int elems = op.GetBytesCount() >> op.Size;
 501  
 502              for (int index = 0; index < elems; index++)
 503              {
 504                  Operand ne = EmitVectorExtract32(context, op.Qn, op.In + index, op.Size, signed);
 505  
 506                  res = EmitVectorInsert(context, res, emit(ne, m), op.Id + index, op.Size);
 507              }
 508  
 509              context.Copy(GetVecA32(op.Qd), res);
 510          }
 511  
 512          public static void EmitVectorByScalarLongOpI32(ArmEmitterContext context, Func2I emit, bool signed)
 513          {
 514              OpCode32SimdRegElem op = (OpCode32SimdRegElem)context.CurrOp;
 515  
 516              Operand m = ExtractElement(context, op.Vm, op.Size, signed);
 517  
 518              if (op.Size == 2)
 519              {
 520                  m = signed ? context.SignExtend32(OperandType.I64, m) : context.ZeroExtend32(OperandType.I64, m);
 521              }
 522  
 523              Operand res = context.VectorZero();
 524  
 525              int elems = op.GetBytesCount() >> op.Size;
 526  
 527              for (int index = 0; index < elems; index++)
 528              {
 529                  Operand ne = EmitVectorExtract32(context, op.Qn, op.In + index, op.Size, signed);
 530  
 531                  if (op.Size == 2)
 532                  {
 533                      ne = signed ? context.SignExtend32(OperandType.I64, ne) : context.ZeroExtend32(OperandType.I64, ne);
 534                  }
 535  
 536                  res = EmitVectorInsert(context, res, emit(ne, m), index, op.Size + 1);
 537              }
 538  
 539              context.Copy(GetVecA32(op.Qd), res);
 540          }
 541  
 542          public static void EmitVectorsByScalarOpF32(ArmEmitterContext context, Func3I emit)
 543          {
 544              OpCode32SimdRegElem op = (OpCode32SimdRegElem)context.CurrOp;
 545  
 546              int sizeF = op.Size & 1;
 547  
 548              OperandType type = sizeF != 0 ? OperandType.FP64 : OperandType.FP32;
 549  
 550              int elems = op.GetBytesCount() >> sizeF + 2;
 551  
 552              Operand m = ExtractScalar(context, type, op.Vm);
 553  
 554              Operand res = GetVecA32(op.Qd);
 555  
 556              for (int index = 0; index < elems; index++)
 557              {
 558                  Operand de = context.VectorExtract(type, GetVecA32(op.Qd), op.Fd + index);
 559                  Operand ne = context.VectorExtract(type, GetVecA32(op.Qn), op.Fn + index);
 560  
 561                  res = context.VectorInsert(res, emit(de, ne, m), op.Fd + index);
 562              }
 563  
 564              context.Copy(GetVecA32(op.Qd), res);
 565          }
 566  
 567          public static void EmitVectorsByScalarOpI32(ArmEmitterContext context, Func3I emit, bool signed)
 568          {
 569              OpCode32SimdRegElem op = (OpCode32SimdRegElem)context.CurrOp;
 570  
 571              Operand m = EmitVectorExtract32(context, op.Vm >> (4 - op.Size), op.Vm & ((1 << (4 - op.Size)) - 1), op.Size, signed);
 572  
 573              Operand res = GetVecA32(op.Qd);
 574  
 575              int elems = op.GetBytesCount() >> op.Size;
 576  
 577              for (int index = 0; index < elems; index++)
 578              {
 579                  Operand de = EmitVectorExtract32(context, op.Qd, op.Id + index, op.Size, signed);
 580                  Operand ne = EmitVectorExtract32(context, op.Qn, op.In + index, op.Size, signed);
 581  
 582                  res = EmitVectorInsert(context, res, emit(de, ne, m), op.Id + index, op.Size);
 583              }
 584  
 585              context.Copy(GetVecA32(op.Qd), res);
 586          }
 587  
 588          // Pairwise
 589  
 590          public static void EmitVectorPairwiseOpF32(ArmEmitterContext context, Func2I emit)
 591          {
 592              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
 593  
 594              int sizeF = op.Size & 1;
 595  
 596              OperandType type = sizeF != 0 ? OperandType.FP64 : OperandType.FP32;
 597  
 598              int elems = op.GetBytesCount() >> (sizeF + 2);
 599              int pairs = elems >> 1;
 600  
 601              Operand res = GetVecA32(op.Qd);
 602              Operand mvec = GetVecA32(op.Qm);
 603              Operand nvec = GetVecA32(op.Qn);
 604  
 605              for (int index = 0; index < pairs; index++)
 606              {
 607                  int pairIndex = index << 1;
 608  
 609                  Operand n1 = context.VectorExtract(type, nvec, op.Fn + pairIndex);
 610                  Operand n2 = context.VectorExtract(type, nvec, op.Fn + pairIndex + 1);
 611  
 612                  res = context.VectorInsert(res, emit(n1, n2), op.Fd + index);
 613  
 614                  Operand m1 = context.VectorExtract(type, mvec, op.Fm + pairIndex);
 615                  Operand m2 = context.VectorExtract(type, mvec, op.Fm + pairIndex + 1);
 616  
 617                  res = context.VectorInsert(res, emit(m1, m2), op.Fd + index + pairs);
 618              }
 619  
 620              context.Copy(GetVecA32(op.Qd), res);
 621          }
 622  
 623          public static void EmitVectorPairwiseOpI32(ArmEmitterContext context, Func2I emit, bool signed)
 624          {
 625              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
 626  
 627              int elems = op.GetBytesCount() >> op.Size;
 628              int pairs = elems >> 1;
 629  
 630              Operand res = GetVecA32(op.Qd);
 631  
 632              for (int index = 0; index < pairs; index++)
 633              {
 634                  int pairIndex = index << 1;
 635                  Operand n1 = EmitVectorExtract32(context, op.Qn, op.In + pairIndex, op.Size, signed);
 636                  Operand n2 = EmitVectorExtract32(context, op.Qn, op.In + pairIndex + 1, op.Size, signed);
 637  
 638                  Operand m1 = EmitVectorExtract32(context, op.Qm, op.Im + pairIndex, op.Size, signed);
 639                  Operand m2 = EmitVectorExtract32(context, op.Qm, op.Im + pairIndex + 1, op.Size, signed);
 640  
 641                  res = EmitVectorInsert(context, res, emit(n1, n2), op.Id + index, op.Size);
 642                  res = EmitVectorInsert(context, res, emit(m1, m2), op.Id + index + pairs, op.Size);
 643              }
 644  
 645              context.Copy(GetVecA32(op.Qd), res);
 646          }
 647  
 648          public static void EmitVectorPairwiseLongOpI32(ArmEmitterContext context, Func2I emit, bool signed)
 649          {
 650              OpCode32Simd op = (OpCode32Simd)context.CurrOp;
 651  
 652              int elems = (op.Q ? 16 : 8) >> op.Size;
 653              int pairs = elems >> 1;
 654              int id = (op.Vd & 1) * pairs;
 655  
 656              Operand res = GetVecA32(op.Qd);
 657  
 658              for (int index = 0; index < pairs; index++)
 659              {
 660                  int pairIndex = index << 1;
 661                  Operand m1 = EmitVectorExtract32(context, op.Qm, op.Im + pairIndex, op.Size, signed);
 662                  Operand m2 = EmitVectorExtract32(context, op.Qm, op.Im + pairIndex + 1, op.Size, signed);
 663  
 664                  if (op.Size == 2)
 665                  {
 666                      m1 = signed ? context.SignExtend32(OperandType.I64, m1) : context.ZeroExtend32(OperandType.I64, m1);
 667                      m2 = signed ? context.SignExtend32(OperandType.I64, m2) : context.ZeroExtend32(OperandType.I64, m2);
 668                  }
 669  
 670                  res = EmitVectorInsert(context, res, emit(m1, m2), id + index, op.Size + 1);
 671              }
 672  
 673              context.Copy(GetVecA32(op.Qd), res);
 674          }
 675  
 676          public static void EmitVectorPairwiseTernaryLongOpI32(ArmEmitterContext context, Func3I emit, bool signed)
 677          {
 678              OpCode32Simd op = (OpCode32Simd)context.CurrOp;
 679  
 680              int elems = op.GetBytesCount() >> op.Size;
 681              int pairs = elems >> 1;
 682  
 683              Operand res = GetVecA32(op.Qd);
 684  
 685              for (int index = 0; index < pairs; index++)
 686              {
 687                  int pairIndex = index * 2;
 688                  Operand m1 = EmitVectorExtract32(context, op.Qm, op.Im + pairIndex, op.Size, signed);
 689                  Operand m2 = EmitVectorExtract32(context, op.Qm, op.Im + pairIndex + 1, op.Size, signed);
 690  
 691                  if (op.Size == 2)
 692                  {
 693                      m1 = signed ? context.SignExtend32(OperandType.I64, m1) : context.ZeroExtend32(OperandType.I64, m1);
 694                      m2 = signed ? context.SignExtend32(OperandType.I64, m2) : context.ZeroExtend32(OperandType.I64, m2);
 695                  }
 696  
 697                  Operand d1 = EmitVectorExtract32(context, op.Qd, op.Id + index, op.Size + 1, signed);
 698  
 699                  res = EmitVectorInsert(context, res, emit(m1, m2, d1), op.Id + index, op.Size + 1);
 700              }
 701  
 702              context.Copy(GetVecA32(op.Qd), res);
 703          }
 704  
 705          // Narrow
 706  
 707          public static void EmitVectorUnaryNarrowOp32(ArmEmitterContext context, Func1I emit, bool signed = false)
 708          {
 709              OpCode32Simd op = (OpCode32Simd)context.CurrOp;
 710  
 711              int elems = 8 >> op.Size; // Size contains the target element size. (for when it becomes a doubleword)
 712  
 713              Operand res = GetVecA32(op.Qd);
 714              int id = (op.Vd & 1) << (3 - op.Size); // Target doubleword base.
 715  
 716              for (int index = 0; index < elems; index++)
 717              {
 718                  Operand m = EmitVectorExtract32(context, op.Qm, index, op.Size + 1, signed);
 719  
 720                  res = EmitVectorInsert(context, res, emit(m), id + index, op.Size);
 721              }
 722  
 723              context.Copy(GetVecA32(op.Qd), res);
 724          }
 725  
 726          // Intrinsic Helpers
 727  
 728          public static Operand EmitMoveDoubleWordToSide(ArmEmitterContext context, Operand input, int originalV, int targetV)
 729          {
 730              Debug.Assert(input.Type == OperandType.V128);
 731  
 732              int originalSide = originalV & 1;
 733              int targetSide = targetV & 1;
 734  
 735              if (originalSide == targetSide)
 736              {
 737                  return input;
 738              }
 739  
 740              if (targetSide == 1)
 741              {
 742                  return context.AddIntrinsic(Intrinsic.X86Movlhps, input, input); // Low to high.
 743              }
 744              else
 745              {
 746                  return context.AddIntrinsic(Intrinsic.X86Movhlps, input, input); // High to low.
 747              }
 748          }
 749  
 750          public static Operand EmitDoubleWordInsert(ArmEmitterContext context, Operand target, Operand value, int targetV)
 751          {
 752              Debug.Assert(target.Type == OperandType.V128 && value.Type == OperandType.V128);
 753  
 754              int targetSide = targetV & 1;
 755              int shuffleMask = 2;
 756  
 757              if (targetSide == 1)
 758              {
 759                  return context.AddIntrinsic(Intrinsic.X86Shufpd, target, value, Const(shuffleMask));
 760              }
 761              else
 762              {
 763                  return context.AddIntrinsic(Intrinsic.X86Shufpd, value, target, Const(shuffleMask));
 764              }
 765          }
 766  
 767          public static Operand EmitScalarInsert(ArmEmitterContext context, Operand target, Operand value, int reg, bool doubleWidth)
 768          {
 769              Debug.Assert(target.Type == OperandType.V128 && value.Type == OperandType.V128);
 770  
 771              // Insert from index 0 in value to index in target.
 772              int index = reg & (doubleWidth ? 1 : 3);
 773  
 774              if (doubleWidth)
 775              {
 776                  if (index == 1)
 777                  {
 778                      return context.AddIntrinsic(Intrinsic.X86Movlhps, target, value); // Low to high.
 779                  }
 780                  else
 781                  {
 782                      return context.AddIntrinsic(Intrinsic.X86Shufpd, value, target, Const(2)); // Low to low, keep high from original.
 783                  }
 784              }
 785              else
 786              {
 787                  if (Optimizations.UseSse41)
 788                  {
 789                      return context.AddIntrinsic(Intrinsic.X86Insertps, target, value, Const(index << 4));
 790                  }
 791                  else
 792                  {
 793                      target = EmitSwapScalar(context, target, index, doubleWidth); // Swap value to replace into element 0.
 794                      target = context.AddIntrinsic(Intrinsic.X86Movss, target, value); // Move the value into element 0 of the vector.
 795                      return EmitSwapScalar(context, target, index, doubleWidth); // Swap new value back to the correct index.
 796                  }
 797              }
 798          }
 799  
 800          public static Operand EmitSwapScalar(ArmEmitterContext context, Operand target, int reg, bool doubleWidth)
 801          {
 802              // Index into 0, 0 into index. This swap happens at the start of an A32 scalar op if required.
 803              int index = reg & (doubleWidth ? 1 : 3);
 804              if (index == 0)
 805              {
 806                  return target;
 807              }
 808  
 809              if (doubleWidth)
 810              {
 811                  int shuffleMask = 1; // Swap top and bottom. (b0 = 1, b1 = 0)
 812                  return context.AddIntrinsic(Intrinsic.X86Shufpd, target, target, Const(shuffleMask));
 813              }
 814              else
 815              {
 816                  int shuffleMask = (3 << 6) | (2 << 4) | (1 << 2) | index; // Swap index and 0. (others remain)
 817                  shuffleMask &= ~(3 << (index * 2));
 818  
 819                  return context.AddIntrinsic(Intrinsic.X86Shufps, target, target, Const(shuffleMask));
 820              }
 821          }
 822  
 823          // Vector Operand Templates
 824  
 825          public static void EmitVectorUnaryOpSimd32(ArmEmitterContext context, Func1I vectorFunc)
 826          {
 827              OpCode32Simd op = (OpCode32Simd)context.CurrOp;
 828  
 829              Operand m = GetVecA32(op.Qm);
 830              Operand d = GetVecA32(op.Qd);
 831  
 832              if (!op.Q) // Register swap: move relevant doubleword to destination side.
 833              {
 834                  m = EmitMoveDoubleWordToSide(context, m, op.Vm, op.Vd);
 835              }
 836  
 837              Operand res = vectorFunc(m);
 838  
 839              if (!op.Q) // Register insert.
 840              {
 841                  res = EmitDoubleWordInsert(context, d, res, op.Vd);
 842              }
 843  
 844              context.Copy(d, res);
 845          }
 846  
 847          public static void EmitVectorUnaryOpF32(ArmEmitterContext context, Intrinsic inst32, Intrinsic inst64)
 848          {
 849              OpCode32Simd op = (OpCode32Simd)context.CurrOp;
 850  
 851              Intrinsic inst = (op.Size & 1) != 0 ? inst64 : inst32;
 852  
 853              EmitVectorUnaryOpSimd32(context, (m) => context.AddIntrinsic(inst, m));
 854          }
 855  
 856          public static void EmitVectorBinaryOpSimd32(ArmEmitterContext context, Func2I vectorFunc, int side = -1)
 857          {
 858              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
 859  
 860              Operand n = GetVecA32(op.Qn);
 861              Operand m = GetVecA32(op.Qm);
 862              Operand d = GetVecA32(op.Qd);
 863  
 864              if (side == -1)
 865              {
 866                  side = op.Vd;
 867              }
 868  
 869              if (!op.Q) // Register swap: move relevant doubleword to destination side.
 870              {
 871                  n = EmitMoveDoubleWordToSide(context, n, op.Vn, side);
 872                  m = EmitMoveDoubleWordToSide(context, m, op.Vm, side);
 873              }
 874  
 875              Operand res = vectorFunc(n, m);
 876  
 877              if (!op.Q) // Register insert.
 878              {
 879                  if (side != op.Vd)
 880                  {
 881                      res = EmitMoveDoubleWordToSide(context, res, side, op.Vd);
 882                  }
 883                  res = EmitDoubleWordInsert(context, d, res, op.Vd);
 884              }
 885  
 886              context.Copy(d, res);
 887          }
 888  
 889          public static void EmitVectorBinaryOpF32(ArmEmitterContext context, Intrinsic inst32, Intrinsic inst64)
 890          {
 891              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
 892  
 893              Intrinsic inst = (op.Size & 1) != 0 ? inst64 : inst32;
 894              EmitVectorBinaryOpSimd32(context, (n, m) => context.AddIntrinsic(inst, n, m));
 895          }
 896  
 897          public static void EmitVectorTernaryOpSimd32(ArmEmitterContext context, Func3I vectorFunc)
 898          {
 899              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
 900  
 901              Operand n = GetVecA32(op.Qn);
 902              Operand m = GetVecA32(op.Qm);
 903              Operand d = GetVecA32(op.Qd);
 904              Operand initialD = d;
 905  
 906              if (!op.Q) // Register swap: move relevant doubleword to destination side.
 907              {
 908                  n = EmitMoveDoubleWordToSide(context, n, op.Vn, op.Vd);
 909                  m = EmitMoveDoubleWordToSide(context, m, op.Vm, op.Vd);
 910              }
 911  
 912              Operand res = vectorFunc(d, n, m);
 913  
 914              if (!op.Q) // Register insert.
 915              {
 916                  res = EmitDoubleWordInsert(context, initialD, res, op.Vd);
 917              }
 918  
 919              context.Copy(initialD, res);
 920          }
 921  
 922          public static void EmitVectorTernaryOpF32(ArmEmitterContext context, Intrinsic inst32pt1, Intrinsic inst64pt1, Intrinsic inst32pt2, Intrinsic inst64pt2)
 923          {
 924              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
 925  
 926              Intrinsic inst1 = (op.Size & 1) != 0 ? inst64pt1 : inst32pt1;
 927              Intrinsic inst2 = (op.Size & 1) != 0 ? inst64pt2 : inst32pt2;
 928  
 929              EmitVectorTernaryOpSimd32(context, (d, n, m) =>
 930              {
 931                  Operand res = context.AddIntrinsic(inst1, n, m);
 932                  return res = context.AddIntrinsic(inst2, d, res);
 933              });
 934          }
 935  
 936          public static void EmitVectorTernaryOpF32(ArmEmitterContext context, Intrinsic inst32)
 937          {
 938              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
 939  
 940              Debug.Assert((op.Size & 1) == 0);
 941  
 942              EmitVectorTernaryOpSimd32(context, (d, n, m) =>
 943              {
 944                  return context.AddIntrinsic(inst32, d, n, m);
 945              });
 946          }
 947  
 948          public static void EmitScalarUnaryOpSimd32(ArmEmitterContext context, Func1I scalarFunc)
 949          {
 950              OpCode32SimdS op = (OpCode32SimdS)context.CurrOp;
 951  
 952              bool doubleSize = (op.Size & 1) != 0;
 953              int shift = doubleSize ? 1 : 2;
 954              Operand m = GetVecA32(op.Vm >> shift);
 955              Operand d = GetVecA32(op.Vd >> shift);
 956  
 957              m = EmitSwapScalar(context, m, op.Vm, doubleSize);
 958  
 959              Operand res = scalarFunc(m);
 960  
 961              // Insert scalar into vector.
 962              res = EmitScalarInsert(context, d, res, op.Vd, doubleSize);
 963  
 964              context.Copy(d, res);
 965          }
 966  
 967          public static void EmitScalarUnaryOpF32(ArmEmitterContext context, Intrinsic inst32, Intrinsic inst64)
 968          {
 969              OpCode32SimdS op = (OpCode32SimdS)context.CurrOp;
 970  
 971              Intrinsic inst = (op.Size & 1) != 0 ? inst64 : inst32;
 972  
 973              EmitScalarUnaryOpSimd32(context, (m) => (inst == 0) ? m : context.AddIntrinsic(inst, m));
 974          }
 975  
 976          public static void EmitScalarBinaryOpSimd32(ArmEmitterContext context, Func2I scalarFunc)
 977          {
 978              OpCode32SimdRegS op = (OpCode32SimdRegS)context.CurrOp;
 979  
 980              bool doubleSize = (op.Size & 1) != 0;
 981              int shift = doubleSize ? 1 : 2;
 982              Operand n = GetVecA32(op.Vn >> shift);
 983              Operand m = GetVecA32(op.Vm >> shift);
 984              Operand d = GetVecA32(op.Vd >> shift);
 985  
 986              n = EmitSwapScalar(context, n, op.Vn, doubleSize);
 987              m = EmitSwapScalar(context, m, op.Vm, doubleSize);
 988  
 989              Operand res = scalarFunc(n, m);
 990  
 991              // Insert scalar into vector.
 992              res = EmitScalarInsert(context, d, res, op.Vd, doubleSize);
 993  
 994              context.Copy(d, res);
 995          }
 996  
 997          public static void EmitScalarBinaryOpF32(ArmEmitterContext context, Intrinsic inst32, Intrinsic inst64)
 998          {
 999              OpCode32SimdRegS op = (OpCode32SimdRegS)context.CurrOp;
1000  
1001              Intrinsic inst = (op.Size & 1) != 0 ? inst64 : inst32;
1002  
1003              EmitScalarBinaryOpSimd32(context, (n, m) => context.AddIntrinsic(inst, n, m));
1004          }
1005  
1006          public static void EmitScalarTernaryOpSimd32(ArmEmitterContext context, Func3I scalarFunc)
1007          {
1008              OpCode32SimdRegS op = (OpCode32SimdRegS)context.CurrOp;
1009  
1010              bool doubleSize = (op.Size & 1) != 0;
1011              int shift = doubleSize ? 1 : 2;
1012              Operand n = GetVecA32(op.Vn >> shift);
1013              Operand m = GetVecA32(op.Vm >> shift);
1014              Operand d = GetVecA32(op.Vd >> shift);
1015              Operand initialD = d;
1016  
1017              n = EmitSwapScalar(context, n, op.Vn, doubleSize);
1018              m = EmitSwapScalar(context, m, op.Vm, doubleSize);
1019              d = EmitSwapScalar(context, d, op.Vd, doubleSize);
1020  
1021              Operand res = scalarFunc(d, n, m);
1022  
1023              // Insert scalar into vector.
1024              res = EmitScalarInsert(context, initialD, res, op.Vd, doubleSize);
1025  
1026              context.Copy(initialD, res);
1027          }
1028  
1029          public static void EmitScalarTernaryOpF32(ArmEmitterContext context, Intrinsic inst32, Intrinsic inst64)
1030          {
1031              OpCode32SimdRegS op = (OpCode32SimdRegS)context.CurrOp;
1032  
1033              bool doubleSize = (op.Size & 1) != 0;
1034  
1035              Intrinsic inst = doubleSize ? inst64 : inst32;
1036  
1037              EmitScalarTernaryOpSimd32(context, (d, n, m) =>
1038              {
1039                  return context.AddIntrinsic(inst, d, n, m);
1040              });
1041          }
1042  
1043          public static void EmitScalarTernaryOpF32(
1044              ArmEmitterContext context,
1045              Intrinsic inst32pt1,
1046              Intrinsic inst64pt1,
1047              Intrinsic inst32pt2,
1048              Intrinsic inst64pt2,
1049              bool isNegD = false)
1050          {
1051              OpCode32SimdRegS op = (OpCode32SimdRegS)context.CurrOp;
1052  
1053              bool doubleSize = (op.Size & 1) != 0;
1054  
1055              Intrinsic inst1 = doubleSize ? inst64pt1 : inst32pt1;
1056              Intrinsic inst2 = doubleSize ? inst64pt2 : inst32pt2;
1057  
1058              EmitScalarTernaryOpSimd32(context, (d, n, m) =>
1059              {
1060                  Operand res = context.AddIntrinsic(inst1, n, m);
1061  
1062                  if (isNegD)
1063                  {
1064                      Operand mask = doubleSize
1065                          ? X86GetScalar(context, -0d)
1066                          : X86GetScalar(context, -0f);
1067  
1068                      d = doubleSize
1069                          ? context.AddIntrinsic(Intrinsic.X86Xorpd, mask, d)
1070                          : context.AddIntrinsic(Intrinsic.X86Xorps, mask, d);
1071                  }
1072  
1073                  return context.AddIntrinsic(inst2, d, res);
1074              });
1075          }
1076  
1077          // By Scalar
1078  
1079          public static void EmitVectorByScalarOpSimd32(ArmEmitterContext context, Func2I vectorFunc)
1080          {
1081              OpCode32SimdRegElem op = (OpCode32SimdRegElem)context.CurrOp;
1082  
1083              Operand n = GetVecA32(op.Qn);
1084              Operand d = GetVecA32(op.Qd);
1085  
1086              int index = op.Vm & 3;
1087              int dupeMask = (index << 6) | (index << 4) | (index << 2) | index;
1088              Operand m = GetVecA32(op.Vm >> 2);
1089              m = context.AddIntrinsic(Intrinsic.X86Shufps, m, m, Const(dupeMask));
1090  
1091              if (!op.Q) // Register swap: move relevant doubleword to destination side.
1092              {
1093                  n = EmitMoveDoubleWordToSide(context, n, op.Vn, op.Vd);
1094              }
1095  
1096              Operand res = vectorFunc(n, m);
1097  
1098              if (!op.Q) // Register insert.
1099              {
1100                  res = EmitDoubleWordInsert(context, d, res, op.Vd);
1101              }
1102  
1103              context.Copy(d, res);
1104          }
1105  
1106          public static void EmitVectorByScalarOpF32(ArmEmitterContext context, Intrinsic inst32, Intrinsic inst64)
1107          {
1108              OpCode32SimdRegElem op = (OpCode32SimdRegElem)context.CurrOp;
1109  
1110              Intrinsic inst = (op.Size & 1) != 0 ? inst64 : inst32;
1111              EmitVectorByScalarOpSimd32(context, (n, m) => context.AddIntrinsic(inst, n, m));
1112          }
1113  
1114          public static void EmitVectorsByScalarOpSimd32(ArmEmitterContext context, Func3I vectorFunc)
1115          {
1116              OpCode32SimdRegElem op = (OpCode32SimdRegElem)context.CurrOp;
1117  
1118              Operand n = GetVecA32(op.Qn);
1119              Operand d = GetVecA32(op.Qd);
1120              Operand initialD = d;
1121  
1122              int index = op.Vm & 3;
1123              int dupeMask = (index << 6) | (index << 4) | (index << 2) | index;
1124              Operand m = GetVecA32(op.Vm >> 2);
1125              m = context.AddIntrinsic(Intrinsic.X86Shufps, m, m, Const(dupeMask));
1126  
1127              if (!op.Q) // Register swap: move relevant doubleword to destination side.
1128              {
1129                  n = EmitMoveDoubleWordToSide(context, n, op.Vn, op.Vd);
1130              }
1131  
1132              Operand res = vectorFunc(d, n, m);
1133  
1134              if (!op.Q) // Register insert.
1135              {
1136                  res = EmitDoubleWordInsert(context, initialD, res, op.Vd);
1137              }
1138  
1139              context.Copy(initialD, res);
1140          }
1141  
1142          public static void EmitVectorsByScalarOpF32(ArmEmitterContext context, Intrinsic inst32pt1, Intrinsic inst64pt1, Intrinsic inst32pt2, Intrinsic inst64pt2)
1143          {
1144              OpCode32SimdRegElem op = (OpCode32SimdRegElem)context.CurrOp;
1145  
1146              Intrinsic inst1 = (op.Size & 1) != 0 ? inst64pt1 : inst32pt1;
1147              Intrinsic inst2 = (op.Size & 1) != 0 ? inst64pt2 : inst32pt2;
1148  
1149              EmitVectorsByScalarOpSimd32(context, (d, n, m) =>
1150              {
1151                  Operand res = context.AddIntrinsic(inst1, n, m);
1152                  return res = context.AddIntrinsic(inst2, d, res);
1153              });
1154          }
1155  
1156          // Pairwise
1157  
1158          public static void EmitSse2VectorPairwiseOpF32(ArmEmitterContext context, Intrinsic inst32)
1159          {
1160              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
1161  
1162              EmitVectorBinaryOpSimd32(context, (n, m) =>
1163              {
1164                  Operand unpck = context.AddIntrinsic(Intrinsic.X86Unpcklps, n, m);
1165  
1166                  Operand part0 = unpck;
1167                  Operand part1 = context.AddIntrinsic(Intrinsic.X86Movhlps, unpck, unpck);
1168  
1169                  return context.AddIntrinsic(inst32, part0, part1);
1170              }, 0);
1171          }
1172  
1173          public static void EmitSsse3VectorPairwiseOp32(ArmEmitterContext context, Intrinsic[] inst)
1174          {
1175              OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
1176  
1177              EmitVectorBinaryOpSimd32(context, (n, m) =>
1178              {
1179                  if (op.RegisterSize == RegisterSize.Simd64)
1180                  {
1181                      Operand zeroEvenMask = X86GetElements(context, ZeroMask, EvenMasks[op.Size]);
1182                      Operand zeroOddMask = X86GetElements(context, ZeroMask, OddMasks[op.Size]);
1183  
1184                      Operand mN = context.AddIntrinsic(Intrinsic.X86Punpcklqdq, n, m); // m:n
1185  
1186                      Operand left = context.AddIntrinsic(Intrinsic.X86Pshufb, mN, zeroEvenMask); // 0:even from m:n
1187                      Operand right = context.AddIntrinsic(Intrinsic.X86Pshufb, mN, zeroOddMask); // 0:odd  from m:n
1188  
1189                      return context.AddIntrinsic(inst[op.Size], left, right);
1190                  }
1191                  else if (op.Size < 3)
1192                  {
1193                      Operand oddEvenMask = X86GetElements(context, OddMasks[op.Size], EvenMasks[op.Size]);
1194  
1195                      Operand oddEvenN = context.AddIntrinsic(Intrinsic.X86Pshufb, n, oddEvenMask); // odd:even from n
1196                      Operand oddEvenM = context.AddIntrinsic(Intrinsic.X86Pshufb, m, oddEvenMask); // odd:even from m
1197  
1198                      Operand left = context.AddIntrinsic(Intrinsic.X86Punpcklqdq, oddEvenN, oddEvenM);
1199                      Operand right = context.AddIntrinsic(Intrinsic.X86Punpckhqdq, oddEvenN, oddEvenM);
1200  
1201                      return context.AddIntrinsic(inst[op.Size], left, right);
1202                  }
1203                  else
1204                  {
1205                      Operand left = context.AddIntrinsic(Intrinsic.X86Punpcklqdq, n, m);
1206                      Operand right = context.AddIntrinsic(Intrinsic.X86Punpckhqdq, n, m);
1207  
1208                      return context.AddIntrinsic(inst[3], left, right);
1209                  }
1210              }, 0);
1211          }
1212  
1213          // Generic Functions
1214  
1215          public static Operand EmitSoftFloatCallDefaultFpscr(ArmEmitterContext context, string name, params Operand[] callArgs)
1216          {
1217              IOpCodeSimd op = (IOpCodeSimd)context.CurrOp;
1218  
1219              MethodInfo info = (op.Size & 1) == 0
1220                  ? typeof(SoftFloat32).GetMethod(name)
1221                  : typeof(SoftFloat64).GetMethod(name);
1222  
1223              Array.Resize(ref callArgs, callArgs.Length + 1);
1224              callArgs[^1] = Const(1);
1225  
1226              context.ExitArmFpMode();
1227              context.StoreToContext();
1228              Operand res = context.Call(info, callArgs);
1229              context.LoadFromContext();
1230              context.EnterArmFpMode();
1231  
1232              return res;
1233          }
1234  
1235          public static Operand EmitVectorExtractSx32(ArmEmitterContext context, int reg, int index, int size)
1236          {
1237              return EmitVectorExtract32(context, reg, index, size, true);
1238          }
1239  
1240          public static Operand EmitVectorExtractZx32(ArmEmitterContext context, int reg, int index, int size)
1241          {
1242              return EmitVectorExtract32(context, reg, index, size, false);
1243          }
1244  
1245          public static Operand EmitVectorExtract32(ArmEmitterContext context, int reg, int index, int size, bool signed)
1246          {
1247              ThrowIfInvalid(index, size);
1248  
1249              Operand res = default;
1250  
1251              switch (size)
1252              {
1253                  case 0:
1254                      res = context.VectorExtract8(GetVec(reg), index);
1255                      break;
1256  
1257                  case 1:
1258                      res = context.VectorExtract16(GetVec(reg), index);
1259                      break;
1260  
1261                  case 2:
1262                      res = context.VectorExtract(OperandType.I32, GetVec(reg), index);
1263                      break;
1264  
1265                  case 3:
1266                      res = context.VectorExtract(OperandType.I64, GetVec(reg), index);
1267                      break;
1268              }
1269  
1270              if (signed)
1271              {
1272                  switch (size)
1273                  {
1274                      case 0:
1275                          res = context.SignExtend8(OperandType.I32, res);
1276                          break;
1277                      case 1:
1278                          res = context.SignExtend16(OperandType.I32, res);
1279                          break;
1280                  }
1281              }
1282              else
1283              {
1284                  switch (size)
1285                  {
1286                      case 0:
1287                          res = context.ZeroExtend8(OperandType.I32, res);
1288                          break;
1289                      case 1:
1290                          res = context.ZeroExtend16(OperandType.I32, res);
1291                          break;
1292                  }
1293              }
1294  
1295              return res;
1296          }
1297  
1298          public static Operand EmitPolynomialMultiply(ArmEmitterContext context, Operand op1, Operand op2, int eSize)
1299          {
1300              Debug.Assert(eSize <= 32);
1301  
1302              Operand result = eSize == 32 ? Const(0L) : Const(0);
1303  
1304              if (eSize == 32)
1305              {
1306                  op1 = context.ZeroExtend32(OperandType.I64, op1);
1307                  op2 = context.ZeroExtend32(OperandType.I64, op2);
1308              }
1309  
1310              for (int i = 0; i < eSize; i++)
1311              {
1312                  Operand mask = context.BitwiseAnd(op1, Const(op1.Type, 1L << i));
1313  
1314                  result = context.BitwiseExclusiveOr(result, context.Multiply(op2, mask));
1315              }
1316  
1317              return result;
1318          }
1319      }
1320  }