1  using ARMeilleure.IntermediateRepresentation;
  2  using ARMeilleure.State;
  3  using ARMeilleure.Translation;
  4  using static ARMeilleure.Instructions.InstEmitHelper;
  5  using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
  6  
  7  namespace ARMeilleure.Instructions
  8  {
  9      static class InstEmitMemoryExHelper
 10      {
 11          private const int ErgSizeLog2 = 4;
 12  
 13          public static Operand EmitLoadExclusive(ArmEmitterContext context, Operand address, bool exclusive, int size)
 14          {
 15              if (exclusive)
 16              {
 17                  Operand value;
 18  
 19                  if (size == 4)
 20                  {
 21                      // Only 128-bit CAS is guaranteed to have a atomic load.
 22                      Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, default, write: false, 4);
 23  
 24                      Operand zero = context.VectorZero();
 25  
 26                      value = context.CompareAndSwap(physAddr, zero, zero);
 27                  }
 28                  else
 29                  {
 30                      value = InstEmitMemoryHelper.EmitReadIntAligned(context, address, size);
 31                  }
 32  
 33                  Operand arg0 = context.LoadArgument(OperandType.I64, 0);
 34  
 35                  Operand exAddrPtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveAddressOffset()));
 36                  Operand exValuePtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveValueOffset()));
 37  
 38                  context.Store(exAddrPtr, context.BitwiseAnd(address, Const(address.Type, GetExclusiveAddressMask())));
 39  
 40                  // Make sure the unused higher bits of the value are cleared.
 41                  if (size < 3)
 42                  {
 43                      context.Store(exValuePtr, Const(0UL));
 44                  }
 45                  if (size < 4)
 46                  {
 47                      context.Store(context.Add(exValuePtr, Const(exValuePtr.Type, 8L)), Const(0UL));
 48                  }
 49  
 50                  // Store the new exclusive value.
 51                  context.Store(exValuePtr, value);
 52  
 53                  return value;
 54              }
 55              else
 56              {
 57                  return InstEmitMemoryHelper.EmitReadIntAligned(context, address, size);
 58              }
 59          }
 60  
 61          public static void EmitStoreExclusive(
 62              ArmEmitterContext context,
 63              Operand address,
 64              Operand value,
 65              bool exclusive,
 66              int size,
 67              int rs,
 68              bool a32)
 69          {
 70              if (size < 3)
 71              {
 72                  value = context.ConvertI64ToI32(value);
 73              }
 74  
 75              if (exclusive)
 76              {
 77                  // We overwrite one of the register (Rs),
 78                  // keep a copy of the values to ensure we are working with the correct values.
 79                  address = context.Copy(address);
 80                  value = context.Copy(value);
 81  
 82                  void SetRs(Operand value)
 83                  {
 84                      if (a32)
 85                      {
 86                          SetIntA32(context, rs, value);
 87                      }
 88                      else
 89                      {
 90                          SetIntOrZR(context, rs, value);
 91                      }
 92                  }
 93  
 94                  Operand arg0 = context.LoadArgument(OperandType.I64, 0);
 95  
 96                  Operand exAddrPtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveAddressOffset()));
 97                  Operand exAddr = context.Load(address.Type, exAddrPtr);
 98  
 99                  // STEP 1: Check if we have exclusive access to this memory region. If not, fail and skip store.
100                  Operand maskedAddress = context.BitwiseAnd(address, Const(address.Type, GetExclusiveAddressMask()));
101  
102                  Operand exFailed = context.ICompareNotEqual(exAddr, maskedAddress);
103  
104                  Operand lblExit = Label();
105  
106                  SetRs(Const(1));
107  
108                  context.BranchIfTrue(lblExit, exFailed);
109  
110                  // STEP 2: We have exclusive access and the address is valid, attempt the store using CAS.
111                  Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, default, write: true, size);
112  
113                  Operand exValuePtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveValueOffset()));
114                  Operand exValue = size switch
115                  {
116                      0 => context.Load8(exValuePtr),
117                      1 => context.Load16(exValuePtr),
118                      2 => context.Load(OperandType.I32, exValuePtr),
119                      3 => context.Load(OperandType.I64, exValuePtr),
120                      _ => context.Load(OperandType.V128, exValuePtr),
121                  };
122  
123                  Operand currValue = size switch
124                  {
125                      0 => context.CompareAndSwap8(physAddr, exValue, value),
126                      1 => context.CompareAndSwap16(physAddr, exValue, value),
127                      _ => context.CompareAndSwap(physAddr, exValue, value),
128                  };
129  
130                  // STEP 3: Check if we succeeded by comparing expected and in-memory values.
131                  Operand storeFailed;
132  
133                  if (size == 4)
134                  {
135                      Operand currValueLow = context.VectorExtract(OperandType.I64, currValue, 0);
136                      Operand currValueHigh = context.VectorExtract(OperandType.I64, currValue, 1);
137  
138                      Operand exValueLow = context.VectorExtract(OperandType.I64, exValue, 0);
139                      Operand exValueHigh = context.VectorExtract(OperandType.I64, exValue, 1);
140  
141                      storeFailed = context.BitwiseOr(
142                          context.ICompareNotEqual(currValueLow, exValueLow),
143                          context.ICompareNotEqual(currValueHigh, exValueHigh));
144                  }
145                  else
146                  {
147                      storeFailed = context.ICompareNotEqual(currValue, exValue);
148                  }
149  
150                  SetRs(storeFailed);
151  
152                  context.MarkLabel(lblExit);
153              }
154              else
155              {
156                  InstEmitMemoryHelper.EmitWriteIntAligned(context, address, value, size);
157              }
158          }
159  
160          public static void EmitClearExclusive(ArmEmitterContext context)
161          {
162              Operand arg0 = context.LoadArgument(OperandType.I64, 0);
163  
164              Operand exAddrPtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveAddressOffset()));
165  
166              // We store ULONG max to force any exclusive address checks to fail,
167              // since this value is not aligned to the ERG mask.
168              context.Store(exAddrPtr, Const(ulong.MaxValue));
169          }
170  
171          private static long GetExclusiveAddressMask() => ~((4L << ErgSizeLog2) - 1);
172      }
173  }