1  /*
   2   * Copyright (C) 2016-2020 Apple Inc. All rights reserved.
   3   *
   4   * Redistribution and use in source and binary forms, with or without
   5   * modification, are permitted provided that the following conditions
   6   * are met:
   7   * 1. Redistributions of source code must retain the above copyright
   8   *    notice, this list of conditions and the following disclaimer.
   9   * 2. Redistributions in binary form must reproduce the above copyright
  10   *    notice, this list of conditions and the following disclaimer in the
  11   *    documentation and/or other materials provided with the distribution.
  12   *
  13   * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14   * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15   * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16   * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18   * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19   * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20   * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21   * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22   * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23   * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24   */
  25  
  26  #include "config.h"
  27  #include "WasmB3IRGenerator.h"
  28  
  29  #if ENABLE(WEBASSEMBLY)
  30  
  31  #include "AllowMacroScratchRegisterUsageIf.h"
  32  #include "B3BasicBlockInlines.h"
  33  #include "B3CCallValue.h"
  34  #include "B3ConstPtrValue.h"
  35  #include "B3FixSSA.h"
  36  #include "B3Generate.h"
  37  #include "B3InsertionSet.h"
  38  #include "B3StackmapGenerationParams.h"
  39  #include "B3SwitchValue.h"
  40  #include "B3UpsilonValue.h"
  41  #include "B3Validate.h"
  42  #include "B3ValueInlines.h"
  43  #include "B3ValueKey.h"
  44  #include "B3Variable.h"
  45  #include "B3VariableValue.h"
  46  #include "B3WasmAddressValue.h"
  47  #include "B3WasmBoundsCheckValue.h"
  48  #include "JSCJSValueInlines.h"
  49  #include "JSWebAssemblyInstance.h"
  50  #include "ScratchRegisterAllocator.h"
  51  #include "WasmCallingConvention.h"
  52  #include "WasmContextInlines.h"
  53  #include "WasmExceptionType.h"
  54  #include "WasmFunctionParser.h"
  55  #include "WasmInstance.h"
  56  #include "WasmMemory.h"
  57  #include "WasmOSREntryData.h"
  58  #include "WasmOpcodeOrigin.h"
  59  #include "WasmOperations.h"
  60  #include "WasmSignatureInlines.h"
  61  #include "WasmThunks.h"
  62  #include <limits>
  63  #include <wtf/Optional.h>
  64  #include <wtf/StdLibExtras.h>
  65  
  66  void dumpProcedure(void* ptr)
  67  {
  68      JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
  69      proc->dump(WTF::dataFile());
  70  }
  71  
  72  namespace JSC { namespace Wasm {
  73  
  74  using namespace B3;
  75  
  76  namespace {
  77  namespace WasmB3IRGeneratorInternal {
  78  static constexpr bool verbose = false;
  79  }
  80  }
  81  
  82  class B3IRGenerator {
  83  public:
  84      using ExpressionType = Value*;
  85      using ResultList = Vector<ExpressionType, 8>;
  86  
  87      struct ControlData {
  88          ControlData(Procedure& proc, Origin origin, BlockSignature signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
  89              : controlBlockType(type)
  90              , m_signature(signature)
  91              , continuation(continuation)
  92              , special(special)
  93          {
  94              if (type == BlockType::Loop) {
  95                  for (unsigned i = 0; i < signature->argumentCount(); ++i)
  96                      phis.append(proc.add<Value>(Phi, toB3Type(signature->argument(i)), origin));
  97              } else {
  98                  for (unsigned i = 0; i < signature->returnCount(); ++i)
  99                      phis.append(proc.add<Value>(Phi, toB3Type(signature->returnType(i)), origin));
 100              }
 101          }
 102  
 103          ControlData()
 104          {
 105          }
 106  
 107          static bool isIf(const ControlData& control) { return control.blockType() == BlockType::If; }
 108          static bool isTopLevel(const ControlData& control) { return control.blockType() == BlockType::TopLevel; }
 109  
 110          void dump(PrintStream& out) const
 111          {
 112              switch (blockType()) {
 113              case BlockType::If:
 114                  out.print("If:       ");
 115                  break;
 116              case BlockType::Block:
 117                  out.print("Block:    ");
 118                  break;
 119              case BlockType::Loop:
 120                  out.print("Loop:     ");
 121                  break;
 122              case BlockType::TopLevel:
 123                  out.print("TopLevel: ");
 124                  break;
 125              }
 126              out.print("Continuation: ", *continuation, ", Special: ");
 127              if (special)
 128                  out.print(*special);
 129              else
 130                  out.print("None");
 131          }
 132  
 133          BlockType blockType() const { return controlBlockType; }
 134  
 135          BlockSignature signature() const { return m_signature; }
 136  
 137          bool hasNonVoidresult() const { return m_signature->returnsVoid(); }
 138  
 139          BasicBlock* targetBlockForBranch()
 140          {
 141              if (blockType() == BlockType::Loop)
 142                  return special;
 143              return continuation;
 144          }
 145  
 146          void convertIfToBlock()
 147          {
 148              ASSERT(blockType() == BlockType::If);
 149              controlBlockType = BlockType::Block;
 150              special = nullptr;
 151          }
 152  
 153          SignatureArgCount branchTargetArity() const
 154          {
 155              if (blockType() == BlockType::Loop)
 156                  return m_signature->argumentCount();
 157              return m_signature->returnCount();
 158          }
 159  
 160          Type branchTargetType(unsigned i) const
 161          {
 162              ASSERT(i < branchTargetArity());
 163              if (blockType() == BlockType::Loop)
 164                  return m_signature->argument(i);
 165              return m_signature->returnType(i);
 166          }
 167  
 168      private:
 169          friend class B3IRGenerator;
 170          BlockType controlBlockType;
 171          BlockSignature m_signature;
 172          BasicBlock* continuation;
 173          BasicBlock* special;
 174          ResultList phis;
 175      };
 176  
 177      using ControlType = ControlData;
 178      using ExpressionList = Vector<ExpressionType, 1>;
 179  
 180      using ControlEntry = FunctionParser<B3IRGenerator>::ControlEntry;
 181      using ControlStack = FunctionParser<B3IRGenerator>::ControlStack;
 182      using Stack = FunctionParser<B3IRGenerator>::Stack;
 183      using TypedExpression = FunctionParser<B3IRGenerator>::TypedExpression;
 184  
 185      static_assert(std::is_same_v<ResultList, FunctionParser<B3IRGenerator>::ResultList>);
 186  
 187      typedef String ErrorType;
 188      typedef Unexpected<ErrorType> UnexpectedResult;
 189      typedef Expected<std::unique_ptr<InternalFunction>, ErrorType> Result;
 190      typedef Expected<void, ErrorType> PartialResult;
 191  
 192      static ExpressionType emptyExpression() { return nullptr; };
 193  
 194      template <typename ...Args>
 195      NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
 196      {
 197          using namespace FailureHelper; // See ADL comment in WasmParser.h.
 198          return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));
 199      }
 200  #define WASM_COMPILE_FAIL_IF(condition, ...) do { \
 201          if (UNLIKELY(condition))                  \
 202              return fail(__VA_ARGS__);             \
 203      } while (0)
 204  
 205      B3IRGenerator(const ModuleInformation&, Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, unsigned& osrEntryScratchBufferSize, MemoryMode, CompilationMode, unsigned functionIndex, unsigned loopIndexForOSREntry, TierUpCount*);
 206  
 207      PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
 208      PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
 209      ExpressionType addConstant(Type, uint64_t);
 210  
 211      // References
 212      PartialResult WARN_UNUSED_RETURN addRefIsNull(ExpressionType value, ExpressionType& result);
 213      PartialResult WARN_UNUSED_RETURN addRefFunc(uint32_t index, ExpressionType& result);
 214  
 215      // Tables
 216      PartialResult WARN_UNUSED_RETURN addTableGet(unsigned, ExpressionType index, ExpressionType& result);
 217      PartialResult WARN_UNUSED_RETURN addTableSet(unsigned, ExpressionType index, ExpressionType value);
 218      PartialResult WARN_UNUSED_RETURN addTableInit(unsigned, unsigned, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length);
 219      PartialResult WARN_UNUSED_RETURN addElemDrop(unsigned);
 220      PartialResult WARN_UNUSED_RETURN addTableSize(unsigned, ExpressionType& result);
 221      PartialResult WARN_UNUSED_RETURN addTableGrow(unsigned, ExpressionType fill, ExpressionType delta, ExpressionType& result);
 222      PartialResult WARN_UNUSED_RETURN addTableFill(unsigned, ExpressionType offset, ExpressionType fill, ExpressionType count);
 223      PartialResult WARN_UNUSED_RETURN addTableCopy(unsigned, unsigned, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length);
 224  
 225      // Locals
 226      PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
 227      PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
 228  
 229      // Globals
 230      PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
 231      PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
 232  
 233      // Memory
 234      PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
 235      PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
 236      PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
 237      PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
 238      PartialResult WARN_UNUSED_RETURN addMemoryFill(ExpressionType dstAddress, ExpressionType targetValue, ExpressionType count);
 239      PartialResult WARN_UNUSED_RETURN addMemoryCopy(ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType count);
 240      PartialResult WARN_UNUSED_RETURN addMemoryInit(unsigned, ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType length);
 241      PartialResult WARN_UNUSED_RETURN addDataDrop(unsigned);
 242  
 243      // Atomics
 244      PartialResult WARN_UNUSED_RETURN atomicLoad(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType& result, uint32_t offset);
 245      PartialResult WARN_UNUSED_RETURN atomicStore(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, uint32_t offset);
 246      PartialResult WARN_UNUSED_RETURN atomicBinaryRMW(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint32_t offset);
 247      PartialResult WARN_UNUSED_RETURN atomicCompareExchange(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType expected, ExpressionType value, ExpressionType& result, uint32_t offset);
 248  
 249      PartialResult WARN_UNUSED_RETURN atomicWait(ExtAtomicOpType, ExpressionType pointer, ExpressionType value, ExpressionType timeout, ExpressionType& result, uint32_t offset);
 250      PartialResult WARN_UNUSED_RETURN atomicNotify(ExtAtomicOpType, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint32_t offset);
 251      PartialResult WARN_UNUSED_RETURN atomicFence(ExtAtomicOpType, uint8_t flags);
 252  
 253      // Basic operators
 254      template<OpType>
 255      PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
 256      template<OpType>
 257      PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
 258      PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
 259  
 260  
 261      // Control flow
 262      ControlData WARN_UNUSED_RETURN addTopLevel(BlockSignature);
 263      PartialResult WARN_UNUSED_RETURN addBlock(BlockSignature, Stack& enclosingStack, ControlType& newBlock, Stack& newStack);
 264      PartialResult WARN_UNUSED_RETURN addLoop(BlockSignature, Stack& enclosingStack, ControlType& block, Stack& newStack, uint32_t loopIndex);
 265      PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, BlockSignature, Stack& enclosingStack, ControlType& result, Stack& newStack);
 266      PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const Stack&);
 267      PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
 268  
 269      PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const Stack& returnValues);
 270      PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const Stack& returnValues);
 271      PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const Stack& expressionStack);
 272      PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, Stack& expressionStack);
 273      PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&, const Stack& = { });
 274  
 275      PartialResult WARN_UNUSED_RETURN endTopLevel(BlockSignature, const Stack&) { return { }; }
 276  
 277      // Calls
 278      PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ResultList& results);
 279      PartialResult WARN_UNUSED_RETURN addCallIndirect(unsigned tableIndex, const Signature&, Vector<ExpressionType>& args, ResultList& results);
 280      PartialResult WARN_UNUSED_RETURN addUnreachable();
 281      B3::Value* createCallPatchpoint(BasicBlock*, Origin, const Signature&, Vector<ExpressionType>& args, const ScopedLambda<void(PatchpointValue*)>& patchpointFunctor);
 282  
 283      void dump(const ControlStack&, const Stack* expressionStack);
 284      void setParser(FunctionParser<B3IRGenerator>* parser) { m_parser = parser; };
 285      void didFinishParsingLocals() { }
 286      void didPopValueFromStack() { }
 287  
 288      Value* constant(B3::Type, uint64_t bits, Optional<Origin> = WTF::nullopt);
 289      Value* framePointer();
 290      void insertConstants();
 291  
 292      B3::Type toB3ResultType(BlockSignature);
 293  
 294  private:
 295      void emitExceptionCheck(CCallHelpers&, ExceptionType);
 296  
 297      void emitEntryTierUpCheck();
 298      void emitLoopTierUpCheck(uint32_t loopIndex, const Stack& enclosingStack);
 299  
 300      void emitWriteBarrierForJSWrapper();
 301      ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
 302      B3::Kind memoryKind(B3::Opcode memoryOp);
 303      ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
 304      void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
 305  
 306      ExpressionType sanitizeAtomicResult(ExtAtomicOpType, Type, ExpressionType result);
 307      ExpressionType emitAtomicLoadOp(ExtAtomicOpType, Type, ExpressionType pointer, uint32_t offset);
 308      void emitAtomicStoreOp(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, uint32_t offset);
 309      ExpressionType emitAtomicBinaryRMWOp(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, uint32_t offset);
 310      ExpressionType emitAtomicCompareExchange(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType expected, ExpressionType value, uint32_t offset);
 311  
 312      void unify(const ExpressionType phi, const ExpressionType source);
 313      void unifyValuesWithBlock(const Stack& resultStack, const ResultList& stack);
 314  
 315      void emitChecksForModOrDiv(B3::Opcode, ExpressionType left, ExpressionType right);
 316  
 317      int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
 318      ExpressionType WARN_UNUSED_RETURN fixupPointerPlusOffsetForAtomicOps(ExtAtomicOpType, ExpressionType, uint32_t);
 319  
 320      void restoreWasmContextInstance(Procedure&, BasicBlock*, Value*);
 321      enum class RestoreCachedStackLimit { No, Yes };
 322      void restoreWebAssemblyGlobalState(RestoreCachedStackLimit, const MemoryInformation&, Value* instance, Procedure&, BasicBlock*);
 323  
 324      Origin origin();
 325  
 326      uint32_t outerLoopIndex() const
 327      {
 328          if (m_outerLoops.isEmpty())
 329              return UINT32_MAX;
 330          return m_outerLoops.last();
 331      }
 332  
 333      FunctionParser<B3IRGenerator>* m_parser { nullptr };
 334      const ModuleInformation& m_info;
 335      const MemoryMode m_mode { MemoryMode::BoundsChecking };
 336      const CompilationMode m_compilationMode { CompilationMode::BBQMode };
 337      const unsigned m_functionIndex { UINT_MAX };
 338      const unsigned m_loopIndexForOSREntry { UINT_MAX };
 339      TierUpCount* m_tierUp { nullptr };
 340  
 341      Procedure& m_proc;
 342      BasicBlock* m_rootBlock { nullptr };
 343      BasicBlock* m_currentBlock { nullptr };
 344      Vector<uint32_t> m_outerLoops;
 345      Vector<Variable*> m_locals;
 346      Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
 347      unsigned& m_osrEntryScratchBufferSize;
 348      HashMap<ValueKey, Value*> m_constantPool;
 349      HashMap<BlockSignature, B3::Type> m_tupleMap;
 350      InsertionSet m_constantInsertionValues;
 351      Value* m_framePointer { nullptr };
 352      GPRReg m_memoryBaseGPR { InvalidGPRReg };
 353      GPRReg m_boundsCheckingSizeGPR { InvalidGPRReg };
 354      GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };
 355      bool m_makesCalls { false };
 356  
 357      Value* m_instanceValue { nullptr }; // Always use the accessor below to ensure the instance value is materialized when used.
 358      bool m_usesInstanceValue { false };
 359      Value* instanceValue()
 360      {
 361          m_usesInstanceValue = true;
 362          return m_instanceValue;
 363      }
 364  
 365      uint32_t m_maxNumJSCallArguments { 0 };
 366      unsigned m_numImportFunctions;
 367  };
 368  
 369  // Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
 370  int32_t B3IRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
 371  {
 372      if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
 373          ptr = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), ptr, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), offset));
 374          return 0;
 375      }
 376      return offset;
 377  }
 378  
 379  void B3IRGenerator::restoreWasmContextInstance(Procedure& proc, BasicBlock* block, Value* arg)
 380  {
 381      if (Context::useFastTLS()) {
 382          PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
 383          if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister())
 384              patchpoint->clobber(RegisterSet::macroScratchRegisters());
 385          patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
 386          patchpoint->setGenerator(
 387              [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 388                  AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister());
 389                  jit.storeWasmContextInstance(params[0].gpr());
 390              });
 391          return;
 392      }
 393  
 394      // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
 395      // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
 396      PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
 397      Effects effects = Effects::none();
 398      effects.writesPinned = true;
 399      effects.reads = B3::HeapRange::top();
 400      patchpoint->effects = effects;
 401      patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR));
 402      patchpoint->append(arg, ValueRep::SomeRegister);
 403      GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR;
 404      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& param) {
 405          jit.move(param[0].gpr(), wasmContextInstanceGPR);
 406      });
 407  }
 408  
 409  B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, unsigned& osrEntryScratchBufferSize, MemoryMode mode, CompilationMode compilationMode, unsigned functionIndex, unsigned loopIndexForOSREntry, TierUpCount* tierUp)
 410      : m_info(info)
 411      , m_mode(mode)
 412      , m_compilationMode(compilationMode)
 413      , m_functionIndex(functionIndex)
 414      , m_loopIndexForOSREntry(loopIndexForOSREntry)
 415      , m_tierUp(tierUp)
 416      , m_proc(procedure)
 417      , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
 418      , m_osrEntryScratchBufferSize(osrEntryScratchBufferSize)
 419      , m_constantInsertionValues(m_proc)
 420      , m_numImportFunctions(info.importFunctionCount())
 421  {
 422      m_rootBlock = m_proc.addBlock();
 423      m_currentBlock = m_rootBlock;
 424  
 425      // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
 426      const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
 427  
 428      m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
 429      m_proc.pinRegister(m_memoryBaseGPR);
 430  
 431      m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;
 432      if (!Context::useFastTLS())
 433          m_proc.pinRegister(m_wasmContextInstanceGPR);
 434  
 435      if (mode != MemoryMode::Signaling) {
 436          m_boundsCheckingSizeGPR = pinnedRegs.boundsCheckingSizeRegister;
 437          m_proc.pinRegister(m_boundsCheckingSizeGPR);
 438      }
 439  
 440      if (info.memory) {
 441          m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR) {
 442              AllowMacroScratchRegisterUsage allowScratch(jit);
 443              switch (m_mode) {
 444              case MemoryMode::BoundsChecking:
 445                  ASSERT_UNUSED(pinnedGPR, m_boundsCheckingSizeGPR == pinnedGPR);
 446                  break;
 447              case MemoryMode::Signaling:
 448                  ASSERT_UNUSED(pinnedGPR, InvalidGPRReg == pinnedGPR);
 449                  break;
 450              }
 451              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
 452          });
 453      }
 454  
 455      {
 456          auto* calleeMoveLocation = &compilation->calleeMoveLocation;
 457          static_assert(CallFrameSlot::codeBlock * sizeof(Register) < WasmCallingConvention::headerSizeInBytes, "We rely on this here for now.");
 458          static_assert(CallFrameSlot::callee * sizeof(Register) < WasmCallingConvention::headerSizeInBytes, "We rely on this here for now.");
 459          B3::PatchpointValue* getCalleePatchpoint = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Int64, Origin());
 460          getCalleePatchpoint->resultConstraints = { B3::ValueRep::SomeRegister };
 461          getCalleePatchpoint->effects = B3::Effects::none();
 462          getCalleePatchpoint->setGenerator(
 463              [=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
 464                  GPRReg result = params[0].gpr();
 465                  MacroAssembler::DataLabelPtr moveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), result);
 466                  jit.addLinkTask([calleeMoveLocation, moveLocation] (LinkBuffer& linkBuffer) {
 467                      *calleeMoveLocation = linkBuffer.locationOf<WasmEntryPtrTag>(moveLocation);
 468                  });
 469              });
 470  
 471          B3::Value* offsetOfCallee = m_currentBlock->appendNew<B3::Const64Value>(m_proc, Origin(), CallFrameSlot::callee * sizeof(Register));
 472          m_currentBlock->appendNew<B3::MemoryValue>(m_proc, B3::Store, Origin(),
 473              getCalleePatchpoint,
 474              m_currentBlock->appendNew<B3::Value>(m_proc, B3::Add, Origin(), framePointer(), offsetOfCallee));
 475  
 476          // FIXME: We shouldn't have to store zero into the CodeBlock* spot in the call frame,
 477          // but there are places that interpret non-null CodeBlock slot to mean a valid CodeBlock.
 478          // When doing unwinding, we'll need to verify that the entire runtime is OK with a non-null
 479          // CodeBlock not implying that the CodeBlock is valid.
 480          // https://bugs.webkit.org/show_bug.cgi?id=165321
 481          B3::Value* offsetOfCodeBlock = m_currentBlock->appendNew<B3::Const64Value>(m_proc, Origin(), CallFrameSlot::codeBlock * sizeof(Register));
 482          m_currentBlock->appendNew<B3::MemoryValue>(m_proc, B3::Store, Origin(),
 483              m_currentBlock->appendNew<B3::Const64Value>(m_proc, Origin(), 0),
 484              m_currentBlock->appendNew<B3::Value>(m_proc, B3::Add, Origin(), framePointer(), offsetOfCodeBlock));
 485      }
 486  
 487      {
 488          B3::PatchpointValue* stackOverflowCheck = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, pointerType(), Origin());
 489          m_instanceValue = stackOverflowCheck;
 490          stackOverflowCheck->appendSomeRegister(framePointer());
 491          stackOverflowCheck->clobber(RegisterSet::macroScratchRegisters());
 492          if (!Context::useFastTLS()) {
 493              // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
 494              // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
 495              stackOverflowCheck->effects.writesPinned = false;
 496              stackOverflowCheck->effects.readsPinned = true;
 497              stackOverflowCheck->resultConstraints = { ValueRep::reg(m_wasmContextInstanceGPR) };
 498          }
 499          stackOverflowCheck->numGPScratchRegisters = 2;
 500          stackOverflowCheck->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
 501              const Checked<int32_t> wasmFrameSize = params.proc().frameSize();
 502              const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024);
 503              const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max<uint32_t>(
 504                  // This allows us to elide stack checks for functions that are terminal nodes in the call
 505                  // tree, (e.g they don't make any calls) and have a small enough frame size. This works by
 506                  // having any such terminal node have its parent caller include some extra size in its
 507                  // own check for it. The goal here is twofold:
 508                  // 1. Emit less code.
 509                  // 2. Try to speed things up by skipping stack checks.
 510                  minimumParentCheckSize,
 511                  // This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will
 512                  // spill all arguments to the stack, we ensure that a stack check here covers the
 513                  // stack that such a stub would use.
 514                  (Checked<uint32_t>(m_maxNumJSCallArguments) * sizeof(Register) + JSCallingConvention::headerSizeInBytes).unsafeGet()
 515              ));
 516              const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).unsafeGet() : wasmFrameSize.unsafeGet();
 517              bool needUnderflowCheck = static_cast<unsigned>(checkSize) > Options::reservedZoneSize();
 518              bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= minimumParentCheckSize || needUnderflowCheck;
 519  
 520              GPRReg contextInstance = Context::useFastTLS() ? params[0].gpr() : m_wasmContextInstanceGPR;
 521  
 522              // This allows leaf functions to not do stack checks if their frame size is within
 523              // certain limits since their caller would have already done the check.
 524              if (needsOverflowCheck) {
 525                  AllowMacroScratchRegisterUsage allowScratch(jit);
 526                  GPRReg fp = params[1].gpr();
 527                  GPRReg scratch1 = params.gpScratch(0);
 528                  GPRReg scratch2 = params.gpScratch(1);
 529  
 530                  if (Context::useFastTLS())
 531                      jit.loadWasmContextInstance(contextInstance);
 532  
 533                  jit.loadPtr(CCallHelpers::Address(contextInstance, Instance::offsetOfCachedStackLimit()), scratch2);
 534                  jit.addPtr(CCallHelpers::TrustedImm32(-checkSize), fp, scratch1);
 535                  MacroAssembler::JumpList overflow;
 536                  if (UNLIKELY(needUnderflowCheck))
 537                      overflow.append(jit.branchPtr(CCallHelpers::Above, scratch1, fp));
 538                  overflow.append(jit.branchPtr(CCallHelpers::Below, scratch1, scratch2));
 539                  jit.addLinkTask([overflow] (LinkBuffer& linkBuffer) {
 540                      linkBuffer.link(overflow, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwStackOverflowFromWasmThunkGenerator).code()));
 541                  });
 542              } else if (m_usesInstanceValue && Context::useFastTLS()) {
 543                  // No overflow check is needed, but the instance values still needs to be correct.
 544                  AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::loadWasmContextInstanceNeedsMacroScratchRegister());
 545                  jit.loadWasmContextInstance(contextInstance);
 546              } else {
 547                  // We said we'd return a pointer. We don't actually need to because it isn't used, but the patchpoint conservatively said it had effects (potential stack check) which prevent it from getting removed.
 548              }
 549          });
 550      }
 551  
 552      emitEntryTierUpCheck();
 553  
 554      if (m_compilationMode == CompilationMode::OMGForOSREntryMode)
 555          m_currentBlock = m_proc.addBlock();
 556  }
 557  
 558  void B3IRGenerator::restoreWebAssemblyGlobalState(RestoreCachedStackLimit restoreCachedStackLimit, const MemoryInformation& memory, Value* instance, Procedure& proc, BasicBlock* block)
 559  {
 560      restoreWasmContextInstance(proc, block, instance);
 561  
 562      if (restoreCachedStackLimit == RestoreCachedStackLimit::Yes) {
 563          // The Instance caches the stack limit, but also knows where its canonical location is.
 564          Value* pointerToActualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfPointerToActualStackLimit()));
 565          Value* actualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), pointerToActualStackLimit);
 566          block->appendNew<MemoryValue>(m_proc, Store, origin(), actualStackLimit, instanceValue(), safeCast<int32_t>(Instance::offsetOfCachedStackLimit()));
 567      }
 568  
 569      if (!!memory) {
 570          const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
 571          RegisterSet clobbers;
 572          clobbers.set(pinnedRegs->baseMemoryPointer);
 573          clobbers.set(pinnedRegs->boundsCheckingSizeRegister);
 574          if (!isARM64())
 575              clobbers.set(RegisterSet::macroScratchRegisters());
 576  
 577          B3::PatchpointValue* patchpoint = block->appendNew<B3::PatchpointValue>(proc, B3::Void, origin());
 578          Effects effects = Effects::none();
 579          effects.writesPinned = true;
 580          effects.reads = B3::HeapRange::top();
 581          patchpoint->effects = effects;
 582          patchpoint->clobber(clobbers);
 583          patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
 584  
 585          patchpoint->append(instance, ValueRep::SomeRegister);
 586          patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
 587              AllowMacroScratchRegisterUsage allowScratch(jit);
 588              GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
 589              GPRReg scratchOrBoundsCheckingSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs->boundsCheckingSizeRegister;
 590  
 591              jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedBoundsCheckingSize()), pinnedRegs->boundsCheckingSizeRegister);
 592              jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemory()), baseMemory);
 593  
 594              jit.cageConditionally(Gigacage::Primitive, baseMemory, pinnedRegs->boundsCheckingSizeRegister, scratchOrBoundsCheckingSize);
 595          });
 596      }
 597  }
 598  
 599  void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
 600  {
 601      jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
 602      auto jumpToExceptionStub = jit.jump();
 603  
 604      jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
 605          linkBuffer.link(jumpToExceptionStub, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
 606      });
 607  }
 608  
 609  Value* B3IRGenerator::constant(B3::Type type, uint64_t bits, Optional<Origin> maybeOrigin)
 610  {
 611      auto result = m_constantPool.ensure(ValueKey(opcodeForConstant(type), type, static_cast<int64_t>(bits)), [&] {
 612          Value* result = m_proc.addConstant(maybeOrigin ? *maybeOrigin : origin(), type, bits);
 613          m_constantInsertionValues.insertValue(0, result);
 614          return result;
 615      });
 616      return result.iterator->value;
 617  }
 618  
 619  Value* B3IRGenerator::framePointer()
 620  {
 621      if (!m_framePointer) {
 622          m_framePointer = m_proc.add<B3::Value>(B3::FramePointer, Origin());
 623          ASSERT(m_framePointer);
 624          m_constantInsertionValues.insertValue(0, m_framePointer);
 625      }
 626      return m_framePointer;
 627  }
 628  
 629  void B3IRGenerator::insertConstants()
 630  {
 631      m_constantInsertionValues.execute(m_proc.at(0));
 632  }
 633  
 634  B3::Type B3IRGenerator::toB3ResultType(BlockSignature returnType)
 635  {
 636      if (returnType->returnsVoid())
 637          return B3::Void;
 638  
 639      if (returnType->returnCount() == 1)
 640          return toB3Type(returnType->returnType(0));
 641  
 642      auto result = m_tupleMap.ensure(returnType, [&] {
 643          Vector<B3::Type> result;
 644          for (unsigned i = 0; i < returnType->returnCount(); ++i)
 645              result.append(toB3Type(returnType->returnType(i)));
 646          return m_proc.addTuple(WTFMove(result));
 647      });
 648      return result.iterator->value;
 649  }
 650  
 651  auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
 652  {
 653      size_t newSize = m_locals.size() + count;
 654      ASSERT(!(CheckedUint32(count) + m_locals.size()).hasOverflowed());
 655      ASSERT(newSize <= maxFunctionLocals);
 656      WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(newSize), "can't allocate memory for ", newSize, " locals");
 657  
 658      for (uint32_t i = 0; i < count; ++i) {
 659          Variable* local = m_proc.addVariable(toB3Type(type));
 660          m_locals.uncheckedAppend(local);
 661          auto val = isRefType(type) ? JSValue::encode(jsNull()) : 0;
 662          m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, constant(toB3Type(type), val, Origin()));
 663      }
 664      return { };
 665  }
 666  
 667  auto B3IRGenerator::addArguments(const Signature& signature) -> PartialResult
 668  {
 669      ASSERT(!m_locals.size());
 670      WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature.argumentCount()), "can't allocate memory for ", signature.argumentCount(), " arguments");
 671  
 672      m_locals.grow(signature.argumentCount());
 673      CallInformation wasmCallInfo = wasmCallingConvention().callInformationFor(signature, CallRole::Callee);
 674  
 675      for (size_t i = 0; i < signature.argumentCount(); ++i) {
 676          B3::Type type = toB3Type(signature.argument(i));
 677          B3::Value* argument;
 678          auto rep = wasmCallInfo.params[i];
 679          if (rep.isReg()) {
 680              argument = m_currentBlock->appendNew<B3::ArgumentRegValue>(m_proc, Origin(), rep.reg());
 681              if (type == B3::Int32 || type == B3::Float)
 682                  argument = m_currentBlock->appendNew<B3::Value>(m_proc, B3::Trunc, Origin(), argument);
 683          } else {
 684              ASSERT(rep.isStack());
 685              B3::Value* address = m_currentBlock->appendNew<B3::Value>(m_proc, B3::Add, Origin(), framePointer(),
 686                  m_currentBlock->appendNew<B3::Const64Value>(m_proc, Origin(), rep.offsetFromFP()));
 687              argument = m_currentBlock->appendNew<B3::MemoryValue>(m_proc, B3::Load, type, Origin(), address);
 688          }
 689  
 690          Variable* argumentVariable = m_proc.addVariable(argument->type());
 691          m_locals[i] = argumentVariable;
 692          m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
 693      }
 694  
 695      return { };
 696  }
 697  
 698  auto B3IRGenerator::addRefIsNull(ExpressionType value, ExpressionType& result) -> PartialResult
 699  {
 700      result = m_currentBlock->appendNew<Value>(m_proc, B3::Equal, origin(), value, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), JSValue::encode(jsNull())));
 701      return { };
 702  }
 703  
 704  auto B3IRGenerator::addTableGet(unsigned tableIndex, ExpressionType index, ExpressionType& result) -> PartialResult
 705  {
 706      // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
 707      result = m_currentBlock->appendNew<CCallValue>(m_proc, toB3Type(Externref), origin(),
 708          m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationGetWasmTableElement)),
 709          instanceValue(), m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex), index);
 710  
 711      {
 712          CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
 713              m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), result, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), 0)));
 714  
 715          check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 716              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess);
 717          });
 718      }
 719  
 720      return { };
 721  }
 722  
 723  auto B3IRGenerator::addTableSet(unsigned tableIndex, ExpressionType index, ExpressionType value) -> PartialResult
 724  {
 725      // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
 726      auto shouldThrow = m_currentBlock->appendNew<CCallValue>(m_proc, B3::Int32, origin(),
 727          m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationSetWasmTableElement)),
 728          instanceValue(), m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex), index, value);
 729  
 730      {
 731          CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
 732              m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), shouldThrow, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0)));
 733  
 734          check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 735              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess);
 736          });
 737      }
 738  
 739      return { };
 740  }
 741  
 742  auto B3IRGenerator::addRefFunc(uint32_t index, ExpressionType& result) -> PartialResult
 743  {
 744      // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
 745  
 746      result = m_currentBlock->appendNew<CCallValue>(m_proc, B3::Int64, origin(),
 747          m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationWasmRefFunc)),
 748          instanceValue(), addConstant(Type::I32, index));
 749  
 750      return { };
 751  }
 752  
 753  auto B3IRGenerator::addTableInit(unsigned elementIndex, unsigned tableIndex, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length) -> PartialResult
 754  {
 755      auto result = m_currentBlock->appendNew<CCallValue>(
 756          m_proc, toB3Type(I32), origin(),
 757          m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationWasmTableInit)),
 758          instanceValue(),
 759          m_currentBlock->appendNew<Const32Value>(m_proc, origin(), elementIndex),
 760          m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex),
 761          dstOffset, srcOffset, length);
 762  
 763      {
 764          CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
 765              m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), result, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0)));
 766  
 767          check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 768              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess);
 769          });
 770      }
 771  
 772      return { };
 773  }
 774  
 775  auto B3IRGenerator::addElemDrop(unsigned elementIndex) -> PartialResult
 776  {
 777      m_currentBlock->appendNew<CCallValue>(
 778          m_proc, B3::Void, origin(),
 779          m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationWasmElemDrop)),
 780          instanceValue(),
 781          m_currentBlock->appendNew<Const32Value>(m_proc, origin(), elementIndex));
 782  
 783      return { };
 784  }
 785  
 786  auto B3IRGenerator::addTableSize(unsigned tableIndex, ExpressionType& result) -> PartialResult
 787  {
 788      // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
 789      result = m_currentBlock->appendNew<CCallValue>(m_proc, toB3Type(I32), origin(),
 790          m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationGetWasmTableSize)),
 791          instanceValue(), m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex));
 792  
 793      return { };
 794  }
 795  
 796  auto B3IRGenerator::addTableGrow(unsigned tableIndex, ExpressionType fill, ExpressionType delta, ExpressionType& result) -> PartialResult
 797  {
 798      result = m_currentBlock->appendNew<CCallValue>(m_proc, toB3Type(I32), origin(),
 799          m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationWasmTableGrow)),
 800          instanceValue(), m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex), fill, delta);
 801  
 802      return { };
 803  }
 804  
 805  auto B3IRGenerator::addTableFill(unsigned tableIndex, ExpressionType offset, ExpressionType fill, ExpressionType count) -> PartialResult
 806  {
 807      auto result = m_currentBlock->appendNew<CCallValue>(m_proc, toB3Type(I32), origin(),
 808          m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationWasmTableFill)),
 809          instanceValue(), m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex), offset, fill, count);
 810  
 811      {
 812          CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
 813              m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), result, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0)));
 814  
 815          check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 816              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess);
 817          });
 818      }
 819  
 820      return { };
 821  }
 822  
 823  auto B3IRGenerator::addTableCopy(unsigned dstTableIndex, unsigned srcTableIndex, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length) -> PartialResult
 824  {
 825      auto result = m_currentBlock->appendNew<CCallValue>(
 826          m_proc, toB3Type(I32), origin(),
 827          m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationWasmTableCopy)),
 828          instanceValue(),
 829          m_currentBlock->appendNew<Const32Value>(m_proc, origin(), dstTableIndex),
 830          m_currentBlock->appendNew<Const32Value>(m_proc, origin(), srcTableIndex),
 831          dstOffset, srcOffset, length);
 832  
 833      {
 834          CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
 835              m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), result, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0)));
 836  
 837          check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 838              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess);
 839          });
 840      }
 841  
 842      return { };
 843  }
 844  
 845  auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
 846  {
 847      ASSERT(m_locals[index]);
 848      result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin(), m_locals[index]);
 849      return { };
 850  }
 851  
 852  auto B3IRGenerator::addUnreachable() -> PartialResult
 853  {
 854      B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
 855      unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 856          this->emitExceptionCheck(jit, ExceptionType::Unreachable);
 857      });
 858      unreachable->effects.terminal = true;
 859      return { };
 860  }
 861  
 862  auto B3IRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
 863  {
 864      result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
 865          m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationGrowMemory)),
 866          framePointer(), instanceValue(), delta);
 867  
 868      restoreWebAssemblyGlobalState(RestoreCachedStackLimit::No, m_info.memory, instanceValue(), m_proc, m_currentBlock);
 869  
 870      return { };
 871  }
 872  
 873  auto B3IRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
 874  {
 875      static_assert(sizeof(decltype(static_cast<Memory*>(nullptr)->size())) == sizeof(uint64_t), "codegen relies on this size");
 876      Value* memory = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfMemory()));
 877      Value* handle = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), memory, safeCast<int32_t>(Memory::offsetOfHandle()));
 878      Value* size = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), handle, safeCast<int32_t>(MemoryHandle::offsetOfSize()));
 879  
 880      constexpr uint32_t shiftValue = 16;
 881      static_assert(PageCount::pageSize == 1ull << shiftValue, "This must hold for the code below to be correct.");
 882      Value* numPages = m_currentBlock->appendNew<Value>(m_proc, ZShr, origin(),
 883          size, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), shiftValue));
 884  
 885      result = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), numPages);
 886  
 887      return { };
 888  }
 889  
 890  auto B3IRGenerator::addMemoryFill(ExpressionType dstAddress, ExpressionType targetValue, ExpressionType count) -> PartialResult
 891  {
 892      auto result = m_currentBlock->appendNew<CCallValue>(
 893          m_proc, toB3Type(I32), origin(),
 894          m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationWasmMemoryFill)),
 895          instanceValue(),
 896          dstAddress, targetValue, count);
 897  
 898      {
 899          CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
 900              m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), result, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0)));
 901  
 902          check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 903              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess);
 904          });
 905      }
 906  
 907      return { };
 908  }
 909  
 910  auto B3IRGenerator::addMemoryInit(unsigned dataSegmentIndex, ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType length) -> PartialResult
 911  {
 912      auto result = m_currentBlock->appendNew<CCallValue>(
 913          m_proc, toB3Type(I32), origin(),
 914          m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationWasmMemoryInit)),
 915          instanceValue(),
 916          m_currentBlock->appendNew<Const32Value>(m_proc, origin(), dataSegmentIndex),
 917          dstAddress, srcAddress, length);
 918  
 919      {
 920          CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
 921              m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), result, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0)));
 922  
 923          check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 924              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess);
 925          });
 926      }
 927  
 928      return { };
 929  }
 930  
 931  auto B3IRGenerator::addMemoryCopy(ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType count) -> PartialResult
 932  {
 933      auto result = m_currentBlock->appendNew<CCallValue>(
 934          m_proc, toB3Type(I32), origin(),
 935          m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationWasmMemoryCopy)),
 936          instanceValue(),
 937          dstAddress, srcAddress, count);
 938  
 939      {
 940          CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
 941              m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), result, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0)));
 942  
 943          check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 944              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess);
 945          });
 946      }
 947  
 948      return { };
 949  }
 950  
 951  auto B3IRGenerator::addDataDrop(unsigned dataSegmentIndex) -> PartialResult
 952  {
 953      m_currentBlock->appendNew<CCallValue>(
 954          m_proc, B3::Void, origin(),
 955          m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationWasmDataDrop)),
 956          instanceValue(),
 957          m_currentBlock->appendNew<Const32Value>(m_proc, origin(), dataSegmentIndex));
 958  
 959      return { };
 960  }
 961  
 962  auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
 963  {
 964      ASSERT(m_locals[index]);
 965      m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, origin(), m_locals[index], value);
 966      return { };
 967  }
 968  
 969  auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
 970  {
 971      const Wasm::GlobalInformation& global = m_info.globals[index];
 972      Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
 973      switch (global.bindingMode) {
 974      case Wasm::GlobalInformation::BindingMode::EmbeddedInInstance:
 975          result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(global.type), origin(), globalsArray, safeCast<int32_t>(index * sizeof(Register)));
 976          break;
 977      case Wasm::GlobalInformation::BindingMode::Portable: {
 978          ASSERT(global.mutability == Wasm::GlobalInformation::Mutability::Mutable);
 979          Value* pointer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, B3::Int64, origin(), globalsArray, safeCast<int32_t>(index * sizeof(Register)));
 980          result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(global.type), origin(), pointer);
 981          break;
 982      }
 983      }
 984      return { };
 985  }
 986  
 987  auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
 988  {
 989      const Wasm::GlobalInformation& global = m_info.globals[index];
 990      ASSERT(toB3Type(global.type) == value->type());
 991      Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
 992      switch (global.bindingMode) {
 993      case Wasm::GlobalInformation::BindingMode::EmbeddedInInstance:
 994          m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin(), value, globalsArray, safeCast<int32_t>(index * sizeof(Register)));
 995          if (isRefType(global.type))
 996              emitWriteBarrierForJSWrapper();
 997          break;
 998      case Wasm::GlobalInformation::BindingMode::Portable: {
 999          ASSERT(global.mutability == Wasm::GlobalInformation::Mutability::Mutable);
1000          Value* pointer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, B3::Int64, origin(), globalsArray, safeCast<int32_t>(index * sizeof(Register)));
1001          m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin(), value, pointer);
1002          // We emit a write-barrier onto JSWebAssemblyGlobal, not JSWebAssemblyInstance.
1003          if (isRefType(global.type)) {
1004              Value* instance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfOwner()));
1005              Value* cell = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), pointer, Wasm::Global::offsetOfOwner() - Wasm::Global::offsetOfValue());
1006              Value* cellState = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Int32, origin(), cell, safeCast<int32_t>(JSCell::cellStateOffset()));
1007              Value* vm = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instance, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfVM()));
1008              Value* threshold = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(), vm, safeCast<int32_t>(VM::offsetOfHeapBarrierThreshold()));
1009  
1010              BasicBlock* fenceCheckPath = m_proc.addBlock();
1011              BasicBlock* fencePath = m_proc.addBlock();
1012              BasicBlock* doSlowPath = m_proc.addBlock();
1013              BasicBlock* continuation = m_proc.addBlock();
1014  
1015              m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
1016                  m_currentBlock->appendNew<Value>(m_proc, Above, origin(), cellState, threshold),
1017                  FrequentedBlock(continuation), FrequentedBlock(fenceCheckPath, FrequencyClass::Rare));
1018              fenceCheckPath->addPredecessor(m_currentBlock);
1019              continuation->addPredecessor(m_currentBlock);
1020              m_currentBlock = fenceCheckPath;
1021  
1022              Value* shouldFence = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Int32, origin(), vm, safeCast<int32_t>(VM::offsetOfHeapMutatorShouldBeFenced()));
1023              m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
1024                  shouldFence,
1025                  FrequentedBlock(fencePath), FrequentedBlock(doSlowPath));
1026              fencePath->addPredecessor(m_currentBlock);
1027              doSlowPath->addPredecessor(m_currentBlock);
1028              m_currentBlock = fencePath;
1029  
1030              B3::PatchpointValue* doFence = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
1031              doFence->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1032                  jit.memoryFence();
1033              });
1034  
1035              Value* cellStateLoadAfterFence = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Int32, origin(), cell, safeCast<int32_t>(JSCell::cellStateOffset()));
1036              m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
1037                  m_currentBlock->appendNew<Value>(m_proc, Above, origin(), cellStateLoadAfterFence, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), blackThreshold)),
1038                  FrequentedBlock(continuation), FrequentedBlock(doSlowPath, FrequencyClass::Rare));
1039              doSlowPath->addPredecessor(m_currentBlock);
1040              continuation->addPredecessor(m_currentBlock);
1041              m_currentBlock = doSlowPath;
1042  
1043              Value* writeBarrierAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationWasmWriteBarrierSlowPath));
1044              m_currentBlock->appendNew<CCallValue>(m_proc, B3::Void, origin(), writeBarrierAddress, cell, vm);
1045              m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1046  
1047              continuation->addPredecessor(m_currentBlock);
1048              m_currentBlock = continuation;
1049          }
1050          break;
1051      }
1052      }
1053      return { };
1054  }
1055  
1056  inline void B3IRGenerator::emitWriteBarrierForJSWrapper()
1057  {
1058      Value* cell = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfOwner()));
1059      Value* cellState = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Int32, origin(), cell, safeCast<int32_t>(JSCell::cellStateOffset()));
1060      Value* vm = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), cell, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfVM()));
1061      Value* threshold = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(), vm, safeCast<int32_t>(VM::offsetOfHeapBarrierThreshold()));
1062  
1063      BasicBlock* fenceCheckPath = m_proc.addBlock();
1064      BasicBlock* fencePath = m_proc.addBlock();
1065      BasicBlock* doSlowPath = m_proc.addBlock();
1066      BasicBlock* continuation = m_proc.addBlock();
1067  
1068      m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
1069          m_currentBlock->appendNew<Value>(m_proc, Above, origin(), cellState, threshold),
1070          FrequentedBlock(continuation), FrequentedBlock(fenceCheckPath, FrequencyClass::Rare));
1071      fenceCheckPath->addPredecessor(m_currentBlock);
1072      continuation->addPredecessor(m_currentBlock);
1073      m_currentBlock = fenceCheckPath;
1074  
1075      Value* shouldFence = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Int32, origin(), vm, safeCast<int32_t>(VM::offsetOfHeapMutatorShouldBeFenced()));
1076      m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
1077          shouldFence,
1078          FrequentedBlock(fencePath), FrequentedBlock(doSlowPath));
1079      fencePath->addPredecessor(m_currentBlock);
1080      doSlowPath->addPredecessor(m_currentBlock);
1081      m_currentBlock = fencePath;
1082  
1083      B3::PatchpointValue* doFence = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
1084      doFence->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1085          jit.memoryFence();
1086      });
1087  
1088      Value* cellStateLoadAfterFence = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Int32, origin(), cell, safeCast<int32_t>(JSCell::cellStateOffset()));
1089      m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
1090          m_currentBlock->appendNew<Value>(m_proc, Above, origin(), cellStateLoadAfterFence, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), blackThreshold)),
1091          FrequentedBlock(continuation), FrequentedBlock(doSlowPath, FrequencyClass::Rare));
1092      doSlowPath->addPredecessor(m_currentBlock);
1093      continuation->addPredecessor(m_currentBlock);
1094      m_currentBlock = doSlowPath;
1095  
1096      Value* writeBarrierAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationWasmWriteBarrierSlowPath));
1097      m_currentBlock->appendNew<CCallValue>(m_proc, B3::Void, origin(), writeBarrierAddress, cell, vm);
1098      m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1099  
1100      continuation->addPredecessor(m_currentBlock);
1101      m_currentBlock = continuation;
1102  }
1103  
1104  inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
1105  {
1106      ASSERT(m_memoryBaseGPR);
1107  
1108      switch (m_mode) {
1109      case MemoryMode::BoundsChecking: {
1110          // We're not using signal handling only when the memory is not shared.
1111          // Regardless of signaling, we must check that no memory access exceeds the current memory size.
1112          ASSERT(m_boundsCheckingSizeGPR);
1113          ASSERT(sizeOfOperation + offset > offset);
1114          m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), m_boundsCheckingSizeGPR, pointer, sizeOfOperation + offset - 1);
1115          break;
1116      }
1117  
1118      case MemoryMode::Signaling: {
1119          // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],
1120          // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register
1121          // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above
1122          // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any
1123          // register + large offset access. We don't think this will be generated frequently.
1124          //
1125          // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the
1126          // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater
1127          // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that
1128          // any access equal to or greater than 4GiB will trap, no need to add the redzone.
1129          if (offset >= Memory::fastMappedRedzoneBytes()) {
1130              size_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();
1131              m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), pointer, sizeOfOperation + offset - 1, maximum);
1132          }
1133          break;
1134      }
1135      }
1136  
1137      pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), pointer);
1138      return m_currentBlock->appendNew<WasmAddressValue>(m_proc, origin(), pointer, m_memoryBaseGPR);
1139  }
1140  
1141  inline uint32_t sizeOfLoadOp(LoadOpType op)
1142  {
1143      switch (op) {
1144      case LoadOpType::I32Load8S:
1145      case LoadOpType::I32Load8U:
1146      case LoadOpType::I64Load8S:
1147      case LoadOpType::I64Load8U:
1148          return 1;
1149      case LoadOpType::I32Load16S:
1150      case LoadOpType::I64Load16S:
1151      case LoadOpType::I32Load16U:
1152      case LoadOpType::I64Load16U:
1153          return 2;
1154      case LoadOpType::I32Load:
1155      case LoadOpType::I64Load32S:
1156      case LoadOpType::I64Load32U:
1157      case LoadOpType::F32Load:
1158          return 4;
1159      case LoadOpType::I64Load:
1160      case LoadOpType::F64Load:
1161          return 8;
1162      }
1163      RELEASE_ASSERT_NOT_REACHED();
1164  }
1165  
1166  inline B3::Kind B3IRGenerator::memoryKind(B3::Opcode memoryOp)
1167  {
1168      if (m_mode == MemoryMode::Signaling || m_info.memory.isShared())
1169          return trapping(memoryOp);
1170      return memoryOp;
1171  }
1172  
1173  inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
1174  {
1175      int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
1176  
1177      switch (op) {
1178      case LoadOpType::I32Load8S: {
1179          return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
1180      }
1181  
1182      case LoadOpType::I64Load8S: {
1183          Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
1184          return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
1185      }
1186  
1187      case LoadOpType::I32Load8U: {
1188          return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
1189      }
1190  
1191      case LoadOpType::I64Load8U: {
1192          Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
1193          return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
1194      }
1195  
1196      case LoadOpType::I32Load16S: {
1197          return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
1198      }
1199  
1200      case LoadOpType::I64Load16S: {
1201          Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
1202          return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
1203      }
1204  
1205      case LoadOpType::I32Load16U: {
1206          return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
1207      }
1208  
1209      case LoadOpType::I64Load16U: {
1210          Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
1211          return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
1212      }
1213  
1214      case LoadOpType::I32Load: {
1215          return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
1216      }
1217  
1218      case LoadOpType::I64Load32U: {
1219          Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
1220          return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
1221      }
1222  
1223      case LoadOpType::I64Load32S: {
1224          Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
1225          return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
1226      }
1227  
1228      case LoadOpType::I64Load: {
1229          return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int64, origin(), pointer, offset);
1230      }
1231  
1232      case LoadOpType::F32Load: {
1233          return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Float, origin(), pointer, offset);
1234      }
1235  
1236      case LoadOpType::F64Load: {
1237          return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Double, origin(), pointer, offset);
1238      }
1239      }
1240      RELEASE_ASSERT_NOT_REACHED();
1241  }
1242  
1243  auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
1244  {
1245      ASSERT(pointer->type() == Int32);
1246  
1247      if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
1248          // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1249          // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1250          B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
1251          throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1252              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
1253          });
1254  
1255          switch (op) {
1256          case LoadOpType::I32Load8S:
1257          case LoadOpType::I32Load16S:
1258          case LoadOpType::I32Load:
1259          case LoadOpType::I32Load16U:
1260          case LoadOpType::I32Load8U:
1261              result = constant(Int32, 0);
1262              break;
1263          case LoadOpType::I64Load8S:
1264          case LoadOpType::I64Load8U:
1265          case LoadOpType::I64Load16S:
1266          case LoadOpType::I64Load32U:
1267          case LoadOpType::I64Load32S:
1268          case LoadOpType::I64Load:
1269          case LoadOpType::I64Load16U:
1270              result = constant(Int64, 0);
1271              break;
1272          case LoadOpType::F32Load:
1273              result = constant(Float, 0);
1274              break;
1275          case LoadOpType::F64Load:
1276              result = constant(Double, 0);
1277              break;
1278          }
1279  
1280      } else
1281          result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
1282  
1283      return { };
1284  }
1285  
1286  inline uint32_t sizeOfStoreOp(StoreOpType op)
1287  {
1288      switch (op) {
1289      case StoreOpType::I32Store8:
1290      case StoreOpType::I64Store8:
1291          return 1;
1292      case StoreOpType::I32Store16:
1293      case StoreOpType::I64Store16:
1294          return 2;
1295      case StoreOpType::I32Store:
1296      case StoreOpType::I64Store32:
1297      case StoreOpType::F32Store:
1298          return 4;
1299      case StoreOpType::I64Store:
1300      case StoreOpType::F64Store:
1301          return 8;
1302      }
1303      RELEASE_ASSERT_NOT_REACHED();
1304  }
1305  
1306  
1307  inline void B3IRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
1308  {
1309      int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
1310  
1311      switch (op) {
1312      case StoreOpType::I64Store8:
1313          value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
1314          FALLTHROUGH;
1315  
1316      case StoreOpType::I32Store8:
1317          m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store8), origin(), value, pointer, offset);
1318          return;
1319  
1320      case StoreOpType::I64Store16:
1321          value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
1322          FALLTHROUGH;
1323  
1324      case StoreOpType::I32Store16:
1325          m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store16), origin(), value, pointer, offset);
1326          return;
1327  
1328      case StoreOpType::I64Store32:
1329          value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
1330          FALLTHROUGH;
1331  
1332      case StoreOpType::I64Store:
1333      case StoreOpType::I32Store:
1334      case StoreOpType::F32Store:
1335      case StoreOpType::F64Store:
1336          m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store), origin(), value, pointer, offset);
1337          return;
1338      }
1339      RELEASE_ASSERT_NOT_REACHED();
1340  }
1341  
1342  auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
1343  {
1344      ASSERT(pointer->type() == Int32);
1345  
1346      if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
1347          // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1348          // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1349          B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
1350          throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1351              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
1352          });
1353      } else
1354          emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
1355  
1356      return { };
1357  }
1358  
1359  inline B3::Width accessWidth(ExtAtomicOpType op)
1360  {
1361      return static_cast<B3::Width>(memoryLog2Alignment(op));
1362  }
1363  
1364  inline uint32_t sizeOfAtomicOpMemoryAccess(ExtAtomicOpType op)
1365  {
1366      return bytesForWidth(accessWidth(op));
1367  }
1368  
1369  inline Value* B3IRGenerator::sanitizeAtomicResult(ExtAtomicOpType op, Type valueType, ExpressionType result)
1370  {
1371      auto sanitize32 = [&](ExpressionType result) {
1372          switch (accessWidth(op)) {
1373          case B3::Width8:
1374              return m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), result, constant(Int32, 0xff));
1375          case B3::Width16:
1376              return m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), result, constant(Int32, 0xffff));
1377          default:
1378              return result;
1379          }
1380      };
1381  
1382      switch (valueType) {
1383      case Type::I64: {
1384          if (accessWidth(op) == B3::Width64)
1385              return result;
1386          return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), sanitize32(result));
1387      }
1388      case Type::I32:
1389          return sanitize32(result);
1390      default:
1391          RELEASE_ASSERT_NOT_REACHED();
1392          return nullptr;
1393      }
1394  }
1395  
1396  Value* B3IRGenerator::fixupPointerPlusOffsetForAtomicOps(ExtAtomicOpType op, ExpressionType ptr, uint32_t offset)
1397  {
1398      auto pointer = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), ptr, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), offset));
1399      if (accessWidth(op) != B3::Width8) {
1400          CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1401              m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), pointer, constant(pointerType(), sizeOfAtomicOpMemoryAccess(op) - 1)));
1402          check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1403              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
1404          });
1405      }
1406      return pointer;
1407  }
1408  
1409  inline Value* B3IRGenerator::emitAtomicLoadOp(ExtAtomicOpType op, Type valueType, ExpressionType pointer, uint32_t uoffset)
1410  {
1411      pointer = fixupPointerPlusOffsetForAtomicOps(op, pointer, uoffset);
1412  
1413      ExpressionType value = nullptr;
1414      switch (accessWidth(op)) {
1415      case B3::Width8:
1416      case B3::Width16:
1417      case B3::Width32:
1418          value = constant(Int32, 0);
1419          break;
1420      case B3::Width64:
1421          value = constant(Int64, 0);
1422          break;
1423      }
1424  
1425      return sanitizeAtomicResult(op, valueType, m_currentBlock->appendNew<AtomicValue>(m_proc, memoryKind(AtomicXchgAdd), origin(), accessWidth(op), value, pointer));
1426  }
1427  
1428  auto B3IRGenerator::atomicLoad(ExtAtomicOpType op, Type valueType, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
1429  {
1430      ASSERT(pointer->type() == Int32);
1431  
1432      if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfAtomicOpMemoryAccess(op)))) {
1433          // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1434          // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1435          B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
1436          throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1437              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
1438          });
1439  
1440          switch (valueType) {
1441          case Type::I32:
1442              result = constant(Int32, 0);
1443              break;
1444          case Type::I64:
1445              result = constant(Int64, 0);
1446              break;
1447          default:
1448              RELEASE_ASSERT_NOT_REACHED();
1449              break;
1450          }
1451      } else
1452          result = emitAtomicLoadOp(op, valueType, emitCheckAndPreparePointer(pointer, offset, sizeOfAtomicOpMemoryAccess(op)), offset);
1453  
1454      return { };
1455  }
1456  
1457  inline void B3IRGenerator::emitAtomicStoreOp(ExtAtomicOpType op, Type valueType, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
1458  {
1459      pointer = fixupPointerPlusOffsetForAtomicOps(op, pointer, uoffset);
1460  
1461      if (valueType == Type::I64 && accessWidth(op) != B3::Width64)
1462          value = m_currentBlock->appendNew<B3::Value>(m_proc, B3::Trunc, Origin(), value);
1463      m_currentBlock->appendNew<AtomicValue>(m_proc, memoryKind(AtomicXchg), origin(), accessWidth(op), value, pointer);
1464  }
1465  
1466  auto B3IRGenerator::atomicStore(ExtAtomicOpType op, Type valueType, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
1467  {
1468      ASSERT(pointer->type() == Int32);
1469  
1470      if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfAtomicOpMemoryAccess(op)))) {
1471          // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1472          // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1473          B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
1474          throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1475              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
1476          });
1477      } else
1478          emitAtomicStoreOp(op, valueType, emitCheckAndPreparePointer(pointer, offset, sizeOfAtomicOpMemoryAccess(op)), value, offset);
1479  
1480      return { };
1481  }
1482  
1483  inline Value* B3IRGenerator::emitAtomicBinaryRMWOp(ExtAtomicOpType op, Type valueType, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
1484  {
1485      pointer = fixupPointerPlusOffsetForAtomicOps(op, pointer, uoffset);
1486  
1487      B3::Opcode opcode = B3::Nop;
1488      switch (op) {
1489      case ExtAtomicOpType::I32AtomicRmw8AddU:
1490      case ExtAtomicOpType::I32AtomicRmw16AddU:
1491      case ExtAtomicOpType::I32AtomicRmwAdd:
1492      case ExtAtomicOpType::I64AtomicRmw8AddU:
1493      case ExtAtomicOpType::I64AtomicRmw16AddU:
1494      case ExtAtomicOpType::I64AtomicRmw32AddU:
1495      case ExtAtomicOpType::I64AtomicRmwAdd:
1496          opcode = AtomicXchgAdd;
1497          break;
1498      case ExtAtomicOpType::I32AtomicRmw8SubU:
1499      case ExtAtomicOpType::I32AtomicRmw16SubU:
1500      case ExtAtomicOpType::I32AtomicRmwSub:
1501      case ExtAtomicOpType::I64AtomicRmw8SubU:
1502      case ExtAtomicOpType::I64AtomicRmw16SubU:
1503      case ExtAtomicOpType::I64AtomicRmw32SubU:
1504      case ExtAtomicOpType::I64AtomicRmwSub:
1505          opcode = AtomicXchgSub;
1506          break;
1507      case ExtAtomicOpType::I32AtomicRmw8AndU:
1508      case ExtAtomicOpType::I32AtomicRmw16AndU:
1509      case ExtAtomicOpType::I32AtomicRmwAnd:
1510      case ExtAtomicOpType::I64AtomicRmw8AndU:
1511      case ExtAtomicOpType::I64AtomicRmw16AndU:
1512      case ExtAtomicOpType::I64AtomicRmw32AndU:
1513      case ExtAtomicOpType::I64AtomicRmwAnd:
1514          opcode = AtomicXchgAnd;
1515          break;
1516      case ExtAtomicOpType::I32AtomicRmw8OrU:
1517      case ExtAtomicOpType::I32AtomicRmw16OrU:
1518      case ExtAtomicOpType::I32AtomicRmwOr:
1519      case ExtAtomicOpType::I64AtomicRmw8OrU:
1520      case ExtAtomicOpType::I64AtomicRmw16OrU:
1521      case ExtAtomicOpType::I64AtomicRmw32OrU:
1522      case ExtAtomicOpType::I64AtomicRmwOr:
1523          opcode = AtomicXchgOr;
1524          break;
1525      case ExtAtomicOpType::I32AtomicRmw8XorU:
1526      case ExtAtomicOpType::I32AtomicRmw16XorU:
1527      case ExtAtomicOpType::I32AtomicRmwXor:
1528      case ExtAtomicOpType::I64AtomicRmw8XorU:
1529      case ExtAtomicOpType::I64AtomicRmw16XorU:
1530      case ExtAtomicOpType::I64AtomicRmw32XorU:
1531      case ExtAtomicOpType::I64AtomicRmwXor:
1532          opcode = AtomicXchgXor;
1533          break;
1534      case ExtAtomicOpType::I32AtomicRmw8XchgU:
1535      case ExtAtomicOpType::I32AtomicRmw16XchgU:
1536      case ExtAtomicOpType::I32AtomicRmwXchg:
1537      case ExtAtomicOpType::I64AtomicRmw8XchgU:
1538      case ExtAtomicOpType::I64AtomicRmw16XchgU:
1539      case ExtAtomicOpType::I64AtomicRmw32XchgU:
1540      case ExtAtomicOpType::I64AtomicRmwXchg:
1541          opcode = AtomicXchg;
1542          break;
1543      default:
1544          RELEASE_ASSERT_NOT_REACHED();
1545          break;
1546      }
1547  
1548      if (valueType == Type::I64 && accessWidth(op) != B3::Width64)
1549          value = m_currentBlock->appendNew<B3::Value>(m_proc, B3::Trunc, Origin(), value);
1550  
1551      return sanitizeAtomicResult(op, valueType, m_currentBlock->appendNew<AtomicValue>(m_proc, memoryKind(opcode), origin(), accessWidth(op), value, pointer));
1552  }
1553  
1554  auto B3IRGenerator::atomicBinaryRMW(ExtAtomicOpType op, Type valueType, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint32_t offset) -> PartialResult
1555  {
1556      ASSERT(pointer->type() == Int32);
1557  
1558      if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfAtomicOpMemoryAccess(op)))) {
1559          // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1560          // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1561          B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
1562          throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1563              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
1564          });
1565  
1566          switch (valueType) {
1567          case Type::I32:
1568              result = constant(Int32, 0);
1569              break;
1570          case Type::I64:
1571              result = constant(Int64, 0);
1572              break;
1573          default:
1574              RELEASE_ASSERT_NOT_REACHED();
1575              break;
1576          }
1577      } else
1578          result = emitAtomicBinaryRMWOp(op, valueType, emitCheckAndPreparePointer(pointer, offset, sizeOfAtomicOpMemoryAccess(op)), value, offset);
1579  
1580      return { };
1581  }
1582  
1583  Value* B3IRGenerator::emitAtomicCompareExchange(ExtAtomicOpType op, Type valueType, ExpressionType pointer, ExpressionType expected, ExpressionType value, uint32_t uoffset)
1584  {
1585      pointer = fixupPointerPlusOffsetForAtomicOps(op, pointer, uoffset);
1586  
1587      B3::Width accessWidth = Wasm::accessWidth(op);
1588  
1589      if (widthForType(toB3Type(valueType)) == accessWidth)
1590          return sanitizeAtomicResult(op, valueType, m_currentBlock->appendNew<AtomicValue>(m_proc, memoryKind(AtomicStrongCAS), origin(), accessWidth, expected, value, pointer));
1591  
1592      Value* maximum = nullptr;
1593      switch (valueType) {
1594      case Type::I64: {
1595          switch (accessWidth) {
1596          case B3::Width8:
1597              maximum = constant(Int64, UINT8_MAX);
1598              break;
1599          case B3::Width16:
1600              maximum = constant(Int64, UINT16_MAX);
1601              break;
1602          case B3::Width32:
1603              maximum = constant(Int64, UINT32_MAX);
1604              break;
1605          case B3::Width64:
1606              RELEASE_ASSERT_NOT_REACHED();
1607          }
1608          break;
1609      }
1610      case Type::I32:
1611          switch (accessWidth) {
1612          case B3::Width8:
1613              maximum = constant(Int32, UINT8_MAX);
1614              break;
1615          case B3::Width16:
1616              maximum = constant(Int32, UINT16_MAX);
1617              break;
1618          case B3::Width32:
1619          case B3::Width64:
1620              RELEASE_ASSERT_NOT_REACHED();
1621          }
1622          break;
1623      default:
1624          RELEASE_ASSERT_NOT_REACHED();
1625      }
1626  
1627      BasicBlock* failureCase = m_proc.addBlock();
1628      BasicBlock* successCase = m_proc.addBlock();
1629      BasicBlock* continuation = m_proc.addBlock();
1630  
1631      auto condition = m_currentBlock->appendNew<Value>(m_proc, Above, origin(), expected, maximum);
1632      m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), condition, FrequentedBlock(failureCase, FrequencyClass::Rare), FrequentedBlock(successCase, FrequencyClass::Normal));
1633      failureCase->addPredecessor(m_currentBlock);
1634      successCase->addPredecessor(m_currentBlock);
1635  
1636      m_currentBlock = successCase;
1637      B3::UpsilonValue* successValue = nullptr;
1638      {
1639          auto truncatedExpected = expected;
1640          auto truncatedValue = value;
1641          if (valueType == Type::I64) {
1642              truncatedExpected = m_currentBlock->appendNew<B3::Value>(m_proc, B3::Trunc, Origin(), expected);
1643              truncatedValue = m_currentBlock->appendNew<B3::Value>(m_proc, B3::Trunc, Origin(), value);
1644          }
1645  
1646          auto result = m_currentBlock->appendNew<AtomicValue>(m_proc, memoryKind(AtomicStrongCAS), origin(), accessWidth, truncatedExpected, truncatedValue, pointer);
1647          successValue = m_currentBlock->appendNew<B3::UpsilonValue>(m_proc, origin(), result);
1648          m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1649          continuation->addPredecessor(m_currentBlock);
1650      }
1651  
1652      m_currentBlock = failureCase;
1653      B3::UpsilonValue* failureValue = nullptr;
1654      {
1655          Value* addingValue = nullptr;
1656          switch (accessWidth) {
1657          case B3::Width8:
1658          case B3::Width16:
1659          case B3::Width32:
1660              addingValue = constant(Int32, 0);
1661              break;
1662          case B3::Width64:
1663              addingValue = constant(Int64, 0);
1664              break;
1665          }
1666          auto result = m_currentBlock->appendNew<AtomicValue>(m_proc, memoryKind(AtomicXchgAdd), origin(), accessWidth, addingValue, pointer);
1667          failureValue = m_currentBlock->appendNew<B3::UpsilonValue>(m_proc, origin(), result);
1668          m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1669          continuation->addPredecessor(m_currentBlock);
1670      }
1671  
1672      m_currentBlock = continuation;
1673      Value* phi = continuation->appendNew<Value>(m_proc, Phi, accessWidth == B3::Width64 ? Int64 : Int32, origin());
1674      successValue->setPhi(phi);
1675      failureValue->setPhi(phi);
1676      return sanitizeAtomicResult(op, valueType, phi);
1677  }
1678  
1679  auto B3IRGenerator::atomicCompareExchange(ExtAtomicOpType op, Type valueType, ExpressionType pointer, ExpressionType expected, ExpressionType value, ExpressionType& result, uint32_t offset) -> PartialResult
1680  {
1681      ASSERT(pointer->type() == Int32);
1682  
1683      if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfAtomicOpMemoryAccess(op)))) {
1684          // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1685          // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1686          B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
1687          throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1688              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
1689          });
1690  
1691          switch (valueType) {
1692          case Type::I32:
1693              result = constant(Int32, 0);
1694              break;
1695          case Type::I64:
1696              result = constant(Int64, 0);
1697              break;
1698          default:
1699              RELEASE_ASSERT_NOT_REACHED();
1700              break;
1701          }
1702      } else
1703          result = emitAtomicCompareExchange(op, valueType, emitCheckAndPreparePointer(pointer, offset, sizeOfAtomicOpMemoryAccess(op)), expected, value, offset);
1704  
1705      return { };
1706  }
1707  
1708  auto B3IRGenerator::atomicWait(ExtAtomicOpType op, ExpressionType pointer, ExpressionType value, ExpressionType timeout, ExpressionType& result, uint32_t offset) -> PartialResult
1709  {
1710      if (op == ExtAtomicOpType::MemoryAtomicWait32) {
1711          result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
1712              m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationMemoryAtomicWait32)),
1713              instanceValue(), pointer, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), offset), value, timeout);
1714      } else {
1715          result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
1716              m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationMemoryAtomicWait64)),
1717              instanceValue(), pointer, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), offset), value, timeout);
1718      }
1719  
1720      {
1721          CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1722              m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), result, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0)));
1723  
1724          check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1725              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
1726          });
1727      }
1728  
1729      return { };
1730  }
1731  
1732  auto B3IRGenerator::atomicNotify(ExtAtomicOpType, ExpressionType pointer, ExpressionType count, ExpressionType& result, uint32_t offset) -> PartialResult
1733  {
1734      result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
1735          m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationMemoryAtomicNotify)),
1736          instanceValue(), pointer, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), offset), count);
1737  
1738      {
1739          CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1740              m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), result, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0)));
1741  
1742          check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1743              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
1744          });
1745      }
1746  
1747      return { };
1748  }
1749  
1750  auto B3IRGenerator::atomicFence(ExtAtomicOpType, uint8_t) -> PartialResult
1751  {
1752      m_currentBlock->appendNew<FenceValue>(m_proc, origin());
1753      return { };
1754  }
1755  
1756  auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
1757  {
1758      result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, origin(), condition, nonZero, zero);
1759      return { };
1760  }
1761  
1762  B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
1763  {
1764  
1765      return constant(toB3Type(type), value);
1766  }
1767  
1768  void B3IRGenerator::emitEntryTierUpCheck()
1769  {
1770      if (!m_tierUp)
1771          return;
1772  
1773      ASSERT(m_tierUp);
1774      Value* countDownLocation = constant(pointerType(), reinterpret_cast<uint64_t>(&m_tierUp->m_counter), Origin());
1775  
1776      PatchpointValue* patch = m_currentBlock->appendNew<PatchpointValue>(m_proc, B3::Void, Origin());
1777      Effects effects = Effects::none();
1778      // FIXME: we should have a more precise heap range for the tier up count.
1779      effects.reads = B3::HeapRange::top();
1780      effects.writes = B3::HeapRange::top();
1781      patch->effects = effects;
1782      patch->clobber(RegisterSet::macroScratchRegisters());
1783  
1784      patch->append(countDownLocation, ValueRep::SomeRegister);
1785      patch->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1786          AllowMacroScratchRegisterUsage allowScratch(jit);
1787          CCallHelpers::Jump tierUp = jit.branchAdd32(CCallHelpers::PositiveOrZero, CCallHelpers::TrustedImm32(TierUpCount::functionEntryIncrement()), CCallHelpers::Address(params[0].gpr()));
1788          CCallHelpers::Label tierUpResume = jit.label();
1789  
1790          params.addLatePath([=] (CCallHelpers& jit) {
1791              tierUp.link(&jit);
1792  
1793              const unsigned extraPaddingBytes = 0;
1794              RegisterSet registersToSpill = { };
1795              registersToSpill.add(GPRInfo::argumentGPR1);
1796              unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(jit, registersToSpill, extraPaddingBytes);
1797  
1798              jit.move(MacroAssembler::TrustedImm32(m_functionIndex), GPRInfo::argumentGPR1);
1799              MacroAssembler::Call call = jit.nearCall();
1800  
1801              ScratchRegisterAllocator::restoreRegistersFromStackForCall(jit, registersToSpill, RegisterSet(), numberOfStackBytesUsedForRegisterPreservation, extraPaddingBytes);
1802              jit.jump(tierUpResume);
1803  
1804              jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
1805                  MacroAssembler::repatchNearCall(linkBuffer.locationOfNearCall<NoPtrTag>(call), CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(triggerOMGEntryTierUpThunkGenerator).code()));
1806              });
1807          });
1808      });
1809  }
1810  
1811  void B3IRGenerator::emitLoopTierUpCheck(uint32_t loopIndex, const Stack& enclosingStack)
1812  {
1813      uint32_t outerLoopIndex = this->outerLoopIndex();
1814      m_outerLoops.append(loopIndex);
1815  
1816      if (!m_tierUp)
1817          return;
1818  
1819      Origin origin = this->origin();
1820      ASSERT(m_tierUp->osrEntryTriggers().size() == loopIndex);
1821      m_tierUp->osrEntryTriggers().append(TierUpCount::TriggerReason::DontTrigger);
1822      m_tierUp->outerLoops().append(outerLoopIndex);
1823  
1824      Value* countDownLocation = constant(pointerType(), reinterpret_cast<uint64_t>(&m_tierUp->m_counter), origin);
1825  
1826      Vector<ExpressionType> stackmap;
1827      for (auto& local : m_locals) {
1828          Value* result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin, local);
1829          stackmap.append(result);
1830      }
1831      for (unsigned controlIndex = 0; controlIndex < m_parser->controlStack().size(); ++controlIndex) {
1832          auto& expressionStack = m_parser->controlStack()[controlIndex].enclosedExpressionStack;
1833          for (TypedExpression value : expressionStack)
1834              stackmap.append(value);
1835      }
1836      for (TypedExpression value : enclosingStack)
1837          stackmap.append(value);
1838  
1839      PatchpointValue* patch = m_currentBlock->appendNew<PatchpointValue>(m_proc, B3::Void, origin);
1840      Effects effects = Effects::none();
1841      // FIXME: we should have a more precise heap range for the tier up count.
1842      effects.reads = B3::HeapRange::top();
1843      effects.writes = B3::HeapRange::top();
1844      effects.exitsSideways = true;
1845      patch->effects = effects;
1846  
1847      patch->clobber(RegisterSet::macroScratchRegisters());
1848      RegisterSet clobberLate;
1849      clobberLate.add(GPRInfo::argumentGPR0);
1850      patch->clobberLate(clobberLate);
1851  
1852      patch->append(countDownLocation, ValueRep::SomeRegister);
1853      patch->appendVectorWithRep(stackmap, ValueRep::ColdAny);
1854  
1855      TierUpCount::TriggerReason* forceEntryTrigger = &(m_tierUp->osrEntryTriggers().last());
1856      static_assert(!static_cast<uint8_t>(TierUpCount::TriggerReason::DontTrigger), "the JIT code assumes non-zero means 'enter'");
1857      static_assert(sizeof(TierUpCount::TriggerReason) == 1, "branchTest8 assumes this size");
1858      patch->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1859          AllowMacroScratchRegisterUsage allowScratch(jit);
1860          CCallHelpers::Jump forceOSREntry = jit.branchTest8(CCallHelpers::NonZero, CCallHelpers::AbsoluteAddress(forceEntryTrigger));
1861          CCallHelpers::Jump tierUp = jit.branchAdd32(CCallHelpers::PositiveOrZero, CCallHelpers::TrustedImm32(TierUpCount::loopIncrement()), CCallHelpers::Address(params[0].gpr()));
1862          MacroAssembler::Label tierUpResume = jit.label();
1863  
1864          OSREntryData& osrEntryData = m_tierUp->addOSREntryData(m_functionIndex, loopIndex);
1865          // First argument is the countdown location.
1866          for (unsigned i = 1; i < params.value()->numChildren(); ++i)
1867              osrEntryData.values().constructAndAppend(params[i], params.value()->child(i)->type());
1868          OSREntryData* osrEntryDataPtr = &osrEntryData;
1869  
1870          params.addLatePath([=] (CCallHelpers& jit) {
1871              AllowMacroScratchRegisterUsage allowScratch(jit);
1872              forceOSREntry.link(&jit);
1873              tierUp.link(&jit);
1874  
1875              jit.probe(tagCFunction<JITProbePtrTag>(operationWasmTriggerOSREntryNow), osrEntryDataPtr);
1876              jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::argumentGPR0).linkTo(tierUpResume, &jit);
1877              jit.farJump(GPRInfo::argumentGPR1, WasmEntryPtrTag);
1878          });
1879      });
1880  }
1881  
1882  auto B3IRGenerator::addLoop(BlockSignature signature, Stack& enclosingStack, ControlType& block, Stack& newStack, uint32_t loopIndex) -> PartialResult
1883  {
1884      BasicBlock* body = m_proc.addBlock();
1885      BasicBlock* continuation = m_proc.addBlock();
1886  
1887      block = ControlData(m_proc, origin(), signature, BlockType::Loop, continuation, body);
1888  
1889      ExpressionList args;
1890      {
1891          unsigned offset = enclosingStack.size() - signature->argumentCount();
1892          for (unsigned i = 0; i < signature->argumentCount(); ++i) {
1893              TypedExpression value = enclosingStack.at(offset + i);
1894              auto* upsilon = m_currentBlock->appendNew<UpsilonValue>(m_proc, origin(), value);
1895              Value* phi = block.phis[i];
1896              body->append(phi);
1897              upsilon->setPhi(phi);
1898              newStack.constructAndAppend(value.type(), phi);
1899          }
1900          enclosingStack.shrink(offset);
1901      }
1902  
1903      m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body);
1904      if (loopIndex == m_loopIndexForOSREntry) {
1905          dataLogLnIf(WasmB3IRGeneratorInternal::verbose, "Setting up for OSR entry");
1906  
1907          m_currentBlock = m_rootBlock;
1908          Value* pointer = m_rootBlock->appendNew<ArgumentRegValue>(m_proc, Origin(), GPRInfo::argumentGPR0);
1909  
1910          unsigned indexInBuffer = 0;
1911          auto loadFromScratchBuffer = [&] (B3::Type type) {
1912              size_t offset = sizeof(uint64_t) * indexInBuffer++;
1913              RELEASE_ASSERT(type.isNumeric());
1914              return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, type, origin(), pointer, offset);
1915          };
1916  
1917          for (auto& local : m_locals)
1918              m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, loadFromScratchBuffer(local->type()));
1919  
1920          auto connectControlEntry = [&](const ControlData& data, Stack& expressionStack) {
1921              // For each stack entry enclosed by this loop we need to replace the value with a phi so we can fill it on OSR entry.
1922              BasicBlock* sourceBlock = nullptr;
1923              unsigned blockIndex = 0;
1924              B3::InsertionSet insertionSet(m_proc);
1925              for (unsigned i = 0; i < expressionStack.size(); i++) {
1926                  TypedExpression value = expressionStack[i];
1927                  if (value->isConstant()) {
1928                      ++indexInBuffer;
1929                      continue;
1930                  }
1931  
1932                  if (value->owner != sourceBlock) {
1933                      if (sourceBlock)
1934                          insertionSet.execute(sourceBlock);
1935                      ASSERT(insertionSet.isEmpty());
1936                      dataLogLnIf(WasmB3IRGeneratorInternal::verbose && sourceBlock, "Executed insertion set into: ", *sourceBlock);
1937                      blockIndex = 0;
1938                      sourceBlock = value->owner;
1939                  }
1940  
1941                  while (sourceBlock->at(blockIndex++) != value)
1942                      ASSERT(blockIndex < sourceBlock->size());
1943                  ASSERT(sourceBlock->at(blockIndex - 1) == value);
1944  
1945                  auto* phi = data.continuation->appendNew<Value>(m_proc, Phi,  value->type(), value->origin());
1946                  expressionStack[i] = TypedExpression { value.type(), phi };
1947                  m_currentBlock->appendNew<UpsilonValue>(m_proc, value->origin(), loadFromScratchBuffer(value->type()), phi);
1948  
1949                  auto* sourceUpsilon = m_proc.add<UpsilonValue>(value->origin(), value, phi);
1950                  insertionSet.insertValue(blockIndex, sourceUpsilon);
1951              }
1952              if (sourceBlock)
1953                  insertionSet.execute(sourceBlock);
1954          };
1955  
1956          for (unsigned controlIndex = 0; controlIndex < m_parser->controlStack().size(); ++controlIndex) {
1957              auto& data = m_parser->controlStack()[controlIndex].controlData;
1958              auto& expressionStack = m_parser->controlStack()[controlIndex].enclosedExpressionStack;
1959              connectControlEntry(data, expressionStack);
1960          }
1961          connectControlEntry(block, enclosingStack);
1962  
1963          m_osrEntryScratchBufferSize = indexInBuffer;
1964          m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body);
1965          body->addPredecessor(m_currentBlock);
1966      }
1967  
1968      m_currentBlock = body;
1969      emitLoopTierUpCheck(loopIndex, enclosingStack);
1970      return { };
1971  }
1972  
1973  B3IRGenerator::ControlData B3IRGenerator::addTopLevel(BlockSignature signature)
1974  {
1975      return ControlData(m_proc, Origin(), signature, BlockType::TopLevel, m_proc.addBlock());
1976  }
1977  
1978  auto B3IRGenerator::addBlock(BlockSignature signature, Stack& enclosingStack, ControlType& newBlock, Stack& newStack) -> PartialResult
1979  {
1980      BasicBlock* continuation = m_proc.addBlock();
1981  
1982      splitStack(signature, enclosingStack, newStack);
1983      newBlock = ControlData(m_proc, origin(), signature, BlockType::Block, continuation);
1984      return { };
1985  }
1986  
1987  auto B3IRGenerator::addIf(ExpressionType condition, BlockSignature signature, Stack& enclosingStack, ControlType& result, Stack& newStack) -> PartialResult
1988  {
1989      // FIXME: This needs to do some kind of stack passing.
1990  
1991      BasicBlock* taken = m_proc.addBlock();
1992      BasicBlock* notTaken = m_proc.addBlock();
1993      BasicBlock* continuation = m_proc.addBlock();
1994  
1995      m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
1996      m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
1997      taken->addPredecessor(m_currentBlock);
1998      notTaken->addPredecessor(m_currentBlock);
1999  
2000      m_currentBlock = taken;
2001      splitStack(signature, enclosingStack, newStack);
2002      result = ControlData(m_proc, origin(), signature, BlockType::If, continuation, notTaken);
2003      return { };
2004  }
2005  
2006  auto B3IRGenerator::addElse(ControlData& data, const Stack& currentStack) -> PartialResult
2007  {
2008      unifyValuesWithBlock(currentStack, data.phis);
2009      m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
2010      return addElseToUnreachable(data);
2011  }
2012  
2013  auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
2014  {
2015      ASSERT(data.blockType() == BlockType::If);
2016      m_currentBlock = data.special;
2017      data.convertIfToBlock();
2018      return { };
2019  }
2020  
2021  auto B3IRGenerator::addReturn(const ControlData&, const Stack& returnValues) -> PartialResult
2022  {
2023      CallInformation wasmCallInfo = wasmCallingConvention().callInformationFor(m_parser->signature(), CallRole::Callee);
2024  
2025      PatchpointValue* patch = m_proc.add<PatchpointValue>(B3::Void, origin());
2026      patch->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2027          auto calleeSaves = params.code().calleeSaveRegisterAtOffsetList();
2028  
2029          for (RegisterAtOffset calleeSave : calleeSaves)
2030              jit.load64ToReg(CCallHelpers::Address(GPRInfo::callFrameRegister, calleeSave.offset()), calleeSave.reg());
2031  
2032          jit.emitFunctionEpilogue();
2033          jit.ret();
2034      });
2035      patch->effects.terminal = true;
2036  
2037      RELEASE_ASSERT(returnValues.size() >= wasmCallInfo.results.size());
2038      unsigned offset = returnValues.size() - wasmCallInfo.results.size();
2039      for (unsigned i = 0; i < wasmCallInfo.results.size(); ++i) {
2040          B3::ValueRep rep = wasmCallInfo.results[i];
2041          if (rep.isStack()) {
2042              B3::Value* address = m_currentBlock->appendNew<B3::Value>(m_proc, B3::Add, Origin(), framePointer(), constant(pointerType(), rep.offsetFromFP()));
2043              m_currentBlock->appendNew<B3::MemoryValue>(m_proc, B3::Store, Origin(), returnValues[offset + i], address);
2044          } else {
2045              ASSERT(rep.isReg());
2046              patch->append(returnValues[offset + i], rep);
2047          }
2048      }
2049  
2050      m_currentBlock->append(patch);
2051      return { };
2052  }
2053  
2054  auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const Stack& returnValues) -> PartialResult
2055  {
2056      unifyValuesWithBlock(returnValues, data.phis);
2057  
2058      BasicBlock* target = data.targetBlockForBranch();
2059      if (condition) {
2060          BasicBlock* continuation = m_proc.addBlock();
2061          m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
2062          m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
2063          target->addPredecessor(m_currentBlock);
2064          continuation->addPredecessor(m_currentBlock);
2065          m_currentBlock = continuation;
2066      } else {
2067          m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), FrequentedBlock(target));
2068          target->addPredecessor(m_currentBlock);
2069      }
2070  
2071      return { };
2072  }
2073  
2074  auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const Stack& expressionStack) -> PartialResult
2075  {
2076      for (size_t i = 0; i < targets.size(); ++i)
2077          unifyValuesWithBlock(expressionStack, targets[i]->phis);
2078      unifyValuesWithBlock(expressionStack, defaultTarget.phis);
2079  
2080      SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, origin(), condition);
2081      switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
2082      for (size_t i = 0; i < targets.size(); ++i)
2083          switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
2084  
2085      return { };
2086  }
2087  
2088  auto B3IRGenerator::endBlock(ControlEntry& entry, Stack& expressionStack) -> PartialResult
2089  {
2090      ControlData& data = entry.controlData;
2091  
2092      ASSERT(expressionStack.size() == data.signature()->returnCount());
2093      if (data.blockType() != BlockType::Loop)
2094          unifyValuesWithBlock(expressionStack, data.phis);
2095  
2096      m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
2097      data.continuation->addPredecessor(m_currentBlock);
2098  
2099      return addEndToUnreachable(entry, expressionStack);
2100  }
2101  
2102  auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry, const Stack& expressionStack) -> PartialResult
2103  {
2104      ControlData& data = entry.controlData;
2105      m_currentBlock = data.continuation;
2106  
2107      if (data.blockType() == BlockType::If) {
2108          data.special->appendNewControlValue(m_proc, Jump, origin(), m_currentBlock);
2109          m_currentBlock->addPredecessor(data.special);
2110      }
2111  
2112      if (data.blockType() != BlockType::Loop) {
2113          for (unsigned i = 0; i < data.signature()->returnCount(); ++i) {
2114              Value* result = data.phis[i];
2115              m_currentBlock->append(result);
2116              entry.enclosedExpressionStack.constructAndAppend(data.signature()->returnType(i), result);
2117          }
2118      } else {
2119          m_outerLoops.removeLast();
2120          for (unsigned i = 0; i < data.signature()->returnCount(); ++i) {
2121              if (i < expressionStack.size())
2122                  entry.enclosedExpressionStack.append(expressionStack[i]);
2123              else {
2124                  Type returnType = data.signature()->returnType(i);
2125                  entry.enclosedExpressionStack.constructAndAppend(returnType, constant(toB3Type(returnType), 0xbbadbeef));
2126              }
2127          }
2128      }
2129  
2130      // TopLevel does not have any code after this so we need to make sure we emit a return here.
2131      if (data.blockType() == BlockType::TopLevel)
2132          return addReturn(entry.controlData, entry.enclosedExpressionStack);
2133  
2134      return { };
2135  }
2136  
2137  
2138  B3::Value* B3IRGenerator::createCallPatchpoint(BasicBlock* block, Origin origin, const Signature& signature, Vector<ExpressionType>& args, const ScopedLambda<void(PatchpointValue*)>& patchpointFunctor)
2139  {
2140      Vector<B3::ConstrainedValue> constrainedArguments;
2141      CallInformation wasmCallInfo = wasmCallingConvention().callInformationFor(signature);
2142      for (unsigned i = 0; i < args.size(); ++i)
2143          constrainedArguments.append(B3::ConstrainedValue(args[i], wasmCallInfo.params[i]));
2144  
2145      m_proc.requestCallArgAreaSizeInBytes(WTF::roundUpToMultipleOf(stackAlignmentBytes(), wasmCallInfo.headerAndArgumentStackSizeInBytes));
2146  
2147      B3::Type returnType = toB3ResultType(&signature);
2148      B3::PatchpointValue* patchpoint = block->appendNew<B3::PatchpointValue>(m_proc, returnType, origin);
2149      patchpoint->clobberEarly(RegisterSet::macroScratchRegisters());
2150      patchpoint->clobberLate(RegisterSet::volatileRegistersForJSCall());
2151      patchpointFunctor(patchpoint);
2152      patchpoint->appendVector(constrainedArguments);
2153  
2154      if (returnType != B3::Void)
2155          patchpoint->resultConstraints = WTFMove(wasmCallInfo.results);
2156      return patchpoint;
2157  }
2158  
2159  auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ResultList& results) -> PartialResult
2160  {
2161      ASSERT(signature.argumentCount() == args.size());
2162  
2163      m_makesCalls = true;
2164      B3::Type returnType = toB3ResultType(&signature);
2165  
2166      auto fillResults = [&] (Value* callResult) {
2167          ASSERT(returnType == callResult->type());
2168  
2169          switch (returnType.kind()) {
2170          case B3::Void: {
2171              break;
2172          }
2173          case B3::Tuple: {
2174              const Vector<B3::Type>& tuple = m_proc.tupleForType(returnType);
2175              ASSERT(signature.returnCount() == tuple.size());
2176              for (unsigned i = 0; i < signature.returnCount(); ++i)
2177                  results.append(m_currentBlock->appendNew<ExtractValue>(m_proc, origin(), tuple[i], callResult, i));
2178              break;
2179          }
2180          default: {
2181              results.append(callResult);
2182              break;
2183          }
2184          }
2185      };
2186  
2187      Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
2188  
2189      if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
2190          m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
2191  
2192          // FIXME: imports can be linked here, instead of generating a patchpoint, because all import stubs are generated before B3 compilation starts. https://bugs.webkit.org/show_bug.cgi?id=166462
2193          Value* targetInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfTargetInstance(functionIndex)));
2194          // The target instance is 0 unless the call is wasm->wasm.
2195          Value* isWasmCall = m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), targetInstance, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), 0));
2196  
2197          BasicBlock* isWasmBlock = m_proc.addBlock();
2198          BasicBlock* isEmbedderBlock = m_proc.addBlock();
2199          BasicBlock* continuation = m_proc.addBlock();
2200          m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isEmbedderBlock));
2201  
2202          Value* wasmCallResult = createCallPatchpoint(isWasmBlock, origin(), signature, args,
2203              scopedLambdaRef<void(PatchpointValue*)>([=] (PatchpointValue* patchpoint) -> void {
2204                  patchpoint->effects.writesPinned = true;
2205                  patchpoint->effects.readsPinned = true;
2206                  // We need to clobber all potential pinned registers since we might be leaving the instance.
2207                  // We pessimistically assume we could be calling to something that is bounds checking.
2208                  // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
2209                  patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
2210                  patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2211                      AllowMacroScratchRegisterUsage allowScratch(jit);
2212                      CCallHelpers::Call call = jit.threadSafePatchableNearCall();
2213                      jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
2214                          unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
2215                      });
2216                  });
2217              }));
2218          UpsilonValue* wasmCallResultUpsilon = returnType == B3::Void ? nullptr : isWasmBlock->appendNew<UpsilonValue>(m_proc, origin(), wasmCallResult);
2219          isWasmBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
2220  
2221          // FIXME: Let's remove this indirection by creating a PIC friendly IC
2222          // for calls out to the embedder. This shouldn't be that hard to do. We could probably
2223          // implement the IC to be over Context*.
2224          // https://bugs.webkit.org/show_bug.cgi?id=170375
2225          Value* jumpDestination = isEmbedderBlock->appendNew<MemoryValue>(m_proc,
2226              Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfWasmToEmbedderStub(functionIndex)));
2227  
2228          Value* embedderCallResult = createCallPatchpoint(isEmbedderBlock, origin(), signature, args,
2229              scopedLambdaRef<void(PatchpointValue*)>([=] (PatchpointValue* patchpoint) -> void {
2230                  patchpoint->effects.writesPinned = true;
2231                  patchpoint->effects.readsPinned = true;
2232                  patchpoint->append(jumpDestination, ValueRep::SomeRegister);
2233                  // We need to clobber all potential pinned registers since we might be leaving the instance.
2234                  // We pessimistically assume we could be calling to something that is bounds checking.
2235                  // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
2236                  patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
2237                  patchpoint->setGenerator([returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2238                      AllowMacroScratchRegisterUsage allowScratch(jit);
2239                      jit.call(params[params.proc().resultCount(returnType)].gpr(), WasmEntryPtrTag);
2240                  });
2241              }));
2242          UpsilonValue* embedderCallResultUpsilon = returnType == B3::Void ? nullptr : isEmbedderBlock->appendNew<UpsilonValue>(m_proc, origin(), embedderCallResult);
2243          isEmbedderBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
2244  
2245          m_currentBlock = continuation;
2246  
2247          if (returnType != B3::Void) {
2248              Value* phi = continuation->appendNew<Value>(m_proc, Phi, returnType, origin());
2249              wasmCallResultUpsilon->setPhi(phi);
2250              embedderCallResultUpsilon->setPhi(phi);
2251              fillResults(phi);
2252          }
2253  
2254          // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
2255          restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, continuation);
2256      } else {
2257  
2258          Value* patch = createCallPatchpoint(m_currentBlock, origin(), signature, args,
2259              scopedLambdaRef<void(PatchpointValue*)>([=] (PatchpointValue* patchpoint) -> void {
2260                  patchpoint->effects.writesPinned = true;
2261                  patchpoint->effects.readsPinned = true;
2262  
2263                  // We need to clobber the size register since the LLInt always bounds checks
2264                  if (m_mode == MemoryMode::Signaling || m_info.memory.isShared())
2265                      patchpoint->clobberLate(RegisterSet { PinnedRegisterInfo::get().boundsCheckingSizeRegister });
2266                  patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2267                      AllowMacroScratchRegisterUsage allowScratch(jit);
2268                      CCallHelpers::Call call = jit.threadSafePatchableNearCall();
2269                      jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
2270                          unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
2271                      });
2272                  });
2273              }));
2274          fillResults(patch);
2275      }
2276  
2277      return { };
2278  }
2279  
2280  auto B3IRGenerator::addCallIndirect(unsigned tableIndex, const Signature& signature, Vector<ExpressionType>& args, ResultList& results) -> PartialResult
2281  {
2282      ExpressionType calleeIndex = args.takeLast();
2283      ASSERT(signature.argumentCount() == args.size());
2284  
2285      m_makesCalls = true;
2286      // Note: call indirect can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because
2287      // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects
2288      // can be to the embedder for our stack check calculation.
2289      m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
2290  
2291      ExpressionType callableFunctionBuffer;
2292      ExpressionType instancesBuffer;
2293      ExpressionType callableFunctionBufferLength;
2294      {
2295          ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
2296              instanceValue(), safeCast<int32_t>(Instance::offsetOfTablePtr(m_numImportFunctions, tableIndex)));
2297          callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
2298              table, safeCast<int32_t>(FuncRefTable::offsetOfFunctions()));
2299          instancesBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
2300              table, safeCast<int32_t>(FuncRefTable::offsetOfInstances()));
2301          callableFunctionBufferLength = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
2302              table, safeCast<int32_t>(Table::offsetOfLength()));
2303      }
2304  
2305      // Check the index we are looking for is valid.
2306      {
2307          CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
2308              m_currentBlock->appendNew<Value>(m_proc, AboveEqual, origin(), calleeIndex, callableFunctionBufferLength));
2309  
2310          check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2311              this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
2312          });
2313      }
2314  
2315      calleeIndex = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), calleeIndex);
2316  
2317      ExpressionType callableFunction;
2318      {
2319          // Compute the offset in the table index space we are looking for.
2320          ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
2321              calleeIndex, constant(pointerType(), sizeof(WasmToWasmImportableFunction)));
2322          callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), callableFunctionBuffer, offset);
2323  
2324          // Check that the WasmToWasmImportableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
2325          // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210
2326          static_assert(sizeof(WasmToWasmImportableFunction::signatureIndex) == sizeof(uint64_t), "Load codegen assumes i64");
2327          ExpressionType calleeSignatureIndex = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), callableFunction, safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfSignatureIndex()));
2328          {
2329              CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
2330                  m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
2331                      calleeSignatureIndex,
2332                      m_currentBlock->appendNew<Const64Value>(m_proc, origin(), Signature::invalidIndex)));
2333  
2334              check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2335                  this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
2336              });
2337          }
2338  
2339          // Check the signature matches the value we expect.
2340          {
2341              ExpressionType expectedSignatureIndex = m_currentBlock->appendNew<Const64Value>(m_proc, origin(), SignatureInformation::get(signature));
2342              CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
2343                  m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), calleeSignatureIndex, expectedSignatureIndex));
2344  
2345              check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2346                  this->emitExceptionCheck(jit, ExceptionType::BadSignature);
2347              });
2348          }
2349      }
2350  
2351      // Do a context switch if needed.
2352      {
2353          Value* offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
2354              calleeIndex, constant(pointerType(), sizeof(Instance*)));
2355          Value* newContextInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
2356              m_currentBlock->appendNew<Value>(m_proc, Add, origin(), instancesBuffer, offset));
2357  
2358          BasicBlock* continuation = m_proc.addBlock();
2359          BasicBlock* doContextSwitch = m_proc.addBlock();
2360  
2361          Value* isSameContextInstance = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
2362              newContextInstance, instanceValue());
2363          m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
2364              isSameContextInstance, FrequentedBlock(continuation), FrequentedBlock(doContextSwitch));
2365  
2366          PatchpointValue* patchpoint = doContextSwitch->appendNew<PatchpointValue>(m_proc, B3::Void, origin());
2367          patchpoint->effects.writesPinned = true;
2368          // We pessimistically assume we're calling something with BoundsChecking memory.
2369          // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
2370          patchpoint->clobber(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
2371          patchpoint->clobber(RegisterSet::macroScratchRegisters());
2372          patchpoint->append(newContextInstance, ValueRep::SomeRegister);
2373          patchpoint->append(instanceValue(), ValueRep::SomeRegister);
2374          patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
2375  
2376          patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2377              AllowMacroScratchRegisterUsage allowScratch(jit);
2378              GPRReg newContextInstance = params[0].gpr();
2379              GPRReg oldContextInstance = params[1].gpr();
2380              const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
2381              GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
2382              ASSERT(newContextInstance != baseMemory);
2383              jit.loadPtr(CCallHelpers::Address(oldContextInstance, Instance::offsetOfCachedStackLimit()), baseMemory);
2384              jit.storePtr(baseMemory, CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedStackLimit()));
2385              jit.storeWasmContextInstance(newContextInstance);
2386              ASSERT(pinnedRegs.boundsCheckingSizeRegister != baseMemory);
2387              // FIXME: We should support more than one memory size register
2388              //   see: https://bugs.webkit.org/show_bug.cgi?id=162952
2389              ASSERT(pinnedRegs.boundsCheckingSizeRegister != newContextInstance);
2390              GPRReg scratchOrBoundsCheckingSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs.boundsCheckingSizeRegister;
2391  
2392              jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedBoundsCheckingSize()), pinnedRegs.boundsCheckingSizeRegister); // Memory size.
2393              jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemory()), baseMemory); // Memory::void*.
2394  
2395              jit.cageConditionally(Gigacage::Primitive, baseMemory, pinnedRegs.boundsCheckingSizeRegister, scratchOrBoundsCheckingSize);
2396          });
2397          doContextSwitch->appendNewControlValue(m_proc, Jump, origin(), continuation);
2398  
2399          m_currentBlock = continuation;
2400      }
2401  
2402      ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
2403          m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), callableFunction,
2404              safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation())));
2405  
2406      B3::Type returnType = toB3ResultType(&signature);
2407      ExpressionType callResult = createCallPatchpoint(m_currentBlock, origin(), signature, args,
2408          scopedLambdaRef<void(PatchpointValue*)>([=] (PatchpointValue* patchpoint) -> void {
2409              patchpoint->effects.writesPinned = true;
2410              patchpoint->effects.readsPinned = true;
2411              // We need to clobber all potential pinned registers since we might be leaving the instance.
2412              // We pessimistically assume we're always calling something that is bounds checking so
2413              // because the wasm->wasm thunk unconditionally overrides the size registers.
2414              // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can
2415              // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181
2416              patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
2417  
2418              patchpoint->append(calleeCode, ValueRep::SomeRegister);
2419              patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2420                  AllowMacroScratchRegisterUsage allowScratch(jit);
2421                  jit.call(params[params.proc().resultCount(returnType)].gpr(), WasmEntryPtrTag);
2422              });
2423          }));
2424  
2425      switch (returnType.kind()) {
2426      case B3::Void: {
2427          break;
2428      }
2429      case B3::Tuple: {
2430          const Vector<B3::Type>& tuple = m_proc.tupleForType(returnType);
2431          for (unsigned i = 0; i < signature.returnCount(); ++i)
2432              results.append(m_currentBlock->appendNew<ExtractValue>(m_proc, origin(), tuple[i], callResult, i));
2433          break;
2434      }
2435      default: {
2436          results.append(callResult);
2437          break;
2438      }
2439      }
2440  
2441      // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
2442      restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, m_currentBlock);
2443  
2444      return { };
2445  }
2446  
2447  void B3IRGenerator::unify(const ExpressionType phi, const ExpressionType source)
2448  {
2449      m_currentBlock->appendNew<UpsilonValue>(m_proc, origin(), source, phi);
2450  }
2451  
2452  void B3IRGenerator::unifyValuesWithBlock(const Stack& resultStack, const ResultList& result)
2453  {
2454      ASSERT(result.size() <= resultStack.size());
2455  
2456      for (size_t i = 0; i < result.size(); ++i)
2457          unify(result[result.size() - 1 - i], resultStack.at(resultStack.size() - 1 - i));
2458  }
2459  
2460  static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::Stack& expressionStack)
2461  {
2462      dataLog(comma, "ExpressionStack:");
2463      for (const auto& expression : expressionStack)
2464          dataLog(comma, *expression);
2465  }
2466  
2467  void B3IRGenerator::dump(const ControlStack& controlStack, const Stack* expressionStack)
2468  {
2469      dataLogLn("Constants:");
2470      for (const auto& constant : m_constantPool)
2471          dataLogLn(deepDump(m_proc, constant.value));
2472  
2473      dataLogLn("Processing Graph:");
2474      dataLog(m_proc);
2475      dataLogLn("With current block:", *m_currentBlock);
2476      dataLogLn("Control stack:");
2477      ASSERT(controlStack.size());
2478      for (size_t i = controlStack.size(); i--;) {
2479          dataLog("  ", controlStack[i].controlData, ": ");
2480          CommaPrinter comma(", ", "");
2481          dumpExpressionStack(comma, *expressionStack);
2482          expressionStack = &controlStack[i].enclosedExpressionStack;
2483          dataLogLn();
2484      }
2485      dataLogLn();
2486  }
2487  
2488  auto B3IRGenerator::origin() -> Origin
2489  {
2490      OpcodeOrigin origin(m_parser->currentOpcode(), m_parser->currentOpcodeStartingOffset());
2491      ASSERT(isValidOpType(static_cast<uint8_t>(origin.opcode())));
2492      return bitwise_cast<Origin>(origin);
2493  }
2494  
2495  Expected<std::unique_ptr<InternalFunction>, String> parseAndCompile(CompilationContext& compilationContext, const FunctionData& function, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, unsigned& osrEntryScratchBufferSize, const ModuleInformation& info, MemoryMode mode, CompilationMode compilationMode, uint32_t functionIndex, uint32_t loopIndexForOSREntry, TierUpCount* tierUp)
2496  {
2497      auto result = makeUnique<InternalFunction>();
2498  
2499      compilationContext.embedderEntrypointJIT = makeUnique<CCallHelpers>();
2500      compilationContext.wasmEntrypointJIT = makeUnique<CCallHelpers>();
2501  
2502      Procedure procedure;
2503  
2504      procedure.setOriginPrinter([] (PrintStream& out, Origin origin) {
2505          if (origin.data())
2506              out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
2507      });
2508      
2509      // This means we cannot use either StackmapGenerationParams::usedRegisters() or
2510      // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
2511      // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
2512      // optLevel=1.
2513      procedure.setNeedsUsedRegisters(false);
2514      
2515      procedure.setOptLevel(compilationMode == CompilationMode::BBQMode
2516          ? Options::webAssemblyBBQB3OptimizationLevel()
2517          : Options::webAssemblyOMGOptimizationLevel());
2518  
2519      B3IRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, osrEntryScratchBufferSize, mode, compilationMode, functionIndex, loopIndexForOSREntry, tierUp);
2520      FunctionParser<B3IRGenerator> parser(irGenerator, function.data.data(), function.data.size(), signature, info);
2521      WASM_FAIL_IF_HELPER_FAILS(parser.parse());
2522  
2523      irGenerator.insertConstants();
2524  
2525      procedure.resetReachability();
2526      if (ASSERT_ENABLED)
2527          validate(procedure, "After parsing:\n");
2528  
2529      dataLogIf(WasmB3IRGeneratorInternal::verbose, "Pre SSA: ", procedure);
2530      fixSSA(procedure);
2531      dataLogIf(WasmB3IRGeneratorInternal::verbose, "Post SSA: ", procedure);
2532      
2533      {
2534          B3::prepareForGeneration(procedure);
2535          B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
2536          compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
2537          result->entrypoint.calleeSaveRegisters = procedure.calleeSaveRegisterAtOffsetList();
2538      }
2539  
2540      return result;
2541  }
2542  
2543  // Custom wasm ops. These are the ones too messy to do in wasm.json.
2544  
2545  void B3IRGenerator::emitChecksForModOrDiv(B3::Opcode operation, ExpressionType left, ExpressionType right)
2546  {
2547      ASSERT(operation == Div || operation == Mod || operation == UDiv || operation == UMod);
2548      const B3::Type type = left->type();
2549  
2550      {
2551          CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
2552              m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, 0)));
2553  
2554          check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2555              this->emitExceptionCheck(jit, ExceptionType::DivisionByZero);
2556          });
2557      }
2558  
2559      if (operation == Div) {
2560          int64_t min = type == Int32 ? std::numeric_limits<int32_t>::min() : std::numeric_limits<int64_t>::min();
2561  
2562          CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
2563              m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
2564                  m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), left, constant(type, min)),
2565                  m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, -1))));
2566  
2567          check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2568              this->emitExceptionCheck(jit, ExceptionType::IntegerOverflow);
2569          });
2570      }
2571  }
2572  
2573  template<>
2574  auto B3IRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2575  {
2576      const B3::Opcode op = Div;
2577      emitChecksForModOrDiv(op, left, right);
2578      result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
2579      return { };
2580  }
2581  
2582  template<>
2583  auto B3IRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2584  {
2585      const B3::Opcode op = Mod;
2586      emitChecksForModOrDiv(op, left, right);
2587      result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
2588      return { };
2589  }
2590  
2591  template<>
2592  auto B3IRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2593  {
2594      const B3::Opcode op = UDiv;
2595      emitChecksForModOrDiv(op, left, right);
2596      result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
2597      return { };
2598  }
2599  
2600  template<>
2601  auto B3IRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2602  {
2603      const B3::Opcode op = UMod;
2604      emitChecksForModOrDiv(op, left, right);
2605      result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
2606      return { };
2607  }
2608  
2609  template<>
2610  auto B3IRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2611  {
2612      const B3::Opcode op = Div;
2613      emitChecksForModOrDiv(op, left, right);
2614      result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
2615      return { };
2616  }
2617  
2618  template<>
2619  auto B3IRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2620  {
2621      const B3::Opcode op = Mod;
2622      emitChecksForModOrDiv(op, left, right);
2623      result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
2624      return { };
2625  }
2626  
2627  template<>
2628  auto B3IRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2629  {
2630      const B3::Opcode op = UDiv;
2631      emitChecksForModOrDiv(op, left, right);
2632      result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
2633      return { };
2634  }
2635  
2636  template<>
2637  auto B3IRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2638  {
2639      const B3::Opcode op = UMod;
2640      emitChecksForModOrDiv(op, left, right);
2641      result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
2642      return { };
2643  }
2644  
2645  template<>
2646  auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
2647  {
2648      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
2649      patchpoint->append(arg, ValueRep::SomeRegister);
2650      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2651          jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
2652      });
2653      patchpoint->effects = Effects::none();
2654      result = patchpoint;
2655      return { };
2656  }
2657  
2658  template<>
2659  auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
2660  {
2661      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
2662      patchpoint->append(arg, ValueRep::SomeRegister);
2663      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2664          jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
2665      });
2666      patchpoint->effects = Effects::none();
2667      result = patchpoint;
2668      return { };
2669  }
2670  
2671  template<>
2672  auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
2673  {
2674  #if CPU(X86_64)
2675      if (MacroAssembler::supportsCountPopulation()) {
2676          PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
2677          patchpoint->append(arg, ValueRep::SomeRegister);
2678          patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2679              jit.countPopulation32(params[1].gpr(), params[0].gpr());
2680          });
2681          patchpoint->effects = Effects::none();
2682          result = patchpoint;
2683          return { };
2684      }
2685  #endif
2686  
2687      Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationPopcount32));
2688      result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(), Effects::none(), funcAddress, arg);
2689      return { };
2690  }
2691  
2692  template<>
2693  auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
2694  {
2695  #if CPU(X86_64)
2696      if (MacroAssembler::supportsCountPopulation()) {
2697          PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
2698          patchpoint->append(arg, ValueRep::SomeRegister);
2699          patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2700              jit.countPopulation64(params[1].gpr(), params[0].gpr());
2701          });
2702          patchpoint->effects = Effects::none();
2703          result = patchpoint;
2704          return { };
2705      }
2706  #endif
2707  
2708      Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operationPopcount64));
2709      result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, origin(), Effects::none(), funcAddress, arg);
2710      return { };
2711  }
2712  
2713  template<>
2714  auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2715  {
2716      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
2717      if (isX86())
2718          patchpoint->numGPScratchRegisters = 1;
2719      patchpoint->clobber(RegisterSet::macroScratchRegisters());
2720      patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
2721      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2722          AllowMacroScratchRegisterUsage allowScratch(jit);
2723  #if CPU(X86_64)
2724          jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
2725  #else
2726          jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
2727  #endif
2728      });
2729      patchpoint->effects = Effects::none();
2730      result = patchpoint;
2731      return { };
2732  }
2733  
2734  template<>
2735  auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2736  {
2737      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
2738      if (isX86())
2739          patchpoint->numGPScratchRegisters = 1;
2740      patchpoint->clobber(RegisterSet::macroScratchRegisters());
2741      patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
2742      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2743          AllowMacroScratchRegisterUsage allowScratch(jit);
2744  #if CPU(X86_64)
2745          jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
2746  #else
2747          jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
2748  #endif
2749      });
2750      patchpoint->effects = Effects::none();
2751      result = patchpoint;
2752      return { };
2753  }
2754  
2755  template<>
2756  auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
2757  {
2758      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
2759      patchpoint->append(arg, ValueRep::SomeRegister);
2760      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2761          jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
2762      });
2763      patchpoint->effects = Effects::none();
2764      result = patchpoint;
2765      return { };
2766  }
2767  
2768  template<>
2769  auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
2770  {
2771      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
2772      patchpoint->append(arg, ValueRep::SomeRegister);
2773      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2774          jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
2775      });
2776      patchpoint->effects = Effects::none();
2777      result = patchpoint;
2778      return { };
2779  }
2780  
2781  template<>
2782  auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
2783  {
2784      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
2785      patchpoint->append(arg, ValueRep::SomeRegister);
2786      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2787          jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
2788      });
2789      patchpoint->effects = Effects::none();
2790      result = patchpoint;
2791      return { };
2792  }
2793  
2794  template<>
2795  auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
2796  {
2797      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
2798      patchpoint->append(arg, ValueRep::SomeRegister);
2799      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2800          jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
2801      });
2802      patchpoint->effects = Effects::none();
2803      result = patchpoint;
2804      return { };
2805  }
2806  
2807  template<>
2808  auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2809  {
2810      Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
2811      Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) - 1.0));
2812      Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
2813          m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
2814          m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
2815      outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
2816      CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
2817      trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
2818          this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
2819      });
2820      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
2821      patchpoint->append(arg, ValueRep::SomeRegister);
2822      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2823          jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
2824      });
2825      patchpoint->effects = Effects::none();
2826      result = patchpoint;
2827      return { };
2828  }
2829  
2830  template<>
2831  auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2832  {
2833      Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
2834      Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
2835      Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
2836          m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
2837          m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
2838      outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
2839      CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
2840      trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
2841          this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
2842      });
2843      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
2844      patchpoint->append(arg, ValueRep::SomeRegister);
2845      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2846          jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
2847      });
2848      patchpoint->effects = Effects::none();
2849      result = patchpoint;
2850      return { };
2851  }
2852  
2853  
2854  template<>
2855  auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2856  {
2857      Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
2858      Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
2859      Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
2860          m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
2861          m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
2862      outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
2863      CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
2864      trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
2865          this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
2866      });
2867      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
2868      patchpoint->append(arg, ValueRep::SomeRegister);
2869      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2870          jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
2871      });
2872      patchpoint->effects = Effects::none();
2873      result = patchpoint;
2874      return { };
2875  }
2876  
2877  template<>
2878  auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2879  {
2880      Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
2881      Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
2882      Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
2883          m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
2884          m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
2885      outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
2886      CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
2887      trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
2888          this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
2889      });
2890      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
2891      patchpoint->append(arg, ValueRep::SomeRegister);
2892      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2893          jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
2894      });
2895      patchpoint->effects = Effects::none();
2896      result = patchpoint;
2897      return { };
2898  }
2899  
2900  template<>
2901  auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2902  {
2903      Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
2904      Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
2905      Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
2906          m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
2907          m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
2908      outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
2909      CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
2910      trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
2911          this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
2912      });
2913      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
2914      patchpoint->append(arg, ValueRep::SomeRegister);
2915      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2916          jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
2917      });
2918      patchpoint->effects = Effects::none();
2919      result = patchpoint;
2920      return { };
2921  }
2922  
2923  template<>
2924  auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2925  {
2926      Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
2927      Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
2928      Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
2929          m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
2930          m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
2931      outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
2932      CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
2933      trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
2934          this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
2935      });
2936  
2937      Value* signBitConstant;
2938      if (isX86()) {
2939          // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
2940          // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
2941          // so we can pool them if needed.
2942          signBitConstant = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2943      }
2944      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
2945      patchpoint->append(arg, ValueRep::SomeRegister);
2946      if (isX86()) {
2947          patchpoint->append(signBitConstant, ValueRep::SomeRegister);
2948          patchpoint->numFPScratchRegisters = 1;
2949      }
2950      patchpoint->clobber(RegisterSet::macroScratchRegisters());
2951      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2952          AllowMacroScratchRegisterUsage allowScratch(jit);
2953          FPRReg scratch = InvalidFPRReg;
2954          FPRReg constant = InvalidFPRReg;
2955          if (isX86()) {
2956              scratch = params.fpScratch(0);
2957              constant = params[2].fpr();
2958          }
2959          jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2960      });
2961      patchpoint->effects = Effects::none();
2962      result = patchpoint;
2963      return { };
2964  }
2965  
2966  template<>
2967  auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2968  {
2969      Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
2970      Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
2971      Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
2972          m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
2973          m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
2974      outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
2975      CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
2976      trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
2977          this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
2978      });
2979      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
2980      patchpoint->append(arg, ValueRep::SomeRegister);
2981      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2982          jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
2983      });
2984      patchpoint->effects = Effects::none();
2985      result = patchpoint;
2986      return { };
2987  }
2988  
2989  template<>
2990  auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2991  {
2992      Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
2993      Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
2994      Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
2995          m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
2996          m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
2997      outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
2998      CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
2999      trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
3000          this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
3001      });
3002  
3003      Value* signBitConstant;
3004      if (isX86()) {
3005          // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
3006          // the numbers would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
3007          // so we can pool them if needed.
3008          signBitConstant = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
3009      }
3010      PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
3011      patchpoint->append(arg, ValueRep::SomeRegister);
3012      if (isX86()) {
3013          patchpoint->append(signBitConstant, ValueRep::SomeRegister);
3014          patchpoint->numFPScratchRegisters = 1;
3015      }
3016      patchpoint->clobber(RegisterSet::macroScratchRegisters());
3017      patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
3018          AllowMacroScratchRegisterUsage allowScratch(jit);
3019          FPRReg scratch = InvalidFPRReg;
3020          FPRReg constant = InvalidFPRReg;
3021          if (isX86()) {
3022              scratch = params.fpScratch(0);
3023              constant = params[2].fpr();
3024          }
3025          jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
3026      });
3027      patchpoint->effects = Effects::none();
3028      result = patchpoint;
3029      return { };
3030  }
3031  
3032  } } // namespace JSC::Wasm
3033  
3034  #include "WasmB3IRGeneratorInlines.h"
3035  
3036  #endif // ENABLE(WEBASSEMBLY)