1  #include <biscuit/assert.hpp>
   2  #include <biscuit/assembler.hpp>
   3  
   4  #include <bit>
   5  #include <cstring>
   6  #include <utility>
   7  
   8  #include "assembler_util.hpp"
   9  
  10  namespace biscuit {
  11  
  12  Assembler::Assembler(size_t capacity)
  13      : m_buffer(capacity) {}
  14  
  15  Assembler::Assembler(uint8_t* buffer, size_t capacity, ArchFeature features)
  16      : m_buffer(buffer, capacity), m_features{features} {}
  17  
  18  Assembler::~Assembler() = default;
  19  
  20  CodeBuffer& Assembler::GetCodeBuffer() {
  21      return m_buffer;
  22  }
  23  
  24  CodeBuffer Assembler::SwapCodeBuffer(CodeBuffer&& buffer) noexcept {
  25      return std::exchange(m_buffer, std::move(buffer));
  26  }
  27  
  28  void Assembler::Bind(Label* label) {
  29      BindToOffset(label, m_buffer.GetCursorOffset());
  30  }
  31  
  32  void Assembler::ADD(GPR rd, GPR lhs, GPR rhs) noexcept {
  33      EmitRType(m_buffer, 0b0000000, rhs, lhs, 0b000, rd, 0b0110011);
  34  }
  35  
  36  void Assembler::ADDI(GPR rd, GPR rs, int32_t imm) noexcept {
  37      EmitIType(m_buffer, static_cast<uint32_t>(imm), rs, 0b000, rd, 0b0010011);
  38  }
  39  
  40  void Assembler::AND(GPR rd, GPR lhs, GPR rhs) noexcept {
  41      EmitRType(m_buffer, 0b0000000, rhs, lhs, 0b111, rd, 0b0110011);
  42  }
  43  
  44  void Assembler::ANDI(GPR rd, GPR rs, uint32_t imm) noexcept {
  45      EmitIType(m_buffer, imm, rs, 0b111, rd, 0b0010011);
  46  }
  47  
  48  void Assembler::AUIPC(GPR rd, int32_t imm) noexcept {
  49      EmitUType(m_buffer, static_cast<uint32_t>(imm), rd, 0b0010111);
  50  }
  51  
  52  void Assembler::BEQ(GPR rs1, GPR rs2, Label* label) noexcept {
  53      const auto address = LinkAndGetOffset(label);
  54      BEQ(rs1, rs2, static_cast<int32_t>(address));
  55  }
  56  
  57  void Assembler::BEQZ(GPR rs, Label* label) noexcept {
  58      const auto address = LinkAndGetOffset(label);
  59      BEQZ(rs, static_cast<int32_t>(address));
  60  }
  61  
  62  void Assembler::BGE(GPR rs1, GPR rs2, Label* label) noexcept {
  63      const auto address = LinkAndGetOffset(label);
  64      BGE(rs1, rs2, static_cast<int32_t>(address));
  65  }
  66  
  67  void Assembler::BGEU(GPR rs1, GPR rs2, Label* label) noexcept {
  68      const auto address = LinkAndGetOffset(label);
  69      BGEU(rs1, rs2, static_cast<int32_t>(address));
  70  }
  71  
  72  void Assembler::BGEZ(GPR rs, Label* label) noexcept {
  73      const auto address = LinkAndGetOffset(label);
  74      BGEZ(rs, static_cast<int32_t>(address));
  75  }
  76  
  77  void Assembler::BGT(GPR rs, GPR rt, Label* label) noexcept {
  78      const auto address = LinkAndGetOffset(label);
  79      BGT(rs, rt, static_cast<int32_t>(address));
  80  }
  81  
  82  void Assembler::BGTU(GPR rs, GPR rt, Label* label) noexcept {
  83      const auto address = LinkAndGetOffset(label);
  84      BGTU(rs, rt, static_cast<int32_t>(address));
  85  }
  86  
  87  void Assembler::BGTZ(GPR rs, Label* label) noexcept {
  88      const auto address = LinkAndGetOffset(label);
  89      BGTZ(rs, static_cast<int32_t>(address));
  90  }
  91  
  92  void Assembler::BLE(GPR rs, GPR rt, Label* label) noexcept {
  93      const auto address = LinkAndGetOffset(label);
  94      BLE(rs, rt, static_cast<int32_t>(address));
  95  }
  96  
  97  void Assembler::BLEU(GPR rs, GPR rt, Label* label) noexcept {
  98      const auto address = LinkAndGetOffset(label);
  99      BLEU(rs, rt, static_cast<int32_t>(address));
 100  }
 101  
 102  void Assembler::BLEZ(GPR rs, Label* label) noexcept {
 103      const auto address = LinkAndGetOffset(label);
 104      BLEZ(rs, static_cast<int32_t>(address));
 105  }
 106  
 107  void Assembler::BLT(GPR rs1, GPR rs2, Label* label) noexcept {
 108      const auto address = LinkAndGetOffset(label);
 109      BLT(rs1, rs2, static_cast<int32_t>(address));
 110  }
 111  
 112  void Assembler::BLTU(GPR rs1, GPR rs2, Label* label) noexcept {
 113      const auto address = LinkAndGetOffset(label);
 114      BLTU(rs1, rs2, static_cast<int32_t>(address));
 115  }
 116  
 117  void Assembler::BLTZ(GPR rs, Label* label) noexcept {
 118      const auto address = LinkAndGetOffset(label);
 119      BLTZ(rs, static_cast<int32_t>(address));
 120  }
 121  
 122  void Assembler::BNE(GPR rs1, GPR rs2, Label* label) noexcept {
 123      const auto address = LinkAndGetOffset(label);
 124      BNE(rs1, rs2, static_cast<int32_t>(address));
 125  }
 126  
 127  void Assembler::BNEZ(GPR rs, Label* label) noexcept {
 128      const auto address = LinkAndGetOffset(label);
 129      BNEZ(rs, static_cast<int32_t>(address));
 130  }
 131  
 132  void Assembler::BEQ(GPR rs1, GPR rs2, int32_t imm) noexcept {
 133      BISCUIT_ASSERT(IsValidBTypeImm(imm));
 134      EmitBType(m_buffer, static_cast<uint32_t>(imm), rs2, rs1, 0b000, 0b1100011);
 135  }
 136  
 137  void Assembler::BEQZ(GPR rs, int32_t imm) noexcept {
 138      BEQ(rs, x0, imm);
 139  }
 140  
 141  void Assembler::BGE(GPR rs1, GPR rs2, int32_t imm) noexcept {
 142      BISCUIT_ASSERT(IsValidBTypeImm(imm));
 143      EmitBType(m_buffer, static_cast<uint32_t>(imm), rs2, rs1, 0b101, 0b1100011);
 144  }
 145  
 146  void Assembler::BGEU(GPR rs1, GPR rs2, int32_t imm) noexcept {
 147      BISCUIT_ASSERT(IsValidBTypeImm(imm));
 148      EmitBType(m_buffer, static_cast<uint32_t>(imm), rs2, rs1, 0b111, 0b1100011);
 149  }
 150  
 151  void Assembler::BGEZ(GPR rs, int32_t imm) noexcept {
 152      BGE(rs, x0, imm);
 153  }
 154  
 155  void Assembler::BGT(GPR rs, GPR rt, int32_t imm) noexcept {
 156      BLT(rt, rs, imm);
 157  }
 158  
 159  void Assembler::BGTU(GPR rs, GPR rt, int32_t imm) noexcept {
 160      BLTU(rt, rs, imm);
 161  }
 162  
 163  void Assembler::BGTZ(GPR rs, int32_t imm) noexcept {
 164      BLT(x0, rs, imm);
 165  }
 166  
 167  void Assembler::BLE(GPR rs, GPR rt, int32_t imm) noexcept {
 168      BGE(rt, rs, imm);
 169  }
 170  
 171  void Assembler::BLEU(GPR rs, GPR rt, int32_t imm) noexcept {
 172      BGEU(rt, rs, imm);
 173  }
 174  
 175  void Assembler::BLEZ(GPR rs, int32_t imm) noexcept {
 176      BGE(x0, rs, imm);
 177  }
 178  
 179  void Assembler::BLT(GPR rs1, GPR rs2, int32_t imm) noexcept {
 180      BISCUIT_ASSERT(IsValidBTypeImm(imm));
 181      EmitBType(m_buffer, static_cast<uint32_t>(imm), rs2, rs1, 0b100, 0b1100011);
 182  }
 183  
 184  void Assembler::BLTU(GPR rs1, GPR rs2, int32_t imm) noexcept {
 185      BISCUIT_ASSERT(IsValidBTypeImm(imm));
 186      EmitBType(m_buffer, static_cast<uint32_t>(imm), rs2, rs1, 0b110, 0b1100011);
 187  }
 188  
 189  void Assembler::BLTZ(GPR rs, int32_t imm) noexcept {
 190      BLT(rs, x0, imm);
 191  }
 192  
 193  void Assembler::BNE(GPR rs1, GPR rs2, int32_t imm) noexcept {
 194      BISCUIT_ASSERT(IsValidBTypeImm(imm));
 195      EmitBType(m_buffer, static_cast<uint32_t>(imm), rs2, rs1, 0b001, 0b1100011);
 196  }
 197  
 198  void Assembler::BNEZ(GPR rs, int32_t imm) noexcept {
 199      BNE(x0, rs, imm);
 200  }
 201  
 202  void Assembler::CALL(int32_t offset) noexcept {
 203      const auto uimm = static_cast<uint32_t>(offset);
 204      const auto lower = uimm & 0xFFF;
 205      const auto upper = (uimm & 0xFFFFF000) >> 12;
 206      const auto needs_increment = (uimm & 0x800) != 0;
 207  
 208      // Sign-extend the lower portion if the MSB of it is set.
 209      const auto new_lower = needs_increment ? static_cast<int32_t>(lower << 20) >> 20
 210                                             : static_cast<int32_t>(lower);
 211      const auto new_upper = needs_increment ? upper + 1 : upper;
 212  
 213      AUIPC(x1, static_cast<int32_t>(new_upper));
 214      JALR(x1, new_lower, x1);
 215  }
 216  
 217  void Assembler::EBREAK() noexcept {
 218      m_buffer.Emit32(0x00100073);
 219  }
 220  
 221  void Assembler::ECALL() noexcept {
 222      m_buffer.Emit32(0x00000073);
 223  }
 224  
 225  void Assembler::FENCE() noexcept {
 226      FENCE(FenceOrder::IORW, FenceOrder::IORW);
 227  }
 228  
 229  void Assembler::FENCE(FenceOrder pred, FenceOrder succ) noexcept {
 230      EmitFENCE(m_buffer, 0b0000, pred, succ, x0, 0b000, x0, 0b0001111);
 231  }
 232  
 233  void Assembler::FENCEI(GPR rd, GPR rs, uint32_t imm) noexcept {
 234      m_buffer.Emit32(((imm & 0xFFF) << 20) | (rs.Index() << 15) | 0x1000U | (rd.Index() << 7) | 0b0001111);
 235  }
 236  
 237  void Assembler::FENCETSO() noexcept {
 238      EmitFENCE(m_buffer, 0b1000, FenceOrder::RW, FenceOrder::RW, x0, 0b000, x0, 0b0001111);
 239  }
 240  
 241  void Assembler::J(Label* label) noexcept {
 242      const auto address = LinkAndGetOffset(label);
 243      BISCUIT_ASSERT(IsValidJTypeImm(address));
 244      J(static_cast<int32_t>(address));
 245  }
 246  
 247  void Assembler::JAL(Label* label) noexcept {
 248      const auto address = LinkAndGetOffset(label);
 249      BISCUIT_ASSERT(IsValidJTypeImm(address));
 250      JAL(static_cast<int32_t>(address));
 251  }
 252  
 253  void Assembler::JAL(GPR rd, Label* label) noexcept {
 254      const auto address = LinkAndGetOffset(label);
 255      BISCUIT_ASSERT(IsValidJTypeImm(address));
 256      JAL(rd, static_cast<int32_t>(address));
 257  }
 258  
 259  void Assembler::J(int32_t imm) noexcept {
 260      BISCUIT_ASSERT(IsValidJTypeImm(imm));
 261      JAL(x0, imm);
 262  }
 263  
 264  void Assembler::JAL(int32_t imm) noexcept {
 265      BISCUIT_ASSERT(IsValidJTypeImm(imm));
 266      EmitJType(m_buffer, static_cast<uint32_t>(imm), x1, 0b1101111);
 267  }
 268  
 269  void Assembler::JAL(GPR rd, int32_t imm) noexcept {
 270      BISCUIT_ASSERT(IsValidJTypeImm(imm));
 271      EmitJType(m_buffer, static_cast<uint32_t>(imm), rd, 0b1101111);
 272  }
 273  
 274  void Assembler::JALR(GPR rs) noexcept {
 275      JALR(x1, 0, rs);
 276  }
 277  
 278  void Assembler::JALR(GPR rd, int32_t imm, GPR rs1) noexcept {
 279      BISCUIT_ASSERT(IsValidSigned12BitImm(imm));
 280      EmitIType(m_buffer, static_cast<uint32_t>(imm), rs1, 0b000, rd, 0b1100111);
 281  }
 282  
 283  void Assembler::JR(GPR rs) noexcept {
 284      JALR(x0, 0, rs);
 285  }
 286  
 287  void Assembler::LB(GPR rd, int32_t imm, GPR rs) noexcept {
 288      BISCUIT_ASSERT(IsValidSigned12BitImm(imm));
 289      EmitIType(m_buffer, static_cast<uint32_t>(imm), rs, 0b000, rd, 0b0000011);
 290  }
 291  
 292  void Assembler::LBU(GPR rd, int32_t imm, GPR rs) noexcept {
 293      BISCUIT_ASSERT(IsValidSigned12BitImm(imm));
 294      EmitIType(m_buffer, static_cast<uint32_t>(imm), rs, 0b100, rd, 0b0000011);
 295  }
 296  
 297  void Assembler::LH(GPR rd, int32_t imm, GPR rs) noexcept {
 298      BISCUIT_ASSERT(IsValidSigned12BitImm(imm));
 299      EmitIType(m_buffer, static_cast<uint32_t>(imm), rs, 0b001, rd, 0b0000011);
 300  }
 301  
 302  void Assembler::LHU(GPR rd, int32_t imm, GPR rs) noexcept {
 303      BISCUIT_ASSERT(IsValidSigned12BitImm(imm));
 304      EmitIType(m_buffer, static_cast<uint32_t>(imm), rs, 0b101, rd, 0b0000011);
 305  }
 306  
 307  void Assembler::LI(GPR rd, uint64_t imm) noexcept {
 308      if (IsRV32(m_features)) {
 309          // Depending on imm, the following instructions are emitted.
 310          // hi20 == 0              -> ADDI
 311          // lo12 == 0 && hi20 != 0 -> LUI
 312          // otherwise              -> LUI+ADDI
 313  
 314          // Add 0x800 to cancel out the signed extension of ADDI.
 315          const auto uimm32 = static_cast<uint32_t>(imm);
 316          const auto hi20 = (uimm32 + 0x800) >> 12 & 0xFFFFF;
 317          const auto lo12 = static_cast<int32_t>(uimm32) & 0xFFF;
 318          GPR rs1 = zero;
 319  
 320          if (hi20 != 0) {
 321              LUI(rd, hi20);
 322              rs1 = rd;
 323          }
 324  
 325          if (lo12 != 0 || hi20 == 0) {
 326              ADDI(rd, rs1, lo12);
 327          }
 328      } else {
 329          // For 64-bit imm, a sequence of up to 8 instructions (i.e. LUI+ADDIW+SLLI+
 330          // ADDI+SLLI+ADDI+SLLI+ADDI) is emitted.
 331          // In the following, imm is processed from LSB to MSB while instruction emission
 332          // is performed from MSB to LSB by calling LI() recursively. In each recursion,
 333          // the lowest 12 bits are removed from imm and the optimal shift amount is
 334          // calculated. Then, the remaining part of imm is processed recursively and
 335          // LI() get called as soon as it fits into 32 bits.
 336  
 337          if (static_cast<uint64_t>(static_cast<int64_t>(imm << 32) >> 32) == imm) {
 338              // Depending on imm, the following instructions are emitted.
 339              // hi20 == 0              -> ADDIW
 340              // lo12 == 0 && hi20 != 0 -> LUI
 341              // otherwise              -> LUI+ADDIW
 342  
 343              // Add 0x800 to cancel out the signed extension of ADDIW.
 344              const auto hi20 = (static_cast<uint32_t>(imm) + 0x800) >> 12 & 0xFFFFF;
 345              const auto lo12 = static_cast<int32_t>(imm) & 0xFFF;
 346              GPR rs1 = zero;
 347  
 348              if (hi20 != 0) {
 349                  LUI(rd, hi20);
 350                  rs1 = rd;
 351              }
 352  
 353              if (lo12 != 0 || hi20 == 0) {
 354                  ADDIW(rd, rs1, lo12);
 355              }
 356              return;
 357          }
 358  
 359          const auto lo12 = static_cast<int32_t>(static_cast<int64_t>(imm << 52) >> 52);
 360          // Add 0x800 to cancel out the signed extension of ADDI.
 361          uint64_t hi52 = (imm + 0x800) >> 12;
 362          const uint32_t shift = 12 + static_cast<uint32_t>(std::countr_zero(hi52));
 363          hi52 = static_cast<uint64_t>((static_cast<int64_t>(hi52 >> (shift - 12)) << shift) >> shift);
 364          LI(rd, hi52);
 365          SLLI(rd, rd, shift);
 366          if (lo12 != 0) {
 367              ADDI(rd, rd, lo12);
 368          }
 369      }
 370  }
 371  
 372  void Assembler::LUI(GPR rd, uint32_t imm) noexcept {
 373      EmitUType(m_buffer, imm, rd, 0b0110111);
 374  }
 375  
 376  void Assembler::LW(GPR rd, int32_t imm, GPR rs) noexcept {
 377      BISCUIT_ASSERT(IsValidSigned12BitImm(imm));
 378      EmitIType(m_buffer, static_cast<uint32_t>(imm), rs, 0b010, rd, 0b0000011);
 379  }
 380  
 381  void Assembler::MV(GPR rd, GPR rs) noexcept {
 382      ADDI(rd, rs, 0);
 383  }
 384  
 385  void Assembler::NEG(GPR rd, GPR rs) noexcept {
 386      SUB(rd, x0, rs);
 387  }
 388  
 389  void Assembler::NOP() noexcept {
 390      ADDI(x0, x0, 0);
 391  }
 392  
 393  void Assembler::NOT(GPR rd, GPR rs) noexcept {
 394      XORI(rd, rs, UINT32_MAX);
 395  }
 396  
 397  void Assembler::OR(GPR rd, GPR lhs, GPR rhs) noexcept {
 398      EmitRType(m_buffer, 0b0000000, rhs, lhs, 0b110, rd, 0b0110011);
 399  }
 400  
 401  void Assembler::ORI(GPR rd, GPR rs, uint32_t imm) noexcept {
 402      EmitIType(m_buffer, imm, rs, 0b110, rd, 0b0010011);
 403  }
 404  
 405  void Assembler::PAUSE() noexcept {
 406      m_buffer.Emit32(0x0100000F);
 407  }
 408  
 409  void Assembler::RET() noexcept {
 410      JALR(x0, 0, x1);
 411  }
 412  
 413  void Assembler::SB(GPR rs2, int32_t imm, GPR rs1) noexcept {
 414      BISCUIT_ASSERT(IsValidSigned12BitImm(imm));
 415      EmitSType(m_buffer, static_cast<uint32_t>(imm), rs2, rs1, 0b000, 0b0100011);
 416  }
 417  
 418  void Assembler::SEQZ(GPR rd, GPR rs) noexcept {
 419      SLTIU(rd, rs, 1);
 420  }
 421  
 422  void Assembler::SGTZ(GPR rd, GPR rs) noexcept {
 423      SLT(rd, x0, rs);
 424  }
 425  
 426  void Assembler::SH(GPR rs2, int32_t imm, GPR rs1) noexcept {
 427      BISCUIT_ASSERT(IsValidSigned12BitImm(imm));
 428      EmitSType(m_buffer, static_cast<uint32_t>(imm), rs2, rs1, 0b001, 0b0100011);
 429  }
 430  
 431  void Assembler::SLL(GPR rd, GPR lhs, GPR rhs) noexcept {
 432      EmitRType(m_buffer, 0b0000000, rhs, lhs, 0b001, rd, 0b0110011);
 433  }
 434  
 435  void Assembler::SLLI(GPR rd, GPR rs, uint32_t shift) noexcept {
 436      if (IsRV32(m_features)) {
 437          BISCUIT_ASSERT(shift <= 31);
 438          EmitIType(m_buffer, shift & 0x1F, rs, 0b001, rd, 0b0010011);
 439      } else {
 440          BISCUIT_ASSERT(shift <= 63);
 441          EmitIType(m_buffer, shift & 0x3F, rs, 0b001, rd, 0b0010011);
 442      }
 443  }
 444  
 445  void Assembler::SLT(GPR rd, GPR lhs, GPR rhs) noexcept {
 446      EmitRType(m_buffer, 0b0000000, rhs, lhs, 0b010, rd, 0b0110011);
 447  }
 448  
 449  void Assembler::SLTI(GPR rd, GPR rs, int32_t imm) noexcept {
 450      BISCUIT_ASSERT(IsValidSigned12BitImm(imm));
 451      EmitIType(m_buffer, static_cast<uint32_t>(imm), rs, 0b010, rd, 0b0010011);
 452  }
 453  
 454  void Assembler::SLTIU(GPR rd, GPR rs, int32_t imm) noexcept {
 455      BISCUIT_ASSERT(IsValidSigned12BitImm(imm));
 456      EmitIType(m_buffer, static_cast<uint32_t>(imm), rs, 0b011, rd, 0b0010011);
 457  }
 458  
 459  void Assembler::SLTU(GPR rd, GPR lhs, GPR rhs) noexcept {
 460      EmitRType(m_buffer, 0b0000000, rhs, lhs, 0b011, rd, 0b0110011);
 461  }
 462  
 463  void Assembler::SLTZ(GPR rd, GPR rs) noexcept {
 464      SLT(rd, rs, x0);
 465  }
 466  
 467  void Assembler::SNEZ(GPR rd, GPR rs) noexcept {
 468      SLTU(rd, x0, rs);
 469  }
 470  
 471  void Assembler::SRA(GPR rd, GPR lhs, GPR rhs) noexcept {
 472      EmitRType(m_buffer, 0b0100000, rhs, lhs, 0b101, rd, 0b0110011);
 473  }
 474  
 475  void Assembler::SRAI(GPR rd, GPR rs, uint32_t shift) noexcept {
 476      if (IsRV32(m_features)) {
 477          BISCUIT_ASSERT(shift <= 31);
 478          EmitIType(m_buffer, (0b0100000 << 5) | (shift & 0x1F), rs, 0b101, rd, 0b0010011);
 479      } else {
 480          BISCUIT_ASSERT(shift <= 63);
 481          EmitIType(m_buffer, (0b0100000 << 5) | (shift & 0x3F), rs, 0b101, rd, 0b0010011);
 482      }
 483  }
 484  
 485  void Assembler::SRL(GPR rd, GPR lhs, GPR rhs) noexcept {
 486      EmitRType(m_buffer, 0b0000000, rhs, lhs, 0b101, rd, 0b0110011);
 487  }
 488  
 489  void Assembler::SRLI(GPR rd, GPR rs, uint32_t shift) noexcept {
 490      if (IsRV32(m_features)) {
 491          BISCUIT_ASSERT(shift <= 31);
 492          EmitIType(m_buffer, shift & 0x1F, rs, 0b101, rd, 0b0010011);
 493      } else {
 494          BISCUIT_ASSERT(shift <= 63);
 495          EmitIType(m_buffer, shift & 0x3F, rs, 0b101, rd, 0b0010011);
 496      }
 497  }
 498  
 499  void Assembler::SUB(GPR rd, GPR lhs, GPR rhs) noexcept {
 500      EmitRType(m_buffer, 0b0100000, rhs, lhs, 0b000, rd, 0b0110011);
 501  }
 502  
 503  void Assembler::SW(GPR rs2, int32_t imm, GPR rs1) noexcept {
 504      BISCUIT_ASSERT(IsValidSigned12BitImm(imm));
 505      EmitSType(m_buffer, static_cast<uint32_t>(imm), rs2, rs1, 0b010, 0b0100011);
 506  }
 507  
 508  void Assembler::XOR(GPR rd, GPR lhs, GPR rhs) noexcept {
 509      EmitRType(m_buffer, 0b0000000, rhs, lhs, 0b100, rd, 0b0110011);
 510  }
 511  
 512  void Assembler::XORI(GPR rd, GPR rs, uint32_t imm) noexcept {
 513      EmitIType(m_buffer, imm, rs, 0b100, rd, 0b0010011);
 514  }
 515  
 516  // RV64I Instructions
 517  
 518  void Assembler::ADDIW(GPR rd, GPR rs, int32_t imm) noexcept {
 519      BISCUIT_ASSERT(IsRV64(m_features));
 520      EmitIType(m_buffer, static_cast<uint32_t>(imm), rs, 0b000, rd, 0b0011011);
 521  }
 522  
 523  void Assembler::ADDW(GPR rd, GPR lhs, GPR rhs) noexcept {
 524      BISCUIT_ASSERT(IsRV64(m_features));
 525      EmitRType(m_buffer, 0b0000000, rhs, lhs, 0b000, rd, 0b0111011);
 526  }
 527  
 528  void Assembler::LD(GPR rd, int32_t imm, GPR rs) noexcept {
 529      BISCUIT_ASSERT(IsRV64(m_features));
 530      BISCUIT_ASSERT(IsValidSigned12BitImm(imm));
 531      EmitIType(m_buffer, static_cast<uint32_t>(imm), rs, 0b011, rd, 0b0000011);
 532  }
 533  
 534  void Assembler::LWU(GPR rd, int32_t imm, GPR rs) noexcept {
 535      BISCUIT_ASSERT(IsRV64(m_features));
 536      BISCUIT_ASSERT(IsValidSigned12BitImm(imm));
 537      EmitIType(m_buffer, static_cast<uint32_t>(imm), rs, 0b110, rd, 0b0000011);
 538  }
 539  
 540  void Assembler::SD(GPR rs2, int32_t imm, GPR rs1) noexcept {
 541      BISCUIT_ASSERT(IsRV64(m_features));
 542      BISCUIT_ASSERT(IsValidSigned12BitImm(imm));
 543      EmitSType(m_buffer, static_cast<uint32_t>(imm), rs2, rs1, 0b011, 0b0100011);
 544  }
 545  
 546  void Assembler::SLLIW(GPR rd, GPR rs, uint32_t shift) noexcept {
 547      BISCUIT_ASSERT(IsRV64(m_features));
 548      BISCUIT_ASSERT(shift <= 31);
 549      EmitIType(m_buffer, shift & 0x1F, rs, 0b001, rd, 0b0011011);
 550  }
 551  void Assembler::SRAIW(GPR rd, GPR rs, uint32_t shift) noexcept {
 552      BISCUIT_ASSERT(IsRV64(m_features));
 553      BISCUIT_ASSERT(shift <= 31);
 554      EmitIType(m_buffer, (0b0100000 << 5) | (shift & 0x1F), rs, 0b101, rd, 0b0011011);
 555  }
 556  void Assembler::SRLIW(GPR rd, GPR rs, uint32_t shift) noexcept {
 557      BISCUIT_ASSERT(IsRV64(m_features));
 558      BISCUIT_ASSERT(shift <= 31);
 559      EmitIType(m_buffer, shift & 0x1F, rs, 0b101, rd, 0b0011011);
 560  }
 561  
 562  void Assembler::SLLW(GPR rd, GPR lhs, GPR rhs) noexcept {
 563      BISCUIT_ASSERT(IsRV64(m_features));
 564      EmitRType(m_buffer, 0b0000000, rhs, lhs, 0b001, rd, 0b0111011);
 565  }
 566  void Assembler::SRAW(GPR rd, GPR lhs, GPR rhs) noexcept {
 567      BISCUIT_ASSERT(IsRV64(m_features));
 568      EmitRType(m_buffer, 0b0100000, rhs, lhs, 0b101, rd, 0b0111011);
 569  }
 570  void Assembler::SRLW(GPR rd, GPR lhs, GPR rhs) noexcept {
 571      BISCUIT_ASSERT(IsRV64(m_features));
 572      EmitRType(m_buffer, 0b0000000, rhs, lhs, 0b101, rd, 0b0111011);
 573  }
 574  
 575  void Assembler::SUBW(GPR rd, GPR lhs, GPR rhs) noexcept {
 576      BISCUIT_ASSERT(IsRV64(m_features));
 577      EmitRType(m_buffer, 0b0100000, rhs, lhs, 0b000, rd, 0b0111011);
 578  }
 579  
 580  // Zawrs Extension Instructions
 581  
 582  void Assembler::WRS_NTO() noexcept {
 583      EmitIType(m_buffer, 0b01101, x0, 0, x0, 0b1110011);
 584  }
 585  void Assembler::WRS_STO() noexcept {
 586      EmitIType(m_buffer, 0b11101, x0, 0, x0, 0b1110011);
 587  }
 588  
 589  // Zacas Extension Instructions
 590  
 591  void Assembler::AMOCAS_D(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 592      if (IsRV32(m_features)) {
 593          BISCUIT_ASSERT((rd.Index() % 2) == 0);
 594          BISCUIT_ASSERT((rs1.Index() % 2) == 0);
 595          BISCUIT_ASSERT((rs2.Index() % 2) == 0);
 596      }
 597      EmitAtomic(m_buffer, 0b00101, ordering, rs2, rs1, 0b011, rd, 0b0101111);
 598  }
 599  void Assembler::AMOCAS_Q(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 600      BISCUIT_ASSERT(IsRV64(m_features));
 601  
 602      // Both rd and rs2 indicate a register pair, so they need to be even-numbered.
 603      BISCUIT_ASSERT((rd.Index() % 2) == 0);
 604      BISCUIT_ASSERT((rs1.Index() % 2) == 0);
 605      BISCUIT_ASSERT((rs2.Index() % 2) == 0);
 606      EmitAtomic(m_buffer, 0b00101, ordering, rs2, rs1, 0b100, rd, 0b0101111);
 607  }
 608  void Assembler::AMOCAS_W(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 609      EmitAtomic(m_buffer, 0b00101, ordering, rs2, rs1, 0b010, rd, 0b0101111);
 610  }
 611  
 612  // Zicond Extension Instructions
 613  
 614  void Assembler::CZERO_EQZ(GPR rd, GPR value, GPR condition) noexcept {
 615      EmitRType(m_buffer, 0b0000111, condition, value, 0b101, rd, 0b0110011);
 616  }
 617  void Assembler::CZERO_NEZ(GPR rd, GPR value, GPR condition) noexcept {
 618      EmitRType(m_buffer, 0b0000111, condition, value, 0b111, rd, 0b0110011);
 619  }
 620  
 621  // Zicsr Extension Instructions
 622  
 623  void Assembler::CSRRC(GPR rd, CSR csr, GPR rs) noexcept {
 624      EmitIType(m_buffer, static_cast<uint32_t>(csr), rs, 0b011, rd, 0b1110011);
 625  }
 626  void Assembler::CSRRCI(GPR rd, CSR csr, uint32_t imm) noexcept {
 627      BISCUIT_ASSERT(imm <= 0x1F);
 628      EmitIType(m_buffer, static_cast<uint32_t>(csr), GPR{imm & 0x1F}, 0b111, rd, 0b1110011);
 629  }
 630  void Assembler::CSRRS(GPR rd, CSR csr, GPR rs) noexcept {
 631      EmitIType(m_buffer, static_cast<uint32_t>(csr), rs, 0b010, rd, 0b1110011);
 632  }
 633  void Assembler::CSRRSI(GPR rd, CSR csr, uint32_t imm) noexcept {
 634      BISCUIT_ASSERT(imm <= 0x1F);
 635      EmitIType(m_buffer, static_cast<uint32_t>(csr), GPR{imm & 0x1F}, 0b110, rd, 0b1110011);
 636  }
 637  void Assembler::CSRRW(GPR rd, CSR csr, GPR rs) noexcept {
 638      EmitIType(m_buffer, static_cast<uint32_t>(csr), rs, 0b001, rd, 0b1110011);
 639  }
 640  void Assembler::CSRRWI(GPR rd, CSR csr, uint32_t imm) noexcept {
 641      BISCUIT_ASSERT(imm <= 0x1F);
 642      EmitIType(m_buffer, static_cast<uint32_t>(csr), GPR{imm & 0x1F}, 0b101, rd, 0b1110011);
 643  }
 644  
 645  void Assembler::CSRR(GPR rd, CSR csr) noexcept {
 646      CSRRS(rd, csr, x0);
 647  }
 648  void Assembler::CSWR(CSR csr, GPR rs) noexcept {
 649      CSRRW(x0, csr, rs);
 650  }
 651  
 652  void Assembler::CSRS(CSR csr, GPR rs) noexcept {
 653      CSRRS(x0, csr, rs);
 654  }
 655  void Assembler::CSRC(CSR csr, GPR rs) noexcept {
 656      CSRRC(x0, csr, rs);
 657  }
 658  
 659  void Assembler::CSRCI(CSR csr, uint32_t imm) noexcept {
 660      CSRRCI(x0, csr, imm);
 661  }
 662  void Assembler::CSRSI(CSR csr, uint32_t imm) noexcept {
 663      CSRRSI(x0, csr, imm);
 664  }
 665  void Assembler::CSRWI(CSR csr, uint32_t imm) noexcept {
 666      CSRRWI(x0, csr, imm);
 667  }
 668  
 669  void Assembler::FRCSR(GPR rd) noexcept {
 670      CSRRS(rd, CSR::FCSR, x0);
 671  }
 672  void Assembler::FSCSR(GPR rd, GPR rs) noexcept {
 673      CSRRW(rd, CSR::FCSR, rs);
 674  }
 675  void Assembler::FSCSR(GPR rs) noexcept {
 676      CSRRW(x0, CSR::FCSR, rs);
 677  }
 678  
 679  void Assembler::FRRM(GPR rd) noexcept {
 680      CSRRS(rd, CSR::FRM, x0);
 681  }
 682  void Assembler::FSRM(GPR rd, GPR rs) noexcept {
 683      CSRRW(rd, CSR::FRM, rs);
 684  }
 685  void Assembler::FSRM(GPR rs) noexcept {
 686      CSRRW(x0, CSR::FRM, rs);
 687  }
 688  
 689  void Assembler::FSRMI(GPR rd, uint32_t imm) noexcept {
 690      CSRRWI(rd, CSR::FRM, imm);
 691  }
 692  void Assembler::FSRMI(uint32_t imm) noexcept {
 693      CSRRWI(x0, CSR::FRM, imm);
 694  }
 695  
 696  void Assembler::FRFLAGS(GPR rd) noexcept {
 697      CSRRS(rd, CSR::FFlags, x0);
 698  }
 699  void Assembler::FSFLAGS(GPR rd, GPR rs) noexcept {
 700      CSRRW(rd, CSR::FFlags, rs);
 701  }
 702  void Assembler::FSFLAGS(GPR rs) noexcept {
 703      CSRRW(x0, CSR::FFlags, rs);
 704  }
 705  
 706  void Assembler::FSFLAGSI(GPR rd, uint32_t imm) noexcept {
 707      CSRRWI(rd, CSR::FFlags, imm);
 708  }
 709  void Assembler::FSFLAGSI(uint32_t imm) noexcept {
 710      CSRRWI(x0, CSR::FFlags, imm);
 711  }
 712  
 713  void Assembler::RDCYCLE(GPR rd) noexcept {
 714      CSRRS(rd, CSR::Cycle, x0);
 715  }
 716  void Assembler::RDCYCLEH(GPR rd) noexcept {
 717      CSRRS(rd, CSR::CycleH, x0);
 718  }
 719  
 720  void Assembler::RDINSTRET(GPR rd) noexcept {
 721      CSRRS(rd, CSR::InstRet, x0);
 722  }
 723  void Assembler::RDINSTRETH(GPR rd) noexcept {
 724      CSRRS(rd, CSR::InstRetH, x0);
 725  }
 726  
 727  void Assembler::RDTIME(GPR rd) noexcept {
 728      CSRRS(rd, CSR::Time, x0);
 729  }
 730  void Assembler::RDTIMEH(GPR rd) noexcept {
 731      CSRRS(rd, CSR::TimeH, x0);
 732  }
 733  
 734  // Zihintntl Extension Instructions
 735  
 736  void Assembler::C_NTL_ALL() noexcept {
 737      C_ADD(x0, x5);
 738  }
 739  void Assembler::C_NTL_S1() noexcept {
 740      C_ADD(x0, x4);
 741  }
 742  void Assembler::C_NTL_P1() noexcept {
 743      C_ADD(x0, x2);
 744  }
 745  void Assembler::C_NTL_PALL() noexcept {
 746      C_ADD(x0, x3);
 747  }
 748  void Assembler::NTL_ALL() noexcept {
 749      ADD(x0, x0, x5);
 750  }
 751  void Assembler::NTL_S1() noexcept {
 752      ADD(x0, x0, x4);
 753  }
 754  void Assembler::NTL_P1() noexcept {
 755      ADD(x0, x0, x2);
 756  }
 757  void Assembler::NTL_PALL() noexcept {
 758      ADD(x0, x0, x3);
 759  }
 760  
 761  // RV32M Extension Instructions
 762  
 763  void Assembler::DIV(GPR rd, GPR rs1, GPR rs2) noexcept {
 764      EmitRType(m_buffer, 0b0000001, rs2, rs1, 0b100, rd, 0b0110011);
 765  }
 766  void Assembler::DIVU(GPR rd, GPR rs1, GPR rs2) noexcept {
 767      EmitRType(m_buffer, 0b0000001, rs2, rs1, 0b101, rd, 0b0110011);
 768  }
 769  void Assembler::MUL(GPR rd, GPR rs1, GPR rs2) noexcept {
 770      EmitRType(m_buffer, 0b0000001, rs2, rs1, 0b000, rd, 0b0110011);
 771  }
 772  void Assembler::MULH(GPR rd, GPR rs1, GPR rs2) noexcept {
 773      EmitRType(m_buffer, 0b0000001, rs2, rs1, 0b001, rd, 0b0110011);
 774  }
 775  void Assembler::MULHSU(GPR rd, GPR rs1, GPR rs2) noexcept {
 776      EmitRType(m_buffer, 0b0000001, rs2, rs1, 0b010, rd, 0b0110011);
 777  }
 778  void Assembler::MULHU(GPR rd, GPR rs1, GPR rs2) noexcept {
 779      EmitRType(m_buffer, 0b0000001, rs2, rs1, 0b011, rd, 0b0110011);
 780  }
 781  void Assembler::REM(GPR rd, GPR rs1, GPR rs2) noexcept {
 782      EmitRType(m_buffer, 0b0000001, rs2, rs1, 0b110, rd, 0b0110011);
 783  }
 784  void Assembler::REMU(GPR rd, GPR rs1, GPR rs2) noexcept {
 785      EmitRType(m_buffer, 0b0000001, rs2, rs1, 0b111, rd, 0b0110011);
 786  }
 787  
 788  // RV64M Extension Instructions
 789  
 790  void Assembler::DIVW(GPR rd, GPR rs1, GPR rs2) noexcept {
 791      EmitRType(m_buffer, 0b0000001, rs2, rs1, 0b100, rd, 0b0111011);
 792  }
 793  void Assembler::DIVUW(GPR rd, GPR rs1, GPR rs2) noexcept {
 794      EmitRType(m_buffer, 0b0000001, rs2, rs1, 0b101, rd, 0b0111011);
 795  }
 796  void Assembler::MULW(GPR rd, GPR rs1, GPR rs2) noexcept {
 797      EmitRType(m_buffer, 0b0000001, rs2, rs1, 0b000, rd, 0b0111011);
 798  }
 799  void Assembler::REMW(GPR rd, GPR rs1, GPR rs2) noexcept {
 800      EmitRType(m_buffer, 0b0000001, rs2, rs1, 0b110, rd, 0b0111011);
 801  }
 802  void Assembler::REMUW(GPR rd, GPR rs1, GPR rs2) noexcept {
 803      EmitRType(m_buffer, 0b0000001, rs2, rs1, 0b111, rd, 0b0111011);
 804  }
 805  
 806  // RV32A Extension Instructions
 807  
 808  void Assembler::AMOADD_W(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 809      EmitAtomic(m_buffer, 0b00000, ordering, rs2, rs1, 0b010, rd, 0b0101111);
 810  }
 811  void Assembler::AMOAND_W(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 812      EmitAtomic(m_buffer, 0b01100, ordering, rs2, rs1, 0b010, rd, 0b0101111);
 813  }
 814  void Assembler::AMOMAX_W(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 815      EmitAtomic(m_buffer, 0b10100, ordering, rs2, rs1, 0b010, rd, 0b0101111);
 816  }
 817  void Assembler::AMOMAXU_W(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 818      EmitAtomic(m_buffer, 0b11100, ordering, rs2, rs1, 0b010, rd, 0b0101111);
 819  }
 820  void Assembler::AMOMIN_W(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 821      EmitAtomic(m_buffer, 0b10000, ordering, rs2, rs1, 0b010, rd, 0b0101111);
 822  }
 823  void Assembler::AMOMINU_W(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 824      EmitAtomic(m_buffer, 0b11000, ordering, rs2, rs1, 0b010, rd, 0b0101111);
 825  }
 826  void Assembler::AMOOR_W(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 827      EmitAtomic(m_buffer, 0b01000, ordering, rs2, rs1, 0b010, rd, 0b0101111);
 828  }
 829  void Assembler::AMOSWAP_W(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 830      EmitAtomic(m_buffer, 0b00001, ordering, rs2, rs1, 0b010, rd, 0b0101111);
 831  }
 832  void Assembler::AMOXOR_W(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 833      EmitAtomic(m_buffer, 0b00100, ordering, rs2, rs1, 0b010, rd, 0b0101111);
 834  }
 835  void Assembler::LR_W(Ordering ordering, GPR rd, GPR rs) noexcept {
 836      EmitAtomic(m_buffer, 0b00010, ordering, x0, rs, 0b010, rd, 0b0101111);
 837  }
 838  void Assembler::SC_W(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 839      EmitAtomic(m_buffer, 0b00011, ordering, rs2, rs1, 0b010, rd, 0b0101111);
 840  }
 841  
 842  // RV64A Extension Instructions
 843  
 844  void Assembler::AMOADD_D(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 845      BISCUIT_ASSERT(IsRV64(m_features));
 846      EmitAtomic(m_buffer, 0b00000, ordering, rs2, rs1, 0b011, rd, 0b0101111);
 847  }
 848  void Assembler::AMOAND_D(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 849      BISCUIT_ASSERT(IsRV64(m_features));
 850      EmitAtomic(m_buffer, 0b01100, ordering, rs2, rs1, 0b011, rd, 0b0101111);
 851  }
 852  void Assembler::AMOMAX_D(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 853      BISCUIT_ASSERT(IsRV64(m_features));
 854      EmitAtomic(m_buffer, 0b10100, ordering, rs2, rs1, 0b011, rd, 0b0101111);
 855  }
 856  void Assembler::AMOMAXU_D(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 857      BISCUIT_ASSERT(IsRV64(m_features));
 858      EmitAtomic(m_buffer, 0b11100, ordering, rs2, rs1, 0b011, rd, 0b0101111);
 859  }
 860  void Assembler::AMOMIN_D(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 861      BISCUIT_ASSERT(IsRV64(m_features));
 862      EmitAtomic(m_buffer, 0b10000, ordering, rs2, rs1, 0b011, rd, 0b0101111);
 863  }
 864  void Assembler::AMOMINU_D(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 865      BISCUIT_ASSERT(IsRV64(m_features));
 866      EmitAtomic(m_buffer, 0b11000, ordering, rs2, rs1, 0b011, rd, 0b0101111);
 867  }
 868  void Assembler::AMOOR_D(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 869      BISCUIT_ASSERT(IsRV64(m_features));
 870      EmitAtomic(m_buffer, 0b01000, ordering, rs2, rs1, 0b011, rd, 0b0101111);
 871  }
 872  void Assembler::AMOSWAP_D(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 873      BISCUIT_ASSERT(IsRV64(m_features));
 874      EmitAtomic(m_buffer, 0b00001, ordering, rs2, rs1, 0b011, rd, 0b0101111);
 875  }
 876  void Assembler::AMOXOR_D(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 877      BISCUIT_ASSERT(IsRV64(m_features));
 878      EmitAtomic(m_buffer, 0b00100, ordering, rs2, rs1, 0b011, rd, 0b0101111);
 879  }
 880  void Assembler::LR_D(Ordering ordering, GPR rd, GPR rs) noexcept {
 881      BISCUIT_ASSERT(IsRV64(m_features));
 882      EmitAtomic(m_buffer, 0b00010, ordering, x0, rs, 0b011, rd, 0b0101111);
 883  }
 884  void Assembler::SC_D(Ordering ordering, GPR rd, GPR rs2, GPR rs1) noexcept {
 885      BISCUIT_ASSERT(IsRV64(m_features));
 886      EmitAtomic(m_buffer, 0b00011, ordering, rs2, rs1, 0b011, rd, 0b0101111);
 887  }
 888  
 889  // RVB Extension Instructions
 890  
 891  void Assembler::ADDUW(GPR rd, GPR rs1, GPR rs2) noexcept {
 892      BISCUIT_ASSERT(IsRV64(m_features));
 893      EmitRType(m_buffer, 0b0000100, rs2, rs1, 0b000, rd, 0b0111011);
 894  }
 895  
 896  void Assembler::ANDN(GPR rd, GPR rs1, GPR rs2) noexcept {
 897      EmitRType(m_buffer, 0b0100000, rs2, rs1, 0b111, rd, 0b0110011);
 898  }
 899  
 900  void Assembler::BCLR(GPR rd, GPR rs1, GPR rs2) noexcept {
 901      EmitRType(m_buffer, 0b0100100, rs2, rs1, 0b001, rd, 0b0110011);
 902  }
 903  
 904  void Assembler::BCLRI(GPR rd, GPR rs, uint32_t bit) noexcept {
 905      if (IsRV32(m_features)) {
 906          BISCUIT_ASSERT(bit <= 31);
 907      } else {
 908          BISCUIT_ASSERT(bit <= 63);
 909      }
 910  
 911      const auto imm = (0b010010U << 6) | bit;
 912      EmitIType(m_buffer, imm, rs, 0b001, rd, 0b0010011);
 913  }
 914  
 915  void Assembler::BEXT(GPR rd, GPR rs1, GPR rs2) noexcept {
 916      EmitRType(m_buffer, 0b0100100, rs2, rs1, 0b101, rd, 0b0110011);
 917  }
 918  
 919  void Assembler::BEXTI(GPR rd, GPR rs, uint32_t bit) noexcept {
 920      if (IsRV32(m_features)) {
 921          BISCUIT_ASSERT(bit <= 31);
 922      } else {
 923          BISCUIT_ASSERT(bit <= 63);
 924      }
 925  
 926      const auto imm = (0b010010U << 6) | bit;
 927      EmitIType(m_buffer, imm, rs, 0b101, rd, 0b0010011);
 928  }
 929  
 930  void Assembler::BINV(GPR rd, GPR rs1, GPR rs2) noexcept {
 931      EmitRType(m_buffer, 0b0110100, rs2, rs1, 0b001, rd, 0b0110011);
 932  }
 933  
 934  void Assembler::BINVI(GPR rd, GPR rs, uint32_t bit) noexcept {
 935      if (IsRV32(m_features)) {
 936          BISCUIT_ASSERT(bit <= 31);
 937      } else {
 938          BISCUIT_ASSERT(bit <= 63);
 939      }
 940  
 941      const auto imm = (0b011010U << 6) | bit;
 942      EmitIType(m_buffer, imm, rs, 0b001, rd, 0b0010011);
 943  }
 944  
 945  void Assembler::BREV8(GPR rd, GPR rs) noexcept {
 946      EmitIType(m_buffer, 0b011010000111, rs, 0b101, rd, 0b0010011);
 947  }
 948  
 949  void Assembler::BSET(GPR rd, GPR rs1, GPR rs2) noexcept {
 950      EmitRType(m_buffer, 0b0010100, rs2, rs1, 0b001, rd, 0b0110011);
 951  }
 952  
 953  void Assembler::BSETI(GPR rd, GPR rs, uint32_t bit) noexcept {
 954      if (IsRV32(m_features)) {
 955          BISCUIT_ASSERT(bit <= 31);
 956      } else {
 957          BISCUIT_ASSERT(bit <= 63);
 958      }
 959  
 960      const auto imm = (0b001010U << 6) | bit;
 961      EmitIType(m_buffer, imm, rs, 0b001, rd, 0b0110011);
 962  }
 963  
 964  void Assembler::CLMUL(GPR rd, GPR rs1, GPR rs2) noexcept {
 965      EmitRType(m_buffer, 0b0000101, rs2, rs1, 0b001, rd, 0b0110011);
 966  }
 967  
 968  void Assembler::CLMULH(GPR rd, GPR rs1, GPR rs2) noexcept {
 969      EmitRType(m_buffer, 0b0000101, rs2, rs1, 0b011, rd, 0b0110011);
 970  }
 971  
 972  void Assembler::CLMULR(GPR rd, GPR rs1, GPR rs2) noexcept {
 973      EmitRType(m_buffer, 0b0000101, rs2, rs1, 0b010, rd, 0b0110011);
 974  }
 975  
 976  void Assembler::CLZ(GPR rd, GPR rs) noexcept {
 977      EmitIType(m_buffer, 0b011000000000, rs, 0b001, rd, 0b0010011);
 978  }
 979  
 980  void Assembler::CLZW(GPR rd, GPR rs) noexcept {
 981      BISCUIT_ASSERT(IsRV64(m_features));
 982      EmitIType(m_buffer, 0b011000000000, rs, 0b001, rd, 0b0011011);
 983  }
 984  
 985  void Assembler::CPOP(GPR rd, GPR rs) noexcept {
 986      EmitIType(m_buffer, 0b011000000010, rs, 0b001, rd, 0b0010011);
 987  }
 988  
 989  void Assembler::CPOPW(GPR rd, GPR rs) noexcept {
 990      BISCUIT_ASSERT(IsRV64(m_features));
 991      EmitIType(m_buffer, 0b011000000010, rs, 0b001, rd, 0b0011011);
 992  }
 993  
 994  void Assembler::CTZ(GPR rd, GPR rs) noexcept {
 995      EmitIType(m_buffer, 0b011000000001, rs, 0b001, rd, 0b0010011);
 996  }
 997  
 998  void Assembler::CTZW(GPR rd, GPR rs) noexcept {
 999      BISCUIT_ASSERT(IsRV64(m_features));
1000      EmitIType(m_buffer, 0b011000000001, rs, 0b001, rd, 0b0011011);
1001  }
1002  
1003  void Assembler::MAX(GPR rd, GPR rs1, GPR rs2) noexcept {
1004      EmitRType(m_buffer, 0b0000101, rs2, rs1, 0b110, rd, 0b0110011);
1005  }
1006  
1007  void Assembler::MAXU(GPR rd, GPR rs1, GPR rs2) noexcept {
1008      EmitRType(m_buffer, 0b0000101, rs2, rs1, 0b111, rd, 0b0110011);
1009  }
1010  
1011  void Assembler::MIN(GPR rd, GPR rs1, GPR rs2) noexcept {
1012      EmitRType(m_buffer, 0b0000101, rs2, rs1, 0b100, rd, 0b0110011);
1013  }
1014  
1015  void Assembler::MINU(GPR rd, GPR rs1, GPR rs2) noexcept {
1016      EmitRType(m_buffer, 0b0000101, rs2, rs1, 0b101, rd, 0b0110011);
1017  }
1018  
1019  void Assembler::ORCB(GPR rd, GPR rs) noexcept {
1020      EmitIType(m_buffer, 0b001010000111, rs, 0b101, rd, 0b0010011);
1021  }
1022  
1023  void Assembler::ORN(GPR rd, GPR rs1, GPR rs2) noexcept {
1024      EmitRType(m_buffer, 0b0100000, rs2, rs1, 0b110, rd, 0b0110011);
1025  }
1026  
1027  void Assembler::PACK(GPR rd, GPR rs1, GPR rs2) noexcept {
1028      EmitRType(m_buffer, 0b0000100, rs2, rs1, 0b100, rd, 0b0110011);
1029  }
1030  
1031  void Assembler::PACKH(GPR rd, GPR rs1, GPR rs2) noexcept {
1032      EmitRType(m_buffer, 0b0000100, rs2, rs1, 0b111, rd, 0b0110011);
1033  }
1034  
1035  void Assembler::PACKW(GPR rd, GPR rs1, GPR rs2) noexcept {
1036      BISCUIT_ASSERT(IsRV64(m_features));
1037      EmitRType(m_buffer, 0b0000100, rs2, rs1, 0b100, rd, 0b0111011);
1038  }
1039  
1040  void Assembler::REV8(GPR rd, GPR rs) noexcept {
1041      if (IsRV32(m_features)) {
1042          EmitIType(m_buffer, 0b011010011000, rs, 0b101, rd, 0b0010011);
1043      } else {
1044          EmitIType(m_buffer, 0b011010111000, rs, 0b101, rd, 0b0010011);
1045      }
1046  }
1047  
1048  void Assembler::ROL(GPR rd, GPR rs1, GPR rs2) noexcept {
1049      EmitRType(m_buffer, 0b0110000, rs2, rs1, 0b001, rd, 0b0110011);
1050  }
1051  
1052  void Assembler::ROLW(GPR rd, GPR rs1, GPR rs2) noexcept {
1053      BISCUIT_ASSERT(IsRV64(m_features));
1054      EmitRType(m_buffer, 0b0110000, rs2, rs1, 0b001, rd, 0b0111011);
1055  }
1056  
1057  void Assembler::ROR(GPR rd, GPR rs1, GPR rs2) noexcept {
1058      EmitRType(m_buffer, 0b0110000, rs2, rs1, 0b101, rd, 0b0110011);
1059  }
1060  
1061  void Assembler::RORI(GPR rd, GPR rs, uint32_t rotate_amount) noexcept {
1062      BISCUIT_ASSERT(rotate_amount <= 63);
1063      const auto imm = (0b011000U << 6) | rotate_amount;
1064      EmitIType(m_buffer, imm, rs, 0b101, rd, 0b0010011);
1065  }
1066  
1067  void Assembler::RORIW(GPR rd, GPR rs, uint32_t rotate_amount) noexcept {
1068      BISCUIT_ASSERT(IsRV64(m_features));
1069      BISCUIT_ASSERT(rotate_amount <= 63);
1070      const auto imm = (0b011000U << 6) | rotate_amount;
1071      EmitIType(m_buffer, imm, rs, 0b101, rd, 0b0011011);
1072  }
1073  
1074  void Assembler::RORW(GPR rd, GPR rs1, GPR rs2) noexcept {
1075      BISCUIT_ASSERT(IsRV64(m_features));
1076      EmitRType(m_buffer, 0b0110000, rs2, rs1, 0b101, rd, 0b0111011);
1077  }
1078  
1079  void Assembler::SEXTB(GPR rd, GPR rs) noexcept {
1080      EmitIType(m_buffer, 0b011000000100, rs, 0b001, rd, 0b0010011);
1081  }
1082  
1083  void Assembler::SEXTH(GPR rd, GPR rs) noexcept {
1084      EmitIType(m_buffer, 0b011000000101, rs, 0b001, rd, 0b0010011);
1085  }
1086  
1087  void Assembler::SH1ADD(GPR rd, GPR rs1, GPR rs2) noexcept {
1088      EmitRType(m_buffer, 0b0010000, rs2, rs1, 0b010, rd, 0b0110011);
1089  }
1090  
1091  void Assembler::SH1ADDUW(GPR rd, GPR rs1, GPR rs2) noexcept {
1092      BISCUIT_ASSERT(IsRV64(m_features));
1093      EmitRType(m_buffer, 0b0010000, rs2, rs1, 0b010, rd, 0b0111011);
1094  }
1095  
1096  void Assembler::SH2ADD(GPR rd, GPR rs1, GPR rs2) noexcept {
1097      EmitRType(m_buffer, 0b0010000, rs2, rs1, 0b100, rd, 0b0110011);
1098  }
1099  
1100  void Assembler::SH2ADDUW(GPR rd, GPR rs1, GPR rs2) noexcept {
1101      BISCUIT_ASSERT(IsRV64(m_features));
1102      EmitRType(m_buffer, 0b0010000, rs2, rs1, 0b100, rd, 0b0111011);
1103  }
1104  
1105  void Assembler::SH3ADD(GPR rd, GPR rs1, GPR rs2) noexcept {
1106      EmitRType(m_buffer, 0b0010000, rs2, rs1, 0b110, rd, 0b0110011);
1107  }
1108  
1109  void Assembler::SH3ADDUW(GPR rd, GPR rs1, GPR rs2) noexcept {
1110      BISCUIT_ASSERT(IsRV64(m_features));
1111      EmitRType(m_buffer, 0b0010000, rs2, rs1, 0b110, rd, 0b0111011);
1112  }
1113  
1114  void Assembler::SLLIUW(GPR rd, GPR rs, uint32_t shift_amount) noexcept {
1115      BISCUIT_ASSERT(IsRV64(m_features));
1116      BISCUIT_ASSERT(shift_amount <= 63);
1117      const auto imm = (0b000010U << 6) | shift_amount;
1118      EmitIType(m_buffer, imm, rs, 0b001, rd, 0b0011011);
1119  }
1120  
1121  void Assembler::UNZIP(GPR rd, GPR rs) noexcept {
1122      BISCUIT_ASSERT(IsRV32(m_features));
1123      EmitIType(m_buffer, 0b000010011111, rs, 0b101, rd, 0b0010011);
1124  }
1125  
1126  void Assembler::XNOR(GPR rd, GPR rs1, GPR rs2) noexcept {
1127      EmitRType(m_buffer, 0b0100000, rs2, rs1, 0b100, rd, 0b0110011);
1128  }
1129  
1130  void Assembler::XPERM4(GPR rd, GPR rs1, GPR rs2) noexcept {
1131      EmitRType(m_buffer, 0b0010100, rs2, rs1, 0b010, rd, 0b0110011);
1132  }
1133  
1134  void Assembler::XPERM8(GPR rd, GPR rs1, GPR rs2) noexcept {
1135      EmitRType(m_buffer, 0b0010100, rs2, rs1, 0b100, rd, 0b0110011);
1136  }
1137  
1138  void Assembler::ZEXTH(GPR rd, GPR rs) noexcept {
1139      if (IsRV32(m_features)) {
1140          EmitIType(m_buffer, 0b000010000000, rs, 0b100, rd, 0b0110011);
1141      } else {
1142          EmitIType(m_buffer, 0b000010000000, rs, 0b100, rd, 0b0111011);
1143      }
1144  }
1145  
1146  void Assembler::ZEXTW(GPR rd, GPR rs) noexcept {
1147      ADDUW(rd, rs, x0);
1148  }
1149  
1150  void Assembler::ZIP(GPR rd, GPR rs) noexcept {
1151      BISCUIT_ASSERT(IsRV32(m_features));
1152      EmitIType(m_buffer, 0b000010011110, rs, 0b001, rd, 0b0010011);
1153  }
1154  
1155  // Cache Management Operation Extension Instructions
1156  
1157  void Assembler::CBO_CLEAN(GPR rs) noexcept {
1158      EmitRType(m_buffer, 0b0000000, x1, rs, 0b010, x0, 0b0001111);
1159  }
1160  
1161  void Assembler::CBO_FLUSH(GPR rs) noexcept {
1162      EmitRType(m_buffer, 0b0000000, x2, rs, 0b010, x0, 0b0001111);
1163  }
1164  
1165  void Assembler::CBO_INVAL(GPR rs) noexcept {
1166      EmitRType(m_buffer, 0b0000000, x0, rs, 0b010, x0, 0b0001111);
1167  }
1168  
1169  void Assembler::CBO_ZERO(GPR rs) noexcept {
1170      EmitRType(m_buffer, 0b0000000, x4, rs, 0b010, x0, 0b0001111);
1171  }
1172  
1173  void Assembler::PREFETCH_I(GPR rs, int32_t offset) noexcept {
1174      // Offset must be able to fit in a 12-bit signed immediate and be
1175      // cleanly divisible by 32 since the bottom 5 bits are encoded as zero.
1176      BISCUIT_ASSERT(IsValidSigned12BitImm(offset));
1177      BISCUIT_ASSERT(offset % 32 == 0);
1178      EmitIType(m_buffer, static_cast<uint32_t>(offset), rs, 0b110, x0, 0b0010011);
1179  }
1180  
1181  void Assembler::PREFETCH_R(GPR rs, int32_t offset) noexcept {
1182      // Offset must be able to fit in a 12-bit signed immediate and be
1183      // cleanly divisible by 32 since the bottom 5 bits are encoded as zero.
1184      BISCUIT_ASSERT(IsValidSigned12BitImm(offset));
1185      BISCUIT_ASSERT(offset % 32 == 0);
1186      EmitIType(m_buffer, static_cast<uint32_t>(offset) | 0b01, rs, 0b110, x0, 0b0010011);
1187  }
1188  
1189  void Assembler::PREFETCH_W(GPR rs, int32_t offset) noexcept {
1190      // Offset must be able to fit in a 12-bit signed immediate and be
1191      // cleanly divisible by 32 since the bottom 5 bits are encoded as zero.
1192      BISCUIT_ASSERT(IsValidSigned12BitImm(offset));
1193      BISCUIT_ASSERT(offset % 32 == 0);
1194      EmitIType(m_buffer, static_cast<uint32_t>(offset) | 0b11, rs, 0b110, x0, 0b0010011);
1195  }
1196  
1197  // Privileged Instructions
1198  
1199  void Assembler::HFENCE_GVMA(GPR rs1, GPR rs2) noexcept {
1200      EmitRType(m_buffer, 0b0110001, rs2, rs1, 0b000, x0, 0b1110011);
1201  }
1202  
1203  void Assembler::HFENCE_VVMA(GPR rs1, GPR rs2) noexcept {
1204      EmitRType(m_buffer, 0b0010001, rs2, rs1, 0b000, x0, 0b1110011);
1205  }
1206  
1207  void Assembler::HINVAL_GVMA(GPR rs1, GPR rs2) noexcept {
1208      EmitRType(m_buffer, 0b0110011, rs2, rs1, 0b000, x0, 0b1110011);
1209  }
1210  
1211  void Assembler::HINVAL_VVMA(GPR rs1, GPR rs2) noexcept {
1212      EmitRType(m_buffer, 0b0010011, rs2, rs1, 0b000, x0, 0b1110011);
1213  }
1214  
1215  void Assembler::HLV_B(GPR rd, GPR rs) noexcept {
1216      EmitRType(m_buffer, 0b0110000, x0, rs, 0b100, rd, 0b1110011);
1217  }
1218  
1219  void Assembler::HLV_BU(GPR rd, GPR rs) noexcept {
1220      EmitRType(m_buffer, 0b0110000, x1, rs, 0b100, rd, 0b1110011);
1221  }
1222  
1223  void Assembler::HLV_D(GPR rd, GPR rs) noexcept {
1224      BISCUIT_ASSERT(IsRV64(m_features));
1225      EmitRType(m_buffer, 0b0110110, x0, rs, 0b100, rd, 0b1110011);
1226  }
1227  
1228  void Assembler::HLV_H(GPR rd, GPR rs) noexcept {
1229      EmitRType(m_buffer, 0b0110010, x0, rs, 0b100, rd, 0b1110011);
1230  }
1231  
1232  void Assembler::HLV_HU(GPR rd, GPR rs) noexcept {
1233      EmitRType(m_buffer, 0b0110010, x1, rs, 0b100, rd, 0b1110011);
1234  }
1235  
1236  void Assembler::HLV_W(GPR rd, GPR rs) noexcept {
1237      EmitRType(m_buffer, 0b0110100, x0, rs, 0b100, rd, 0b1110011);
1238  }
1239  
1240  void Assembler::HLV_WU(GPR rd, GPR rs) noexcept {
1241      BISCUIT_ASSERT(IsRV64(m_features));
1242      EmitRType(m_buffer, 0b0110100, x1, rs, 0b100, rd, 0b1110011);
1243  }
1244  
1245  void Assembler::HLVX_HU(GPR rd, GPR rs) noexcept {
1246      EmitRType(m_buffer, 0b0110010, x3, rs, 0b100, rd, 0b1110011);
1247  }
1248  
1249  void Assembler::HLVX_WU(GPR rd, GPR rs) noexcept {
1250      EmitRType(m_buffer, 0b0110100, x3, rs, 0b100, rd, 0b1110011);
1251  }
1252  
1253  void Assembler::HSV_B(GPR rs2, GPR rs1) noexcept {
1254      EmitRType(m_buffer, 0b0110001, rs2, rs1, 0b100, x0, 0b1110011);
1255  }
1256  
1257  void Assembler::HSV_D(GPR rs2, GPR rs1) noexcept {
1258      BISCUIT_ASSERT(IsRV64(m_features));
1259      EmitRType(m_buffer, 0b0110111, rs2, rs1, 0b100, x0, 0b1110011);
1260  }
1261  
1262  void Assembler::HSV_H(GPR rs2, GPR rs1) noexcept {
1263      EmitRType(m_buffer, 0b0110011, rs2, rs1, 0b100, x0, 0b1110011);
1264  }
1265  
1266  void Assembler::HSV_W(GPR rs2, GPR rs1) noexcept {
1267      EmitRType(m_buffer, 0b0110101, rs2, rs1, 0b100, x0, 0b1110011);
1268  }
1269  
1270  void Assembler::MRET() noexcept {
1271      m_buffer.Emit32(0x30200073);
1272  }
1273  
1274  void Assembler::SFENCE_INVAL_IR() noexcept {
1275      m_buffer.Emit32(0x18100073U);
1276  }
1277  
1278  void Assembler::SFENCE_VMA(GPR rs1, GPR rs2) noexcept {
1279      EmitRType(m_buffer, 0b0001001, rs2, rs1, 0b000, x0, 0b1110011);
1280  }
1281  
1282  void Assembler::SFENCE_W_INVAL() noexcept {
1283      m_buffer.Emit32(0x18000073U);
1284  }
1285  
1286  void Assembler::SINVAL_VMA(GPR rs1, GPR rs2) noexcept {
1287      EmitRType(m_buffer, 0b0001011, rs2, rs1, 0b000, x0, 0b1110011);
1288  }
1289  
1290  void Assembler::SRET() noexcept {
1291      m_buffer.Emit32(0x10200073);
1292  }
1293  
1294  void Assembler::URET() noexcept {
1295      m_buffer.Emit32(0x00200073);
1296  }
1297  
1298  void Assembler::WFI() noexcept {
1299      m_buffer.Emit32(0x10500073);
1300  }
1301  
1302  void Assembler::BindToOffset(Label* label, Label::LocationOffset offset) {
1303      BISCUIT_ASSERT(label != nullptr);
1304      BISCUIT_ASSERT(offset >= 0 && offset <= m_buffer.GetCursorOffset());
1305  
1306      label->Bind(offset);
1307      ResolveLabelOffsets(label);
1308      label->ClearOffsets();
1309  }
1310  
1311  ptrdiff_t Assembler::LinkAndGetOffset(Label* label) {
1312      BISCUIT_ASSERT(label != nullptr);
1313  
1314      // If we have a bound label, then it's straightforward to calculate
1315      // the offsets.
1316      if (label->IsBound()) {
1317          const auto cursor_address = m_buffer.GetCursorAddress();
1318          const auto label_offset = m_buffer.GetOffsetAddress(*label->GetLocation());
1319          return static_cast<ptrdiff_t>(label_offset - cursor_address);
1320      }
1321  
1322      // If we don't have a bound location, we return an offset of zero.
1323      // While the emitter will emit a bogus branch instruction initially,
1324      // the offset will be patched over once the label has been properly
1325      // bound to a location.
1326      label->AddOffset(m_buffer.GetCursorOffset());
1327      return 0;
1328  }
1329  
1330  void Assembler::ResolveLabelOffsets(Label* label) {
1331      // Conditional branch instructions make use of the B-type immediate encoding for offsets.
1332      const auto is_b_type = [](uint32_t instruction) {
1333          return (instruction & 0x7F) == 0b1100011;
1334      };
1335      // JAL makes use of the J-type immediate encoding for offsets.
1336      const auto is_j_type = [](uint32_t instruction) {
1337          return (instruction & 0x7F) == 0b1101111;
1338      };
1339      // C.BEQZ and C.BNEZ make use of this encoding type.
1340      const auto is_cb_type = [](uint32_t instruction) {
1341          const auto op = instruction & 0b11;
1342          const auto funct3 = instruction & 0xE000;
1343          return op == 0b01 && funct3 >= 0xC000;
1344      };
1345      // C.JAL and C.J make use of this encoding type.
1346      const auto is_cj_type = [](uint32_t instruction) {
1347          const auto op = instruction & 0b11;
1348          const auto funct3 = instruction & 0xE000;
1349          return op == 0b01 && (funct3 == 0x2000 || funct3 == 0xA000);
1350      };
1351      // If we know an instruction is a compressed branch, then it's a 16-bit instruction
1352      // Otherwise it's a regular-sized 32-bit instruction.
1353      const auto determine_inst_size = [&](uint32_t instruction) -> size_t {
1354          if (is_cj_type(instruction) || is_cb_type(instruction)) {
1355              return 2;
1356          } else {
1357              return 4;
1358          }
1359      };
1360  
1361      const auto label_location = *label->GetLocation();
1362  
1363      for (const auto offset : label->m_offsets) {
1364          const auto address = m_buffer.GetOffsetAddress(offset);
1365          auto* const ptr = reinterpret_cast<uint8_t*>(address);
1366          const auto inst_size = determine_inst_size(uint32_t{*ptr} | (uint32_t{*(ptr + 1)} << 8));
1367  
1368          uint32_t instruction = 0;
1369          std::memcpy(&instruction, ptr, inst_size);
1370  
1371          // Given all branch instructions we need to patch have 0 encoded as
1372          // their branch offset, we don't need to worry about any masking work.
1373          //
1374          // It's enough to verify that the immediate is going to be valid
1375          // and then OR it into the instruction.
1376  
1377          const auto encoded_offset = label_location - offset;
1378  
1379          if (inst_size == sizeof(uint32_t)) {
1380              if (is_b_type(instruction)) {
1381                  BISCUIT_ASSERT(IsValidBTypeImm(encoded_offset));
1382                  instruction |= TransformToBTypeImm(static_cast<uint32_t>(encoded_offset));
1383              } else if (is_j_type(instruction)) {
1384                  BISCUIT_ASSERT(IsValidJTypeImm(encoded_offset));
1385                  instruction |= TransformToJTypeImm(static_cast<uint32_t>(encoded_offset));
1386              }
1387          } else {
1388              if (is_cb_type(instruction)) {
1389                  BISCUIT_ASSERT(IsValidCBTypeImm(encoded_offset));
1390                  instruction |= TransformToCBTypeImm(static_cast<uint32_t>(encoded_offset));
1391              } else if (is_cj_type(instruction)) {
1392                  BISCUIT_ASSERT(IsValidCJTypeImm(encoded_offset));
1393                  instruction |= TransformToCJTypeImm(static_cast<uint32_t>(encoded_offset));
1394              }
1395          }
1396  
1397          std::memcpy(ptr, &instruction, inst_size);
1398      }
1399  }
1400  
1401  } // namespace biscuit