1  using ARMeilleure.CodeGen;
   2  using ARMeilleure.CodeGen.Linking;
   3  using ARMeilleure.CodeGen.Unwinding;
   4  using ARMeilleure.Common;
   5  using ARMeilleure.Memory;
   6  using Ryujinx.Common;
   7  using Ryujinx.Common.Configuration;
   8  using Ryujinx.Common.Logging;
   9  using Ryujinx.Common.Memory;
  10  using System;
  11  using System.Buffers.Binary;
  12  using System.Collections.Generic;
  13  using System.Diagnostics;
  14  using System.IO;
  15  using System.IO.Compression;
  16  using System.Runtime;
  17  using System.Runtime.CompilerServices;
  18  using System.Runtime.InteropServices;
  19  using System.Threading;
  20  using static ARMeilleure.Translation.PTC.PtcFormatter;
  21  
  22  namespace ARMeilleure.Translation.PTC
  23  {
  24      using Arm64HardwareCapabilities = CodeGen.Arm64.HardwareCapabilities;
  25      using X86HardwareCapabilities = CodeGen.X86.HardwareCapabilities;
  26  
  27      class Ptc : IPtcLoadState
  28      {
  29          private const string OuterHeaderMagicString = "PTCohd\0\0";
  30          private const string InnerHeaderMagicString = "PTCihd\0\0";
  31  
  32          private const uint InternalVersion = 6950; //! To be incremented manually for each change to the ARMeilleure project.
  33  
  34          private const string ActualDir = "0";
  35          private const string BackupDir = "1";
  36  
  37          private const string TitleIdTextDefault = "0000000000000000";
  38          private const string DisplayVersionDefault = "0";
  39  
  40          public static readonly Symbol PageTableSymbol = new(SymbolType.Special, 1);
  41          public static readonly Symbol CountTableSymbol = new(SymbolType.Special, 2);
  42          public static readonly Symbol DispatchStubSymbol = new(SymbolType.Special, 3);
  43  
  44          private const byte FillingByte = 0x00;
  45          private const CompressionLevel SaveCompressionLevel = CompressionLevel.Fastest;
  46  
  47          public PtcProfiler Profiler { get; }
  48  
  49          // Carriers.
  50          private MemoryStream _infosStream;
  51          private List<byte[]> _codesList;
  52          private MemoryStream _relocsStream;
  53          private MemoryStream _unwindInfosStream;
  54  
  55          private readonly ulong _outerHeaderMagic;
  56          private readonly ulong _innerHeaderMagic;
  57  
  58          private readonly ManualResetEvent _waitEvent;
  59  
  60          private readonly object _lock;
  61  
  62          private bool _disposed;
  63  
  64          public string TitleIdText { get; private set; }
  65          public string DisplayVersion { get; private set; }
  66  
  67          private MemoryManagerType _memoryMode;
  68  
  69          public string CachePathActual { get; private set; }
  70          public string CachePathBackup { get; private set; }
  71  
  72          public PtcState State { get; private set; }
  73  
  74          // Progress reporting helpers.
  75          private volatile int _translateCount;
  76          private volatile int _translateTotalCount;
  77          public event Action<PtcLoadingState, int, int> PtcStateChanged;
  78  
  79          public Ptc()
  80          {
  81              Profiler = new PtcProfiler(this);
  82  
  83              InitializeCarriers();
  84  
  85              _outerHeaderMagic = BinaryPrimitives.ReadUInt64LittleEndian(EncodingCache.UTF8NoBOM.GetBytes(OuterHeaderMagicString).AsSpan());
  86              _innerHeaderMagic = BinaryPrimitives.ReadUInt64LittleEndian(EncodingCache.UTF8NoBOM.GetBytes(InnerHeaderMagicString).AsSpan());
  87  
  88              _waitEvent = new ManualResetEvent(true);
  89  
  90              _lock = new object();
  91  
  92              _disposed = false;
  93  
  94              TitleIdText = TitleIdTextDefault;
  95              DisplayVersion = DisplayVersionDefault;
  96  
  97              CachePathActual = string.Empty;
  98              CachePathBackup = string.Empty;
  99  
 100              Disable();
 101          }
 102  
 103          public void Initialize(string titleIdText, string displayVersion, bool enabled, MemoryManagerType memoryMode)
 104          {
 105              Wait();
 106  
 107              Profiler.Wait();
 108              Profiler.ClearEntries();
 109  
 110              Logger.Info?.Print(LogClass.Ptc, $"Initializing Profiled Persistent Translation Cache (enabled: {enabled}).");
 111  
 112              if (!enabled || string.IsNullOrEmpty(titleIdText) || titleIdText == TitleIdTextDefault)
 113              {
 114                  TitleIdText = TitleIdTextDefault;
 115                  DisplayVersion = DisplayVersionDefault;
 116  
 117                  CachePathActual = string.Empty;
 118                  CachePathBackup = string.Empty;
 119  
 120                  Disable();
 121  
 122                  return;
 123              }
 124  
 125              TitleIdText = titleIdText;
 126              DisplayVersion = !string.IsNullOrEmpty(displayVersion) ? displayVersion : DisplayVersionDefault;
 127              _memoryMode = memoryMode;
 128  
 129              string workPathActual = Path.Combine(AppDataManager.GamesDirPath, TitleIdText, "cache", "cpu", ActualDir);
 130              string workPathBackup = Path.Combine(AppDataManager.GamesDirPath, TitleIdText, "cache", "cpu", BackupDir);
 131  
 132              if (!Directory.Exists(workPathActual))
 133              {
 134                  Directory.CreateDirectory(workPathActual);
 135              }
 136  
 137              if (!Directory.Exists(workPathBackup))
 138              {
 139                  Directory.CreateDirectory(workPathBackup);
 140              }
 141  
 142              CachePathActual = Path.Combine(workPathActual, DisplayVersion);
 143              CachePathBackup = Path.Combine(workPathBackup, DisplayVersion);
 144  
 145              PreLoad();
 146              Profiler.PreLoad();
 147  
 148              Enable();
 149          }
 150  
 151          private void InitializeCarriers()
 152          {
 153              _infosStream = MemoryStreamManager.Shared.GetStream();
 154              _codesList = new List<byte[]>();
 155              _relocsStream = MemoryStreamManager.Shared.GetStream();
 156              _unwindInfosStream = MemoryStreamManager.Shared.GetStream();
 157          }
 158  
 159          private void DisposeCarriers()
 160          {
 161              _infosStream.Dispose();
 162              _codesList.Clear();
 163              _relocsStream.Dispose();
 164              _unwindInfosStream.Dispose();
 165          }
 166  
 167          private bool AreCarriersEmpty()
 168          {
 169              return _infosStream.Length == 0L && _codesList.Count == 0 && _relocsStream.Length == 0L && _unwindInfosStream.Length == 0L;
 170          }
 171  
 172          private void ResetCarriersIfNeeded()
 173          {
 174              if (AreCarriersEmpty())
 175              {
 176                  return;
 177              }
 178  
 179              DisposeCarriers();
 180  
 181              InitializeCarriers();
 182          }
 183  
 184          private void PreLoad()
 185          {
 186              string fileNameActual = $"{CachePathActual}.cache";
 187              string fileNameBackup = $"{CachePathBackup}.cache";
 188  
 189              FileInfo fileInfoActual = new(fileNameActual);
 190              FileInfo fileInfoBackup = new(fileNameBackup);
 191  
 192              if (fileInfoActual.Exists && fileInfoActual.Length != 0L)
 193              {
 194                  if (!Load(fileNameActual, false))
 195                  {
 196                      if (fileInfoBackup.Exists && fileInfoBackup.Length != 0L)
 197                      {
 198                          Load(fileNameBackup, true);
 199                      }
 200                  }
 201              }
 202              else if (fileInfoBackup.Exists && fileInfoBackup.Length != 0L)
 203              {
 204                  Load(fileNameBackup, true);
 205              }
 206          }
 207  
 208          private unsafe bool Load(string fileName, bool isBackup)
 209          {
 210              using (FileStream compressedStream = new(fileName, FileMode.Open))
 211              using (DeflateStream deflateStream = new(compressedStream, CompressionMode.Decompress, true))
 212              {
 213                  OuterHeader outerHeader = DeserializeStructure<OuterHeader>(compressedStream);
 214  
 215                  if (!outerHeader.IsHeaderValid())
 216                  {
 217                      InvalidateCompressedStream(compressedStream);
 218  
 219                      return false;
 220                  }
 221  
 222                  if (outerHeader.Magic != _outerHeaderMagic)
 223                  {
 224                      InvalidateCompressedStream(compressedStream);
 225  
 226                      return false;
 227                  }
 228  
 229                  if (outerHeader.CacheFileVersion != InternalVersion)
 230                  {
 231                      InvalidateCompressedStream(compressedStream);
 232  
 233                      return false;
 234                  }
 235  
 236                  if (outerHeader.Endianness != GetEndianness())
 237                  {
 238                      InvalidateCompressedStream(compressedStream);
 239  
 240                      return false;
 241                  }
 242  
 243                  if (outerHeader.FeatureInfo != GetFeatureInfo())
 244                  {
 245                      InvalidateCompressedStream(compressedStream);
 246  
 247                      return false;
 248                  }
 249  
 250                  if (outerHeader.MemoryManagerMode != GetMemoryManagerMode())
 251                  {
 252                      InvalidateCompressedStream(compressedStream);
 253  
 254                      return false;
 255                  }
 256  
 257                  if (outerHeader.OSPlatform != GetOSPlatform())
 258                  {
 259                      InvalidateCompressedStream(compressedStream);
 260  
 261                      return false;
 262                  }
 263  
 264                  if (outerHeader.Architecture != (uint)RuntimeInformation.ProcessArchitecture)
 265                  {
 266                      InvalidateCompressedStream(compressedStream);
 267  
 268                      return false;
 269                  }
 270  
 271                  IntPtr intPtr = IntPtr.Zero;
 272  
 273                  try
 274                  {
 275                      intPtr = Marshal.AllocHGlobal(new IntPtr(outerHeader.UncompressedStreamSize));
 276  
 277                      using UnmanagedMemoryStream stream = new((byte*)intPtr.ToPointer(), outerHeader.UncompressedStreamSize, outerHeader.UncompressedStreamSize, FileAccess.ReadWrite);
 278                      try
 279                      {
 280                          deflateStream.CopyTo(stream);
 281                      }
 282                      catch
 283                      {
 284                          InvalidateCompressedStream(compressedStream);
 285  
 286                          return false;
 287                      }
 288  
 289                      Debug.Assert(stream.Position == stream.Length);
 290  
 291                      stream.Seek(0L, SeekOrigin.Begin);
 292  
 293                      InnerHeader innerHeader = DeserializeStructure<InnerHeader>(stream);
 294  
 295                      if (!innerHeader.IsHeaderValid())
 296                      {
 297                          InvalidateCompressedStream(compressedStream);
 298  
 299                          return false;
 300                      }
 301  
 302                      if (innerHeader.Magic != _innerHeaderMagic)
 303                      {
 304                          InvalidateCompressedStream(compressedStream);
 305  
 306                          return false;
 307                      }
 308  
 309                      ReadOnlySpan<byte> infosBytes = new(stream.PositionPointer, innerHeader.InfosLength);
 310                      stream.Seek(innerHeader.InfosLength, SeekOrigin.Current);
 311  
 312                      Hash128 infosHash = XXHash128.ComputeHash(infosBytes);
 313  
 314                      if (innerHeader.InfosHash != infosHash)
 315                      {
 316                          InvalidateCompressedStream(compressedStream);
 317  
 318                          return false;
 319                      }
 320  
 321                      ReadOnlySpan<byte> codesBytes = (int)innerHeader.CodesLength > 0 ? new(stream.PositionPointer, (int)innerHeader.CodesLength) : ReadOnlySpan<byte>.Empty;
 322                      stream.Seek(innerHeader.CodesLength, SeekOrigin.Current);
 323  
 324                      Hash128 codesHash = XXHash128.ComputeHash(codesBytes);
 325  
 326                      if (innerHeader.CodesHash != codesHash)
 327                      {
 328                          InvalidateCompressedStream(compressedStream);
 329  
 330                          return false;
 331                      }
 332  
 333                      ReadOnlySpan<byte> relocsBytes = new(stream.PositionPointer, innerHeader.RelocsLength);
 334                      stream.Seek(innerHeader.RelocsLength, SeekOrigin.Current);
 335  
 336                      Hash128 relocsHash = XXHash128.ComputeHash(relocsBytes);
 337  
 338                      if (innerHeader.RelocsHash != relocsHash)
 339                      {
 340                          InvalidateCompressedStream(compressedStream);
 341  
 342                          return false;
 343                      }
 344  
 345                      ReadOnlySpan<byte> unwindInfosBytes = new(stream.PositionPointer, innerHeader.UnwindInfosLength);
 346                      stream.Seek(innerHeader.UnwindInfosLength, SeekOrigin.Current);
 347  
 348                      Hash128 unwindInfosHash = XXHash128.ComputeHash(unwindInfosBytes);
 349  
 350                      if (innerHeader.UnwindInfosHash != unwindInfosHash)
 351                      {
 352                          InvalidateCompressedStream(compressedStream);
 353  
 354                          return false;
 355                      }
 356  
 357                      Debug.Assert(stream.Position == stream.Length);
 358  
 359                      stream.Seek((long)Unsafe.SizeOf<InnerHeader>(), SeekOrigin.Begin);
 360  
 361                      _infosStream.Write(infosBytes);
 362                      stream.Seek(innerHeader.InfosLength, SeekOrigin.Current);
 363  
 364                      _codesList.ReadFrom(stream);
 365  
 366                      _relocsStream.Write(relocsBytes);
 367                      stream.Seek(innerHeader.RelocsLength, SeekOrigin.Current);
 368  
 369                      _unwindInfosStream.Write(unwindInfosBytes);
 370                      stream.Seek(innerHeader.UnwindInfosLength, SeekOrigin.Current);
 371  
 372                      Debug.Assert(stream.Position == stream.Length);
 373                  }
 374                  finally
 375                  {
 376                      if (intPtr != IntPtr.Zero)
 377                      {
 378                          Marshal.FreeHGlobal(intPtr);
 379                      }
 380                  }
 381              }
 382  
 383              long fileSize = new FileInfo(fileName).Length;
 384  
 385              Logger.Info?.Print(LogClass.Ptc, $"{(isBackup ? "Loaded Backup Translation Cache" : "Loaded Translation Cache")} (size: {fileSize} bytes, translated functions: {GetEntriesCount()}).");
 386  
 387              return true;
 388          }
 389  
 390          private static void InvalidateCompressedStream(FileStream compressedStream)
 391          {
 392              compressedStream.SetLength(0L);
 393          }
 394  
 395          private void PreSave()
 396          {
 397              _waitEvent.Reset();
 398  
 399              try
 400              {
 401                  string fileNameActual = $"{CachePathActual}.cache";
 402                  string fileNameBackup = $"{CachePathBackup}.cache";
 403  
 404                  FileInfo fileInfoActual = new(fileNameActual);
 405  
 406                  if (fileInfoActual.Exists && fileInfoActual.Length != 0L)
 407                  {
 408                      File.Copy(fileNameActual, fileNameBackup, true);
 409                  }
 410  
 411                  Save(fileNameActual);
 412              }
 413              finally
 414              {
 415                  ResetCarriersIfNeeded();
 416  
 417                  GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
 418              }
 419  
 420              _waitEvent.Set();
 421          }
 422  
 423          private unsafe void Save(string fileName)
 424          {
 425              int translatedFuncsCount;
 426  
 427              InnerHeader innerHeader = new()
 428              {
 429                  Magic = _innerHeaderMagic,
 430  
 431                  InfosLength = (int)_infosStream.Length,
 432                  CodesLength = _codesList.Length(),
 433                  RelocsLength = (int)_relocsStream.Length,
 434                  UnwindInfosLength = (int)_unwindInfosStream.Length,
 435              };
 436  
 437              OuterHeader outerHeader = new()
 438              {
 439                  Magic = _outerHeaderMagic,
 440  
 441                  CacheFileVersion = InternalVersion,
 442                  Endianness = GetEndianness(),
 443                  FeatureInfo = GetFeatureInfo(),
 444                  MemoryManagerMode = GetMemoryManagerMode(),
 445                  OSPlatform = GetOSPlatform(),
 446                  Architecture = (uint)RuntimeInformation.ProcessArchitecture,
 447  
 448                  UncompressedStreamSize =
 449                  (long)Unsafe.SizeOf<InnerHeader>() +
 450                  innerHeader.InfosLength +
 451                  innerHeader.CodesLength +
 452                  innerHeader.RelocsLength +
 453                  innerHeader.UnwindInfosLength,
 454              };
 455  
 456              outerHeader.SetHeaderHash();
 457  
 458              IntPtr intPtr = IntPtr.Zero;
 459  
 460              try
 461              {
 462                  intPtr = Marshal.AllocHGlobal(new IntPtr(outerHeader.UncompressedStreamSize));
 463  
 464                  using UnmanagedMemoryStream stream = new((byte*)intPtr.ToPointer(), outerHeader.UncompressedStreamSize, outerHeader.UncompressedStreamSize, FileAccess.ReadWrite);
 465                  stream.Seek((long)Unsafe.SizeOf<InnerHeader>(), SeekOrigin.Begin);
 466  
 467                  ReadOnlySpan<byte> infosBytes = new(stream.PositionPointer, innerHeader.InfosLength);
 468                  _infosStream.WriteTo(stream);
 469  
 470                  ReadOnlySpan<byte> codesBytes = (int)innerHeader.CodesLength > 0 ? new(stream.PositionPointer, (int)innerHeader.CodesLength) : ReadOnlySpan<byte>.Empty;
 471                  _codesList.WriteTo(stream);
 472  
 473                  ReadOnlySpan<byte> relocsBytes = new(stream.PositionPointer, innerHeader.RelocsLength);
 474                  _relocsStream.WriteTo(stream);
 475  
 476                  ReadOnlySpan<byte> unwindInfosBytes = new(stream.PositionPointer, innerHeader.UnwindInfosLength);
 477                  _unwindInfosStream.WriteTo(stream);
 478  
 479                  Debug.Assert(stream.Position == stream.Length);
 480  
 481                  innerHeader.InfosHash = XXHash128.ComputeHash(infosBytes);
 482                  innerHeader.CodesHash = XXHash128.ComputeHash(codesBytes);
 483                  innerHeader.RelocsHash = XXHash128.ComputeHash(relocsBytes);
 484                  innerHeader.UnwindInfosHash = XXHash128.ComputeHash(unwindInfosBytes);
 485  
 486                  innerHeader.SetHeaderHash();
 487  
 488                  stream.Seek(0L, SeekOrigin.Begin);
 489                  SerializeStructure(stream, innerHeader);
 490  
 491                  translatedFuncsCount = GetEntriesCount();
 492  
 493                  ResetCarriersIfNeeded();
 494  
 495                  using FileStream compressedStream = new(fileName, FileMode.OpenOrCreate);
 496                  using DeflateStream deflateStream = new(compressedStream, SaveCompressionLevel, true);
 497                  try
 498                  {
 499                      SerializeStructure(compressedStream, outerHeader);
 500  
 501                      stream.Seek(0L, SeekOrigin.Begin);
 502                      stream.CopyTo(deflateStream);
 503                  }
 504                  catch
 505                  {
 506                      compressedStream.Position = 0L;
 507                  }
 508  
 509                  if (compressedStream.Position < compressedStream.Length)
 510                  {
 511                      compressedStream.SetLength(compressedStream.Position);
 512                  }
 513              }
 514              finally
 515              {
 516                  if (intPtr != IntPtr.Zero)
 517                  {
 518                      Marshal.FreeHGlobal(intPtr);
 519                  }
 520              }
 521  
 522              long fileSize = new FileInfo(fileName).Length;
 523  
 524              if (fileSize != 0L)
 525              {
 526                  Logger.Info?.Print(LogClass.Ptc, $"Saved Translation Cache (size: {fileSize} bytes, translated functions: {translatedFuncsCount}).");
 527              }
 528          }
 529  
 530          public void LoadTranslations(Translator translator)
 531          {
 532              if (AreCarriersEmpty())
 533              {
 534                  return;
 535              }
 536  
 537              long infosStreamLength = _infosStream.Length;
 538              long relocsStreamLength = _relocsStream.Length;
 539              long unwindInfosStreamLength = _unwindInfosStream.Length;
 540  
 541              _infosStream.Seek(0L, SeekOrigin.Begin);
 542              _relocsStream.Seek(0L, SeekOrigin.Begin);
 543              _unwindInfosStream.Seek(0L, SeekOrigin.Begin);
 544  
 545              using (BinaryReader relocsReader = new(_relocsStream, EncodingCache.UTF8NoBOM, true))
 546              using (BinaryReader unwindInfosReader = new(_unwindInfosStream, EncodingCache.UTF8NoBOM, true))
 547              {
 548                  for (int index = 0; index < GetEntriesCount(); index++)
 549                  {
 550                      InfoEntry infoEntry = DeserializeStructure<InfoEntry>(_infosStream);
 551  
 552                      if (infoEntry.Stubbed)
 553                      {
 554                          SkipCode(index, infoEntry.CodeLength);
 555                          SkipReloc(infoEntry.RelocEntriesCount);
 556                          SkipUnwindInfo(unwindInfosReader);
 557  
 558                          continue;
 559                      }
 560  
 561                      bool isEntryChanged = infoEntry.Hash != ComputeHash(translator.Memory, infoEntry.Address, infoEntry.GuestSize);
 562  
 563                      if (isEntryChanged || (!infoEntry.HighCq && Profiler.ProfiledFuncs.TryGetValue(infoEntry.Address, out var value) && value.HighCq))
 564                      {
 565                          infoEntry.Stubbed = true;
 566                          infoEntry.CodeLength = 0;
 567                          UpdateInfo(infoEntry);
 568  
 569                          StubCode(index);
 570                          StubReloc(infoEntry.RelocEntriesCount);
 571                          StubUnwindInfo(unwindInfosReader);
 572  
 573                          if (isEntryChanged)
 574                          {
 575                              Logger.Info?.Print(LogClass.Ptc, $"Invalidated translated function (address: 0x{infoEntry.Address:X16})");
 576                          }
 577  
 578                          continue;
 579                      }
 580  
 581                      byte[] code = ReadCode(index, infoEntry.CodeLength);
 582  
 583                      Counter<uint> callCounter = null;
 584  
 585                      if (infoEntry.RelocEntriesCount != 0)
 586                      {
 587                          RelocEntry[] relocEntries = GetRelocEntries(relocsReader, infoEntry.RelocEntriesCount);
 588  
 589                          PatchCode(translator, code, relocEntries, out callCounter);
 590                      }
 591  
 592                      UnwindInfo unwindInfo = ReadUnwindInfo(unwindInfosReader);
 593  
 594                      TranslatedFunction func = FastTranslate(code, callCounter, infoEntry.GuestSize, unwindInfo, infoEntry.HighCq);
 595  
 596                      translator.RegisterFunction(infoEntry.Address, func);
 597  
 598                      bool isAddressUnique = translator.Functions.TryAdd(infoEntry.Address, infoEntry.GuestSize, func);
 599  
 600                      Debug.Assert(isAddressUnique, $"The address 0x{infoEntry.Address:X16} is not unique.");
 601                  }
 602              }
 603  
 604              if (_infosStream.Length != infosStreamLength || _infosStream.Position != infosStreamLength ||
 605                  _relocsStream.Length != relocsStreamLength || _relocsStream.Position != relocsStreamLength ||
 606                  _unwindInfosStream.Length != unwindInfosStreamLength || _unwindInfosStream.Position != unwindInfosStreamLength)
 607              {
 608                  throw new Exception("The length of a memory stream has changed, or its position has not reached or has exceeded its end.");
 609              }
 610  
 611              Logger.Info?.Print(LogClass.Ptc, $"{translator.Functions.Count} translated functions loaded");
 612          }
 613  
 614          private int GetEntriesCount()
 615          {
 616              return _codesList.Count;
 617          }
 618  
 619          [Conditional("DEBUG")]
 620          private void SkipCode(int index, int codeLength)
 621          {
 622              Debug.Assert(_codesList[index].Length == 0);
 623              Debug.Assert(codeLength == 0);
 624          }
 625  
 626          private void SkipReloc(int relocEntriesCount)
 627          {
 628              _relocsStream.Seek(relocEntriesCount * RelocEntry.Stride, SeekOrigin.Current);
 629          }
 630  
 631          private void SkipUnwindInfo(BinaryReader unwindInfosReader)
 632          {
 633              int pushEntriesLength = unwindInfosReader.ReadInt32();
 634  
 635              _unwindInfosStream.Seek(pushEntriesLength * UnwindPushEntry.Stride + UnwindInfo.Stride, SeekOrigin.Current);
 636          }
 637  
 638          private byte[] ReadCode(int index, int codeLength)
 639          {
 640              Debug.Assert(_codesList[index].Length == codeLength);
 641  
 642              return _codesList[index];
 643          }
 644  
 645          private static RelocEntry[] GetRelocEntries(BinaryReader relocsReader, int relocEntriesCount)
 646          {
 647              RelocEntry[] relocEntries = new RelocEntry[relocEntriesCount];
 648  
 649              for (int i = 0; i < relocEntriesCount; i++)
 650              {
 651                  int position = relocsReader.ReadInt32();
 652                  SymbolType type = (SymbolType)relocsReader.ReadByte();
 653                  ulong value = relocsReader.ReadUInt64();
 654  
 655                  relocEntries[i] = new RelocEntry(position, new Symbol(type, value));
 656              }
 657  
 658              return relocEntries;
 659          }
 660  
 661          private static void PatchCode(Translator translator, Span<byte> code, RelocEntry[] relocEntries, out Counter<uint> callCounter)
 662          {
 663              callCounter = null;
 664  
 665              foreach (RelocEntry relocEntry in relocEntries)
 666              {
 667                  IntPtr? imm = null;
 668                  Symbol symbol = relocEntry.Symbol;
 669  
 670                  if (symbol.Type == SymbolType.FunctionTable)
 671                  {
 672                      ulong guestAddress = symbol.Value;
 673  
 674                      if (translator.FunctionTable.IsValid(guestAddress))
 675                      {
 676                          unsafe
 677                          {
 678                              imm = (IntPtr)Unsafe.AsPointer(ref translator.FunctionTable.GetValue(guestAddress));
 679                          }
 680                      }
 681                  }
 682                  else if (symbol.Type == SymbolType.DelegateTable)
 683                  {
 684                      int index = (int)symbol.Value;
 685  
 686                      if (Delegates.TryGetDelegateFuncPtrByIndex(index, out IntPtr funcPtr))
 687                      {
 688                          imm = funcPtr;
 689                      }
 690                  }
 691                  else if (symbol == PageTableSymbol)
 692                  {
 693                      imm = translator.Memory.PageTablePointer;
 694                  }
 695                  else if (symbol == CountTableSymbol)
 696                  {
 697                      callCounter ??= new Counter<uint>(translator.CountTable);
 698  
 699                      unsafe
 700                      {
 701                          imm = (IntPtr)Unsafe.AsPointer(ref callCounter.Value);
 702                      }
 703                  }
 704                  else if (symbol == DispatchStubSymbol)
 705                  {
 706                      imm = translator.Stubs.DispatchStub;
 707                  }
 708  
 709                  if (imm == null)
 710                  {
 711                      throw new Exception($"Unexpected reloc entry {relocEntry}.");
 712                  }
 713  
 714                  BinaryPrimitives.WriteUInt64LittleEndian(code.Slice(relocEntry.Position, 8), (ulong)imm.Value);
 715              }
 716          }
 717  
 718          private static UnwindInfo ReadUnwindInfo(BinaryReader unwindInfosReader)
 719          {
 720              int pushEntriesLength = unwindInfosReader.ReadInt32();
 721  
 722              UnwindPushEntry[] pushEntries = new UnwindPushEntry[pushEntriesLength];
 723  
 724              for (int i = 0; i < pushEntriesLength; i++)
 725              {
 726                  int pseudoOp = unwindInfosReader.ReadInt32();
 727                  int prologOffset = unwindInfosReader.ReadInt32();
 728                  int regIndex = unwindInfosReader.ReadInt32();
 729                  int stackOffsetOrAllocSize = unwindInfosReader.ReadInt32();
 730  
 731                  pushEntries[i] = new UnwindPushEntry((UnwindPseudoOp)pseudoOp, prologOffset, regIndex, stackOffsetOrAllocSize);
 732              }
 733  
 734              int prologueSize = unwindInfosReader.ReadInt32();
 735  
 736              return new UnwindInfo(pushEntries, prologueSize);
 737          }
 738  
 739          private static TranslatedFunction FastTranslate(
 740              byte[] code,
 741              Counter<uint> callCounter,
 742              ulong guestSize,
 743              UnwindInfo unwindInfo,
 744              bool highCq)
 745          {
 746              var cFunc = new CompiledFunction(code, unwindInfo, RelocInfo.Empty);
 747              var gFunc = cFunc.MapWithPointer<GuestFunction>(out IntPtr gFuncPointer);
 748  
 749              return new TranslatedFunction(gFunc, gFuncPointer, callCounter, guestSize, highCq);
 750          }
 751  
 752          private void UpdateInfo(InfoEntry infoEntry)
 753          {
 754              _infosStream.Seek(-Unsafe.SizeOf<InfoEntry>(), SeekOrigin.Current);
 755  
 756              SerializeStructure(_infosStream, infoEntry);
 757          }
 758  
 759          private void StubCode(int index)
 760          {
 761              _codesList[index] = Array.Empty<byte>();
 762          }
 763  
 764          private void StubReloc(int relocEntriesCount)
 765          {
 766              for (int i = 0; i < relocEntriesCount * RelocEntry.Stride; i++)
 767              {
 768                  _relocsStream.WriteByte(FillingByte);
 769              }
 770          }
 771  
 772          private void StubUnwindInfo(BinaryReader unwindInfosReader)
 773          {
 774              int pushEntriesLength = unwindInfosReader.ReadInt32();
 775  
 776              for (int i = 0; i < pushEntriesLength * UnwindPushEntry.Stride + UnwindInfo.Stride; i++)
 777              {
 778                  _unwindInfosStream.WriteByte(FillingByte);
 779              }
 780          }
 781  
 782          public void MakeAndSaveTranslations(Translator translator)
 783          {
 784              var profiledFuncsToTranslate = Profiler.GetProfiledFuncsToTranslate(translator.Functions);
 785  
 786              _translateCount = 0;
 787              _translateTotalCount = profiledFuncsToTranslate.Count;
 788  
 789              if (_translateTotalCount == 0)
 790              {
 791                  ResetCarriersIfNeeded();
 792  
 793                  GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
 794  
 795                  return;
 796              }
 797  
 798              int degreeOfParallelism = Environment.ProcessorCount;
 799  
 800              // If there are enough cores lying around, we leave one alone for other tasks.
 801              if (degreeOfParallelism > 4)
 802              {
 803                  degreeOfParallelism--;
 804              }
 805  
 806              Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism}");
 807  
 808              PtcStateChanged?.Invoke(PtcLoadingState.Start, _translateCount, _translateTotalCount);
 809  
 810              using AutoResetEvent progressReportEvent = new(false);
 811  
 812              Thread progressReportThread = new(ReportProgress)
 813              {
 814                  Name = "Ptc.ProgressReporter",
 815                  Priority = ThreadPriority.Lowest,
 816                  IsBackground = true,
 817              };
 818  
 819              progressReportThread.Start(progressReportEvent);
 820  
 821              void TranslateFuncs()
 822              {
 823                  while (profiledFuncsToTranslate.TryDequeue(out var item))
 824                  {
 825                      ulong address = item.address;
 826  
 827                      Debug.Assert(Profiler.IsAddressInStaticCodeRange(address));
 828  
 829                      TranslatedFunction func = translator.Translate(address, item.funcProfile.Mode, item.funcProfile.HighCq);
 830  
 831                      bool isAddressUnique = translator.Functions.TryAdd(address, func.GuestSize, func);
 832  
 833                      Debug.Assert(isAddressUnique, $"The address 0x{address:X16} is not unique.");
 834  
 835                      Interlocked.Increment(ref _translateCount);
 836  
 837                      translator.RegisterFunction(address, func);
 838  
 839                      if (State != PtcState.Enabled)
 840                      {
 841                          break;
 842                      }
 843                  }
 844              }
 845  
 846              List<Thread> threads = new();
 847  
 848              for (int i = 0; i < degreeOfParallelism; i++)
 849              {
 850                  Thread thread = new(TranslateFuncs)
 851                  {
 852                      IsBackground = true,
 853                  };
 854  
 855                  threads.Add(thread);
 856              }
 857  
 858              Stopwatch sw = Stopwatch.StartNew();
 859  
 860              foreach (var thread in threads)
 861              {
 862                  thread.Start();
 863              }
 864              foreach (var thread in threads)
 865              {
 866                  thread.Join();
 867              }
 868  
 869              threads.Clear();
 870  
 871              progressReportEvent.Set();
 872              progressReportThread.Join();
 873  
 874              sw.Stop();
 875  
 876              PtcStateChanged?.Invoke(PtcLoadingState.Loaded, _translateCount, _translateTotalCount);
 877  
 878              Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s");
 879  
 880              Thread preSaveThread = new(PreSave)
 881              {
 882                  IsBackground = true,
 883              };
 884              preSaveThread.Start();
 885          }
 886  
 887          private void ReportProgress(object state)
 888          {
 889              const int RefreshRate = 50; // ms.
 890  
 891              AutoResetEvent endEvent = (AutoResetEvent)state;
 892  
 893              int count = 0;
 894  
 895              do
 896              {
 897                  int newCount = _translateCount;
 898  
 899                  if (count != newCount)
 900                  {
 901                      PtcStateChanged?.Invoke(PtcLoadingState.Loading, newCount, _translateTotalCount);
 902                      count = newCount;
 903                  }
 904              }
 905              while (!endEvent.WaitOne(RefreshRate));
 906          }
 907  
 908          public static Hash128 ComputeHash(IMemoryManager memory, ulong address, ulong guestSize)
 909          {
 910              return XXHash128.ComputeHash(memory.GetSpan(address, checked((int)(guestSize))));
 911          }
 912  
 913          public void WriteCompiledFunction(ulong address, ulong guestSize, Hash128 hash, bool highCq, CompiledFunction compiledFunc)
 914          {
 915              lock (_lock)
 916              {
 917                  byte[] code = compiledFunc.Code;
 918                  RelocInfo relocInfo = compiledFunc.RelocInfo;
 919                  UnwindInfo unwindInfo = compiledFunc.UnwindInfo;
 920  
 921                  InfoEntry infoEntry = new()
 922                  {
 923                      Address = address,
 924                      GuestSize = guestSize,
 925                      Hash = hash,
 926                      HighCq = highCq,
 927                      Stubbed = false,
 928                      CodeLength = code.Length,
 929                      RelocEntriesCount = relocInfo.Entries.Length,
 930                  };
 931  
 932                  SerializeStructure(_infosStream, infoEntry);
 933  
 934                  WriteCode(code.AsSpan());
 935  
 936                  // WriteReloc.
 937                  using var relocInfoWriter = new BinaryWriter(_relocsStream, EncodingCache.UTF8NoBOM, true);
 938  
 939                  foreach (RelocEntry entry in relocInfo.Entries)
 940                  {
 941                      relocInfoWriter.Write(entry.Position);
 942                      relocInfoWriter.Write((byte)entry.Symbol.Type);
 943                      relocInfoWriter.Write(entry.Symbol.Value);
 944                  }
 945  
 946                  // WriteUnwindInfo.
 947                  using var unwindInfoWriter = new BinaryWriter(_unwindInfosStream, EncodingCache.UTF8NoBOM, true);
 948  
 949                  unwindInfoWriter.Write(unwindInfo.PushEntries.Length);
 950  
 951                  foreach (UnwindPushEntry unwindPushEntry in unwindInfo.PushEntries)
 952                  {
 953                      unwindInfoWriter.Write((int)unwindPushEntry.PseudoOp);
 954                      unwindInfoWriter.Write(unwindPushEntry.PrologOffset);
 955                      unwindInfoWriter.Write(unwindPushEntry.RegIndex);
 956                      unwindInfoWriter.Write(unwindPushEntry.StackOffsetOrAllocSize);
 957                  }
 958  
 959                  unwindInfoWriter.Write(unwindInfo.PrologSize);
 960              }
 961          }
 962  
 963          private void WriteCode(ReadOnlySpan<byte> code)
 964          {
 965              _codesList.Add(code.ToArray());
 966          }
 967  
 968          public static bool GetEndianness()
 969          {
 970              return BitConverter.IsLittleEndian;
 971          }
 972  
 973          private static FeatureInfo GetFeatureInfo()
 974          {
 975              if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
 976              {
 977                  return new FeatureInfo(
 978                      (ulong)Arm64HardwareCapabilities.LinuxFeatureInfoHwCap,
 979                      (ulong)Arm64HardwareCapabilities.LinuxFeatureInfoHwCap2,
 980                      (ulong)Arm64HardwareCapabilities.MacOsFeatureInfo,
 981                      0,
 982                      0);
 983              }
 984              else if (RuntimeInformation.ProcessArchitecture == Architecture.X64)
 985              {
 986                  return new FeatureInfo(
 987                      (ulong)X86HardwareCapabilities.FeatureInfo1Ecx,
 988                      (ulong)X86HardwareCapabilities.FeatureInfo1Edx,
 989                      (ulong)X86HardwareCapabilities.FeatureInfo7Ebx,
 990                      (ulong)X86HardwareCapabilities.FeatureInfo7Ecx,
 991                      (ulong)X86HardwareCapabilities.Xcr0InfoEax);
 992              }
 993              else
 994              {
 995                  return new FeatureInfo(0, 0, 0, 0, 0);
 996              }
 997          }
 998  
 999          private byte GetMemoryManagerMode()
1000          {
1001              return (byte)_memoryMode;
1002          }
1003  
1004          private static uint GetOSPlatform()
1005          {
1006              uint osPlatform = 0u;
1007  
1008  #pragma warning disable IDE0055 // Disable formatting
1009              osPlatform |= (OperatingSystem.IsFreeBSD() ? 1u : 0u) << 0;
1010              osPlatform |= (OperatingSystem.IsLinux()   ? 1u : 0u) << 1;
1011              osPlatform |= (OperatingSystem.IsMacOS()   ? 1u : 0u) << 2;
1012              osPlatform |= (OperatingSystem.IsWindows() ? 1u : 0u) << 3;
1013  #pragma warning restore IDE0055
1014  
1015              return osPlatform;
1016          }
1017  
1018          [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 86*/)]
1019          private struct OuterHeader
1020          {
1021              public ulong Magic;
1022  
1023              public uint CacheFileVersion;
1024  
1025              public bool Endianness;
1026              public FeatureInfo FeatureInfo;
1027              public byte MemoryManagerMode;
1028              public uint OSPlatform;
1029              public uint Architecture;
1030  
1031              public long UncompressedStreamSize;
1032  
1033              public Hash128 HeaderHash;
1034  
1035              public void SetHeaderHash()
1036              {
1037                  Span<OuterHeader> spanHeader = MemoryMarshal.CreateSpan(ref this, 1);
1038  
1039                  HeaderHash = XXHash128.ComputeHash(MemoryMarshal.AsBytes(spanHeader)[..(Unsafe.SizeOf<OuterHeader>() - Unsafe.SizeOf<Hash128>())]);
1040              }
1041  
1042              public bool IsHeaderValid()
1043              {
1044                  Span<OuterHeader> spanHeader = MemoryMarshal.CreateSpan(ref this, 1);
1045  
1046                  return XXHash128.ComputeHash(MemoryMarshal.AsBytes(spanHeader)[..(Unsafe.SizeOf<OuterHeader>() - Unsafe.SizeOf<Hash128>())]) == HeaderHash;
1047              }
1048          }
1049  
1050          [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 40*/)]
1051          private record struct FeatureInfo(ulong FeatureInfo0, ulong FeatureInfo1, ulong FeatureInfo2, ulong FeatureInfo3, ulong FeatureInfo4);
1052  
1053          [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 128*/)]
1054          private struct InnerHeader
1055          {
1056              public ulong Magic;
1057  
1058              public int InfosLength;
1059              public long CodesLength;
1060              public int RelocsLength;
1061              public int UnwindInfosLength;
1062  
1063              public Hash128 InfosHash;
1064              public Hash128 CodesHash;
1065              public Hash128 RelocsHash;
1066              public Hash128 UnwindInfosHash;
1067  
1068              public Hash128 HeaderHash;
1069  
1070              public void SetHeaderHash()
1071              {
1072                  Span<InnerHeader> spanHeader = MemoryMarshal.CreateSpan(ref this, 1);
1073  
1074                  HeaderHash = XXHash128.ComputeHash(MemoryMarshal.AsBytes(spanHeader)[..(Unsafe.SizeOf<InnerHeader>() - Unsafe.SizeOf<Hash128>())]);
1075              }
1076  
1077              public bool IsHeaderValid()
1078              {
1079                  Span<InnerHeader> spanHeader = MemoryMarshal.CreateSpan(ref this, 1);
1080  
1081                  return XXHash128.ComputeHash(MemoryMarshal.AsBytes(spanHeader)[..(Unsafe.SizeOf<InnerHeader>() - Unsafe.SizeOf<Hash128>())]) == HeaderHash;
1082              }
1083          }
1084  
1085          [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 42*/)]
1086          private struct InfoEntry
1087          {
1088              public ulong Address;
1089              public ulong GuestSize;
1090              public Hash128 Hash;
1091              public bool HighCq;
1092              public bool Stubbed;
1093              public int CodeLength;
1094              public int RelocEntriesCount;
1095          }
1096  
1097          private void Enable()
1098          {
1099              State = PtcState.Enabled;
1100          }
1101  
1102          public void Continue()
1103          {
1104              if (State == PtcState.Enabled)
1105              {
1106                  State = PtcState.Continuing;
1107              }
1108          }
1109  
1110          public void Close()
1111          {
1112              if (State == PtcState.Enabled ||
1113                  State == PtcState.Continuing)
1114              {
1115                  State = PtcState.Closing;
1116              }
1117          }
1118  
1119          public void Disable()
1120          {
1121              State = PtcState.Disabled;
1122          }
1123  
1124          private void Wait()
1125          {
1126              _waitEvent.WaitOne();
1127          }
1128  
1129          public void Dispose()
1130          {
1131              if (!_disposed)
1132              {
1133                  _disposed = true;
1134  
1135                  Wait();
1136                  _waitEvent.Dispose();
1137  
1138                  DisposeCarriers();
1139              }
1140          }
1141      }
1142  }