1  using ARMeilleure.Memory;
  2  using Ryujinx.Memory;
  3  using Ryujinx.Memory.Range;
  4  using Ryujinx.Memory.Tracking;
  5  using System;
  6  using System.Buffers;
  7  using System.Collections.Generic;
  8  using System.Linq;
  9  using System.Runtime.CompilerServices;
 10  using System.Runtime.InteropServices;
 11  using System.Threading;
 12  
 13  namespace Ryujinx.Cpu.Jit
 14  {
 15      /// <summary>
 16      /// Represents a CPU memory manager.
 17      /// </summary>
 18      public sealed class MemoryManager : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked
 19      {
 20          private const int PteSize = 8;
 21  
 22          private const int PointerTagBit = 62;
 23  
 24          private readonly MemoryBlock _backingMemory;
 25          private readonly InvalidAccessHandler _invalidAccessHandler;
 26  
 27          /// <inheritdoc/>
 28          public bool UsesPrivateAllocations => false;
 29  
 30          /// <summary>
 31          /// Address space width in bits.
 32          /// </summary>
 33          public int AddressSpaceBits { get; }
 34  
 35          private readonly MemoryBlock _pageTable;
 36  
 37          private readonly ManagedPageFlags _pages;
 38  
 39          /// <summary>
 40          /// Page table base pointer.
 41          /// </summary>
 42          public IntPtr PageTablePointer => _pageTable.Pointer;
 43  
 44          public MemoryManagerType Type => MemoryManagerType.SoftwarePageTable;
 45  
 46          public MemoryTracking Tracking { get; }
 47  
 48          public event Action<ulong, ulong> UnmapEvent;
 49  
 50          protected override ulong AddressSpaceSize { get; }
 51  
 52          /// <summary>
 53          /// Creates a new instance of the memory manager.
 54          /// </summary>
 55          /// <param name="backingMemory">Physical backing memory where virtual memory will be mapped to</param>
 56          /// <param name="addressSpaceSize">Size of the address space</param>
 57          /// <param name="invalidAccessHandler">Optional function to handle invalid memory accesses</param>
 58          public MemoryManager(MemoryBlock backingMemory, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler = null)
 59          {
 60              _backingMemory = backingMemory;
 61              _invalidAccessHandler = invalidAccessHandler;
 62  
 63              ulong asSize = PageSize;
 64              int asBits = PageBits;
 65  
 66              while (asSize < addressSpaceSize)
 67              {
 68                  asSize <<= 1;
 69                  asBits++;
 70              }
 71  
 72              AddressSpaceBits = asBits;
 73              AddressSpaceSize = asSize;
 74              _pageTable = new MemoryBlock((asSize / PageSize) * PteSize);
 75  
 76              _pages = new ManagedPageFlags(AddressSpaceBits);
 77  
 78              Tracking = new MemoryTracking(this, PageSize);
 79          }
 80  
 81          /// <inheritdoc/>
 82          public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags)
 83          {
 84              AssertValidAddressAndSize(va, size);
 85  
 86              ulong remainingSize = size;
 87              ulong oVa = va;
 88              while (remainingSize != 0)
 89              {
 90                  _pageTable.Write((va / PageSize) * PteSize, PaToPte(pa));
 91  
 92                  va += PageSize;
 93                  pa += PageSize;
 94                  remainingSize -= PageSize;
 95              }
 96  
 97              _pages.AddMapping(oVa, size);
 98              Tracking.Map(oVa, size);
 99          }
100  
101          /// <inheritdoc/>
102          public void Unmap(ulong va, ulong size)
103          {
104              // If size is 0, there's nothing to unmap, just exit early.
105              if (size == 0)
106              {
107                  return;
108              }
109  
110              AssertValidAddressAndSize(va, size);
111  
112              UnmapEvent?.Invoke(va, size);
113              Tracking.Unmap(va, size);
114              _pages.RemoveMapping(va, size);
115  
116              ulong remainingSize = size;
117              while (remainingSize != 0)
118              {
119                  _pageTable.Write((va / PageSize) * PteSize, 0UL);
120  
121                  va += PageSize;
122                  remainingSize -= PageSize;
123              }
124          }
125  
126          public override T ReadTracked<T>(ulong va)
127          {
128              try
129              {
130                  return base.ReadTracked<T>(va);
131              }
132              catch (InvalidMemoryRegionException)
133              {
134                  if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
135                  {
136                      throw;
137                  }
138  
139                  return default;
140              }
141          }
142  
143          /// <inheritdoc/>
144          public T ReadGuest<T>(ulong va) where T : unmanaged
145          {
146              try
147              {
148                  SignalMemoryTrackingImpl(va, (ulong)Unsafe.SizeOf<T>(), false, true);
149  
150                  return Read<T>(va);
151              }
152              catch (InvalidMemoryRegionException)
153              {
154                  if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
155                  {
156                      throw;
157                  }
158  
159                  return default;
160              }
161          }
162  
163          /// <inheritdoc/>
164          public override void Read(ulong va, Span<byte> data)
165          {
166              try
167              {
168                  base.Read(va, data);
169              }
170              catch (InvalidMemoryRegionException)
171              {
172                  if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
173                  {
174                      throw;
175                  }
176              }
177          }
178  
179          public override void Write(ulong va, ReadOnlySpan<byte> data)
180          {
181              try
182              {
183                  base.Write(va, data);
184              }
185              catch (InvalidMemoryRegionException)
186              {
187                  if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
188                  {
189                      throw;
190                  }
191              }
192          }
193  
194          /// <inheritdoc/>
195          public void WriteGuest<T>(ulong va, T value) where T : unmanaged
196          {
197              Span<byte> data = MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateSpan(ref value, 1));
198  
199              SignalMemoryTrackingImpl(va, (ulong)data.Length, true, true);
200  
201              Write(va, data);
202          }
203  
204          public override void WriteUntracked(ulong va, ReadOnlySpan<byte> data)
205          {
206              try
207              {
208                  base.WriteUntracked(va, data);
209              }
210              catch (InvalidMemoryRegionException)
211              {
212                  if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
213                  {
214                      throw;
215                  }
216              }
217          }
218  
219          public override ReadOnlySequence<byte> GetReadOnlySequence(ulong va, int size, bool tracked = false)
220          {
221              try
222              {
223                  return base.GetReadOnlySequence(va, size, tracked);
224              }
225              catch (InvalidMemoryRegionException)
226              {
227                  if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
228                  {
229                      throw;
230                  }
231  
232                  return ReadOnlySequence<byte>.Empty;
233              }
234          }
235  
236          public ref T GetRef<T>(ulong va) where T : unmanaged
237          {
238              if (!IsContiguous(va, Unsafe.SizeOf<T>()))
239              {
240                  ThrowMemoryNotContiguous();
241              }
242  
243              SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), true);
244  
245              return ref _backingMemory.GetRef<T>(GetPhysicalAddressInternal(va));
246          }
247  
248          /// <inheritdoc/>
249          public IEnumerable<HostMemoryRange> GetHostRegions(ulong va, ulong size)
250          {
251              if (size == 0)
252              {
253                  return Enumerable.Empty<HostMemoryRange>();
254              }
255  
256              var guestRegions = GetPhysicalRegionsImpl(va, size);
257              if (guestRegions == null)
258              {
259                  return null;
260              }
261  
262              var regions = new HostMemoryRange[guestRegions.Count];
263  
264              for (int i = 0; i < regions.Length; i++)
265              {
266                  var guestRegion = guestRegions[i];
267                  IntPtr pointer = _backingMemory.GetPointer(guestRegion.Address, guestRegion.Size);
268                  regions[i] = new HostMemoryRange((nuint)(ulong)pointer, guestRegion.Size);
269              }
270  
271              return regions;
272          }
273  
274          /// <inheritdoc/>
275          public IEnumerable<MemoryRange> GetPhysicalRegions(ulong va, ulong size)
276          {
277              if (size == 0)
278              {
279                  return Enumerable.Empty<MemoryRange>();
280              }
281  
282              return GetPhysicalRegionsImpl(va, size);
283          }
284  
285          private List<MemoryRange> GetPhysicalRegionsImpl(ulong va, ulong size)
286          {
287              if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size))
288              {
289                  return null;
290              }
291  
292              int pages = GetPagesCount(va, (uint)size, out va);
293  
294              var regions = new List<MemoryRange>();
295  
296              ulong regionStart = GetPhysicalAddressInternal(va);
297              ulong regionSize = PageSize;
298  
299              for (int page = 0; page < pages - 1; page++)
300              {
301                  if (!ValidateAddress(va + PageSize))
302                  {
303                      return null;
304                  }
305  
306                  ulong newPa = GetPhysicalAddressInternal(va + PageSize);
307  
308                  if (GetPhysicalAddressInternal(va) + PageSize != newPa)
309                  {
310                      regions.Add(new MemoryRange(regionStart, regionSize));
311                      regionStart = newPa;
312                      regionSize = 0;
313                  }
314  
315                  va += PageSize;
316                  regionSize += PageSize;
317              }
318  
319              regions.Add(new MemoryRange(regionStart, regionSize));
320  
321              return regions;
322          }
323  
324          /// <inheritdoc/>
325          public bool IsRangeMapped(ulong va, ulong size)
326          {
327              if (size == 0UL)
328              {
329                  return true;
330              }
331  
332              if (!ValidateAddressAndSize(va, size))
333              {
334                  return false;
335              }
336  
337              int pages = GetPagesCount(va, (uint)size, out va);
338  
339              for (int page = 0; page < pages; page++)
340              {
341                  if (!IsMapped(va))
342                  {
343                      return false;
344                  }
345  
346                  va += PageSize;
347              }
348  
349              return true;
350          }
351  
352          [MethodImpl(MethodImplOptions.AggressiveInlining)]
353          public override bool IsMapped(ulong va)
354          {
355              if (!ValidateAddress(va))
356              {
357                  return false;
358              }
359  
360              return _pageTable.Read<ulong>((va / PageSize) * PteSize) != 0;
361          }
362  
363          private nuint GetPhysicalAddressInternal(ulong va)
364          {
365              return (nuint)(PteToPa(_pageTable.Read<ulong>((va / PageSize) * PteSize) & ~(0xffffUL << 48)) + (va & PageMask));
366          }
367  
368          /// <inheritdoc/>
369          public void Reprotect(ulong va, ulong size, MemoryPermission protection)
370          {
371              // TODO
372          }
373  
374          /// <inheritdoc/>
375          public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest)
376          {
377              AssertValidAddressAndSize(va, size);
378  
379              if (guest)
380              {
381                  // Protection is inverted on software pages, since the default value is 0.
382                  protection = (~protection) & MemoryPermission.ReadAndWrite;
383  
384                  long tag = protection switch
385                  {
386                      MemoryPermission.None => 0L,
387                      MemoryPermission.Write => 2L << PointerTagBit,
388                      _ => 3L << PointerTagBit,
389                  };
390  
391                  int pages = GetPagesCount(va, (uint)size, out va);
392                  ulong pageStart = va >> PageBits;
393                  long invTagMask = ~(0xffffL << 48);
394  
395                  for (int page = 0; page < pages; page++)
396                  {
397                      ref long pageRef = ref _pageTable.GetRef<long>(pageStart * PteSize);
398  
399                      long pte;
400  
401                      do
402                      {
403                          pte = Volatile.Read(ref pageRef);
404                      }
405                      while (pte != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte);
406  
407                      pageStart++;
408                  }
409              }
410              else
411              {
412                  _pages.TrackingReprotect(va, size, protection);
413              }
414          }
415  
416          /// <inheritdoc/>
417          public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None)
418          {
419              return Tracking.BeginTracking(address, size, id, flags);
420          }
421  
422          /// <inheritdoc/>
423          public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None)
424          {
425              return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags);
426          }
427  
428          /// <inheritdoc/>
429          public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id)
430          {
431              return Tracking.BeginSmartGranularTracking(address, size, granularity, id);
432          }
433  
434          private void SignalMemoryTrackingImpl(ulong va, ulong size, bool write, bool guest, bool precise = false, int? exemptId = null)
435          {
436              AssertValidAddressAndSize(va, size);
437  
438              if (precise)
439              {
440                  Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId);
441                  return;
442              }
443  
444              // If the memory tracking is coming from the guest, use the tag bits in the page table entry.
445              // Otherwise, use the managed page flags.
446  
447              if (guest)
448              {
449                  // We emulate guard pages for software memory access. This makes for an easy transition to
450                  // tracking using host guard pages in future, but also supporting platforms where this is not possible.
451  
452                  // Write tag includes read protection, since we don't have any read actions that aren't performed before write too.
453                  long tag = (write ? 3L : 1L) << PointerTagBit;
454  
455                  int pages = GetPagesCount(va, (uint)size, out _);
456                  ulong pageStart = va >> PageBits;
457  
458                  for (int page = 0; page < pages; page++)
459                  {
460                      ref long pageRef = ref _pageTable.GetRef<long>(pageStart * PteSize);
461  
462                      long pte = Volatile.Read(ref pageRef);
463  
464                      if ((pte & tag) != 0)
465                      {
466                          Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId, true);
467                          break;
468                      }
469  
470                      pageStart++;
471                  }
472              }
473              else
474              {
475                  _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId);
476              }
477          }
478  
479          /// <inheritdoc/>
480          public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
481          {
482              SignalMemoryTrackingImpl(va, size, write, false, precise, exemptId);
483          }
484  
485          private ulong PaToPte(ulong pa)
486          {
487              return (ulong)_backingMemory.GetPointer(pa, PageSize);
488          }
489  
490          private ulong PteToPa(ulong pte)
491          {
492              return (ulong)((long)pte - _backingMemory.Pointer.ToInt64());
493          }
494  
495          /// <summary>
496          /// Disposes of resources used by the memory manager.
497          /// </summary>
498          protected override void Destroy() => _pageTable.Dispose();
499  
500          protected override Memory<byte> GetPhysicalAddressMemory(nuint pa, int size)
501              => _backingMemory.GetMemory(pa, size);
502  
503          protected override Span<byte> GetPhysicalAddressSpan(nuint pa, int size)
504              => _backingMemory.GetSpan(pa, size);
505  
506          protected override nuint TranslateVirtualAddressChecked(ulong va)
507              => GetPhysicalAddressInternal(va);
508  
509          protected override nuint TranslateVirtualAddressUnchecked(ulong va)
510              => GetPhysicalAddressInternal(va);
511      }
512  }