1  using Ryujinx.Common.Memory;
  2  using Ryujinx.Graphics.Gpu.Image;
  3  using Ryujinx.Memory;
  4  using Ryujinx.Memory.Range;
  5  using System;
  6  using System.Collections.Generic;
  7  using System.Runtime.CompilerServices;
  8  using System.Runtime.InteropServices;
  9  
 10  namespace Ryujinx.Graphics.Gpu.Memory
 11  {
 12      /// <summary>
 13      /// GPU memory manager.
 14      /// </summary>
 15      public class MemoryManager : IWritableBlock
 16      {
 17          private const int PtLvl0Bits = 14;
 18          private const int PtLvl1Bits = 14;
 19          public const int PtPageBits = 12;
 20  
 21          private const ulong PtLvl0Size = 1UL << PtLvl0Bits;
 22          private const ulong PtLvl1Size = 1UL << PtLvl1Bits;
 23          public const ulong PageSize = 1UL << PtPageBits;
 24  
 25          private const ulong PtLvl0Mask = PtLvl0Size - 1;
 26          private const ulong PtLvl1Mask = PtLvl1Size - 1;
 27          public const ulong PageMask = PageSize - 1;
 28  
 29          private const int PtLvl0Bit = PtPageBits + PtLvl1Bits;
 30          private const int PtLvl1Bit = PtPageBits;
 31          private const int AddressSpaceBits = PtPageBits + PtLvl1Bits + PtLvl0Bits;
 32  
 33          public const ulong PteUnmapped = ulong.MaxValue;
 34  
 35          private readonly ulong[][] _pageTable;
 36  
 37          public event EventHandler<UnmapEventArgs> MemoryUnmapped;
 38  
 39          /// <summary>
 40          /// Physical memory where the virtual memory is mapped into.
 41          /// </summary>
 42          internal PhysicalMemory Physical { get; }
 43  
 44          /// <summary>
 45          /// Virtual range cache.
 46          /// </summary>
 47          internal VirtualRangeCache VirtualRangeCache { get; }
 48  
 49          /// <summary>
 50          /// Cache of GPU counters.
 51          /// </summary>
 52          internal CounterCache CounterCache { get; }
 53  
 54          /// <summary>
 55          /// Creates a new instance of the GPU memory manager.
 56          /// </summary>
 57          /// <param name="physicalMemory">Physical memory that this memory manager will map into</param>
 58          internal MemoryManager(PhysicalMemory physicalMemory)
 59          {
 60              Physical = physicalMemory;
 61              VirtualRangeCache = new VirtualRangeCache(this);
 62              CounterCache = new CounterCache();
 63              _pageTable = new ulong[PtLvl0Size][];
 64              MemoryUnmapped += Physical.TextureCache.MemoryUnmappedHandler;
 65              MemoryUnmapped += Physical.BufferCache.MemoryUnmappedHandler;
 66              MemoryUnmapped += VirtualRangeCache.MemoryUnmappedHandler;
 67              MemoryUnmapped += CounterCache.MemoryUnmappedHandler;
 68              Physical.TextureCache.Initialize();
 69          }
 70  
 71          /// <summary>
 72          /// Reads data from GPU mapped memory.
 73          /// </summary>
 74          /// <typeparam name="T">Type of the data</typeparam>
 75          /// <param name="va">GPU virtual address where the data is located</param>
 76          /// <param name="tracked">True if read tracking is triggered on the memory region</param>
 77          /// <returns>The data at the specified memory location</returns>
 78          public T Read<T>(ulong va, bool tracked = false) where T : unmanaged
 79          {
 80              int size = Unsafe.SizeOf<T>();
 81  
 82              if (IsContiguous(va, size))
 83              {
 84                  ulong address = Translate(va);
 85  
 86                  if (tracked)
 87                  {
 88                      return Physical.ReadTracked<T>(address);
 89                  }
 90                  else
 91                  {
 92                      return Physical.Read<T>(address);
 93                  }
 94              }
 95              else
 96              {
 97                  Span<byte> data = new byte[size];
 98  
 99                  ReadImpl(va, data, tracked);
100  
101                  return MemoryMarshal.Cast<byte, T>(data)[0];
102              }
103          }
104  
105          /// <summary>
106          /// Gets a read-only span of data from GPU mapped memory.
107          /// </summary>
108          /// <param name="va">GPU virtual address where the data is located</param>
109          /// <param name="size">Size of the data</param>
110          /// <param name="tracked">True if read tracking is triggered on the span</param>
111          /// <returns>The span of the data at the specified memory location</returns>
112          public ReadOnlySpan<byte> GetSpan(ulong va, int size, bool tracked = false)
113          {
114              if (IsContiguous(va, size))
115              {
116                  return Physical.GetSpan(Translate(va), size, tracked);
117              }
118              else
119              {
120                  Span<byte> data = new byte[size];
121  
122                  ReadImpl(va, data, tracked);
123  
124                  return data;
125              }
126          }
127  
128          /// <summary>
129          /// Gets a read-only span of data from GPU mapped memory, up to the entire range specified,
130          /// or the last mapped page if the range is not fully mapped.
131          /// </summary>
132          /// <param name="va">GPU virtual address where the data is located</param>
133          /// <param name="size">Size of the data</param>
134          /// <param name="tracked">True if read tracking is triggered on the span</param>
135          /// <returns>The span of the data at the specified memory location</returns>
136          public ReadOnlySpan<byte> GetSpanMapped(ulong va, int size, bool tracked = false)
137          {
138              bool isContiguous = true;
139              int mappedSize;
140  
141              if (ValidateAddress(va) && GetPte(va) != PteUnmapped && Physical.IsMapped(Translate(va)))
142              {
143                  ulong endVa = va + (ulong)size;
144                  ulong endVaAligned = (endVa + PageMask) & ~PageMask;
145                  ulong currentVa = va & ~PageMask;
146  
147                  int pages = (int)((endVaAligned - currentVa) / PageSize);
148  
149                  for (int page = 0; page < pages - 1; page++)
150                  {
151                      ulong nextVa = currentVa + PageSize;
152                      ulong nextPa = Translate(nextVa);
153  
154                      if (!ValidateAddress(nextVa) || GetPte(nextVa) == PteUnmapped || !Physical.IsMapped(nextPa))
155                      {
156                          break;
157                      }
158  
159                      if (Translate(currentVa) + PageSize != nextPa)
160                      {
161                          isContiguous = false;
162                      }
163  
164                      currentVa += PageSize;
165                  }
166  
167                  currentVa += PageSize;
168  
169                  if (currentVa > endVa)
170                  {
171                      currentVa = endVa;
172                  }
173  
174                  mappedSize = (int)(currentVa - va);
175              }
176              else
177              {
178                  return ReadOnlySpan<byte>.Empty;
179              }
180  
181              if (isContiguous)
182              {
183                  return Physical.GetSpan(Translate(va), mappedSize, tracked);
184              }
185              else
186              {
187                  Span<byte> data = new byte[mappedSize];
188  
189                  ReadImpl(va, data, tracked);
190  
191                  return data;
192              }
193          }
194  
195          /// <summary>
196          /// Reads data from a possibly non-contiguous region of GPU mapped memory.
197          /// </summary>
198          /// <param name="va">GPU virtual address of the data</param>
199          /// <param name="data">Span to write the read data into</param>
200          /// <param name="tracked">True to enable write tracking on read, false otherwise</param>
201          private void ReadImpl(ulong va, Span<byte> data, bool tracked)
202          {
203              if (data.Length == 0)
204              {
205                  return;
206              }
207  
208              int offset = 0, size;
209  
210              if ((va & PageMask) != 0)
211              {
212                  ulong pa = Translate(va);
213  
214                  size = Math.Min(data.Length, (int)PageSize - (int)(va & PageMask));
215  
216                  Physical.GetSpan(pa, size, tracked).CopyTo(data[..size]);
217  
218                  offset += size;
219              }
220  
221              for (; offset < data.Length; offset += size)
222              {
223                  ulong pa = Translate(va + (ulong)offset);
224  
225                  size = Math.Min(data.Length - offset, (int)PageSize);
226  
227                  Physical.GetSpan(pa, size, tracked).CopyTo(data.Slice(offset, size));
228              }
229          }
230  
231          /// <summary>
232          /// Gets a writable region from GPU mapped memory.
233          /// </summary>
234          /// <param name="va">Start address of the range</param>
235          /// <param name="size">Size in bytes to be range</param>
236          /// <param name="tracked">True if write tracking is triggered on the span</param>
237          /// <returns>A writable region with the data at the specified memory location</returns>
238          public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
239          {
240              if (IsContiguous(va, size))
241              {
242                  return Physical.GetWritableRegion(Translate(va), size, tracked);
243              }
244              else
245              {
246                  MemoryOwner<byte> memoryOwner = MemoryOwner<byte>.Rent(size);
247  
248                  ReadImpl(va, memoryOwner.Span, tracked);
249  
250                  return new WritableRegion(this, va, memoryOwner, tracked);
251              }
252          }
253  
254          /// <summary>
255          /// Writes data to GPU mapped memory.
256          /// </summary>
257          /// <typeparam name="T">Type of the data</typeparam>
258          /// <param name="va">GPU virtual address to write the value into</param>
259          /// <param name="value">The value to be written</param>
260          public void Write<T>(ulong va, T value) where T : unmanaged
261          {
262              Write(va, MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateSpan(ref value, 1)));
263          }
264  
265          /// <summary>
266          /// Writes data to GPU mapped memory.
267          /// </summary>
268          /// <param name="va">GPU virtual address to write the data into</param>
269          /// <param name="data">The data to be written</param>
270          public void Write(ulong va, ReadOnlySpan<byte> data)
271          {
272              WriteImpl(va, data, Physical.Write);
273          }
274  
275          /// <summary>
276          /// Writes data to GPU mapped memory, destined for a tracked resource.
277          /// </summary>
278          /// <param name="va">GPU virtual address to write the data into</param>
279          /// <param name="data">The data to be written</param>
280          public void WriteTrackedResource(ulong va, ReadOnlySpan<byte> data)
281          {
282              WriteImpl(va, data, Physical.WriteTrackedResource);
283          }
284  
285          /// <summary>
286          /// Writes data to GPU mapped memory without write tracking.
287          /// </summary>
288          /// <param name="va">GPU virtual address to write the data into</param>
289          /// <param name="data">The data to be written</param>
290          public void WriteUntracked(ulong va, ReadOnlySpan<byte> data)
291          {
292              WriteImpl(va, data, Physical.WriteUntracked);
293          }
294  
295          private delegate void WriteCallback(ulong address, ReadOnlySpan<byte> data);
296  
297          /// <summary>
298          /// Writes data to possibly non-contiguous GPU mapped memory.
299          /// </summary>
300          /// <param name="va">GPU virtual address of the region to write into</param>
301          /// <param name="data">Data to be written</param>
302          /// <param name="writeCallback">Write callback</param>
303          private void WriteImpl(ulong va, ReadOnlySpan<byte> data, WriteCallback writeCallback)
304          {
305              if (IsContiguous(va, data.Length))
306              {
307                  writeCallback(Translate(va), data);
308              }
309              else
310              {
311                  int offset = 0, size;
312  
313                  if ((va & PageMask) != 0)
314                  {
315                      ulong pa = Translate(va);
316  
317                      size = Math.Min(data.Length, (int)PageSize - (int)(va & PageMask));
318  
319                      writeCallback(pa, data[..size]);
320  
321                      offset += size;
322                  }
323  
324                  for (; offset < data.Length; offset += size)
325                  {
326                      ulong pa = Translate(va + (ulong)offset);
327  
328                      size = Math.Min(data.Length - offset, (int)PageSize);
329  
330                      writeCallback(pa, data.Slice(offset, size));
331                  }
332              }
333          }
334  
335          /// <summary>
336          /// Runs remap actions that are added to an unmap event.
337          /// These must run after the mapping completes.
338          /// </summary>
339          /// <param name="e">Event with remap actions</param>
340          private static void RunRemapActions(UnmapEventArgs e)
341          {
342              if (e.RemapActions != null)
343              {
344                  foreach (Action action in e.RemapActions)
345                  {
346                      action();
347                  }
348              }
349          }
350  
351          /// <summary>
352          /// Maps a given range of pages to the specified CPU virtual address.
353          /// </summary>
354          /// <remarks>
355          /// All addresses and sizes must be page aligned.
356          /// </remarks>
357          /// <param name="pa">CPU virtual address to map into</param>
358          /// <param name="va">GPU virtual address to be mapped</param>
359          /// <param name="size">Size in bytes of the mapping</param>
360          /// <param name="kind">Kind of the resource located at the mapping</param>
361          public void Map(ulong pa, ulong va, ulong size, PteKind kind)
362          {
363              lock (_pageTable)
364              {
365                  UnmapEventArgs e = new(va, size);
366                  MemoryUnmapped?.Invoke(this, e);
367  
368                  for (ulong offset = 0; offset < size; offset += PageSize)
369                  {
370                      SetPte(va + offset, PackPte(pa + offset, kind));
371                  }
372  
373                  RunRemapActions(e);
374              }
375          }
376  
377          /// <summary>
378          /// Unmaps a given range of pages at the specified GPU virtual memory region.
379          /// </summary>
380          /// <param name="va">GPU virtual address to unmap</param>
381          /// <param name="size">Size in bytes of the region being unmapped</param>
382          public void Unmap(ulong va, ulong size)
383          {
384              lock (_pageTable)
385              {
386                  // Event handlers are not expected to be thread safe.
387                  UnmapEventArgs e = new(va, size);
388                  MemoryUnmapped?.Invoke(this, e);
389  
390                  for (ulong offset = 0; offset < size; offset += PageSize)
391                  {
392                      SetPte(va + offset, PteUnmapped);
393                  }
394  
395                  RunRemapActions(e);
396              }
397          }
398  
399          /// <summary>
400          /// Checks if a region of GPU mapped memory is contiguous.
401          /// </summary>
402          /// <param name="va">GPU virtual address of the region</param>
403          /// <param name="size">Size of the region</param>
404          /// <returns>True if the region is contiguous, false otherwise</returns>
405          [MethodImpl(MethodImplOptions.AggressiveInlining)]
406          private bool IsContiguous(ulong va, int size)
407          {
408              if (!ValidateAddress(va) || GetPte(va) == PteUnmapped)
409              {
410                  return false;
411              }
412  
413              ulong endVa = (va + (ulong)size + PageMask) & ~PageMask;
414  
415              va &= ~PageMask;
416  
417              int pages = (int)((endVa - va) / PageSize);
418  
419              for (int page = 0; page < pages - 1; page++)
420              {
421                  if (!ValidateAddress(va + PageSize) || GetPte(va + PageSize) == PteUnmapped)
422                  {
423                      return false;
424                  }
425  
426                  if (Translate(va) + PageSize != Translate(va + PageSize))
427                  {
428                      return false;
429                  }
430  
431                  va += PageSize;
432              }
433  
434              return true;
435          }
436  
437          /// <summary>
438          /// Gets the physical regions that make up the given virtual address region.
439          /// </summary>
440          /// <param name="va">Virtual address of the range</param>
441          /// <param name="size">Size of the range</param>
442          /// <returns>Multi-range with the physical regions</returns>
443          public MultiRange GetPhysicalRegions(ulong va, ulong size)
444          {
445              if (IsContiguous(va, (int)size))
446              {
447                  return new MultiRange(Translate(va), size);
448              }
449  
450              ulong regionStart = Translate(va);
451              ulong regionSize = Math.Min(size, PageSize - (va & PageMask));
452  
453              ulong endVa = va + size;
454              ulong endVaRounded = (endVa + PageMask) & ~PageMask;
455  
456              va &= ~PageMask;
457  
458              int pages = (int)((endVaRounded - va) / PageSize);
459  
460              var regions = new List<MemoryRange>();
461  
462              for (int page = 0; page < pages - 1; page++)
463              {
464                  ulong currPa = Translate(va);
465                  ulong newPa = Translate(va + PageSize);
466  
467                  if ((currPa != PteUnmapped || newPa != PteUnmapped) && currPa + PageSize != newPa)
468                  {
469                      regions.Add(new MemoryRange(regionStart, regionSize));
470                      regionStart = newPa;
471                      regionSize = 0;
472                  }
473  
474                  va += PageSize;
475                  regionSize += Math.Min(endVa - va, PageSize);
476              }
477  
478              if (regions.Count == 0)
479              {
480                  return new MultiRange(regionStart, regionSize);
481              }
482  
483              regions.Add(new MemoryRange(regionStart, regionSize));
484  
485              return new MultiRange(regions.ToArray());
486          }
487  
488          /// <summary>
489          /// Checks if a given GPU virtual memory range is mapped to the same physical regions
490          /// as the specified physical memory multi-range.
491          /// </summary>
492          /// <param name="range">Physical memory multi-range</param>
493          /// <param name="va">GPU virtual memory address</param>
494          /// <returns>True if the virtual memory region is mapped into the specified physical one, false otherwise</returns>
495          public bool CompareRange(MultiRange range, ulong va)
496          {
497              va &= ~PageMask;
498  
499              for (int i = 0; i < range.Count; i++)
500              {
501                  MemoryRange currentRange = range.GetSubRange(i);
502  
503                  if (currentRange.Address != PteUnmapped)
504                  {
505                      ulong address = currentRange.Address & ~PageMask;
506                      ulong endAddress = (currentRange.EndAddress + PageMask) & ~PageMask;
507  
508                      while (address < endAddress)
509                      {
510                          if (Translate(va) != address)
511                          {
512                              return false;
513                          }
514  
515                          va += PageSize;
516                          address += PageSize;
517                      }
518                  }
519                  else
520                  {
521                      ulong endVa = va + (((currentRange.Size) + PageMask) & ~PageMask);
522  
523                      while (va < endVa)
524                      {
525                          if (Translate(va) != PteUnmapped)
526                          {
527                              return false;
528                          }
529  
530                          va += PageSize;
531                      }
532                  }
533              }
534  
535              return true;
536          }
537  
538          /// <summary>
539          /// Validates a GPU virtual address.
540          /// </summary>
541          /// <param name="va">Address to validate</param>
542          /// <returns>True if the address is valid, false otherwise</returns>
543          private static bool ValidateAddress(ulong va)
544          {
545              return va < (1UL << AddressSpaceBits);
546          }
547  
548          /// <summary>
549          /// Checks if a given page is mapped.
550          /// </summary>
551          /// <param name="va">GPU virtual address of the page to check</param>
552          /// <returns>True if the page is mapped, false otherwise</returns>
553          public bool IsMapped(ulong va)
554          {
555              return Translate(va) != PteUnmapped;
556          }
557  
558          /// <summary>
559          /// Translates a GPU virtual address to a CPU virtual address.
560          /// </summary>
561          /// <param name="va">GPU virtual address to be translated</param>
562          /// <returns>CPU virtual address, or <see cref="PteUnmapped"/> if unmapped</returns>
563          public ulong Translate(ulong va)
564          {
565              if (!ValidateAddress(va))
566              {
567                  return PteUnmapped;
568              }
569  
570              ulong pte = GetPte(va);
571  
572              if (pte == PteUnmapped)
573              {
574                  return PteUnmapped;
575              }
576  
577              return UnpackPaFromPte(pte) + (va & PageMask);
578          }
579  
580          /// <summary>
581          /// Translates a GPU virtual address to a CPU virtual address on the first mapped page of memory
582          /// on the specified region.
583          /// If no page is mapped on the specified region, <see cref="PteUnmapped"/> is returned.
584          /// </summary>
585          /// <param name="va">GPU virtual address to be translated</param>
586          /// <param name="size">Size of the range to be translated</param>
587          /// <returns>CPU virtual address, or <see cref="PteUnmapped"/> if unmapped</returns>
588          public ulong TranslateFirstMapped(ulong va, ulong size)
589          {
590              if (!ValidateAddress(va))
591              {
592                  return PteUnmapped;
593              }
594  
595              ulong endVa = va + size;
596  
597              ulong pte = GetPte(va);
598  
599              for (; va < endVa && pte == PteUnmapped; va += PageSize - (va & PageMask))
600              {
601                  pte = GetPte(va);
602              }
603  
604              if (pte == PteUnmapped)
605              {
606                  return PteUnmapped;
607              }
608  
609              return UnpackPaFromPte(pte) + (va & PageMask);
610          }
611  
612          /// <summary>
613          /// Translates a GPU virtual address and returns the number of bytes that are mapped after it.
614          /// </summary>
615          /// <param name="va">GPU virtual address to be translated</param>
616          /// <param name="maxSize">Maximum size in bytes to scan</param>
617          /// <returns>Number of bytes, 0 if unmapped</returns>
618          public ulong GetMappedSize(ulong va, ulong maxSize)
619          {
620              if (!ValidateAddress(va))
621              {
622                  return 0;
623              }
624  
625              ulong startVa = va;
626              ulong endVa = va + maxSize;
627  
628              ulong pte = GetPte(va);
629  
630              while (pte != PteUnmapped && va < endVa)
631              {
632                  va += PageSize - (va & PageMask);
633                  pte = GetPte(va);
634              }
635  
636              return Math.Min(maxSize, va - startVa);
637          }
638  
639          /// <summary>
640          /// Gets the kind of a given memory page.
641          /// This might indicate the type of resource that can be allocated on the page, and also texture tiling.
642          /// </summary>
643          /// <param name="va">GPU virtual address</param>
644          /// <returns>Kind of the memory page</returns>
645          public PteKind GetKind(ulong va)
646          {
647              if (!ValidateAddress(va))
648              {
649                  return PteKind.Invalid;
650              }
651  
652              ulong pte = GetPte(va);
653  
654              if (pte == PteUnmapped)
655              {
656                  return PteKind.Invalid;
657              }
658  
659              return UnpackKindFromPte(pte);
660          }
661  
662          /// <summary>
663          /// Gets the Page Table entry for a given GPU virtual address.
664          /// </summary>
665          /// <param name="va">GPU virtual address</param>
666          /// <returns>Page table entry (CPU virtual address)</returns>
667          private ulong GetPte(ulong va)
668          {
669              ulong l0 = (va >> PtLvl0Bit) & PtLvl0Mask;
670              ulong l1 = (va >> PtLvl1Bit) & PtLvl1Mask;
671  
672              if (_pageTable[l0] == null)
673              {
674                  return PteUnmapped;
675              }
676  
677              return _pageTable[l0][l1];
678          }
679  
680          /// <summary>
681          /// Sets a Page Table entry at a given GPU virtual address.
682          /// </summary>
683          /// <param name="va">GPU virtual address</param>
684          /// <param name="pte">Page table entry (CPU virtual address)</param>
685          private void SetPte(ulong va, ulong pte)
686          {
687              ulong l0 = (va >> PtLvl0Bit) & PtLvl0Mask;
688              ulong l1 = (va >> PtLvl1Bit) & PtLvl1Mask;
689  
690              if (_pageTable[l0] == null)
691              {
692                  _pageTable[l0] = new ulong[PtLvl1Size];
693  
694                  for (ulong index = 0; index < PtLvl1Size; index++)
695                  {
696                      _pageTable[l0][index] = PteUnmapped;
697                  }
698              }
699  
700              _pageTable[l0][l1] = pte;
701          }
702  
703          /// <summary>
704          /// Creates a page table entry from a physical address and kind.
705          /// </summary>
706          /// <param name="pa">Physical address</param>
707          /// <param name="kind">Kind</param>
708          /// <returns>Page table entry</returns>
709          private static ulong PackPte(ulong pa, PteKind kind)
710          {
711              return pa | ((ulong)kind << 56);
712          }
713  
714          /// <summary>
715          /// Unpacks kind from a page table entry.
716          /// </summary>
717          /// <param name="pte">Page table entry</param>
718          /// <returns>Kind</returns>
719          private static PteKind UnpackKindFromPte(ulong pte)
720          {
721              return (PteKind)(pte >> 56);
722          }
723  
724          /// <summary>
725          /// Unpacks physical address from a page table entry.
726          /// </summary>
727          /// <param name="pte">Page table entry</param>
728          /// <returns>Physical address</returns>
729          private static ulong UnpackPaFromPte(ulong pte)
730          {
731              return pte & 0xffffffffffffffUL;
732          }
733      }
734  }