PhysicalMemory.cs
1 using Ryujinx.Common.Memory; 2 using Ryujinx.Cpu; 3 using Ryujinx.Graphics.Device; 4 using Ryujinx.Graphics.Gpu.Image; 5 using Ryujinx.Graphics.Gpu.Shader; 6 using Ryujinx.Memory; 7 using Ryujinx.Memory.Range; 8 using Ryujinx.Memory.Tracking; 9 using System; 10 using System.Buffers; 11 using System.Collections.Generic; 12 using System.Linq; 13 using System.Runtime.InteropServices; 14 using System.Threading; 15 16 namespace Ryujinx.Graphics.Gpu.Memory 17 { 18 /// <summary> 19 /// Represents physical memory, accessible from the GPU. 20 /// This is actually working CPU virtual addresses, of memory mapped on the application process. 21 /// </summary> 22 class PhysicalMemory : IDisposable 23 { 24 private readonly GpuContext _context; 25 private readonly IVirtualMemoryManagerTracked _cpuMemory; 26 private int _referenceCount; 27 28 /// <summary> 29 /// In-memory shader cache. 30 /// </summary> 31 public ShaderCache ShaderCache { get; } 32 33 /// <summary> 34 /// GPU buffer manager. 35 /// </summary> 36 public BufferCache BufferCache { get; } 37 38 /// <summary> 39 /// GPU texture manager. 40 /// </summary> 41 public TextureCache TextureCache { get; } 42 43 /// <summary> 44 /// Creates a new instance of the physical memory. 45 /// </summary> 46 /// <param name="context">GPU context that the physical memory belongs to</param> 47 /// <param name="cpuMemory">CPU memory manager of the application process</param> 48 public PhysicalMemory(GpuContext context, IVirtualMemoryManagerTracked cpuMemory) 49 { 50 _context = context; 51 _cpuMemory = cpuMemory; 52 ShaderCache = new ShaderCache(context); 53 BufferCache = new BufferCache(context, this); 54 TextureCache = new TextureCache(context, this); 55 56 if (cpuMemory is IRefCounted rc) 57 { 58 rc.IncrementReferenceCount(); 59 } 60 61 _referenceCount = 1; 62 } 63 64 /// <summary> 65 /// Increments the memory reference count. 66 /// </summary> 67 public void IncrementReferenceCount() 68 { 69 Interlocked.Increment(ref _referenceCount); 70 } 71 72 /// <summary> 73 /// Decrements the memory reference count. 74 /// </summary> 75 public void DecrementReferenceCount() 76 { 77 if (Interlocked.Decrement(ref _referenceCount) == 0 && _cpuMemory is IRefCounted rc) 78 { 79 rc.DecrementReferenceCount(); 80 } 81 } 82 83 /// <summary> 84 /// Creates a new device memory manager. 85 /// </summary> 86 /// <returns>The memory manager</returns> 87 public DeviceMemoryManager CreateDeviceMemoryManager() 88 { 89 return new DeviceMemoryManager(_cpuMemory); 90 } 91 92 /// <summary> 93 /// Gets a host pointer for a given range of application memory. 94 /// If the memory region is not a single contiguous block, this method returns 0. 95 /// </summary> 96 /// <remarks> 97 /// Getting a host pointer is unsafe. It should be considered invalid immediately if the GPU memory is unmapped. 98 /// </remarks> 99 /// <param name="range">Ranges of physical memory where the target data is located</param> 100 /// <returns>Pointer to the range of memory</returns> 101 public nint GetHostPointer(MultiRange range) 102 { 103 if (range.Count == 1) 104 { 105 var singleRange = range.GetSubRange(0); 106 if (singleRange.Address != MemoryManager.PteUnmapped) 107 { 108 var regions = _cpuMemory.GetHostRegions(singleRange.Address, singleRange.Size); 109 110 if (regions != null && regions.Count() == 1) 111 { 112 return (nint)regions.First().Address; 113 } 114 } 115 } 116 117 return 0; 118 } 119 120 /// <summary> 121 /// Gets a span of data from the application process. 122 /// </summary> 123 /// <param name="address">Start address of the range</param> 124 /// <param name="size">Size in bytes to be range</param> 125 /// <param name="tracked">True if read tracking is triggered on the span</param> 126 /// <returns>A read only span of the data at the specified memory location</returns> 127 public ReadOnlySpan<byte> GetSpan(ulong address, int size, bool tracked = false) 128 { 129 return _cpuMemory.GetSpan(address, size, tracked); 130 } 131 132 /// <summary> 133 /// Gets a span of data from the application process. 134 /// </summary> 135 /// <param name="range">Ranges of physical memory where the data is located</param> 136 /// <param name="tracked">True if read tracking is triggered on the span</param> 137 /// <returns>A read only span of the data at the specified memory location</returns> 138 public ReadOnlySpan<byte> GetSpan(MultiRange range, bool tracked = false) 139 { 140 if (range.Count == 1) 141 { 142 var singleRange = range.GetSubRange(0); 143 if (singleRange.Address != MemoryManager.PteUnmapped) 144 { 145 return _cpuMemory.GetSpan(singleRange.Address, (int)singleRange.Size, tracked); 146 } 147 } 148 149 Span<byte> data = new byte[range.GetSize()]; 150 151 int offset = 0; 152 153 for (int i = 0; i < range.Count; i++) 154 { 155 var currentRange = range.GetSubRange(i); 156 int size = (int)currentRange.Size; 157 if (currentRange.Address != MemoryManager.PteUnmapped) 158 { 159 _cpuMemory.GetSpan(currentRange.Address, size, tracked).CopyTo(data.Slice(offset, size)); 160 } 161 offset += size; 162 } 163 164 return data; 165 } 166 167 /// <summary> 168 /// Gets a writable region from the application process. 169 /// </summary> 170 /// <param name="address">Start address of the range</param> 171 /// <param name="size">Size in bytes to be range</param> 172 /// <param name="tracked">True if write tracking is triggered on the span</param> 173 /// <returns>A writable region with the data at the specified memory location</returns> 174 public WritableRegion GetWritableRegion(ulong address, int size, bool tracked = false) 175 { 176 return _cpuMemory.GetWritableRegion(address, size, tracked); 177 } 178 179 /// <summary> 180 /// Gets a writable region from GPU mapped memory. 181 /// </summary> 182 /// <param name="range">Range</param> 183 /// <param name="tracked">True if write tracking is triggered on the span</param> 184 /// <returns>A writable region with the data at the specified memory location</returns> 185 public WritableRegion GetWritableRegion(MultiRange range, bool tracked = false) 186 { 187 if (range.Count == 1) 188 { 189 MemoryRange subrange = range.GetSubRange(0); 190 191 return GetWritableRegion(subrange.Address, (int)subrange.Size, tracked); 192 } 193 else 194 { 195 MemoryOwner<byte> memoryOwner = MemoryOwner<byte>.Rent(checked((int)range.GetSize())); 196 197 Span<byte> memorySpan = memoryOwner.Span; 198 199 int offset = 0; 200 for (int i = 0; i < range.Count; i++) 201 { 202 var currentRange = range.GetSubRange(i); 203 int size = (int)currentRange.Size; 204 if (currentRange.Address != MemoryManager.PteUnmapped) 205 { 206 GetSpan(currentRange.Address, size).CopyTo(memorySpan.Slice(offset, size)); 207 } 208 offset += size; 209 } 210 211 return new WritableRegion(new MultiRangeWritableBlock(range, this), 0, memoryOwner, tracked); 212 } 213 } 214 215 /// <summary> 216 /// Reads data from the application process. 217 /// </summary> 218 /// <typeparam name="T">Type of the structure</typeparam> 219 /// <param name="address">Address to read from</param> 220 /// <returns>The data at the specified memory location</returns> 221 public T Read<T>(ulong address) where T : unmanaged 222 { 223 return _cpuMemory.Read<T>(address); 224 } 225 226 /// <summary> 227 /// Reads data from the application process, with write tracking. 228 /// </summary> 229 /// <typeparam name="T">Type of the structure</typeparam> 230 /// <param name="address">Address to read from</param> 231 /// <returns>The data at the specified memory location</returns> 232 public T ReadTracked<T>(ulong address) where T : unmanaged 233 { 234 return _cpuMemory.ReadTracked<T>(address); 235 } 236 237 /// <summary> 238 /// Writes data to the application process, triggering a precise memory tracking event. 239 /// </summary> 240 /// <param name="address">Address to write into</param> 241 /// <param name="data">Data to be written</param> 242 public void WriteTrackedResource(ulong address, ReadOnlySpan<byte> data) 243 { 244 _cpuMemory.SignalMemoryTracking(address, (ulong)data.Length, true, precise: true); 245 _cpuMemory.WriteUntracked(address, data); 246 } 247 248 /// <summary> 249 /// Writes data to the application process, triggering a precise memory tracking event. 250 /// </summary> 251 /// <param name="address">Address to write into</param> 252 /// <param name="data">Data to be written</param> 253 /// <param name="kind">Kind of the resource being written, which will not be signalled as CPU modified</param> 254 public void WriteTrackedResource(ulong address, ReadOnlySpan<byte> data, ResourceKind kind) 255 { 256 _cpuMemory.SignalMemoryTracking(address, (ulong)data.Length, true, precise: true, exemptId: (int)kind); 257 _cpuMemory.WriteUntracked(address, data); 258 } 259 260 /// <summary> 261 /// Writes data to the application process. 262 /// </summary> 263 /// <param name="address">Address to write into</param> 264 /// <param name="data">Data to be written</param> 265 public void Write(ulong address, ReadOnlySpan<byte> data) 266 { 267 _cpuMemory.Write(address, data); 268 } 269 270 /// <summary> 271 /// Writes data to the application process. 272 /// </summary> 273 /// <param name="range">Ranges of physical memory where the data is located</param> 274 /// <param name="data">Data to be written</param> 275 public void Write(MultiRange range, ReadOnlySpan<byte> data) 276 { 277 WriteImpl(range, data, _cpuMemory.Write); 278 } 279 280 /// <summary> 281 /// Writes data to the application process, without any tracking. 282 /// </summary> 283 /// <param name="address">Address to write into</param> 284 /// <param name="data">Data to be written</param> 285 public void WriteUntracked(ulong address, ReadOnlySpan<byte> data) 286 { 287 _cpuMemory.WriteUntracked(address, data); 288 } 289 290 /// <summary> 291 /// Writes data to the application process, without any tracking. 292 /// </summary> 293 /// <param name="range">Ranges of physical memory where the data is located</param> 294 /// <param name="data">Data to be written</param> 295 public void WriteUntracked(MultiRange range, ReadOnlySpan<byte> data) 296 { 297 WriteImpl(range, data, _cpuMemory.WriteUntracked); 298 } 299 300 /// <summary> 301 /// Writes data to the application process, returning false if the data was not changed. 302 /// This triggers read memory tracking, as a redundancy check would be useless if the data is not up to date. 303 /// </summary> 304 /// <remarks>The memory manager can return that memory has changed when it hasn't to avoid expensive data copies.</remarks> 305 /// <param name="address">Address to write into</param> 306 /// <param name="data">Data to be written</param> 307 /// <returns>True if the data was changed, false otherwise</returns> 308 public bool WriteWithRedundancyCheck(ulong address, ReadOnlySpan<byte> data) 309 { 310 return _cpuMemory.WriteWithRedundancyCheck(address, data); 311 } 312 313 private delegate void WriteCallback(ulong address, ReadOnlySpan<byte> data); 314 315 /// <summary> 316 /// Writes data to the application process, using the supplied callback method. 317 /// </summary> 318 /// <param name="range">Ranges of physical memory where the data is located</param> 319 /// <param name="data">Data to be written</param> 320 /// <param name="writeCallback">Callback method that will perform the write</param> 321 private static void WriteImpl(MultiRange range, ReadOnlySpan<byte> data, WriteCallback writeCallback) 322 { 323 if (range.Count == 1) 324 { 325 var singleRange = range.GetSubRange(0); 326 if (singleRange.Address != MemoryManager.PteUnmapped) 327 { 328 writeCallback(singleRange.Address, data); 329 } 330 } 331 else 332 { 333 int offset = 0; 334 335 for (int i = 0; i < range.Count; i++) 336 { 337 var currentRange = range.GetSubRange(i); 338 int size = (int)currentRange.Size; 339 if (currentRange.Address != MemoryManager.PteUnmapped) 340 { 341 writeCallback(currentRange.Address, data.Slice(offset, size)); 342 } 343 offset += size; 344 } 345 } 346 } 347 348 /// <summary> 349 /// Fills the specified memory region with a 32-bit integer value. 350 /// </summary> 351 /// <param name="address">CPU virtual address of the region</param> 352 /// <param name="size">Size of the region</param> 353 /// <param name="value">Value to fill the region with</param> 354 /// <param name="kind">Kind of the resource being filled, which will not be signalled as CPU modified</param> 355 public void FillTrackedResource(ulong address, ulong size, uint value, ResourceKind kind) 356 { 357 _cpuMemory.SignalMemoryTracking(address, size, write: true, precise: true, (int)kind); 358 359 using WritableRegion region = _cpuMemory.GetWritableRegion(address, (int)size); 360 361 MemoryMarshal.Cast<byte, uint>(region.Memory.Span).Fill(value); 362 } 363 364 /// <summary> 365 /// Obtains a memory tracking handle for the given virtual region. This should be disposed when finished with. 366 /// </summary> 367 /// <param name="address">CPU virtual address of the region</param> 368 /// <param name="size">Size of the region</param> 369 /// <param name="kind">Kind of the resource being tracked</param> 370 /// <param name="flags">Region flags</param> 371 /// <returns>The memory tracking handle</returns> 372 public RegionHandle BeginTracking(ulong address, ulong size, ResourceKind kind, RegionFlags flags = RegionFlags.None) 373 { 374 return _cpuMemory.BeginTracking(address, size, (int)kind, flags); 375 } 376 377 /// <summary> 378 /// Obtains a memory tracking handle for the given virtual region. This should be disposed when finished with. 379 /// </summary> 380 /// <param name="range">Ranges of physical memory where the data is located</param> 381 /// <param name="kind">Kind of the resource being tracked</param> 382 /// <returns>The memory tracking handle</returns> 383 public GpuRegionHandle BeginTracking(MultiRange range, ResourceKind kind) 384 { 385 var cpuRegionHandles = new RegionHandle[range.Count]; 386 int count = 0; 387 388 for (int i = 0; i < range.Count; i++) 389 { 390 var currentRange = range.GetSubRange(i); 391 if (currentRange.Address != MemoryManager.PteUnmapped) 392 { 393 cpuRegionHandles[count++] = _cpuMemory.BeginTracking(currentRange.Address, currentRange.Size, (int)kind); 394 } 395 } 396 397 if (count != range.Count) 398 { 399 Array.Resize(ref cpuRegionHandles, count); 400 } 401 402 return new GpuRegionHandle(cpuRegionHandles); 403 } 404 405 /// <summary> 406 /// Obtains a memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with. 407 /// </summary> 408 /// <param name="address">CPU virtual address of the region</param> 409 /// <param name="size">Size of the region</param> 410 /// <param name="kind">Kind of the resource being tracked</param> 411 /// <param name="flags">Region flags</param> 412 /// <param name="handles">Handles to inherit state from or reuse</param> 413 /// <param name="granularity">Desired granularity of write tracking</param> 414 /// <returns>The memory tracking handle</returns> 415 public MultiRegionHandle BeginGranularTracking( 416 ulong address, 417 ulong size, 418 ResourceKind kind, 419 RegionFlags flags = RegionFlags.None, 420 IEnumerable<IRegionHandle> handles = null, 421 ulong granularity = 4096) 422 { 423 return _cpuMemory.BeginGranularTracking(address, size, handles, granularity, (int)kind, flags); 424 } 425 426 /// <summary> 427 /// Obtains a smart memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with. 428 /// </summary> 429 /// <param name="address">CPU virtual address of the region</param> 430 /// <param name="size">Size of the region</param> 431 /// <param name="kind">Kind of the resource being tracked</param> 432 /// <param name="granularity">Desired granularity of write tracking</param> 433 /// <returns>The memory tracking handle</returns> 434 public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ResourceKind kind, ulong granularity = 4096) 435 { 436 return _cpuMemory.BeginSmartGranularTracking(address, size, granularity, (int)kind); 437 } 438 439 /// <summary> 440 /// Checks if a given memory page is mapped. 441 /// </summary> 442 /// <param name="address">CPU virtual address of the page</param> 443 /// <returns>True if mapped, false otherwise</returns> 444 public bool IsMapped(ulong address) 445 { 446 return _cpuMemory.IsMapped(address); 447 } 448 449 /// <summary> 450 /// Release our reference to the CPU memory manager. 451 /// </summary> 452 public void Dispose() 453 { 454 _context.DeferredActions.Enqueue(Destroy); 455 } 456 457 /// <summary> 458 /// Performs disposal of the host GPU caches with resources mapped on this physical memory. 459 /// This must only be called from the render thread. 460 /// </summary> 461 private void Destroy() 462 { 463 ShaderCache.Dispose(); 464 BufferCache.Dispose(); 465 TextureCache.Dispose(); 466 467 DecrementReferenceCount(); 468 } 469 } 470 }