MemoryManagerHostMapped.cs
1 using ARMeilleure.Memory; 2 using Ryujinx.Memory; 3 using Ryujinx.Memory.Range; 4 using Ryujinx.Memory.Tracking; 5 using System; 6 using System.Buffers; 7 using System.Collections.Generic; 8 using System.Linq; 9 using System.Runtime.CompilerServices; 10 11 namespace Ryujinx.Cpu.Jit 12 { 13 /// <summary> 14 /// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region. 15 /// </summary> 16 public sealed class MemoryManagerHostMapped : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked 17 { 18 private readonly InvalidAccessHandler _invalidAccessHandler; 19 private readonly bool _unsafeMode; 20 21 private readonly AddressSpace _addressSpace; 22 23 private readonly PageTable<ulong> _pageTable; 24 25 private readonly MemoryEhMeilleure _memoryEh; 26 27 private readonly ManagedPageFlags _pages; 28 29 /// <inheritdoc/> 30 public bool UsesPrivateAllocations => false; 31 32 public int AddressSpaceBits { get; } 33 34 public IntPtr PageTablePointer => _addressSpace.Base.Pointer; 35 36 public MemoryManagerType Type => _unsafeMode ? MemoryManagerType.HostMappedUnsafe : MemoryManagerType.HostMapped; 37 38 public MemoryTracking Tracking { get; } 39 40 public event Action<ulong, ulong> UnmapEvent; 41 42 protected override ulong AddressSpaceSize { get; } 43 44 /// <summary> 45 /// Creates a new instance of the host mapped memory manager. 46 /// </summary> 47 /// <param name="addressSpace">Address space instance to use</param> 48 /// <param name="unsafeMode">True if unmanaged access should not be masked (unsafe), false otherwise.</param> 49 /// <param name="invalidAccessHandler">Optional function to handle invalid memory accesses</param> 50 public MemoryManagerHostMapped(AddressSpace addressSpace, bool unsafeMode, InvalidAccessHandler invalidAccessHandler) 51 { 52 _addressSpace = addressSpace; 53 _pageTable = new PageTable<ulong>(); 54 _invalidAccessHandler = invalidAccessHandler; 55 _unsafeMode = unsafeMode; 56 AddressSpaceSize = addressSpace.AddressSpaceSize; 57 58 ulong asSize = PageSize; 59 int asBits = PageBits; 60 61 while (asSize < AddressSpaceSize) 62 { 63 asSize <<= 1; 64 asBits++; 65 } 66 67 AddressSpaceBits = asBits; 68 69 _pages = new ManagedPageFlags(AddressSpaceBits); 70 71 Tracking = new MemoryTracking(this, (int)MemoryBlock.GetPageSize(), invalidAccessHandler); 72 _memoryEh = new MemoryEhMeilleure(_addressSpace.Base, _addressSpace.Mirror, Tracking); 73 } 74 75 /// <summary> 76 /// Ensures the combination of virtual address and size is part of the addressable space and fully mapped. 77 /// </summary> 78 /// <param name="va">Virtual address of the range</param> 79 /// <param name="size">Size of the range in bytes</param> 80 private void AssertMapped(ulong va, ulong size) 81 { 82 if (!ValidateAddressAndSize(va, size) || !_pages.IsRangeMapped(va, size)) 83 { 84 throw new InvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); 85 } 86 } 87 88 /// <inheritdoc/> 89 public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags) 90 { 91 AssertValidAddressAndSize(va, size); 92 93 _addressSpace.Map(va, pa, size, flags); 94 _pages.AddMapping(va, size); 95 PtMap(va, pa, size); 96 97 Tracking.Map(va, size); 98 } 99 100 /// <inheritdoc/> 101 public void Unmap(ulong va, ulong size) 102 { 103 AssertValidAddressAndSize(va, size); 104 105 UnmapEvent?.Invoke(va, size); 106 Tracking.Unmap(va, size); 107 108 _pages.RemoveMapping(va, size); 109 PtUnmap(va, size); 110 _addressSpace.Unmap(va, size); 111 } 112 113 private void PtMap(ulong va, ulong pa, ulong size) 114 { 115 while (size != 0) 116 { 117 _pageTable.Map(va, pa); 118 119 va += PageSize; 120 pa += PageSize; 121 size -= PageSize; 122 } 123 } 124 125 private void PtUnmap(ulong va, ulong size) 126 { 127 while (size != 0) 128 { 129 _pageTable.Unmap(va); 130 131 va += PageSize; 132 size -= PageSize; 133 } 134 } 135 136 public override T Read<T>(ulong va) 137 { 138 try 139 { 140 AssertMapped(va, (ulong)Unsafe.SizeOf<T>()); 141 142 return _addressSpace.Mirror.Read<T>(va); 143 } 144 catch (InvalidMemoryRegionException) 145 { 146 if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) 147 { 148 throw; 149 } 150 151 return default; 152 } 153 } 154 155 public override T ReadTracked<T>(ulong va) 156 { 157 try 158 { 159 return base.ReadTracked<T>(va); 160 } 161 catch (InvalidMemoryRegionException) 162 { 163 if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) 164 { 165 throw; 166 } 167 168 return default; 169 } 170 } 171 172 public override void Read(ulong va, Span<byte> data) 173 { 174 try 175 { 176 AssertMapped(va, (ulong)data.Length); 177 178 _addressSpace.Mirror.Read(va, data); 179 } 180 catch (InvalidMemoryRegionException) 181 { 182 if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) 183 { 184 throw; 185 } 186 } 187 } 188 189 public override void Write<T>(ulong va, T value) 190 { 191 try 192 { 193 SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), write: true); 194 195 _addressSpace.Mirror.Write(va, value); 196 } 197 catch (InvalidMemoryRegionException) 198 { 199 if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) 200 { 201 throw; 202 } 203 } 204 } 205 206 public override void Write(ulong va, ReadOnlySpan<byte> data) 207 { 208 try 209 { 210 SignalMemoryTracking(va, (ulong)data.Length, write: true); 211 212 _addressSpace.Mirror.Write(va, data); 213 } 214 catch (InvalidMemoryRegionException) 215 { 216 if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) 217 { 218 throw; 219 } 220 } 221 } 222 223 public override void WriteUntracked(ulong va, ReadOnlySpan<byte> data) 224 { 225 try 226 { 227 AssertMapped(va, (ulong)data.Length); 228 229 _addressSpace.Mirror.Write(va, data); 230 } 231 catch (InvalidMemoryRegionException) 232 { 233 if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) 234 { 235 throw; 236 } 237 } 238 } 239 240 public override bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan<byte> data) 241 { 242 try 243 { 244 SignalMemoryTracking(va, (ulong)data.Length, false); 245 246 Span<byte> target = _addressSpace.Mirror.GetSpan(va, data.Length); 247 bool changed = !data.SequenceEqual(target); 248 249 if (changed) 250 { 251 data.CopyTo(target); 252 } 253 254 return changed; 255 } 256 catch (InvalidMemoryRegionException) 257 { 258 if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) 259 { 260 throw; 261 } 262 263 return true; 264 } 265 } 266 267 public override ReadOnlySequence<byte> GetReadOnlySequence(ulong va, int size, bool tracked = false) 268 { 269 if (tracked) 270 { 271 SignalMemoryTracking(va, (ulong)size, write: false); 272 } 273 else 274 { 275 AssertMapped(va, (ulong)size); 276 } 277 278 return new ReadOnlySequence<byte>(_addressSpace.Mirror.GetMemory(va, size)); 279 } 280 281 public override ReadOnlySpan<byte> GetSpan(ulong va, int size, bool tracked = false) 282 { 283 if (tracked) 284 { 285 SignalMemoryTracking(va, (ulong)size, write: false); 286 } 287 else 288 { 289 AssertMapped(va, (ulong)size); 290 } 291 292 return _addressSpace.Mirror.GetSpan(va, size); 293 } 294 295 public override WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) 296 { 297 if (tracked) 298 { 299 SignalMemoryTracking(va, (ulong)size, true); 300 } 301 else 302 { 303 AssertMapped(va, (ulong)size); 304 } 305 306 return _addressSpace.Mirror.GetWritableRegion(va, size); 307 } 308 309 public ref T GetRef<T>(ulong va) where T : unmanaged 310 { 311 SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), true); 312 313 return ref _addressSpace.Mirror.GetRef<T>(va); 314 } 315 316 [MethodImpl(MethodImplOptions.AggressiveInlining)] 317 public override bool IsMapped(ulong va) 318 { 319 return ValidateAddress(va) && _pages.IsMapped(va); 320 } 321 322 /// <inheritdoc/> 323 public bool IsRangeMapped(ulong va, ulong size) 324 { 325 AssertValidAddressAndSize(va, size); 326 327 return _pages.IsRangeMapped(va, size); 328 } 329 330 /// <inheritdoc/> 331 public IEnumerable<HostMemoryRange> GetHostRegions(ulong va, ulong size) 332 { 333 AssertValidAddressAndSize(va, size); 334 335 return Enumerable.Repeat(new HostMemoryRange((nuint)(ulong)_addressSpace.Mirror.GetPointer(va, size), size), 1); 336 } 337 338 /// <inheritdoc/> 339 public IEnumerable<MemoryRange> GetPhysicalRegions(ulong va, ulong size) 340 { 341 int pages = GetPagesCount(va, (uint)size, out va); 342 343 var regions = new List<MemoryRange>(); 344 345 ulong regionStart = GetPhysicalAddressChecked(va); 346 ulong regionSize = PageSize; 347 348 for (int page = 0; page < pages - 1; page++) 349 { 350 if (!ValidateAddress(va + PageSize)) 351 { 352 return null; 353 } 354 355 ulong newPa = GetPhysicalAddressChecked(va + PageSize); 356 357 if (GetPhysicalAddressChecked(va) + PageSize != newPa) 358 { 359 regions.Add(new MemoryRange(regionStart, regionSize)); 360 regionStart = newPa; 361 regionSize = 0; 362 } 363 364 va += PageSize; 365 regionSize += PageSize; 366 } 367 368 regions.Add(new MemoryRange(regionStart, regionSize)); 369 370 return regions; 371 } 372 373 private ulong GetPhysicalAddressChecked(ulong va) 374 { 375 if (!IsMapped(va)) 376 { 377 ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}"); 378 } 379 380 return GetPhysicalAddressInternal(va); 381 } 382 383 private ulong GetPhysicalAddressInternal(ulong va) 384 { 385 return _pageTable.Read(va) + (va & PageMask); 386 } 387 388 /// <remarks> 389 /// This function also validates that the given range is both valid and mapped, and will throw if it is not. 390 /// </remarks> 391 public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) 392 { 393 AssertValidAddressAndSize(va, size); 394 395 if (precise) 396 { 397 Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId); 398 return; 399 } 400 401 _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId); 402 } 403 404 /// <inheritdoc/> 405 public void Reprotect(ulong va, ulong size, MemoryPermission protection) 406 { 407 // TODO 408 } 409 410 /// <inheritdoc/> 411 public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest) 412 { 413 if (guest) 414 { 415 _addressSpace.Base.Reprotect(va, size, protection, false); 416 } 417 else 418 { 419 _pages.TrackingReprotect(va, size, protection); 420 } 421 } 422 423 /// <inheritdoc/> 424 public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None) 425 { 426 return Tracking.BeginTracking(address, size, id, flags); 427 } 428 429 /// <inheritdoc/> 430 public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None) 431 { 432 return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags); 433 } 434 435 /// <inheritdoc/> 436 public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id) 437 { 438 return Tracking.BeginSmartGranularTracking(address, size, granularity, id); 439 } 440 441 /// <summary> 442 /// Disposes of resources used by the memory manager. 443 /// </summary> 444 protected override void Destroy() 445 { 446 _addressSpace.Dispose(); 447 _memoryEh.Dispose(); 448 } 449 450 protected override Memory<byte> GetPhysicalAddressMemory(nuint pa, int size) 451 => _addressSpace.Mirror.GetMemory(pa, size); 452 453 protected override Span<byte> GetPhysicalAddressSpan(nuint pa, int size) 454 => _addressSpace.Mirror.GetSpan(pa, size); 455 456 protected override nuint TranslateVirtualAddressChecked(ulong va) 457 => (nuint)GetPhysicalAddressChecked(va); 458 459 protected override nuint TranslateVirtualAddressUnchecked(ulong va) 460 => (nuint)GetPhysicalAddressInternal(va); 461 } 462 }