BufferManager.cs
1 using Ryujinx.Common.Logging; 2 using Ryujinx.Graphics.GAL; 3 using Silk.NET.Vulkan; 4 using System; 5 using System.Runtime.CompilerServices; 6 using System.Runtime.InteropServices; 7 using VkBuffer = Silk.NET.Vulkan.Buffer; 8 using VkFormat = Silk.NET.Vulkan.Format; 9 10 namespace Ryujinx.Graphics.Vulkan 11 { 12 readonly struct ScopedTemporaryBuffer : IDisposable 13 { 14 private readonly BufferManager _bufferManager; 15 private readonly bool _isReserved; 16 17 public readonly BufferRange Range; 18 public readonly BufferHolder Holder; 19 20 public BufferHandle Handle => Range.Handle; 21 public int Offset => Range.Offset; 22 23 public ScopedTemporaryBuffer(BufferManager bufferManager, BufferHolder holder, BufferHandle handle, int offset, int size, bool isReserved) 24 { 25 _bufferManager = bufferManager; 26 27 Range = new BufferRange(handle, offset, size); 28 Holder = holder; 29 30 _isReserved = isReserved; 31 } 32 33 public void Dispose() 34 { 35 if (!_isReserved) 36 { 37 _bufferManager.Delete(Range.Handle); 38 } 39 } 40 } 41 42 class BufferManager : IDisposable 43 { 44 public const MemoryPropertyFlags DefaultBufferMemoryFlags = 45 MemoryPropertyFlags.HostVisibleBit | 46 MemoryPropertyFlags.HostCoherentBit | 47 MemoryPropertyFlags.HostCachedBit; 48 49 // Some drivers don't expose a "HostCached" memory type, 50 // so we need those alternative flags for the allocation to succeed there. 51 private const MemoryPropertyFlags DefaultBufferMemoryNoCacheFlags = 52 MemoryPropertyFlags.HostVisibleBit | 53 MemoryPropertyFlags.HostCoherentBit; 54 55 private const MemoryPropertyFlags DeviceLocalBufferMemoryFlags = 56 MemoryPropertyFlags.DeviceLocalBit; 57 58 private const MemoryPropertyFlags DeviceLocalMappedBufferMemoryFlags = 59 MemoryPropertyFlags.DeviceLocalBit | 60 MemoryPropertyFlags.HostVisibleBit | 61 MemoryPropertyFlags.HostCoherentBit; 62 63 private const BufferUsageFlags DefaultBufferUsageFlags = 64 BufferUsageFlags.TransferSrcBit | 65 BufferUsageFlags.TransferDstBit | 66 BufferUsageFlags.UniformTexelBufferBit | 67 BufferUsageFlags.StorageTexelBufferBit | 68 BufferUsageFlags.UniformBufferBit | 69 BufferUsageFlags.StorageBufferBit | 70 BufferUsageFlags.IndexBufferBit | 71 BufferUsageFlags.VertexBufferBit | 72 BufferUsageFlags.TransformFeedbackBufferBitExt; 73 74 private const BufferUsageFlags HostImportedBufferUsageFlags = 75 BufferUsageFlags.TransferSrcBit | 76 BufferUsageFlags.TransferDstBit; 77 78 private readonly Device _device; 79 80 private readonly IdList<BufferHolder> _buffers; 81 82 public int BufferCount { get; private set; } 83 84 public StagingBuffer StagingBuffer { get; } 85 86 public MemoryRequirements HostImportedBufferMemoryRequirements { get; } 87 88 public BufferManager(VulkanRenderer gd, Device device) 89 { 90 _device = device; 91 _buffers = new IdList<BufferHolder>(); 92 StagingBuffer = new StagingBuffer(gd, this); 93 94 HostImportedBufferMemoryRequirements = GetHostImportedUsageRequirements(gd); 95 } 96 97 public unsafe BufferHandle CreateHostImported(VulkanRenderer gd, nint pointer, int size) 98 { 99 var usage = HostImportedBufferUsageFlags; 100 101 if (gd.Capabilities.SupportsIndirectParameters) 102 { 103 usage |= BufferUsageFlags.IndirectBufferBit; 104 } 105 106 var externalMemoryBuffer = new ExternalMemoryBufferCreateInfo 107 { 108 SType = StructureType.ExternalMemoryBufferCreateInfo, 109 HandleTypes = ExternalMemoryHandleTypeFlags.HostAllocationBitExt, 110 }; 111 112 var bufferCreateInfo = new BufferCreateInfo 113 { 114 SType = StructureType.BufferCreateInfo, 115 Size = (ulong)size, 116 Usage = usage, 117 SharingMode = SharingMode.Exclusive, 118 PNext = &externalMemoryBuffer, 119 }; 120 121 gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError(); 122 123 (Auto<MemoryAllocation> allocation, ulong offset) = gd.HostMemoryAllocator.GetExistingAllocation(pointer, (ulong)size); 124 125 gd.Api.BindBufferMemory(_device, buffer, allocation.GetUnsafe().Memory, allocation.GetUnsafe().Offset + offset); 126 127 var holder = new BufferHolder(gd, _device, buffer, allocation, size, BufferAllocationType.HostMapped, BufferAllocationType.HostMapped, (int)offset); 128 129 BufferCount++; 130 131 ulong handle64 = (uint)_buffers.Add(holder); 132 133 return Unsafe.As<ulong, BufferHandle>(ref handle64); 134 } 135 136 public unsafe BufferHandle CreateSparse(VulkanRenderer gd, ReadOnlySpan<BufferRange> storageBuffers) 137 { 138 var usage = DefaultBufferUsageFlags; 139 140 if (gd.Capabilities.SupportsIndirectParameters) 141 { 142 usage |= BufferUsageFlags.IndirectBufferBit; 143 } 144 145 ulong size = 0; 146 147 foreach (BufferRange range in storageBuffers) 148 { 149 size += (ulong)range.Size; 150 } 151 152 var bufferCreateInfo = new BufferCreateInfo() 153 { 154 SType = StructureType.BufferCreateInfo, 155 Size = size, 156 Usage = usage, 157 SharingMode = SharingMode.Exclusive, 158 Flags = BufferCreateFlags.SparseBindingBit | BufferCreateFlags.SparseAliasedBit 159 }; 160 161 gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError(); 162 163 var memoryBinds = new SparseMemoryBind[storageBuffers.Length]; 164 var storageAllocations = new Auto<MemoryAllocation>[storageBuffers.Length]; 165 int storageAllocationsCount = 0; 166 167 ulong dstOffset = 0; 168 169 for (int index = 0; index < storageBuffers.Length; index++) 170 { 171 BufferRange range = storageBuffers[index]; 172 173 if (TryGetBuffer(range.Handle, out var existingHolder)) 174 { 175 (var memory, var offset) = existingHolder.GetDeviceMemoryAndOffset(); 176 177 memoryBinds[index] = new SparseMemoryBind() 178 { 179 ResourceOffset = dstOffset, 180 Size = (ulong)range.Size, 181 Memory = memory, 182 MemoryOffset = offset + (ulong)range.Offset, 183 Flags = SparseMemoryBindFlags.None 184 }; 185 186 storageAllocations[storageAllocationsCount++] = existingHolder.GetAllocation(); 187 } 188 else 189 { 190 memoryBinds[index] = new SparseMemoryBind() 191 { 192 ResourceOffset = dstOffset, 193 Size = (ulong)range.Size, 194 Memory = default, 195 MemoryOffset = 0UL, 196 Flags = SparseMemoryBindFlags.None 197 }; 198 } 199 200 dstOffset += (ulong)range.Size; 201 } 202 203 if (storageAllocations.Length != storageAllocationsCount) 204 { 205 Array.Resize(ref storageAllocations, storageAllocationsCount); 206 } 207 208 fixed (SparseMemoryBind* pMemoryBinds = memoryBinds) 209 { 210 SparseBufferMemoryBindInfo bufferBind = new SparseBufferMemoryBindInfo() 211 { 212 Buffer = buffer, 213 BindCount = (uint)memoryBinds.Length, 214 PBinds = pMemoryBinds 215 }; 216 217 BindSparseInfo bindSparseInfo = new BindSparseInfo() 218 { 219 SType = StructureType.BindSparseInfo, 220 BufferBindCount = 1, 221 PBufferBinds = &bufferBind 222 }; 223 224 gd.Api.QueueBindSparse(gd.Queue, 1, in bindSparseInfo, default).ThrowOnError(); 225 } 226 227 var holder = new BufferHolder(gd, _device, buffer, (int)size, storageAllocations); 228 229 BufferCount++; 230 231 ulong handle64 = (uint)_buffers.Add(holder); 232 233 return Unsafe.As<ulong, BufferHandle>(ref handle64); 234 } 235 236 public BufferHandle CreateWithHandle( 237 VulkanRenderer gd, 238 int size, 239 bool sparseCompatible = false, 240 BufferAllocationType baseType = BufferAllocationType.HostMapped, 241 bool forceMirrors = false) 242 { 243 return CreateWithHandle(gd, size, out _, sparseCompatible, baseType, forceMirrors); 244 } 245 246 public BufferHandle CreateWithHandle( 247 VulkanRenderer gd, 248 int size, 249 out BufferHolder holder, 250 bool sparseCompatible = false, 251 BufferAllocationType baseType = BufferAllocationType.HostMapped, 252 bool forceMirrors = false) 253 { 254 holder = Create(gd, size, forConditionalRendering: false, sparseCompatible, baseType); 255 if (holder == null) 256 { 257 return BufferHandle.Null; 258 } 259 260 if (forceMirrors) 261 { 262 holder.UseMirrors(); 263 } 264 265 BufferCount++; 266 267 ulong handle64 = (uint)_buffers.Add(holder); 268 269 return Unsafe.As<ulong, BufferHandle>(ref handle64); 270 } 271 272 public ScopedTemporaryBuffer ReserveOrCreate(VulkanRenderer gd, CommandBufferScoped cbs, int size) 273 { 274 StagingBufferReserved? result = StagingBuffer.TryReserveData(cbs, size); 275 276 if (result.HasValue) 277 { 278 return new ScopedTemporaryBuffer(this, result.Value.Buffer, StagingBuffer.Handle, result.Value.Offset, result.Value.Size, true); 279 } 280 else 281 { 282 // Create a temporary buffer. 283 BufferHandle handle = CreateWithHandle(gd, size, out BufferHolder holder); 284 285 return new ScopedTemporaryBuffer(this, holder, handle, 0, size, false); 286 } 287 } 288 289 public unsafe MemoryRequirements GetHostImportedUsageRequirements(VulkanRenderer gd) 290 { 291 var usage = HostImportedBufferUsageFlags; 292 293 if (gd.Capabilities.SupportsIndirectParameters) 294 { 295 usage |= BufferUsageFlags.IndirectBufferBit; 296 } 297 298 var bufferCreateInfo = new BufferCreateInfo 299 { 300 SType = StructureType.BufferCreateInfo, 301 Size = (ulong)Environment.SystemPageSize, 302 Usage = usage, 303 SharingMode = SharingMode.Exclusive, 304 }; 305 306 gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError(); 307 308 gd.Api.GetBufferMemoryRequirements(_device, buffer, out var requirements); 309 310 gd.Api.DestroyBuffer(_device, buffer, null); 311 312 return requirements; 313 } 314 315 public unsafe (VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) CreateBacking( 316 VulkanRenderer gd, 317 int size, 318 BufferAllocationType type, 319 bool forConditionalRendering = false, 320 bool sparseCompatible = false, 321 BufferAllocationType fallbackType = BufferAllocationType.Auto) 322 { 323 var usage = DefaultBufferUsageFlags; 324 325 if (forConditionalRendering && gd.Capabilities.SupportsConditionalRendering) 326 { 327 usage |= BufferUsageFlags.ConditionalRenderingBitExt; 328 } 329 else if (gd.Capabilities.SupportsIndirectParameters) 330 { 331 usage |= BufferUsageFlags.IndirectBufferBit; 332 } 333 334 var bufferCreateInfo = new BufferCreateInfo 335 { 336 SType = StructureType.BufferCreateInfo, 337 Size = (ulong)size, 338 Usage = usage, 339 SharingMode = SharingMode.Exclusive, 340 }; 341 342 gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError(); 343 gd.Api.GetBufferMemoryRequirements(_device, buffer, out var requirements); 344 345 if (sparseCompatible) 346 { 347 requirements.Alignment = Math.Max(requirements.Alignment, Constants.SparseBufferAlignment); 348 } 349 350 MemoryAllocation allocation; 351 352 do 353 { 354 var allocateFlags = type switch 355 { 356 BufferAllocationType.HostMappedNoCache => DefaultBufferMemoryNoCacheFlags, 357 BufferAllocationType.HostMapped => DefaultBufferMemoryFlags, 358 BufferAllocationType.DeviceLocal => DeviceLocalBufferMemoryFlags, 359 BufferAllocationType.DeviceLocalMapped => DeviceLocalMappedBufferMemoryFlags, 360 _ => DefaultBufferMemoryFlags, 361 }; 362 363 // If an allocation with this memory type fails, fall back to the previous one. 364 try 365 { 366 allocation = gd.MemoryAllocator.AllocateDeviceMemory(requirements, allocateFlags, true); 367 } 368 catch (VulkanException) 369 { 370 allocation = default; 371 } 372 } 373 while (allocation.Memory.Handle == 0 && (--type != fallbackType)); 374 375 if (allocation.Memory.Handle == 0UL) 376 { 377 gd.Api.DestroyBuffer(_device, buffer, null); 378 return default; 379 } 380 381 gd.Api.BindBufferMemory(_device, buffer, allocation.Memory, allocation.Offset); 382 383 return (buffer, allocation, type); 384 } 385 386 public BufferHolder Create( 387 VulkanRenderer gd, 388 int size, 389 bool forConditionalRendering = false, 390 bool sparseCompatible = false, 391 BufferAllocationType baseType = BufferAllocationType.HostMapped) 392 { 393 BufferAllocationType type = baseType; 394 395 if (baseType == BufferAllocationType.Auto) 396 { 397 type = BufferAllocationType.HostMapped; 398 } 399 400 (VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) = 401 CreateBacking(gd, size, type, forConditionalRendering, sparseCompatible); 402 403 if (buffer.Handle != 0) 404 { 405 var holder = new BufferHolder(gd, _device, buffer, allocation, size, baseType, resultType); 406 407 return holder; 408 } 409 410 Logger.Error?.Print(LogClass.Gpu, $"Failed to create buffer with size 0x{size:X} and type \"{baseType}\"."); 411 412 return null; 413 } 414 415 public Auto<DisposableBufferView> CreateView(BufferHandle handle, VkFormat format, int offset, int size, Action invalidateView) 416 { 417 if (TryGetBuffer(handle, out var holder)) 418 { 419 return holder.CreateView(format, offset, size, invalidateView); 420 } 421 422 return null; 423 } 424 425 public Auto<DisposableBuffer> GetBuffer(CommandBuffer commandBuffer, BufferHandle handle, bool isWrite, bool isSSBO = false) 426 { 427 if (TryGetBuffer(handle, out var holder)) 428 { 429 return holder.GetBuffer(commandBuffer, isWrite, isSSBO); 430 } 431 432 return null; 433 } 434 435 public Auto<DisposableBuffer> GetBuffer(CommandBuffer commandBuffer, BufferHandle handle, int offset, int size, bool isWrite) 436 { 437 if (TryGetBuffer(handle, out var holder)) 438 { 439 return holder.GetBuffer(commandBuffer, offset, size, isWrite); 440 } 441 442 return null; 443 } 444 445 public Auto<DisposableBuffer> GetBufferI8ToI16(CommandBufferScoped cbs, BufferHandle handle, int offset, int size) 446 { 447 if (TryGetBuffer(handle, out var holder)) 448 { 449 return holder.GetBufferI8ToI16(cbs, offset, size); 450 } 451 452 return null; 453 } 454 455 public Auto<DisposableBuffer> GetAlignedVertexBuffer(CommandBufferScoped cbs, BufferHandle handle, int offset, int size, int stride, int alignment) 456 { 457 if (TryGetBuffer(handle, out var holder)) 458 { 459 return holder.GetAlignedVertexBuffer(cbs, offset, size, stride, alignment); 460 } 461 462 return null; 463 } 464 465 public Auto<DisposableBuffer> GetBufferTopologyConversion(CommandBufferScoped cbs, BufferHandle handle, int offset, int size, IndexBufferPattern pattern, int indexSize) 466 { 467 if (TryGetBuffer(handle, out var holder)) 468 { 469 return holder.GetBufferTopologyConversion(cbs, offset, size, pattern, indexSize); 470 } 471 472 return null; 473 } 474 475 public (Auto<DisposableBuffer>, Auto<DisposableBuffer>) GetBufferTopologyConversionIndirect( 476 VulkanRenderer gd, 477 CommandBufferScoped cbs, 478 BufferRange indexBuffer, 479 BufferRange indirectBuffer, 480 BufferRange drawCountBuffer, 481 IndexBufferPattern pattern, 482 int indexSize, 483 bool hasDrawCount, 484 int maxDrawCount, 485 int indirectDataStride) 486 { 487 BufferHolder drawCountBufferHolder = null; 488 489 if (!TryGetBuffer(indexBuffer.Handle, out var indexBufferHolder) || 490 !TryGetBuffer(indirectBuffer.Handle, out var indirectBufferHolder) || 491 (hasDrawCount && !TryGetBuffer(drawCountBuffer.Handle, out drawCountBufferHolder))) 492 { 493 return (null, null); 494 } 495 496 var indexBufferKey = new TopologyConversionIndirectCacheKey( 497 gd, 498 pattern, 499 indexSize, 500 indirectBufferHolder, 501 indirectBuffer.Offset, 502 indirectBuffer.Size); 503 504 bool hasConvertedIndexBuffer = indexBufferHolder.TryGetCachedConvertedBuffer( 505 indexBuffer.Offset, 506 indexBuffer.Size, 507 indexBufferKey, 508 out var convertedIndexBuffer); 509 510 var indirectBufferKey = new IndirectDataCacheKey(pattern); 511 bool hasConvertedIndirectBuffer = indirectBufferHolder.TryGetCachedConvertedBuffer( 512 indirectBuffer.Offset, 513 indirectBuffer.Size, 514 indirectBufferKey, 515 out var convertedIndirectBuffer); 516 517 var drawCountBufferKey = new DrawCountCacheKey(); 518 bool hasCachedDrawCount = true; 519 520 if (hasDrawCount) 521 { 522 hasCachedDrawCount = drawCountBufferHolder.TryGetCachedConvertedBuffer( 523 drawCountBuffer.Offset, 524 drawCountBuffer.Size, 525 drawCountBufferKey, 526 out _); 527 } 528 529 if (!hasConvertedIndexBuffer || !hasConvertedIndirectBuffer || !hasCachedDrawCount) 530 { 531 // The destination index size is always I32. 532 533 int indexCount = indexBuffer.Size / indexSize; 534 535 int convertedCount = pattern.GetConvertedCount(indexCount); 536 537 if (!hasConvertedIndexBuffer) 538 { 539 convertedIndexBuffer = Create(gd, convertedCount * 4); 540 indexBufferKey.SetBuffer(convertedIndexBuffer.GetBuffer()); 541 indexBufferHolder.AddCachedConvertedBuffer(indexBuffer.Offset, indexBuffer.Size, indexBufferKey, convertedIndexBuffer); 542 } 543 544 if (!hasConvertedIndirectBuffer) 545 { 546 convertedIndirectBuffer = Create(gd, indirectBuffer.Size); 547 indirectBufferHolder.AddCachedConvertedBuffer(indirectBuffer.Offset, indirectBuffer.Size, indirectBufferKey, convertedIndirectBuffer); 548 } 549 550 gd.PipelineInternal.EndRenderPass(); 551 gd.HelperShader.ConvertIndexBufferIndirect( 552 gd, 553 cbs, 554 indirectBufferHolder, 555 convertedIndirectBuffer, 556 drawCountBuffer, 557 indexBufferHolder, 558 convertedIndexBuffer, 559 pattern, 560 indexSize, 561 indexBuffer.Offset, 562 indexBuffer.Size, 563 indirectBuffer.Offset, 564 hasDrawCount, 565 maxDrawCount, 566 indirectDataStride); 567 568 // Any modification of the indirect buffer should invalidate the index buffers that are associated with it, 569 // since we used the indirect data to find the range of the index buffer that is used. 570 571 var indexBufferDependency = new Dependency( 572 indexBufferHolder, 573 indexBuffer.Offset, 574 indexBuffer.Size, 575 indexBufferKey); 576 577 indirectBufferHolder.AddCachedConvertedBufferDependency( 578 indirectBuffer.Offset, 579 indirectBuffer.Size, 580 indirectBufferKey, 581 indexBufferDependency); 582 583 if (hasDrawCount) 584 { 585 if (!hasCachedDrawCount) 586 { 587 drawCountBufferHolder.AddCachedConvertedBuffer(drawCountBuffer.Offset, drawCountBuffer.Size, drawCountBufferKey, null); 588 } 589 590 // If we have a draw count, any modification of the draw count should invalidate all indirect buffers 591 // where we used it to find the range of indirect data that is actually used. 592 593 var indirectBufferDependency = new Dependency( 594 indirectBufferHolder, 595 indirectBuffer.Offset, 596 indirectBuffer.Size, 597 indirectBufferKey); 598 599 drawCountBufferHolder.AddCachedConvertedBufferDependency( 600 drawCountBuffer.Offset, 601 drawCountBuffer.Size, 602 drawCountBufferKey, 603 indirectBufferDependency); 604 } 605 } 606 607 return (convertedIndexBuffer.GetBuffer(), convertedIndirectBuffer.GetBuffer()); 608 } 609 610 public Auto<DisposableBuffer> GetBuffer(CommandBuffer commandBuffer, BufferHandle handle, bool isWrite, out int size) 611 { 612 if (TryGetBuffer(handle, out var holder)) 613 { 614 size = holder.Size; 615 return holder.GetBuffer(commandBuffer, isWrite); 616 } 617 618 size = 0; 619 return null; 620 } 621 622 public PinnedSpan<byte> GetData(BufferHandle handle, int offset, int size) 623 { 624 if (TryGetBuffer(handle, out var holder)) 625 { 626 return holder.GetData(offset, size); 627 } 628 629 return new PinnedSpan<byte>(); 630 } 631 632 public void SetData<T>(BufferHandle handle, int offset, ReadOnlySpan<T> data) where T : unmanaged 633 { 634 SetData(handle, offset, MemoryMarshal.Cast<T, byte>(data), null, null); 635 } 636 637 public void SetData(BufferHandle handle, int offset, ReadOnlySpan<byte> data, CommandBufferScoped? cbs, Action endRenderPass) 638 { 639 if (TryGetBuffer(handle, out var holder)) 640 { 641 holder.SetData(offset, data, cbs, endRenderPass); 642 } 643 } 644 645 public void Delete(BufferHandle handle) 646 { 647 if (TryGetBuffer(handle, out var holder)) 648 { 649 holder.Dispose(); 650 _buffers.Remove((int)Unsafe.As<BufferHandle, ulong>(ref handle)); 651 } 652 } 653 654 private bool TryGetBuffer(BufferHandle handle, out BufferHolder holder) 655 { 656 return _buffers.TryGetValue((int)Unsafe.As<BufferHandle, ulong>(ref handle), out holder); 657 } 658 659 protected virtual void Dispose(bool disposing) 660 { 661 if (disposing) 662 { 663 StagingBuffer.Dispose(); 664 665 foreach (BufferHolder buffer in _buffers) 666 { 667 buffer.Dispose(); 668 } 669 670 _buffers.Clear(); 671 } 672 } 673 674 public void Dispose() 675 { 676 Dispose(true); 677 } 678 } 679 }