1  using Ryujinx.Common;
  2  using Ryujinx.Graphics.GAL;
  3  using Silk.NET.Vulkan;
  4  using System;
  5  using System.Collections.Generic;
  6  using System.Numerics;
  7  using System.Runtime.CompilerServices;
  8  using Format = Ryujinx.Graphics.GAL.Format;
  9  using VkBuffer = Silk.NET.Vulkan.Buffer;
 10  using VkFormat = Silk.NET.Vulkan.Format;
 11  
 12  namespace Ryujinx.Graphics.Vulkan
 13  {
 14      class TextureStorage : IDisposable
 15      {
 16          private struct TextureSliceInfo
 17          {
 18              public int BindCount;
 19          }
 20  
 21          private const MemoryPropertyFlags DefaultImageMemoryFlags =
 22              MemoryPropertyFlags.DeviceLocalBit;
 23  
 24          private const ImageUsageFlags DefaultUsageFlags =
 25              ImageUsageFlags.SampledBit |
 26              ImageUsageFlags.TransferSrcBit |
 27              ImageUsageFlags.TransferDstBit;
 28  
 29          public const AccessFlags DefaultAccessMask =
 30              AccessFlags.ShaderReadBit |
 31              AccessFlags.ShaderWriteBit |
 32              AccessFlags.ColorAttachmentReadBit |
 33              AccessFlags.ColorAttachmentWriteBit |
 34              AccessFlags.DepthStencilAttachmentReadBit |
 35              AccessFlags.DepthStencilAttachmentWriteBit |
 36              AccessFlags.TransferReadBit |
 37              AccessFlags.TransferWriteBit;
 38  
 39          private readonly VulkanRenderer _gd;
 40  
 41          private readonly Device _device;
 42  
 43          private TextureCreateInfo _info;
 44  
 45          public TextureCreateInfo Info => _info;
 46  
 47          public bool Disposed { get; private set; }
 48  
 49          private readonly Image _image;
 50          private readonly Auto<DisposableImage> _imageAuto;
 51          private readonly Auto<MemoryAllocation> _allocationAuto;
 52          private readonly int _depthOrLayers;
 53          private Auto<MemoryAllocation> _foreignAllocationAuto;
 54  
 55          private Dictionary<Format, TextureStorage> _aliasedStorages;
 56  
 57          private AccessFlags _lastModificationAccess;
 58          private PipelineStageFlags _lastModificationStage;
 59          private AccessFlags _lastReadAccess;
 60          private PipelineStageFlags _lastReadStage;
 61  
 62          private int _viewsCount;
 63          private readonly ulong _size;
 64  
 65          private int _bindCount;
 66          private readonly TextureSliceInfo[] _slices;
 67  
 68          public VkFormat VkFormat { get; }
 69  
 70          public unsafe TextureStorage(
 71              VulkanRenderer gd,
 72              Device device,
 73              TextureCreateInfo info,
 74              Auto<MemoryAllocation> foreignAllocation = null)
 75          {
 76              _gd = gd;
 77              _device = device;
 78              _info = info;
 79  
 80              var format = _gd.FormatCapabilities.ConvertToVkFormat(info.Format);
 81              var levels = (uint)info.Levels;
 82              var layers = (uint)info.GetLayers();
 83              var depth = (uint)(info.Target == Target.Texture3D ? info.Depth : 1);
 84  
 85              VkFormat = format;
 86              _depthOrLayers = info.GetDepthOrLayers();
 87  
 88              var type = info.Target.Convert();
 89  
 90              var extent = new Extent3D((uint)info.Width, (uint)info.Height, depth);
 91  
 92              var sampleCountFlags = ConvertToSampleCountFlags(gd.Capabilities.SupportedSampleCounts, (uint)info.Samples);
 93  
 94              var usage = GetImageUsage(info.Format, info.Target, gd.Capabilities);
 95  
 96              var flags = ImageCreateFlags.CreateMutableFormatBit | ImageCreateFlags.CreateExtendedUsageBit;
 97  
 98              // This flag causes mipmapped texture arrays to break on AMD GCN, so for that copy dependencies are forced for aliasing as cube.
 99              bool isCube = info.Target == Target.Cubemap || info.Target == Target.CubemapArray;
100              bool cubeCompatible = gd.IsAmdGcn ? isCube : (info.Width == info.Height && layers >= 6);
101  
102              if (type == ImageType.Type2D && cubeCompatible)
103              {
104                  flags |= ImageCreateFlags.CreateCubeCompatibleBit;
105              }
106  
107              if (type == ImageType.Type3D && !gd.Capabilities.PortabilitySubset.HasFlag(PortabilitySubsetFlags.No3DImageView))
108              {
109                  flags |= ImageCreateFlags.Create2DArrayCompatibleBit;
110              }
111  
112              var imageCreateInfo = new ImageCreateInfo
113              {
114                  SType = StructureType.ImageCreateInfo,
115                  ImageType = type,
116                  Format = format,
117                  Extent = extent,
118                  MipLevels = levels,
119                  ArrayLayers = layers,
120                  Samples = sampleCountFlags,
121                  Tiling = ImageTiling.Optimal,
122                  Usage = usage,
123                  SharingMode = SharingMode.Exclusive,
124                  InitialLayout = ImageLayout.Undefined,
125                  Flags = flags,
126              };
127  
128              gd.Api.CreateImage(device, in imageCreateInfo, null, out _image).ThrowOnError();
129  
130              if (foreignAllocation == null)
131              {
132                  gd.Api.GetImageMemoryRequirements(device, _image, out var requirements);
133                  var allocation = gd.MemoryAllocator.AllocateDeviceMemory(requirements, DefaultImageMemoryFlags);
134  
135                  if (allocation.Memory.Handle == 0UL)
136                  {
137                      gd.Api.DestroyImage(device, _image, null);
138                      throw new Exception("Image initialization failed.");
139                  }
140  
141                  _size = requirements.Size;
142  
143                  gd.Api.BindImageMemory(device, _image, allocation.Memory, allocation.Offset).ThrowOnError();
144  
145                  _allocationAuto = new Auto<MemoryAllocation>(allocation);
146                  _imageAuto = new Auto<DisposableImage>(new DisposableImage(_gd.Api, device, _image), null, _allocationAuto);
147  
148                  InitialTransition(ImageLayout.Undefined, ImageLayout.General);
149              }
150              else
151              {
152                  _foreignAllocationAuto = foreignAllocation;
153                  foreignAllocation.IncrementReferenceCount();
154                  var allocation = foreignAllocation.GetUnsafe();
155  
156                  gd.Api.BindImageMemory(device, _image, allocation.Memory, allocation.Offset).ThrowOnError();
157  
158                  _imageAuto = new Auto<DisposableImage>(new DisposableImage(_gd.Api, device, _image));
159  
160                  InitialTransition(ImageLayout.Preinitialized, ImageLayout.General);
161              }
162  
163              _slices = new TextureSliceInfo[levels * _depthOrLayers];
164          }
165  
166          public TextureStorage CreateAliasedColorForDepthStorageUnsafe(Format format)
167          {
168              var colorFormat = format switch
169              {
170                  Format.S8Uint => Format.R8Unorm,
171                  Format.D16Unorm => Format.R16Unorm,
172                  Format.D24UnormS8Uint or Format.S8UintD24Unorm or Format.X8UintD24Unorm => Format.R8G8B8A8Unorm,
173                  Format.D32Float => Format.R32Float,
174                  Format.D32FloatS8Uint => Format.R32G32Float,
175                  _ => throw new ArgumentException($"\"{format}\" is not a supported depth or stencil format."),
176              };
177  
178              return CreateAliasedStorageUnsafe(colorFormat);
179          }
180  
181          public TextureStorage CreateAliasedStorageUnsafe(Format format)
182          {
183              if (_aliasedStorages == null || !_aliasedStorages.TryGetValue(format, out var storage))
184              {
185                  _aliasedStorages ??= new Dictionary<Format, TextureStorage>();
186  
187                  var info = NewCreateInfoWith(ref _info, format, _info.BytesPerPixel);
188  
189                  storage = new TextureStorage(_gd, _device, info, _allocationAuto);
190  
191                  _aliasedStorages.Add(format, storage);
192              }
193  
194              return storage;
195          }
196  
197          public static TextureCreateInfo NewCreateInfoWith(ref TextureCreateInfo info, Format format, int bytesPerPixel)
198          {
199              return NewCreateInfoWith(ref info, format, bytesPerPixel, info.Width, info.Height);
200          }
201  
202          public static TextureCreateInfo NewCreateInfoWith(
203              ref TextureCreateInfo info,
204              Format format,
205              int bytesPerPixel,
206              int width,
207              int height)
208          {
209              return new TextureCreateInfo(
210                  width,
211                  height,
212                  info.Depth,
213                  info.Levels,
214                  info.Samples,
215                  info.BlockWidth,
216                  info.BlockHeight,
217                  bytesPerPixel,
218                  format,
219                  info.DepthStencilMode,
220                  info.Target,
221                  info.SwizzleR,
222                  info.SwizzleG,
223                  info.SwizzleB,
224                  info.SwizzleA);
225          }
226  
227          public Auto<DisposableImage> GetImage()
228          {
229              return _imageAuto;
230          }
231  
232          public Image GetImageForViewCreation()
233          {
234              return _image;
235          }
236  
237          public bool HasCommandBufferDependency(CommandBufferScoped cbs)
238          {
239              if (_foreignAllocationAuto != null)
240              {
241                  return _foreignAllocationAuto.HasCommandBufferDependency(cbs);
242              }
243              else if (_allocationAuto != null)
244              {
245                  return _allocationAuto.HasCommandBufferDependency(cbs);
246              }
247  
248              return false;
249          }
250  
251          private unsafe void InitialTransition(ImageLayout srcLayout, ImageLayout dstLayout)
252          {
253              CommandBufferScoped cbs;
254              bool useTempCbs = !_gd.CommandBufferPool.OwnedByCurrentThread;
255  
256              if (useTempCbs)
257              {
258                  cbs = _gd.BackgroundResources.Get().GetPool().Rent();
259              }
260              else
261              {
262                  if (_gd.PipelineInternal != null)
263                  {
264                      cbs = _gd.PipelineInternal.GetPreloadCommandBuffer();
265                  }
266                  else
267                  {
268                      cbs = _gd.CommandBufferPool.Rent();
269                      useTempCbs = true;
270                  }
271              }
272  
273              var aspectFlags = _info.Format.ConvertAspectFlags();
274  
275              var subresourceRange = new ImageSubresourceRange(aspectFlags, 0, (uint)_info.Levels, 0, (uint)_info.GetLayers());
276  
277              var barrier = new ImageMemoryBarrier
278              {
279                  SType = StructureType.ImageMemoryBarrier,
280                  SrcAccessMask = 0,
281                  DstAccessMask = DefaultAccessMask,
282                  OldLayout = srcLayout,
283                  NewLayout = dstLayout,
284                  SrcQueueFamilyIndex = Vk.QueueFamilyIgnored,
285                  DstQueueFamilyIndex = Vk.QueueFamilyIgnored,
286                  Image = _imageAuto.Get(cbs).Value,
287                  SubresourceRange = subresourceRange,
288              };
289  
290              _gd.Api.CmdPipelineBarrier(
291                  cbs.CommandBuffer,
292                  PipelineStageFlags.TopOfPipeBit,
293                  PipelineStageFlags.AllCommandsBit,
294                  0,
295                  0,
296                  null,
297                  0,
298                  null,
299                  1,
300                  in barrier);
301  
302              if (useTempCbs)
303              {
304                  cbs.Dispose();
305              }
306          }
307  
308          public static ImageUsageFlags GetImageUsage(Format format, Target target, in HardwareCapabilities capabilities)
309          {
310              var usage = DefaultUsageFlags;
311  
312              if (format.IsDepthOrStencil())
313              {
314                  usage |= ImageUsageFlags.DepthStencilAttachmentBit;
315              }
316              else if (format.IsRtColorCompatible())
317              {
318                  usage |= ImageUsageFlags.ColorAttachmentBit;
319              }
320  
321              bool supportsMsStorage = capabilities.SupportsShaderStorageImageMultisample;
322  
323              if (format.IsImageCompatible() && (supportsMsStorage || !target.IsMultisample()))
324              {
325                  usage |= ImageUsageFlags.StorageBit;
326              }
327  
328              if (capabilities.SupportsAttachmentFeedbackLoop &&
329                  (usage & (ImageUsageFlags.DepthStencilAttachmentBit | ImageUsageFlags.ColorAttachmentBit)) != 0)
330              {
331                  usage |= ImageUsageFlags.AttachmentFeedbackLoopBitExt;
332              }
333  
334              return usage;
335          }
336  
337          public static SampleCountFlags ConvertToSampleCountFlags(SampleCountFlags supportedSampleCounts, uint samples)
338          {
339              if (samples == 0 || samples > (uint)SampleCountFlags.Count64Bit)
340              {
341                  return SampleCountFlags.Count1Bit;
342              }
343  
344              // Round up to the nearest power of two.
345              SampleCountFlags converted = (SampleCountFlags)(1u << (31 - BitOperations.LeadingZeroCount(samples)));
346  
347              // Pick nearest sample count that the host actually supports.
348              while (converted != SampleCountFlags.Count1Bit && (converted & supportedSampleCounts) == 0)
349              {
350                  converted = (SampleCountFlags)((uint)converted >> 1);
351              }
352  
353              return converted;
354          }
355  
356          public TextureView CreateView(TextureCreateInfo info, int firstLayer, int firstLevel)
357          {
358              return new TextureView(_gd, _device, info, this, firstLayer, firstLevel);
359          }
360  
361          public void CopyFromOrToBuffer(
362              CommandBuffer commandBuffer,
363              VkBuffer buffer,
364              Image image,
365              int size,
366              bool to,
367              int x,
368              int y,
369              int dstLayer,
370              int dstLevel,
371              int dstLayers,
372              int dstLevels,
373              bool singleSlice,
374              ImageAspectFlags aspectFlags,
375              bool forFlush)
376          {
377              bool is3D = Info.Target == Target.Texture3D;
378              int width = Info.Width;
379              int height = Info.Height;
380              int depth = is3D && !singleSlice ? Info.Depth : 1;
381              int layer = is3D ? 0 : dstLayer;
382              int layers = dstLayers;
383              int levels = dstLevels;
384  
385              int offset = 0;
386  
387              for (int level = 0; level < levels; level++)
388              {
389                  int mipSize = Info.GetMipSize(level);
390  
391                  if (forFlush)
392                  {
393                      mipSize = GetBufferDataLength(mipSize);
394                  }
395  
396                  int endOffset = offset + mipSize;
397  
398                  if ((uint)endOffset > (uint)size)
399                  {
400                      break;
401                  }
402  
403                  int rowLength = (Info.GetMipStride(level) / Info.BytesPerPixel) * Info.BlockWidth;
404  
405                  var sl = new ImageSubresourceLayers(
406                      aspectFlags,
407                      (uint)(dstLevel + level),
408                      (uint)layer,
409                      (uint)layers);
410  
411                  var extent = new Extent3D((uint)width, (uint)height, (uint)depth);
412  
413                  int z = is3D ? dstLayer : 0;
414  
415                  var region = new BufferImageCopy(
416                      (ulong)offset,
417                      (uint)BitUtils.AlignUp(rowLength, Info.BlockWidth),
418                      (uint)BitUtils.AlignUp(height, Info.BlockHeight),
419                      sl,
420                      new Offset3D(x, y, z),
421                      extent);
422  
423                  if (to)
424                  {
425                      _gd.Api.CmdCopyImageToBuffer(commandBuffer, image, ImageLayout.General, buffer, 1, in region);
426                  }
427                  else
428                  {
429                      _gd.Api.CmdCopyBufferToImage(commandBuffer, buffer, image, ImageLayout.General, 1, in region);
430                  }
431  
432                  offset += mipSize;
433  
434                  width = Math.Max(1, width >> 1);
435                  height = Math.Max(1, height >> 1);
436  
437                  if (Info.Target == Target.Texture3D)
438                  {
439                      depth = Math.Max(1, depth >> 1);
440                  }
441              }
442          }
443  
444          private int GetBufferDataLength(int length)
445          {
446              if (NeedsD24S8Conversion())
447              {
448                  return length * 2;
449              }
450  
451              return length;
452          }
453  
454          private bool NeedsD24S8Conversion()
455          {
456              return FormatCapabilities.IsD24S8(Info.Format) && VkFormat == VkFormat.D32SfloatS8Uint;
457          }
458  
459          public void AddStoreOpUsage(bool depthStencil)
460          {
461              _lastModificationStage = depthStencil ?
462                  PipelineStageFlags.LateFragmentTestsBit :
463                  PipelineStageFlags.ColorAttachmentOutputBit;
464  
465              _lastModificationAccess = depthStencil ?
466                  AccessFlags.DepthStencilAttachmentWriteBit :
467                  AccessFlags.ColorAttachmentWriteBit;
468          }
469  
470          public void QueueLoadOpBarrier(CommandBufferScoped cbs, bool depthStencil)
471          {
472              PipelineStageFlags srcStageFlags = _lastReadStage | _lastModificationStage;
473              PipelineStageFlags dstStageFlags = depthStencil ?
474                  PipelineStageFlags.EarlyFragmentTestsBit | PipelineStageFlags.LateFragmentTestsBit :
475                  PipelineStageFlags.ColorAttachmentOutputBit;
476  
477              AccessFlags srcAccessFlags = _lastModificationAccess | _lastReadAccess;
478              AccessFlags dstAccessFlags = depthStencil ?
479                  AccessFlags.DepthStencilAttachmentWriteBit | AccessFlags.DepthStencilAttachmentReadBit :
480                  AccessFlags.ColorAttachmentWriteBit | AccessFlags.ColorAttachmentReadBit;
481  
482              if (srcAccessFlags != AccessFlags.None)
483              {
484                  ImageAspectFlags aspectFlags = Info.Format.ConvertAspectFlags();
485                  ImageMemoryBarrier barrier = TextureView.GetImageBarrier(
486                      _imageAuto.Get(cbs).Value,
487                      srcAccessFlags,
488                      dstAccessFlags,
489                      aspectFlags,
490                      0,
491                      0,
492                      _info.GetLayers(),
493                      _info.Levels);
494  
495                  _gd.Barriers.QueueBarrier(barrier, this, srcStageFlags, dstStageFlags);
496  
497                  _lastReadStage = PipelineStageFlags.None;
498                  _lastReadAccess = AccessFlags.None;
499              }
500  
501              _lastModificationStage = depthStencil ?
502                  PipelineStageFlags.LateFragmentTestsBit :
503                  PipelineStageFlags.ColorAttachmentOutputBit;
504  
505              _lastModificationAccess = depthStencil ?
506                  AccessFlags.DepthStencilAttachmentWriteBit :
507                  AccessFlags.ColorAttachmentWriteBit;
508          }
509  
510          public void QueueWriteToReadBarrier(CommandBufferScoped cbs, AccessFlags dstAccessFlags, PipelineStageFlags dstStageFlags)
511          {
512              _lastReadAccess |= dstAccessFlags;
513              _lastReadStage |= dstStageFlags;
514  
515              if (_lastModificationAccess != AccessFlags.None)
516              {
517                  ImageAspectFlags aspectFlags = Info.Format.ConvertAspectFlags();
518                  ImageMemoryBarrier barrier = TextureView.GetImageBarrier(
519                      _imageAuto.Get(cbs).Value,
520                      _lastModificationAccess,
521                      dstAccessFlags,
522                      aspectFlags,
523                      0,
524                      0,
525                      _info.GetLayers(),
526                      _info.Levels);
527  
528                  _gd.Barriers.QueueBarrier(barrier, this, _lastModificationStage, dstStageFlags);
529  
530                  _lastModificationAccess = AccessFlags.None;
531              }
532          }
533  
534          public void AddBinding(TextureView view)
535          {
536              // Assumes a view only has a first level.
537  
538              int index = view.FirstLevel * _depthOrLayers + view.FirstLayer;
539              int layers = view.Layers;
540  
541              for (int i = 0; i < layers; i++)
542              {
543                  ref TextureSliceInfo info = ref _slices[index++];
544  
545                  info.BindCount++;
546              }
547  
548              _bindCount++;
549          }
550  
551          public void ClearBindings()
552          {
553              if (_bindCount != 0)
554              {
555                  Array.Clear(_slices, 0, _slices.Length);
556  
557                  _bindCount = 0;
558              }
559          }
560  
561          [MethodImpl(MethodImplOptions.AggressiveInlining)]
562          public bool IsBound(TextureView view)
563          {
564              if (_bindCount != 0)
565              {
566                  int index = view.FirstLevel * _depthOrLayers + view.FirstLayer;
567                  int layers = view.Layers;
568  
569                  for (int i = 0; i < layers; i++)
570                  {
571                      ref TextureSliceInfo info = ref _slices[index++];
572  
573                      if (info.BindCount != 0)
574                      {
575                          return true;
576                      }
577                  }
578              }
579  
580              return false;
581          }
582  
583          public void IncrementViewsCount()
584          {
585              _viewsCount++;
586          }
587  
588          public void DecrementViewsCount()
589          {
590              if (--_viewsCount == 0)
591              {
592                  _gd.PipelineInternal?.FlushCommandsIfWeightExceeding(_imageAuto, _size);
593  
594                  Dispose();
595              }
596          }
597  
598          public void Dispose()
599          {
600              Disposed = true;
601  
602              if (_aliasedStorages != null)
603              {
604                  foreach (var storage in _aliasedStorages.Values)
605                  {
606                      storage.Dispose();
607                  }
608  
609                  _aliasedStorages.Clear();
610              }
611  
612              _imageAuto.Dispose();
613              _allocationAuto?.Dispose();
614              _foreignAllocationAuto?.DecrementReferenceCount();
615              _foreignAllocationAuto = null;
616          }
617      }
618  }