1  using Ryujinx.Common.Memory;
   2  using Ryujinx.Graphics.GAL;
   3  using Silk.NET.Vulkan;
   4  using System;
   5  using System.Collections.Generic;
   6  using System.Linq;
   7  using System.Threading;
   8  using Format = Ryujinx.Graphics.GAL.Format;
   9  using VkBuffer = Silk.NET.Vulkan.Buffer;
  10  using VkFormat = Silk.NET.Vulkan.Format;
  11  
  12  namespace Ryujinx.Graphics.Vulkan
  13  {
  14      class TextureView : ITexture, IDisposable
  15      {
  16          private readonly VulkanRenderer _gd;
  17  
  18          private readonly Device _device;
  19  
  20          private readonly Auto<DisposableImageView> _imageView;
  21          private readonly Auto<DisposableImageView> _imageViewDraw;
  22          private readonly Auto<DisposableImageView> _imageViewIdentity;
  23          private readonly Auto<DisposableImageView> _imageView2dArray;
  24          private Dictionary<Format, TextureView> _selfManagedViews;
  25  
  26          private int _hazardUses;
  27  
  28          private readonly TextureCreateInfo _info;
  29  
  30          private HashTableSlim<RenderPassCacheKey, RenderPassHolder> _renderPasses;
  31  
  32          public TextureCreateInfo Info => _info;
  33  
  34          public TextureStorage Storage { get; }
  35  
  36          public int Width => Info.Width;
  37          public int Height => Info.Height;
  38          public int Layers => Info.GetDepthOrLayers();
  39          public int FirstLayer { get; }
  40          public int FirstLevel { get; }
  41          public VkFormat VkFormat { get; }
  42          private int _isValid;
  43          public bool Valid => Volatile.Read(ref _isValid) != 0;
  44  
  45          public TextureView(
  46              VulkanRenderer gd,
  47              Device device,
  48              TextureCreateInfo info,
  49              TextureStorage storage,
  50              int firstLayer,
  51              int firstLevel)
  52          {
  53              _gd = gd;
  54              _device = device;
  55              _info = info;
  56              Storage = storage;
  57              FirstLayer = firstLayer;
  58              FirstLevel = firstLevel;
  59  
  60              storage.IncrementViewsCount();
  61  
  62              gd.Textures.Add(this);
  63  
  64              var format = _gd.FormatCapabilities.ConvertToVkFormat(info.Format);
  65              var usage = TextureStorage.GetImageUsage(info.Format, info.Target, gd.Capabilities);
  66              var levels = (uint)info.Levels;
  67              var layers = (uint)info.GetLayers();
  68  
  69              VkFormat = format;
  70  
  71              var type = info.Target.ConvertView();
  72  
  73              var swizzleR = info.SwizzleR.Convert();
  74              var swizzleG = info.SwizzleG.Convert();
  75              var swizzleB = info.SwizzleB.Convert();
  76              var swizzleA = info.SwizzleA.Convert();
  77  
  78              if (info.Format == Format.R5G5B5A1Unorm ||
  79                  info.Format == Format.R5G5B5X1Unorm ||
  80                  info.Format == Format.R5G6B5Unorm)
  81              {
  82                  (swizzleB, swizzleR) = (swizzleR, swizzleB);
  83              }
  84              else if (VkFormat == VkFormat.R4G4B4A4UnormPack16 || info.Format == Format.A1B5G5R5Unorm)
  85              {
  86                  var tempB = swizzleB;
  87                  var tempA = swizzleA;
  88  
  89                  swizzleB = swizzleG;
  90                  swizzleA = swizzleR;
  91                  swizzleR = tempA;
  92                  swizzleG = tempB;
  93              }
  94  
  95              var componentMapping = new ComponentMapping(swizzleR, swizzleG, swizzleB, swizzleA);
  96  
  97              var aspectFlags = info.Format.ConvertAspectFlags(info.DepthStencilMode);
  98              var aspectFlagsDepth = info.Format.ConvertAspectFlags();
  99  
 100              var subresourceRange = new ImageSubresourceRange(aspectFlags, (uint)firstLevel, levels, (uint)firstLayer, layers);
 101              var subresourceRangeDepth = new ImageSubresourceRange(aspectFlagsDepth, (uint)firstLevel, levels, (uint)firstLayer, layers);
 102  
 103              unsafe Auto<DisposableImageView> CreateImageView(ComponentMapping cm, ImageSubresourceRange sr, ImageViewType viewType, ImageUsageFlags usageFlags)
 104              {
 105                  var imageViewUsage = new ImageViewUsageCreateInfo
 106                  {
 107                      SType = StructureType.ImageViewUsageCreateInfo,
 108                      Usage = usageFlags,
 109                  };
 110  
 111                  var imageCreateInfo = new ImageViewCreateInfo
 112                  {
 113                      SType = StructureType.ImageViewCreateInfo,
 114                      Image = storage.GetImageForViewCreation(),
 115                      ViewType = viewType,
 116                      Format = format,
 117                      Components = cm,
 118                      SubresourceRange = sr,
 119                      PNext = &imageViewUsage,
 120                  };
 121  
 122                  gd.Api.CreateImageView(device, in imageCreateInfo, null, out var imageView).ThrowOnError();
 123                  return new Auto<DisposableImageView>(new DisposableImageView(gd.Api, device, imageView), null, storage.GetImage());
 124              }
 125  
 126              ImageUsageFlags shaderUsage = ImageUsageFlags.SampledBit;
 127  
 128              if (info.Format.IsImageCompatible() && (_gd.Capabilities.SupportsShaderStorageImageMultisample || !info.Target.IsMultisample()))
 129              {
 130                  shaderUsage |= ImageUsageFlags.StorageBit;
 131              }
 132  
 133              _imageView = CreateImageView(componentMapping, subresourceRange, type, shaderUsage);
 134  
 135              // Framebuffer attachments and storage images requires a identity component mapping.
 136              var identityComponentMapping = new ComponentMapping(
 137                  ComponentSwizzle.R,
 138                  ComponentSwizzle.G,
 139                  ComponentSwizzle.B,
 140                  ComponentSwizzle.A);
 141  
 142              _imageViewDraw = CreateImageView(identityComponentMapping, subresourceRangeDepth, type, usage);
 143              _imageViewIdentity = aspectFlagsDepth == aspectFlags ? _imageViewDraw : CreateImageView(identityComponentMapping, subresourceRange, type, usage);
 144  
 145              // Framebuffer attachments also require 3D textures to be bound as 2D array.
 146              if (info.Target == Target.Texture3D)
 147              {
 148                  if (gd.Capabilities.PortabilitySubset.HasFlag(PortabilitySubsetFlags.No3DImageView))
 149                  {
 150                      if (levels == 1 && (info.Format.IsRtColorCompatible() || info.Format.IsDepthOrStencil()))
 151                      {
 152                          subresourceRange = new ImageSubresourceRange(aspectFlags, (uint)firstLevel, levels, (uint)firstLayer, 1);
 153  
 154                          _imageView2dArray = CreateImageView(identityComponentMapping, subresourceRange, ImageViewType.Type2D, ImageUsageFlags.ColorAttachmentBit);
 155                      }
 156                  }
 157                  else
 158                  {
 159                      subresourceRange = new ImageSubresourceRange(aspectFlags, (uint)firstLevel, 1, (uint)firstLayer, (uint)info.Depth);
 160  
 161                      _imageView2dArray = CreateImageView(identityComponentMapping, subresourceRange, ImageViewType.Type2DArray, usage);
 162                  }
 163              }
 164  
 165              _isValid = 1;
 166          }
 167  
 168          /// <summary>
 169          /// Create a texture view for an existing swapchain image view.
 170          /// Does not set storage, so only appropriate for swapchain use.
 171          /// </summary>
 172          /// <remarks>Do not use this for normal textures, and make sure uses do not try to read storage.</remarks>
 173          public TextureView(VulkanRenderer gd, Device device, DisposableImageView view, TextureCreateInfo info, VkFormat format)
 174          {
 175              _gd = gd;
 176              _device = device;
 177  
 178              _imageView = new Auto<DisposableImageView>(view);
 179              _imageViewDraw = _imageView;
 180              _imageViewIdentity = _imageView;
 181              _info = info;
 182  
 183              VkFormat = format;
 184  
 185              _isValid = 1;
 186          }
 187  
 188          public Auto<DisposableImage> GetImage()
 189          {
 190              return Storage.GetImage();
 191          }
 192  
 193          public Auto<DisposableImageView> GetImageView()
 194          {
 195              return _imageView;
 196          }
 197  
 198          public Auto<DisposableImageView> GetIdentityImageView()
 199          {
 200              return _imageViewIdentity;
 201          }
 202  
 203          public Auto<DisposableImageView> GetImageViewForAttachment()
 204          {
 205              return _imageView2dArray ?? _imageViewDraw;
 206          }
 207  
 208          public void CopyTo(ITexture destination, int firstLayer, int firstLevel)
 209          {
 210              var src = this;
 211              var dst = (TextureView)destination;
 212  
 213              if (!Valid || !dst.Valid)
 214              {
 215                  return;
 216              }
 217  
 218              _gd.PipelineInternal.EndRenderPass();
 219  
 220              var cbs = _gd.PipelineInternal.CurrentCommandBuffer;
 221  
 222              var srcImage = src.GetImage().Get(cbs).Value;
 223              var dstImage = dst.GetImage().Get(cbs).Value;
 224  
 225              if (!dst.Info.Target.IsMultisample() && Info.Target.IsMultisample())
 226              {
 227                  int layers = Math.Min(Info.GetLayers(), dst.Info.GetLayers() - firstLayer);
 228                  _gd.HelperShader.CopyMSToNonMS(_gd, cbs, src, dst, 0, firstLayer, layers);
 229              }
 230              else if (dst.Info.Target.IsMultisample() && !Info.Target.IsMultisample())
 231              {
 232                  int layers = Math.Min(Info.GetLayers(), dst.Info.GetLayers() - firstLayer);
 233                  _gd.HelperShader.CopyNonMSToMS(_gd, cbs, src, dst, 0, firstLayer, layers);
 234              }
 235              else if (dst.Info.BytesPerPixel != Info.BytesPerPixel)
 236              {
 237                  int layers = Math.Min(Info.GetLayers(), dst.Info.GetLayers() - firstLayer);
 238                  int levels = Math.Min(Info.Levels, dst.Info.Levels - firstLevel);
 239                  _gd.HelperShader.CopyIncompatibleFormats(_gd, cbs, src, dst, 0, firstLayer, 0, firstLevel, layers, levels);
 240              }
 241              else if (src.Info.Format.IsDepthOrStencil() != dst.Info.Format.IsDepthOrStencil())
 242              {
 243                  int layers = Math.Min(Info.GetLayers(), dst.Info.GetLayers() - firstLayer);
 244                  int levels = Math.Min(Info.Levels, dst.Info.Levels - firstLevel);
 245  
 246                  _gd.HelperShader.CopyColor(_gd, cbs, src, dst, 0, firstLayer, 0, FirstLevel, layers, levels);
 247              }
 248              else
 249              {
 250                  TextureCopy.Copy(
 251                      _gd.Api,
 252                      cbs.CommandBuffer,
 253                      srcImage,
 254                      dstImage,
 255                      src.Info,
 256                      dst.Info,
 257                      src.FirstLayer,
 258                      dst.FirstLayer,
 259                      src.FirstLevel,
 260                      dst.FirstLevel,
 261                      0,
 262                      firstLayer,
 263                      0,
 264                      firstLevel);
 265              }
 266          }
 267  
 268          public void CopyTo(ITexture destination, int srcLayer, int dstLayer, int srcLevel, int dstLevel)
 269          {
 270              var src = this;
 271              var dst = (TextureView)destination;
 272  
 273              if (!Valid || !dst.Valid)
 274              {
 275                  return;
 276              }
 277  
 278              _gd.PipelineInternal.EndRenderPass();
 279  
 280              var cbs = _gd.PipelineInternal.CurrentCommandBuffer;
 281  
 282              var srcImage = src.GetImage().Get(cbs).Value;
 283              var dstImage = dst.GetImage().Get(cbs).Value;
 284  
 285              if (!dst.Info.Target.IsMultisample() && Info.Target.IsMultisample())
 286              {
 287                  _gd.HelperShader.CopyMSToNonMS(_gd, cbs, src, dst, srcLayer, dstLayer, 1);
 288              }
 289              else if (dst.Info.Target.IsMultisample() && !Info.Target.IsMultisample())
 290              {
 291                  _gd.HelperShader.CopyNonMSToMS(_gd, cbs, src, dst, srcLayer, dstLayer, 1);
 292              }
 293              else if (dst.Info.BytesPerPixel != Info.BytesPerPixel)
 294              {
 295                  _gd.HelperShader.CopyIncompatibleFormats(_gd, cbs, src, dst, srcLayer, dstLayer, srcLevel, dstLevel, 1, 1);
 296              }
 297              else if (src.Info.Format.IsDepthOrStencil() != dst.Info.Format.IsDepthOrStencil())
 298              {
 299                  _gd.HelperShader.CopyColor(_gd, cbs, src, dst, srcLayer, dstLayer, srcLevel, dstLevel, 1, 1);
 300              }
 301              else
 302              {
 303                  TextureCopy.Copy(
 304                      _gd.Api,
 305                      cbs.CommandBuffer,
 306                      srcImage,
 307                      dstImage,
 308                      src.Info,
 309                      dst.Info,
 310                      src.FirstLayer,
 311                      dst.FirstLayer,
 312                      src.FirstLevel,
 313                      dst.FirstLevel,
 314                      srcLayer,
 315                      dstLayer,
 316                      srcLevel,
 317                      dstLevel,
 318                      1,
 319                      1);
 320              }
 321          }
 322  
 323          public void CopyTo(ITexture destination, Extents2D srcRegion, Extents2D dstRegion, bool linearFilter)
 324          {
 325              var dst = (TextureView)destination;
 326  
 327              if (_gd.CommandBufferPool.OwnedByCurrentThread)
 328              {
 329                  _gd.PipelineInternal.EndRenderPass();
 330  
 331                  var cbs = _gd.PipelineInternal.CurrentCommandBuffer;
 332  
 333                  CopyToImpl(cbs, dst, srcRegion, dstRegion, linearFilter);
 334              }
 335              else
 336              {
 337                  var cbp = _gd.BackgroundResources.Get().GetPool();
 338  
 339                  using var cbs = cbp.Rent();
 340  
 341                  CopyToImpl(cbs, dst, srcRegion, dstRegion, linearFilter);
 342              }
 343          }
 344  
 345          private void CopyToImpl(CommandBufferScoped cbs, TextureView dst, Extents2D srcRegion, Extents2D dstRegion, bool linearFilter)
 346          {
 347              var src = this;
 348  
 349              var srcFormat = GetCompatibleGalFormat(src.Info.Format);
 350              var dstFormat = GetCompatibleGalFormat(dst.Info.Format);
 351  
 352              bool srcUsesStorageFormat = src.VkFormat == src.Storage.VkFormat;
 353              bool dstUsesStorageFormat = dst.VkFormat == dst.Storage.VkFormat;
 354  
 355              int layers = Math.Min(dst.Info.GetDepthOrLayers(), src.Info.GetDepthOrLayers());
 356              int levels = Math.Min(dst.Info.Levels, src.Info.Levels);
 357  
 358              if (srcUsesStorageFormat && dstUsesStorageFormat)
 359              {
 360                  if ((srcRegion.X1 | dstRegion.X1) == 0 &&
 361                      (srcRegion.Y1 | dstRegion.Y1) == 0 &&
 362                      srcRegion.X2 == src.Width &&
 363                      srcRegion.Y2 == src.Height &&
 364                      dstRegion.X2 == dst.Width &&
 365                      dstRegion.Y2 == dst.Height &&
 366                      src.Width == dst.Width &&
 367                      src.Height == dst.Height &&
 368                      src.VkFormat == dst.VkFormat)
 369                  {
 370                      if (src.Info.Samples > 1 && src.Info.Samples != dst.Info.Samples && src.Info.Format.IsDepthOrStencil())
 371                      {
 372                          // CmdResolveImage does not support depth-stencil resolve, so we need to use an alternative path
 373                          // for those textures.
 374                          TextureCopy.ResolveDepthStencil(_gd, _device, cbs, src, dst);
 375                      }
 376                      else
 377                      {
 378                          TextureCopy.Copy(
 379                              _gd.Api,
 380                              cbs.CommandBuffer,
 381                              src.GetImage().Get(cbs).Value,
 382                              dst.GetImage().Get(cbs).Value,
 383                              src.Info,
 384                              dst.Info,
 385                              src.FirstLayer,
 386                              dst.FirstLayer,
 387                              src.FirstLevel,
 388                              dst.FirstLevel,
 389                              0,
 390                              0,
 391                              0,
 392                              0,
 393                              layers,
 394                              levels);
 395                      }
 396  
 397                      return;
 398                  }
 399  
 400                  if (_gd.FormatCapabilities.OptimalFormatSupports(FormatFeatureFlags.BlitSrcBit, srcFormat) &&
 401                      _gd.FormatCapabilities.OptimalFormatSupports(FormatFeatureFlags.BlitDstBit, dstFormat))
 402                  {
 403                      TextureCopy.Blit(
 404                          _gd.Api,
 405                          cbs.CommandBuffer,
 406                          src.GetImage().Get(cbs).Value,
 407                          dst.GetImage().Get(cbs).Value,
 408                          src.Info,
 409                          dst.Info,
 410                          srcRegion,
 411                          dstRegion,
 412                          src.FirstLayer,
 413                          dst.FirstLayer,
 414                          src.FirstLevel,
 415                          dst.FirstLevel,
 416                          layers,
 417                          levels,
 418                          linearFilter);
 419  
 420                      return;
 421                  }
 422              }
 423  
 424              bool isDepthOrStencil = dst.Info.Format.IsDepthOrStencil();
 425  
 426              if (!VulkanConfiguration.UseUnsafeBlit || (_gd.Vendor != Vendor.Nvidia && _gd.Vendor != Vendor.Intel))
 427              {
 428                  _gd.HelperShader.Blit(
 429                      _gd,
 430                      src,
 431                      dst,
 432                      srcRegion,
 433                      dstRegion,
 434                      layers,
 435                      levels,
 436                      isDepthOrStencil,
 437                      linearFilter);
 438  
 439                  return;
 440              }
 441  
 442              Auto<DisposableImage> srcImage;
 443              Auto<DisposableImage> dstImage;
 444  
 445              if (isDepthOrStencil)
 446              {
 447                  srcImage = src.Storage.CreateAliasedColorForDepthStorageUnsafe(srcFormat).GetImage();
 448                  dstImage = dst.Storage.CreateAliasedColorForDepthStorageUnsafe(dstFormat).GetImage();
 449              }
 450              else
 451              {
 452                  srcImage = src.Storage.CreateAliasedStorageUnsafe(srcFormat).GetImage();
 453                  dstImage = dst.Storage.CreateAliasedStorageUnsafe(dstFormat).GetImage();
 454              }
 455  
 456              TextureCopy.Blit(
 457                  _gd.Api,
 458                  cbs.CommandBuffer,
 459                  srcImage.Get(cbs).Value,
 460                  dstImage.Get(cbs).Value,
 461                  src.Info,
 462                  dst.Info,
 463                  srcRegion,
 464                  dstRegion,
 465                  src.FirstLayer,
 466                  dst.FirstLayer,
 467                  src.FirstLevel,
 468                  dst.FirstLevel,
 469                  layers,
 470                  levels,
 471                  linearFilter,
 472                  ImageAspectFlags.ColorBit,
 473                  ImageAspectFlags.ColorBit);
 474          }
 475  
 476          public static unsafe void InsertMemoryBarrier(
 477              Vk api,
 478              CommandBuffer commandBuffer,
 479              AccessFlags srcAccessMask,
 480              AccessFlags dstAccessMask,
 481              PipelineStageFlags srcStageMask,
 482              PipelineStageFlags dstStageMask)
 483          {
 484              MemoryBarrier memoryBarrier = new()
 485              {
 486                  SType = StructureType.MemoryBarrier,
 487                  SrcAccessMask = srcAccessMask,
 488                  DstAccessMask = dstAccessMask,
 489              };
 490  
 491              api.CmdPipelineBarrier(
 492                  commandBuffer,
 493                  srcStageMask,
 494                  dstStageMask,
 495                  DependencyFlags.None,
 496                  1,
 497                  in memoryBarrier,
 498                  0,
 499                  null,
 500                  0,
 501                  null);
 502          }
 503  
 504          public static ImageMemoryBarrier GetImageBarrier(
 505              Image image,
 506              AccessFlags srcAccessMask,
 507              AccessFlags dstAccessMask,
 508              ImageAspectFlags aspectFlags,
 509              int firstLayer,
 510              int firstLevel,
 511              int layers,
 512              int levels)
 513          {
 514              return new()
 515              {
 516                  SType = StructureType.ImageMemoryBarrier,
 517                  SrcAccessMask = srcAccessMask,
 518                  DstAccessMask = dstAccessMask,
 519                  SrcQueueFamilyIndex = Vk.QueueFamilyIgnored,
 520                  DstQueueFamilyIndex = Vk.QueueFamilyIgnored,
 521                  Image = image,
 522                  OldLayout = ImageLayout.General,
 523                  NewLayout = ImageLayout.General,
 524                  SubresourceRange = new ImageSubresourceRange(aspectFlags, (uint)firstLevel, (uint)levels, (uint)firstLayer, (uint)layers),
 525              };
 526          }
 527  
 528          public static unsafe void InsertImageBarrier(
 529              Vk api,
 530              CommandBuffer commandBuffer,
 531              Image image,
 532              AccessFlags srcAccessMask,
 533              AccessFlags dstAccessMask,
 534              PipelineStageFlags srcStageMask,
 535              PipelineStageFlags dstStageMask,
 536              ImageAspectFlags aspectFlags,
 537              int firstLayer,
 538              int firstLevel,
 539              int layers,
 540              int levels)
 541          {
 542              ImageMemoryBarrier memoryBarrier = GetImageBarrier(
 543                  image,
 544                  srcAccessMask,
 545                  dstAccessMask,
 546                  aspectFlags,
 547                  firstLayer,
 548                  firstLevel,
 549                  layers,
 550                  levels);
 551  
 552              api.CmdPipelineBarrier(
 553                  commandBuffer,
 554                  srcStageMask,
 555                  dstStageMask,
 556                  0,
 557                  0,
 558                  null,
 559                  0,
 560                  null,
 561                  1,
 562                  in memoryBarrier);
 563          }
 564  
 565          public TextureView GetView(Format format)
 566          {
 567              if (format == Info.Format)
 568              {
 569                  return this;
 570              }
 571  
 572              if (_selfManagedViews != null && _selfManagedViews.TryGetValue(format, out var view))
 573              {
 574                  return view;
 575              }
 576  
 577              view = CreateViewImpl(new TextureCreateInfo(
 578                  Info.Width,
 579                  Info.Height,
 580                  Info.Depth,
 581                  Info.Levels,
 582                  Info.Samples,
 583                  Info.BlockWidth,
 584                  Info.BlockHeight,
 585                  Info.BytesPerPixel,
 586                  format,
 587                  Info.DepthStencilMode,
 588                  Info.Target,
 589                  Info.SwizzleR,
 590                  Info.SwizzleG,
 591                  Info.SwizzleB,
 592                  Info.SwizzleA), 0, 0);
 593  
 594              (_selfManagedViews ??= new Dictionary<Format, TextureView>()).Add(format, view);
 595  
 596              return view;
 597          }
 598  
 599          public ITexture CreateView(TextureCreateInfo info, int firstLayer, int firstLevel)
 600          {
 601              return CreateViewImpl(info, firstLayer, firstLevel);
 602          }
 603  
 604          public TextureView CreateViewImpl(TextureCreateInfo info, int firstLayer, int firstLevel)
 605          {
 606              return new TextureView(_gd, _device, info, Storage, FirstLayer + firstLayer, FirstLevel + firstLevel);
 607          }
 608  
 609          public byte[] GetData(int x, int y, int width, int height)
 610          {
 611              int size = width * height * Info.BytesPerPixel;
 612              using var bufferHolder = _gd.BufferManager.Create(_gd, size);
 613  
 614              using (var cbs = _gd.CommandBufferPool.Rent())
 615              {
 616                  var buffer = bufferHolder.GetBuffer(cbs.CommandBuffer).Get(cbs).Value;
 617                  var image = GetImage().Get(cbs).Value;
 618  
 619                  CopyFromOrToBuffer(cbs.CommandBuffer, buffer, image, size, true, 0, 0, x, y, width, height);
 620              }
 621  
 622              bufferHolder.WaitForFences();
 623              byte[] bitmap = new byte[size];
 624              GetDataFromBuffer(bufferHolder.GetDataStorage(0, size), size, Span<byte>.Empty).CopyTo(bitmap);
 625              return bitmap;
 626          }
 627  
 628          public PinnedSpan<byte> GetData()
 629          {
 630              BackgroundResource resources = _gd.BackgroundResources.Get();
 631  
 632              if (_gd.CommandBufferPool.OwnedByCurrentThread)
 633              {
 634                  _gd.FlushAllCommands();
 635  
 636                  return PinnedSpan<byte>.UnsafeFromSpan(GetData(_gd.CommandBufferPool, resources.GetFlushBuffer()));
 637              }
 638  
 639              return PinnedSpan<byte>.UnsafeFromSpan(GetData(resources.GetPool(), resources.GetFlushBuffer()));
 640          }
 641  
 642          public PinnedSpan<byte> GetData(int layer, int level)
 643          {
 644              BackgroundResource resources = _gd.BackgroundResources.Get();
 645  
 646              if (_gd.CommandBufferPool.OwnedByCurrentThread)
 647              {
 648                  _gd.FlushAllCommands();
 649  
 650                  return PinnedSpan<byte>.UnsafeFromSpan(GetData(_gd.CommandBufferPool, resources.GetFlushBuffer(), layer, level));
 651              }
 652  
 653              return PinnedSpan<byte>.UnsafeFromSpan(GetData(resources.GetPool(), resources.GetFlushBuffer(), layer, level));
 654          }
 655  
 656          public void CopyTo(BufferRange range, int layer, int level, int stride)
 657          {
 658              _gd.PipelineInternal.EndRenderPass();
 659              var cbs = _gd.PipelineInternal.CurrentCommandBuffer;
 660  
 661              int outSize = Info.GetMipSize(level);
 662              int hostSize = GetBufferDataLength(outSize);
 663  
 664              var image = GetImage().Get(cbs).Value;
 665              int offset = range.Offset;
 666  
 667              Auto<DisposableBuffer> autoBuffer = _gd.BufferManager.GetBuffer(cbs.CommandBuffer, range.Handle, true);
 668              VkBuffer buffer = autoBuffer.Get(cbs, range.Offset, outSize).Value;
 669  
 670              if (PrepareOutputBuffer(cbs, hostSize, buffer, out VkBuffer copyToBuffer, out BufferHolder tempCopyHolder))
 671              {
 672                  // No barrier necessary, as this is a temporary copy buffer.
 673                  offset = 0;
 674              }
 675              else
 676              {
 677                  BufferHolder.InsertBufferBarrier(
 678                      _gd,
 679                      cbs.CommandBuffer,
 680                      copyToBuffer,
 681                      BufferHolder.DefaultAccessFlags,
 682                      AccessFlags.TransferWriteBit,
 683                      PipelineStageFlags.AllCommandsBit,
 684                      PipelineStageFlags.TransferBit,
 685                      offset,
 686                      outSize);
 687              }
 688  
 689              InsertImageBarrier(
 690                  _gd.Api,
 691                  cbs.CommandBuffer,
 692                  image,
 693                  TextureStorage.DefaultAccessMask,
 694                  AccessFlags.TransferReadBit,
 695                  PipelineStageFlags.AllCommandsBit,
 696                  PipelineStageFlags.TransferBit,
 697                  Info.Format.ConvertAspectFlags(),
 698                  FirstLayer + layer,
 699                  FirstLevel + level,
 700                  1,
 701                  1);
 702  
 703              CopyFromOrToBuffer(cbs.CommandBuffer, copyToBuffer, image, hostSize, true, layer, level, 1, 1, singleSlice: true, offset, stride);
 704  
 705              if (tempCopyHolder != null)
 706              {
 707                  CopyDataToOutputBuffer(cbs, tempCopyHolder, autoBuffer, hostSize, range.Offset);
 708                  tempCopyHolder.Dispose();
 709              }
 710              else
 711              {
 712                  BufferHolder.InsertBufferBarrier(
 713                      _gd,
 714                      cbs.CommandBuffer,
 715                      copyToBuffer,
 716                      AccessFlags.TransferWriteBit,
 717                      BufferHolder.DefaultAccessFlags,
 718                      PipelineStageFlags.TransferBit,
 719                      PipelineStageFlags.AllCommandsBit,
 720                      offset,
 721                      outSize);
 722              }
 723          }
 724  
 725          private ReadOnlySpan<byte> GetData(CommandBufferPool cbp, PersistentFlushBuffer flushBuffer)
 726          {
 727              int size = 0;
 728  
 729              for (int level = 0; level < Info.Levels; level++)
 730              {
 731                  size += Info.GetMipSize(level);
 732              }
 733  
 734              size = GetBufferDataLength(size);
 735  
 736              Span<byte> result = flushBuffer.GetTextureData(cbp, this, size);
 737              return GetDataFromBuffer(result, size, result);
 738          }
 739  
 740          private ReadOnlySpan<byte> GetData(CommandBufferPool cbp, PersistentFlushBuffer flushBuffer, int layer, int level)
 741          {
 742              int size = GetBufferDataLength(Info.GetMipSize(level));
 743  
 744              Span<byte> result = flushBuffer.GetTextureData(cbp, this, size, layer, level);
 745              return GetDataFromBuffer(result, size, result);
 746          }
 747  
 748          /// <inheritdoc/>
 749          public void SetData(MemoryOwner<byte> data)
 750          {
 751              SetData(data.Span, 0, 0, Info.GetLayers(), Info.Levels, singleSlice: false);
 752              data.Dispose();
 753          }
 754  
 755          /// <inheritdoc/>
 756          public void SetData(MemoryOwner<byte> data, int layer, int level)
 757          {
 758              SetData(data.Span, layer, level, 1, 1, singleSlice: true);
 759              data.Dispose();
 760          }
 761  
 762          /// <inheritdoc/>
 763          public void SetData(MemoryOwner<byte> data, int layer, int level, Rectangle<int> region)
 764          {
 765              SetData(data.Span, layer, level, 1, 1, singleSlice: true, region);
 766              data.Dispose();
 767          }
 768  
 769          private void SetData(ReadOnlySpan<byte> data, int layer, int level, int layers, int levels, bool singleSlice, Rectangle<int>? region = null)
 770          {
 771              int bufferDataLength = GetBufferDataLength(data.Length);
 772  
 773              using var bufferHolder = _gd.BufferManager.Create(_gd, bufferDataLength);
 774  
 775              Auto<DisposableImage> imageAuto = GetImage();
 776  
 777              // Load texture data inline if the texture has been used on the current command buffer.
 778  
 779              bool loadInline = Storage.HasCommandBufferDependency(_gd.PipelineInternal.CurrentCommandBuffer);
 780  
 781              var cbs = loadInline ? _gd.PipelineInternal.CurrentCommandBuffer : _gd.PipelineInternal.GetPreloadCommandBuffer();
 782  
 783              if (loadInline)
 784              {
 785                  _gd.PipelineInternal.EndRenderPass();
 786              }
 787  
 788              CopyDataToBuffer(bufferHolder.GetDataStorage(0, bufferDataLength), data);
 789  
 790              var buffer = bufferHolder.GetBuffer(cbs.CommandBuffer).Get(cbs).Value;
 791              var image = imageAuto.Get(cbs).Value;
 792  
 793              if (region.HasValue)
 794              {
 795                  CopyFromOrToBuffer(
 796                      cbs.CommandBuffer,
 797                      buffer,
 798                      image,
 799                      bufferDataLength,
 800                      false,
 801                      layer,
 802                      level,
 803                      region.Value.X,
 804                      region.Value.Y,
 805                      region.Value.Width,
 806                      region.Value.Height);
 807              }
 808              else
 809              {
 810                  CopyFromOrToBuffer(cbs.CommandBuffer, buffer, image, bufferDataLength, false, layer, level, layers, levels, singleSlice);
 811              }
 812          }
 813  
 814          private int GetBufferDataLength(int length)
 815          {
 816              if (NeedsD24S8Conversion())
 817              {
 818                  return length * 2;
 819              }
 820  
 821              return length;
 822          }
 823  
 824          private Format GetCompatibleGalFormat(Format format)
 825          {
 826              if (NeedsD24S8Conversion())
 827              {
 828                  return Format.D32FloatS8Uint;
 829              }
 830  
 831              return format;
 832          }
 833  
 834          private void CopyDataToBuffer(Span<byte> storage, ReadOnlySpan<byte> input)
 835          {
 836              if (NeedsD24S8Conversion())
 837              {
 838                  FormatConverter.ConvertD24S8ToD32FS8(storage, input);
 839                  return;
 840              }
 841  
 842              input.CopyTo(storage);
 843          }
 844  
 845          private ReadOnlySpan<byte> GetDataFromBuffer(ReadOnlySpan<byte> storage, int size, Span<byte> output)
 846          {
 847              if (NeedsD24S8Conversion())
 848              {
 849                  if (output.IsEmpty)
 850                  {
 851                      output = new byte[GetBufferDataLength(size)];
 852                  }
 853  
 854                  FormatConverter.ConvertD32FS8ToD24S8(output, storage);
 855                  return output;
 856              }
 857  
 858              return storage;
 859          }
 860  
 861          private bool PrepareOutputBuffer(CommandBufferScoped cbs, int hostSize, VkBuffer target, out VkBuffer copyTarget, out BufferHolder copyTargetHolder)
 862          {
 863              if (NeedsD24S8Conversion())
 864              {
 865                  copyTargetHolder = _gd.BufferManager.Create(_gd, hostSize);
 866                  copyTarget = copyTargetHolder.GetBuffer().Get(cbs, 0, hostSize).Value;
 867  
 868                  return true;
 869              }
 870  
 871              copyTarget = target;
 872              copyTargetHolder = null;
 873  
 874              return false;
 875          }
 876  
 877          private void CopyDataToOutputBuffer(CommandBufferScoped cbs, BufferHolder hostData, Auto<DisposableBuffer> copyTarget, int hostSize, int dstOffset)
 878          {
 879              if (NeedsD24S8Conversion())
 880              {
 881                  _gd.HelperShader.ConvertD32S8ToD24S8(_gd, cbs, hostData, copyTarget, hostSize / (2 * sizeof(int)), dstOffset);
 882              }
 883          }
 884  
 885          private bool NeedsD24S8Conversion()
 886          {
 887              return FormatCapabilities.IsD24S8(Info.Format) && VkFormat == VkFormat.D32SfloatS8Uint;
 888          }
 889  
 890          public void CopyFromOrToBuffer(
 891              CommandBuffer commandBuffer,
 892              VkBuffer buffer,
 893              Image image,
 894              int size,
 895              bool to,
 896              int dstLayer,
 897              int dstLevel,
 898              int dstLayers,
 899              int dstLevels,
 900              bool singleSlice,
 901              int offset = 0,
 902              int stride = 0)
 903          {
 904              bool is3D = Info.Target == Target.Texture3D;
 905              int width = Math.Max(1, Info.Width >> dstLevel);
 906              int height = Math.Max(1, Info.Height >> dstLevel);
 907              int depth = is3D && !singleSlice ? Math.Max(1, Info.Depth >> dstLevel) : 1;
 908              int layer = is3D ? 0 : dstLayer;
 909              int layers = dstLayers;
 910              int levels = dstLevels;
 911  
 912              for (int level = 0; level < levels; level++)
 913              {
 914                  int mipSize = GetBufferDataLength(is3D && !singleSlice
 915                      ? Info.GetMipSize(dstLevel + level)
 916                      : Info.GetMipSize2D(dstLevel + level) * dstLayers);
 917  
 918                  int endOffset = offset + mipSize;
 919  
 920                  if ((uint)endOffset > (uint)size)
 921                  {
 922                      break;
 923                  }
 924  
 925                  int rowLength = ((stride == 0 ? Info.GetMipStride(dstLevel + level) : stride) / Info.BytesPerPixel) * Info.BlockWidth;
 926  
 927                  var aspectFlags = Info.Format.ConvertAspectFlags();
 928  
 929                  if (aspectFlags == (ImageAspectFlags.DepthBit | ImageAspectFlags.StencilBit))
 930                  {
 931                      aspectFlags = ImageAspectFlags.DepthBit;
 932                  }
 933  
 934                  var sl = new ImageSubresourceLayers(
 935                      aspectFlags,
 936                      (uint)(FirstLevel + dstLevel + level),
 937                      (uint)(FirstLayer + layer),
 938                      (uint)layers);
 939  
 940                  var extent = new Extent3D((uint)width, (uint)height, (uint)depth);
 941  
 942                  int z = is3D ? dstLayer : 0;
 943  
 944                  var region = new BufferImageCopy(
 945                      (ulong)offset,
 946                      (uint)AlignUpNpot(rowLength, Info.BlockWidth),
 947                      (uint)AlignUpNpot(height, Info.BlockHeight),
 948                      sl,
 949                      new Offset3D(0, 0, z),
 950                      extent);
 951  
 952                  if (to)
 953                  {
 954                      _gd.Api.CmdCopyImageToBuffer(commandBuffer, image, ImageLayout.General, buffer, 1, in region);
 955                  }
 956                  else
 957                  {
 958                      _gd.Api.CmdCopyBufferToImage(commandBuffer, buffer, image, ImageLayout.General, 1, in region);
 959                  }
 960  
 961                  offset += mipSize;
 962  
 963                  width = Math.Max(1, width >> 1);
 964                  height = Math.Max(1, height >> 1);
 965  
 966                  if (Info.Target == Target.Texture3D)
 967                  {
 968                      depth = Math.Max(1, depth >> 1);
 969                  }
 970              }
 971          }
 972  
 973          private void CopyFromOrToBuffer(
 974              CommandBuffer commandBuffer,
 975              VkBuffer buffer,
 976              Image image,
 977              int size,
 978              bool to,
 979              int dstLayer,
 980              int dstLevel,
 981              int x,
 982              int y,
 983              int width,
 984              int height)
 985          {
 986              var aspectFlags = Info.Format.ConvertAspectFlags();
 987  
 988              if (aspectFlags == (ImageAspectFlags.DepthBit | ImageAspectFlags.StencilBit))
 989              {
 990                  aspectFlags = ImageAspectFlags.DepthBit;
 991              }
 992  
 993              var sl = new ImageSubresourceLayers(aspectFlags, (uint)(FirstLevel + dstLevel), (uint)(FirstLayer + dstLayer), 1);
 994  
 995              var extent = new Extent3D((uint)width, (uint)height, 1);
 996  
 997              int rowLengthAlignment = Info.BlockWidth;
 998  
 999              // We expect all data being written into the texture to have a stride aligned by 4.
1000              if (!to && Info.BytesPerPixel < 4)
1001              {
1002                  rowLengthAlignment = 4 / Info.BytesPerPixel;
1003              }
1004  
1005              var region = new BufferImageCopy(
1006                  0,
1007                  (uint)AlignUpNpot(width, rowLengthAlignment),
1008                  (uint)AlignUpNpot(height, Info.BlockHeight),
1009                  sl,
1010                  new Offset3D(x, y, 0),
1011                  extent);
1012  
1013              if (to)
1014              {
1015                  _gd.Api.CmdCopyImageToBuffer(commandBuffer, image, ImageLayout.General, buffer, 1, in region);
1016              }
1017              else
1018              {
1019                  _gd.Api.CmdCopyBufferToImage(commandBuffer, buffer, image, ImageLayout.General, 1, in region);
1020              }
1021          }
1022  
1023          private static int AlignUpNpot(int size, int alignment)
1024          {
1025              int remainder = size % alignment;
1026              if (remainder == 0)
1027              {
1028                  return size;
1029              }
1030  
1031              return size + (alignment - remainder);
1032          }
1033  
1034          public void SetStorage(BufferRange buffer)
1035          {
1036              throw new NotImplementedException();
1037          }
1038  
1039          public void PrepareForUsage(CommandBufferScoped cbs, PipelineStageFlags flags, List<TextureView> feedbackLoopHazards)
1040          {
1041              Storage.QueueWriteToReadBarrier(cbs, AccessFlags.ShaderReadBit, flags);
1042  
1043              if (feedbackLoopHazards != null && Storage.IsBound(this))
1044              {
1045                  feedbackLoopHazards.Add(this);
1046                  _hazardUses++;
1047              }
1048          }
1049  
1050          public void ClearUsage(List<TextureView> feedbackLoopHazards)
1051          {
1052              if (_hazardUses != 0 && feedbackLoopHazards != null)
1053              {
1054                  feedbackLoopHazards.Remove(this);
1055                  _hazardUses--;
1056              }
1057          }
1058  
1059          public void DecrementHazardUses()
1060          {
1061              if (_hazardUses != 0)
1062              {
1063                  _hazardUses--;
1064              }
1065          }
1066  
1067          public (RenderPassHolder rpHolder, Auto<DisposableFramebuffer> framebuffer) GetPassAndFramebuffer(
1068              VulkanRenderer gd,
1069              Device device,
1070              CommandBufferScoped cbs,
1071              FramebufferParams fb)
1072          {
1073              var key = fb.GetRenderPassCacheKey();
1074  
1075              if (_renderPasses == null || !_renderPasses.TryGetValue(ref key, out RenderPassHolder rpHolder))
1076              {
1077                  rpHolder = new RenderPassHolder(gd, device, key, fb);
1078              }
1079  
1080              return (rpHolder, rpHolder.GetFramebuffer(gd, cbs, fb));
1081          }
1082  
1083          public void AddRenderPass(RenderPassCacheKey key, RenderPassHolder renderPass)
1084          {
1085              _renderPasses ??= new HashTableSlim<RenderPassCacheKey, RenderPassHolder>();
1086  
1087              _renderPasses.Add(ref key, renderPass);
1088          }
1089  
1090          public void RemoveRenderPass(RenderPassCacheKey key)
1091          {
1092              _renderPasses.Remove(ref key);
1093          }
1094  
1095          protected virtual void Dispose(bool disposing)
1096          {
1097              if (disposing)
1098              {
1099                  bool wasValid = Interlocked.Exchange(ref _isValid, 0) != 0;
1100                  if (wasValid)
1101                  {
1102                      _gd.Textures.Remove(this);
1103  
1104                      _imageView.Dispose();
1105                      _imageView2dArray?.Dispose();
1106  
1107                      if (_imageViewIdentity != _imageView)
1108                      {
1109                          _imageViewIdentity.Dispose();
1110                      }
1111  
1112                      if (_imageViewDraw != _imageViewIdentity)
1113                      {
1114                          _imageViewDraw.Dispose();
1115                      }
1116  
1117                      Storage?.DecrementViewsCount();
1118  
1119                      if (_renderPasses != null)
1120                      {
1121                          var renderPasses = _renderPasses.Values.ToArray();
1122  
1123                          foreach (var pass in renderPasses)
1124                          {
1125                              pass.Dispose();
1126                          }
1127                      }
1128  
1129                      if (_selfManagedViews != null)
1130                      {
1131                          foreach (var view in _selfManagedViews.Values)
1132                          {
1133                              view.Dispose();
1134                          }
1135  
1136                          _selfManagedViews = null;
1137                      }
1138                  }
1139              }
1140          }
1141  
1142          public void Dispose()
1143          {
1144              Dispose(true);
1145          }
1146  
1147          public void Release()
1148          {
1149              Dispose();
1150          }
1151      }
1152  }