1  using Ryujinx.Common;
  2  using Ryujinx.Common.Logging;
  3  using Ryujinx.Graphics.GAL;
  4  using System;
  5  using System.Collections.Generic;
  6  using System.Diagnostics;
  7  
  8  namespace Ryujinx.Graphics.Vulkan
  9  {
 10      readonly struct StagingBufferReserved
 11      {
 12          public readonly BufferHolder Buffer;
 13          public readonly int Offset;
 14          public readonly int Size;
 15  
 16          public StagingBufferReserved(BufferHolder buffer, int offset, int size)
 17          {
 18              Buffer = buffer;
 19              Offset = offset;
 20              Size = size;
 21          }
 22      }
 23  
 24      class StagingBuffer : IDisposable
 25      {
 26          private const int BufferSize = 32 * 1024 * 1024;
 27  
 28          private int _freeOffset;
 29          private int _freeSize;
 30  
 31          private readonly VulkanRenderer _gd;
 32          private readonly BufferHolder _buffer;
 33          private readonly int _resourceAlignment;
 34  
 35          public readonly BufferHandle Handle;
 36  
 37          private readonly struct PendingCopy
 38          {
 39              public FenceHolder Fence { get; }
 40              public int Size { get; }
 41  
 42              public PendingCopy(FenceHolder fence, int size)
 43              {
 44                  Fence = fence;
 45                  Size = size;
 46                  fence.Get();
 47              }
 48          }
 49  
 50          private readonly Queue<PendingCopy> _pendingCopies;
 51  
 52          public StagingBuffer(VulkanRenderer gd, BufferManager bufferManager)
 53          {
 54              _gd = gd;
 55              Handle = bufferManager.CreateWithHandle(gd, BufferSize, out _buffer);
 56              _pendingCopies = new Queue<PendingCopy>();
 57              _freeSize = BufferSize;
 58              _resourceAlignment = (int)gd.Capabilities.MinResourceAlignment;
 59          }
 60  
 61          public void PushData(CommandBufferPool cbp, CommandBufferScoped? cbs, Action endRenderPass, BufferHolder dst, int dstOffset, ReadOnlySpan<byte> data)
 62          {
 63              bool isRender = cbs != null;
 64              CommandBufferScoped scoped = cbs ?? cbp.Rent();
 65  
 66              // Must push all data to the buffer. If it can't fit, split it up.
 67  
 68              endRenderPass?.Invoke();
 69  
 70              while (data.Length > 0)
 71              {
 72                  if (_freeSize < data.Length)
 73                  {
 74                      FreeCompleted();
 75                  }
 76  
 77                  while (_freeSize == 0)
 78                  {
 79                      if (!WaitFreeCompleted(cbp))
 80                      {
 81                          if (isRender)
 82                          {
 83                              _gd.FlushAllCommands();
 84                              scoped = cbp.Rent();
 85                              isRender = false;
 86                          }
 87                          else
 88                          {
 89                              scoped = cbp.ReturnAndRent(scoped);
 90                          }
 91                      }
 92                  }
 93  
 94                  int chunkSize = Math.Min(_freeSize, data.Length);
 95  
 96                  PushDataImpl(scoped, dst, dstOffset, data[..chunkSize]);
 97  
 98                  dstOffset += chunkSize;
 99                  data = data[chunkSize..];
100              }
101  
102              if (!isRender)
103              {
104                  scoped.Dispose();
105              }
106          }
107  
108          private void PushDataImpl(CommandBufferScoped cbs, BufferHolder dst, int dstOffset, ReadOnlySpan<byte> data)
109          {
110              var srcBuffer = _buffer.GetBuffer();
111              var dstBuffer = dst.GetBuffer(cbs.CommandBuffer, dstOffset, data.Length, true);
112  
113              int offset = _freeOffset;
114              int capacity = BufferSize - offset;
115              if (capacity < data.Length)
116              {
117                  _buffer.SetDataUnchecked(offset, data[..capacity]);
118                  _buffer.SetDataUnchecked(0, data[capacity..]);
119  
120                  BufferHolder.Copy(_gd, cbs, srcBuffer, dstBuffer, offset, dstOffset, capacity);
121                  BufferHolder.Copy(_gd, cbs, srcBuffer, dstBuffer, 0, dstOffset + capacity, data.Length - capacity);
122              }
123              else
124              {
125                  _buffer.SetDataUnchecked(offset, data);
126  
127                  BufferHolder.Copy(_gd, cbs, srcBuffer, dstBuffer, offset, dstOffset, data.Length);
128              }
129  
130              _freeOffset = (offset + data.Length) & (BufferSize - 1);
131              _freeSize -= data.Length;
132              Debug.Assert(_freeSize >= 0);
133  
134              _pendingCopies.Enqueue(new PendingCopy(cbs.GetFence(), data.Length));
135          }
136  
137          public bool TryPushData(CommandBufferScoped cbs, Action endRenderPass, BufferHolder dst, int dstOffset, ReadOnlySpan<byte> data)
138          {
139              if (data.Length > BufferSize)
140              {
141                  return false;
142              }
143  
144              if (_freeSize < data.Length)
145              {
146                  FreeCompleted();
147  
148                  if (_freeSize < data.Length)
149                  {
150                      return false;
151                  }
152              }
153  
154              endRenderPass?.Invoke();
155  
156              PushDataImpl(cbs, dst, dstOffset, data);
157  
158              return true;
159          }
160  
161          private StagingBufferReserved ReserveDataImpl(CommandBufferScoped cbs, int size, int alignment)
162          {
163              // Assumes the caller has already determined that there is enough space.
164              int offset = BitUtils.AlignUp(_freeOffset, alignment);
165              int padding = offset - _freeOffset;
166  
167              int capacity = Math.Min(_freeSize, BufferSize - offset);
168              int reservedLength = size + padding;
169              if (capacity < size)
170              {
171                  offset = 0; // Place at start.
172                  reservedLength += capacity;
173              }
174  
175              _freeOffset = (_freeOffset + reservedLength) & (BufferSize - 1);
176              _freeSize -= reservedLength;
177              Debug.Assert(_freeSize >= 0);
178  
179              _pendingCopies.Enqueue(new PendingCopy(cbs.GetFence(), reservedLength));
180  
181              return new StagingBufferReserved(_buffer, offset, size);
182          }
183  
184          private int GetContiguousFreeSize(int alignment)
185          {
186              int alignedFreeOffset = BitUtils.AlignUp(_freeOffset, alignment);
187              int padding = alignedFreeOffset - _freeOffset;
188  
189              // Free regions:
190              // - Aligned free offset to end (minimum free size - padding)
191              // - 0 to _freeOffset + freeSize wrapped (only if free area contains 0)
192  
193              int endOffset = (_freeOffset + _freeSize) & (BufferSize - 1);
194  
195              return Math.Max(
196                  Math.Min(_freeSize - padding, BufferSize - alignedFreeOffset),
197                  endOffset <= _freeOffset ? Math.Min(_freeSize, endOffset) : 0
198                  );
199          }
200  
201          /// <summary>
202          /// Reserve a range on the staging buffer for the current command buffer and upload data to it.
203          /// </summary>
204          /// <param name="cbs">Command buffer to reserve the data on</param>
205          /// <param name="size">The minimum size the reserved data requires</param>
206          /// <param name="alignment">The required alignment for the buffer offset</param>
207          /// <returns>The reserved range of the staging buffer</returns>
208          public unsafe StagingBufferReserved? TryReserveData(CommandBufferScoped cbs, int size, int alignment)
209          {
210              if (size > BufferSize)
211              {
212                  return null;
213              }
214  
215              // Temporary reserved data cannot be fragmented.
216  
217              if (GetContiguousFreeSize(alignment) < size)
218              {
219                  FreeCompleted();
220  
221                  if (GetContiguousFreeSize(alignment) < size)
222                  {
223                      Logger.Debug?.PrintMsg(LogClass.Gpu, $"Staging buffer out of space to reserve data of size {size}.");
224                      return null;
225                  }
226              }
227  
228              return ReserveDataImpl(cbs, size, alignment);
229          }
230  
231          /// <summary>
232          /// Reserve a range on the staging buffer for the current command buffer and upload data to it.
233          /// Uses the most permissive byte alignment.
234          /// </summary>
235          /// <param name="cbs">Command buffer to reserve the data on</param>
236          /// <param name="size">The minimum size the reserved data requires</param>
237          /// <returns>The reserved range of the staging buffer</returns>
238          public unsafe StagingBufferReserved? TryReserveData(CommandBufferScoped cbs, int size)
239          {
240              return TryReserveData(cbs, size, _resourceAlignment);
241          }
242  
243          private bool WaitFreeCompleted(CommandBufferPool cbp)
244          {
245              if (_pendingCopies.TryPeek(out var pc))
246              {
247                  if (!pc.Fence.IsSignaled())
248                  {
249                      if (cbp.IsFenceOnRentedCommandBuffer(pc.Fence))
250                      {
251                          return false;
252                      }
253  
254                      pc.Fence.Wait();
255                  }
256  
257                  var dequeued = _pendingCopies.Dequeue();
258                  Debug.Assert(dequeued.Fence == pc.Fence);
259                  _freeSize += pc.Size;
260                  pc.Fence.Put();
261              }
262  
263              return true;
264          }
265  
266          public void FreeCompleted()
267          {
268              FenceHolder signalledFence = null;
269              while (_pendingCopies.TryPeek(out var pc) && (pc.Fence == signalledFence || pc.Fence.IsSignaled()))
270              {
271                  signalledFence = pc.Fence; // Already checked - don't need to do it again.
272                  var dequeued = _pendingCopies.Dequeue();
273                  Debug.Assert(dequeued.Fence == pc.Fence);
274                  _freeSize += pc.Size;
275                  pc.Fence.Put();
276              }
277          }
278  
279          protected virtual void Dispose(bool disposing)
280          {
281              if (disposing)
282              {
283                  _gd.BufferManager.Delete(Handle);
284  
285                  while (_pendingCopies.TryDequeue(out var pc))
286                  {
287                      pc.Fence.Put();
288                  }
289              }
290          }
291  
292          public void Dispose()
293          {
294              Dispose(true);
295          }
296      }
297  }