1  using Ryujinx.Graphics.GAL;
  2  using System;
  3  using System.Collections.Generic;
  4  
  5  namespace Ryujinx.Graphics.Gpu.Memory
  6  {
  7      /// <summary>
  8      /// Type of backing memory.
  9      /// In ascending order of priority when merging multiple buffer backing states.
 10      /// </summary>
 11      internal enum BufferBackingType
 12      {
 13          HostMemory,
 14          DeviceMemory,
 15          DeviceMemoryWithFlush
 16      }
 17  
 18      /// <summary>
 19      /// Keeps track of buffer usage to decide what memory heap that buffer memory is placed on.
 20      /// Dedicated GPUs prefer certain types of resources to be device local,
 21      /// and if we need data to be read back, we might prefer that they're in host memory.
 22      /// 
 23      /// The measurements recorded here compare to a set of heruristics (thresholds and conditions)
 24      /// that appear to produce good performance in most software.
 25      /// </summary>
 26      internal struct BufferBackingState
 27      {
 28          private const int DeviceLocalSizeThreshold = 256 * 1024; // 256kb
 29  
 30          private const int SetCountThreshold = 100;
 31          private const int WriteCountThreshold = 50;
 32          private const int FlushCountThreshold = 5;
 33          private const int DeviceLocalForceExpiry = 100;
 34  
 35          public readonly bool IsDeviceLocal => _activeType != BufferBackingType.HostMemory;
 36  
 37          private readonly SystemMemoryType _systemMemoryType;
 38          private BufferBackingType _activeType;
 39          private BufferBackingType _desiredType;
 40  
 41          private bool _canSwap;
 42  
 43          private int _setCount;
 44          private int _writeCount;
 45          private int _flushCount;
 46          private int _flushTemp;
 47          private int _lastFlushWrite;
 48          private int _deviceLocalForceCount;
 49  
 50          private readonly int _size;
 51  
 52          /// <summary>
 53          /// Initialize the buffer backing state for a given parent buffer.
 54          /// </summary>
 55          /// <param name="context">GPU context</param>
 56          /// <param name="parent">Parent buffer</param>
 57          /// <param name="stage">Initial buffer stage</param>
 58          /// <param name="baseBuffers">Buffers to inherit state from</param>
 59          public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, IEnumerable<Buffer> baseBuffers = null)
 60          {
 61              _size = (int)parent.Size;
 62              _systemMemoryType = context.Capabilities.MemoryType;
 63  
 64              // Backend managed is always auto, unified memory is always host.
 65              _desiredType = BufferBackingType.HostMemory;
 66              _canSwap = _systemMemoryType != SystemMemoryType.BackendManaged && _systemMemoryType != SystemMemoryType.UnifiedMemory;
 67  
 68              if (_canSwap)
 69              {
 70                  // Might want to start certain buffers as being device local,
 71                  // and the usage might also lock those buffers into being device local.
 72  
 73                  BufferStage storageFlags = stage & BufferStage.StorageMask;
 74  
 75                  if (parent.Size > DeviceLocalSizeThreshold && baseBuffers == null)
 76                  {
 77                      _desiredType = BufferBackingType.DeviceMemory;
 78                  }
 79  
 80                  if (storageFlags != 0)
 81                  {
 82                      // Storage buffer bindings may require special treatment.
 83  
 84                      var rawStage = stage & BufferStage.StageMask;
 85  
 86                      if (rawStage == BufferStage.Fragment)
 87                      {
 88                          // Fragment read should start device local.
 89  
 90                          _desiredType = BufferBackingType.DeviceMemory;
 91  
 92                          if (storageFlags != BufferStage.StorageRead)
 93                          {
 94                              // Fragment write should stay device local until the use doesn't happen anymore.
 95  
 96                              _deviceLocalForceCount = DeviceLocalForceExpiry;
 97                          }
 98                      }
 99  
100                      // TODO: Might be nice to force atomic access to be device local for any stage.
101                  }
102  
103                  if (baseBuffers != null)
104                  {
105                      foreach (Buffer buffer in baseBuffers)
106                      {
107                          CombineState(buffer.BackingState);
108                      }
109                  }
110              }
111          }
112  
113          /// <summary>
114          /// Combine buffer backing types, selecting the one with highest priority.
115          /// </summary>
116          /// <param name="left">First buffer backing type</param>
117          /// <param name="right">Second buffer backing type</param>
118          /// <returns>Combined buffer backing type</returns>
119          private static BufferBackingType CombineTypes(BufferBackingType left, BufferBackingType right)
120          {
121              return (BufferBackingType)Math.Max((int)left, (int)right);
122          }
123  
124          /// <summary>
125          /// Combine the state from the given buffer backing state with this one,
126          /// so that the state isn't lost when migrating buffers.
127          /// </summary>
128          /// <param name="oldState">Buffer state to combine into this state</param>
129          private void CombineState(BufferBackingState oldState)
130          {
131              _setCount += oldState._setCount;
132              _writeCount += oldState._writeCount;
133              _flushCount += oldState._flushCount;
134              _flushTemp += oldState._flushTemp;
135              _lastFlushWrite = -1;
136              _deviceLocalForceCount = Math.Max(_deviceLocalForceCount, oldState._deviceLocalForceCount);
137  
138              _canSwap &= oldState._canSwap;
139  
140              _desiredType = CombineTypes(_desiredType, oldState._desiredType);
141          }
142  
143          /// <summary>
144          /// Get the buffer access for the desired backing type, and record that type as now being active.
145          /// </summary>
146          /// <param name="parent">Parent buffer</param>
147          /// <returns>Buffer access</returns>
148          public BufferAccess SwitchAccess(Buffer parent)
149          {
150              BufferAccess access = parent.SparseCompatible ? BufferAccess.SparseCompatible : BufferAccess.Default;
151  
152              bool isBackendManaged = _systemMemoryType == SystemMemoryType.BackendManaged;
153  
154              if (!isBackendManaged)
155              {
156                  switch (_desiredType)
157                  {
158                      case BufferBackingType.HostMemory:
159                          access |= BufferAccess.HostMemory;
160                          break;
161                      case BufferBackingType.DeviceMemory:
162                          access |= BufferAccess.DeviceMemory;
163                          break;
164                      case BufferBackingType.DeviceMemoryWithFlush:
165                          access |= BufferAccess.DeviceMemoryMapped;
166                          break;
167                  }
168              }
169  
170              _activeType = _desiredType;
171  
172              return access;
173          }
174  
175          /// <summary>
176          /// Record when data has been uploaded to the buffer.
177          /// </summary>
178          public void RecordSet()
179          {
180              _setCount++;
181  
182              ConsiderUseCounts();
183          }
184  
185          /// <summary>
186          /// Record when data has been flushed from the buffer.
187          /// </summary>
188          public void RecordFlush()
189          {
190              if (_lastFlushWrite != _writeCount)
191              {
192                  // If it's on the same page as the last flush, ignore it.
193                  _lastFlushWrite = _writeCount;
194                  _flushCount++;
195              }
196          }
197  
198          /// <summary>
199          /// Determine if the buffer backing should be changed.
200          /// </summary>
201          /// <returns>True if the desired backing type is different from the current type</returns>
202          public readonly bool ShouldChangeBacking()
203          {
204              return _desiredType != _activeType;
205          }
206  
207          /// <summary>
208          /// Determine if the buffer backing should be changed, considering a new use with the given buffer stage.
209          /// </summary>
210          /// <param name="stage">Buffer stage for the use</param>
211          /// <returns>True if the desired backing type is different from the current type</returns>
212          public bool ShouldChangeBacking(BufferStage stage)
213          {
214              if (!_canSwap)
215              {
216                  return false;
217              }
218  
219              BufferStage storageFlags = stage & BufferStage.StorageMask;
220  
221              if (storageFlags != 0)
222              {
223                  if (storageFlags != BufferStage.StorageRead)
224                  {
225                      // Storage write.
226                      _writeCount++;
227  
228                      var rawStage = stage & BufferStage.StageMask;
229  
230                      if (rawStage == BufferStage.Fragment)
231                      {
232                          // Switch to device memory, swap back only if this use disappears.
233  
234                          _desiredType = CombineTypes(_desiredType, BufferBackingType.DeviceMemory);
235                          _deviceLocalForceCount = DeviceLocalForceExpiry;
236  
237                          // TODO: Might be nice to force atomic access to be device local for any stage.
238                      }
239                  }
240  
241                  ConsiderUseCounts();
242              }
243  
244              return _desiredType != _activeType;
245          }
246  
247          /// <summary>
248          /// Evaluate the current counts to determine what the buffer's desired backing type is.
249          /// This method depends on heuristics devised by testing a variety of software.
250          /// </summary>
251          private void ConsiderUseCounts()
252          {
253              if (_canSwap)
254              {
255                  if (_writeCount >= WriteCountThreshold || _setCount >= SetCountThreshold || _flushCount >= FlushCountThreshold)
256                  {
257                      if (_deviceLocalForceCount > 0 && --_deviceLocalForceCount != 0)
258                      {
259                          // Some buffer usage demanded that the buffer stay device local.
260                          // The desired type was selected when this counter was set.
261                      }
262                      else if (_flushCount > 0 || _flushTemp-- > 0)
263                      {
264                          // Buffers that flush should ideally be mapped in host address space for easy copies.
265                          // If the buffer is large it will do better on GPU memory, as there will be more writes than data flushes (typically individual pages).
266                          // If it is small, then it's likely most of the buffer will be flushed so we want it on host memory, as access is cached.
267                          _desiredType = _size > DeviceLocalSizeThreshold ? BufferBackingType.DeviceMemoryWithFlush : BufferBackingType.HostMemory;
268                      }
269                      else if (_writeCount >= WriteCountThreshold)
270                      {
271                          // Buffers that are written often should ideally be in the device local heap. (Storage buffers)
272                          _desiredType = BufferBackingType.DeviceMemory;
273                      }
274                      else if (_setCount > SetCountThreshold)
275                      {
276                          // Buffers that have their data set often should ideally be host mapped. (Constant buffers)
277                          _desiredType = BufferBackingType.HostMemory;
278                      }
279  
280                      // It's harder for a buffer that is flushed to revert to another type of mapping.
281                      if (_flushCount > 0)
282                      {
283                          _flushTemp = 1000;
284                      }
285  
286                      _lastFlushWrite = -1;
287                      _flushCount = 0;
288                      _writeCount = 0;
289                      _setCount = 0;
290                  }
291              }
292          }
293      }
294  }