1  using Ryujinx.Graphics.Gpu.Synchronization;
  2  using Ryujinx.Memory.Tracking;
  3  using System;
  4  using System.Collections.Generic;
  5  using System.Threading;
  6  
  7  namespace Ryujinx.Graphics.Gpu.Image
  8  {
  9      /// <summary>
 10      /// A tracking handle for a texture group, which represents a range of views in a storage texture.
 11      /// Retains a list of overlapping texture views, a modified flag, and tracking for each
 12      /// CPU VA range that the views cover.
 13      /// Also tracks copy dependencies for the handle - references to other handles that must be kept
 14      /// in sync with this one before use.
 15      /// </summary>
 16      class TextureGroupHandle : ISyncActionHandler, IDisposable
 17      {
 18          private const int FlushBalanceIncrement = 6;
 19          private const int FlushBalanceWriteCost = 1;
 20          private const int FlushBalanceThreshold = 7;
 21          private const int FlushBalanceMax = 60;
 22          private const int FlushBalanceMin = -10;
 23  
 24          private readonly TextureGroup _group;
 25          private int _bindCount;
 26          private readonly int _firstLevel;
 27          private readonly int _firstLayer;
 28  
 29          // Sync state for texture flush.
 30  
 31          /// <summary>
 32          /// The sync number last registered.
 33          /// </summary>
 34          private ulong _registeredSync;
 35          private ulong _registeredBufferSync = ulong.MaxValue;
 36          private ulong _registeredBufferGuestSync = ulong.MaxValue;
 37  
 38          /// <summary>
 39          /// The sync number when the texture was last modified by GPU.
 40          /// </summary>
 41          private ulong _modifiedSync;
 42  
 43          /// <summary>
 44          /// Whether a tracking action is currently registered or not. (0/1)
 45          /// </summary>
 46          private int _actionRegistered;
 47  
 48          /// <summary>
 49          /// Whether a sync action is currently registered or not.
 50          /// </summary>
 51          private bool _syncActionRegistered;
 52  
 53          /// <summary>
 54          /// Determines the balance of synced writes to flushes.
 55          /// Used to determine if the texture should always write data to a persistent buffer for flush.
 56          /// </summary>
 57          private int _flushBalance;
 58  
 59          /// <summary>
 60          /// The byte offset from the start of the storage of this handle.
 61          /// </summary>
 62          public int Offset { get; }
 63  
 64          /// <summary>
 65          /// The size in bytes covered by this handle.
 66          /// </summary>
 67          public int Size { get; }
 68  
 69          /// <summary>
 70          /// The base slice index for this handle.
 71          /// </summary>
 72          public int BaseSlice { get; }
 73  
 74          /// <summary>
 75          /// The number of slices covered by this handle.
 76          /// </summary>
 77          public int SliceCount { get; }
 78  
 79          /// <summary>
 80          /// The textures which this handle overlaps with.
 81          /// </summary>
 82          public List<Texture> Overlaps { get; }
 83  
 84          /// <summary>
 85          /// The CPU memory tracking handles that cover this handle.
 86          /// </summary>
 87          public RegionHandle[] Handles { get; }
 88  
 89          /// <summary>
 90          /// True if a texture overlapping this handle has been modified. Is set false when the flush action is called.
 91          /// </summary>
 92          public bool Modified { get; set; }
 93  
 94          /// <summary>
 95          /// Dependencies to handles from other texture groups.
 96          /// </summary>
 97          public List<TextureDependency> Dependencies { get; }
 98  
 99          /// <summary>
100          /// A flag indicating that a copy is required from one of the dependencies.
101          /// </summary>
102          public bool NeedsCopy => DeferredCopy != null;
103  
104          /// <summary>
105          /// A data copy that must be acknowledged the next time this handle is used.
106          /// </summary>
107          public TextureGroupHandle DeferredCopy { get; set; }
108  
109          /// <summary>
110          /// Create a new texture group handle, representing a range of views in a storage texture.
111          /// </summary>
112          /// <param name="group">The TextureGroup that the handle belongs to</param>
113          /// <param name="offset">The byte offset from the start of the storage of the handle</param>
114          /// <param name="size">The size in bytes covered by the handle</param>
115          /// <param name="views">All views of the storage texture, used to calculate overlaps</param>
116          /// <param name="firstLayer">The first layer of this handle in the storage texture</param>
117          /// <param name="firstLevel">The first level of this handle in the storage texture</param>
118          /// <param name="baseSlice">The base slice index of this handle</param>
119          /// <param name="sliceCount">The number of slices this handle covers</param>
120          /// <param name="handles">The memory tracking handles that cover this handle</param>
121          public TextureGroupHandle(TextureGroup group,
122                                    int offset,
123                                    ulong size,
124                                    IEnumerable<Texture> views,
125                                    int firstLayer,
126                                    int firstLevel,
127                                    int baseSlice,
128                                    int sliceCount,
129                                    RegionHandle[] handles)
130          {
131              _group = group;
132              _firstLayer = firstLayer;
133              _firstLevel = firstLevel;
134  
135              Offset = offset;
136              Size = (int)size;
137              Overlaps = new List<Texture>();
138              Dependencies = new List<TextureDependency>();
139  
140              BaseSlice = baseSlice;
141              SliceCount = sliceCount;
142  
143              if (views != null)
144              {
145                  RecalculateOverlaps(group, views);
146              }
147  
148              Handles = handles;
149  
150              if (group.Storage.Info.IsLinear)
151              {
152                  // Linear textures are presumed to be used for readback initially.
153                  _flushBalance = FlushBalanceThreshold + FlushBalanceIncrement;
154              }
155  
156              foreach (RegionHandle handle in handles)
157              {
158                  handle.RegisterDirtyEvent(DirtyAction);
159              }
160          }
161  
162          /// <summary>
163          /// The action to perform when a memory tracking handle is flipped to dirty.
164          /// This notifies overlapping textures that the memory needs to be synchronized.
165          /// </summary>
166          private void DirtyAction()
167          {
168              // Notify all textures that belong to this handle.
169  
170              _group.Storage.SignalGroupDirty();
171  
172              lock (Overlaps)
173              {
174                  foreach (Texture overlap in Overlaps)
175                  {
176                      overlap.SignalGroupDirty();
177                  }
178              }
179  
180              DeferredCopy = null;
181          }
182  
183          /// <summary>
184          /// Discards all data for this handle.
185          /// This clears all dirty flags and pending copies from other handles.
186          /// </summary>
187          public void DiscardData()
188          {
189              DeferredCopy = null;
190  
191              foreach (RegionHandle handle in Handles)
192              {
193                  if (handle.Dirty)
194                  {
195                      handle.Reprotect();
196                  }
197              }
198          }
199  
200          /// <summary>
201          /// Calculate a list of which views overlap this handle.
202          /// </summary>
203          /// <param name="group">The parent texture group, used to find a view's base CPU VA offset</param>
204          /// <param name="views">The views to search for overlaps</param>
205          public void RecalculateOverlaps(TextureGroup group, IEnumerable<Texture> views)
206          {
207              // Overlaps can be accessed from the memory tracking signal handler, so access must be atomic.
208              lock (Overlaps)
209              {
210                  int endOffset = Offset + Size;
211  
212                  Overlaps.Clear();
213  
214                  foreach (Texture view in views)
215                  {
216                      int viewOffset = group.FindOffset(view);
217                      if (viewOffset < endOffset && Offset < viewOffset + (int)view.Size)
218                      {
219                          Overlaps.Add(view);
220                      }
221                  }
222              }
223          }
224  
225          /// <summary>
226          /// Determine if the next sync will copy into the flush buffer.
227          /// </summary>
228          /// <returns>True if it will copy, false otherwise</returns>
229          private bool NextSyncCopies()
230          {
231              return _flushBalance - FlushBalanceWriteCost > FlushBalanceThreshold;
232          }
233  
234          /// <summary>
235          /// Alters the flush balance by the given value. Should increase significantly with each sync, decrease with each write.
236          /// A flush balance higher than the threshold will cause a texture to repeatedly copy to a flush buffer on each use.
237          /// </summary>
238          /// <param name="modifier">Value to add to the existing flush balance</param>
239          /// <returns>True if the new balance is over the threshold, false otherwise</returns>
240          private bool ModifyFlushBalance(int modifier)
241          {
242              int result;
243              int existingBalance;
244              do
245              {
246                  existingBalance = _flushBalance;
247                  result = Math.Max(FlushBalanceMin, Math.Min(FlushBalanceMax, existingBalance + modifier));
248              }
249              while (Interlocked.CompareExchange(ref _flushBalance, result, existingBalance) != existingBalance);
250  
251              return result > FlushBalanceThreshold;
252          }
253  
254          /// <summary>
255          /// Adds a single texture view as an overlap if its range overlaps.
256          /// </summary>
257          /// <param name="offset">The offset of the view in the group</param>
258          /// <param name="view">The texture to add as an overlap</param>
259          public void AddOverlap(int offset, Texture view)
260          {
261              // Overlaps can be accessed from the memory tracking signal handler, so access must be atomic.
262  
263              if (OverlapsWith(offset, (int)view.Size))
264              {
265                  lock (Overlaps)
266                  {
267                      Overlaps.Add(view);
268                  }
269              }
270          }
271  
272          /// <summary>
273          /// Removes a single texture view as an overlap if its range overlaps.
274          /// </summary>
275          /// <param name="offset">The offset of the view in the group</param>
276          /// <param name="view">The texture to add as an overlap</param>
277          public void RemoveOverlap(int offset, Texture view)
278          {
279              // Overlaps can be accessed from the memory tracking signal handler, so access must be atomic.
280  
281              if (OverlapsWith(offset, (int)view.Size))
282              {
283                  lock (Overlaps)
284                  {
285                      Overlaps.Remove(view);
286                  }
287              }
288          }
289  
290          /// <summary>
291          /// Registers a sync action to happen for this handle, and an interim flush action on the tracking handle.
292          /// </summary>
293          /// <param name="context">The GPU context to register a sync action on</param>
294          private void RegisterSync(GpuContext context)
295          {
296              if (!_syncActionRegistered)
297              {
298                  _modifiedSync = context.SyncNumber;
299                  context.RegisterSyncAction(this, true);
300                  _syncActionRegistered = true;
301              }
302  
303              if (Interlocked.Exchange(ref _actionRegistered, 1) == 0)
304              {
305                  _group.RegisterAction(this);
306              }
307          }
308  
309          /// <summary>
310          /// Signal that this handle has been modified to any existing dependencies, and set the modified flag.
311          /// </summary>
312          /// <param name="context">The GPU context to register a sync action on</param>
313          public void SignalModified(GpuContext context)
314          {
315              Modified = true;
316  
317              // If this handle has any copy dependencies, notify the other handle that a copy needs to be performed.
318  
319              foreach (TextureDependency dependency in Dependencies)
320              {
321                  dependency.SignalModified();
322              }
323  
324              RegisterSync(context);
325          }
326  
327          /// <summary>
328          /// Signal that this handle has either started or ended being modified.
329          /// </summary>
330          /// <param name="bound">True if this handle is being bound, false if unbound</param>
331          /// <param name="context">The GPU context to register a sync action on</param>
332          public void SignalModifying(bool bound, GpuContext context)
333          {
334              SignalModified(context);
335  
336              if (!bound && _syncActionRegistered && NextSyncCopies())
337              {
338                  // On unbind, textures that flush often should immediately create sync so their result can be obtained as soon as possible.
339  
340                  context.CreateHostSyncIfNeeded(HostSyncFlags.Force);
341              }
342  
343              // Note: Bind count currently resets to 0 on inherit for safety, as the handle <-> view relationship can change.
344              _bindCount = Math.Max(0, _bindCount + (bound ? 1 : -1));
345          }
346  
347          /// <summary>
348          /// Synchronize dependent textures, if any of them have deferred a copy from this texture.
349          /// </summary>
350          public void SynchronizeDependents()
351          {
352              foreach (TextureDependency dependency in Dependencies)
353              {
354                  TextureGroupHandle otherHandle = dependency.Other.Handle;
355  
356                  if (otherHandle.DeferredCopy == this)
357                  {
358                      otherHandle._group.Storage.SynchronizeMemory();
359                  }
360              }
361          }
362  
363          /// <summary>
364          /// Wait for the latest sync number that the texture handle was written to,
365          /// removing the modified flag if it was reached, or leaving it set if it has not yet been created.
366          /// </summary>
367          /// <param name="context">The GPU context used to wait for sync</param>
368          /// <returns>True if the texture data can be read from the flush buffer</returns>
369          public bool Sync(GpuContext context)
370          {
371              // Currently assumes the calling thread is a guest thread.
372  
373              bool inBuffer = _registeredBufferGuestSync != ulong.MaxValue;
374              ulong sync = inBuffer ? _registeredBufferGuestSync : _registeredSync;
375  
376              long diff = (long)(context.SyncNumber - sync);
377  
378              ModifyFlushBalance(FlushBalanceIncrement);
379  
380              if (diff > 0)
381              {
382                  context.Renderer.WaitSync(sync);
383  
384                  if ((long)(_modifiedSync - sync) > 0)
385                  {
386                      // Flush the data in a previous state. Do not remove the modified flag - it will be removed to ignore following writes.
387                      return inBuffer;
388                  }
389  
390                  Modified = false;
391  
392                  return inBuffer;
393              }
394  
395              // If the difference is <= 0, no data is not ready yet. Flush any data we can without waiting or removing modified flag.
396              return false;
397          }
398  
399          /// <summary>
400          /// Clears the action registered variable, indicating that the tracking action should be
401          /// re-registered on the next modification.
402          /// </summary>
403          public void ClearActionRegistered()
404          {
405              Interlocked.Exchange(ref _actionRegistered, 0);
406          }
407  
408          /// <summary>
409          /// Action to perform before a sync number is registered after modification.
410          /// This action will copy the texture data to the flush buffer if this texture
411          /// flushes often enough, which is determined by the flush balance.
412          /// </summary>
413          /// <inheritdoc/>
414          public void SyncPreAction(bool syncpoint)
415          {
416              if (syncpoint || NextSyncCopies())
417              {
418                  if (ModifyFlushBalance(0) && _registeredBufferSync != _modifiedSync)
419                  {
420                      _group.FlushIntoBuffer(this);
421                      _registeredBufferSync = _modifiedSync;
422                  }
423              }
424          }
425  
426          /// <summary>
427          /// Action to perform when a sync number is registered after modification.
428          /// This action will register a read tracking action on the memory tracking handle so that a flush from CPU can happen.
429          /// </summary>
430          /// <inheritdoc/>
431          public bool SyncAction(bool syncpoint)
432          {
433              // The storage will need to signal modified again to update the sync number in future.
434              _group.Storage.SignalModifiedDirty();
435  
436              bool lastInBuffer = _registeredBufferSync == _modifiedSync;
437  
438              if (!lastInBuffer)
439              {
440                  _registeredBufferSync = ulong.MaxValue;
441              }
442  
443              lock (Overlaps)
444              {
445                  foreach (Texture texture in Overlaps)
446                  {
447                      texture.SignalModifiedDirty();
448                  }
449              }
450  
451              // Register region tracking for CPU? (again)
452  
453              _registeredSync = _modifiedSync;
454              _syncActionRegistered = false;
455  
456              if (Interlocked.Exchange(ref _actionRegistered, 1) == 0)
457              {
458                  _group.RegisterAction(this);
459              }
460  
461              if (syncpoint)
462              {
463                  _registeredBufferGuestSync = _registeredBufferSync;
464              }
465  
466              // If the last modification is in the buffer, keep this sync action alive until it sees a syncpoint.
467              return syncpoint || !lastInBuffer;
468          }
469  
470          /// <summary>
471          /// Signal that a copy dependent texture has been modified, and must have its data copied to this one.
472          /// </summary>
473          /// <param name="copyFrom">The texture handle that must defer a copy to this one</param>
474          public void DeferCopy(TextureGroupHandle copyFrom)
475          {
476              Modified = false;
477              DeferredCopy = copyFrom;
478  
479              _group.Storage.SignalGroupDirty();
480  
481              foreach (Texture overlap in Overlaps)
482              {
483                  overlap.SignalGroupDirty();
484              }
485          }
486  
487          /// <summary>
488          /// Create a copy dependency between this handle, and another.
489          /// </summary>
490          /// <param name="other">The handle to create a copy dependency to</param>
491          /// <param name="copyToOther">True if a copy should be deferred to all of the other handle's dependencies</param>
492          public void CreateCopyDependency(TextureGroupHandle other, bool copyToOther = false)
493          {
494              // Does this dependency already exist?
495              foreach (TextureDependency existing in Dependencies)
496              {
497                  if (existing.Other.Handle == other)
498                  {
499                      // Do not need to create it again. May need to set the dirty flag.
500                      return;
501                  }
502              }
503  
504              _group.HasCopyDependencies = true;
505              other._group.HasCopyDependencies = true;
506  
507              TextureDependency dependency = new(this);
508              TextureDependency otherDependency = new(other);
509  
510              dependency.Other = otherDependency;
511              otherDependency.Other = dependency;
512  
513              Dependencies.Add(dependency);
514              other.Dependencies.Add(otherDependency);
515  
516              // Recursively create dependency:
517              // All of this handle's dependencies must depend on the other.
518              foreach (TextureDependency existing in Dependencies.ToArray())
519              {
520                  if (existing != dependency && existing.Other.Handle != other)
521                  {
522                      existing.Other.Handle.CreateCopyDependency(other);
523                  }
524              }
525  
526              // All of the other handle's dependencies must depend on this.
527              foreach (TextureDependency existing in other.Dependencies.ToArray())
528              {
529                  if (existing != otherDependency && existing.Other.Handle != this)
530                  {
531                      existing.Other.Handle.CreateCopyDependency(this);
532  
533                      if (copyToOther && Modified)
534                      {
535                          existing.Other.Handle.DeferCopy(this);
536                      }
537                  }
538              }
539          }
540  
541          /// <summary>
542          /// Remove a dependency from this handle's dependency list.
543          /// </summary>
544          /// <param name="dependency">The dependency to remove</param>
545          public void RemoveDependency(TextureDependency dependency)
546          {
547              Dependencies.Remove(dependency);
548          }
549  
550          /// <summary>
551          /// Check if any of this handle's memory tracking handles are dirty.
552          /// </summary>
553          /// <returns>True if at least one of the handles is dirty</returns>
554          private bool CheckDirty()
555          {
556              return Array.Exists(Handles, handle => handle.Dirty);
557          }
558  
559          /// <summary>
560          /// Perform a copy from the provided handle to this one, or perform a deferred copy if none is provided.
561          /// </summary>
562          /// <param name="context">GPU context to register sync for modified handles</param>
563          /// <param name="fromHandle">The handle to copy from. If not provided, this method will copy from and clear the deferred copy instead</param>
564          /// <returns>True if the copy was performed, false otherwise</returns>
565          public bool Copy(GpuContext context, TextureGroupHandle fromHandle = null)
566          {
567              bool result = false;
568              bool shouldCopy = false;
569  
570              if (fromHandle == null)
571              {
572                  fromHandle = DeferredCopy;
573  
574                  if (fromHandle != null)
575                  {
576                      // Only copy if the copy texture is still modified.
577                      // DeferredCopy will be set to null if new data is written from CPU (see the DirtyAction method).
578                      // It will also set as unmodified if a copy is deferred to it.
579  
580                      shouldCopy = true;
581  
582                      if (fromHandle._bindCount == 0)
583                      {
584                          // Repeat the copy in future if the bind count is greater than 0.
585                          DeferredCopy = null;
586                      }
587                  }
588              }
589              else
590              {
591                  // Copies happen directly when initializing a copy dependency.
592                  // If dirty, do not copy. Its data no longer matters, and this handle should also be dirty.
593                  // Also, only direct copy if the data in this handle is not already modified (can be set by copies from modified handles).
594                  shouldCopy = !fromHandle.CheckDirty() && (fromHandle.Modified || !Modified);
595              }
596  
597              if (shouldCopy)
598              {
599                  Texture from = fromHandle._group.Storage;
600                  Texture to = _group.Storage;
601  
602                  if (from.ScaleFactor != to.ScaleFactor)
603                  {
604                      to.PropagateScale(from);
605                  }
606  
607                  from.HostTexture.CopyTo(
608                      to.HostTexture,
609                      fromHandle._firstLayer,
610                      _firstLayer,
611                      fromHandle._firstLevel,
612                      _firstLevel);
613  
614                  if (fromHandle.Modified)
615                  {
616                      Modified = true;
617  
618                      RegisterSync(context);
619                  }
620  
621                  result = true;
622              }
623  
624              return result;
625          }
626  
627          /// <summary>
628          /// Check if this handle has a dependency to a given texture group.
629          /// </summary>
630          /// <param name="group">The texture group to check for</param>
631          /// <returns>True if there is a dependency, false otherwise</returns>
632          public bool HasDependencyTo(TextureGroup group)
633          {
634              foreach (TextureDependency dep in Dependencies)
635              {
636                  if (dep.Other.Handle._group == group)
637                  {
638                      return true;
639                  }
640              }
641  
642              return false;
643          }
644  
645          /// <summary>
646          /// Inherit modified flags and dependencies from another texture handle.
647          /// </summary>
648          /// <param name="old">The texture handle to inherit from</param>
649          /// <param name="withCopies">Whether the handle should inherit copy dependencies or not</param>
650          public void Inherit(TextureGroupHandle old, bool withCopies)
651          {
652              Modified |= old.Modified;
653  
654              if (withCopies)
655              {
656                  foreach (TextureDependency dependency in old.Dependencies.ToArray())
657                  {
658                      CreateCopyDependency(dependency.Other.Handle);
659  
660                      if (dependency.Other.Handle.DeferredCopy == old)
661                      {
662                          dependency.Other.Handle.DeferredCopy = this;
663                      }
664                  }
665  
666                  DeferredCopy = old.DeferredCopy;
667              }
668          }
669  
670          /// <summary>
671          /// Check if this region overlaps with another.
672          /// </summary>
673          /// <param name="address">Base address</param>
674          /// <param name="size">Size of the region</param>
675          /// <returns>True if overlapping, false otherwise</returns>
676          public bool OverlapsWith(int offset, int size)
677          {
678              return Offset < offset + size && offset < Offset + Size;
679          }
680  
681          /// <summary>
682          /// Dispose this texture group handle, removing all its dependencies and disposing its memory tracking handles.
683          /// </summary>
684          public void Dispose()
685          {
686              foreach (RegionHandle handle in Handles)
687              {
688                  handle.Dispose();
689              }
690  
691              foreach (TextureDependency dependency in Dependencies.ToArray())
692              {
693                  dependency.Other.Handle.RemoveDependency(dependency.Other);
694              }
695          }
696      }
697  }