1  namespace Utils
  2  {
  3      public class Retry
  4      {
  5          private readonly string description;
  6          private readonly TimeSpan maxTimeout;
  7          private readonly TimeSpan sleepAfterFail;
  8          private readonly Action<Failure> onFail;
  9          private readonly bool failFast;
 10  
 11          public Retry(string description, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail, bool failFast)
 12          {
 13              this.description = description;
 14              this.maxTimeout = maxTimeout;
 15              this.sleepAfterFail = sleepAfterFail;
 16              this.onFail = onFail;
 17              this.failFast = failFast;
 18          }
 19  
 20          public void Run(Action task)
 21          {
 22              var run = new RetryRun(description, task, maxTimeout, sleepAfterFail, onFail, failFast);
 23              run.Run();
 24          }
 25  
 26          public T Run<T>(Func<T> task)
 27          {
 28              T? result = default;
 29  
 30              var run = new RetryRun(description, () =>
 31              {
 32                  result = task();
 33              }, maxTimeout, sleepAfterFail, onFail, failFast);
 34              run.Run();
 35  
 36              return result!;
 37          }
 38  
 39          private class RetryRun
 40          {
 41              private readonly string description;
 42              private readonly Action task;
 43              private readonly TimeSpan maxTimeout;
 44              private readonly TimeSpan sleepAfterFail;
 45              private readonly Action<Failure> onFail;
 46              private readonly DateTime start = DateTime.UtcNow;
 47              private readonly List<Failure> failures = new List<Failure>();
 48              private readonly bool failFast;
 49              private int tryNumber;
 50              private DateTime tryStart;
 51  
 52              public RetryRun(string description, Action task, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail, bool failFast)
 53              {
 54                  this.description = description;
 55                  this.task = task;
 56                  this.maxTimeout = maxTimeout;
 57                  this.sleepAfterFail = sleepAfterFail;
 58                  this.onFail = onFail;
 59                  this.failFast = failFast;
 60  
 61                  tryNumber = 0;
 62                  tryStart = DateTime.UtcNow;
 63              }
 64  
 65              public void Run()
 66              {
 67                  while (true)
 68                  {
 69                      CheckMaximums();
 70  
 71                      tryNumber++;
 72                      tryStart = DateTime.UtcNow;
 73                      try
 74                      {
 75                          task();
 76                          return;
 77                      }
 78                      catch (OperationCanceledException)
 79                      {
 80                          return;
 81                      }
 82                      catch (Exception ex)
 83                      {
 84                          var failure = CaptureFailure(ex);
 85                          onFail(failure);
 86                          Time.Sleep(sleepAfterFail);
 87                      }
 88                  }
 89              }
 90  
 91              private Failure CaptureFailure(Exception ex)
 92              {
 93                  var f = new Failure(ex, DateTime.UtcNow - tryStart, tryNumber);
 94                  failures.Add(f);
 95                  return f;
 96              }
 97  
 98              private void CheckMaximums()
 99              {
100                  if (Duration() > maxTimeout) Fail();
101                  if (tryNumber > 30) Fail();
102  
103                  // If we have a few very fast failures, retrying won't help us. There's probably something wrong with our operation.
104                  // In this case, don't wait the full duration and fail quickly.
105                  if (failFast && failures.Count > 5 && failures.All(f => f.Duration < TimeSpan.FromSeconds(1.0))) Fail();
106              }
107  
108              private void Fail()
109              {
110                  throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport()}",
111                          new AggregateException(failures.Select(f => f.Exception)));
112              }
113  
114              private string GetFailureReport()
115              {
116                  return Environment.NewLine + string.Join(Environment.NewLine, failures.Select(f => f.Describe()));
117              }
118  
119              private TimeSpan Duration()
120              {
121                  return DateTime.UtcNow - start;
122              }
123          }
124      }
125  
126      public class Failure
127      {
128          public Failure(Exception exception, TimeSpan duration, int tryNumber)
129          {
130              Exception = exception;
131              Duration = duration;
132              TryNumber = tryNumber;
133          }
134  
135          public Exception Exception { get; }
136          public TimeSpan Duration { get; }
137          public int TryNumber { get; }
138  
139          public string Describe()
140          {
141              return $"Try {TryNumber} failed after {Time.FormatDuration(Duration)} with exception '{Exception}'";
142          }
143      }
144  }