GithubRetry.py
1 ############################ Copyrights and license ############################ 2 # # 3 # Copyright 2022 Enrico Minack <github@enrico.minack.dev> # 4 # # 5 # This file is part of PyGithub. # 6 # http://pygithub.readthedocs.io/ # 7 # # 8 # PyGithub is free software: you can redistribute it and/or modify it under # 9 # the terms of the GNU Lesser General Public License as published by the Free # 10 # Software Foundation, either version 3 of the License, or (at your option) # 11 # any later version. # 12 # # 13 # PyGithub is distributed in the hope that it will be useful, but WITHOUT ANY # 14 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # 15 # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more # 16 # details. # 17 # # 18 # You should have received a copy of the GNU Lesser General Public License # 19 # along with PyGithub. If not, see <http://www.gnu.org/licenses/>. # 20 # # 21 ################################################################################ 22 import contextlib 23 import logging 24 import sys 25 import unittest 26 from datetime import datetime 27 from io import BytesIO 28 from unittest import mock 29 30 import urllib3.response 31 from urllib3 import Retry 32 33 import github 34 from github.GithubRetry import DEFAULT_SECONDARY_RATE_WAIT 35 36 from . import Requester 37 38 PrimaryRateLimitMessage = Requester.Requester.PrimaryRateLimitErrors[0] 39 PrimaryRateLimitJson = ( 40 '{"message":"' 41 + PrimaryRateLimitMessage 42 + '","documentation_url":"https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting"}' 43 ) 44 45 SecondaryRateLimitMessage = Requester.Requester.SecondaryRateLimitErrors[0] 46 SecondaryRateLimitJson = ( 47 '{"message":"' 48 + SecondaryRateLimitMessage 49 + '","documentation_url": "https://docs.github.com/en/free-pro-team@latest/rest/overview/resources-in-the-rest-api#secondary-rate-limits"}' 50 ) 51 52 53 class GithubRetry(unittest.TestCase): 54 def get_test_increment_func(self, expected_rate_limit_error): 55 is_primary = expected_rate_limit_error == PrimaryRateLimitMessage 56 57 def test_increment( 58 retry, 59 response, 60 expected_total=None, 61 expected_backoff=None, 62 expected_retry_backoff=None, 63 expect_retry_error=False, 64 has_reset=False, 65 ): 66 # fmt: off 67 self.assertTrue( 68 expected_total is not None and expected_backoff is not None and not expect_retry_error or # noqa: W504 69 expected_total is None and expected_backoff is None and expect_retry_error 70 ) 71 # fmt: on 72 73 orig_retry = retry 74 with mock.patch.object(retry, "_GithubRetry__log") as log: 75 if expect_retry_error: 76 with self.assertRaises(urllib3.exceptions.MaxRetryError): 77 retry.increment("TEST", "URL", response) 78 retry = None 79 else: 80 retry = retry.increment("TEST", "URL", response) 81 82 self.assertEqual(expected_total, retry.total) 83 self.assertEqual( 84 expected_backoff if expected_retry_backoff is None else expected_retry_backoff, 85 retry.get_backoff_time(), 86 ) 87 self.assertEqual(orig_retry.secondary_rate_wait, retry.secondary_rate_wait) 88 89 # fmt: off 90 log.assert_has_calls( 91 [ 92 mock.call(20, "Request TEST URL failed with 403: None"), 93 mock.call(10, f"Response body indicates retry-able {'primary' if is_primary else 'secondary'} rate limit error: {expected_rate_limit_error}"), 94 ] + ([ 95 mock.call(10, "Reset occurs in 0:00:12 (1644768012 / 2022-02-13 16:00:12+00:00)") 96 ] if has_reset else []) + ([ 97 mock.call(10, f"Retry backoff of {expected_retry_backoff}s exceeds required rate limit backoff of {expected_backoff}s") 98 ] if expected_retry_backoff and expected_backoff > 0 else []) + ([ 99 mock.call(20, f"Setting next backoff to {expected_backoff if expected_retry_backoff is None else expected_retry_backoff}s") 100 ] if not expect_retry_error else []), 101 any_order=False, 102 ) 103 # fmt: on 104 return retry 105 106 return test_increment 107 108 @staticmethod 109 def response_func(content, reset=None): 110 def response(): 111 stream = BytesIO(content.encode("utf8")) 112 return urllib3.response.HTTPResponse( 113 body=stream, 114 preload_content=False, 115 headers={"X-RateLimit-Reset": f"{reset}"} if reset else {}, 116 status=403, 117 ) 118 119 return response 120 121 @contextlib.contextmanager 122 def mock_retry_now(self, now): 123 if sys.version_info[0] > 3 or sys.version_info[0] == 3 and sys.version_info[1] >= 11: 124 attr = "github.GithubRetry.GithubRetry._GithubRetry__datetime" 125 else: 126 attr = "github.GithubRetry._GithubRetry__datetime" 127 with mock.patch(attr) as dt: 128 dt.now = lambda tz=None: datetime.fromtimestamp(now, tz=tz) 129 dt.fromtimestamp = datetime.fromtimestamp 130 yield 131 132 def test_primary_rate_error_with_reset(self): 133 retry = github.GithubRetry(total=3) 134 response = self.response_func(PrimaryRateLimitJson, 1644768012) 135 test_increment = self.get_test_increment_func(PrimaryRateLimitMessage) 136 137 # test 12 seconds before reset, note backoff will be 12+1 second 138 with self.mock_retry_now(1644768000): 139 retry = test_increment( 140 retry, 141 response(), 142 expected_total=2, 143 expected_backoff=12 + 1, 144 has_reset=True, 145 ) 146 with self.mock_retry_now(1644768000): 147 retry = test_increment( 148 retry, 149 response(), 150 expected_total=1, 151 expected_backoff=12 + 1, 152 has_reset=True, 153 ) 154 155 # test 2 seconds after reset, no backoff expected 156 with self.mock_retry_now(1644768014): 157 retry = test_increment(retry, response(), expected_total=0, expected_backoff=0) 158 test_increment(retry, response(), expect_retry_error=True) 159 160 def test_primary_rate_error_with_reset_and_exponential_backoff(self): 161 retry = github.GithubRetry(total=3, backoff_factor=10) 162 response = self.response_func(PrimaryRateLimitJson, 1644768012) 163 test_increment = self.get_test_increment_func(PrimaryRateLimitMessage) 164 165 # test 12 seconds before reset, note backoff will be 12+1 second 166 with self.mock_retry_now(1644768000): 167 retry = test_increment( 168 retry, 169 response(), 170 expected_total=2, 171 expected_backoff=12 + 1, 172 has_reset=True, 173 ) 174 with self.mock_retry_now(1644768000): 175 retry = test_increment( 176 retry, 177 response(), 178 expected_total=1, 179 expected_backoff=12 + 1, 180 expected_retry_backoff=20, 181 has_reset=True, 182 ) 183 184 # test 2 seconds after reset, no backoff expected 185 with self.mock_retry_now(1644768014): 186 retry = test_increment( 187 retry, 188 response(), 189 expected_total=0, 190 expected_backoff=-2, 191 expected_retry_backoff=40, 192 ) 193 test_increment(retry, response(), expect_retry_error=True) 194 195 def test_primary_rate_error_without_reset(self): 196 retry = github.GithubRetry(total=3) 197 response = self.response_func(PrimaryRateLimitJson, reset=None) 198 test_increment = self.get_test_increment_func(PrimaryRateLimitMessage) 199 200 # test without reset 201 retry = test_increment(retry, response(), expected_total=2, expected_backoff=0) 202 retry = test_increment(retry, response(), expected_total=1, expected_backoff=0) 203 retry = test_increment(retry, response(), expected_total=0, expected_backoff=0) 204 test_increment(retry, response(), expect_retry_error=True) 205 206 def test_primary_rate_error_without_reset_with_exponential_backoff(self): 207 retry = github.GithubRetry(total=3, backoff_factor=10) 208 response = self.response_func(PrimaryRateLimitJson, reset=None) 209 test_increment = self.get_test_increment_func(PrimaryRateLimitMessage) 210 211 # test without reset 212 retry = test_increment( 213 retry, 214 response(), 215 expected_total=2, 216 expected_backoff=0, 217 expected_retry_backoff=0, 218 ) 219 retry = test_increment( 220 retry, 221 response(), 222 expected_total=1, 223 expected_backoff=0, 224 expected_retry_backoff=20, 225 ) 226 retry = test_increment( 227 retry, 228 response(), 229 expected_total=0, 230 expected_backoff=0, 231 expected_retry_backoff=40, 232 ) 233 test_increment(retry, response(), expect_retry_error=True) 234 235 def test_secondary_rate_error_with_reset(self): 236 retry = github.GithubRetry(total=3) 237 response = self.response_func(SecondaryRateLimitJson, 1644768012) 238 test_increment = self.get_test_increment_func(SecondaryRateLimitMessage) 239 240 # test 12 seconds before reset, expect secondary wait seconds of 60 241 with self.mock_retry_now(1644768000): 242 retry = test_increment( 243 retry, 244 response(), 245 expected_total=2, 246 expected_backoff=60, 247 has_reset=False, 248 ) 249 with self.mock_retry_now(1644768000): 250 retry = test_increment( 251 retry, 252 response(), 253 expected_total=1, 254 expected_backoff=60, 255 has_reset=False, 256 ) 257 258 # test 2 seconds after reset, still expect secondary wait seconds of 60 259 with self.mock_retry_now(1644768014): 260 retry = test_increment(retry, response(), expected_total=0, expected_backoff=60) 261 test_increment(retry, response(), expect_retry_error=True) 262 263 def test_secondary_rate_error_with_reset_and_exponential_backoff(self): 264 retry = github.GithubRetry(total=3, backoff_factor=10, secondary_rate_wait=15) 265 response = self.response_func(SecondaryRateLimitJson, 1644768012) 266 test_increment = self.get_test_increment_func(SecondaryRateLimitMessage) 267 268 # test 12 seconds before reset, expect secondary wait seconds of 15 269 with self.mock_retry_now(1644768000): 270 retry = test_increment( 271 retry, 272 response(), 273 expected_total=2, 274 expected_backoff=15, 275 has_reset=False, 276 ) 277 with self.mock_retry_now(1644768000): 278 retry = test_increment( 279 retry, 280 response(), 281 expected_total=1, 282 expected_backoff=15, 283 expected_retry_backoff=20, 284 has_reset=False, 285 ) 286 287 # test 2 seconds after reset, exponential backoff exceeds secondary wait seconds of 15 288 with self.mock_retry_now(1644768014): 289 retry = test_increment( 290 retry, 291 response(), 292 expected_total=0, 293 expected_backoff=15, 294 expected_retry_backoff=40, 295 ) 296 test_increment(retry, response(), expect_retry_error=True) 297 298 def test_secondary_rate_error_without_reset(self): 299 retry = github.GithubRetry(total=3) 300 response = self.response_func(SecondaryRateLimitJson, reset=None) 301 test_increment = self.get_test_increment_func(SecondaryRateLimitMessage) 302 303 retry = test_increment( 304 retry, 305 response(), 306 expected_total=2, 307 expected_backoff=DEFAULT_SECONDARY_RATE_WAIT, 308 ) 309 retry = test_increment( 310 retry, 311 response(), 312 expected_total=1, 313 expected_backoff=DEFAULT_SECONDARY_RATE_WAIT, 314 ) 315 retry = test_increment( 316 retry, 317 response(), 318 expected_total=0, 319 expected_backoff=DEFAULT_SECONDARY_RATE_WAIT, 320 ) 321 test_increment(retry, response(), expect_retry_error=True) 322 323 def test_secondary_rate_error_without_reset_with_exponential_backoff(self): 324 retry = github.GithubRetry(total=3, backoff_factor=10, secondary_rate_wait=5) 325 response = self.response_func(SecondaryRateLimitJson, reset=None) 326 test_increment = self.get_test_increment_func(SecondaryRateLimitMessage) 327 328 retry = test_increment(retry, response(), expected_total=2, expected_backoff=5) 329 retry = test_increment( 330 retry, 331 response(), 332 expected_total=1, 333 expected_backoff=5, 334 expected_retry_backoff=20, 335 ) 336 retry = test_increment( 337 retry, 338 response(), 339 expected_total=0, 340 expected_backoff=5, 341 expected_retry_backoff=40, 342 ) 343 test_increment(retry, response(), expect_retry_error=True) 344 345 def do_test_default_behaviour(self, retry, response): 346 expected = Retry(total=retry.total, backoff_factor=retry.backoff_factor) 347 self.assertTrue(retry.total > 0) 348 for _ in range(retry.total): 349 retry = retry.increment("TEST", "URL", response) 350 expected = expected.increment("TEST", "URL", response) 351 self.assertEqual(expected.total, retry.total) 352 self.assertEqual(expected.get_backoff_time(), retry.get_backoff_time()) 353 354 with self.assertRaises(urllib3.exceptions.MaxRetryError): 355 retry.increment("TEST", "URL", response) 356 with self.assertRaises(urllib3.exceptions.MaxRetryError): 357 expected.increment("TEST", "URL", response) 358 359 def test_403_with_retry_after(self): 360 retry = github.GithubRetry(total=3) 361 response = urllib3.response.HTTPResponse(status=403, headers={"Retry-After": "123"}) 362 self.do_test_default_behaviour(retry, response) 363 364 def test_403_with_non_retryable_error(self): 365 retry = github.GithubRetry(total=3) 366 with self.assertRaises(github.BadUserAgentException): 367 retry.increment( 368 "TEST", 369 "URL", 370 self.response_func('{"message":"Missing or invalid User Agent string."}')(), 371 ) 372 373 def test_misc_response(self): 374 retry = github.GithubRetry(total=3) 375 response = urllib3.response.HTTPResponse() 376 self.do_test_default_behaviour(retry, response) 377 378 def test_misc_response_exponential_backoff(self): 379 retry = github.GithubRetry(total=3, backoff_factor=10) 380 response = urllib3.response.HTTPResponse() 381 self.do_test_default_behaviour(retry, response) 382 383 def test_error_in_get_content(self): 384 retry = github.GithubRetry(total=3) 385 response = urllib3.response.HTTPResponse(status=403, reason="NOT GOOD") 386 387 with mock.patch.object(retry, "_GithubRetry__log") as log: 388 with self.assertRaises(github.GithubException) as exp: 389 retry.increment("TEST", "URL", response) 390 self.assertEqual(403, exp.exception.status) 391 self.assertEqual("NOT GOOD", exp.exception.data) 392 self.assertEqual({}, exp.exception.headers) 393 394 self.assertIsInstance(exp.exception.__cause__, RuntimeError) 395 self.assertEqual(("Failed to inspect response message",), exp.exception.__cause__.args) 396 397 self.assertIsInstance(exp.exception.__cause__.__cause__, ValueError) 398 self.assertEqual( 399 ("Unable to determine whether fp is closed.",), 400 exp.exception.__cause__.__cause__.args, 401 ) 402 403 log.assert_called_once_with(logging.INFO, "Request TEST URL failed with 403: NOT GOOD")