Requester.py
1 ############################ Copyrights and license ############################ 2 # # 3 # Copyright 2022 Enrico Minack <github@enrico.minack.dev> # 4 # # 5 # This file is part of PyGithub. # 6 # http://pygithub.readthedocs.io/ # 7 # # 8 # PyGithub is free software: you can redistribute it and/or modify it under # 9 # the terms of the GNU Lesser General Public License as published by the Free # 10 # Software Foundation, either version 3 of the License, or (at your option) # 11 # any later version. # 12 # # 13 # PyGithub is distributed in the hope that it will be useful, but WITHOUT ANY # 14 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # 15 # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more # 16 # details. # 17 # # 18 # You should have received a copy of the GNU Lesser General Public License # 19 # along with PyGithub. If not, see <http://www.gnu.org/licenses/>. # 20 # # 21 ################################################################################ 22 import contextlib 23 from datetime import datetime, timedelta, timezone 24 from unittest import mock 25 26 import github 27 28 from . import Framework 29 from .GithubIntegration import APP_ID, PRIVATE_KEY 30 31 REPO_NAME = "PyGithub/PyGithub" 32 33 34 class Requester(Framework.TestCase): 35 logger = None 36 37 def setUp(self): 38 super().setUp() 39 self.logger = mock.MagicMock() 40 github.Requester.Requester.injectLogger(self.logger) 41 42 def tearDown(self): 43 github.Requester.Requester.resetLogger() 44 super().tearDown() 45 46 def testRecreation(self): 47 class TestAuth(github.Auth.AppAuth): 48 pass 49 50 # create a Requester with non-default arguments 51 auth = TestAuth(123, "key") 52 requester = github.Requester.Requester( 53 auth=auth, 54 base_url="https://base.url", 55 timeout=1, 56 user_agent="user agent", 57 per_page=123, 58 verify=False, 59 retry=3, 60 pool_size=5, 61 seconds_between_requests=1.2, 62 seconds_between_writes=3.4, 63 ) 64 kwargs = requester.kwargs 65 66 # assert kwargs consists of ALL constructor arguments 67 self.assertEqual(kwargs.keys(), github.Requester.Requester.__init__.__annotations__.keys()) 68 self.assertEqual( 69 kwargs, 70 dict( 71 auth=auth, 72 base_url="https://base.url", 73 timeout=1, 74 user_agent="user agent", 75 per_page=123, 76 verify=False, 77 retry=3, 78 pool_size=5, 79 seconds_between_requests=1.2, 80 seconds_between_writes=3.4, 81 ), 82 ) 83 84 # create a copy Requester, assert identity via kwargs 85 copy = github.Requester.Requester(**kwargs) 86 self.assertEqual(copy.kwargs, kwargs) 87 88 # create Github instance, assert identity requester 89 gh = github.Github(**kwargs) 90 self.assertEqual(gh._Github__requester.kwargs, kwargs) 91 92 # create GithubIntegration instance, assert identity requester 93 gi = github.GithubIntegration(**kwargs) 94 self.assertEqual(gi._GithubIntegration__requester.kwargs, kwargs) 95 96 def testWithAuth(self): 97 class TestAuth(github.Auth.AppAuth): 98 pass 99 100 # create a Requester with non-default arguments 101 auth = TestAuth(123, "key") 102 requester = github.Requester.Requester( 103 auth=auth, 104 base_url="https://base.url", 105 timeout=1, 106 user_agent="user agent", 107 per_page=123, 108 verify=False, 109 retry=3, 110 pool_size=5, 111 seconds_between_requests=1.2, 112 seconds_between_writes=3.4, 113 ) 114 115 # create a copy with different auth 116 auth2 = TestAuth(456, "key2") 117 copy = requester.withAuth(auth2) 118 119 # assert kwargs of copy 120 self.assertEqual( 121 copy.kwargs, 122 dict( 123 auth=auth2, 124 base_url="https://base.url", 125 timeout=1, 126 user_agent="user agent", 127 per_page=123, 128 verify=False, 129 retry=3, 130 pool_size=5, 131 seconds_between_requests=1.2, 132 seconds_between_writes=3.4, 133 ), 134 ) 135 136 def testCloseGithub(self): 137 mocked_connection = mock.MagicMock() 138 mocked_custom_connection = mock.MagicMock() 139 140 with github.Github() as gh: 141 requester = gh._Github__requester 142 requester._Requester__connection = mocked_connection 143 requester._Requester__custom_connections.append(mocked_custom_connection) 144 145 mocked_connection.close.assert_called_once_with() 146 mocked_custom_connection.close.assert_called_once_with() 147 self.assertIsNone(requester._Requester__connection) 148 149 def testCloseGithubIntegration(self): 150 mocked_connection = mock.MagicMock() 151 mocked_custom_connection = mock.MagicMock() 152 153 auth = github.Auth.AppAuth(APP_ID, PRIVATE_KEY) 154 with github.GithubIntegration(auth=auth) as gi: 155 requester = gi._GithubIntegration__requester 156 requester._Requester__connection = mocked_connection 157 requester._Requester__custom_connections.append(mocked_custom_connection) 158 159 mocked_connection.close.assert_called_once_with() 160 mocked_custom_connection.close.assert_called_once_with() 161 self.assertIsNone(requester._Requester__connection) 162 163 def testLoggingRedirection(self): 164 self.assertEqual(self.g.get_repo("EnricoMi/test").name, "test-renamed") 165 self.logger.info.assert_called_once_with( 166 "Following Github server redirection from /repos/EnricoMi/test to /repositories/638123443" 167 ) 168 169 def testBaseUrlSchemeRedirection(self): 170 gh = github.Github(base_url="http://api.github.com") 171 with self.assertRaises(RuntimeError) as exc: 172 gh.get_repo("PyGithub/PyGithub") 173 self.assertEqual( 174 exc.exception.args, 175 ( 176 "Github server redirected from http protocol to https, please correct your " 177 "Github server URL via base_url: Github(base_url=...)", 178 ), 179 ) 180 181 def testBaseUrlHostRedirection(self): 182 gh = github.Github(base_url="https://www.github.com") 183 with self.assertRaises(RuntimeError) as exc: 184 gh.get_repo("PyGithub/PyGithub") 185 self.assertEqual( 186 exc.exception.args, 187 ( 188 "Github server redirected from host www.github.com to github.com, " 189 "please correct your Github server URL via base_url: Github(base_url=...)", 190 ), 191 ) 192 193 def testBaseUrlPortRedirection(self): 194 # replay data forged 195 gh = github.Github(base_url="https://api.github.com") 196 with self.assertRaises(RuntimeError) as exc: 197 gh.get_repo("PyGithub/PyGithub") 198 self.assertEqual( 199 exc.exception.args, 200 ( 201 "Requested https://api.github.com/repos/PyGithub/PyGithub but server " 202 "redirected to https://api.github.com:443/repos/PyGithub/PyGithub, " 203 "you may need to correct your Github server URL " 204 "via base_url: Github(base_url=...)", 205 ), 206 ) 207 208 def testBaseUrlPrefixRedirection(self): 209 # replay data forged 210 gh = github.Github(base_url="https://api.github.com/api/v3") 211 self.assertEqual(gh.get_repo("PyGithub/PyGithub").name, "PyGithub") 212 self.logger.info.assert_called_once_with( 213 "Following Github server redirection from /api/v3/repos/PyGithub/PyGithub to /repos/PyGithub/PyGithub" 214 ) 215 216 PrimaryRateLimitErrors = [ 217 "API rate limit exceeded for x.x.x.x. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.)", 218 ] 219 SecondaryRateLimitErrors = [ 220 "You have triggered an abuse detection mechanism. Please wait a few minutes before you try again.", 221 "You have triggered an abuse detection mechanism and have been temporarily blocked from content creation. Please retry your request again later." 222 "You have exceeded a secondary rate limit and have been temporarily blocked from content creation. Please retry your request again later.", 223 "You have exceeded a secondary rate limit. Please wait a few minutes before you try again.", 224 "Something else here. Please wait a few minutes before you try again.", 225 ] 226 OtherErrors = ["User does not exist or is not a member of the organization"] 227 228 def testIsRateLimitError(self): 229 for message in self.PrimaryRateLimitErrors + self.SecondaryRateLimitErrors: 230 self.assertTrue(github.Requester.Requester.isRateLimitError(message), message) 231 for message in self.OtherErrors: 232 self.assertFalse(github.Requester.Requester.isRateLimitError(message), message) 233 234 def testIsPrimaryRateLimitError(self): 235 for message in self.PrimaryRateLimitErrors: 236 self.assertTrue(github.Requester.Requester.isPrimaryRateLimitError(message), message) 237 for message in self.OtherErrors + self.SecondaryRateLimitErrors: 238 self.assertFalse(github.Requester.Requester.isPrimaryRateLimitError(message), message) 239 240 def testIsSecondaryRateLimitError(self): 241 for message in self.SecondaryRateLimitErrors: 242 self.assertTrue(github.Requester.Requester.isSecondaryRateLimitError(message), message) 243 for message in self.OtherErrors + self.PrimaryRateLimitErrors: 244 self.assertFalse(github.Requester.Requester.isSecondaryRateLimitError(message), message) 245 246 def assertException(self, exception, exception_type, status, data, headers, string): 247 self.assertIsInstance(exception, exception_type) 248 self.assertEqual(exception.status, status) 249 if data is None: 250 self.assertIsNone(exception.data) 251 else: 252 self.assertEqual(exception.data, data) 253 self.assertEqual(exception.headers, headers) 254 self.assertEqual(str(exception), string) 255 256 def testShouldCreateBadCredentialsException(self): 257 exc = self.g._Github__requester.createException(401, {"header": "value"}, {"message": "Bad credentials"}) 258 self.assertException( 259 exc, 260 github.BadCredentialsException, 261 401, 262 {"message": "Bad credentials"}, 263 {"header": "value"}, 264 '401 {"message": "Bad credentials"}', 265 ) 266 267 def testShouldCreateTwoFactorException(self): 268 exc = self.g._Github__requester.createException( 269 401, 270 {"x-github-otp": "required; app"}, 271 { 272 "message": "Must specify two-factor authentication OTP code.", 273 "documentation_url": "https://developer.github.com/v3/auth#working-with-two-factor-authentication", 274 }, 275 ) 276 self.assertException( 277 exc, 278 github.TwoFactorException, 279 401, 280 { 281 "message": "Must specify two-factor authentication OTP code.", 282 "documentation_url": "https://developer.github.com/v3/auth#working-with-two-factor-authentication", 283 }, 284 {"x-github-otp": "required; app"}, 285 '401 {"message": "Must specify two-factor authentication OTP code.", "documentation_url": "https://developer.github.com/v3/auth#working-with-two-factor-authentication"}', 286 ) 287 288 def testShouldCreateBadUserAgentException(self): 289 exc = self.g._Github__requester.createException( 290 403, 291 {"header": "value"}, 292 {"message": "Missing or invalid User Agent string"}, 293 ) 294 self.assertException( 295 exc, 296 github.BadUserAgentException, 297 403, 298 {"message": "Missing or invalid User Agent string"}, 299 {"header": "value"}, 300 '403 {"message": "Missing or invalid User Agent string"}', 301 ) 302 303 def testShouldCreateRateLimitExceededException(self): 304 for message in self.PrimaryRateLimitErrors + self.SecondaryRateLimitErrors: 305 with self.subTest(message=message): 306 exc = self.g._Github__requester.createException(403, {"header": "value"}, {"message": message}) 307 self.assertException( 308 exc, 309 github.RateLimitExceededException, 310 403, 311 {"message": message}, 312 {"header": "value"}, 313 f'403 {{"message": "{message}"}}', 314 ) 315 316 def testShouldCreateUnknownObjectException(self): 317 exc = self.g._Github__requester.createException(404, {"header": "value"}, {"message": "Not Found"}) 318 self.assertException( 319 exc, 320 github.UnknownObjectException, 321 404, 322 {"message": "Not Found"}, 323 {"header": "value"}, 324 '404 {"message": "Not Found"}', 325 ) 326 327 def testShouldCreateGithubException(self): 328 for status in range(400, 600): 329 with self.subTest(status=status): 330 exc = self.g._Github__requester.createException( 331 status, {"header": "value"}, {"message": "Something unknown"} 332 ) 333 self.assertException( 334 exc, 335 github.GithubException, 336 status, 337 {"message": "Something unknown"}, 338 {"header": "value"}, 339 f'{status} {{"message": "Something unknown"}}', 340 ) 341 342 def testShouldCreateExceptionWithoutMessage(self): 343 for status in range(400, 600): 344 with self.subTest(status=status): 345 exc = self.g._Github__requester.createException(status, {}, {}) 346 self.assertException(exc, github.GithubException, status, {}, {}, f"{status} {{}}") 347 348 def testShouldCreateExceptionWithoutOutput(self): 349 for status in range(400, 600): 350 with self.subTest(status=status): 351 exc = self.g._Github__requester.createException(status, {}, None) 352 self.assertException(exc, github.GithubException, status, None, {}, f"{status}") 353 354 355 class RequesterThrottleTestCase(Framework.TestCase): 356 per_page = 10 357 358 mock_time = [datetime.now(timezone.utc)] 359 360 def sleep(self, seconds): 361 self.mock_time[0] = self.mock_time[0] + timedelta(seconds=seconds) 362 363 def now(self, tz=None): 364 return self.mock_time[0] 365 366 @contextlib.contextmanager 367 def mock_sleep(self): 368 with mock.patch("github.Requester.time.sleep", side_effect=self.sleep) as sleep_mock, mock.patch( 369 "github.Requester.datetime" 370 ) as datetime_mock: 371 datetime_mock.now = self.now 372 yield sleep_mock 373 374 375 class RequesterUnThrottled(RequesterThrottleTestCase): 376 def testShouldNotDeferRequests(self): 377 with self.mock_sleep() as sleep_mock: 378 # same test setup as in RequesterThrottled.testShouldDeferRequests 379 repository = self.g.get_repo(REPO_NAME) 380 releases = list(repository.get_releases()) 381 self.assertEqual(len(releases), 30) 382 383 sleep_mock.assert_not_called() 384 385 386 class RequesterThrottled(RequesterThrottleTestCase): 387 seconds_between_requests = 1.0 388 seconds_between_writes = 3.0 389 390 def testShouldDeferRequests(self): 391 with self.mock_sleep() as sleep_mock: 392 # same test setup as in RequesterUnThrottled.testShouldNotDeferRequests 393 repository = self.g.get_repo(REPO_NAME) 394 releases = [release for release in repository.get_releases()] 395 self.assertEqual(len(releases), 30) 396 397 self.assertEqual(sleep_mock.call_args_list, [mock.call(1), mock.call(1), mock.call(1)]) 398 399 def testShouldDeferWrites(self): 400 with self.mock_sleep() as sleep_mock: 401 # same test setup as in AuthenticatedUser.testEmail 402 user = self.g.get_user() 403 emails = user.get_emails() 404 self.assertEqual( 405 [item.email for item in emails], 406 ["vincent@vincent-jacques.net", "github.com@vincent-jacques.net"], 407 ) 408 self.assertTrue(emails[0].primary) 409 self.assertTrue(emails[0].verified) 410 self.assertEqual(emails[0].visibility, "private") 411 user.add_to_emails("1@foobar.com", "2@foobar.com") 412 self.assertEqual( 413 [item.email for item in user.get_emails()], 414 [ 415 "vincent@vincent-jacques.net", 416 "1@foobar.com", 417 "2@foobar.com", 418 "github.com@vincent-jacques.net", 419 ], 420 ) 421 user.remove_from_emails("1@foobar.com", "2@foobar.com") 422 self.assertEqual( 423 [item.email for item in user.get_emails()], 424 ["vincent@vincent-jacques.net", "github.com@vincent-jacques.net"], 425 ) 426 427 self.assertEqual( 428 sleep_mock.call_args_list, 429 [ 430 # g.get_user() does not call into GitHub API 431 # user.get_emails() is the first request so no waiting needed 432 # user.add_to_emails is a write request, this is the first write request 433 mock.call(1), 434 # user.get_emails() is a read request 435 mock.call(1), 436 # user.remove_from_emails is a write request, it has to be 3 seconds after the last write 437 mock.call(2), 438 # user.get_emails() is a read request 439 mock.call(1), 440 ], 441 )