Requester.py
1 ############################ Copyrights and license ############################ 2 # # 3 # Copyright 2012 Andrew Bettison <andrewb@zip.com.au> # 4 # Copyright 2012 Dima Kukushkin <dima@kukushkin.me> # 5 # Copyright 2012 Michael Woodworth <mwoodworth@upverter.com> # 6 # Copyright 2012 Petteri Muilu <pmuilu@xena.(none)> # 7 # Copyright 2012 Steve English <steve.english@navetas.com> # 8 # Copyright 2012 Vincent Jacques <vincent@vincent-jacques.net> # 9 # Copyright 2012 Zearin <zearin@gonk.net> # 10 # Copyright 2013 AKFish <akfish@gmail.com> # 11 # Copyright 2013 Cameron White <cawhite@pdx.edu> # 12 # Copyright 2013 Ed Jackson <ed.jackson@gmail.com> # 13 # Copyright 2013 Jonathan J Hunt <hunt@braincorporation.com> # 14 # Copyright 2013 Mark Roddy <markroddy@gmail.com> # 15 # Copyright 2013 Vincent Jacques <vincent@vincent-jacques.net> # 16 # Copyright 2014 Jimmy Zelinskie <jimmyzelinskie@gmail.com> # 17 # Copyright 2014 Vincent Jacques <vincent@vincent-jacques.net> # 18 # Copyright 2015 Brian Eugley <Brian.Eugley@capitalone.com> # 19 # Copyright 2015 Daniel Pocock <daniel@pocock.pro> # 20 # Copyright 2015 Jimmy Zelinskie <jimmyzelinskie@gmail.com> # 21 # Copyright 2016 Denis K <f1nal@cgaming.org> # 22 # Copyright 2016 Jared K. Smith <jaredsmith@jaredsmith.net> # 23 # Copyright 2016 Jimmy Zelinskie <jimmy.zelinskie+git@gmail.com> # 24 # Copyright 2016 Mathieu Mitchell <mmitchell@iweb.com> # 25 # Copyright 2016 Peter Buckley <dx-pbuckley@users.noreply.github.com> # 26 # Copyright 2017 Chris McBride <thehighlander@users.noreply.github.com> # 27 # Copyright 2017 Hugo <hugovk@users.noreply.github.com> # 28 # Copyright 2017 Simon <spam@esemi.ru> # 29 # Copyright 2018 Dylan <djstein@ncsu.edu> # 30 # Copyright 2018 Maarten Fonville <mfonville@users.noreply.github.com> # 31 # Copyright 2018 Mike Miller <github@mikeage.net> # 32 # Copyright 2018 R1kk3r <R1kk3r@users.noreply.github.com> # 33 # Copyright 2018 sfdye <tsfdye@gmail.com> # 34 # Copyright 2022 Enrico Minack <github@enrico.minack.dev> # 35 # # 36 # This file is part of PyGithub. # 37 # http://pygithub.readthedocs.io/ # 38 # # 39 # PyGithub is free software: you can redistribute it and/or modify it under # 40 # the terms of the GNU Lesser General Public License as published by the Free # 41 # Software Foundation, either version 3 of the License, or (at your option) # 42 # any later version. # 43 # # 44 # PyGithub is distributed in the hope that it will be useful, but WITHOUT ANY # 45 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # 46 # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more # 47 # details. # 48 # # 49 # You should have received a copy of the GNU Lesser General Public License # 50 # along with PyGithub. If not, see <http://www.gnu.org/licenses/>. # 51 # # 52 ################################################################################ 53 54 import io 55 import json 56 import logging 57 import mimetypes 58 import os 59 import re 60 import threading 61 import time 62 import urllib 63 import urllib.parse 64 from collections import deque 65 from datetime import datetime, timezone 66 from io import IOBase 67 from typing import ( 68 TYPE_CHECKING, 69 Any, 70 BinaryIO, 71 Callable, 72 Deque, 73 Dict, 74 Generic, 75 ItemsView, 76 List, 77 Optional, 78 Tuple, 79 Type, 80 TypeVar, 81 Union, 82 ) 83 84 import requests 85 import requests.adapters 86 from urllib3 import Retry 87 88 import github.Consts as Consts 89 import github.GithubException as GithubException 90 91 if TYPE_CHECKING: 92 from .AppAuthentication import AppAuthentication 93 from .Auth import Auth 94 from .GithubObject import GithubObject 95 from .InstallationAuthorization import InstallationAuthorization 96 97 T = TypeVar("T") 98 99 # For App authentication, time remaining before token expiration to request a new one 100 ACCESS_TOKEN_REFRESH_THRESHOLD_SECONDS = 20 101 102 103 class RequestsResponse: 104 # mimic the httplib response object 105 def __init__(self, r: requests.Response): 106 self.status = r.status_code 107 self.headers = r.headers 108 self.text = r.text 109 110 def getheaders(self) -> ItemsView[str, str]: 111 return self.headers.items() 112 113 def read(self) -> str: 114 return self.text 115 116 117 class HTTPSRequestsConnectionClass: 118 retry: Union[int, Retry] 119 120 # mimic the httplib connection object 121 def __init__( 122 self, 123 host: str, 124 port: Optional[int] = None, 125 strict: bool = False, 126 timeout: Optional[int] = None, 127 retry: Optional[Union[int, Retry]] = None, 128 pool_size: Optional[int] = None, 129 **kwargs: Any, 130 ) -> None: 131 self.port = port if port else 443 132 self.host = host 133 self.protocol = "https" 134 self.timeout = timeout 135 self.verify = kwargs.get("verify", True) 136 self.session = requests.Session() 137 # having Session.auth set something other than None disables falling back to .netrc file 138 # https://github.com/psf/requests/blob/d63e94f552ebf77ccf45d97e5863ac46500fa2c7/src/requests/sessions.py#L480-L481 139 # see https://github.com/PyGithub/PyGithub/pull/2703 140 self.session.auth = Requester.noopAuth 141 142 if retry is None: 143 self.retry = requests.adapters.DEFAULT_RETRIES 144 else: 145 self.retry = retry 146 147 if pool_size is None: 148 self.pool_size = requests.adapters.DEFAULT_POOLSIZE 149 else: 150 self.pool_size = pool_size 151 152 self.adapter = requests.adapters.HTTPAdapter( 153 max_retries=self.retry, 154 pool_connections=self.pool_size, 155 pool_maxsize=self.pool_size, 156 ) 157 self.session.mount("https://", self.adapter) 158 159 def request( 160 self, 161 verb: str, 162 url: str, 163 input: Optional[Union[str, io.BufferedReader]], 164 headers: Dict[str, str], 165 ) -> None: 166 self.verb = verb 167 self.url = url 168 self.input = input 169 self.headers = headers 170 171 def getresponse(self) -> RequestsResponse: 172 verb = getattr(self.session, self.verb.lower()) 173 url = f"{self.protocol}://{self.host}:{self.port}{self.url}" 174 r = verb( 175 url, 176 headers=self.headers, 177 data=self.input, 178 timeout=self.timeout, 179 verify=self.verify, 180 allow_redirects=False, 181 ) 182 return RequestsResponse(r) 183 184 def close(self) -> None: 185 self.session.close() 186 187 188 class HTTPRequestsConnectionClass: 189 # mimic the httplib connection object 190 def __init__( 191 self, 192 host: str, 193 port: Optional[int] = None, 194 strict: bool = False, 195 timeout: Optional[int] = None, 196 retry: Optional[Union[int, Retry]] = None, 197 pool_size: Optional[int] = None, 198 **kwargs: Any, 199 ): 200 self.port = port if port else 80 201 self.host = host 202 self.protocol = "http" 203 self.timeout = timeout 204 self.verify = kwargs.get("verify", True) 205 self.session = requests.Session() 206 # having Session.auth set something other than None disables falling back to .netrc file 207 # https://github.com/psf/requests/blob/d63e94f552ebf77ccf45d97e5863ac46500fa2c7/src/requests/sessions.py#L480-L481 208 # see https://github.com/PyGithub/PyGithub/pull/2703 209 self.session.auth = Requester.noopAuth 210 211 if retry is None: 212 self.retry = requests.adapters.DEFAULT_RETRIES 213 else: 214 self.retry = retry # type: ignore 215 216 if pool_size is None: 217 self.pool_size = requests.adapters.DEFAULT_POOLSIZE 218 else: 219 self.pool_size = pool_size 220 221 self.adapter = requests.adapters.HTTPAdapter( 222 max_retries=self.retry, 223 pool_connections=self.pool_size, 224 pool_maxsize=self.pool_size, 225 ) 226 self.session.mount("http://", self.adapter) 227 228 def request(self, verb: str, url: str, input: None, headers: Dict[str, str]) -> None: 229 self.verb = verb 230 self.url = url 231 self.input = input 232 self.headers = headers 233 234 def getresponse(self) -> RequestsResponse: 235 verb = getattr(self.session, self.verb.lower()) 236 url = f"{self.protocol}://{self.host}:{self.port}{self.url}" 237 r = verb( 238 url, 239 headers=self.headers, 240 data=self.input, 241 timeout=self.timeout, 242 verify=self.verify, 243 allow_redirects=False, 244 ) 245 return RequestsResponse(r) 246 247 def close(self) -> None: 248 self.session.close() 249 250 251 class Requester: 252 __installation_authorization: Optional["InstallationAuthorization"] 253 __app_auth: Optional["AppAuthentication"] 254 255 __httpConnectionClass = HTTPRequestsConnectionClass 256 __httpsConnectionClass = HTTPSRequestsConnectionClass 257 __persist = True 258 __logger: Optional[logging.Logger] = None 259 260 _frameBuffer: List[Any] 261 262 @staticmethod 263 def noopAuth(request: requests.models.PreparedRequest) -> requests.models.PreparedRequest: 264 return request 265 266 @classmethod 267 def injectConnectionClasses( 268 cls, 269 httpConnectionClass: Type[HTTPRequestsConnectionClass], 270 httpsConnectionClass: Type[HTTPSRequestsConnectionClass], 271 ) -> None: 272 cls.__persist = False 273 cls.__httpConnectionClass = httpConnectionClass 274 cls.__httpsConnectionClass = httpsConnectionClass 275 276 @classmethod 277 def resetConnectionClasses(cls) -> None: 278 cls.__persist = True 279 cls.__httpConnectionClass = HTTPRequestsConnectionClass 280 cls.__httpsConnectionClass = HTTPSRequestsConnectionClass 281 282 @classmethod 283 def injectLogger(cls, logger: logging.Logger) -> None: 284 cls.__logger = logger 285 286 @classmethod 287 def resetLogger(cls) -> None: 288 cls.__logger = None 289 290 ############################################################# 291 # For Debug 292 @classmethod 293 def setDebugFlag(cls, flag: bool) -> None: 294 cls.DEBUG_FLAG = flag 295 296 @classmethod 297 def setOnCheckMe(cls, onCheckMe: Callable) -> None: 298 cls.ON_CHECK_ME = onCheckMe 299 300 DEBUG_FLAG = False 301 302 DEBUG_FRAME_BUFFER_SIZE = 1024 303 304 DEBUG_HEADER_KEY = "DEBUG_FRAME" 305 306 ON_CHECK_ME: Optional[Callable] = None 307 308 def NEW_DEBUG_FRAME(self, requestHeader: Dict[str, str]) -> None: 309 """ 310 Initialize a debug frame with requestHeader 311 Frame count is updated and will be attached to respond header 312 The structure of a frame: [requestHeader, statusCode, responseHeader, raw_data] 313 Some of them may be None 314 """ 315 if self.DEBUG_FLAG: # pragma no branch (Flag always set in tests) 316 new_frame = [requestHeader, None, None, None] 317 if self._frameCount < self.DEBUG_FRAME_BUFFER_SIZE - 1: # pragma no branch (Should be covered) 318 self._frameBuffer.append(new_frame) 319 else: 320 self._frameBuffer[0] = new_frame # pragma no cover (Should be covered) 321 322 self._frameCount = len(self._frameBuffer) - 1 323 324 def DEBUG_ON_RESPONSE(self, statusCode: int, responseHeader: Dict[str, Union[str, int]], data: str) -> None: 325 """ 326 Update current frame with response 327 Current frame index will be attached to responseHeader 328 """ 329 if self.DEBUG_FLAG: # pragma no branch (Flag always set in tests) 330 self._frameBuffer[self._frameCount][1:4] = [ 331 statusCode, 332 responseHeader, 333 data, 334 ] 335 responseHeader[self.DEBUG_HEADER_KEY] = self._frameCount 336 337 def check_me(self, obj: "GithubObject") -> None: 338 if self.DEBUG_FLAG and self.ON_CHECK_ME is not None: # pragma no branch (Flag always set in tests) 339 frame = None 340 if self.DEBUG_HEADER_KEY in obj._headers: 341 frame_index = obj._headers[self.DEBUG_HEADER_KEY] 342 frame = self._frameBuffer[frame_index] # type: ignore 343 self.ON_CHECK_ME(obj, frame) 344 345 def _initializeDebugFeature(self) -> None: 346 self._frameCount = 0 347 self._frameBuffer = [] 348 349 ############################################################# 350 351 _frameCount: int 352 __connectionClass: Union[Type[HTTPRequestsConnectionClass], Type[HTTPSRequestsConnectionClass]] 353 __hostname: str 354 __authorizationHeader: Optional[str] 355 __seconds_between_requests: Optional[float] 356 __seconds_between_writes: Optional[float] 357 358 # keep arguments in-sync with github.MainClass and GithubIntegration 359 def __init__( 360 self, 361 auth: Optional["Auth"], 362 base_url: str, 363 timeout: int, 364 user_agent: str, 365 per_page: int, 366 verify: Union[bool, str], 367 retry: Optional[Union[int, Retry]], 368 pool_size: Optional[int], 369 seconds_between_requests: Optional[float] = None, 370 seconds_between_writes: Optional[float] = None, 371 ): 372 self._initializeDebugFeature() 373 374 self.__auth = auth 375 self.__base_url = base_url 376 377 o = urllib.parse.urlparse(base_url) 378 self.__hostname = o.hostname # type: ignore 379 self.__port = o.port 380 self.__prefix = o.path 381 self.__timeout = timeout 382 self.__retry = retry # NOTE: retry can be either int or an urllib3 Retry object 383 self.__pool_size = pool_size 384 self.__seconds_between_requests = seconds_between_requests 385 self.__seconds_between_writes = seconds_between_writes 386 self.__last_requests: Dict[str, float] = dict() 387 self.__scheme = o.scheme 388 if o.scheme == "https": 389 self.__connectionClass = self.__httpsConnectionClass 390 elif o.scheme == "http": 391 self.__connectionClass = self.__httpConnectionClass 392 else: 393 assert False, "Unknown URL scheme" 394 self.__connection: Optional[Union[HTTPRequestsConnectionClass, HTTPSRequestsConnectionClass]] = None 395 self.__connection_lock = threading.Lock() 396 self.__custom_connections: Deque[Union[HTTPRequestsConnectionClass, HTTPSRequestsConnectionClass]] = deque() 397 self.rate_limiting = (-1, -1) 398 self.rate_limiting_resettime = 0 399 self.FIX_REPO_GET_GIT_REF = True 400 self.per_page = per_page 401 402 self.oauth_scopes = None 403 404 assert user_agent is not None, ( 405 "github now requires a user-agent. " 406 "See https://docs.github.com/en/rest/overview/resources-in-the-rest-api#user-agent-required" 407 ) 408 self.__userAgent = user_agent 409 self.__verify = verify 410 411 self.__installation_authorization = None 412 413 # provide auth implementations that require a requester with this requester 414 if isinstance(self.__auth, WithRequester): 415 self.__auth.withRequester(self) 416 417 def __getstate__(self) -> Dict[str, Any]: 418 state = self.__dict__.copy() 419 # __connection_lock is not picklable 420 del state["_Requester__connection_lock"] 421 # __connection is not usable on remote, so ignore it 422 del state["_Requester__connection"] 423 # __custom_connections is not usable on remote, so ignore it 424 del state["_Requester__custom_connections"] 425 return state 426 427 def __setstate__(self, state: Dict[str, Any]) -> None: 428 self.__dict__.update(state) 429 self.__connection_lock = threading.Lock() 430 self.__connection = None 431 self.__custom_connections = deque() 432 433 def close(self) -> None: 434 """ 435 Close the connection to the server. 436 """ 437 with self.__connection_lock: 438 if self.__connection is not None: 439 self.__connection.close() 440 self.__connection = None 441 while self.__custom_connections: 442 self.__custom_connections.popleft().close() 443 444 @property 445 def kwargs(self) -> Dict[str, Any]: 446 """ 447 Returns arguments required to recreate this Requester with Requester.__init__, as well as 448 with MainClass.__init__ and GithubIntegration.__init__. 449 :return: 450 """ 451 return dict( 452 auth=self.__auth, 453 base_url=self.__base_url, 454 timeout=self.__timeout, 455 user_agent=self.__userAgent, 456 per_page=self.per_page, 457 verify=self.__verify, 458 retry=self.__retry, 459 pool_size=self.__pool_size, 460 seconds_between_requests=self.__seconds_between_requests, 461 seconds_between_writes=self.__seconds_between_writes, 462 ) 463 464 @property 465 def base_url(self) -> str: 466 return self.__base_url 467 468 @property 469 def hostname(self) -> str: 470 return self.__hostname 471 472 @property 473 def auth(self) -> Optional["Auth"]: 474 return self.__auth 475 476 def withAuth(self, auth: Optional["Auth"]) -> "Requester": 477 """ 478 Create a new requester instance with identical configuration but the given authentication method. 479 :param auth: authentication method 480 :return: new Requester implementation 481 """ 482 kwargs = self.kwargs 483 kwargs.update(auth=auth) 484 return Requester(**kwargs) 485 486 def requestJsonAndCheck( 487 self, 488 verb: str, 489 url: str, 490 parameters: Optional[Dict[str, Any]] = None, 491 headers: Optional[Dict[str, str]] = None, 492 input: Optional[Any] = None, 493 ) -> Tuple[Dict[str, Any], Any]: 494 return self.__check(*self.requestJson(verb, url, parameters, headers, input, self.__customConnection(url))) 495 496 def requestMultipartAndCheck( 497 self, 498 verb: str, 499 url: str, 500 parameters: Optional[Dict[str, Any]] = None, 501 headers: Optional[Dict[str, Any]] = None, 502 input: Optional[Dict[str, str]] = None, 503 ) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]]]: 504 return self.__check(*self.requestMultipart(verb, url, parameters, headers, input, self.__customConnection(url))) 505 506 def requestBlobAndCheck( 507 self, 508 verb: str, 509 url: str, 510 parameters: Optional[Dict[str, str]] = None, 511 headers: Optional[Dict[str, str]] = None, 512 input: Optional[str] = None, 513 cnx: Optional[Union[HTTPRequestsConnectionClass, HTTPSRequestsConnectionClass]] = None, 514 ) -> Tuple[Dict[str, Any], Dict[str, Any]]: 515 return self.__check(*self.requestBlob(verb, url, parameters, headers, input, self.__customConnection(url))) 516 517 def __check( 518 self, 519 status: int, 520 responseHeaders: Dict[str, Any], 521 output: str, 522 ) -> Tuple[Dict[str, Any], Any]: 523 data = self.__structuredFromJson(output) 524 if status >= 400: 525 raise self.createException(status, responseHeaders, data) 526 return responseHeaders, data 527 528 def __customConnection( 529 self, url: str 530 ) -> Optional[Union[HTTPRequestsConnectionClass, HTTPSRequestsConnectionClass]]: 531 cnx: Optional[Union[HTTPRequestsConnectionClass, HTTPSRequestsConnectionClass]] = None 532 if not url.startswith("/"): 533 o = urllib.parse.urlparse(url) 534 if ( 535 o.hostname != self.__hostname 536 or (o.port and o.port != self.__port) 537 or (o.scheme != self.__scheme and not (o.scheme == "https" and self.__scheme == "http")) 538 ): # issue80 539 if o.scheme == "http": 540 cnx = self.__httpConnectionClass( 541 o.hostname, # type: ignore 542 o.port, 543 retry=self.__retry, 544 pool_size=self.__pool_size, 545 ) 546 self.__custom_connections.append(cnx) 547 elif o.scheme == "https": 548 cnx = self.__httpsConnectionClass( 549 o.hostname, # type: ignore 550 o.port, 551 retry=self.__retry, 552 pool_size=self.__pool_size, 553 ) 554 self.__custom_connections.append(cnx) 555 return cnx 556 557 @classmethod 558 def createException( 559 cls, 560 status: int, 561 headers: Dict[str, Any], 562 output: Dict[str, Any], 563 ) -> GithubException.GithubException: 564 message = output.get("message", "").lower() if output is not None else "" 565 566 exc = GithubException.GithubException 567 if status == 401 and message == "bad credentials": 568 exc = GithubException.BadCredentialsException 569 elif status == 401 and Consts.headerOTP in headers and re.match(r".*required.*", headers[Consts.headerOTP]): 570 exc = GithubException.TwoFactorException 571 elif status == 403 and message.startswith("missing or invalid user agent string"): 572 exc = GithubException.BadUserAgentException 573 elif status == 403 and cls.isRateLimitError(message): 574 exc = GithubException.RateLimitExceededException 575 elif status == 404 and message == "not found": 576 exc = GithubException.UnknownObjectException 577 578 return exc(status, output, headers) 579 580 @classmethod 581 def isRateLimitError(cls, message: str) -> bool: 582 return cls.isPrimaryRateLimitError(message) or cls.isSecondaryRateLimitError(message) 583 584 @classmethod 585 def isPrimaryRateLimitError(cls, message: str) -> bool: 586 if not message: 587 return False 588 589 message = message.lower() 590 return message.startswith("api rate limit exceeded") 591 592 @classmethod 593 def isSecondaryRateLimitError(cls, message: str) -> bool: 594 if not message: 595 return False 596 597 message = message.lower() 598 return ( 599 message.startswith("you have exceeded a secondary rate limit") 600 or message.endswith("please retry your request again later.") 601 or message.endswith("please wait a few minutes before you try again.") 602 ) 603 604 def __structuredFromJson(self, data: str) -> Any: 605 if len(data) == 0: 606 return None 607 else: 608 if isinstance(data, bytes): 609 data = data.decode("utf-8") 610 try: 611 return json.loads(data) 612 except ValueError: 613 if data.startswith("{") or data.startswith("["): 614 raise 615 return {"data": data} 616 617 def requestJson( 618 self, 619 verb: str, 620 url: str, 621 parameters: Optional[Dict[str, Any]] = None, 622 headers: Optional[Dict[str, Any]] = None, 623 input: Optional[Any] = None, 624 cnx: Optional[Union[HTTPRequestsConnectionClass, HTTPSRequestsConnectionClass]] = None, 625 ) -> Tuple[int, Dict[str, Any], str]: 626 def encode(input: Any) -> Tuple[str, str]: 627 return "application/json", json.dumps(input) 628 629 return self.__requestEncode(cnx, verb, url, parameters, headers, input, encode) 630 631 def requestMultipart( 632 self, 633 verb: str, 634 url: str, 635 parameters: Optional[Dict[str, Any]] = None, 636 headers: Optional[Dict[str, Any]] = None, 637 input: Optional[Dict[str, str]] = None, 638 cnx: Optional[Union[HTTPRequestsConnectionClass, HTTPSRequestsConnectionClass]] = None, 639 ) -> Tuple[int, Dict[str, Any], str]: 640 def encode(input: Dict[str, Any]) -> Tuple[str, str]: 641 boundary = "----------------------------3c3ba8b523b2" 642 eol = "\r\n" 643 644 encoded_input = "" 645 for name, value in input.items(): 646 encoded_input += f"--{boundary}{eol}" 647 encoded_input += f'Content-Disposition: form-data; name="{name}"{eol}' 648 encoded_input += eol 649 encoded_input += value + eol 650 encoded_input += f"--{boundary}--{eol}" 651 return f"multipart/form-data; boundary={boundary}", encoded_input 652 653 return self.__requestEncode(cnx, verb, url, parameters, headers, input, encode) 654 655 def requestBlob( 656 self, 657 verb: str, 658 url: str, 659 parameters: Optional[Dict[str, str]] = None, 660 headers: Optional[Dict[str, str]] = None, 661 input: Optional[str] = None, 662 cnx: Optional[Union[HTTPRequestsConnectionClass, HTTPSRequestsConnectionClass]] = None, 663 ) -> Tuple[int, Dict[str, Any], str]: 664 if headers is None: 665 headers = {} 666 667 def encode(local_path: str) -> Tuple[str, Any]: 668 if "Content-Type" in headers: # type: ignore 669 mime_type = headers["Content-Type"] # type: ignore 670 else: 671 guessed_type = mimetypes.guess_type(local_path) 672 mime_type = guessed_type[0] if guessed_type[0] is not None else Consts.defaultMediaType 673 f = open(local_path, "rb") 674 return mime_type, f 675 676 if input: 677 headers["Content-Length"] = str(os.path.getsize(input)) 678 return self.__requestEncode(cnx, verb, url, parameters, headers, input, encode) 679 680 def requestMemoryBlobAndCheck( 681 self, 682 verb: str, 683 url: str, 684 parameters: Any, 685 headers: Dict[str, Any], 686 file_like: BinaryIO, 687 cnx: Optional[Union[HTTPRequestsConnectionClass, HTTPSRequestsConnectionClass]] = None, 688 ) -> Tuple[Dict[str, Any], Any]: 689 # The expected signature of encode means that the argument is ignored. 690 def encode(_: Any) -> Tuple[str, Any]: 691 return headers["Content-Type"], file_like 692 693 if not cnx: 694 cnx = self.__customConnection(url) 695 return self.__check(*self.__requestEncode(cnx, verb, url, parameters, headers, file_like, encode)) 696 697 def __requestEncode( 698 self, 699 cnx: Optional[Union[HTTPRequestsConnectionClass, HTTPSRequestsConnectionClass]], 700 verb: str, 701 url: str, 702 parameters: Optional[Dict[str, str]], 703 requestHeaders: Optional[Dict[str, str]], 704 input: Optional[T], 705 encode: Callable[[T], Tuple[str, Any]], 706 ) -> Tuple[int, Dict[str, Any], str]: 707 assert verb in ["HEAD", "GET", "POST", "PATCH", "PUT", "DELETE"] 708 if parameters is None: 709 parameters = {} 710 if requestHeaders is None: 711 requestHeaders = {} 712 713 if self.__auth is not None: 714 requestHeaders["Authorization"] = f"{self.__auth.token_type} {self.__auth.token}" 715 requestHeaders["User-Agent"] = self.__userAgent 716 717 url = self.__makeAbsoluteUrl(url) 718 url = self.__addParametersToUrl(url, parameters) 719 720 encoded_input = None 721 if input is not None: 722 requestHeaders["Content-Type"], encoded_input = encode(input) 723 724 self.NEW_DEBUG_FRAME(requestHeaders) 725 726 status, responseHeaders, output = self.__requestRaw(cnx, verb, url, requestHeaders, encoded_input) 727 728 if Consts.headerRateRemaining in responseHeaders and Consts.headerRateLimit in responseHeaders: 729 self.rate_limiting = ( 730 # ints expected but sometimes floats returned: https://github.com/PyGithub/PyGithub/pull/2697 731 int(float(responseHeaders[Consts.headerRateRemaining])), 732 int(float(responseHeaders[Consts.headerRateLimit])), 733 ) 734 if Consts.headerRateReset in responseHeaders: 735 # ints expected but sometimes floats returned: https://github.com/PyGithub/PyGithub/pull/2697 736 self.rate_limiting_resettime = int(float(responseHeaders[Consts.headerRateReset])) 737 738 if Consts.headerOAuthScopes in responseHeaders: 739 self.oauth_scopes = responseHeaders[Consts.headerOAuthScopes].split(", ") 740 741 self.DEBUG_ON_RESPONSE(status, responseHeaders, output) 742 743 return status, responseHeaders, output 744 745 def __requestRaw( 746 self, 747 cnx: Optional[Union[HTTPRequestsConnectionClass, HTTPSRequestsConnectionClass]], 748 verb: str, 749 url: str, 750 requestHeaders: Dict[str, str], 751 input: Optional[Any], 752 ) -> Tuple[int, Dict[str, Any], str]: 753 self.__deferRequest(verb) 754 755 try: 756 original_cnx = cnx 757 if cnx is None: 758 cnx = self.__createConnection() 759 cnx.request(verb, url, input, requestHeaders) 760 response = cnx.getresponse() 761 762 status = response.status 763 responseHeaders = {k.lower(): v for k, v in response.getheaders()} 764 output = response.read() 765 766 if input: 767 if isinstance(input, IOBase): 768 input.close() 769 770 self.__log(verb, url, requestHeaders, input, status, responseHeaders, output) 771 772 if status == 202 and ( 773 verb == "GET" or verb == "HEAD" 774 ): # only for requests that are considered 'safe' in RFC 2616 775 time.sleep(Consts.PROCESSING_202_WAIT_TIME) 776 return self.__requestRaw(original_cnx, verb, url, requestHeaders, input) 777 778 if status == 301 and "location" in responseHeaders: 779 location = responseHeaders["location"] 780 o = urllib.parse.urlparse(location) 781 if o.scheme != self.__scheme: 782 raise RuntimeError( 783 f"Github server redirected from {self.__scheme} protocol to {o.scheme}, " 784 f"please correct your Github server URL via base_url: Github(base_url=...)" 785 ) 786 if o.hostname != self.__hostname: 787 raise RuntimeError( 788 f"Github server redirected from host {self.__hostname} to {o.hostname}, " 789 f"please correct your Github server URL via base_url: Github(base_url=...)" 790 ) 791 if o.path == url: 792 port = ":" + str(self.__port) if self.__port is not None else "" 793 requested_location = f"{self.__scheme}://{self.__hostname}{port}{url}" 794 raise RuntimeError( 795 f"Requested {requested_location} but server redirected to {location}, " 796 f"you may need to correct your Github server URL " 797 f"via base_url: Github(base_url=...)" 798 ) 799 if self._logger.isEnabledFor(logging.INFO): 800 self._logger.info(f"Following Github server redirection from {url} to {o.path}") 801 return self.__requestRaw(original_cnx, verb, o.path, requestHeaders, input) 802 803 return status, responseHeaders, output 804 finally: 805 # we record the time of this request after it finished 806 # to defer next request starting from this request's end, not start 807 self.__recordRequestTime(verb) 808 809 def __deferRequest(self, verb: str) -> None: 810 # Ensures at least self.__seconds_between_requests seconds have passed since any last request 811 # and self.__seconds_between_writes seconds have passed since last write request (if verb refers to a write). 812 # Uses self.__last_requests. 813 requests = self.__last_requests.values() 814 writes = [l for v, l in self.__last_requests.items() if v != "GET"] 815 816 last_request = max(requests) if requests else 0 817 last_write = max(writes) if writes else 0 818 819 next_request = (last_request + self.__seconds_between_requests) if self.__seconds_between_requests else 0 820 next_write = (last_write + self.__seconds_between_writes) if self.__seconds_between_writes else 0 821 822 next = next_request if verb == "GET" else max(next_request, next_write) 823 defer = max(next - datetime.now(timezone.utc).timestamp(), 0) 824 if defer > 0: 825 if self.__logger is None: 826 self.__logger = logging.getLogger(__name__) 827 self.__logger.debug(f"sleeping {defer}s before next GitHub request") 828 time.sleep(defer) 829 830 def __recordRequestTime(self, verb: str) -> None: 831 # Updates self.__last_requests with current timestamp for given verb 832 self.__last_requests[verb] = datetime.now(timezone.utc).timestamp() 833 834 def __makeAbsoluteUrl(self, url: str) -> str: 835 # URLs generated locally will be relative to __base_url 836 # URLs returned from the server will start with __base_url 837 if url.startswith("/"): 838 url = f"{self.__prefix}{url}" 839 else: 840 o = urllib.parse.urlparse(url) 841 assert o.hostname in [ 842 self.__hostname, 843 "uploads.github.com", 844 "status.github.com", 845 "github.com", 846 ], o.hostname 847 assert o.path.startswith((self.__prefix, "/api/")) 848 assert o.port == self.__port 849 url = o.path 850 if o.query != "": 851 url += f"?{o.query}" 852 return url 853 854 def __addParametersToUrl( 855 self, 856 url: str, 857 parameters: Dict[str, Any], 858 ) -> str: 859 if len(parameters) == 0: 860 return url 861 else: 862 return f"{url}?{urllib.parse.urlencode(parameters)}" 863 864 def __createConnection( 865 self, 866 ) -> Union[HTTPRequestsConnectionClass, HTTPSRequestsConnectionClass]: 867 if self.__persist and self.__connection is not None: 868 return self.__connection 869 870 with self.__connection_lock: 871 if self.__connection is not None: 872 if self.__persist: 873 return self.__connection 874 self.__connection.close() 875 self.__connection = self.__connectionClass( 876 self.__hostname, 877 self.__port, 878 retry=self.__retry, 879 pool_size=self.__pool_size, 880 timeout=self.__timeout, 881 verify=self.__verify, 882 ) 883 884 return self.__connection 885 886 @property 887 def _logger(self) -> logging.Logger: 888 if self.__logger is None: 889 self.__logger = logging.getLogger(__name__) 890 return self.__logger 891 892 def __log( 893 self, 894 verb: str, 895 url: str, 896 requestHeaders: Dict[str, str], 897 input: Optional[Any], 898 status: Optional[int], 899 responseHeaders: Dict[str, Any], 900 output: Optional[str], 901 ) -> None: 902 if self._logger.isEnabledFor(logging.DEBUG): 903 headersForRequest = requestHeaders.copy() 904 if "Authorization" in requestHeaders: 905 if requestHeaders["Authorization"].startswith("Basic"): 906 headersForRequest["Authorization"] = "Basic (login and password removed)" 907 elif requestHeaders["Authorization"].startswith("token"): 908 headersForRequest["Authorization"] = "token (oauth token removed)" 909 elif requestHeaders["Authorization"].startswith("Bearer"): 910 headersForRequest["Authorization"] = "Bearer (jwt removed)" 911 else: # pragma no cover (Cannot happen, but could if we add an authentication method => be prepared) 912 headersForRequest[ 913 "Authorization" 914 ] = "(unknown auth removed)" # pragma no cover (Cannot happen, but could if we add an authentication method => be prepared) 915 self._logger.debug( 916 "%s %s://%s%s %s %s ==> %i %s %s", 917 verb, 918 self.__scheme, 919 self.__hostname, 920 url, 921 headersForRequest, 922 input, 923 status, 924 responseHeaders, 925 output, 926 ) 927 928 929 class WithRequester(Generic[T]): 930 """ 931 Mixin class that allows to set a requester. 932 """ 933 934 __requester: Requester 935 936 def __init__(self) -> None: 937 self.__requester: Optional[Requester] = None # type: ignore 938 939 @property 940 def requester(self) -> Requester: 941 return self.__requester 942 943 def withRequester(self, requester: Requester) -> "WithRequester[T]": 944 assert isinstance(requester, Requester), requester 945 self.__requester = requester 946 return self