rpc.php
1 <?php 2 3 $rpc_internal_timeout = 10; 4 $rpc_external_timeout = 4; 5 6 function rpc_start_request_with_options($url, $options) 7 { 8 global $rpc_internal_timeout, $rpc_external_timeout; 9 10 // FIXME: $internal was undefined 11 $internal = false; 12 13 $curlopts = array( 14 CURLOPT_FAILONERROR => true, //...? 15 CURLOPT_FOLLOWLOCATION => true, 16 CURLOPT_RETURNTRANSFER => true, 17 18 CURLOPT_CONNECTTIMEOUT => 2, 19 CURLOPT_TIMEOUT => $internal ? $rpc_internal_timeout : $rpc_external_timeout, 20 CURLOPT_USERAGENT => "4chan.org", 21 ); 22 23 $curlopts = $options + $curlopts; 24 25 global $rpc_mh; 26 global $rpc_chs; 27 28 $ch = curl_init($url); 29 if (!$ch || !is_resource($ch)) { 30 internal_error_log("rpc", "couldn't curl_init '$ch' '$url': '".curl_error($ch)."'"); 31 curl_close($ch); 32 return null; 33 } 34 35 $optstr = print_r($curlopts, TRUE); 36 // quick_log_to( "/www/perhost/curls.log", "curl: '$ch' URL: $url\n$optstr\n"); 37 38 foreach ($curlopts as $opt=>$value) { 39 if (curl_setopt($ch, $opt, $value) === false) { 40 internal_error_log("rpc", "couldn't curl_setopt '$ch' '$opt' '$value': '".curl_error($ch)."'"); 41 curl_close($ch); 42 return null; 43 } 44 } 45 46 if (!isset($rpc_mh)) { 47 rpc_multi_init(); 48 } 49 50 $ret = curl_multi_add_handle($rpc_mh, $ch); 51 52 if ($ret != 0) { 53 internal_error_log("rpc", "couldn't add curl handle $ch $ret: ".curl_error($ch)); 54 return null; 55 } 56 57 $rpc_chs[] = $ch; 58 59 return $ch; 60 } 61 62 // returns a request ID, or null if it failed 63 function rpc_start_request($url, $post, $cookies, $internal) 64 { 65 global $rpc_internal_timeout, $rpc_external_timeout; 66 67 $cookiestr = ''; 68 if ($cookies) { 69 $carray = array(); 70 foreach($cookies as $name=>$value) { 71 $name = urlencode($name); 72 $value = urlencode($value); 73 $carray[] = "$name=$value"; 74 } 75 $cookiestr = implode("; ", $carray).";"; 76 } 77 78 if ($internal) { 79 $curlopts[CURLOPT_SSL_VERIFYHOST] = false; 80 $curlopts[CURLOPT_SSL_VERIFYPEER] = false; 81 } 82 else { 83 $curlopts[CURLINFO_HEADER_OUT] = true; 84 // $curlopts[CURLOPT_VERBOSE] = true; 85 } 86 87 if ($post) { 88 $curlopts[CURLOPT_POSTFIELDS] = $post; 89 } 90 91 if ($cookiestr) { 92 $curlopts[CURLOPT_COOKIE] = $cookiestr; 93 } 94 95 return rpc_start_request_with_options($url, $curlopts); 96 } 97 98 function rpc_start_captcha_request($url, $post, $cookies, $internal) { 99 global $rpc_internal_timeout, $rpc_external_timeout; 100 101 $cookiestr = ''; 102 if ($cookies) { 103 $carray = array(); 104 foreach($cookies as $name=>$value) { 105 $name = urlencode($name); 106 $value = urlencode($value); 107 $carray[] = "$name=$value"; 108 } 109 $cookiestr = implode("; ", $carray).";"; 110 } 111 112 if ($internal) { 113 $curlopts[CURLOPT_SSL_VERIFYHOST] = false; 114 $curlopts[CURLOPT_SSL_VERIFYPEER] = false; 115 } 116 else { 117 $curlopts[CURLINFO_HEADER_OUT] = true; 118 //$curlopts[CURLOPT_RESOLVE] = array('www.google.com:443:172.217.4.132'); 119 // $curlopts[CURLOPT_VERBOSE] = true; 120 } 121 122 if ($post) { 123 $curlopts[CURLOPT_POSTFIELDS] = $post; 124 } 125 126 if ($cookiestr) { 127 $curlopts[CURLOPT_COOKIE] = $cookiestr; 128 } 129 130 return rpc_start_request_with_options($url, $curlopts); 131 } 132 133 function rpc_new_multi_handle() 134 { 135 $mh = curl_multi_init(); 136 137 curl_multi_setopt($mh, CURLMOPT_PIPELINING, 1); 138 curl_multi_setopt($mh, CURLMOPT_MAXCONNECTS, 16); 139 140 return $mh; 141 } 142 143 function rpc_multi_init() 144 { 145 global $rpc_mh; 146 global $rpc_chs; 147 148 $rpc_mh = rpc_new_multi_handle(); 149 $rpc_chs = array(); 150 151 register_shutdown_function('rpc_finish_all'); 152 } 153 154 // call at idle points, calls curl's task until it stops having immediate work 155 function rpc_task() 156 { 157 global $rpc_mh; 158 159 if (!is_resource($rpc_mh)) return false; 160 161 $still_running = false; 162 163 do { 164 $ret = curl_multi_exec($rpc_mh, $still_running); 165 } while ($ret == CURLM_CALL_MULTI_PERFORM); 166 167 return $still_running; 168 } 169 170 // blocks till all requests are no longer 'running' and clears rpc_mh 171 // this can block for a few seconds, watch out! 172 function rpc_finish_all() 173 { 174 global $rpc_mh; 175 global $rpc_chs; 176 177 if (!is_resource($rpc_mh) || !count($rpc_chs)) return; 178 179 flush_output_buffers(); 180 181 do { 182 if (rpc_task() == false) break; 183 curl_multi_select($rpc_mh); 184 } while (true); 185 186 // clear out the curl_multi handle 187 foreach ($rpc_chs as $ch) { 188 curl_multi_remove_handle($rpc_mh, $ch); 189 } 190 191 //quick_log_to("/www/perhost/rpc.log", getmypid()." $n rpcs finished in $rpc_mh\n"); 192 193 //deallocate all curl handles 194 $rpc_chs = array(); 195 196 //hopefully rpc_mh is empty now. 197 //we don't want to close it because curl uses the state for http pipelining 198 } 199 200 function rpc_close_multi() 201 { 202 global $rpc_mh; 203 global $rpc_chs; 204 205 // explicitly close these objects since curl debug seems to not print otherwise? 206 // don't wait for them to finish. this can be used to prevent double-submits. maybe... 207 unset($rpc_chs); 208 unset($rpc_mh); 209 } 210 211 function rpc_debug_fd() 212 { 213 static $fd = -1; 214 215 if ($fd == -1) { 216 $fd = fopen( "/www/perhost/curl-debug.log", "a" ); 217 fwrite($fd, "--------------\n"); 218 } 219 220 return $fd; 221 } 222 223 function rpc_debug_request($ch) 224 { 225 $errno = curl_errno($ch); 226 $error = curl_error($ch); 227 $sent = curl_getinfo($ch, CURLINFO_HEADER_OUT); 228 $bsent = curl_getinfo($ch, CURLINFO_SIZE_UPLOAD); 229 $brec = curl_getinfo($ch, CURLINFO_SIZE_DOWNLOAD); 230 231 quick_log_to("/www/perhost/rpc-failures.log", getmypid()." curl error $errno '$error'\nbytes up $bsent down $brec\ndata: $sent"); 232 } 233 234 // returns the response, or sets $error if null 235 // DANGER: if you call this on a curl after rpc_finish_all() it seems to send the POST over again 236 function rpc_finish_request($ch, &$error, &$httperror = null) 237 { 238 rpc_task(); 239 240 // Move the request into the foreground and block (hopefully not actually blocking) 241 global $rpc_mh; 242 global $rpc_chs; 243 244 if ($rpc_mh===null || !is_resource($ch)) { 245 $error = "Connections not started ($rpc_mh $ch)"; 246 return null; 247 } 248 249 curl_multi_remove_handle($rpc_mh, $ch); 250 $ret = curl_exec($ch); 251 252 // Get contents 253 if ($ret === false) { 254 $errstr = curl_error($ch); 255 $errno = curl_errno($ch); 256 $error = "Curl error: $errstr ($errno)"; 257 if ($httperror !== null) { 258 $httperror = curl_getinfo($ch, CURLINFO_HTTP_CODE); 259 } 260 rpc_debug_request($ch); 261 $ret = null; 262 } 263 264 curl_close($ch); 265 266 if (($pos = array_search($ch, $rpc_chs, true)) !== FALSE) { 267 unset($rpc_chs[$pos]); 268 } 269 270 return $ret; 271 } 272 273 // some dumb shit for sending HTTP POST to another server. 274 // only use this function internally 275 function rpc_send_request($host, $url, $request, &$error, $internal=true) { 276 $port = 80; 277 $proto = 'tcp://'; 278 $internal_network = preg_match( '#\.int$#', $host ) || strpos( $host, '10.0' ) === 0; 279 280 if(strpos($host, "4chan.org") !== false || $internal_network ) { 281 if( strpos( $url, 'imgboard.php' ) !== false || strpos( $url, 'admin.php' ) !== false || strpos( $host, 'www.' ) !== false || $internal_network ) { 282 $proto = 'ssl://'; 283 $port = 443; 284 } 285 } 286 287 $timeout = $internal_network ? 60 : 4; 288 289 $cookie = ''; 290 foreach($request['COOKIE'] as $name=>$value) { 291 $name = urlencode($name); 292 $value = urlencode($value); 293 $cookie .= "$name=$value;"; 294 } 295 296 $postbody = ''; 297 foreach($request['POST'] as $name=>$value) { 298 $name = urlencode($name); 299 $value = urlencode($value); 300 $postbody .= "$name=$value&"; 301 } 302 303 // POSTing with HTTP 1.1 tends to make responders send 304 // back Transfer-encoding: chunked, so use 1.0 305 306 $header = "POST $url HTTP/1.0\r\n"; 307 $header .= "Host: $host\r\n"; 308 $header .= "Content-Type: application/x-www-form-urlencoded\r\n"; 309 $header .= "Content-Length: ". strlen($postbody) . "\r\n"; 310 $header .= "User-Agent: 4chan.org\r\n"; 311 if ($cookie && $internal) $header .= "Cookie: $cookie\r\n"; 312 $header .= "Connection: close\r\n"; 313 $header .= "\r\n"; 314 315 $header .= "$postbody\r\n"; 316 317 $rpc_start_time = microtime(true); 318 $socket = fsockopen($proto.$host, $port, $errno, $errstr, $timeout); 319 if(!$socket) { 320 $error = $errstr; return; 321 } 322 323 if(fwrite($socket, $header) != strlen($header)) { 324 $error = 'Could not write to socket'; return; 325 } 326 327 $rpc_connect_time = microtime(true); 328 $rpc_connect_took = $rpc_connect_time - $rpc_start_time; 329 stream_set_timeout($socket, $timeout - $rpc_connect_took); 330 331 $response = ''; 332 do { 333 $response .= fgets($socket, 1160); 334 $info = stream_get_meta_data($socket); 335 } while(!feof($socket) && !$info['timed_out']); 336 337 fclose($socket); 338 if(!preg_match('!^HTTP/1\.. 200 OK!', $response)) { 339 $lines = explode("\n", $response); 340 $error = 'Error response from server ('.strlen($response).' bytes): '. $lines[0]; 341 $response = null; 342 } 343 344 // $rpc_end_time = microtime(true); 345 // $rpc_took = $rpc_end_time - $rpc_start_time; 346 347 /* 348 if ($error) { 349 quick_log_to("/www/perhost/rpc-slow.log", "ERROR: $host$url ct $rpc_connect_took took $rpc_took error: ".implode("\n",$lines)."\n".$postbody); 350 } else 351 if ($rpc_took > 1) { 352 quick_log_to("/www/perhost/rpc-slow.log", "SLOW: $host$url ct $rpc_connect_took took $rpc_took errored ".($response?0:1)."\n".$postbody); 353 } 354 */ 355 356 return $response; 357 } 358 359 // a shortcut to create the request object (with cookie set and POST initialized) 360 function rpc_blank_request() { 361 return array('COOKIE' => $_COOKIE, 'POST' => array() ); 362 } 363 364 function rpc_log_url($s,$r) { 365 $s = nl2br($s); 366 $r = nl2br($r); 367 $rh = fopen("/www/perhost/rpc.log", "a"); 368 flock($rh, LOCK_EX); 369 fwrite($rh, "$s --> $r\n"); 370 fclose($rh); 371 } 372 373 // TODO: This isn't really used since we just block link shorteners instead. 374 // But it should be optimized into curl_multi if possible. 375 function rpc_find_real_url($short) { 376 $cu = curl_init($short); 377 curl_setopt($cu, CURLOPT_FOLLOWLOCATION, true); 378 curl_setopt($cu, CURLOPT_MAXREDIRS, 4); 379 curl_setopt($cu, CURLOPT_NOBODY, true); 380 curl_setopt($cu, CURLOPT_TIMEOUT, 10); 381 curl_setopt($cu, CURLOPT_USERAGENT, "4chan.org"); 382 383 if (curl_exec($cu)) { 384 $ret = curl_getinfo($cu, CURLINFO_EFFECTIVE_URL); 385 } else $ret = FALSE; 386 387 curl_close($cu); 388 //rpc_log_url($short,$ret); 389 return $ret; 390 }