subprocess.h
1 // Based on the https://github.com/arun11299/cpp-subprocess project. 2 3 /*! 4 5 Documentation for C++ subprocessing library. 6 7 @copyright The code is licensed under the [MIT 8 License](http://opensource.org/licenses/MIT): 9 <br> 10 Copyright © 2016-2018 Arun Muralidharan. 11 <br> 12 Permission is hereby granted, free of charge, to any person obtaining a copy 13 of this software and associated documentation files (the "Software"), to deal 14 in the Software without restriction, including without limitation the rights 15 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 16 copies of the Software, and to permit persons to whom the Software is 17 furnished to do so, subject to the following conditions: 18 <br> 19 The above copyright notice and this permission notice shall be included in 20 all copies or substantial portions of the Software. 21 <br> 22 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 28 SOFTWARE. 29 30 @author [Arun Muralidharan] 31 @see https://github.com/arun11299/cpp-subprocess to download the source code 32 33 @version 1.0.0 34 */ 35 36 #ifndef BITCOIN_UTIL_SUBPROCESS_H 37 #define BITCOIN_UTIL_SUBPROCESS_H 38 39 #include <util/syserror.h> 40 41 #include <algorithm> 42 #include <cassert> 43 #include <csignal> 44 #include <cstdio> 45 #include <cstdlib> 46 #include <cstring> 47 #include <exception> 48 #include <future> 49 #include <initializer_list> 50 #include <iostream> 51 #include <locale> 52 #include <map> 53 #include <memory> 54 #include <sstream> 55 #include <string> 56 #include <vector> 57 58 #ifdef WIN32 59 #include <codecvt> 60 #endif 61 62 extern "C" { 63 #ifdef WIN32 64 #include <windows.h> 65 #include <io.h> 66 #include <cwchar> 67 #else 68 #include <sys/wait.h> 69 #include <unistd.h> 70 #endif 71 #include <csignal> 72 #include <fcntl.h> 73 #include <sys/types.h> 74 } 75 76 // The Microsoft C++ compiler issues deprecation warnings 77 // for the standard POSIX function names. 78 // Its preferred implementations have a leading underscore. 79 // See: https://learn.microsoft.com/en-us/cpp/c-runtime-library/compatibility. 80 #if (defined _MSC_VER) 81 #define subprocess_close _close 82 #define subprocess_fileno _fileno 83 #define subprocess_open _open 84 #define subprocess_write _write 85 #else 86 #define subprocess_close close 87 #define subprocess_fileno fileno 88 #define subprocess_open open 89 #define subprocess_write write 90 #endif 91 92 /*! 93 * Getting started with reading this source code. 94 * The source is mainly divided into four parts: 95 * 1. Exception Classes: 96 * These are very basic exception classes derived from 97 * runtime_error exception. 98 * There are two types of exception thrown from subprocess 99 * library: OSError and CalledProcessError 100 * 101 * 2. Popen Class 102 * This is the main class the users will deal with. It 103 * provides with all the API's to deal with processes. 104 * 105 * 3. Util namespace 106 * It includes some helper functions to split/join a string, 107 * reading from file descriptors, waiting on a process, fcntl 108 * options on file descriptors etc. 109 * 110 * 4. Detail namespace 111 * This includes some metaprogramming and helper classes. 112 */ 113 114 115 namespace subprocess { 116 117 // Max buffer size allocated on stack for read error 118 // from pipe 119 static const size_t SP_MAX_ERR_BUF_SIZ = 1024; 120 121 // Default buffer capacity for OutBuffer and ErrBuffer. 122 // If the data exceeds this capacity, the buffer size is grown 123 // by 1.5 times its previous capacity 124 static const size_t DEFAULT_BUF_CAP_BYTES = 8192; 125 126 127 /*----------------------------------------------- 128 * EXCEPTION CLASSES 129 *----------------------------------------------- 130 */ 131 132 /*! 133 * class: CalledProcessError 134 * Thrown when there was error executing the command. 135 * Check Popen class API's to know when this exception 136 * can be thrown. 137 * 138 */ 139 class CalledProcessError: public std::runtime_error 140 { 141 public: 142 int retcode; 143 CalledProcessError(const std::string& error_msg, int retcode): 144 std::runtime_error(error_msg), retcode(retcode) 145 {} 146 }; 147 148 149 /*! 150 * class: OSError 151 * Thrown when some system call fails to execute or give result. 152 * The exception message contains the name of the failed system call 153 * with the stringisized errno code. 154 * Check Popen class API's to know when this exception would be 155 * thrown. 156 * Its usual that the API exception specification would have 157 * this exception together with CalledProcessError. 158 */ 159 class OSError: public std::runtime_error 160 { 161 public: 162 OSError(const std::string& err_msg, int err_code): 163 std::runtime_error(err_msg + ": " + SysErrorString(err_code)) 164 {} 165 }; 166 167 //-------------------------------------------------------------------- 168 namespace util 169 { 170 #ifdef WIN32 171 inline void quote_argument(const std::wstring &argument, std::wstring &command_line, 172 bool force) 173 { 174 // 175 // Unless we're told otherwise, don't quote unless we actually 176 // need to do so --- hopefully avoid problems if programs won't 177 // parse quotes properly 178 // 179 180 if (force == false && argument.empty() == false && 181 argument.find_first_of(L" \t\n\v") == argument.npos) { 182 command_line.append(argument); 183 } 184 else { 185 command_line.push_back(L'"'); 186 187 for (auto it = argument.begin();; ++it) { 188 unsigned number_backslashes = 0; 189 190 while (it != argument.end() && *it == L'\\') { 191 ++it; 192 ++number_backslashes; 193 } 194 195 if (it == argument.end()) { 196 197 // 198 // Escape all backslashes, but let the terminating 199 // double quotation mark we add below be interpreted 200 // as a metacharacter. 201 // 202 203 command_line.append(number_backslashes * 2, L'\\'); 204 break; 205 } 206 else if (*it == L'"') { 207 208 // 209 // Escape all backslashes and the following 210 // double quotation mark. 211 // 212 213 command_line.append(number_backslashes * 2 + 1, L'\\'); 214 command_line.push_back(*it); 215 } 216 else { 217 218 // 219 // Backslashes aren't special here. 220 // 221 222 command_line.append(number_backslashes, L'\\'); 223 command_line.push_back(*it); 224 } 225 } 226 227 command_line.push_back(L'"'); 228 } 229 } 230 231 inline std::string get_last_error(DWORD errorMessageID) 232 { 233 if (errorMessageID == 0) 234 return std::string(); 235 236 LPSTR messageBuffer = nullptr; 237 size_t size = FormatMessageA( 238 FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | 239 FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_MAX_WIDTH_MASK, 240 NULL, errorMessageID, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), 241 (LPSTR)&messageBuffer, 0, NULL); 242 243 std::string message(messageBuffer, size); 244 245 LocalFree(messageBuffer); 246 247 return message; 248 } 249 250 inline FILE *file_from_handle(HANDLE h, const char *mode) 251 { 252 int md; 253 if (!mode) { 254 throw OSError("invalid_mode", 0); 255 } 256 257 if (mode[0] == 'w') { 258 md = _O_WRONLY; 259 } 260 else if (mode[0] == 'r') { 261 md = _O_RDONLY; 262 } 263 else { 264 throw OSError("file_from_handle", 0); 265 } 266 267 int os_fhandle = _open_osfhandle((intptr_t)h, md); 268 if (os_fhandle == -1) { 269 CloseHandle(h); 270 throw OSError("_open_osfhandle", 0); 271 } 272 273 FILE *fp = _fdopen(os_fhandle, mode); 274 if (fp == 0) { 275 subprocess_close(os_fhandle); 276 throw OSError("_fdopen", 0); 277 } 278 279 return fp; 280 } 281 282 inline void configure_pipe(HANDLE* read_handle, HANDLE* write_handle, HANDLE* child_handle) 283 { 284 SECURITY_ATTRIBUTES saAttr; 285 286 // Set the bInheritHandle flag so pipe handles are inherited. 287 saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); 288 saAttr.bInheritHandle = TRUE; 289 saAttr.lpSecurityDescriptor = NULL; 290 291 // Create a pipe for the child process's STDIN. 292 if (!CreatePipe(read_handle, write_handle, &saAttr,0)) 293 throw OSError("CreatePipe", 0); 294 295 // Ensure the write handle to the pipe for STDIN is not inherited. 296 if (!SetHandleInformation(*child_handle, HANDLE_FLAG_INHERIT, 0)) 297 throw OSError("SetHandleInformation", 0); 298 } 299 #endif 300 301 /*! 302 * Function: split 303 * Parameters: 304 * [in] str : Input string which needs to be split based upon the 305 * delimiters provided. 306 * [in] deleims : Delimiter characters based upon which the string needs 307 * to be split. Default constructed to ' '(space) and '\t'(tab) 308 * [out] vector<string> : Vector of strings split at deleimiter. 309 */ 310 static inline std::vector<std::string> 311 split(const std::string& str, const std::string& delims=" \t") 312 { 313 std::vector<std::string> res; 314 size_t init = 0; 315 316 while (true) { 317 auto pos = str.find_first_of(delims, init); 318 if (pos == std::string::npos) { 319 res.emplace_back(str.substr(init, str.length())); 320 break; 321 } 322 res.emplace_back(str.substr(init, pos - init)); 323 pos++; 324 init = pos; 325 } 326 327 return res; 328 } 329 330 331 #ifndef WIN32 332 /*! 333 * Function: set_clo_on_exec 334 * Sets/Resets the FD_CLOEXEC flag on the provided file descriptor 335 * based upon the `set` parameter. 336 * Parameters: 337 * [in] fd : The descriptor on which FD_CLOEXEC needs to be set/reset. 338 * [in] set : If 'true', set FD_CLOEXEC. 339 * If 'false' unset FD_CLOEXEC. 340 */ 341 static inline 342 void set_clo_on_exec(int fd, bool set = true) 343 { 344 int flags = fcntl(fd, F_GETFD, 0); 345 if (flags == -1) { 346 throw OSError("fcntl F_GETFD failed", errno); 347 } 348 if (set) flags |= FD_CLOEXEC; 349 else flags &= ~FD_CLOEXEC; 350 if (fcntl(fd, F_SETFD, flags) == -1) { 351 throw OSError("fcntl F_SETFD failed", errno); 352 } 353 } 354 355 356 /*! 357 * Function: pipe_cloexec 358 * Creates a pipe and sets FD_CLOEXEC flag on both 359 * read and write descriptors of the pipe. 360 * Parameters: 361 * [out] : A pair of file descriptors. 362 * First element of pair is the read descriptor of pipe. 363 * Second element is the write descriptor of pipe. 364 */ 365 static inline 366 std::pair<int, int> pipe_cloexec() noexcept(false) 367 { 368 int pipe_fds[2]; 369 int res = pipe(pipe_fds); 370 if (res) { 371 throw OSError("pipe failure", errno); 372 } 373 374 set_clo_on_exec(pipe_fds[0]); 375 set_clo_on_exec(pipe_fds[1]); 376 377 return std::make_pair(pipe_fds[0], pipe_fds[1]); 378 } 379 #endif 380 381 382 /*! 383 * Function: write_n 384 * Writes `length` bytes to the file descriptor `fd` 385 * from the buffer `buf`. 386 * Parameters: 387 * [in] fd : The file descriptotr to write to. 388 * [in] buf: Buffer from which data needs to be written to fd. 389 * [in] length: The number of bytes that needs to be written from 390 * `buf` to `fd`. 391 * [out] int : Number of bytes written or -1 in case of failure. 392 */ 393 static inline 394 int write_n(int fd, const char* buf, size_t length) 395 { 396 size_t nwritten = 0; 397 while (nwritten < length) { 398 int written = subprocess_write(fd, buf + nwritten, length - nwritten); 399 if (written == -1) return -1; 400 nwritten += written; 401 } 402 return nwritten; 403 } 404 405 406 /*! 407 * Function: read_atmost_n 408 * Reads at the most `read_upto` bytes from the 409 * file object `fp` before returning. 410 * Parameters: 411 * [in] fp : The file object from which it needs to read. 412 * [in] buf : The buffer into which it needs to write the data. 413 * [in] read_upto: Max number of bytes which must be read from `fd`. 414 * [out] int : Number of bytes written to `buf` or read from `fd` 415 * OR -1 in case of error. 416 * NOTE: In case of EINTR while reading from socket, this API 417 * will retry to read from `fd`, but only till the EINTR counter 418 * reaches 50 after which it will return with whatever data it read. 419 */ 420 static inline 421 int read_atmost_n(FILE* fp, char* buf, size_t read_upto) 422 { 423 #ifdef WIN32 424 return (int)fread(buf, 1, read_upto, fp); 425 #else 426 int fd = subprocess_fileno(fp); 427 int rbytes = 0; 428 int eintr_cnter = 0; 429 430 while (1) { 431 int read_bytes = read(fd, buf + rbytes, read_upto - rbytes); 432 if (read_bytes == -1) { 433 if (errno == EINTR) { 434 if (eintr_cnter >= 50) return -1; 435 eintr_cnter++; 436 continue; 437 } 438 return -1; 439 } 440 if (read_bytes == 0) return rbytes; 441 442 rbytes += read_bytes; 443 } 444 return rbytes; 445 #endif 446 } 447 448 449 /*! 450 * Function: read_all 451 * Reads all the available data from `fp` into 452 * `buf`. Internally calls read_atmost_n. 453 * Parameters: 454 * [in] fp : The file object from which to read from. 455 * [in] buf : The buffer of type `class Buffer` into which 456 * the read data is written to. 457 * [out] int: Number of bytes read OR -1 in case of failure. 458 * 459 * NOTE: `class Buffer` is a exposed public class. See below. 460 */ 461 462 static inline int read_all(FILE* fp, std::vector<char>& buf) 463 { 464 auto buffer = buf.data(); 465 int total_bytes_read = 0; 466 int fill_sz = buf.size(); 467 468 while (1) { 469 const int rd_bytes = read_atmost_n(fp, buffer, fill_sz); 470 471 if (rd_bytes == -1) { // Read finished 472 if (total_bytes_read == 0) return -1; 473 break; 474 475 } else if (rd_bytes == fill_sz) { // Buffer full 476 const auto orig_sz = buf.size(); 477 const auto new_sz = orig_sz * 2; 478 buf.resize(new_sz); 479 fill_sz = new_sz - orig_sz; 480 481 //update the buffer pointer 482 buffer = buf.data(); 483 total_bytes_read += rd_bytes; 484 buffer += total_bytes_read; 485 486 } else { // Partial data ? Continue reading 487 total_bytes_read += rd_bytes; 488 fill_sz -= rd_bytes; 489 break; 490 } 491 } 492 buf.erase(buf.begin()+total_bytes_read, buf.end()); // remove extra nulls 493 return total_bytes_read; 494 } 495 496 #ifndef WIN32 497 /*! 498 * Function: wait_for_child_exit 499 * Waits for the process with pid `pid` to exit 500 * and returns its status. 501 * Parameters: 502 * [in] pid : The pid of the process. 503 * [out] pair<int, int>: 504 * pair.first : Return code of the waitpid call. 505 * pair.second : Exit status of the process. 506 * 507 * NOTE: This is a blocking call as in, it will loop 508 * till the child is exited. 509 */ 510 static inline 511 std::pair<int, int> wait_for_child_exit(int pid) 512 { 513 int status = 0; 514 int ret = -1; 515 while (1) { 516 ret = waitpid(pid, &status, 0); 517 if (ret == -1) break; 518 if (ret == 0) continue; 519 return std::make_pair(ret, status); 520 } 521 522 return std::make_pair(ret, status); 523 } 524 #endif 525 526 } // end namespace util 527 528 529 530 /* ------------------------------- 531 * Popen Arguments 532 * ------------------------------- 533 */ 534 535 /*! 536 * Base class for all arguments involving string value. 537 */ 538 struct string_arg 539 { 540 string_arg(const char* arg): arg_value(arg) {} 541 string_arg(std::string&& arg): arg_value(std::move(arg)) {} 542 string_arg(const std::string& arg): arg_value(arg) {} 543 std::string arg_value; 544 }; 545 546 /*! 547 * Option to specify the executable name separately 548 * from the args sequence. 549 * In this case the cmd args must only contain the 550 * options required for this executable. 551 * 552 * Eg: executable{"ls"} 553 */ 554 struct executable: string_arg 555 { 556 template <typename T> 557 executable(T&& arg): string_arg(std::forward<T>(arg)) {} 558 }; 559 560 /*! 561 * Used for redirecting input/output/error 562 */ 563 enum IOTYPE { 564 STDOUT = 1, 565 STDERR, 566 PIPE, 567 }; 568 569 //TODO: A common base/interface for below stream structures ?? 570 571 /*! 572 * Option to specify the input channel for the child 573 * process. It can be: 574 * 1. An already open file descriptor. 575 * 2. A file name. 576 * 3. IOTYPE. Usual a PIPE 577 * 578 * Eg: input{PIPE} 579 * OR in case of redirection, output of another Popen 580 * input{popen.output()} 581 */ 582 struct input 583 { 584 // For an already existing file descriptor. 585 explicit input(int fd): rd_ch_(fd) {} 586 587 // FILE pointer. 588 explicit input (FILE* fp):input(subprocess_fileno(fp)) { assert(fp); } 589 590 explicit input(const char* filename) { 591 int fd = subprocess_open(filename, O_RDONLY); 592 if (fd == -1) throw OSError("File not found: ", errno); 593 rd_ch_ = fd; 594 } 595 explicit input(IOTYPE typ) { 596 assert (typ == PIPE && "STDOUT/STDERR not allowed"); 597 #ifndef WIN32 598 std::tie(rd_ch_, wr_ch_) = util::pipe_cloexec(); 599 #endif 600 } 601 602 int rd_ch_ = -1; 603 int wr_ch_ = -1; 604 }; 605 606 607 /*! 608 * Option to specify the output channel for the child 609 * process. It can be: 610 * 1. An already open file descriptor. 611 * 2. A file name. 612 * 3. IOTYPE. Usually a PIPE. 613 * 614 * Eg: output{PIPE} 615 * OR output{"output.txt"} 616 */ 617 struct output 618 { 619 explicit output(int fd): wr_ch_(fd) {} 620 621 explicit output (FILE* fp):output(subprocess_fileno(fp)) { assert(fp); } 622 623 explicit output(const char* filename) { 624 int fd = subprocess_open(filename, O_APPEND | O_CREAT | O_RDWR, 0640); 625 if (fd == -1) throw OSError("File not found: ", errno); 626 wr_ch_ = fd; 627 } 628 explicit output(IOTYPE typ) { 629 assert (typ == PIPE && "STDOUT/STDERR not allowed"); 630 #ifndef WIN32 631 std::tie(rd_ch_, wr_ch_) = util::pipe_cloexec(); 632 #endif 633 } 634 635 int rd_ch_ = -1; 636 int wr_ch_ = -1; 637 }; 638 639 640 /*! 641 * Option to specify the error channel for the child 642 * process. It can be: 643 * 1. An already open file descriptor. 644 * 2. A file name. 645 * 3. IOTYPE. Usually a PIPE or STDOUT 646 * 647 */ 648 struct error 649 { 650 explicit error(int fd): wr_ch_(fd) {} 651 652 explicit error(FILE* fp):error(subprocess_fileno(fp)) { assert(fp); } 653 654 explicit error(const char* filename) { 655 int fd = subprocess_open(filename, O_APPEND | O_CREAT | O_RDWR, 0640); 656 if (fd == -1) throw OSError("File not found: ", errno); 657 wr_ch_ = fd; 658 } 659 explicit error(IOTYPE typ) { 660 assert ((typ == PIPE || typ == STDOUT) && "STDERR not allowed"); 661 if (typ == PIPE) { 662 #ifndef WIN32 663 std::tie(rd_ch_, wr_ch_) = util::pipe_cloexec(); 664 #endif 665 } else { 666 // Need to defer it till we have checked all arguments 667 deferred_ = true; 668 } 669 } 670 671 bool deferred_ = false; 672 int rd_ch_ = -1; 673 int wr_ch_ = -1; 674 }; 675 676 // ~~~~ End Popen Args ~~~~ 677 678 679 /*! 680 * class: Buffer 681 * This class is a very thin wrapper around std::vector<char> 682 * This is basically used to determine the length of the actual 683 * data stored inside the dynamically resized vector. 684 * 685 * This is what is returned as the output to the communicate 686 * function, so, users must know about this class. 687 * 688 * OutBuffer and ErrBuffer are just different typedefs to this class. 689 */ 690 class Buffer 691 { 692 public: 693 Buffer() = default; 694 explicit Buffer(size_t cap) { buf.resize(cap); } 695 void add_cap(size_t cap) { buf.resize(cap); } 696 697 public: 698 std::vector<char> buf; 699 size_t length = 0; 700 }; 701 702 // Buffer for storing output written to output fd 703 using OutBuffer = Buffer; 704 // Buffer for storing output written to error fd 705 using ErrBuffer = Buffer; 706 707 708 // Fwd Decl. 709 class Popen; 710 711 /*--------------------------------------------------- 712 * DETAIL NAMESPACE 713 *--------------------------------------------------- 714 */ 715 716 namespace detail { 717 /*! 718 * A helper class to Popen class for setting 719 * options as provided in the Popen constructor. 720 * This design allows us to _not_ have any fixed position 721 * to any arguments and specify them in a way similar to what 722 * can be done in python. 723 */ 724 struct ArgumentDeducer 725 { 726 ArgumentDeducer(Popen* p): popen_(p) {} 727 728 void set_option(executable&& exe); 729 void set_option(input&& inp); 730 void set_option(output&& out); 731 void set_option(error&& err); 732 733 private: 734 Popen* popen_ = nullptr; 735 }; 736 737 #ifndef WIN32 738 /*! 739 * A helper class to Popen. 740 * This takes care of all the fork-exec logic 741 * in the execute_child API. 742 */ 743 class Child 744 { 745 public: 746 Child(Popen* p, int err_wr_pipe): 747 parent_(p), 748 err_wr_pipe_(err_wr_pipe) 749 {} 750 751 void execute_child(); 752 753 private: 754 // Lets call it parent even though 755 // technically a bit incorrect 756 Popen* parent_ = nullptr; 757 int err_wr_pipe_ = -1; 758 }; 759 #endif 760 761 // Fwd Decl. 762 class Streams; 763 764 /*! 765 * A helper class to Streams. 766 * This takes care of management of communicating 767 * with the child process with the means of the correct 768 * file descriptor. 769 */ 770 class Communication 771 { 772 public: 773 Communication(Streams* stream): stream_(stream) 774 {} 775 Communication(const Communication&) = delete; 776 Communication& operator=(const Communication&) = delete; 777 Communication(Communication&&) = default; 778 Communication& operator=(Communication&&) = default; 779 public: 780 int send(const char* msg, size_t length); 781 int send(const std::vector<char>& msg); 782 783 std::pair<OutBuffer, ErrBuffer> communicate(const char* msg, size_t length); 784 std::pair<OutBuffer, ErrBuffer> communicate(const std::vector<char>& msg) 785 { return communicate(msg.data(), msg.size()); } 786 787 void set_out_buf_cap(size_t cap) { out_buf_cap_ = cap; } 788 void set_err_buf_cap(size_t cap) { err_buf_cap_ = cap; } 789 790 private: 791 std::pair<OutBuffer, ErrBuffer> communicate_threaded( 792 const char* msg, size_t length); 793 794 private: 795 Streams* stream_; 796 size_t out_buf_cap_ = DEFAULT_BUF_CAP_BYTES; 797 size_t err_buf_cap_ = DEFAULT_BUF_CAP_BYTES; 798 }; 799 800 801 802 /*! 803 * This is a helper class to Popen. 804 * It takes care of management of all the file descriptors 805 * and file pointers. 806 * It dispatches of the communication aspects to the 807 * Communication class. 808 * Read through the data members to understand about the 809 * various file descriptors used. 810 */ 811 class Streams 812 { 813 public: 814 Streams():comm_(this) {} 815 Streams(const Streams&) = delete; 816 Streams& operator=(const Streams&) = delete; 817 Streams(Streams&&) = default; 818 Streams& operator=(Streams&&) = default; 819 820 public: 821 void setup_comm_channels(); 822 823 void cleanup_fds() 824 { 825 if (write_to_child_ != -1 && read_from_parent_ != -1) { 826 subprocess_close(write_to_child_); 827 } 828 if (write_to_parent_ != -1 && read_from_child_ != -1) { 829 subprocess_close(read_from_child_); 830 } 831 if (err_write_ != -1 && err_read_ != -1) { 832 subprocess_close(err_read_); 833 } 834 } 835 836 void close_parent_fds() 837 { 838 if (write_to_child_ != -1) subprocess_close(write_to_child_); 839 if (read_from_child_ != -1) subprocess_close(read_from_child_); 840 if (err_read_ != -1) subprocess_close(err_read_); 841 } 842 843 void close_child_fds() 844 { 845 if (write_to_parent_ != -1) subprocess_close(write_to_parent_); 846 if (read_from_parent_ != -1) subprocess_close(read_from_parent_); 847 if (err_write_ != -1) subprocess_close(err_write_); 848 } 849 850 FILE* input() { return input_.get(); } 851 FILE* output() { return output_.get(); } 852 FILE* error() { return error_.get(); } 853 854 void input(FILE* fp) { input_.reset(fp, fclose); } 855 void output(FILE* fp) { output_.reset(fp, fclose); } 856 void error(FILE* fp) { error_.reset(fp, fclose); } 857 858 void set_out_buf_cap(size_t cap) { comm_.set_out_buf_cap(cap); } 859 void set_err_buf_cap(size_t cap) { comm_.set_err_buf_cap(cap); } 860 861 public: /* Communication forwarding API's */ 862 int send(const char* msg, size_t length) 863 { return comm_.send(msg, length); } 864 865 int send(const std::vector<char>& msg) 866 { return comm_.send(msg); } 867 868 std::pair<OutBuffer, ErrBuffer> communicate(const char* msg, size_t length) 869 { return comm_.communicate(msg, length); } 870 871 std::pair<OutBuffer, ErrBuffer> communicate(const std::vector<char>& msg) 872 { return comm_.communicate(msg); } 873 874 875 public:// Yes they are public 876 877 std::shared_ptr<FILE> input_ = nullptr; 878 std::shared_ptr<FILE> output_ = nullptr; 879 std::shared_ptr<FILE> error_ = nullptr; 880 881 #ifdef WIN32 882 HANDLE g_hChildStd_IN_Rd = nullptr; 883 HANDLE g_hChildStd_IN_Wr = nullptr; 884 HANDLE g_hChildStd_OUT_Rd = nullptr; 885 HANDLE g_hChildStd_OUT_Wr = nullptr; 886 HANDLE g_hChildStd_ERR_Rd = nullptr; 887 HANDLE g_hChildStd_ERR_Wr = nullptr; 888 #endif 889 890 // Pipes for communicating with child 891 892 // Emulates stdin 893 int write_to_child_ = -1; // Parent owned descriptor 894 int read_from_parent_ = -1; // Child owned descriptor 895 896 // Emulates stdout 897 int write_to_parent_ = -1; // Child owned descriptor 898 int read_from_child_ = -1; // Parent owned descriptor 899 900 // Emulates stderr 901 int err_write_ = -1; // Write error to parent (Child owned) 902 int err_read_ = -1; // Read error from child (Parent owned) 903 904 private: 905 Communication comm_; 906 }; 907 908 } // end namespace detail 909 910 911 912 /*! 913 * class: Popen 914 * This is the single most important class in the whole library 915 * and glues together all the helper classes to provide a common 916 * interface to the client. 917 * 918 * API's provided by the class: 919 * Popen({"cmd"}, output{..}, error{..}, ....) 920 * Command provided as a sequence. 921 * wait() - Wait for the child to exit. 922 * retcode() - The return code of the exited child. 923 * send(...) - Send input to the input channel of the child. 924 * communicate(...) - Get the output/error from the child and close the channels 925 * from the parent side. 926 */ 927 class Popen 928 { 929 public: 930 friend struct detail::ArgumentDeducer; 931 #ifndef WIN32 932 friend class detail::Child; 933 #endif 934 935 template <typename... Args> 936 Popen(std::initializer_list<const char*> cmd_args, Args&& ...args) 937 { 938 vargs_.insert(vargs_.end(), cmd_args.begin(), cmd_args.end()); 939 init_args(std::forward<Args>(args)...); 940 941 // Setup the communication channels of the Popen class 942 stream_.setup_comm_channels(); 943 944 execute_process(); 945 } 946 947 template <typename... Args> 948 Popen(std::vector<std::string> vargs_, Args &&... args) : vargs_(vargs_) 949 { 950 init_args(std::forward<Args>(args)...); 951 952 // Setup the communication channels of the Popen class 953 stream_.setup_comm_channels(); 954 955 execute_process(); 956 } 957 958 int retcode() const noexcept { return retcode_; } 959 960 int wait() noexcept(false); 961 962 void set_out_buf_cap(size_t cap) { stream_.set_out_buf_cap(cap); } 963 964 void set_err_buf_cap(size_t cap) { stream_.set_err_buf_cap(cap); } 965 966 int send(const char* msg, size_t length) 967 { return stream_.send(msg, length); } 968 969 int send(const std::string& msg) 970 { return send(msg.c_str(), msg.size()); } 971 972 int send(const std::vector<char>& msg) 973 { return stream_.send(msg); } 974 975 std::pair<OutBuffer, ErrBuffer> communicate(const char* msg, size_t length) 976 { 977 auto res = stream_.communicate(msg, length); 978 retcode_ = wait(); 979 return res; 980 } 981 982 std::pair<OutBuffer, ErrBuffer> communicate(const std::string& msg) 983 { 984 return communicate(msg.c_str(), msg.size()); 985 } 986 987 std::pair<OutBuffer, ErrBuffer> communicate(const std::vector<char>& msg) 988 { 989 auto res = stream_.communicate(msg); 990 retcode_ = wait(); 991 return res; 992 } 993 994 std::pair<OutBuffer, ErrBuffer> communicate() 995 { 996 return communicate(nullptr, 0); 997 } 998 999 private: 1000 template <typename F, typename... Args> 1001 void init_args(F&& farg, Args&&... args); 1002 void init_args(); 1003 void populate_c_argv(); 1004 void execute_process() noexcept(false); 1005 1006 private: 1007 detail::Streams stream_; 1008 1009 #ifdef WIN32 1010 HANDLE process_handle_; 1011 std::future<void> cleanup_future_; 1012 #else 1013 // Pid of the child process 1014 int child_pid_ = -1; 1015 #endif 1016 1017 std::string exe_name_; 1018 1019 // Command provided as sequence 1020 std::vector<std::string> vargs_; 1021 std::vector<char*> cargv_; 1022 1023 int retcode_ = -1; 1024 }; 1025 1026 inline void Popen::init_args() { 1027 populate_c_argv(); 1028 } 1029 1030 template <typename F, typename... Args> 1031 inline void Popen::init_args(F&& farg, Args&&... args) 1032 { 1033 detail::ArgumentDeducer argd(this); 1034 argd.set_option(std::forward<F>(farg)); 1035 init_args(std::forward<Args>(args)...); 1036 } 1037 1038 inline void Popen::populate_c_argv() 1039 { 1040 cargv_.clear(); 1041 cargv_.reserve(vargs_.size() + 1); 1042 for (auto& arg : vargs_) cargv_.push_back(&arg[0]); 1043 cargv_.push_back(nullptr); 1044 } 1045 1046 inline int Popen::wait() noexcept(false) 1047 { 1048 #ifdef WIN32 1049 int ret = WaitForSingleObject(process_handle_, INFINITE); 1050 1051 // WaitForSingleObject with INFINITE should only return when process has signaled 1052 if (ret != WAIT_OBJECT_0) { 1053 throw OSError("Unexpected return code from WaitForSingleObject", 0); 1054 } 1055 1056 DWORD dretcode_; 1057 1058 if (FALSE == GetExitCodeProcess(process_handle_, &dretcode_)) 1059 throw OSError("Failed during call to GetExitCodeProcess", 0); 1060 1061 CloseHandle(process_handle_); 1062 1063 return (int)dretcode_; 1064 #else 1065 int ret, status; 1066 std::tie(ret, status) = util::wait_for_child_exit(child_pid_); 1067 if (ret == -1) { 1068 if (errno != ECHILD) throw OSError("waitpid failed", errno); 1069 return 0; 1070 } 1071 if (WIFEXITED(status)) return WEXITSTATUS(status); 1072 if (WIFSIGNALED(status)) return WTERMSIG(status); 1073 else return 255; 1074 1075 return 0; 1076 #endif 1077 } 1078 1079 inline void Popen::execute_process() noexcept(false) 1080 { 1081 #ifdef WIN32 1082 if (exe_name_.length()) { 1083 this->vargs_.insert(this->vargs_.begin(), this->exe_name_); 1084 this->populate_c_argv(); 1085 } 1086 this->exe_name_ = vargs_[0]; 1087 1088 std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter; 1089 std::wstring argument; 1090 std::wstring command_line; 1091 bool first_arg = true; 1092 1093 for (auto arg : this->vargs_) { 1094 if (!first_arg) { 1095 command_line += L" "; 1096 } else { 1097 first_arg = false; 1098 } 1099 argument = converter.from_bytes(arg); 1100 util::quote_argument(argument, command_line, false); 1101 } 1102 1103 // CreateProcessW can modify szCmdLine so we allocate needed memory 1104 wchar_t *szCmdline = new wchar_t[command_line.size() + 1]; 1105 wcscpy_s(szCmdline, command_line.size() + 1, command_line.c_str()); 1106 PROCESS_INFORMATION piProcInfo; 1107 STARTUPINFOW siStartInfo; 1108 BOOL bSuccess = FALSE; 1109 DWORD creation_flags = CREATE_UNICODE_ENVIRONMENT | CREATE_NO_WINDOW; 1110 1111 // Set up members of the PROCESS_INFORMATION structure. 1112 ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION)); 1113 1114 // Set up members of the STARTUPINFOW structure. 1115 // This structure specifies the STDIN and STDOUT handles for redirection. 1116 1117 ZeroMemory(&siStartInfo, sizeof(STARTUPINFOW)); 1118 siStartInfo.cb = sizeof(STARTUPINFOW); 1119 1120 siStartInfo.hStdError = this->stream_.g_hChildStd_ERR_Wr; 1121 siStartInfo.hStdOutput = this->stream_.g_hChildStd_OUT_Wr; 1122 siStartInfo.hStdInput = this->stream_.g_hChildStd_IN_Rd; 1123 1124 siStartInfo.dwFlags |= STARTF_USESTDHANDLES; 1125 1126 // Create the child process. 1127 bSuccess = CreateProcessW(NULL, 1128 szCmdline, // command line 1129 NULL, // process security attributes 1130 NULL, // primary thread security attributes 1131 TRUE, // handles are inherited 1132 creation_flags, // creation flags 1133 NULL, // use parent's environment 1134 NULL, // use parent's current directory 1135 &siStartInfo, // STARTUPINFOW pointer 1136 &piProcInfo); // receives PROCESS_INFORMATION 1137 1138 // If an error occurs, exit the application. 1139 if (!bSuccess) { 1140 DWORD errorMessageID = ::GetLastError(); 1141 throw CalledProcessError("CreateProcess failed: " + util::get_last_error(errorMessageID), errorMessageID); 1142 } 1143 1144 CloseHandle(piProcInfo.hThread); 1145 1146 /* 1147 TODO: use common apis to close linux handles 1148 */ 1149 1150 this->process_handle_ = piProcInfo.hProcess; 1151 1152 this->cleanup_future_ = std::async(std::launch::async, [this] { 1153 WaitForSingleObject(this->process_handle_, INFINITE); 1154 1155 CloseHandle(this->stream_.g_hChildStd_ERR_Wr); 1156 CloseHandle(this->stream_.g_hChildStd_OUT_Wr); 1157 CloseHandle(this->stream_.g_hChildStd_IN_Rd); 1158 }); 1159 1160 /* 1161 NOTE: In the linux version, there is a check to make sure that the process 1162 has been started. Here, we do nothing because CreateProcess will throw 1163 if we fail to create the process. 1164 */ 1165 1166 1167 #else 1168 1169 int err_rd_pipe, err_wr_pipe; 1170 std::tie(err_rd_pipe, err_wr_pipe) = util::pipe_cloexec(); 1171 1172 if (exe_name_.length()) { 1173 vargs_.insert(vargs_.begin(), exe_name_); 1174 populate_c_argv(); 1175 } 1176 exe_name_ = vargs_[0]; 1177 1178 child_pid_ = fork(); 1179 1180 if (child_pid_ < 0) { 1181 subprocess_close(err_rd_pipe); 1182 subprocess_close(err_wr_pipe); 1183 throw OSError("fork failed", errno); 1184 } 1185 1186 if (child_pid_ == 0) 1187 { 1188 // Close descriptors belonging to parent 1189 stream_.close_parent_fds(); 1190 1191 //Close the read end of the error pipe 1192 subprocess_close(err_rd_pipe); 1193 1194 detail::Child chld(this, err_wr_pipe); 1195 chld.execute_child(); 1196 } 1197 else 1198 { 1199 subprocess_close(err_wr_pipe);// close child side of pipe, else get stuck in read below 1200 1201 stream_.close_child_fds(); 1202 1203 try { 1204 char err_buf[SP_MAX_ERR_BUF_SIZ] = {0,}; 1205 1206 FILE* err_fp = fdopen(err_rd_pipe, "r"); 1207 if (!err_fp) { 1208 subprocess_close(err_rd_pipe); 1209 throw OSError("fdopen failed", errno); 1210 } 1211 int read_bytes = util::read_atmost_n(err_fp, err_buf, SP_MAX_ERR_BUF_SIZ); 1212 fclose(err_fp); 1213 1214 if (read_bytes || strlen(err_buf)) { 1215 // Call waitpid to reap the child process 1216 // waitpid suspends the calling process until the 1217 // child terminates. 1218 int retcode = wait(); 1219 1220 // Throw whatever information we have about child failure 1221 throw CalledProcessError(err_buf, retcode); 1222 } 1223 } catch (std::exception& exp) { 1224 stream_.cleanup_fds(); 1225 throw; 1226 } 1227 1228 } 1229 #endif 1230 } 1231 1232 namespace detail { 1233 1234 inline void ArgumentDeducer::set_option(executable&& exe) { 1235 popen_->exe_name_ = std::move(exe.arg_value); 1236 } 1237 1238 inline void ArgumentDeducer::set_option(input&& inp) { 1239 if (inp.rd_ch_ != -1) popen_->stream_.read_from_parent_ = inp.rd_ch_; 1240 if (inp.wr_ch_ != -1) popen_->stream_.write_to_child_ = inp.wr_ch_; 1241 } 1242 1243 inline void ArgumentDeducer::set_option(output&& out) { 1244 if (out.wr_ch_ != -1) popen_->stream_.write_to_parent_ = out.wr_ch_; 1245 if (out.rd_ch_ != -1) popen_->stream_.read_from_child_ = out.rd_ch_; 1246 } 1247 1248 inline void ArgumentDeducer::set_option(error&& err) { 1249 if (err.deferred_) { 1250 if (popen_->stream_.write_to_parent_) { 1251 popen_->stream_.err_write_ = popen_->stream_.write_to_parent_; 1252 } else { 1253 throw std::runtime_error("Set output before redirecting error to output"); 1254 } 1255 } 1256 if (err.wr_ch_ != -1) popen_->stream_.err_write_ = err.wr_ch_; 1257 if (err.rd_ch_ != -1) popen_->stream_.err_read_ = err.rd_ch_; 1258 } 1259 1260 1261 #ifndef WIN32 1262 inline void Child::execute_child() { 1263 int sys_ret = -1; 1264 auto& stream = parent_->stream_; 1265 1266 try { 1267 if (stream.write_to_parent_ == 0) 1268 stream.write_to_parent_ = dup(stream.write_to_parent_); 1269 1270 if (stream.err_write_ == 0 || stream.err_write_ == 1) 1271 stream.err_write_ = dup(stream.err_write_); 1272 1273 // Make the child owned descriptors as the 1274 // stdin, stdout and stderr for the child process 1275 auto _dup2_ = [](int fd, int to_fd) { 1276 if (fd == to_fd) { 1277 // dup2 syscall does not reset the 1278 // CLOEXEC flag if the descriptors 1279 // provided to it are same. 1280 // But, we need to reset the CLOEXEC 1281 // flag as the provided descriptors 1282 // are now going to be the standard 1283 // input, output and error 1284 util::set_clo_on_exec(fd, false); 1285 } else if(fd != -1) { 1286 int res = dup2(fd, to_fd); 1287 if (res == -1) throw OSError("dup2 failed", errno); 1288 } 1289 }; 1290 1291 // Create the standard streams 1292 _dup2_(stream.read_from_parent_, 0); // Input stream 1293 _dup2_(stream.write_to_parent_, 1); // Output stream 1294 _dup2_(stream.err_write_, 2); // Error stream 1295 1296 // Close the duped descriptors 1297 if (stream.read_from_parent_ != -1 && stream.read_from_parent_ > 2) 1298 subprocess_close(stream.read_from_parent_); 1299 1300 if (stream.write_to_parent_ != -1 && stream.write_to_parent_ > 2) 1301 subprocess_close(stream.write_to_parent_); 1302 1303 if (stream.err_write_ != -1 && stream.err_write_ > 2) 1304 subprocess_close(stream.err_write_); 1305 1306 // Replace the current image with the executable 1307 sys_ret = execvp(parent_->exe_name_.c_str(), parent_->cargv_.data()); 1308 1309 if (sys_ret == -1) throw OSError("execve failed", errno); 1310 1311 } catch (const OSError& exp) { 1312 // Just write the exception message 1313 // TODO: Give back stack trace ? 1314 std::string err_msg(exp.what()); 1315 //ATTN: Can we do something on error here ? 1316 util::write_n(err_wr_pipe_, err_msg.c_str(), err_msg.length()); 1317 } 1318 1319 // Calling application would not get this 1320 // exit failure 1321 _exit (EXIT_FAILURE); 1322 } 1323 #endif 1324 1325 1326 inline void Streams::setup_comm_channels() 1327 { 1328 #ifdef WIN32 1329 util::configure_pipe(&this->g_hChildStd_IN_Rd, &this->g_hChildStd_IN_Wr, &this->g_hChildStd_IN_Wr); 1330 this->input(util::file_from_handle(this->g_hChildStd_IN_Wr, "w")); 1331 this->write_to_child_ = subprocess_fileno(this->input()); 1332 1333 util::configure_pipe(&this->g_hChildStd_OUT_Rd, &this->g_hChildStd_OUT_Wr, &this->g_hChildStd_OUT_Rd); 1334 this->output(util::file_from_handle(this->g_hChildStd_OUT_Rd, "r")); 1335 this->read_from_child_ = subprocess_fileno(this->output()); 1336 1337 util::configure_pipe(&this->g_hChildStd_ERR_Rd, &this->g_hChildStd_ERR_Wr, &this->g_hChildStd_ERR_Rd); 1338 this->error(util::file_from_handle(this->g_hChildStd_ERR_Rd, "r")); 1339 this->err_read_ = subprocess_fileno(this->error()); 1340 #else 1341 1342 if (write_to_child_ != -1) input(fdopen(write_to_child_, "wb")); 1343 if (read_from_child_ != -1) output(fdopen(read_from_child_, "rb")); 1344 if (err_read_ != -1) error(fdopen(err_read_, "rb")); 1345 1346 auto handles = {input(), output(), error()}; 1347 1348 for (auto& h : handles) { 1349 if (h == nullptr) continue; 1350 setvbuf(h, nullptr, _IONBF, BUFSIZ); 1351 } 1352 #endif 1353 } 1354 1355 inline int Communication::send(const char* msg, size_t length) 1356 { 1357 if (stream_->input() == nullptr) return -1; 1358 return std::fwrite(msg, sizeof(char), length, stream_->input()); 1359 } 1360 1361 inline int Communication::send(const std::vector<char>& msg) 1362 { 1363 return send(msg.data(), msg.size()); 1364 } 1365 1366 inline std::pair<OutBuffer, ErrBuffer> 1367 Communication::communicate(const char* msg, size_t length) 1368 { 1369 // Optimization from subprocess.py 1370 // If we are using one pipe, or no pipe 1371 // at all, using select() or threads is unnecessary. 1372 auto hndls = {stream_->input(), stream_->output(), stream_->error()}; 1373 int count = std::count(std::begin(hndls), std::end(hndls), nullptr); 1374 const int len_conv = length; 1375 1376 if (count >= 2) { 1377 OutBuffer obuf; 1378 ErrBuffer ebuf; 1379 if (stream_->input()) { 1380 if (msg) { 1381 int wbytes = std::fwrite(msg, sizeof(char), length, stream_->input()); 1382 if (wbytes < len_conv) { 1383 if (errno != EPIPE && errno != EINVAL) { 1384 throw OSError("fwrite error", errno); 1385 } 1386 } 1387 } 1388 // Close the input stream 1389 stream_->input_.reset(); 1390 } else if (stream_->output()) { 1391 // Read till EOF 1392 // ATTN: This could be blocking, if the process 1393 // at the other end screws up, we get screwed as well 1394 obuf.add_cap(out_buf_cap_); 1395 1396 int rbytes = util::read_all( 1397 stream_->output(), 1398 obuf.buf); 1399 1400 if (rbytes == -1) { 1401 throw OSError("read to obuf failed", errno); 1402 } 1403 1404 obuf.length = rbytes; 1405 // Close the output stream 1406 stream_->output_.reset(); 1407 1408 } else if (stream_->error()) { 1409 // Same screwness applies here as well 1410 ebuf.add_cap(err_buf_cap_); 1411 1412 int rbytes = util::read_atmost_n( 1413 stream_->error(), 1414 ebuf.buf.data(), 1415 ebuf.buf.size()); 1416 1417 if (rbytes == -1) { 1418 throw OSError("read to ebuf failed", errno); 1419 } 1420 1421 ebuf.length = rbytes; 1422 // Close the error stream 1423 stream_->error_.reset(); 1424 } 1425 return std::make_pair(std::move(obuf), std::move(ebuf)); 1426 } 1427 1428 return communicate_threaded(msg, length); 1429 } 1430 1431 1432 inline std::pair<OutBuffer, ErrBuffer> 1433 Communication::communicate_threaded(const char* msg, size_t length) 1434 { 1435 OutBuffer obuf; 1436 ErrBuffer ebuf; 1437 std::future<int> out_fut, err_fut; 1438 const int length_conv = length; 1439 1440 if (stream_->output()) { 1441 obuf.add_cap(out_buf_cap_); 1442 1443 out_fut = std::async(std::launch::async, 1444 [&obuf, this] { 1445 return util::read_all(this->stream_->output(), obuf.buf); 1446 }); 1447 } 1448 if (stream_->error()) { 1449 ebuf.add_cap(err_buf_cap_); 1450 1451 err_fut = std::async(std::launch::async, 1452 [&ebuf, this] { 1453 return util::read_all(this->stream_->error(), ebuf.buf); 1454 }); 1455 } 1456 if (stream_->input()) { 1457 if (msg) { 1458 int wbytes = std::fwrite(msg, sizeof(char), length, stream_->input()); 1459 if (wbytes < length_conv) { 1460 if (errno != EPIPE && errno != EINVAL) { 1461 throw OSError("fwrite error", errno); 1462 } 1463 } 1464 } 1465 stream_->input_.reset(); 1466 } 1467 1468 if (out_fut.valid()) { 1469 int res = out_fut.get(); 1470 if (res != -1) obuf.length = res; 1471 else obuf.length = 0; 1472 } 1473 if (err_fut.valid()) { 1474 int res = err_fut.get(); 1475 if (res != -1) ebuf.length = res; 1476 else ebuf.length = 0; 1477 } 1478 1479 return std::make_pair(std::move(obuf), std::move(ebuf)); 1480 } 1481 1482 } // end namespace detail 1483 1484 } 1485 1486 #endif // BITCOIN_UTIL_SUBPROCESS_H