1  #!/usr/bin/env python3
  2  # Copyright (c) 2021-present The Bitcoin Core developers
  3  # Distributed under the MIT software license, see the accompanying
  4  # file COPYING or http://www.opensource.org/licenses/mit-license.php.
  5  """Tests related to node initialization."""
  6  from concurrent.futures import ThreadPoolExecutor
  7  from pathlib import Path
  8  import os
  9  import platform
 10  import shutil
 11  import signal
 12  import subprocess
 13  import time
 14  
 15  from test_framework.test_framework import BitcoinTestFramework
 16  from test_framework.test_node import (
 17      BITCOIN_PID_FILENAME_DEFAULT,
 18      ErrorMatch,
 19  )
 20  from test_framework.util import assert_equal
 21  
 22  
 23  class InitTest(BitcoinTestFramework):
 24      """
 25      Ensure that initialization can be interrupted at a number of points and not impair
 26      subsequent starts.
 27      """
 28  
 29      def set_test_params(self):
 30          self.setup_clean_chain = False
 31          self.num_nodes = 1
 32          self.uses_wallet = None
 33  
 34      def init_stress_test(self):
 35          """
 36          - test terminating initialization after seeing a certain log line.
 37          - test removing certain essential files to test startup error paths.
 38          """
 39          self.stop_node(0)
 40          node = self.nodes[0]
 41  
 42          def sigterm_node():
 43              if platform.system() == 'Windows':
 44                  # Don't call Python's terminate() since it calls
 45                  # TerminateProcess(), which unlike SIGTERM doesn't allow
 46                  # bitcoind to perform any shutdown logic.
 47                  os.kill(node.process.pid, signal.CTRL_BREAK_EVENT)
 48              else:
 49                  node.process.terminate()
 50              node.process.wait()
 51  
 52          def start_expecting_error(err_fragment, args):
 53              node.assert_start_raises_init_error(
 54                  extra_args=args,
 55                  expected_msg=err_fragment,
 56                  match=ErrorMatch.PARTIAL_REGEX,
 57              )
 58  
 59          def check_clean_start(extra_args):
 60              """Ensure that node restarts successfully after various interrupts."""
 61              node.start(extra_args)
 62              node.wait_for_rpc_connection()
 63              height = node.getblockcount()
 64              assert_equal(200, height)
 65              self.wait_until(lambda: all(i["synced"] and i["best_block_height"] == height for i in node.getindexinfo().values()))
 66  
 67          lines_to_terminate_after = [
 68              b'Validating signatures for all blocks',
 69              b'scheduler thread start',
 70              b'Starting HTTP server',
 71              b'Loading P2P addresses',
 72              b'Loading banlist',
 73              b'Loading block index',
 74              b'Checking all blk files are present',
 75              b'Loaded best chain:',
 76              b'init message: Verifying blocks',
 77              b'init message: Starting network threads',
 78              b'net thread start',
 79              b'addcon thread start',
 80              b'initload thread start',
 81              b'txindex thread start',
 82              b'block filter index thread start',
 83              b'coinstatsindex thread start',
 84              b'msghand thread start',
 85              b'net thread start',
 86              b'addcon thread start',
 87          ]
 88          if self.is_wallet_compiled():
 89              lines_to_terminate_after.append(b'Verifying wallet')
 90  
 91          args = ['-txindex=1', '-blockfilterindex=1', '-coinstatsindex=1']
 92          for terminate_line in lines_to_terminate_after:
 93              self.log.info(f"Starting node and will terminate after line {terminate_line}")
 94              with node.busy_wait_for_debug_log([terminate_line]):
 95                  if platform.system() == 'Windows':
 96                      # CREATE_NEW_PROCESS_GROUP is required in order to be able
 97                      # to terminate the child without terminating the test.
 98                      node.start(extra_args=args, creationflags=subprocess.CREATE_NEW_PROCESS_GROUP)
 99                  else:
100                      node.start(extra_args=args)
101              self.log.debug("Terminating node after terminate line was found")
102              sigterm_node()
103  
104          # Prior to deleting/perturbing index files, start node with all indexes enabled.
105          # 'check_clean_start' will ensure indexes are synchronized (i.e., data exists to modify)
106          check_clean_start(args)
107          self.stop_node(0)
108  
109          self.log.info("Test startup errors after removing certain essential files")
110  
111          deletion_rounds = [
112              {
113                  'filepath_glob': 'blocks/index/*.ldb',
114                  'error_message': 'Error opening block database.',
115                  'startup_args': [],
116              },
117              {
118                  'filepath_glob': 'chainstate/*.ldb',
119                  'error_message': 'Error opening coins database.',
120                  'startup_args': ['-checklevel=4'],
121              },
122              {
123                  'filepath_glob': 'blocks/blk*.dat',
124                  'error_message': 'Error loading block database.',
125                  'startup_args': ['-checkblocks=200', '-checklevel=4'],
126              },
127              {
128                  'filepath_glob': 'indexes/txindex/MANIFEST*',
129                  'error_message': 'LevelDB error: Corruption: CURRENT points to a non-existent file',
130                  'startup_args': ['-txindex=1'],
131              },
132              # Removing these files does not result in a startup error:
133              # 'indexes/blockfilter/basic/*.dat', 'indexes/blockfilter/basic/db/*.*', 'indexes/coinstatsindex/db/*.*',
134              # 'indexes/txindex/*.log', 'indexes/txindex/CURRENT', 'indexes/txindex/LOCK'
135          ]
136  
137          perturbation_rounds = [
138              {
139                  'filepath_glob': 'blocks/index/*.ldb',
140                  'error_message': 'Error loading block database.',
141                  'startup_args': [],
142              },
143              {
144                  'filepath_glob': 'chainstate/*.ldb',
145                  'error_message': 'Error opening coins database.',
146                  'startup_args': [],
147              },
148              {
149                  'filepath_glob': 'blocks/blk*.dat',
150                  'error_message': 'Corrupted block database detected.',
151                  'startup_args': ['-checkblocks=200', '-checklevel=4'],
152              },
153              {
154                  'filepath_glob': 'indexes/blockfilter/basic/db/*.*',
155                  'error_message': 'LevelDB error: Corruption',
156                  'startup_args': ['-blockfilterindex=1'],
157              },
158              {
159                  'filepath_glob': 'indexes/coinstatsindex/db/*.*',
160                  'error_message': 'LevelDB error: Corruption',
161                  'startup_args': ['-coinstatsindex=1'],
162              },
163              {
164                  'filepath_glob': 'indexes/txindex/*.log',
165                  'error_message': 'LevelDB error: Corruption',
166                  'startup_args': ['-txindex=1'],
167              },
168              {
169                  'filepath_glob': 'indexes/txindex/CURRENT',
170                  'error_message': 'LevelDB error: Corruption',
171                  'startup_args': ['-txindex=1'],
172              },
173              # Perturbing these files does not result in a startup error:
174              # 'indexes/blockfilter/basic/*.dat', 'indexes/txindex/MANIFEST*', 'indexes/txindex/LOCK'
175          ]
176  
177          for round_info in deletion_rounds:
178              file_patt = round_info['filepath_glob']
179              err_fragment = round_info['error_message']
180              startup_args = round_info['startup_args']
181              target_files = list(node.chain_path.glob(file_patt))
182  
183              for target_file in target_files:
184                  self.log.info(f"Deleting file to ensure failure {target_file}")
185                  bak_path = str(target_file) + ".bak"
186                  target_file.rename(bak_path)
187  
188              start_expecting_error(err_fragment, startup_args)
189  
190              for target_file in target_files:
191                  bak_path = str(target_file) + ".bak"
192                  self.log.debug(f"Restoring file from {bak_path} and restarting")
193                  Path(bak_path).rename(target_file)
194  
195              check_clean_start(args)
196              self.stop_node(0)
197  
198          self.log.info("Test startup errors after perturbing certain essential files")
199          dirs = ["blocks", "chainstate", "indexes"]
200          for round_info in perturbation_rounds:
201              file_patt = round_info['filepath_glob']
202              err_fragment = round_info['error_message']
203              startup_args = round_info['startup_args']
204  
205              for dir in dirs:
206                  shutil.copytree(node.chain_path / dir, node.chain_path / f"{dir}_bak")
207              target_files = list(node.chain_path.glob(file_patt))
208  
209              for target_file in target_files:
210                  self.log.info(f"Perturbing file to ensure failure {target_file}")
211                  with open(target_file, "r+b") as tf:
212                      # Since the genesis block is not checked by -checkblocks, the
213                      # perturbation window must be chosen such that a higher block
214                      # in blk*.dat is affected.
215                      tf.seek(150)
216                      tf.write(b"1" * 200)
217  
218              start_expecting_error(err_fragment, startup_args)
219  
220              for dir in dirs:
221                  shutil.rmtree(node.chain_path / dir)
222                  shutil.move(node.chain_path / f"{dir}_bak", node.chain_path / dir)
223  
224      def init_pid_test(self):
225          BITCOIN_PID_FILENAME_CUSTOM = "my_fancy_bitcoin_pid_file.foobar"
226  
227          self.log.info("Test specifying custom pid file via -pid command line option")
228          custom_pidfile_relative = BITCOIN_PID_FILENAME_CUSTOM
229          self.log.info(f"-> path relative to datadir ({custom_pidfile_relative})")
230          self.restart_node(0, [f"-pid={custom_pidfile_relative}"])
231          datadir = self.nodes[0].chain_path
232          assert not (datadir / BITCOIN_PID_FILENAME_DEFAULT).exists()
233          assert (datadir / custom_pidfile_relative).exists()
234          self.stop_node(0)
235          assert not (datadir / custom_pidfile_relative).exists()
236  
237          custom_pidfile_absolute = Path(self.options.tmpdir) / BITCOIN_PID_FILENAME_CUSTOM
238          self.log.info(f"-> absolute path ({custom_pidfile_absolute})")
239          self.restart_node(0, [f"-pid={custom_pidfile_absolute}"])
240          assert not (datadir / BITCOIN_PID_FILENAME_DEFAULT).exists()
241          assert custom_pidfile_absolute.exists()
242          self.stop_node(0)
243          assert not custom_pidfile_absolute.exists()
244  
245      def break_wait_test(self):
246          """Test what happens when a break signal is sent during a
247          waitforblockheight RPC call with a long timeout. Ctrl-Break is sent on
248          Windows and SIGTERM is sent on other platforms, to trigger the same node
249          shutdown sequence that would happen if Ctrl-C were pressed in a
250          terminal. (This can be different than the node shutdown sequence that
251          happens when the stop RPC is sent.)
252  
253          The waitforblockheight call should be interrupted and return right away,
254          and not time out."""
255  
256          self.log.info("Testing waitforblockheight RPC call followed by break signal")
257          node = self.nodes[0]
258  
259          if platform.system() == 'Windows':
260              # CREATE_NEW_PROCESS_GROUP prevents python test from exiting
261              # with STATUS_CONTROL_C_EXIT (-1073741510) when break is sent.
262              self.start_node(node.index, creationflags=subprocess.CREATE_NEW_PROCESS_GROUP)
263          else:
264              self.start_node(node.index)
265  
266          current_height = node.getblock(node.getbestblockhash())['height']
267  
268          with ThreadPoolExecutor(max_workers=1) as ex:
269              # Call waitforblockheight with wait timeout longer than RPC timeout,
270              # so it is possible to distinguish whether it times out or returns
271              # early. If it times out it will throw an exception, and if it
272              # returns early it will return the current block height.
273              self.log.debug(f"Calling waitforblockheight with {self.rpc_timeout} sec RPC timeout")
274              fut = ex.submit(node.waitforblockheight, height=current_height+1, timeout=self.rpc_timeout*1000*2)
275              time.sleep(1)
276  
277              self.log.debug(f"Sending break signal to pid {node.process.pid}")
278              if platform.system() == 'Windows':
279                  # Note: CTRL_C_EVENT should not be sent here because unlike
280                  # CTRL_BREAK_EVENT it can not be targeted at a specific process
281                  # group and may behave unpredictably.
282                  node.process.send_signal(signal.CTRL_BREAK_EVENT)
283              else:
284                  # Note: signal.SIGINT would work here as well
285                  node.process.send_signal(signal.SIGTERM)
286              node.process.wait()
287  
288              result = fut.result()
289              self.log.debug(f"waitforblockheight returned {result!r}")
290              assert_equal(result["height"], current_height)
291              node.wait_until_stopped()
292  
293      def run_test(self):
294          self.init_pid_test()
295          self.init_stress_test()
296          self.break_wait_test()
297  
298  
299  if __name__ == '__main__':
300      InitTest(__file__).main()