1  #!/bin/bash
  2  
  3  ######################################
  4  # Utility functions for devnet scripts
  5  ######################################
  6  
  7  # Array to store PIDs of all processes
  8  declare -a PIDS
  9  
 10  # How many cores should each node use?
 11  # (Should be half of the number of (v)CPUs)
 12  # NOTE: when you update this, update TASKSET1/2 as well.
 13  # shellcheck disable=SC2034
 14  CORES_PER_NODE=8
 15  
 16  # Tasksets to pin processes to specific CPUs.
 17  # This is a no-op on MacOS.
 18  if [[ "$(uname)" == "Darwin" ]]; then
 19    # shellcheck disable=SC2034
 20    TASKSET1=""
 21    # shellcheck disable=SC2034
 22    TASKSET2=""
 23  else
 24    # shellcheck disable=SC2034
 25    TASKSET1="taskset -c 0-7"
 26    # shellcheck disable=SC2034
 27    TASKSET2="taskset -c 8-15"
 28  fi
 29  
 30  # Check if any tracked node process has exited.
 31  # Returns 0 if a node stopped, 1 otherwise.
 32  function check_node_stopped() {
 33    for i in "${!PIDS[@]}"; do
 34      local pid="${PIDS[i]}"
 35      if ! kill -0 "$pid" 2>/dev/null; then
 36        echo "Node #${i} (pid=$pid) has exited unexpectedly"
 37        return 0
 38      fi
 39    done
 40    return 1
 41  }
 42  
 43  # Function checking that each node in the given range [start_index, end_index)
 44  # reached a minimum block height.
 45  function check_heights() {
 46    local start_index=$1
 47    local end_index=$2
 48    local min_height=$3
 49    local network_name=$4
 50    local elapsed=$5
 51  
 52    local all_reached=true
 53    local highest_height=0
 54  
 55    for node_index in $(seq "$start_index" $((end_index-1))); do
 56      port=$((3030 + node_index))
 57      height=$(curl -s "http://127.0.0.1:$port/v2/$network_name/block/height/latest" || echo "0")
 58      
 59      # Track highest height for reporting
 60      if (is_integer "$height") && (( height > highest_height )); then
 61        highest_height=$height
 62      fi
 63      
 64      if ! (is_integer "$height") || (( height < min_height )); then
 65        echo "Node #${node_index} (port=$port) only reached height $height, expected at least $min_height"
 66        all_reached=false
 67      fi
 68    done
 69    
 70    if $all_reached; then
 71      echo "✅ SUCCESS: All nodes reached minimum height of $min_height"
 72      return 0
 73    else
 74      if (( elapsed > 0 && ((elapsed % 60) == 0) )); then
 75        elapsed_mins=$((elapsed / 60))
 76        echo "⏳ WAITING: Not all nodes reached minimum height of $min_height (highest so far: $highest_height, elapsed: $elapsed_mins minutes)"
 77      fi
 78  
 79      return 1
 80    fi
 81  }
 82  
 83  # Function checking that nodes created logs on disk and they contain no errors.
 84  function check_logs() {
 85    echo "Checking logs exist for all nodes..."
 86    local log_dir=$1
 87    local total_validators=$2
 88    local total_clients=$3
 89    
 90    local all_reached=true
 91    local highest_height=0
 92  
 93    for ((validator_index = 0; validator_index < total_validators; validator_index++)); do
 94      if [ ! -s "$log_dir/validator-${validator_index}.log" ]; then
 95        echo "❌ Test failed! Validator #${validator_index} did not create any logs in \"$log_dir\"."
 96        return 1
 97      fi
 98  
 99      if grep -q "ERROR" "$log_dir/validator-${validator_index}.log"; then
100        echo "❌ Test failed! Validator #${validator_index} logs contain errors."
101        # Print the errors to the console.
102        grep "ERROR" "$log_dir/validator-${validator_index}.log"
103        return 1
104      fi
105    done
106  
107    for ((client_index = 0; client_index < total_clients; client_index++)); do
108      if [ ! -s "$log_dir/client-${client_index}.log" ]; then
109        echo "❌ Test failed! Client #${client_index} did not create any logs in \"$log_dir\"."
110        return 1
111      fi
112  
113      if grep -q "ERROR" "$log_dir/client-${client_index}.log"; then
114        echo "❌ Test failed! Client #${client_index} logs contain errors."
115        # Print the errors to the console.
116        grep "ERROR" "$log_dir/client-${client_index}.log"
117        return 1
118      fi
119   
120    done
121  
122    return 0
123  }
124  
125  # Determine network name based on network_id
126  function get_network_name() {
127    local network_id=$1
128  
129    case $network_id in
130      0)
131        echo "mainnet"
132        ;;
133      1)
134        echo "testnet"
135        ;;
136      2)
137        echo "canary"
138        ;;
139      *)
140        >&2 echo "Unknown network ID: $network_id, defaulting to mainnet"
141        echo "mainnet"
142        ;;
143    esac
144  }
145  
146  # Stops all running processe in the given list.
147  function stop_nodes() {
148    echo "🚨 Cleaning up ${#PIDS[@]} process(es)…"
149    for pid in "${PIDS[@]}"; do
150      if kill -0 "$pid" 2>/dev/null; then
151        kill -9 "$pid" 2>/dev/null || true
152      fi
153    done
154  
155    # block until all nodes have shut down
156    wait
157  }
158  
159  # Succeeds if all nodes are available.
160  function check_nodes() {
161    local total_validators=$1
162    local total_clients=$2
163  
164    for ((node_index = 0; node_index < total_validators + total_clients; node_index++)); do
165      port=$((3030 + node_index))
166      status=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:3030/v2/$network_name/version")
167      # Fail if the HTTP response is not 2XX.
168      if (( status < 200 || status > 300 )); then
169        return 1
170      fi
171    done
172  
173    return 0
174  }
175  
176  # Succeeds if the given string is an integer.
177  function is_integer() {
178    if [[ $1 =~ ^[0-9]+$ ]]; then
179      return 0
180    else
181      return 1
182    fi
183  }
184  
185  # Succeeds if the given string is a float.
186  function is_float() {  
187    if [[ "$1" =~ ^[+-]?[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?$ ]]; then
188      return 0
189    else
190      return 1
191    fi
192  }
193  
194  # Succeeds if the node with the given index has the specified number of peers (or greater)
195  function wait_for_peers() {
196    local node_index=$1
197    local min_peers=$2
198  
199    local total_wait=0
200    local max_wait=300
201    local poll_interval=1
202    
203    while (( total_wait < max_wait )); do
204      result=$(curl -s "http://localhost:3030/v2/$network_name/peers/count")
205  
206      if (is_integer "$result") && (( result >= min_peers )); then
207        return 0
208      fi
209  
210      # Continue waiting
211      sleep $poll_interval
212      total_wait=$((total_wait+poll_interval))
213    done
214  
215    echo "❌ Nodes did not connect within 5 minutes."
216    return 1
217  }
218  
219  # Blocks until the node with the given index has at least one peer to sync from (or times out).
220  function wait_for_sync_peers() {
221    local node_index=$1
222  
223    local max_wait=300 
224    for ((total_wait=0; total_wait < max_wait; ++total_wait)); do
225      port=$((3030+node_index))
226      result=$(curl -s "http://localhost:${port}/v2/$network_name/sync/peers")
227      echo "$result"
228      num_peers=$(echo "$result" | jq -r '. | length')
229  
230      # Height is set to zero without block locators. So wait for until it is greater than 0 for at least one peer.
231      for ((idx=0; idx<num_peers; ++idx)); do
232        count=$(echo "$result" | jq -r ".[keys[$idx]]")
233        if ((count > 0)); then
234          return 0
235        fi
236      done
237  
238      # Continue waiting
239      sleep 1
240    done
241    
242    return 1
243  }
244  
245  # Blocks until the network is ready.
246  function wait_for_nodes() {
247    echo "Waiting for nodes to become ready"
248    
249    local total_validators=$1
250    local total_clients=$2
251  
252    while true; do
253      if check_node_stopped; then
254        echo "ERROR: one or more nodes stopped unexpectedly"
255        return 1
256      fi
257      
258      if check_nodes "$total_validators" "$total_clients"; then
259        echo "All nodes are ready!"
260        return 0
261      fi
262  
263      # Pause to give the nodes time to start up.
264      sleep 1
265    done
266  }
267  
268  # Compute the throughput for a number of operation over some time.
269  function compute_throughput {
270    local num_ops=$1
271    local duration=$2
272    local decimal_points=2
273    
274    # Use floating point division
275    result=$(bc <<< "scale=$decimal_points; $num_ops/$duration")
276  
277    echo "$result"
278  }
279  
280  # Print the last 20 lines of logs for all nodes.
281  function print_validator_logs() {
282    local log_dir=$1
283    local total_validators=$2
284    local total_clients=$3
285  
286    echo "Last 20 lines of node logs:"
287    for ((validator_index = 0; validator_index < total_validators; validator_index++)); do
288      echo "=== Validator $validator_index logs ==="
289      tail -n 20 "$log_dir/validator-$validator_index.log"
290    done
291  }
292  
293  function print_client_logs() {
294    local log_dir=$1
295    local total_validators=$2
296    local total_clients=$3
297  
298    for ((client_index = 0; client_index < total_clients; client_index++)); do
299      echo "=== Client $client_index logs ==="
300      node_index=$((total_validators + client_index))
301      tail -n 20 "$log_dir/client-$client_index.log"
302    done
303  }