1  #!/bin/bash
  2  # Testnet State Cleanup Script
  3  # Comprehensive cleanup of all persistent state that can block consensus
  4  # Author: Claude Sonnet 4.5
  5  # Date: 2026-01-22
  6  
  7  set -e
  8  
  9  TESTNET_HOSTS=(
 10      "testnet001.ac-dc.network"
 11      "testnet002.ac-dc.network"
 12      "testnet003.ac-dc.network"
 13      "testnet004.ac-dc.network"
 14      "testnet005.ac-dc.network"
 15  )
 16  SSH_PORT=2584
 17  
 18  echo "🧹 Testnet Comprehensive State Cleanup"
 19  echo "======================================"
 20  echo ""
 21  
 22  # Step 1: Stop all validators
 23  echo "Step 1: Stopping all validators..."
 24  for host in "${TESTNET_HOSTS[@]}"; do
 25      echo "  Stopping $host..."
 26      ssh -p $SSH_PORT devops@$host "sudo systemctl stop alphaos-validator" || true
 27  done
 28  echo "✅ All validators stopped"
 29  echo ""
 30  
 31  # Step 2: Wait for processes to fully terminate
 32  echo "Step 2: Waiting for processes to terminate (15 seconds)..."
 33  sleep 15
 34  echo "✅ Process cleanup complete"
 35  echo ""
 36  
 37  # Step 3: Delete all state files
 38  echo "Step 3: Deleting all persistent state..."
 39  for host in "${TESTNET_HOSTS[@]}"; do
 40      echo "  Cleaning $host..."
 41      ssh -p $SSH_PORT devops@$host "
 42          # Delete proposal cache files (CRITICAL - causes round mismatch)
 43          sudo rm -vf /root/.current-proposal-cache-* /root/current-proposal-cache-*
 44  
 45          # Delete BFT storage (RocksDB databases)
 46          sudo rm -rvf /root/.ledger-*
 47  
 48          # Delete blockchain ledgers
 49          sudo rm -rvf /root/.alpha /root/.delta
 50  
 51          # Verify deletion
 52          echo 'Files remaining:'
 53          sudo ls -la /root/ | grep -E 'proposal|ledger|alpha|delta' || echo '  (none - clean)'
 54      "
 55  done
 56  echo "✅ All state deleted"
 57  echo ""
 58  
 59  # Step 4: Restart validators
 60  echo "Step 4: Restarting all validators..."
 61  for host in "${TESTNET_HOSTS[@]}"; do
 62      echo "  Starting $host..."
 63      ssh -p $SSH_PORT devops@$host "sudo systemctl start alphaos-validator"
 64  done
 65  echo "✅ All validators restarted"
 66  echo ""
 67  
 68  # Step 5: Wait for initialization
 69  echo "Step 5: Waiting for validators to initialize (60 seconds)..."
 70  sleep 60
 71  echo ""
 72  
 73  # Step 6: Check status
 74  echo "Step 6: Checking validator status..."
 75  for i in {1..5}; do
 76      host="${TESTNET_HOSTS[$i-1]}"
 77      height=$(ssh -p $SSH_PORT devops@$host "curl -s http://localhost:3030/testnet/latest/height 2>&1" || echo "ERROR")
 78      echo "  $host: $height"
 79  done
 80  echo ""
 81  
 82  # Step 7: Check for round mismatch errors
 83  echo "Step 7: Checking for round mismatch errors..."
 84  for host in "${TESTNET_HOSTS[@]}"; do
 85      echo "  Checking $host logs..."
 86      error_count=$(ssh -p $SSH_PORT devops@$host \
 87          "sudo journalctl -u alphaos-validator --since '5 minutes ago' --no-pager | grep -c 'Cannot propose a batch for round 1' || echo 0")
 88      if [ "$error_count" -gt 0 ]; then
 89          echo "  ⚠️  WARNING: Found $error_count round mismatch errors on $host"
 90      else
 91          echo "  ✅ No round mismatch errors on $host"
 92      fi
 93  done
 94  echo ""
 95  
 96  echo "======================================"
 97  echo "Cleanup complete!"
 98  echo ""
 99  echo "If validators are still stuck, check logs with:"
100  echo "  ssh -p $SSH_PORT devops@testnet001.ac-dc.network 'sudo journalctl -u alphaos-validator -f'"