testnet-shutdown-clean.sh
1 #!/bin/bash 2 # Testnet Server Shutdown and Cleanup Script 3 # Stops all validators and returns servers to virgin state 4 # 5 # Version: 1.0.0 6 # Date: 2026-01-23 7 8 set -euo pipefail 9 10 # Colors 11 RED='\033[0;31m' 12 GREEN='\033[0;32m' 13 YELLOW='\033[1;33m' 14 BLUE='\033[0;34m' 15 NC='\033[0m' 16 17 # Testnet servers 18 SERVERS=( 19 "testnet001.ac-dc.network" 20 "testnet002.ac-dc.network" 21 "testnet003.ac-dc.network" 22 "testnet004.ac-dc.network" 23 "testnet005.ac-dc.network" 24 ) 25 26 SSH_PORT=2584 27 28 log_info() { 29 echo -e "${BLUE}[INFO]${NC} $1" 30 } 31 32 log_success() { 33 echo -e "${GREEN}[OK]${NC} $1" 34 } 35 36 log_warn() { 37 echo -e "${YELLOW}[WARN]${NC} $1" 38 } 39 40 log_error() { 41 echo -e "${RED}[ERROR]${NC} $1" 42 } 43 44 banner() { 45 echo "" 46 echo -e "${BLUE}========================================${NC}" 47 echo -e "${BLUE}$1${NC}" 48 echo -e "${BLUE}========================================${NC}" 49 echo "" 50 } 51 52 # Cleanup function for a single server 53 cleanup_server() { 54 local server=$1 55 local server_name=$(echo "$server" | cut -d'.' -f1) 56 57 banner "Cleaning $server_name" 58 59 # Check connectivity 60 if ! ssh -p $SSH_PORT -o ConnectTimeout=5 devops@"$server" "echo 'Connected'" &>/dev/null; then 61 log_error "Cannot connect to $server" 62 return 1 63 fi 64 65 log_info "Connected to $server" 66 67 # Stop all validator processes 68 log_info "Stopping validator processes..." 69 ssh -p $SSH_PORT devops@"$server" " 70 sudo pkill -9 alphaos 2>/dev/null || true 71 sudo pkill -9 deltaos 2>/dev/null || true 72 sudo pkill -9 adnet 2>/dev/null || true 73 sleep 2 74 " 75 76 # Verify processes stopped 77 RUNNING=$(ssh -p $SSH_PORT devops@"$server" "ps aux | grep -E '(alphaos|deltaos|adnet)' | grep -v grep | wc -l") 78 if [ "$RUNNING" -eq 0 ]; then 79 log_success "All validator processes stopped" 80 else 81 log_warn "$RUNNING processes still running (may be stuck)" 82 fi 83 84 # Clean ledger data 85 log_info "Cleaning ledger data..." 86 ssh -p $SSH_PORT devops@"$server" " 87 # Clean alpha ledger 88 sudo rm -rf /root/.ledger-1-* 2>/dev/null || true 89 sudo rm -rf /root/.alphaos/ledger* 2>/dev/null || true 90 sudo rm -rf ~/.alphaos/ledger* 2>/dev/null || true 91 92 # Clean delta ledger 93 sudo rm -rf /root/.ledger-2-* 2>/dev/null || true 94 sudo rm -rf /root/.deltaos/ledger* 2>/dev/null || true 95 sudo rm -rf ~/.deltaos/ledger* 2>/dev/null || true 96 97 # Clean blockchain data mount if exists 98 if [ -d /mnt/blockchain-data ]; then 99 sudo rm -rf /mnt/blockchain-data/alphaos/* 2>/dev/null || true 100 sudo rm -rf /mnt/blockchain-data/deltaos/* 2>/dev/null || true 101 fi 102 " 103 log_success "Ledger data cleaned" 104 105 # Clean logs 106 log_info "Cleaning log files..." 107 ssh -p $SSH_PORT devops@"$server" " 108 sudo rm -f /var/log/alphaos.log 2>/dev/null || true 109 sudo rm -f /var/log/deltaos.log 2>/dev/null || true 110 sudo rm -f /tmp/alphaos*.log 2>/dev/null || true 111 sudo rm -f /tmp/deltaos*.log 2>/dev/null || true 112 sudo rm -f /root/.alphaos/*.log 2>/dev/null || true 113 sudo rm -f /root/.deltaos/*.log 2>/dev/null || true 114 " 115 log_success "Log files cleaned" 116 117 # Clean /tmp (preserve .testnet_password) 118 log_info "Cleaning /tmp directory..." 119 ssh -p $SSH_PORT devops@"$server" " 120 # Clean old files 121 sudo find /tmp -maxdepth 1 -type f -mtime +1 \ 122 -not -name '.testnet_password' \ 123 -delete 2>/dev/null || true 124 125 # Clean empty old directories 126 sudo find /tmp -maxdepth 1 -type d -mtime +1 -empty \ 127 -delete 2>/dev/null || true 128 129 # Clean non-empty old directories (>2 days) 130 sudo find /tmp -maxdepth 1 -type d -mtime +2 \ 131 -exec rm -rf {} \; 2>/dev/null || true 132 133 # Clean rustc temp directories immediately (known issue) 134 sudo find /tmp -maxdepth 1 -type d -name 'rustc*' \ 135 -mtime +0 -exec rm -rf {} \; 2>/dev/null || true 136 " 137 138 TMP_AFTER=$(ssh -p $SSH_PORT devops@"$server" "ls -la /tmp | wc -l") 139 log_success "/tmp cleaned ($TMP_AFTER entries remaining)" 140 141 # Clean systemd services (if they exist) 142 log_info "Cleaning systemd services..." 143 ssh -p $SSH_PORT devops@"$server" " 144 if [ -f /etc/systemd/system/alphaos.service ]; then 145 sudo systemctl stop alphaos 2>/dev/null || true 146 sudo systemctl disable alphaos 2>/dev/null || true 147 fi 148 149 if [ -f /etc/systemd/system/deltaos.service ]; then 150 sudo systemctl stop deltaos 2>/dev/null || true 151 sudo systemctl disable deltaos 2>/dev/null || true 152 fi 153 154 sudo systemctl daemon-reload 155 " 156 log_success "Systemd services cleaned" 157 158 # System status 159 log_info "Server status after cleanup:" 160 ssh -p $SSH_PORT devops@"$server" " 161 echo ' Uptime: '\$(uptime | awk -F'up ' '{print \$2}' | awk -F',' '{print \$1}') 162 echo ' Load: '\$(uptime | awk -F'load average:' '{print \$2}') 163 echo ' Memory: '\$(free -h | awk '/^Mem:/{printf \"%s / %s (%.0f%%)\", \$3, \$2, \$3/\$2*100}') 164 echo ' Disk: '\$(df -h / | tail -1 | awk '{print \$3\" / \"\$2\" (\"\$5\")\"}') 165 " 166 167 echo "" 168 log_success "$server_name cleaned successfully" 169 return 0 170 } 171 172 # Verify cleanup 173 verify_cleanup() { 174 local server=$1 175 176 log_info "Verifying cleanup on $server..." 177 178 # Check no processes running 179 PROCS=$(ssh -p $SSH_PORT devops@"$server" "ps aux | grep -E '(alphaos|deltaos|adnet)' | grep -v grep | wc -l") 180 if [ "$PROCS" -eq 0 ]; then 181 log_success "No validator processes running" 182 else 183 log_warn "$PROCS processes still found" 184 return 1 185 fi 186 187 # Check /tmp is reasonable 188 TMP_COUNT=$(ssh -p $SSH_PORT devops@"$server" "find /tmp -maxdepth 1 | wc -l") 189 if [ "$TMP_COUNT" -lt 500 ]; then 190 log_success "/tmp is clean ($TMP_COUNT entries)" 191 else 192 log_warn "/tmp still has $TMP_COUNT entries" 193 fi 194 195 # Check ledger data removed 196 LEDGER_COUNT=$(ssh -p $SSH_PORT devops@"$server" "sudo find /root -name '.ledger-*' -o -name 'ledger*' 2>/dev/null | wc -l") 197 if [ "$LEDGER_COUNT" -eq 0 ]; then 198 log_success "Ledger data removed" 199 else 200 log_warn "$LEDGER_COUNT ledger files/dirs still exist" 201 fi 202 203 return 0 204 } 205 206 # Main execution 207 main() { 208 # Check for --yes flag 209 AUTO_YES=false 210 if [ "${1:-}" = "--yes" ] || [ "${1:-}" = "-y" ]; then 211 AUTO_YES=true 212 fi 213 214 echo "" 215 echo -e "${BLUE}╔════════════════════════════════════════════════════════════╗${NC}" 216 echo -e "${BLUE}║ Testnet Server Shutdown & Cleanup ║${NC}" 217 echo -e "${BLUE}║ Version 1.0.0 ║${NC}" 218 echo -e "${BLUE}╚════════════════════════════════════════════════════════════╝${NC}" 219 echo "" 220 221 log_warn "This will:" 222 echo " - Stop all validator processes (alphaos, deltaos, adnet)" 223 echo " - Delete all ledger data" 224 echo " - Clean log files" 225 echo " - Clean /tmp directories" 226 echo " - Disable systemd services" 227 echo "" 228 log_warn "Servers will be returned to virgin state (pre-deployment)" 229 echo "" 230 231 if [ "$AUTO_YES" = false ]; then 232 read -p "Are you sure you want to continue? (yes/no) " -r 233 echo "" 234 235 if [[ ! $REPLY =~ ^[Yy][Ee][Ss]$ ]]; then 236 log_info "Cancelled by user" 237 exit 0 238 fi 239 else 240 log_info "Auto-confirmed with --yes flag" 241 fi 242 243 SUCCESS_COUNT=0 244 FAIL_COUNT=0 245 246 # Clean each server 247 for server in "${SERVERS[@]}"; do 248 if cleanup_server "$server"; then 249 SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) 250 else 251 FAIL_COUNT=$((FAIL_COUNT + 1)) 252 fi 253 echo "" 254 done 255 256 # Verification phase 257 banner "Verification Phase" 258 259 VERIFY_SUCCESS=0 260 VERIFY_FAIL=0 261 262 for server in "${SERVERS[@]}"; do 263 if verify_cleanup "$server"; then 264 VERIFY_SUCCESS=$((VERIFY_SUCCESS + 1)) 265 else 266 VERIFY_FAIL=$((VERIFY_FAIL + 1)) 267 fi 268 done 269 270 # Final summary 271 banner "Cleanup Summary" 272 273 echo -e "Servers cleaned: ${GREEN}${SUCCESS_COUNT}${NC} / ${#SERVERS[@]}" 274 echo -e "Servers failed: ${RED}${FAIL_COUNT}${NC}" 275 echo -e "Verification passed: ${GREEN}${VERIFY_SUCCESS}${NC} / ${#SERVERS[@]}" 276 echo -e "Verification failed: ${YELLOW}${VERIFY_FAIL}${NC}" 277 echo "" 278 279 if [ "$SUCCESS_COUNT" -eq "${#SERVERS[@]}" ] && [ "$VERIFY_SUCCESS" -eq "${#SERVERS[@]}" ]; then 280 log_success "All servers cleaned and verified successfully!" 281 echo "" 282 log_info "Servers are now in virgin state and ready for fresh deployment" 283 return 0 284 else 285 log_warn "Some servers may need manual cleanup" 286 return 1 287 fi 288 } 289 290 # Run main 291 main "$@"