debug.sh
1 #!/bin/bash 2 3 # shellcheck source=Scripts/defaults.sh 4 . "${HOME}"/RoninDojo/Scripts/defaults.sh 5 6 # shellcheck source=Scripts/dojo-defaults.sh 7 . "${HOME}"/RoninDojo/Scripts/dojo-defaults.sh 8 9 # shellcheck source=Scripts/functions.sh 10 . "${HOME}"/RoninDojo/Scripts/functions.sh 11 12 # Check for package dependencies 13 _install_pkg_if_missing "sysstat" "bc" "netcat" "nvme-cli" 14 15 # Import team pgp keys 16 gpg --import "${HOME}"/RoninDojo/Keys/pgp.txt &>/dev/null && gpg --refresh-keys &>/dev/null 17 18 prepare_cpu_readout() { 19 echo 'ENABLED="true"' | sudo tee "/etc/default/sysstat" > /dev/null 20 sudo systemctl start sysstat 21 } 22 23 cleanup_cpu_readout() { 24 sudo rm -f "/etc/default/sysstat" 25 sudo systemctl stop sysstat 26 } 27 28 print_cpu_load() { 29 30 cat <<EOF 31 ##################################################################### 32 CPU load readouts 33 ##################################################################### 34 EOF 35 36 # Get cpu load values and display to user 37 cpus=$(lscpu | grep -e "^CPU(s):" | cut -f2 -d: | awk '{print $1}') 38 39 echo "CURRENT USAGE:" 40 sar -P ALL -u ALL 1 1 | head "-`expr $cpus + 4`" | tail -$cpus | sed -r "s/\S+(\s+AM|\s+PM)?\s+(\S+)\s+(\S+).*/CPU\2 : \3/" 41 echo "" 42 43 echo "AVERAGE USAGE SINCE BOOT:" 44 mpstat -P ALL | head "-`expr $cpus + 4`" | tail -$cpus | sed -r "s/\S+(\s+AM|\s+PM)?\s+(\S+)\s+(\S+).*/CPU\2 : \3/" 45 echo "" 46 47 cat <<EOF 48 ######################################################## 49 CPU Avg Load: <1 Normal, >1 Caution, >2 Unhealthy 50 ######################################################## 51 Load Average : $(uptime | awk -F'load average:' '{ print $2 }' | cut -f1 -d,) 52 Heath Status : $(uptime | awk -F'load average:' '{ print $2 }' | cut -f1 -d, | awk '{if ($1 > 2) print "Unhealthy"; else if ($1 > 1) print "Caution"; else print "Normal"}') 53 EOF 54 55 } 56 57 print_general_info() { 58 59 # Get general system info 60 current_time=$(date) 61 os_descrip=$(grep PRETTY_NAME /etc/os-release | sed 's/PRETTY_NAME=//g') 62 os_version=$(grep VERSION_ID /etc/os-release | sed 's/VERSION_ID=//g') 63 kernel_version=$(uname -r) 64 system_uptime=$(uptime | sed 's/.*up \([^,]*\), .*/\1/') 65 backend_status=$(if cd "${ronin_ui_path}" && pm2 status | grep "online" &>/dev/null ; then printf "Online" ; else printf "Offline" ; fi) 66 tor_status=$(if systemctl is-active --quiet tor ; then printf "Online" ; else printf "Offline" ; fi) 67 docker_version=$(docker --version; docker-compose --version) 68 docker_status=$(if systemctl is-active --quiet docker ; then printf "Online" ; else printf "Offline" ; fi) 69 cpu=$(cat /sys/class/thermal/thermal_zone0/temp) 70 tempC=$((cpu/1000)) 71 temp_output=$(echo $tempC $'\xc2\xb0'C) 72 ronindojo_commit=$(git -C "${ronin_dir}" rev-parse HEAD) 73 74 cat <<EOF 75 ##################################################################### 76 General System Information 77 ##################################################################### 78 Current time : $current_time 79 OS Description : $os_descrip 80 OS Version : $os_version 81 Kernel Version : $kernel_version 82 CPU Temperature : $temp_output 83 Uptime : $system_uptime 84 UI Backend : $backend_status 85 External Tor : $tor_status 86 Docker version : $docker_version 87 Docker : $docker_status 88 RoninDojo commit : $ronindojo_commit 89 EOF 90 91 } 92 93 print_memory_usage() { 94 95 # Get Total Memory, Used Memory, Free Memory, Used Swap and Free Swap values 96 # All variables like this are used to store values as float 97 # Using bc to do all math operations, without bc all values will be integers 98 # Also we use if to add zero before value if value less than 1024, and result of dividing will be less than 1 99 total_mem=$(free -m | head -2 | tail -1| awk '{print $2}') 100 total_bc=$(echo "scale=2;if(${total_mem}<1024 && ${total_mem} > 0) print 0;${total_mem}/1024"| bc -l) 101 used_mem=$(free -m | head -2 | tail -1| awk '{print $3}') 102 used_bc=$(echo "scale=2;if(${used_mem}<1024 && ${used_mem} > 0) print 0;${used_mem}/1024"|bc -l) 103 free_mem=$(free -m | head -2 | tail -1| awk '{print $4}') 104 free_bc=$(echo "scale=2;if(${free_mem}<1024 && ${free_mem} > 0) print 0;${free_mem}/1024"|bc -l) 105 total_swap=$(free -m | tail -1| awk '{print $2}') 106 total_sbc=$(echo "scale=2;if(${total_swap}<1024 && ${total_swap} > 0) print 0;${total_swap}/1024"| bc -l) 107 used_swap=$(free -m | tail -1| awk '{print $3}') 108 used_sbc=$(echo "scale=2;if(${used_swap}<1024 && ${used_swap} > 0) print 0;${used_swap}/1024"|bc -l) 109 free_swap=$(free -m | tail -1| awk '{print $4}') 110 free_sbc=$(echo "scale=2;if(${free_swap}<1024 && ${free_swap} > 0) print 0;${free_swap}/1024"|bc -l) 111 112 cat <<EOF 113 ##################################################################### 114 Memory Usage 115 ##################################################################### 116 Total memory : $total_bc 117 Used memory : $used_bc 118 Free memory : $free_bc 119 Total swap : $total_sbc 120 Used swap : $used_sbc 121 Free swap : $free_sbc 122 EOF 123 124 # Need to fix output not displaying properly 125 #echo -e " 126 #=> Physical Memory 127 #Total\tUsed\tFree\t%Free 128 # as we get values in GB, also we get % of usage dividing Free by Total 129 #${total_bc}GB\t${used_bc}GB \t${free_bc}GB\t$(($free_mem * 100 / $total_mem ))% 130 131 #=> Swap Memory 132 #Total\tUsed\tFree\t%Free 133 #Same as above – values in GB, and in same way we get % of usage 134 #${total_sbc}GB\t${used_sbc}GB\t${free_sbc}GB\t$(($free_swap * 100 / $total_swap ))% 135 #" 136 137 # List of processes that are using most RAM 138 printf "=> Top memory using processes\n" 139 printf "PID %%MEM RSS COMMAND\n" 140 ps aux | awk '{print $2,"\t"$4,"\t"$6,"\t"$11}' | sort -k3rn | head -n 10 141 142 } 143 144 prepare_disk_readout() { 145 df -Pkh | grep -v 'Filesystem' > /tmp/df.status 146 } 147 148 cleanup_disk_readout() { 149 rm /tmp/df.status 150 } 151 152 print_disk_load() { 153 154 cat <<EOF 155 ##################################################################### 156 Disk Usage: Normal <90%, Caution >90%, Unhealthy >95% 157 ##################################################################### 158 EOF 159 160 # Display drive info 161 while read disk ; do 162 line=$(echo $disk | awk '{print $1,"\t",$6,"\t",$5," used","\t",$4," freespace"}') 163 echo -e $line 164 done < /tmp/df.status 165 166 } 167 168 print_disk_smart() { 169 170 if [ ! -b /dev/nvme0n1 ]; then 171 return 1 172 fi 173 174 cat <<EOF 175 ##################################################################### 176 NVME Disk SMART Logs 177 ##################################################################### 178 EOF 179 180 sudo nvme smart-log "/dev/nvme0n1" 181 } 182 183 print_disk_health() { 184 185 cat <<EOF 186 ##################################################################### 187 Disk Health Status 188 ##################################################################### 189 EOF 190 191 while read disk ; do 192 usage=$(echo "$disk" | awk '{print $5}' | cut -f1 -d%) 193 if [ "$usage" -ge 95 ] 194 then 195 status='Unhealthy' 196 elif [ "$usage" -ge 90 ] 197 then 198 status='Caution' 199 else 200 status='Normal' 201 fi 202 line=$(echo "$disk" | awk '{print $1,"\t",$6}') 203 echo -ne "$status" ":" "\t" "$line" 204 printf "\n" 205 done < /tmp/df.status 206 207 printf "\n" 208 209 # Show dmesg error logs if found when piped into grep search 210 if dmesg | grep "error" 1> /dev/null; then 211 _print_message "WARNING - Dmesg Error Logs Detected:" 212 dmesg | grep "error" 213 fi 214 215 } 216 217 print_docker_status() { 218 219 if [ ! -d "${dojo_path}" ]; then 220 return 221 fi 222 223 cat <<EOF 224 ##################################################################### 225 Docker Container Status 226 ##################################################################### 227 EOF 228 229 docker ps 230 231 printf "\n" 232 233 cat <<EOF 234 ##################################################################### 235 Bitcoind Logs 236 ##################################################################### 237 EOF 238 239 cd "$dojo_path_my_dojo" || exit 240 ./dojo.sh logs bitcoind -n 25 241 242 printf "\n" 243 244 cat <<EOF 245 ##################################################################### 246 Tor Logs 247 ##################################################################### 248 EOF 249 250 cd "$dojo_path_my_dojo" || exit 251 ./dojo.sh logs tor -n 25 252 253 printf "\n" 254 255 cat <<EOF 256 ##################################################################### 257 MariaDB Logs 258 ##################################################################### 259 EOF 260 261 cd "$dojo_path_my_dojo" || exit 262 ./dojo.sh logs db -n 25 263 264 printf "\n" 265 266 cat <<EOF 267 ##################################################################### 268 Indexer Logs 269 ##################################################################### 270 EOF 271 272 cd "$dojo_path_my_dojo" || exit 273 _fetch_configured_indexer_type 274 indexer=$? 275 276 if ((indexer==3)); then 277 _print_message "No indexer installed..." 278 elif ((indexer==2)); then 279 _print_message "Fulcrum Server is your current Indexer..." 280 ./dojo.sh logs fulcrum -n 25 281 282 elif ((indexer==1)); then 283 _print_message "SW Addrindexrs is your current Indexer..." 284 ./dojo.sh logs indexer -n 25 285 286 elif ((indexer==0)); then 287 _print_message "Electrs is your current Indexer..." 288 ./dojo.sh logs electrs -n 25 289 else 290 _print_message "Something went wrong! Contact support..." 291 fi 292 293 } 294 295 ronindebug() { 296 prepare_disk_readout 297 prepare_cpu_readout 298 299 print_cpu_load 300 printf "\n" 301 print_general_info 302 printf "\n" 303 print_memory_usage 304 printf "\n" 305 print_disk_load 306 printf "\n" 307 print_disk_smart 308 printf "\n" 309 print_disk_health 310 printf "\n" 311 print_docker_status 312 printf "\n" 313 314 cleanup_disk_readout 315 cleanup_cpu_readout 316 } 317 318 create_output_logs() { 319 320 _create_dir "${ronin_debug_dir}" 321 322 ronindebug > "${ronin_debug_dir}/health.txt" 323 dmesg > "${ronin_debug_dir}/dmesg.txt" 324 325 # shellcheck disable=SC2024 # this is explicit use of sudo only for accessing journalctl 326 sudo journalctl -u ronin-setup > "${ronin_debug_dir}/journal.txt" 327 328 # Include new setup.logs. This mainly replaces the journalctl output, however, there are some good outputs still in journalctl so give both. 329 cp -Rv /home/ronindojo/.logs/setup.logs "${ronin_debug_dir}/setup.logs" 330 331 gpg -e -r btcxzelko@protonmail.com -r s2l1@pm.me -r pajaseviwow@gmail.com -r dammkewl -r admin@ronindojo.io \ 332 --trust-model always -a "${ronin_debug_dir}/health.txt" 333 gpg -e -r btcxzelko@protonmail.com -r s2l1@pm.me -r pajaseviwow@gmail.com -r dammkewl -r admin@ronindojo.io \ 334 --trust-model always -a "${ronin_debug_dir}/dmesg.txt" 335 gpg -e -r btcxzelko@protonmail.com -r s2l1@pm.me -r pajaseviwow@gmail.com -r dammkewl -r admin@ronindojo.io \ 336 --trust-model always -a "${ronin_debug_dir}/journal.txt" 337 gpg -e -r btcxzelko@protonmail.com -r s2l1@pm.me -r pajaseviwow@gmail.com -r dammkewl -r admin@ronindojo.io \ 338 --trust-model always -a "${ronin_debug_dir}/setup.logs" 339 } 340 341 cleanup_output_logs() { 342 rm -rf "${ronin_debug_dir}" 343 } 344 345 upload_logs() { 346 347 # Upload full copy of pgp encrypted logs to termbin.com 348 # Link to termbin github repository: https://github.com/solusipse/fiche. 349 # Life span of single paste is one month. Older pastes are deleted. 350 cat <<EOF 351 ##################################################################### 352 PGP Encrypted Logs 353 ##################################################################### 354 EOF 355 356 357 _print_message "Please wait while URLs are generated..." 358 _sleep 2 359 360 _print_message "PGP Encrypted Logs URLs:" 361 362 printf "health file: " 363 cat "${ronin_debug_dir}"/health.txt.asc | nc termbin.com 9999 364 printf "\n" 365 printf "dmesg file: " 366 cat "${ronin_debug_dir}"/dmesg.txt.asc | nc termbin.com 9999 367 printf "\n" 368 printf "journal file: " 369 cat "${ronin_debug_dir}"/journal.txt.asc | nc termbin.com 9999 370 printf "\n" 371 printf "setup log file: " 372 cat "${ronin_debug_dir}"/setup.logs.asc | nc termbin.com 9999 373 } 374 375 # execute the scripts 376 create_output_logs 377 upload_logs 378 379 380 # Ask user to print the output 381 _print_message "Do you want to see the debugging health script output?" 382 383 while true; do 384 read -rp "[${green}Yes${nc}/${red}No${nc}]: " answer 385 case $answer in 386 [yY][eE][sS]|[yY]) 387 # Display ronindebug function output to user 388 printf "\n" 389 cat "${ronin_debug_dir}"/health.txt 390 391 _pause return 392 break 393 ;; 394 [nN][oO]|[Nn]) 395 break 396 ;; 397 *) 398 _print_message "Invalid answer! Enter Y or N" 399 ;; 400 esac 401 done 402 403 cleanup_output_logs 404 405 exit