/ cehgit
cehgit
1 #!/bin/bash 2 # shellcheck disable=SC2031,SC2030 3 4 # shell sanity options 5 set -ue 6 7 # PLANNED: --private use and install in .git/cehgit.d/ use .git/cehgit.conf load from there too 8 9 # The main function dispatches based on how its called and which sub command is given 10 function cehgit 11 { 12 cehgit_loadconfig 13 case "$0" in 14 # called as cehgit 15 */cehgit|*/.cehgit) 16 case "${1:-}" in 17 help|-h|--help|"") 18 shift 19 show_help 20 ;; 21 init) #O [-f|--force|--update] - initialize or update the local '.cehgit' 22 # PLANNED: --all (bells&whistles) 23 shift 24 cehgit_init "$@" 25 ;; 26 update) #O - update the local '.cehgit', and all installed actions. 27 shift 28 cehgit_init --update 29 cehgit_install_action --update 30 cehgit_install_command --update 31 cehgit_install_modules 32 ;; 33 install-action) #O [-f|--force] [--all|actionglob..] - install actions that are shipped with cehgit 34 shift 35 cehgit_install_action "$@" 36 cehgit_install_modules 37 ;; 38 update-actions) #O - update all actions 39 shift 40 cehgit_install_action --update 41 cehgit_install_modules 42 ;; 43 list-actions) #O - list actions that are installed in the current repository 44 shift 45 cehgit_list_actions 46 ;; 47 available-actions) #O - list actions that are shipped with cehgit 48 shift 49 cehgit_available_actions 50 ;; 51 list-modules) #O - list modules that are installed in the current repository 52 shift 53 cehgit_list_modules 54 ;; 55 available-modules) #O - list modules that are shipped with cehgit 56 shift 57 cehgit_available_modules 58 ;; 59 list-commands) #O - list commands that are installed in the current repository 60 shift 61 cehgit_list_commands 62 ;; 63 available-commands) #O - list commands that are shipped with cehgit 64 shift 65 cehgit_available_commands 66 ;; 67 install-command) #O [-f|--force] [--all|commandglob..] - install commands that are shipped with cehgit 68 shift 69 cehgit_install_command "$@" 70 cehgit_install_modules 71 ;; 72 install-hook) #O [-f|--force] [--all|hooks..] - install a githook to use cehgit 73 shift 74 cehgit_install_hook "$@" 75 ;; 76 remove-hook) #O [hooks..] - delete cehgit controlled githooks 77 shift 78 cehgit_remove_hook "$@" 79 ;; 80 list-hooks) #O - list all githooks that point to cehgit 81 shift 82 cehgit_list_hooks 83 ;; 84 log) #O - show the log of the last test run 85 shift 86 cehgit_log 87 ;; 88 clean) #O - remove all test directories 89 shift 90 cehgit_clean 91 ;; 92 run) #O run '$RUN_HOOK' [pre-commit] with BACKGROUNDING=false and VERBOSITY_LEVEL+1 93 # shellcheck disable=SC2034 94 declare -gr CEHGIT_HOOK="$RUN_HOOK" 95 BACKGROUNDING="false" 96 VERBOSITY_LEVEL=$((VERBOSITY_LEVEL+1)) 97 cehgit_runner "$@" 98 ;; 99 *) 100 if [[ -e "$ACTIONS_DIR/cehgit-$1" ]]; then 101 local cmd="$1" 102 shift 103 # shellcheck disable=SC1090 # dynamic source 104 source "$ACTIONS_DIR/cehgit-$cmd" "$@" 105 else 106 die "unknown sub-command" 107 fi 108 esac 109 ;; 110 # called as hook 111 *.git/hooks/*) 112 # shellcheck disable=SC2034 113 declare -gr CEHGIT_HOOK="${0##*/}" 114 cehgit_runner "$@" 115 ;; 116 *) 117 die "invalid invocation $0" 118 ;; 119 esac 120 } 121 122 # declares/exports global variables, loads config files which may override them and 123 # declares global state variables 124 function cehgit_loadconfig 125 { 126 # constants 127 declare -grx ACTIONS_DIR=".cehgit.d" #C Directory where actions are located 128 129 # defaults overwritten from config files 130 declare -gx KEEP_TESTS=5 #G How many tests are kept must be bigger than 2 131 declare -gx NICE_LEVEL=1 #G The nice level 'run_test' uses 132 declare -gx TIMEOUT_SECS=10 #G Timeout for test actions 133 declare -gx MEMORY_KB=16777216 #G Memory limit for test actions 134 declare -gx TEST_PREFIX=".test-" #G Prefix for test directories, extended by a timestamp and hookname 135 declare -gx TEST_LOG="test.log" #G The name of the file where all test output will be collected 136 declare -gx VERBOSITY_LEVEL="${VERBOSITY_LEVEL:-2}" #G Default verbosity level 137 declare -gx BACKGROUNDING="true" #G Enable the background jobs machinery, when not set to 'true' background jobs will be done in foreground 138 declare -gx RUN_HOOK="pre-commit" #G Which 'hook' should 'cehgit run' emulate 139 declare -gx CEHGIT_HOOKS=(pre-commit pre-merge-commit prepare-commit-msg commit-msg) #G List of hooks to be installed by 'cehgit install-hook --all' 140 declare -gax DISABLED_ACTIONS=() #G List of action globs that are disabled, these will not be run 141 142 # load config files 143 # shellcheck disable=SC2088 # load_existing expands tilde 144 load_existing "~/config/cehgit.conf" "~/.cehgit.conf" ".git/cehgit.conf" ".cehgit.conf" 145 146 # validate config 147 [[ $KEEP_TESTS -ge 2 ]] || die "KEEP_TESTS must be larger or equal to two" 148 149 # state variables 150 # shellcheck disable=SC2155 # pwd should be infallible and more reliable than PWD 151 declare -gx WORKTREE_DIR="$(pwd)" #S Toplevel project directory 152 declare -gx LAST_TEST_DIR #S Directory of the former test 153 declare -gx TEST_DIR #S Directory for the running test 154 } 155 156 # initialize cehgit in a git repository 157 function cehgit_init 158 { 159 [[ -d ".git" ]] || die "not a git repository" 160 161 case "${1:-}" in 162 "-f"|"--force"|"--update") # force installation over existing file 163 FORCE=true 164 shift 165 ;; 166 esac 167 168 if [[ ! -f ".cehgit" ]]; then 169 info "installing .cehgit" 170 cp "$0" "./.cehgit" 171 mkdir -p "$ACTIONS_DIR" 172 elif [[ "${FORCE:-}" == true ]]; then 173 info "updating .cehgit" 174 cp -f "$0" "./.cehgit" 175 else 176 info ".cehgit already installed" 177 fi 178 } 179 180 # install actions from the ones shipped with cehgit 181 function cehgit_install_action 182 { 183 [[ -d "$ACTIONS_DIR" ]] || die "cehgit not initialized" 184 # find system installed actions dir 185 local actions_origin 186 # shellcheck disable=SC2088 # first_dir expands tilde 187 actions_origin=$(first_dir "~/.local/share/cehgit/actions" "/usr/share/cehgit/actions") 188 [[ -n "$actions_origin" ]] || die "no origin actions dir found" 189 190 while [[ $# -ne 0 ]]; do 191 case "${1:-}" in 192 "-f"|"--force") # force installation/update over existing files 193 local FORCE="-f -u" 194 shift 195 ;; 196 "--all") # install all actions 197 debug "installing all actions" 198 # shellcheck disable=SC2086 # $FORCE needs to be split 199 cp -v ${FORCE:--n} "$actions_origin/"[0-9]* "$ACTIONS_DIR/" 200 return 0 201 ;; 202 "--update") # install all actions 203 debug "updating all actions" 204 cp -v -u "$actions_origin/"[0-9]* "$ACTIONS_DIR/" 205 return 0 206 ;; 207 *) 208 break 209 ;; 210 esac 211 done 212 213 for action in "$@"; do 214 # shellcheck disable=SC2086 # $FORCE needs to be split 215 cp -v ${FORCE:--n} "$actions_origin/$action" "$ACTIONS_DIR/" 216 done 217 } 218 219 function cehgit_available_actions 220 { 221 # shellcheck disable=SC2088 # first_dir expands tilde 222 cehgit_describe "$(first_dir "~/.local/share/cehgit/actions" "/usr/share/cehgit/actions" || die "no origin actions dir found" )" '[0-9]*' 223 } 224 225 function cehgit_list_actions 226 { 227 [[ -d "$ACTIONS_DIR" ]] || die "cehgit not initialized" 228 cehgit_describe "$ACTIONS_DIR" '[0-9]*' 229 } 230 231 function cehgit_available_modules 232 { 233 # shellcheck disable=SC2088 # first_dir expands tilde 234 cehgit_describe "$(first_dir "~/.local/share/cehgit/actions" "/usr/share/cehgit/actions" || die "no origin actions dir found" )" 'mod-*' 235 } 236 237 function cehgit_list_modules 238 { 239 [[ -d "$ACTIONS_DIR" ]] || die "cehgit not initialized" 240 cehgit_describe "$ACTIONS_DIR" 'mod-*' 241 } 242 243 function cehgit_available_commands 244 { 245 # shellcheck disable=SC2088 # first_dir expands tilde 246 cehgit_describe "$(first_dir "~/.local/share/cehgit/actions" "/usr/share/cehgit/actions" || die "no origin actions dir found" )" 'cehgit-*' 247 } 248 249 function cehgit_list_commands 250 { 251 [[ -d "$ACTIONS_DIR" ]] || die "cehgit not initialized" 252 cehgit_describe "$ACTIONS_DIR" 'cehgit-*' 253 } 254 255 function any_matching_file 256 { 257 compgen -G "$1" >/dev/null 258 } 259 260 # install commands from the ones shipped with cehgit 261 function cehgit_install_command 262 { 263 [[ -d "$ACTIONS_DIR" ]] || die "cehgit not initialized" 264 # find system installed commands dir 265 local actions_origin 266 # shellcheck disable=SC2088 # first_dir expands tilde 267 actions_origin=$(first_dir "~/.local/share/cehgit/actions" "/usr/share/cehgit/actions") 268 [[ -n "$actions_origin" ]] || die "no origin actions dir found" 269 270 while [[ $# -ne 0 ]]; do 271 case "${1:-}" in 272 "-f"|"--force") # force installation/update over existing files 273 local FORCE="-f -u" 274 shift 275 ;; 276 "--all") # install all commands 277 debug "installing all commands" 278 # shellcheck disable=SC2086 # $FORCE needs to be split 279 any_matching_file "$actions_origin/cehgit-*" && cp -v ${FORCE:--n} "$actions_origin/cehgit-"* "$ACTIONS_DIR/" 280 return 0 281 ;; 282 "--update") # install all commands 283 debug "updating all commands" 284 any_matching_file "$actions_origin/cehgit-*" && cp -v -u "$actions_origin/cehgit-"* "$ACTIONS_DIR/" 285 return 0 286 ;; 287 *) 288 break 289 ;; 290 esac 291 done 292 293 for command in "$@"; do 294 # shellcheck disable=SC2086 # $FORCE needs to be split 295 cp -v ${FORCE:--n} "$actions_origin/cehgit-$command" "$ACTIONS_DIR/" 296 done 297 } 298 299 function cehgit_describe 300 { 301 [[ -d "$1" ]] || die "$1 is not a directory" 302 find "$1/" -maxdepth 1 -type f -name "$2" | sort | 303 while read -r action; do 304 echo "${action##*/}:" 305 awk '/^## ?/{sub(/^## ?/," "); print}' "$action" 306 echo 307 done 308 } 309 310 # autoinstall required modules 311 function cehgit_install_modules 312 { 313 [[ -d "$ACTIONS_DIR" ]] || die "cehgit not initialized" 314 # find system installed modules dir 315 local modules_origin 316 # shellcheck disable=SC2088 # first_dir expands tilde 317 modules_origin=$(first_dir "~/.local/share/cehgit/actions" "/usr/share/cehgit/actions") 318 [[ -n "$modules_origin" ]] || die "no original modules dir found" 319 320 if any_matching_file ""$ACTIONS_DIR/"*"; then 321 # shellcheck disable=SC2013 322 for module in $(awk '/^require/ {for (i=2; i<=NF; i++) print $i}' "$ACTIONS_DIR/"* | sort -u ); do 323 if [[ -e "$modules_origin/mod-$module" ]]; then 324 debug "installing $module" 325 cp -v -u "$modules_origin/mod-$module" "$ACTIONS_DIR/" 326 else 327 error "module $module not found" 328 fi 329 done 330 fi 331 } 332 333 function cehgit_install_hook 334 { 335 [[ -x ".cehgit" ]] || die "cehgit not initialized" 336 337 while [[ $# -ne 0 ]]; do 338 case "${1:-}" in 339 "-f"|"--force") # force installation/update over existing hooks 340 FORCE=true 341 shift 342 ;; 343 "--all") # install all hooks listed in the configuration 344 for hook in "${CEHGIT_HOOKS[@]}"; do 345 debug "installing hook $hook" 346 ln ${FORCE:+-f} -s '../../.cehgit' ".git/hooks/$hook" || error "installing hook $hook failed" 347 done 348 return 0 349 ;; 350 *) # install hooks by name 351 rc=0 352 for hook in "$@"; do 353 debug "installing hook $hook" 354 ln ${FORCE:+-f} -s '../../.cehgit' ".git/hooks/$hook" 2>/dev/null || 355 { 356 error "installing hook $hook failed" 357 rc=1 358 } 359 done 360 return $rc 361 esac 362 done 363 } 364 365 function cehgit_remove_hook 366 { 367 # PLANNED: --all 368 for hook in "$@"; do 369 if [[ $(readlink ".git/hooks/$hook") = ../../.cehgit ]]; then 370 rm ".git/hooks/$hook" 371 debug "removed hook $hook" 372 else 373 error "$hook: not a cehgit controlled hook" 374 fi 375 done 376 } 377 378 function cehgit_list_hooks 379 { 380 find .git/hooks/ -type l -lname '../../.cehgit' -printf "%f\n" 381 } 382 383 function cehgit_clean 384 { 385 [[ -n "$TEST_PREFIX" ]] || die "TEST_PREFIX not set" 386 find . -name "${TEST_PREFIX}*" -type d -exec rm -r {} + 387 } 388 389 function first_dir # finds the first dir that exists 390 { 391 for dir in "$@"; do 392 dir="${dir/#~\//$HOME/}" 393 trace "trying $dir" 394 if [[ -d "$dir" ]]; then 395 echo "$dir" 396 return 0 397 fi 398 done 399 return 1 400 } 401 402 function load_existing # loads (sources) all files from a list, ignoring when they don't exist 403 { 404 for file in "$@"; do 405 file="${file/#~\//$HOME/}" 406 [[ -f "$file" ]] && { 407 debug "$file" 408 # shellcheck disable=SC1090 # dynamic source 409 source "$file" 410 } 411 done 412 return 0 413 } 414 415 # the main loop that calls all actions 416 function cehgit_runner 417 { 418 declare -gx TREE_HASH 419 TREE_HASH="$(git write-tree)" 420 readonly TREE_HASH 421 422 # clean up old tests 423 find . -name "${TEST_PREFIX}*" -type d | sort -n | head -n -$((KEEP_TESTS-1)) | xargs -r rm -r 424 425 # find the dir of the previous test if any 426 LAST_TEST_DIR=$(find . -name "${TEST_PREFIX}*" -type d | sort -rn | head -n 1) 427 LAST_TEST_DIR="${LAST_TEST_DIR:+$WORKTREE_DIR/${LAST_TEST_DIR#./}}" 428 debug "LAST_TEST_DIR = $LAST_TEST_DIR" 429 430 # reuse a test dir that was created from the same git tree 431 TEST_DIR=$(find . -type d -name ".test-*-$TREE_HASH" | tail -1) 432 [[ -z "$TEST_DIR" ]] && TEST_DIR="${TEST_PREFIX}$(awk 'BEGIN {srand(); print srand()}')-$TREE_HASH" 433 debug "TEST_DIR = $TEST_DIR" 434 435 # populate the test dir 436 git archive "$TREE_HASH" --prefix="$TEST_DIR/" | tar xf - 437 ln -sf ../.git "$TEST_DIR/" 438 # and enter it 439 cd "$TEST_DIR" 440 441 lock_wait .cehgit.lock 442 443 # run all test actions 444 # sequence number incremented for each reuse of the test dir 445 state_init .cehgit.log 446 447 # foreground loop over all actions 448 for ACTION in "$WORKTREE_DIR/$ACTIONS_DIR/"[0-9]* ; do 449 [[ -f "$ACTION" ]] || continue 450 ACTION="${ACTION##*/}" 451 452 for glob in "${DISABLED_ACTIONS[@]}"; do 453 # shellcheck disable=SC2053 454 if [[ "$ACTION" == *$glob* ]]; then 455 info "$ACTION is disabled" 456 continue 2 457 fi 458 done 459 460 # shellcheck disable=SC2155 461 local state="$(state_cached)" 462 if [[ -z "$state" ]]; then 463 debug "$ACTION" 464 # shellcheck disable=SC1090 # dynamic source 465 if ! source "$WORKTREE_DIR/$ACTIONS_DIR/$ACTION" "$@"; then 466 exit 1 467 fi 468 elif [[ "$state" == fail ]]; then 469 debug "$ACTION cached $state" 470 exit 1 471 fi 472 done |& tee -a "$TEST_LOG" || true 473 474 [[ ${PIPESTATUS[0]} != 0 ]] && { 475 error "action failed" 476 lock_remove .cehgit.lock 477 exit 1 478 } 479 480 # start backgrounded jobs 481 if state_match _ '*' background; then 482 info "schedule background jobs" 483 declare -a BACKGROUND_ACTIONS 484 mapfile -t BACKGROUND_ACTIONS < <(state_list_actions background) 485 trace "bg: ${BACKGROUND_ACTIONS[*]}" 486 ( 487 lock_wait .cehgit.bg.lock 488 state_init .cehgit.bg.log 489 touch "$TEST_LOG.bg" 490 # shellcheck disable=SC2030 491 declare -rgx CEHGIT_BACKGROUND=true 492 493 # background loop over scheduled actions 494 # shellcheck disable=SC2030 # only interested in local changes 495 for ACTION in "${BACKGROUND_ACTIONS[@]}"; do 496 ACTION="${ACTION##*/}" 497 498 # shellcheck disable=SC2155 499 local state="$(state_cached)" 500 if [[ -z "$state" ]]; then 501 debug "bg running $ACTION" 502 # shellcheck disable=SC1090 # dynamic source 503 source "$WORKTREE_DIR/$ACTIONS_DIR/$ACTION" "$@" || true 504 [[ "$(state_cached)" =~ ok|fail ]] || { 505 die "background_schedule: background job $ACTION must result in ok or fail" 506 } 507 elif [[ "$state" == fail ]]; then 508 debug "bg $ACTION cached $state" 509 exit 1 510 fi 511 done |& cat >>"$TEST_LOG.bg" 512 513 lock_remove .cehgit.bg.lock 514 ) & 515 else 516 debug "no background jobs" 517 fi 518 519 debug "forground runner complete" 520 lock_remove .cehgit.lock 521 } 522 523 function cehgit_log 524 { 525 LAST_TEST_DIR=$(find . -name "${TEST_PREFIX}*" -type d | sort -rn | head -n 1) 526 [[ -z "$LAST_TEST_DIR" ]] && die "no last test dir found" 527 less -R "$LAST_TEST_DIR/$TEST_LOG" 528 } 529 530 # State log: 531 # A statelog records the states of actions in the form of 'SEQ ACTION STATE ...'. 532 # We can have multiple state logs that are distinguished by a tag. 533 # Each state log is used to cache the state of actions and to schedule background jobs. 534 # A state log is appended to after each action is run. 535 536 # (re-) initializes a state log 537 # sets the global STATE_FILE and STATE_SEQ 538 function state_init #api {statefile} - initializes a state log 539 { 540 declare -gx STATE_FILE="$1" 541 declare -gx STATE_SEQ 542 STATE_SEQ=$(awk '{max = ($1 > max) ? $1 : max} END {print max + 1}' "$STATE_FILE" 2>/dev/null || echo 0) 543 touch "$STATE_FILE" 544 trace "STATE_FILE = $STATE_FILE, STATE_SEQ = $STATE_SEQ" 545 } 546 547 function state_cached # returns the cached ok|fail 548 { 549 awk '/^[0-9]+ '"$ACTION"' (ok|fail)/{print $3; exit 0;}' "$STATE_FILE" 2>/dev/null 550 } 551 552 function state_log # {state} - logs action 553 { 554 trace "$STATE_SEQ $ACTION $1 >> $STATE_FILE" 555 echo "$STATE_SEQ $ACTION $1" >>"$STATE_FILE" 556 } 557 558 function state_match # [seq] [action] [state] - checks if any action with a given state exists in the log 559 { 560 local seq="$1" 561 local action="${2##*/}" 562 local state="$3" 563 564 # expand _ to defaults 565 [[ "$seq" = "_" ]] && seq="$STATE_SEQ" 566 [[ "$action" = "_" ]] && action="$ACTION" 567 [[ "$state" = "_" ]] && state="ok|fail" 568 569 # expand * to [^ ]\+ (any word) 570 [[ "$seq" = "*" ]] && seq="[0-9]+" 571 [[ "$action" = "*" ]] && action="\\S+" 572 [[ "$state" = "*" ]] && state="\\S+" 573 574 awk "BEGIN {rc=1} /^$seq $action $state\$/{rc=0} END {exit rc}" "$STATE_FILE" 575 } 576 577 function state_list_actions # {state} - lists all actions with a given state within the current sequence in the log 578 { 579 awk "/^$STATE_SEQ \\S+ $1/{print \$2}" "$STATE_FILE" 580 } 581 582 # Lockfiles: 583 # Manage non recursive ownership of resources. 584 # We can create or wait on a lockfile with lock_wait and remove it with lock_remove. 585 # lock_wait takes a name and an optional command pattern to check if the lock is still valid as parameter. 586 # lock_try takes a name and an optional command pattern to check if the lock is still valid as parameter. 587 # lock_remove takes a name as parameter and removes the lockfile if it is owned by the current process. 588 # The lockfile is named after the name parameter with a '.lock' suffix. 589 # The lockfile contains the BASHPID of the process that owns the lock as first line. 590 591 function lock_wait #api {name} [cmdpat] - lock a lockfile, waits until we have the lock 592 { 593 # Create a temporary lockfile specific to this process 594 local tmp_lockfile="${1}.$$" 595 596 # Put our PID in the temporary file 597 echo $$ > "$tmp_lockfile" 598 599 # Try to atomically acquire the lock 600 while ! ln -s "$tmp_lockfile" "$1" 2>/dev/null; do 601 # Lock exists, check if it's stale 602 if [[ -L "$1" ]]; then 603 # shellcheck disable=SC2155 # want to ignore errors here 604 local lockfile_target=$(readlink "$1") 605 local lockpid="${lockfile_target##*.}" 606 607 # Check if the process still exists 608 if ! kill -0 "$lockpid" 2>/dev/null; then 609 # Process is gone, remove the stale lock 610 rm -f "$1" 611 trace "removed stale lock from process $lockpid" 612 continue 613 fi 614 615 trace "wait: $lockpid to complete" 616 # Wait for the process to exit 617 while kill -0 "$lockpid" 2>/dev/null; do 618 wait "$lockpid" 2>/dev/null || sleep 0.1 619 done 620 621 # Remove the stale lock 622 rm -f "$1" 623 fi 624 done 625 626 trace "locked: $1 as $$" 627 } 628 629 function lock_try #api {name} - try to lock a lockfile, returns 0 if successful, 1 if already locked 630 { 631 # Create a temporary lockfile specific to this process 632 local tmp_lockfile="${1}.$$" 633 634 # Put our PID in the temporary file 635 echo $$ > "$tmp_lockfile" 636 637 # Try to atomically acquire the lock 638 if ln -s "$tmp_lockfile" "$1" 2>/dev/null; then 639 trace "locked: $1 as $$" 640 return 0 641 else 642 # Clean up our temp file since we're not waiting 643 rm -f "$tmp_lockfile" 644 trace "failed to lock $1, already locked" 645 return 1 646 fi 647 } 648 649 function lock_remove #api {name} - remove a lockfile, unlock 650 { 651 if [[ -L "$1" ]]; then 652 # shellcheck disable=SC2155 # want to ignore errors here 653 local lockfile_target=$(readlink "$1") 654 local lockpid="${lockfile_target##*.}" 655 656 if [[ "$lockpid" = "$$" ]]; then 657 trace "unlock $1" 658 # Remove both the lock and the temporary file 659 rm -f "$1" 660 rm -f "$lockfile_target" 661 return 0 662 else 663 error "$1 is not ours (we are $$, owned by $lockpid)" 664 return 1 665 fi 666 elif [[ -f "$1" ]]; then 667 error "$1 is not a symbolic link lockfile" 668 return 1 669 else 670 error "$1 does not exist" 671 return 1 672 fi 673 } 674 675 function run_test #afunc [-n|--nice LEVEL] [-m|--memory KB] [-t|--timeout SECS] [program] [args..] - runs a test in a resource limited subshell 676 { 677 while [[ $# -gt 0 ]]; do 678 case "${1:-}" in 679 -n|--nice) 680 local NICE_LEVEL="$2" 681 shift 2 682 ;; 683 -m|--memory) 684 local MEMORY_KB="$2" 685 shift 2 686 ;; 687 -t|--timeout) 688 local TIMEOUT_SECS="$2" 689 shift 2 690 ;; 691 *) 692 break 693 ;; 694 esac 695 done 696 697 info "run_test $*" 698 if ( 699 renice -n "$NICE_LEVEL" -p $BASHPID >/dev/null 700 ulimit -S -v "$MEMORY_KB" -t "$TIMEOUT_SECS" 701 "$@" 702 ); then 703 state_log ok 704 return 0 705 else 706 state_log fail 707 return 1 708 fi 709 } 710 711 # background lock semantics explained 712 # 713 # we have 2 lockfiles and statelogs: 714 # forground: .cehgit.lock .cehgit.log 715 # background .cehgit.bg.lock .cehgit.bg.log 716 # 717 # background actions must conclusively finish with ok or fail, 718 # this is required because the 'cache' mechanism will only pick up these two states. 719 # 720 # background scheduling with do hand over hand locking from foreground to background 721 # lock. Thus this transition is atomic. 722 723 function background_schedule #afunc - schedules the current action to run in the background when not already scheduled 724 { 725 if [[ "$BACKGROUNDING" != "true" ]] || [[ -n "${CEHGIT_BACKGROUND:-}" ]]; then 726 # called from background job, execute it 727 trace "execute $ACTION" 728 return 1 729 else 730 if state_match _ _ background; then 731 info "already scheduled $ACTION" 732 else 733 info "schedule $ACTION" 734 state_log background 735 fi 736 fi 737 } 738 739 function background_wait #afunc - waits for the background actions to finish and merges the log 740 { 741 if [[ "$BACKGROUNDING" != "true" ]]; then 742 trace "backgrounding disabled" 743 return 1 744 fi 745 746 if [[ -n "${CEHGIT_BACKGROUND:-}" ]]; then 747 trace "running in background $ACTION" 748 return 1 749 else 750 trace "running in foreground $ACTION" 751 # assert that the action was backgrounded 752 state_match '*' _ background || die "background_result: no background action scheduled" 753 754 [[ -n "$(state_cached)" ]] && return 0 755 756 if [[ -f .cehgit.bg.log ]]; then 757 lock_wait .cehgit.bg.lock 758 trace "collect the background logs" 759 cat .cehgit.bg.log >>.cehgit.log 760 cat "$TEST_LOG.bg" >>"$TEST_LOG" 761 rm .cehgit.bg.log 762 rm "$TEST_LOG.bg" 763 lock_remove .cehgit.bg.lock 764 fi 765 return 0 766 fi 767 } 768 769 function background_result #afunc - returns 0 on ok and 1 on fail of a background action 770 { 771 local state 772 state="$(state_cached)" 773 trace "$ACTION $state" 774 if [[ "$state" = "fail" ]]; then 775 echo 1 776 else 777 echo 0 778 fi 779 } 780 781 function show_help 782 { 783 less <<EOF 784 cehgit -- cehtehs personal git assistant 785 786 787 ABOUT 788 789 cehgit is a frontend for githooks that runs bash scripts (actions) in sequence. This acts 790 much like a CI but for your local git repository. Unlike some other 'pre-commit' solutions 791 it will not alter your worktree by stashing changes for the test but run tests in dedicated 792 directories which are kept around for later inspection and improving test performance. 793 794 cehgit caches the state of the last run tests and reuses the test directories when the git 795 tree did not change. This allows for incremental testing and faster turnaround times. 796 It schedule tests to run in background. This means tests may run while you type a commit 797 message. 798 799 It is also possible to extend cehgit with new subcommands. Unlike actions these are not 800 run in a dedicated test directory but in the current worktree. 801 802 When you read this because you seen '.cehgit' used in a repository then you may look at 803 INITIAL INSTALLATION below. 804 805 806 USAGE 807 808 $(sed 's/^ *\([^)]\+\)) #O\(.*\) - \(.*\)/ cehgit \1\2\n \3\n/p;d' < "$0") 809 810 cehgit [-h|--help|help] 811 show this help 812 813 ./.cehgit [..] 814 same as 'cehgit' above but calling the repo local version 815 816 ./.git/hooks/* [OPTIONS..] 817 invoking git hooks manually 818 819 820 SETUP 821 822 To use cehgit in a git repository it has first to be initialized with 'cehgit init'. This 823 copies itself to a repository local '.cehgit' and creates the '.cehgit.d/' 824 directory. 'cehgit init --upgrade' will upgrade an already initialized version. 825 826 Then the desired actions have to be written or installed. 'cehgit install-action --all' will 827 copy all actions shipped with cehgit to '.cehgit.d/'. This should always be safe but may 828 include more than one may want and implement some opinionated workflow. The installed 829 actions are meant to be customized to personal preferences. 830 831 cehgit puts tests in sub-directories starting with '.test-*'. This pattern should be added 832 to '.gitignore'. 833 834 '.cehgit', '.cehgit.d/*' and '.cehgit.conf' are meant to be commited into git and 835 versioned. 836 837 Once this is set up one should 'cehgit install-hooks [--all]' to setup the desired hooks 838 locally. Note that installed hooks are not under version control and every checkout of 839 the repository has to install them manually again. 840 841 This completes the setup, cehgit should now by called whenever git invokes a hook. 842 843 844 HOW CEHGIT WORKS 845 846 Cehgit is implemented in bash the test actions are sourced in sorted order. Bash was chosen 847 because of it's wide availability and more addvanced features than standard shells. We rely 848 on some bashisms. To make shell programming a little more safe it calls 'set -ue' which 849 ensures that variables are declared before used and exits on the first failure. 850 851 Test are run in locally created test directories, the worktree itself is not 852 altered/stashed. This test directories are populated from the currently staged files in the 853 working tree. The '\$TEST_DIR/.git/' directory is symlinked to the original '../.git'. 854 855 Test directries are reused when they orginate from the same git tree (hash), cehgit 856 deliberately does not start from a clean/fresh directory to take advantage of incremental 857 compilation and artifacts from former runs to speed tests up. All actions on a tree are 858 logged and this log is used to query cached results. 859 860 It keeps the last KEEP_TESTS around and removes excess ones. 861 862 When invoked as githook a test directory is created or reused and entered. Then all actions 863 in ACTIONS_DIR are sourced in sorted order. Actions determine by API calls if they should 864 execute, schedule to background or exit early. 865 866 Modules starting with 'mod-' in ACTIONS_DIR can be used to extend the 867 API. actions use 'require' to load. 868 869 API calls with also log the progress, ok/fail states will be reused in subsequent runs. 870 Many function results are memoized with the 'memo' helper function. 871 872 The test directories left behind can be inspected at later time. There will be a 'test.log' 873 where the stdout of all actions that run is collected. 874 875 To debug cehgit execution itself one can set VERBOSITY_LEVEL to a number up to 876 5 (0=none, 1=error, 2=notice, 3=info, 4=debug, 5=trace) 877 878 879 BACKGROUND DETAILS 880 881 Schedule actions to background will finish the currently running hook early before 882 this background actions are completed. Background actions will be scheduled after all 883 foreground actions completed successful. 884 885 Background actions should finish with a conclusive ok or fail. Using 'run_test' will take 886 care of that. 887 888 Example Action: 889 890 # background_schedule will succeed when this action is scheduled and fail when it is already 891 # scheduled and should be executed. We '&& return 0' to exit when scheduled and fall through on fail 892 background_schedule && return 0 893 # To retrieve the result of the background action use background_wait and background_result 894 # this falls through when no background action was scheduled 895 background_wait && return \$(background_result) 896 # eventually run the actual code 897 run_test make 898 899 cehgit utilizes the pre-commit hook to schedule expensive tests into the background. 900 Then while the user enters a commit message the tests run and the commit-msg hook checks 901 for the outcome of the tests. 902 903 904 CONFIGURATION 905 906 cehgit tries to load the following configuration files in this order: 907 "~/config/cehgit.conf" "~/.cehgit.conf" ".git/cehgit.conf" ".cehgit.conf" 908 909 They are all optional, the defaults should be sufficient for most use cases. When not, then 910 one can create the one config file and customize it. Only '.cehgit.conf' are meant to be 911 versioned and distriubuted. The others adds local configuration that is not versioned. 912 913 Following Variables can be configured [=default]: 914 915 $(sed 's/ *declare -[^ ]\+ \([^ ]*\)=\([^ ]*\) *#G *\(.*\)/ \1 [\2]\n \3\n/p;d' < "$0") 916 917 918 EXTEND CEHGIT WITH SUBCOMMANDS 919 920 Bash scripts in the ACTIONS_DIR whose name starts with 'cehgit-' can be called as subcommand 921 for example a '.cehgit.d/cehgit-foo' script can be called with 'cehgit foo'. These scripts 922 are run in the current worktree and not in a test directory but have all cehgit apis 923 available and may load modules with 'require'. They can be used to implement custom 924 workflows or implement other tasks. 925 926 927 WRITING MODULES 928 929 Modules extend the cehgit API. They are loaded with the 'require' function. 930 931 All modules must be prefixed with 'mod-' and be in ACTIONS_DIR. The first word after 'mod-' 932 should be a descriptive name of what the module does. 933 934 Modules should define function to be used by actions. They should not run any code on their 935 own except for initialization tasks. Ideally they use the 'memo'/'memofn' function to cache state. 936 937 Functions they define should contain with the module name and a descriptive name of what the 938 function does. The usual form is 'mod_*' where '*' is a descriptive name of what the 939 function does. For primary predicates we allow 'if_mod_*' forms too. 940 941 Modules are automatically installed to ACTIONS_DIR when required. 942 943 944 WRITING ACTIONS 945 946 cehgit runs all actions in order and aborts execution on the first failure. 947 948 Actions must be prefixed with a double digit number to ensure proper ordering. 949 It is recommended to follow following guides for the naming: 950 951 - 10 configuration and prepopulation 952 When some adjustments are to be done to make the test dir compileable this is the place. 953 954 - 20 validate test dir, toolchain versions 955 Check for presence and validty of files in the test dir. 956 Check for toolchains/tools, required versions 957 958 - 30 linters/formatting check 959 Testing begins here with resonable cheap checks, running linters and similar things. 960 961 - 40 building 962 This is where compilation happens, we are building the project, tests here. But do 963 not run them yet. 964 965 - 50 normal testing 966 Runs the standard testsuite. The shipped actions will background these tests. 967 968 - 60 extensive testing/mutants/fuzzing 969 When there are any expensive tests like running mutant checks, fuzzing, benchmarks this 970 is done here. The shipped actions will background these tests. 971 972 - 70 doc 973 Generate documentation, possibly test it. 974 975 - 80 staging work, release chores, changelogs 976 When some automatic workflow should be implemented like promoting/merging branches, 977 generating changelogs etc, this is done here. 978 979 - 90 packaging/deploy/postprocessing 980 Final step for automatic workflows which may build packages and deploy it. 981 Also if any final processing/cleanup has to be done. 982 983 Actions should have comments starting with '##' which will be extracted on 'cehgit list-actions' 984 and 'cehgit available-actions' giving some info about what an action does. 985 986 Actions can do different things: 987 988 - Calling functions that check whenever the action should be in effect or not. cehgit calls 989 all actions in order. Some actions should only run under certain conditions. Each action 990 may return early when it should not run. The shopped modules provides functions to check 991 the current git branch or the hook that is running and to schedule actions to run in the 992 background and retrieve its results. These functions check for some conditions and return 993 1 when the condition is not met. This must be handled by the caller otherwise 994 cehgit will exit with a failure: 995 996 git_branch_matches master || return 0 997 git_hook_matches pre-commit || return 0 998 background_schedule && return 0 999 background_wait && return \$(background_result) 1000 ... 1001 1002 - Call an actual test. This is usually done by the run_test function that is part of cehgit 1003 API. It takes the command to run as parameters and runs this in a subshell with some 1004 (configurable) resource limits applied. On a Makefile based project this may be something 1005 like: 1006 1007 run_test make check 1008 1009 Actions are run within the TEST_DIR being the current directory. 1010 1011 1012 AVAILABLE ACTIONS 1013 1014 Some actions is shipped with cehgit. More will be added in future. These 1015 implement and eventually evolve into an automated workflow. 1016 1017 $($0 available-actions | awk '/.+/{print " " $0} /^$/{print}') 1018 1019 BUILTIN ACTION FUNCTIONS 1020 1021 cehgit provides a minimal set of built-in functions to be used in actions. Most functionality 1022 should be implemented in modules. 1023 1024 These functions return 0 on success and 1 on failure. A 'return 1' must be handled otherwise 1025 cehgit would exit. Ususally this is done by something like 'some_action_function || return 1026 0'. Using a 'if' or other operators is possible as well. 1027 1028 $(sed 's/^function \(.*\) #afunc\(.*\) - \(.*\)/ \1\2\n \3\n/p;d' < "$0") 1029 1030 1031 API FUNCTIONS 1032 1033 We define a few functions for diagnostics, locking, caching and to record states: 1034 1035 $(sed 's/^function \(.*\) #api\(.*\) - \(.*\)/ \1\2\n \3\n/p;d' < "$0") 1036 1037 For further documentation look into the source. 1038 1039 1040 INITIAL INSTALLATION 1041 1042 cehgit can be invoked in 3 ways: 1043 1044 1. Installed in \$PATH: 1045 This is used to initialize cehgit in git repositories and install actions and hooks. 1046 'cehgit init' copies itself to './.cehgit' and creates a './.cehgit.d/' directory 1047 when called in a git repository. 1048 The recommended way to install cehgit in \$PATH 1049 2. The local './.cehgit' initialized from above: 1050 This should be versioned, so anyone who clones a git repository where cehgit is 1051 initialized can use this local version. 1052 3. githooks symlinked from './.git/hooks/*' -> '../../.cehgit' 1053 When called as githook, then it calls actions in './.cehgit.d/' in order. 1054 1055 To make 1. work it is best to clone cehgit locally and symlink the checked out files 1056 to your '.local/' tree. This allows easy upgrades via git: 1057 1058 1059 # cehgit is distributed on radicle p2p forge when you have a radicle client 1060 # installed you can clone it with the radicle client: 1061 rad clone rad:z3H96oNDcR4kBoREvBJHKr7Z9w7B7 1062 1063 # alternatively you can git-clone it from a seed server: 1064 git clone https://seed.pipapo.org/z3H96oNDcR4kBoREvBJHKr7Z9w7B7.git cehgit 1065 1066 cd cehgit 1067 1068 # symlink the script itself 1069 ln -s $PWD/cehgit $HOME/.local/bin/ 1070 # symlink the actions directory 1071 mkdir -p $HOME/.local/share/cehgit 1072 ln -s $PWD/actions $HOME/.local/share/cehgit/ 1073 1074 You can manually copy or symlink either from above to '/usr/bin' and '/usr/share' as well. 1075 1076 1077 SECURITY 1078 1079 cehgit is completely inert in a initialized or freshly checked out repository. One always 1080 has to './.cehgit install-hook' to enable it. Then as any other build script cehgit actions 1081 run in the context of the calling user. Unlike in a CI there is no isolation. Thus before 1082 hooks are enabled the user is responsible to check or trust the shipped actions. 1083 1084 1085 LICENSE 1086 1087 cehgit -- cehtehs personal git assistant 1088 Copyright (C) 2024 Christian Thäter <ct.cehgit@pipapo.org> 1089 1090 This program is free software: you can redistribute it and/or modify 1091 it under the terms of the GNU Affero General Public License as 1092 published by the Free Software Foundation, either version 3 of the 1093 License, or (at your option) any later version. 1094 1095 This program is distributed in the hope that it will be useful, 1096 but WITHOUT ANY WARRANTY; without even the implied warranty of 1097 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1098 GNU Affero General Public License for more details. 1099 1100 You should have received a copy of the GNU Affero General Public License 1101 along with this program. If not, see <https://www.gnu.org/licenses/>. 1102 EOF 1103 exit 0 1104 } 1105 1106 function require () #api [mod..] - loads a module either by path or from MODULE_PATH 1107 { 1108 declare -gA MODULE_LOADED 1109 declare -ga MODULE_PATH=(".cehgit.d") 1110 declare -g MODULE_PREFIX="mod-" 1111 1112 for mod in "$@"; do 1113 if [[ ${MODULE_LOADED["$mod"]:-false} = false ]]; then 1114 if [[ "$mod" = */* && -f "$mod" ]]; then 1115 debug "$mod" 1116 # shellcheck disable=SC1090 # dynamic source 1117 source "$mod" 1118 MODULE_LOADED["$mod"]=true 1119 else 1120 for path in "${MODULE_PATH[@]}"; do 1121 if [[ -f "$path/$MODULE_PREFIX$mod" ]]; then 1122 debug "$path/$MODULE_PREFIX$mod" 1123 # shellcheck disable=SC1090 # dynamic source 1124 source "$path/$MODULE_PREFIX$mod" 1125 MODULE_LOADED["$mod"]=true 1126 break 1127 elif [[ -f "$path/$mod" ]]; then 1128 debug "$path/$mod" 1129 # shellcheck disable=SC1090 # dynamic source 1130 source "$path/$mod" 1131 MODULE_LOADED["$mod"]=true 1132 break 1133 fi 1134 done 1135 fi 1136 if [[ ${MODULE_LOADED["$mod"]:-false} = false ]]; then 1137 die "failed to load $mod" 1138 fi 1139 else 1140 trace "already loaded: $mod" 1141 fi 1142 done 1143 } 1144 1145 function memo #api [-c|-d] [cmd args..].. - memoize the result/stdout/stderr of commands, will always return the same result again 1146 { 1147 # -c clears the cache completely 1148 if [[ $1 == "-c" ]]; then 1149 unset MEMO_RC MEMO_STDOUT MEMO_STDERR 1150 shift 1151 fi 1152 1153 local delete=false 1154 # -d deletes the cache entry 1155 if [[ $1 == "-d" ]]; then 1156 delete=true 1157 shift 1158 fi 1159 1160 declare -gAi MEMO_RC 1161 declare -gA MEMO_STDOUT MEMO_STDERR 1162 1163 # shellcheck disable=SC2155 1164 local key="$(sha256sum <<<"$$""$*")" 1165 key="${key:0:64}" 1166 1167 if [[ $delete = true && -v MEMO_RC["$key"] ]]; then 1168 unset "MEMO_RC[$key]" "MEMO_STDOUT[$key]" "MEMO_STDERR[$key]" 1169 fi 1170 1171 if [[ -v MEMO_RC["$key"] ]]; then 1172 echo -n "${MEMO_STDOUT[$key]}" 1173 echo -n "${MEMO_STDERR[$key]}" 1>&2 1174 return ${MEMO_RC["$key"]} 1175 elif [[ $# -ge 1 ]]; then 1176 local errexit=false 1177 [[ $- == *e* ]] && errexit=true 1178 set +e 1179 touch "/tmp/$key.stdout" "/tmp/$key.stderr" 1180 eval "$*" > >(tee "/tmp/$key.stdout") 2> >(tee "/tmp/$key.stderr" 1>&2) 1181 local rc=$? 1182 [[ $errexit = true ]] && set -e 1183 MEMO_RC["$key"]=$rc 1184 IFS= read -r -d '' MEMO_STDOUT["$key"] < "/tmp/$key.stdout" 1185 IFS= read -r -d '' MEMO_STDERR["$key"] < "/tmp/$key.stderr" 1186 rm "/tmp/$key.stdout" "/tmp/$key.stderr" 1187 return "$rc" 1188 fi 1189 } 1190 1191 function memofn #api <functionnames..> - rewrites a function into a function that uses 'memo' 1192 { 1193 for fn in "$@"; do 1194 # rename the original function as nomemo_* 1195 eval "nomemo_$(declare -f "$fn")" 1196 # create a new function that calls memo with the original function 1197 eval "function $fn () { memo nomemo_$fn \"\$@\" ; }" 1198 done 1199 } 1200 1201 function memo_ok #api [cmd args..] - memoize success but die on error 1202 { 1203 memo "$@" || die "'$*' failed with $?" 1204 } 1205 1206 function source_info # [N] - returns file:line N (or 0) up the bash call stack 1207 { 1208 echo "${BASH_SOURCE[$((${1:-0}+1))]}:${BASH_LINENO[$((${1:-0}))]}:${FUNCNAME[$((${1:-0}+1))]:+${FUNCNAME[$((${1:-0}+1))]}:}" 1209 } 1210 1211 function die #api [message..] - prints 'message' to stderr and exits with failure 1212 { 1213 if [[ $VERBOSITY_LEVEL -gt 0 ]]; then 1214 echo -e "\033[1;91mPANIC:\033[0m $(source_info 1) $*" >&2 1215 fi 1216 exit 1 1217 } 1218 1219 function error #api [message..] - may print a error message to stderr 1220 { 1221 if [[ $VERBOSITY_LEVEL -gt 0 ]]; then 1222 echo -e "\033[1;31mERROR:\033[0m $(source_info 1) $*" >&2 1223 fi 1224 } 1225 1226 function note #api [message..] - may print an notice to stderr 1227 { 1228 if [[ $VERBOSITY_LEVEL -gt 1 ]]; then 1229 echo -e "\033[1;35m NOTE:\033[0m $*" >&2 1230 fi 1231 } 1232 1233 function info #api [message..] - may print an informal message to stderr 1234 { 1235 if [[ $VERBOSITY_LEVEL -gt 2 ]]; then 1236 echo -e "\033[1;34m INFO:\033[0m $*" >&2 1237 fi 1238 } 1239 1240 function debug #api [message..] - may print a debug message to stderr 1241 { 1242 if [[ $VERBOSITY_LEVEL -gt 3 ]]; then 1243 echo -e "\033[1;36mDEBUG:\033[0m $(source_info 1) $*" >&2 1244 fi 1245 } 1246 1247 function trace #api [message] - may prints a trace message to stderr 1248 { 1249 if [[ $VERBOSITY_LEVEL -gt 4 ]]; then 1250 echo -e "\033[1;96mTRACE:\033[0m $(source_info 1) $*" >&2 1251 fi 1252 } 1253 1254 if [[ ${SHTEST_TESTSUITE:-false} = false ]]; then 1255 cehgit "$@" 1256 fi