1  #!/bin/bash
  2  set -euo pipefail
  3  
  4  #
  5  # check-requirements.sh checks all requirements files for each top-level
  6  # convert*.py script.
  7  #
  8  # WARNING: This is quite IO intensive, because a fresh venv is set up for every
  9  # python script. As of 2023-12-22, this writes ~2.7GB of data. An adequately
 10  # sized tmpfs /tmp or ramdisk is recommended if running this frequently.
 11  #
 12  # usage:    check-requirements.sh [<working_dir>]
 13  #           check-requirements.sh nocleanup [<working_dir>]
 14  #
 15  # where:
 16  #           - <working_dir> is a directory that can be used as the base for
 17  #               setting up the venvs. Defaults to `/tmp`.
 18  #           - 'nocleanup' as the first argument will disable automatic cleanup
 19  #               of the files created by this script.
 20  #
 21  # requires:
 22  #           - bash >= 3.2.57
 23  #           - shellcheck
 24  #
 25  # For each script, it creates a fresh venv, `pip install`s the requirements, and
 26  # finally imports the python script to check for `ImportError`.
 27  #
 28  
 29  log() {
 30      local level=$1 msg=$2
 31      printf >&2 '%s: %s\n' "$level" "$msg"
 32  }
 33  
 34  debug() {
 35      log DEBUG "$@"
 36  }
 37  
 38  info() {
 39      log INFO "$@"
 40  }
 41  
 42  fatal() {
 43      log FATAL "$@"
 44      exit 1
 45  }
 46  
 47  cleanup() {
 48      if [[ -n ${workdir+x} && -d $workdir && -w $workdir ]]; then
 49          info "Removing $workdir"
 50          local count=0
 51          rm -rfv -- "$workdir" | while read -r; do
 52              if (( count++ > 750 )); then
 53                  printf .
 54                  count=0
 55              fi
 56          done
 57          printf '\n'
 58          info "Removed $workdir"
 59      fi
 60  }
 61  
 62  do_cleanup=1
 63  if [[ ${1-} == nocleanup ]]; then
 64      do_cleanup=0; shift
 65  fi
 66  
 67  if (( do_cleanup )); then
 68      trap exit INT TERM
 69      trap cleanup EXIT
 70  fi
 71  
 72  this=$(realpath -- "$0"); readonly this
 73  cd "$(dirname "$this")/.." # PWD should stay in llama.cpp project directory
 74  
 75  shellcheck "$this"
 76  
 77  readonly reqs_dir=requirements
 78  
 79  if [[ ${1+x} ]]; then
 80      tmp_dir=$(realpath -- "$1")
 81      if [[ ! ( -d $tmp_dir && -w $tmp_dir ) ]]; then
 82          fatal "$tmp_dir is not a writable directory"
 83      fi
 84  else
 85      tmp_dir=/tmp
 86  fi
 87  
 88  workdir=$(mktemp -d "$tmp_dir/check-requirements.XXXX"); readonly workdir
 89  info "Working directory: $workdir"
 90  
 91  check_requirements() {
 92      local reqs=$1
 93  
 94      info "$reqs: beginning check"
 95      pip --disable-pip-version-check install -qr "$reqs"
 96      info "$reqs: OK"
 97  }
 98  
 99  check_convert_script() {
100      local py=$1             # e.g. ./convert_hf_to_gguf.py
101      local pyname=${py##*/}  # e.g. convert_hf_to_gguf.py
102      pyname=${pyname%.py}    # e.g. convert_hf_to_gguf
103  
104      info "$py: beginning check"
105  
106      local reqs="$reqs_dir/requirements-$pyname.txt"
107      if [[ ! -r $reqs ]]; then
108          fatal "$py missing requirements. Expected: $reqs"
109      fi
110  
111      # Check that all sub-requirements are added to top-level requirements.txt
112      if ! grep -qF "$reqs" requirements.txt; then
113          fatal "$reqs needs to be added to requirements.txt"
114      fi
115  
116      local venv="$workdir/$pyname-venv"
117      python3 -m venv "$venv"
118  
119      (
120          # shellcheck source=/dev/null
121          source "$venv/bin/activate"
122  
123          check_requirements "$reqs"
124  
125          python - "$py" "$pyname" <<'EOF'
126  import sys
127  from importlib.machinery import SourceFileLoader
128  py, pyname = sys.argv[1:]
129  SourceFileLoader(pyname, py).load_module()
130  EOF
131      )
132  
133      if (( do_cleanup )); then
134          rm -rf -- "$venv"
135      fi
136  
137      info "$py: imports OK"
138  }
139  
140  readonly ignore_eq_eq='check_requirements: ignore "=="'
141  
142  for req in */**/requirements*.txt; do
143      # Make sure exact release versions aren't being pinned in the requirements
144      # Filters out the ignore string
145      if grep -vF "$ignore_eq_eq" "$req" | grep -q '=='; then
146          tab=$'\t'
147          cat >&2 <<EOF
148  FATAL: Avoid pinning exact package versions. Use '~=' instead.
149  You can suppress this error by appending the following to the line:
150  $tab# $ignore_eq_eq
151  EOF
152          exit 1
153      fi
154  done
155  
156  all_venv="$workdir/all-venv"
157  python3 -m venv "$all_venv"
158  
159  (
160      # shellcheck source=/dev/null
161      source "$all_venv/bin/activate"
162      check_requirements requirements.txt
163  )
164  
165  if (( do_cleanup )); then
166      rm -rf -- "$all_venv"
167  fi
168  
169  check_convert_script examples/convert_legacy_llama.py
170  for py in convert_*.py; do
171      # skip convert_hf_to_gguf_update.py
172      # TODO: the check is failing for some reason:
173      #       https://github.com/ggerganov/llama.cpp/actions/runs/8875330981/job/24364557177?pr=6920
174      [[ $py == convert_hf_to_gguf_update.py ]] && continue
175  
176      check_convert_script "$py"
177  done
178  
179  info 'Done! No issues found.'