1  #!/usr/bin/env bash
  2  
  3  MLFLOW_HOME="$(pwd)"
  4  directory="$MLFLOW_HOME/.venvs/mlflow-dev"
  5  REPO_ROOT=$(git rev-parse --show-toplevel)
  6  rd="$REPO_ROOT/requirements"
  7  VENV_DIR="$directory/bin/activate"
  8  # Progress file to resume the script from where it exited previously
  9  PROGRESS_FILE="$MLFLOW_HOME/dev-env-setup-progress"
 10  
 11  showHelp() {
 12  cat << EOF
 13  Usage: ./install-dev-env.sh [-d] [directory to install virtual environment] [-v] [-q] [-f] [-o] [override python version]
 14  Development environment setup script for Python in linux-based Operating Systems (including OSX).
 15  Note: this script will not work on Windows or MacOS M1 arm64 chipsets.
 16  
 17  This script will:
 18  
 19    - Install pyenv if not installed
 20    - Retrieve the appropriate Python version (minimum required) for compatibility support
 21    - Check if the virtual environment already exists
 22      - If it does, prompt for replacement
 23        - If replacing, delete old virtual environment.
 24    - Create a virtual environment using the minimum required Python version based on previous step logic
 25    - Activate the environment
 26    - Install required dependencies for the dev environment
 27  
 28    Example usage:
 29  
 30    From root of MLflow repository on local with a destination virtualenv path of <REPO_ROOT>/.venvs/mlflow-dev:
 31  
 32    dev/dev-env-setup.sh -d $(pwd)/.venvs/mlflow-dev
 33  
 34    Note: it is recommended to preface virtualenv locations with a directory name prefaced by '.' (i.e., ".venvs").
 35  
 36    The default environment setup is for basic functionality, installing the minimum development requirements dependencies.
 37    To install the full development environment that supports working on all flavors and running all tests locally, set
 38    the flag '-f' or '--full'
 39  
 40  -h,     --help        Display help
 41  
 42  -d,     --directory   The path to install the virtual environment into
 43  
 44  -f,     --full        Whether to install all dev requirements (Default: false)
 45  
 46  -q,     --quiet       Whether to have pip install in quiet mode (Default: false)
 47  
 48  -o,     --override    Override the python version
 49  
 50  -c,     --clean       Discard the previous installation progress and restart the setup from scratch
 51  
 52  EOF
 53  }
 54  
 55  while :
 56  do
 57    case "$1" in
 58      -d | --directory)
 59        directory="$2"
 60        shift 2
 61        ;;
 62      -f | --full)
 63        full="full"
 64        shift
 65        ;;
 66      -q | --quiet)
 67        quiet="quiet"
 68        shift
 69        ;;
 70      -o | --override)
 71        override_py_ver="$2"
 72        shift 2
 73        ;;
 74      -h | --help)
 75        showHelp
 76        exit 0
 77        ;;
 78      -c | --clean)
 79        rm $PROGRESS_FILE
 80        shift
 81        ;;
 82      --)
 83        shift
 84        break
 85        ;;
 86      -*)
 87        echo "Error: unknown option: $1" >&2
 88        exit 1
 89        ;;
 90      *)
 91        break
 92        ;;
 93    esac
 94  done
 95  
 96  if [[ -n "$verbose" ]]; then
 97    set -exv
 98  fi
 99  
100  # Acquire the OS for this environment
101  case "$(uname -s)" in
102    Darwin*)                       machine=mac;;
103    Linux*)                        machine=linux;;
104    *)                             machine=unknown;;
105  esac
106  
107  load_progress() {
108    if [[ ! -f "$PROGRESS_FILE" ]]; then
109      echo "0" > "$PROGRESS_FILE"
110    fi
111    cat "$PROGRESS_FILE"
112  }
113  
114  PROGRESS=$(load_progress)
115  
116  save_progress() {
117    echo "$1" > "$PROGRESS_FILE"
118    PROGRESS=$(load_progress)
119  }
120  
121  quiet_command(){
122    echo $( [[ -n $quiet ]] && printf %s '-q' )
123  }
124  
125  minor_to_micro() {
126    case $1 in
127      "3.10") echo "3.10.13" ;;
128    esac
129  }
130  
131  # Check if brew is installed and install it if it isn't present
132  # Note: if xcode isn't installed, this will fail.
133  # $1: name of package that requires brew
134  check_and_install_brew() {
135    # command -v returns exit code 1 if brew does not exist, which directly terminates our test script.
136    # Appending `|| true` to ignore the exit code.
137    if [ -z "$(command -v brew || true)" ]; then
138      echo "Homebrew is required to install $1 on MacOS. Installing in your home directory."
139      bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
140    fi
141    echo "Updating brew..."
142    brew update
143  }
144  
145  # Compare two version numbers
146  # Usage: version_gt version1 version2
147  # Returns 0 (true) if version1 > version2, 1 (false) otherwise
148  version_gt() {
149      IFS='.' read -ra VER1 <<< "$1"
150      IFS='.' read -ra VER2 <<< "$2"
151  
152      # Compare each segment of the version numbers
153      for (( i=0; i<"${#VER1[@]}"; i++ )); do
154          # If VER2 is shorter and we haven't found a difference yet, VER1 is greater
155          if [[ -z ${VER2[i]} ]]; then
156              return 0
157          fi
158  
159          # If some segments are not equal, return their comparison result
160          if (( ${VER1[i]} > ${VER2[i]} )); then
161              return 0
162          elif (( ${VER1[i]} < ${VER2[i]} )); then
163              return 1
164          fi
165      done
166  
167      # If all common length segments are same, the one with more segments is greater
168      return $(( ${#VER1[@]} <= ${#VER2[@]} ))
169  }
170  
171  # Check if pyenv is installed and offer to install it if not present
172  check_and_install_pyenv() {
173    # command -v returns exit code 1 if pyenv does not exist, which directly terminates our test script.
174    # Appending `|| true` to ignore the exit code.
175    pyenv_exist=$(command -v pyenv || true)
176    if [ -z "$pyenv_exist" ]; then
177      if [ -z "$GITHUB_ACTIONS" ]; then
178        read -p "pyenv is required to be installed to manage python versions. Would you like to install it? $(tput bold)(y/n)$(tput sgr0): " -n 1 -r
179        echo
180      fi
181      if [[ $REPLY =~ ^[Yy]$ || -n "$GITHUB_ACTIONS" ]]; then
182        if [[ "$machine" == mac ]]; then
183          check_and_install_brew "pyenv"
184          echo "Installing pyenv..."
185          echo "Note: this will probably take a considerable amount of time."
186          brew install pyenv
187          brew install openssl readline sqlite3 xz zlib libomp
188        elif [[ "$machine" == linux ]]; then
189          sudo apt-get update -y
190          sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \
191            libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
192            libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
193          # Install pyenv from source
194          git clone --depth 1 https://github.com/pyenv/pyenv.git "$HOME/.pyenv"
195          PYENV_ROOT="$HOME/.pyenv"
196          PYENV_BIN="$PYENV_ROOT/bin"
197          PATH="$PYENV_BIN:$PATH"
198          if [ -n "$GITHUB_ACTIONS" ]; then
199            echo "$PYENV_BIN" >>"$GITHUB_PATH"
200            echo "PYENV_ROOT=$PYENV_ROOT" >>"$GITHUB_ENV"
201          fi
202        else
203          echo "Unsupported operating system environment: $machine. This setup script only supports MacOS and Linux. For other operating systems, please follow the manual setup instruction here: https://github.com/mlflow/mlflow/blob/master/CONTRIBUTING.md#manual-python-development-environment-configuration "
204          exit 1
205        fi
206      else
207        PYENV_README=https://github.com/pyenv/pyenv/blob/master/README.md
208        echo "pyenv is required to use this environment setup script. Please install by following instructions here: $PYENV_README"
209        exit 1
210      fi
211    fi
212  }
213  
214  check_and_install_min_py_version() {
215    # Get the minimum supported version for development purposes
216    min_py_version="3.10"
217  
218    echo "The minimum version of Python to ensure backwards compatibility for MLflow development is: $(
219      tput bold
220      tput setaf 3
221    )$min_py_version$(tput sgr0)"
222  
223    if [[ -n "$override_py_ver" ]]; then
224      version_levels=$(grep -o '\.' <<<"$override_py_ver" | wc -l)
225      if [[ $version_levels -eq 1 ]]; then
226        PY_INSTALL_VERSION=$(minor_to_micro $override_py_ver)
227      elif [[ $version_levels -eq 2 ]]; then
228        PY_INSTALL_VERSION=$override_py_ver
229      else
230        echo "You must supply a python override version with either minor (e.g., '3.10') or micro (e.g., '3.10.13'). '$override_py_ver' is invalid."
231        exit 1
232      fi
233    else
234      PY_INSTALL_VERSION=$(minor_to_micro $min_py_version)
235    fi
236  
237    echo "$(tput setaf 2) Installing Python version $(tput bold)$PY_INSTALL_VERSION$(tput sgr0)"
238  
239    # Install the Python version if it cannot be found
240    pyenv install -s "$PY_INSTALL_VERSION"
241    pyenv local "$PY_INSTALL_VERSION"
242    uv pip install --system $(quiet_command) --upgrade pip
243    uv pip install --system $(quiet_command) virtualenv
244  }
245  
246  # Check if the virtualenv already exists at the specified path
247  create_virtualenv() {
248    if [[ -d "$directory" ]]; then
249      if [ -z "$GITHUB_ACTIONS" ]; then
250        read -p "A virtual environment is already located at $directory. Do you wish to replace it? $(
251          tput bold
252          tput setaf 2
253        )(y/n) $(tput sgr0)" -n 1 -r
254        echo
255      fi
256      if [[ $REPLY =~ ^[Yy]$ || -n "$GITHUB_ACTIONS" ]]; then
257        echo "Replacing Virtual environment in '$directory'. Installing new instance."
258        pyenv exec virtualenv --clear "$directory"
259      fi
260    else
261      # Create a virtual environment with the specified Python version
262      pyenv exec virtualenv --python "$PY_INSTALL_VERSION" "$directory"
263    fi
264  
265    # Activate the virtual environment
266    # shellcheck disable=SC1090
267    source "$VENV_DIR"
268  
269    echo "$(tput setaf 2)Current Python version: $(tput bold)$(python --version)$(tput sgr0)"
270    echo "$(tput setaf 3)Activated environment is located: $(tput bold) $directory/bin/activate$(tput sgr0)"
271  }
272  
273  # Install mlflow dev version and required dependencies
274  install_mlflow_and_dependencies() {
275    # Install current checked out version of mlflow (local)
276    uv pip install --system -e .[extras]
277  
278    echo "Installing pip dependencies for development environment."
279    if [[ -n "$full" ]]; then
280      # Install dev requirements
281      uv pip install --system -r "$rd/dev-requirements.txt"
282      # Install test plugin
283      uv pip install --system -e "$MLFLOW_HOME/tests/resources/mlflow-test-plugin"
284    else
285      files=("$rd/test-requirements.txt" "$rd/lint-requirements.txt" "$rd/doc-requirements.txt")
286      for r in "${files[@]}"; do
287        uv pip install --system -r "$r"
288      done
289    fi
290    echo "Finished installing pip dependencies."
291  
292    echo "$(
293      tput setaf 2
294      tput smul
295    )Python packages that have been installed:$(tput rmul)"
296    echo "$(pip freeze)$(tput sgr0)"
297  }
298  
299  check_docker() {
300     command -v docker >/dev/null 2>&1 || echo "$(
301      tput bold
302      tput setaf 1
303    )A docker installation cannot be found. Docker is optional but you may need it for developing some features and running all tests locally. $(tput sgr0)"
304  }
305  
306  # Check if pandoc with required version is installed and offer to install it if not present
307  check_and_install_pandoc() {
308    pandoc_version=$(pandoc --version | grep "pandoc" | awk '{print $2}')
309    if [[ -z "$pandoc_version" ]] || ! version_gt "$pandoc_version" "2.2.1"; then
310      if [ -z "$GITHUB_ACTIONS" ]; then
311        read -p "Pandoc version 2.2.1 or above is an optional requirement for compiling documentation. Would you like to install it? $(tput bold)(y/n)$(tput sgr0): " -n 1 -r
312        echo
313      fi
314  
315      if [[ $REPLY =~ ^[Yy]$ || -n "$GITHUB_ACTIONS" ]]; then
316        echo "Installing Pandoc..."
317        if [[ "$machine" == mac ]]; then
318          check_and_install_brew "pandoc"
319          brew install pandoc
320        elif [[ "$machine" == linux ]]; then
321          # Install pandoc via deb package as `apt-get` gives too old version
322          TEMP_DEB=$(mktemp) &&
323            wget --directory-prefix $TEMP_DEB https://github.com/jgm/pandoc/releases/download/2.16.2/pandoc-2.16.2-1-amd64.deb &&
324            sudo dpkg --install $(find $TEMP_DEB -name '*.deb') &&
325            rm -rf $TEMP_DEB
326        else
327          echo "Unsupported operating system environment: $machine. This setup script only supports MacOS and Linux. For other operating systems, please follow the manual setup instruction here: https://github.com/mlflow/mlflow/blob/master/CONTRIBUTING.md#manual-python-development-environment-configuration "
328          exit 1
329        fi
330      fi
331    fi
332  }
333  
334  # Set up pre-commit hooks and git environment configuration for proper signing of commits
335  set_pre_commit_and_git_signoff() {
336    git_user=$(git config user.name)
337    git_email=$(git config user.email)
338  
339    if [[ -z "$git_email" || -z "$git_user" ]]; then
340      read -p "Your git environment is not setup to automatically sign your commits. Would you like to configure it? $(
341        tput bold
342        tput setaf 2
343      )(y/n): $(tput sgr0)" -n 1 -r
344      echo
345      if [[ $REPLY =~ ^[Yy]$ ]]; then
346        read -p "Enter the user name you would like to have associated with your commit signature: " -r git_user_name
347        echo
348        git config --global user.name "$git_user_name"
349        echo "Git user name set as: $(git config user.name)"
350        read -p "Enter your email address for your commit signature: " -r git_user_email
351        git config --global user.email "$git_user_email"
352        echo "Git user email set as: $(git config user.email)"
353      else
354        echo "Failing to set git 'user.name' and 'user.email' will result in unsigned commits. Ensure that you sign commits manually for CI checks to pass."
355      fi
356    fi
357  
358    # Set up pre-commit hooks
359    pre-commit install --install-hooks
360  }
361  
362  # Execute mandatory setups with strict error handling
363  set +xv && set -e
364  # Mandatory setups
365  if [[ "$PROGRESS" -eq "0" ]]; then
366    check_and_install_pyenv
367    save_progress 1
368  fi
369  if [[ "$PROGRESS" -eq "1" ]]; then
370    check_and_install_min_py_version
371    save_progress 2
372  fi
373  if [[ "$PROGRESS" -eq "2" ]]; then
374    create_virtualenv
375    save_progress 3
376  fi
377  if [[ "$PROGRESS" -eq "3" ]]; then
378    install_mlflow_and_dependencies
379    save_progress 4
380  fi
381  if [[ "$PROGRESS" -eq "4" ]]; then
382    set_pre_commit_and_git_signoff
383    save_progress 5
384  fi
385  if [[ "$PROGRESS" -eq "5" ]]; then
386    # Clear progress file if all mandatory steps are executed successfully
387    rm $PROGRESS_FILE
388  fi
389  
390  # Execute optional setups without strict error handling
391  set +exv
392  # Optional setups
393  check_and_install_pandoc
394  check_docker
395  
396  echo "$(tput setaf 2)Your MLflow development environment can be activated by running: $(tput bold)source $VENV_DIR$(tput sgr0)"