tests.yml
1 # If you change this name also do it in ci_metrics.yml 2 name: Tests 3 4 # The workflow will always run, but the actual tests will only execute when: 5 # - The workflow is triggered manually. 6 # - The push is to main or a release branch. 7 # - There are changes to relevant files on a pull request. 8 # Note: If no conditions are met, the workflow will complete successfully without running tests 9 # to satisfy Branch Protection rules. 10 11 on: 12 workflow_dispatch: # Activate this workflow manually 13 push: 14 branches: 15 - main 16 # release branches have the form v1.9.x 17 - "v[0-9].*[0-9].x" 18 # when we push, we do not need to satisfy Branch Protection rules, so we can ignore PRs that just change docs 19 paths-ignore: 20 - 'docs/**' 21 - 'docs-website/**' 22 pull_request: 23 types: 24 - opened 25 - reopened 26 - synchronize 27 28 env: 29 OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} 30 CORE_AZURE_CS_ENDPOINT: ${{ secrets.CORE_AZURE_CS_ENDPOINT }} 31 CORE_AZURE_CS_API_KEY: ${{ secrets.CORE_AZURE_CS_API_KEY }} 32 AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} 33 AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} 34 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 35 HF_API_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }} 36 PYTHON_VERSION: "3.10" 37 HATCH_VERSION: "1.16.5" 38 39 jobs: 40 check-if-changed: 41 # This job checks if the relevant files have been changed. 42 # We check for changes in the check-if-changed job instead of using paths/paths-ignore at workflow level. 43 # This ensures the "Mark tests as completed" job always runs, which is required by Branch Protection rules. 44 name: Check if changed 45 runs-on: ubuntu-slim 46 permissions: 47 pull-requests: read 48 # Specifying outputs is not needed to make the job work, but only to comply with actionlint 49 outputs: 50 changes: ${{ steps.changes.outputs.changes }} 51 steps: 52 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 53 - name: Check for changed code 54 id: changes 55 uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 56 with: 57 filters: | 58 changes: 59 - "haystack/**/*.py" 60 - "test/**/*.py" 61 - "pyproject.toml" 62 - ".github/utils/*.py" 63 - "scripts/*.py" 64 65 format: 66 needs: check-if-changed 67 # Run tests if: manual trigger, push to main/release, or relevant files changed 68 if: | 69 github.event_name == 'workflow_dispatch' || 70 github.event_name == 'push' || 71 (needs.check-if-changed.outputs.changes == 'true') 72 runs-on: ubuntu-latest 73 steps: 74 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 75 76 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 77 with: 78 python-version: "${{ env.PYTHON_VERSION }}" 79 80 - name: Install Hatch 81 run: pip install hatch==${{ env.HATCH_VERSION }} 82 83 - name: Ruff - check format and linting 84 run: hatch run fmt-check 85 86 - name: Check presence of license header 87 run: docker run --rm -v "$(pwd):/github/workspace" ghcr.io/korandoru/hawkeye check 88 89 check-imports: 90 needs: format 91 runs-on: ubuntu-slim 92 steps: 93 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 94 95 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 96 with: 97 python-version: "${{ env.PYTHON_VERSION }}" 98 99 - name: Install Hatch 100 run: pip install hatch==${{ env.HATCH_VERSION }} 101 102 - name: Check imports 103 run: hatch run python .github/utils/check_imports.py 104 105 unit-tests: 106 name: Unit / ${{ matrix.os }} 107 needs: format 108 timeout-minutes: 30 109 strategy: 110 fail-fast: false 111 matrix: 112 os: 113 - ubuntu-latest 114 - windows-latest 115 - macos-latest 116 runs-on: ${{ matrix.os }} 117 steps: 118 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 119 120 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 121 with: 122 python-version: "${{ env.PYTHON_VERSION }}" 123 124 - name: Install Hatch 125 id: hatch 126 shell: bash 127 run: | 128 pip install hatch==${{ env.HATCH_VERSION }} 129 echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT" 130 131 - name: Run 132 run: hatch run test:unit 133 134 - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 135 id: cache 136 if: matrix.os == 'macos-latest' 137 with: 138 path: ${{ steps.hatch.outputs.env }} 139 key: ${{ runner.os }}-${{ github.sha }} 140 141 - name: Coveralls 142 # We upload only coverage for ubuntu as handling both os 143 # complicates the workflow too much for little to no gain 144 if: matrix.os == 'ubuntu-latest' 145 uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e # v2.3.7 146 continue-on-error: true 147 with: 148 path-to-lcov: coverage.xml 149 150 mypy: 151 needs: unit-tests 152 runs-on: ubuntu-latest 153 steps: 154 - name: Checkout 155 uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 156 with: 157 # With the default value of 1, there are corner cases where tj-actions/changed-files 158 # fails with a `no merge base` error 159 fetch-depth: 0 160 - name: Get changed files 161 id: files 162 uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 163 with: 164 files: | 165 **/*.py 166 pyproject.toml 167 files_ignore: | 168 test/** 169 .github/** 170 scripts/** 171 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 172 if: steps.files.outputs.any_changed == 'true' 173 with: 174 python-version: "${{ env.PYTHON_VERSION }}" 175 176 - name: Install Hatch 177 id: hatch 178 if: steps.files.outputs.any_changed == 'true' 179 run: | 180 pip install hatch==${{ env.HATCH_VERSION }} 181 echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT" 182 183 - name: Mypy 184 if: steps.files.outputs.any_changed == 'true' 185 run: | 186 mkdir .mypy_cache 187 hatch run test:types 188 189 integration-tests-linux: 190 name: Integration / ubuntu-latest 191 needs: unit-tests 192 runs-on: ubuntu-latest 193 timeout-minutes: 30 194 steps: 195 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 196 197 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 198 with: 199 python-version: "${{ env.PYTHON_VERSION }}" 200 201 - name: Install Hatch 202 id: hatch 203 shell: bash 204 run: | 205 pip install hatch==${{ env.HATCH_VERSION }} 206 echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT" 207 208 209 - name: Run 210 run: hatch run test:integration-only-fast 211 212 integration-tests-macos: 213 name: Integration / macos-latest 214 needs: unit-tests 215 runs-on: macos-latest 216 timeout-minutes: 30 217 env: 218 HAYSTACK_MPS_ENABLED: false 219 220 steps: 221 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 222 223 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 224 with: 225 python-version: "${{ env.PYTHON_VERSION }}" 226 227 - name: Install Hatch 228 id: hatch 229 shell: bash 230 run: | 231 pip install hatch==${{ env.HATCH_VERSION }} 232 echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT" 233 234 - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 235 id: cache 236 with: 237 path: ${{ steps.hatch.outputs.env }} 238 key: ${{ runner.os }}-${{ github.sha }} 239 240 241 - name: Run 242 run: hatch run test:integration-only-fast 243 244 integration-tests-windows: 245 name: Integration / windows-latest 246 needs: unit-tests 247 runs-on: windows-latest 248 timeout-minutes: 30 249 env: 250 HAYSTACK_XPU_ENABLED: false 251 252 steps: 253 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 254 255 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 256 with: 257 python-version: "${{ env.PYTHON_VERSION }}" 258 259 - name: Install Hatch 260 id: hatch 261 shell: bash 262 run: | 263 pip install hatch==${{ env.HATCH_VERSION }} 264 echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT" 265 266 - name: Run 267 run: hatch run test:integration-only-fast 268 269 notify-slack-on-failure: 270 if: failure() && github.ref_name == 'main' 271 needs: 272 - check-imports 273 - mypy 274 - integration-tests-linux 275 - integration-tests-macos 276 - integration-tests-windows 277 runs-on: ubuntu-slim 278 steps: 279 - uses: deepset-ai/notify-slack-action@3cda73b77a148f16f703274198e7771340cf862b # v1 280 with: 281 slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }} 282 283 tests-completed: 284 # This job always runs and succeeds if all tests succeed or are skipped. It is required by Branch Protection rules. 285 name: Mark tests as completed 286 runs-on: ubuntu-slim 287 if: ${{ always() && !cancelled() }} 288 needs: 289 - check-imports 290 - mypy 291 - integration-tests-linux 292 - integration-tests-macos 293 - integration-tests-windows 294 295 steps: 296 - name: Mark tests as completed 297 run: | 298 if [ "${{ needs.check-imports.result }}" = "failure" ] || 299 [ "${{ needs.mypy.result }}" = "failure" ] || 300 [ "${{ needs.integration-tests-linux.result }}" = "failure" ] || 301 [ "${{ needs.integration-tests-macos.result }}" = "failure" ] || 302 [ "${{ needs.integration-tests-windows.result }}" = "failure" ]; then 303 echo "Tests failed!" 304 exit 1 305 else 306 echo "Tests completed!" 307 fi