1  name: Real End-to-End Tests
  2  
  3  # ⚠️ WARNING: This workflow makes real API calls and incurs costs!
  4  # Only runs when manually triggered to prevent accidental charges
  5  
  6  on:
  7    workflow_dispatch:  # Manual trigger only
  8      inputs:
  9        framework:
 10          description: 'Framework to test (⚠️ Will incur API costs!)'
 11          required: true
 12          default: 'none'
 13          type: choice
 14          options:
 15          - none
 16          - autogen
 17          - crewai
 18          - all
 19        confirm_costs:
 20          description: 'I understand this will make real API calls and may incur costs'
 21          required: true
 22          type: boolean
 23          default: false
 24  
 25  jobs:
 26    real-tests:
 27      runs-on: ubuntu-latest
 28      if: ${{ github.event.inputs.confirm_costs == 'true' && github.event.inputs.framework != 'none' }}
 29      timeout-minutes: 15
 30      
 31      strategy:
 32        matrix:
 33          python-version: [3.11]  # Single version to minimize costs
 34      
 35      steps:
 36      - name: 🚨 Cost Warning
 37        env:
 38          INPUT_FRAMEWORK: ${{ github.event.inputs.framework }}
 39          INPUT_CONFIRM_COSTS: ${{ github.event.inputs.confirm_costs }}
 40        run: |
 41          echo "⚠️  WARNING: This workflow will make real API calls!"
 42          echo "💰 This may incur charges on your API accounts"
 43          echo "🎯 Framework: $INPUT_FRAMEWORK"
 44          echo "✅ Cost confirmation: $INPUT_CONFIRM_COSTS"
 45          
 46      - name: Checkout code
 47        uses: actions/checkout@v4
 48          with:
 49            persist-credentials: false
 50          with:
 51            persist-credentials: false
 52  
 53      - name: Set up Python ${{ matrix.python-version }}
 54        uses: actions/setup-python@v5
 55        with:
 56          python-version: ${{ matrix.python-version }}
 57  
 58      - name: Install UV
 59        run: |
 60          curl -LsSf https://astral.sh/uv/install.sh | sh
 61          echo "$HOME/.local/bin" >> $GITHUB_PATH
 62  
 63  
 64      - name: Install dependencies
 65        run: |
 66          cd src/praisonai
 67          uv pip install --system ."[ui,gradio,api,agentops,google,openai,anthropic,cohere,chat,code,realtime,call,crewai,autogen]"
 68          uv pip install --system pytest pytest-asyncio pytest-cov
 69          # Install knowledge dependencies from praisonai-agents
 70          uv pip install --system "praisonaiagents[knowledge]"
 71  
 72      - name: Set environment variables
 73        run: |
 74          echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> $GITHUB_ENV
 75          echo "ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}" >> $GITHUB_ENV
 76          echo "GOOGLE_API_KEY=${{ secrets.GOOGLE_API_KEY }}" >> $GITHUB_ENV
 77  
 78      - name: Verify API Keys
 79        run: |
 80          if [ -z "${{ secrets.OPENAI_API_KEY }}" ]; then
 81            echo "❌ OPENAI_API_KEY not set in secrets"
 82            echo "🔧 Add your API key to repository secrets"
 83            exit 1
 84          fi
 85          echo "✅ API keys configured"
 86  
 87      - name: Run Real AutoGen Tests
 88        if: ${{ github.event.inputs.framework == 'autogen' || github.event.inputs.framework == 'all' }}
 89        run: |
 90          echo "🤖 Running REAL AutoGen tests (⚠️  API costs may apply)"
 91          cd src/praisonai && python -m pytest tests/e2e/autogen/ -v -m real --tb=short
 92        continue-on-error: false
 93  
 94      - name: Run Real CrewAI Tests
 95        if: ${{ github.event.inputs.framework == 'crewai' || github.event.inputs.framework == 'all' }}
 96        run: |
 97          echo "⛵ Running REAL CrewAI tests (⚠️  API costs may apply)"
 98          cd src/praisonai && python -m pytest tests/e2e/crewai/ -v -m real --tb=short
 99        continue-on-error: false
100  
101      - name: Generate Real Test Report
102        if: always()
103        env:
104          INPUT_FRAMEWORK: ${{ github.event.inputs.framework }}
105        run: |
106          echo "# 🔥 Real End-to-End Test Report" > real_test_report.md
107          echo "" >> real_test_report.md
108          echo "⚠️  **WARNING: This report represents tests that made real API calls**" >> real_test_report.md
109          echo "" >> real_test_report.md
110          echo "**Framework Tested:** $INPUT_FRAMEWORK" >> real_test_report.md
111          echo "**Python Version:** ${{ matrix.python-version }}" >> real_test_report.md
112          echo "**Date:** $(date -u)" >> real_test_report.md
113          echo "**Triggered by:** $GITHUB_ACTOR" >> real_test_report.md
114          echo "" >> real_test_report.md
115          
116          echo "## 🧪 Test Results" >> real_test_report.md
117          echo "" >> real_test_report.md
118          
119          if [ "$INPUT_FRAMEWORK" == "autogen" ] || [ "$INPUT_FRAMEWORK" == "all" ]; then
120            echo "### AutoGen Real Tests:" >> real_test_report.md
121            echo "- Environment verification" >> real_test_report.md
122            echo "- Agent creation with real API calls" >> real_test_report.md
123            echo "- Configuration validation" >> real_test_report.md
124            echo "" >> real_test_report.md
125          fi
126          
127          if [ "$INPUT_FRAMEWORK" == "crewai" ] || [ "$INPUT_FRAMEWORK" == "all" ]; then
128            echo "### CrewAI Real Tests:" >> real_test_report.md
129            echo "- Environment verification" >> real_test_report.md
130            echo "- Crew creation with real API calls" >> real_test_report.md
131            echo "- Multi-agent setup validation" >> real_test_report.md
132            echo "" >> real_test_report.md
133          fi
134          
135          echo "## 💰 Cost Considerations" >> real_test_report.md
136          echo "- These tests made actual API calls to LLM providers" >> real_test_report.md
137          echo "- Costs depend on your API pricing tier" >> real_test_report.md
138          echo "- Tests are designed to be minimal to reduce costs" >> real_test_report.md
139          echo "- Check your API provider dashboard for actual usage" >> real_test_report.md
140          
141          echo "## 📋 Next Steps" >> real_test_report.md
142          echo "- Review test results for any failures" >> real_test_report.md
143          echo "- Check API usage in your provider dashboard" >> real_test_report.md
144          echo "- Use mock tests (tests/integration/) for routine testing" >> real_test_report.md
145  
146      - name: Upload Real Test Results
147        uses: actions/upload-artifact@v4
148        if: always()
149        with:
150          name: real-test-results-${{ github.event.inputs.framework }}-python-${{ matrix.python-version }}
151          path: |
152            real_test_report.md
153          retention-days: 30
154  
155    # Safety job that runs when costs not confirmed
156    safety-check:
157      runs-on: ubuntu-latest
158      if: ${{ github.event.inputs.confirm_costs != 'true' || github.event.inputs.framework == 'none' }}
159      
160      steps:
161      - name: 🛡️ Safety Check Failed
162        env:
163          INPUT_FRAMEWORK: ${{ github.event.inputs.framework }}
164          INPUT_CONFIRM_COSTS: ${{ github.event.inputs.confirm_costs }}
165        run: |
166          echo "🚨 Real tests not executed due to safety checks:"
167          echo ""
168          echo "✅ Costs confirmed: $INPUT_CONFIRM_COSTS"
169          echo "✅ Framework selected: $INPUT_FRAMEWORK"
170          echo ""
171          echo "To run real tests:"
172          echo "1. Select a framework (autogen, crewai, or all)"
173          echo "2. Check 'I understand this will make real API calls and may incur costs'"
174          echo "3. Ensure API keys are set in repository secrets"
175          echo ""
176          echo "💡 For cost-free testing, use mock tests instead:"
177          echo "   - Run 'python -m pytest tests/integration/' locally"
178          echo "   - Or trigger other workflows that use mock tests"
179          
180          exit 1