lm-evaluation-harness/.github/workflows/unit_tests.yml

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
# just comment out unwanted steps to turn off the test.
name: Unit Tests

on:
  push:
    branches:
      - 'main'
  pull_request:
    branches:
      - 'main'
  workflow_dispatch:
# Jobs run concurrently and steps run sequentially within a job.
# jobs: linter and cpu_tests. Add more jobs/steps as required.
jobs:
  linter:
    name: Linters
    runs-on: ubuntu-latest
    timeout-minutes: 5

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
      - name: Set up Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
          cache: pip
          cache-dependency-path: pyproject.toml
      - name: Pre-Commit
        env:
          SKIP: "no-commit-to-branch,mypy"
        uses: pre-commit/action@v3.0.1
  # Job 2
  testcpu:
    name: CPU Tests
    runs-on: ubuntu-latest
    strategy:
      fail-fast: true
      matrix:
        python-version: ["3.9", "3.10", "3.11"]
    timeout-minutes: 30
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
          cache-dependency-path: pyproject.toml

      # Cache HuggingFace cache directory for CPU tests
      - name: Cache HuggingFace cache (CPU tests)
        uses: actions/cache@v3
        id: cache-hf-cpu
        with:
          path: ~/.cache/huggingface
          key: ${{ runner.os }}-hf-cache-cpu
          restore-keys: |
            ${{ runner.os }}-hf-cache-cpu

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -e '.[dev]' --extra-index-url https://download.pytorch.org/whl/cpu
          pip install hf_xet

      - name: Test with pytest
        run: python -m pytest --showlocals -s -vv -n=auto --ignore=tests/models/test_neuralmagic.py --ignore=tests/models/test_openvino.py --ignore=tests/models/test_hf_steered.py
        continue-on-error: true  # Continue workflow even if tests fail

      # Save test artifacts
      - name: Archive test artifacts
        uses: actions/upload-artifact@v4
        with:
          name: output_testcpu${{ matrix.python-version }}
          path: |
            test_logs/*

#  testmodels:
#    name: External LM Tests
#    runs-on: ubuntu-latest
#    timeout-minutes: 30
#    steps:
#      - name: Checkout Code
#        uses: actions/checkout@v4
#      - name: Set up Python 3.9
#        uses: actions/setup-python@v5
#        with:
#          python-version: 3.9
#          cache: pip
#          cache-dependency-path: pyproject.toml
#
#      # Cache HuggingFace cache directory for External LM tests
#      - name: Cache HuggingFace cache (External LM tests)
#        uses: actions/cache@v3
#        id: cache-hf-lm
#        with:
#          path: ~/.cache/huggingface
#          key: ${{ runner.os }}-hf-cache-external-lm
#          restore-keys: |
#            ${{ runner.os }}-hf-cache-external-lm
#
#      - name: Install dependencies
#        run: |
#          python -m pip install --upgrade pip
#          pip install -e '.[dev,optimum,deepsparse,sparseml,api]' --extra-index-url https://download.pytorch.org/whl/cpu
#          pip install -U transformers peft accelerate
#
#      - name: Test with pytest
#        run: python -m pytest tests/models --showlocals -s -vv
#        continue-on-error: true  # Continue workflow even if tests fail