From 69e1078c7b0b2cc57090a9b5418e4c0a71191caf Mon Sep 17 00:00:00 2001 From: doxav Date: Tue, 17 Mar 2026 18:36:30 +0100 Subject: [PATCH 1/3] try to fix CI --- .github/workflows/ci.yml | 164 ++++++++++++++++++++++++--------------- 1 file changed, 103 insertions(+), 61 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 622c9626..b6e0f109 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,71 +2,113 @@ name: CI on: push: - branches: [ main, dev, experimental, ci-multi ] + branches: [main, dev, experimental, ci-multi] pull_request: - branches: [ main, dev, experimental, ci-multi ] + branches: [main, dev, experimental, ci-multi] jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 180 + env: + PYTHON_VERSION: "3.13" + OLLAMA_VERSION: "0.18.0" + OLLAMA_HOST: "127.0.0.1:11434" + OLLAMA_CONTEXT_LENGTH: "4000" + OLLAMA_MODELS: "${{ github.workspace }}/.ollama/models" + steps: - - name: Checkout code - uses: actions/checkout@v4 - - # 1) Restore any cached Ollama data (~2 GB) - - name: Restore Ollama cache - uses: actions/cache@v4 - with: - path: ~/.ollama - key: qwen3-4b-gguf-v1 - - # 2) Install Ollama - - name: Install Ollama - run: | - curl -fsSL https://ollama.com/install.sh | sh - - # 3) Drop-in override to bump context window to 4k tokens - - name: Configure Ollama for 4K context - run: | - sudo mkdir -p /etc/systemd/system/ollama.service.d - sudo tee /etc/systemd/system/ollama.service.d/override.conf << 'EOF' - [Service] - ExecStart= - ExecStart=/usr/local/bin/ollama serve --num_ctx 4000 - EOF - sudo systemctl daemon-reload - - # 4) Enable & start the systemd-managed Ollama daemon - - name: Enable & start Ollama - run: | - sudo systemctl enable --now ollama - - # 5) Pull the phi4-mini:3.8b model (uses cache if present) - - name: Pull phi4-mini:3.8b model - run: ollama pull phi4-mini:3.8b - - # 6) Set up Python & install dependencies - - uses: actions/setup-python@v5 - with: { python-version: "3.13" } - - name: Install Python deps - run: | - pip install -e . - pip install pytest datasets numpy - - # 7) Point LiteLLM/OpenAI to our local Ollama server - - name: Configure LLM env - run: | - echo "OPENAI_API_KEY=ollama" >> $GITHUB_ENV - echo "OPENAI_API_BASE=http://localhost:11434/v1" >> $GITHUB_ENV - echo "TRACE_LITELLM_MODEL=openai/phi4-mini:3.8b" >> $GITHUB_ENV - - # 8) Run all Trace unit tests - - name: Run unit tests - run: pytest tests/unit_tests/ - - # 9) Run basic tests for each optimizer (some will fail due to the small LLM model chosen for free GitHub CI) - - name: Run optimizers test suite - run: pytest tests/llm_optimizers_tests/test_optimizer.py || true - continue-on-error: true + - name: Checkout code + uses: actions/checkout@v4 + + # Cache the actual model directory we control and reuse across runs. + - name: Restore Ollama model cache + uses: actions/cache@v4 + with: + path: ${{ env.OLLAMA_MODELS }} + key: ollama-${{ runner.os }}-${{ env.OLLAMA_VERSION }}-phi4-mini-3.8b-v1 + restore-keys: | + ollama-${{ runner.os }}-${{ env.OLLAMA_VERSION }}- + ollama-${{ runner.os }}- + + - name: Prepare Ollama model directory + run: | + mkdir -p "${OLLAMA_MODELS}" + + - name: Install Ollama (pinned) + run: | + curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION=${OLLAMA_VERSION} sh + + # Use environment variables instead of replacing ExecStart. + - name: Configure Ollama service + run: | + sudo mkdir -p /etc/systemd/system/ollama.service.d + sudo tee /etc/systemd/system/ollama.service.d/override.conf > /dev/null < /dev/null; then + echo "Ollama is ready" + exit 0 + fi + sleep 2 + done + + echo "Ollama failed to become ready" + sudo systemctl status ollama --no-pager || true + sudo journalctl -u ollama -n 200 --no-pager || true + exit 1 + + - name: Pull phi4-mini:3.8b model + run: | + ollama pull phi4-mini:3.8b + + - name: Show Ollama info + run: | + ollama list + curl -fsS "http://${OLLAMA_HOST}/api/tags" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Python deps + run: | + python -m pip install --upgrade pip + pip install -e . + pip install pytest datasets numpy + + - name: Configure LLM env + run: | + echo "OPENAI_API_KEY=ollama" >> "$GITHUB_ENV" + echo "OPENAI_API_BASE=http://${OLLAMA_HOST}/v1" >> "$GITHUB_ENV" + echo "TRACE_LITELLM_MODEL=openai/phi4-mini:3.8b" >> "$GITHUB_ENV" + + - name: Run unit tests + run: | + pytest tests/unit_tests/ + + - name: Run optimizers test suite + run: | + pytest tests/llm_optimizers_tests/test_optimizer.py || true + continue-on-error: true + + - name: Dump Ollama logs on failure + if: failure() + run: | + sudo systemctl status ollama --no-pager || true + sudo journalctl -u ollama -n 300 --no-pager || true + ls -R "${OLLAMA_MODELS}" || true From 113e3a9c087f0afe0adafbe37667ff70e409f387 Mon Sep 17 00:00:00 2001 From: doxav Date: Tue, 17 Mar 2026 18:46:24 +0100 Subject: [PATCH 2/3] try to fix CI v2 --- .github/workflows/ci.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6e0f109..1e380b70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,18 +58,24 @@ jobs: shell: bash run: | set -euo pipefail - for i in {1..60}; do + + ready=0 + for i in $(seq 1 60); do if curl -fsS "http://${OLLAMA_HOST}/api/tags" > /dev/null; then echo "Ollama is ready" - exit 0 + ready=1 + break fi + echo "Waiting for Ollama... attempt $i/60" sleep 2 done - echo "Ollama failed to become ready" - sudo systemctl status ollama --no-pager || true - sudo journalctl -u ollama -n 200 --no-pager || true - exit 1 + if [ "$ready" -ne 1 ]; then + echo "Ollama failed to become ready" + sudo systemctl status ollama --no-pager || true + sudo journalctl -u ollama -n 200 --no-pager || true + exit 1 + fi - name: Pull phi4-mini:3.8b model run: | From 8c22637cce236a0899b440b29d03a72119883af7 Mon Sep 17 00:00:00 2001 From: doxav Date: Tue, 17 Mar 2026 18:49:54 +0100 Subject: [PATCH 3/3] try to fix CI v3 --- .github/workflows/ci.yml | 54 +++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e380b70..c6704c8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,31 +16,30 @@ jobs: OLLAMA_VERSION: "0.18.0" OLLAMA_HOST: "127.0.0.1:11434" OLLAMA_CONTEXT_LENGTH: "4000" - OLLAMA_MODELS: "${{ github.workspace }}/.ollama/models" + OLLAMA_MODELS: "/usr/share/ollama/.ollama/models" steps: - name: Checkout code uses: actions/checkout@v4 - # Cache the actual model directory we control and reuse across runs. - name: Restore Ollama model cache uses: actions/cache@v4 with: - path: ${{ env.OLLAMA_MODELS }} - key: ollama-${{ runner.os }}-${{ env.OLLAMA_VERSION }}-phi4-mini-3.8b-v1 + path: /usr/share/ollama/.ollama/models + key: ollama-${{ runner.os }}-${{ env.OLLAMA_VERSION }}-phi4-mini-3.8b-v2 restore-keys: | ollama-${{ runner.os }}-${{ env.OLLAMA_VERSION }}- ollama-${{ runner.os }}- - - name: Prepare Ollama model directory + - name: Install Ollama run: | - mkdir -p "${OLLAMA_MODELS}" + curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION=${OLLAMA_VERSION} sh - - name: Install Ollama (pinned) + - name: Prepare Ollama directories run: | - curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION=${OLLAMA_VERSION} sh + sudo mkdir -p /usr/share/ollama/.ollama/models + sudo chown -R ollama:ollama /usr/share/ollama - # Use environment variables instead of replacing ExecStart. - name: Configure Ollama service run: | sudo mkdir -p /etc/systemd/system/ollama.service.d @@ -54,38 +53,39 @@ jobs: sudo systemctl enable ollama sudo systemctl restart ollama + - name: Verify Ollama service started + shell: bash + run: | + set -euxo pipefail + sudo systemctl status ollama --no-pager || true + sudo journalctl -u ollama -n 100 --no-pager || true + - name: Wait for Ollama to become ready shell: bash run: | set -euo pipefail - ready=0 for i in $(seq 1 60); do if curl -fsS "http://${OLLAMA_HOST}/api/tags" > /dev/null; then echo "Ollama is ready" - ready=1 break fi + + if [ "$i" -eq 60 ]; then + echo "Ollama failed to become ready" + sudo systemctl status ollama --no-pager || true + sudo journalctl -u ollama -n 200 --no-pager || true + exit 1 + fi + echo "Waiting for Ollama... attempt $i/60" sleep 2 done - if [ "$ready" -ne 1 ]; then - echo "Ollama failed to become ready" - sudo systemctl status ollama --no-pager || true - sudo journalctl -u ollama -n 200 --no-pager || true - exit 1 - fi - - name: Pull phi4-mini:3.8b model run: | ollama pull phi4-mini:3.8b - - name: Show Ollama info - run: | - ollama list - curl -fsS "http://${OLLAMA_HOST}/api/tags" - - name: Set up Python uses: actions/setup-python@v5 with: @@ -104,12 +104,10 @@ jobs: echo "TRACE_LITELLM_MODEL=openai/phi4-mini:3.8b" >> "$GITHUB_ENV" - name: Run unit tests - run: | - pytest tests/unit_tests/ + run: pytest tests/unit_tests/ - name: Run optimizers test suite - run: | - pytest tests/llm_optimizers_tests/test_optimizer.py || true + run: pytest tests/llm_optimizers_tests/test_optimizer.py || true continue-on-error: true - name: Dump Ollama logs on failure @@ -117,4 +115,4 @@ jobs: run: | sudo systemctl status ollama --no-pager || true sudo journalctl -u ollama -n 300 --no-pager || true - ls -R "${OLLAMA_MODELS}" || true + sudo ls -R /usr/share/ollama/.ollama || true