Add GH Action for nightly evals

.github/workflows/huggingface-upload.yml

@@ -3,6 +3,8 @@ name: Upload to Hugging Face
 on:
   push:
     branches: [ main, master ]
+  repository_dispatch:
+    types: [trigger-hf-upload]
 
 jobs:
   upload:
@@ -23,7 +25,7 @@ jobs:
       - name: Upload to Hugging Face
         env:
           HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
-          COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
+          COMMIT_MESSAGE: ${{ github.event.head_commit.message || 'Update from nightly evaluation' }}
         run: |
           python -c '
           from huggingface_hub import upload_folder

.github/workflows/nightly-evals.yml

@@ -0,0 +1,44 @@
+name: Nightly Evaluation Run
+
+on:
+  schedule:
+    - cron: '0 3 * * *'  # Run at 3am UTC every day
+  workflow_dispatch:  # Allow manual triggering
+
+jobs:
+  run-evals:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+          
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          
+      - name: Run evaluations
+        env:
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+        run: |
+          cd evals
+          python main.py
+          
+      - name: Commit changes
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add results.json models.json languages.json
+          git commit -m "Update evaluation results [skip ci]" || echo "No changes to commit"
+          git push
+          
+      - name: Trigger Hugging Face upload
+        if: success()
+        uses: peter-evans/repository-dispatch@v2
+        with:
+          event-type: trigger-hf-upload
+          token: ${{ secrets.GITHUB_TOKEN }} 

requirements.txt

@@ -1,168 +1,55 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile pyproject.toml -o requirements.txt
-aiofiles==23.2.1
-    # via gradio
 annotated-types==0.7.0
     # via pydantic
 anyio==4.8.0
-    # via
-    #   gradio
-    #   httpx
-    #   starlette
-certifi==2025.1.31
-    # via
-    #   httpcore
-    #   httpx
-    #   requests
-charset-normalizer==3.4.1
-    # via requests
+    # via starlette
 click==8.1.8
-    # via
-    #   typer
-    #   uvicorn
+    # via uvicorn
 exceptiongroup==1.2.2
     # via anyio
 fastapi==0.115.8
-    # via gradio
-ffmpy==0.5.0
-    # via gradio
-filelock==3.17.0
-    # via huggingface-hub
-fsspec==2025.2.0
-    # via
-    #   gradio-client
-    #   huggingface-hub
-gradio==5.16.2
-    # via
-    #   languagebench (pyproject.toml)
-    #   gradio-rangeslider
-gradio-client==1.7.1
-    # via gradio
-gradio-rangeslider==0.0.8
     # via languagebench (pyproject.toml)
 h11==0.14.0
-    # via
-    #   httpcore
-    #   uvicorn
-httpcore==1.0.7
-    # via httpx
-httpx==0.28.1
-    # via
-    #   gradio
-    #   gradio-client
-    #   safehttpx
-huggingface-hub==0.29.1
-    # via
-    #   gradio
-    #   gradio-client
+    # via uvicorn
 idna==3.10
-    # via
-    #   anyio
-    #   httpx
-    #   requests
-jinja2==3.1.5
-    # via gradio
+    # via anyio
+joblib==1.5.0
+    # via languagebench (pyproject.toml)
 language-data==1.3.0
     # via languagebench (pyproject.toml)
 marisa-trie==1.2.1
     # via language-data
-markdown-it-py==3.0.0
-    # via rich
-markupsafe==2.1.5
-    # via
-    #   gradio
-    #   jinja2
-mdurl==0.1.2
-    # via markdown-it-py
-narwhals==1.27.1
-    # via plotly
 numpy==2.2.3
     # via
-    #   gradio
+    #   languagebench (pyproject.toml)
     #   pandas
-orjson==3.10.15
-    # via gradio
-packaging==24.2
-    # via
-    #   gradio
-    #   gradio-client
-    #   huggingface-hub
-    #   plotly
 pandas==2.2.3
-    # via
-    #   languagebench (pyproject.toml)
-    #   gradio
-pillow==11.1.0
-    # via gradio
-plotly==6.0.0
-    # via languagebench (pyproject.toml)
-pycountry==24.6.1
     # via languagebench (pyproject.toml)
 pydantic==2.10.6
-    # via
-    #   fastapi
-    #   gradio
+    # via fastapi
 pydantic-core==2.27.2
     # via pydantic
-pydub==0.25.1
-    # via gradio
-pygments==2.19.1
-    # via rich
 python-dateutil==2.9.0.post0
     # via pandas
-python-multipart==0.0.20
-    # via gradio
 pytz==2025.1
     # via pandas
-pyyaml==6.0.2
-    # via
-    #   gradio
-    #   huggingface-hub
-requests==2.32.3
-    # via huggingface-hub
-rich==13.9.4
-    # via typer
-ruff==0.9.7
-    # via gradio
-safehttpx==0.1.6
-    # via gradio
-semantic-version==2.10.0
-    # via gradio
 setuptools==75.8.2
     # via marisa-trie
-shellingham==1.5.4
-    # via typer
 six==1.17.0
     # via python-dateutil
 sniffio==1.3.1
     # via anyio
 starlette==0.45.3
-    # via
-    #   fastapi
-    #   gradio
-tomlkit==0.13.2
-    # via gradio
-tqdm==4.67.1
-    # via huggingface-hub
-typer==0.15.1
-    # via gradio
+    # via fastapi
 typing-extensions==4.12.2
     # via
     #   anyio
     #   fastapi
-    #   gradio
-    #   gradio-client
-    #   huggingface-hub
     #   pydantic
     #   pydantic-core
-    #   rich
-    #   typer
     #   uvicorn
 tzdata==2025.1
     # via pandas
-urllib3==2.3.0
-    # via requests
 uvicorn==0.34.0
-    # via gradio
-websockets==14.2
-    # via gradio-client
+    # via languagebench (pyproject.toml)