243 lines
7.4 KiB
YAML
243 lines
7.4 KiB
YAML
name: LLM Hello World
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
pr_number:
|
|
description: "PR number to review"
|
|
required: true
|
|
type: number
|
|
phase:
|
|
description: "Which phase to run"
|
|
required: true
|
|
type: choice
|
|
options:
|
|
- hello-world
|
|
- token-probe
|
|
default: hello-world
|
|
|
|
env:
|
|
LLM_ENDPOINT: "https://models.github.ai/inference/chat/completions"
|
|
LLM_MODEL: "openai/gpt-4.1"
|
|
|
|
permissions:
|
|
contents: read
|
|
pull-requests: write
|
|
models: read
|
|
|
|
jobs:
|
|
hello-world:
|
|
if: ${{ inputs.phase == 'hello-world' }}
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Fetch PR details
|
|
id: pr
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
echo "Fetching PR #${{ inputs.pr_number }}..."
|
|
|
|
# Fetch PR title and body
|
|
PR_JSON=$(gh pr view ${{ inputs.pr_number }} --json title,body,author)
|
|
|
|
if [ $? -ne 0 ]; then
|
|
echo "::error::Failed to fetch PR #${{ inputs.pr_number }}. Does it exist?"
|
|
exit 1
|
|
fi
|
|
|
|
PR_TITLE=$(echo "$PR_JSON" | jq -r '.title')
|
|
PR_BODY=$(echo "$PR_JSON" | jq -r '.body // "No description provided"')
|
|
PR_AUTHOR=$(echo "$PR_JSON" | jq -r '.author.login')
|
|
|
|
echo "PR Title: $PR_TITLE"
|
|
echo "PR Author: $PR_AUTHOR"
|
|
|
|
# Store for next step (escape newlines for multiline)
|
|
{
|
|
echo "title<<EOF"
|
|
echo "$PR_TITLE"
|
|
echo "EOF"
|
|
} >> $GITHUB_OUTPUT
|
|
|
|
{
|
|
echo "body<<EOF"
|
|
echo "$PR_BODY"
|
|
echo "EOF"
|
|
} >> $GITHUB_OUTPUT
|
|
|
|
echo "author=$PR_AUTHOR" >> $GITHUB_OUTPUT
|
|
|
|
- name: Call LLM for summary
|
|
id: llm
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
echo "Calling LLM to summarize PR..."
|
|
|
|
# Build the prompt
|
|
PROMPT="Summarize this PR in one sentence.
|
|
|
|
Title: ${{ steps.pr.outputs.title }}
|
|
|
|
Description:
|
|
${{ steps.pr.outputs.body }}"
|
|
|
|
# Escape for JSON
|
|
PROMPT_ESCAPED=$(echo "$PROMPT" | jq -Rs .)
|
|
|
|
# Call the LLM
|
|
RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "$LLM_ENDPOINT" \
|
|
-H "Authorization: Bearer $GITHUB_TOKEN" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{
|
|
\"model\": \"$LLM_MODEL\",
|
|
\"messages\": [
|
|
{\"role\": \"system\", \"content\": \"You are a helpful assistant that summarizes pull requests concisely.\"},
|
|
{\"role\": \"user\", \"content\": $PROMPT_ESCAPED}
|
|
],
|
|
\"max_tokens\": 150
|
|
}")
|
|
|
|
# Extract HTTP status code (last line)
|
|
HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
|
|
BODY=$(echo "$RESPONSE" | sed '$d')
|
|
|
|
echo "HTTP Status: $HTTP_CODE"
|
|
|
|
if [ "$HTTP_CODE" != "200" ]; then
|
|
echo "::error::LLM API call failed with status $HTTP_CODE"
|
|
echo "Response: $BODY"
|
|
exit 1
|
|
fi
|
|
|
|
# Extract the summary from the response
|
|
SUMMARY=$(echo "$BODY" | jq -r '.choices[0].message.content')
|
|
|
|
echo "LLM Summary: $SUMMARY"
|
|
|
|
{
|
|
echo "summary<<EOF"
|
|
echo "$SUMMARY"
|
|
echo "EOF"
|
|
} >> $GITHUB_OUTPUT
|
|
|
|
- name: Post comment to PR
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
SUMMARY: ${{ steps.llm.outputs.summary }}
|
|
AUTHOR: ${{ steps.pr.outputs.author }}
|
|
PR_NUM: ${{ inputs.pr_number }}
|
|
run: |
|
|
echo "Posting comment to PR #$PR_NUM..."
|
|
|
|
gh pr comment "$PR_NUM" --body "$(cat <<EOF
|
|
## LLM Hello World Test
|
|
|
|
**Model:** \`$LLM_MODEL\`
|
|
**PR Author:** @$AUTHOR
|
|
|
|
### Summary
|
|
$SUMMARY
|
|
|
|
---
|
|
*This comment was generated by the LLM Hello World workflow (Phase 0a)*
|
|
EOF
|
|
)"
|
|
|
|
echo "Comment posted successfully!"
|
|
|
|
token-probe:
|
|
if: ${{ inputs.phase == 'token-probe' }}
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Probe token limits
|
|
env:
|
|
AUTH_TOKEN: ${{ secrets.COPILOT_PAT }}
|
|
FALLBACK_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
# Try Azure endpoint (Gemini's suggestion) - deprecated Oct 2025 but may have different limits
|
|
LLM_ENDPOINT_AZURE: "https://models.inference.ai.azure.com/chat/completions"
|
|
LLM_ENDPOINT_GITHUB: "https://models.github.ai/inference/chat/completions"
|
|
run: |
|
|
echo "=== Token Limit Probe ==="
|
|
|
|
# Diagnostic: which token are we using?
|
|
if [ -n "$AUTH_TOKEN" ]; then
|
|
echo "Using: COPILOT_PAT (length: ${#AUTH_TOKEN} chars)"
|
|
TOKEN="$AUTH_TOKEN"
|
|
else
|
|
echo "Using: GITHUB_TOKEN (fallback)"
|
|
TOKEN="$FALLBACK_TOKEN"
|
|
fi
|
|
|
|
# Test both endpoints
|
|
for ENDPOINT_NAME in "AZURE" "GITHUB"; do
|
|
if [ "$ENDPOINT_NAME" = "AZURE" ]; then
|
|
ENDPOINT="$LLM_ENDPOINT_AZURE"
|
|
else
|
|
ENDPOINT="$LLM_ENDPOINT_GITHUB"
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== Testing $ENDPOINT_NAME endpoint ==="
|
|
echo "URL: $ENDPOINT"
|
|
echo "Model: gpt-4o"
|
|
echo ""
|
|
|
|
# Test sizes in characters (roughly 4 chars per token)
|
|
SIZES=(2000 4000 8000 16000 32000 64000 128000)
|
|
|
|
echo "| Size (chars) | ~Tokens | HTTP Status | Result |"
|
|
echo "|--------------|---------|-------------|--------|"
|
|
|
|
for SIZE in "${SIZES[@]}"; do
|
|
APPROX_TOKENS=$((SIZE / 4))
|
|
|
|
# Generate filler
|
|
FILLER=$(python3 -c "print(('The quick brown fox jumps over the lazy dog. ' * ($SIZE // 45 + 1))[:$SIZE])")
|
|
|
|
PROMPT="Respond with exactly one word: 'received'. Here is padding text to test token limits: $FILLER"
|
|
PROMPT_ESCAPED=$(echo "$PROMPT" | jq -Rs .)
|
|
|
|
RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "$ENDPOINT" \
|
|
-H "Authorization: Bearer $TOKEN" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{
|
|
\"model\": \"gpt-4o\",
|
|
\"messages\": [
|
|
{\"role\": \"user\", \"content\": $PROMPT_ESCAPED}
|
|
],
|
|
\"max_tokens\": 10
|
|
}" 2>/dev/null)
|
|
|
|
HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
|
|
BODY=$(echo "$RESPONSE" | sed '$d')
|
|
|
|
if [ "$HTTP_CODE" = "200" ]; then
|
|
RESULT="OK"
|
|
else
|
|
ERROR_MSG=$(echo "$BODY" | jq -r '.error.message // .message // "Unknown error"' 2>/dev/null | head -c 50)
|
|
RESULT="FAIL: $ERROR_MSG"
|
|
echo ""
|
|
echo "::warning::Failed at ${SIZE} chars (~${APPROX_TOKENS} tokens) on $ENDPOINT_NAME"
|
|
echo "Full error: $BODY"
|
|
echo ""
|
|
fi
|
|
|
|
echo "| $SIZE | ~$APPROX_TOKENS | $HTTP_CODE | $RESULT |"
|
|
|
|
if [ "$HTTP_CODE" != "200" ]; then
|
|
echo ""
|
|
echo "=== $ENDPOINT_NAME Ceiling: ~$((SIZE / 8)) tokens ==="
|
|
break
|
|
fi
|
|
|
|
sleep 1
|
|
done
|
|
done
|
|
|
|
echo ""
|
|
echo "=== Probe Complete ==="
|