Visitar URL original
Add PR comment summary for CI/CD test results · python-mode/python-mode@8f33e58 · GitHub
Skip to content

Commit 8f33e58

Browse files
diraoldiraol
committed
Add PR comment summary for CI/CD test results
- Add summary job to workflow that collects test results from all Python versions - Create generate_pr_summary.sh script to parse test results and generate markdown summary - Post test summary as PR comment using actions-comment-pull-request - Summary includes per-version results and overall test status - Comment is automatically updated on subsequent runs (no duplicates) - Only runs on pull requests, not on regular pushes
1 parent 3855fe5 commit 8f33e58

2 files changed

Lines changed: 221 additions & 0 deletions

File tree

‎.github/workflows/test.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,39 @@ jobs:
5151
with:
5252
file: ./coverage.xml
5353
flags: python-${{ matrix.python-version }}
54+
55+
summary:
56+
runs-on: ubuntu-latest
57+
needs: test
58+
if: github.event_name == 'pull_request'
59+
60+
steps:
61+
- name: Checkout code
62+
uses: actions/checkout@v4
63+
with:
64+
submodules: recursive
65+
66+
- name: Download all test results
67+
uses: actions/download-artifact@v4
68+
with:
69+
path: test-results-artifacts
70+
pattern: test-results-*
71+
merge-multiple: false
72+
73+
- name: Install jq for JSON parsing
74+
run: |
75+
sudo apt-get update
76+
sudo apt-get install -y jq
77+
78+
- name: Generate PR summary
79+
id: generate_summary
80+
run: |
81+
bash scripts/cicd/generate_pr_summary.sh test-results-artifacts pr-summary.md
82+
continue-on-error: true
83+
84+
- name: Post PR comment
85+
uses: thollander/actions-comment-pull-request@v3
86+
if: always() && github.event_name == 'pull_request'
87+
with:
88+
file-path: pr-summary.md
89+
comment-tag: test-summary

‎scripts/cicd/generate_pr_summary.sh‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
#!/bin/bash
2+
# Generate PR summary from test results JSON files
3+
set -euo pipefail
4+
5+
# Colors for output
6+
RED='\033[0;31m'
7+
GREEN='\033[0;32m'
8+
YELLOW='\033[1;33m'
9+
BLUE='\033[0;34m'
10+
NC='\033[0m' # No Color
11+
12+
ARTIFACTS_DIR="${1:-test-results-artifacts}"
13+
OUTPUT_FILE="${2:-pr-summary.md}"
14+
15+
echo "Generating PR summary from test results..."
16+
echo "Artifacts directory: $ARTIFACTS_DIR"
17+
18+
# Initialize summary variables
19+
TOTAL_PYTHON_VERSIONS=0
20+
TOTAL_TESTS=0
21+
TOTAL_PASSED=0
22+
TOTAL_FAILED=0
23+
TOTAL_ASSERTIONS=0
24+
PASSED_ASSERTIONS=0
25+
ALL_PASSED=true
26+
FAILED_VERSIONS=()
27+
PASSED_VERSIONS=()
28+
29+
# Start markdown output
30+
cat > "$OUTPUT_FILE" << 'EOF'
31+
## 🧪 Test Results Summary
32+
33+
This comment will be updated automatically as tests complete.
34+
35+
EOF
36+
37+
# Check if artifacts directory exists and has content
38+
if [ ! -d "$ARTIFACTS_DIR" ] || [ -z "$(ls -A "$ARTIFACTS_DIR" 2>/dev/null)" ]; then
39+
echo "⚠️ No test artifacts found in $ARTIFACTS_DIR" >> "$OUTPUT_FILE"
40+
echo "Tests may still be running or failed to upload artifacts." >> "$OUTPUT_FILE"
41+
exit 0
42+
fi
43+
44+
# Process each Python version's test results
45+
# Handle both direct artifact structure and nested structure
46+
for artifact_dir in "$ARTIFACTS_DIR"/*/; do
47+
if [ ! -d "$artifact_dir" ]; then
48+
continue
49+
fi
50+
51+
# Extract Python version from directory name (e.g., "test-results-3.10" -> "3.10")
52+
dir_name=$(basename "$artifact_dir")
53+
python_version="${dir_name#test-results-}"
54+
55+
# Look for test-results.json in the artifact directory
56+
results_file="$artifact_dir/test-results.json"
57+
58+
if [ ! -f "$results_file" ]; then
59+
echo "⚠️ Warning: test-results.json not found for Python $python_version (looked in: $results_file)" >> "$OUTPUT_FILE"
60+
echo "Available files in $artifact_dir:" >> "$OUTPUT_FILE"
61+
ls -la "$artifact_dir" >> "$OUTPUT_FILE" 2>&1 || true
62+
continue
63+
fi
64+
65+
# Parse JSON (using jq if available, otherwise use basic parsing)
66+
if command -v jq &> /dev/null; then
67+
total_tests=$(jq -r '.total_tests // 0' "$results_file")
68+
passed_tests=$(jq -r '.passed_tests // 0' "$results_file")
69+
failed_tests=$(jq -r '.failed_tests // 0' "$results_file")
70+
total_assertions=$(jq -r '.total_assertions // 0' "$results_file")
71+
passed_assertions=$(jq -r '.passed_assertions // 0' "$results_file")
72+
python_ver=$(jq -r '.python_version // "unknown"' "$results_file")
73+
vim_ver=$(jq -r '.vim_version // "unknown"' "$results_file")
74+
75+
# Get failed test names
76+
failed_test_names=$(jq -r '.results.failed[]?' "$results_file" 2>/dev/null | tr '\n' ',' | sed 's/,$//' || echo "")
77+
else
78+
# Fallback: basic parsing without jq
79+
total_tests=$(grep -o '"total_tests":[0-9]*' "$results_file" | grep -o '[0-9]*' || echo "0")
80+
passed_tests=$(grep -o '"passed_tests":[0-9]*' "$results_file" | grep -o '[0-9]*' || echo "0")
81+
failed_tests=$(grep -o '"failed_tests":[0-9]*' "$results_file" | grep -o '[0-9]*' || echo "0")
82+
total_assertions=$(grep -o '"total_assertions":[0-9]*' "$results_file" | grep -o '[0-9]*' || echo "0")
83+
passed_assertions=$(grep -o '"passed_assertions":[0-9]*' "$results_file" | grep -o '[0-9]*' || echo "0")
84+
python_ver="Python $python_version"
85+
vim_ver="unknown"
86+
failed_test_names=""
87+
fi
88+
89+
TOTAL_PYTHON_VERSIONS=$((TOTAL_PYTHON_VERSIONS + 1))
90+
TOTAL_TESTS=$((TOTAL_TESTS + total_tests))
91+
TOTAL_PASSED=$((TOTAL_PASSED + passed_tests))
92+
TOTAL_FAILED=$((TOTAL_FAILED + failed_tests))
93+
TOTAL_ASSERTIONS=$((TOTAL_ASSERTIONS + total_assertions))
94+
PASSED_ASSERTIONS=$((PASSED_ASSERTIONS + passed_assertions))
95+
96+
# Determine status
97+
if [ "$failed_tests" -gt 0 ]; then
98+
ALL_PASSED=false
99+
FAILED_VERSIONS+=("$python_version")
100+
status_icon="❌"
101+
status_text="FAILED"
102+
else
103+
PASSED_VERSIONS+=("$python_version")
104+
status_icon="✅"
105+
status_text="PASSED"
106+
fi
107+
108+
# Add version summary to markdown
109+
cat >> "$OUTPUT_FILE" << EOF
110+
111+
### Python $python_version $status_icon
112+
113+
- **Status**: $status_text
114+
- **Python Version**: $python_ver
115+
- **Vim Version**: $vim_ver
116+
- **Tests**: $passed_tests/$total_tests passed
117+
- **Assertions**: $passed_assertions/$total_assertions passed
118+
119+
EOF
120+
121+
# Add failed tests if any
122+
if [ "$failed_tests" -gt 0 ] && [ -n "$failed_test_names" ]; then
123+
echo "**Failed tests:**" >> "$OUTPUT_FILE"
124+
if command -v jq &> /dev/null; then
125+
jq -r '.results.failed[]?' "$results_file" 2>/dev/null | while read -r test_name; do
126+
echo "- \`$test_name\`" >> "$OUTPUT_FILE"
127+
done || true
128+
else
129+
# Basic parsing fallback
130+
echo "- See test logs for details" >> "$OUTPUT_FILE"
131+
fi
132+
echo "" >> "$OUTPUT_FILE"
133+
fi
134+
done
135+
136+
# Add overall summary
137+
cat >> "$OUTPUT_FILE" << EOF
138+
139+
---
140+
141+
### 📊 Overall Summary
142+
143+
- **Python Versions Tested**: $TOTAL_PYTHON_VERSIONS
144+
- **Total Tests**: $TOTAL_TESTS
145+
- **Passed**: $TOTAL_PASSED
146+
- **Failed**: $TOTAL_FAILED
147+
- **Total Assertions**: $TOTAL_ASSERTIONS
148+
- **Passed Assertions**: $PASSED_ASSERTIONS
149+
150+
EOF
151+
152+
# Add status summary
153+
if [ "$ALL_PASSED" = true ]; then
154+
cat >> "$OUTPUT_FILE" << EOF
155+
**🎉 All tests passed across all Python versions!**
156+
157+
EOF
158+
else
159+
cat >> "$OUTPUT_FILE" << EOF
160+
**⚠️ Some tests failed:**
161+
162+
EOF
163+
for version in "${FAILED_VERSIONS[@]}"; do
164+
echo "- Python $version" >> "$OUTPUT_FILE"
165+
done
166+
echo "" >> "$OUTPUT_FILE"
167+
fi
168+
169+
# Add footer
170+
cat >> "$OUTPUT_FILE" << EOF
171+
172+
---
173+
*Generated automatically by CI/CD workflow*
174+
EOF
175+
176+
echo "Summary generated: $OUTPUT_FILE"
177+
cat "$OUTPUT_FILE"
178+
179+
# Exit with error if any tests failed
180+
if [ "$ALL_PASSED" = false ]; then
181+
exit 1
182+
fi
183+
184+
exit 0
185+

0 commit comments

Comments
 (0)