diff --git a/.github/workflows/coverage-nightly.yaml b/.github/workflows/coverage-nightly.yaml index dfa896867bbf..fed93dbea91f 100644 --- a/.github/workflows/coverage-nightly.yaml +++ b/.github/workflows/coverage-nightly.yaml @@ -8,6 +8,8 @@ on: workflow_dispatch: # Allow being called from other workflows workflow_call: + # Run on pull requests for testing + pull_request: concurrency: group: coverage-${{ github.ref }} @@ -32,9 +34,17 @@ jobs: - name: Install dependencies run: bash -x .github/scripts/setup.sh + - name: Install Clang 18 for coverage + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 18 + sudo ln -sf /usr/bin/clang-18 /usr/bin/clang + sudo ln -sf /usr/bin/clang++-18 /usr/bin/clang++ + - name: Build with coverage instrumentation run: | - ./configure --enable-debugbuild --enable-coverage CC=clang + ./configure --enable-debugbuild --enable-coverage CC=clang-18 uv run make -j $(nproc) testpack.tar.bz2 - name: Upload build artifact @@ -51,12 +61,26 @@ jobs: fail-fast: false matrix: include: - - name: sqlite + - name: sqlite-1 + db: sqlite3 + pytest_par: 10 + test_group: 1 + test_group_count: 2 + - name: sqlite-2 db: sqlite3 pytest_par: 10 - - name: postgres + test_group: 2 + test_group_count: 2 + - name: postgres-1 + db: postgres + pytest_par: 10 + test_group: 1 + test_group_count: 2 + - name: postgres-2 db: postgres pytest_par: 10 + test_group: 2 + test_group_count: 2 steps: - name: Checkout @@ -84,24 +108,83 @@ jobs: - name: Unpack build run: tar -xaf testpack.tar.bz2 + - name: Verify coverage instrumentation + run: | + echo "Checking if binaries are instrumented for coverage..." + if nm lightningd/lightningd | grep -q '__llvm_profile'; then + echo "✓ Binaries appear to be instrumented for coverage" + else + echo "⚠ WARNING: Binaries may not be properly instrumented" + echo "Checking build flags in config.vars:" + grep -E "COVERAGE|CFLAGS" config.vars || echo "No coverage flags found" + fi + - name: Run tests with coverage env: - CLN_COVERAGE_DIR: ${{ github.workspace }}/coverage-raw TEST_DB_PROVIDER: ${{ matrix.db }} PYTEST_PAR: ${{ matrix.pytest_par }} SLOW_MACHINE: 1 TIMEOUT: 900 + VALGRIND: 0 run: | + export CLN_COVERAGE_DIR="${{ github.workspace }}/coverage-raw" mkdir -p "$CLN_COVERAGE_DIR" - uv run eatmydata pytest tests/ -n ${PYTEST_PAR} -vvv + echo "CLN_COVERAGE_DIR=$CLN_COVERAGE_DIR" + echo "Running test group ${{ matrix.test_group }} of ${{ matrix.test_group_count }}" + echo "Current directory: $(pwd)" + ls -la + VALGRIND=0 uv run eatmydata pytest tests/ -n ${PYTEST_PAR} -vvv \ + --test-group-count ${{ matrix.test_group_count }} \ + --test-group ${{ matrix.test_group }} + + - name: Check coverage files generated + if: always() + run: | + echo "Checking for profraw files in coverage-raw" + find "${{ github.workspace }}/coverage-raw" -name "*.profraw" -ls || echo "No profraw files found" + COUNT=$(find "${{ github.workspace }}/coverage-raw" -name "*.profraw" 2>/dev/null | wc -l) + echo "Total profraw files: $COUNT" + if [ "$COUNT" -eq 0 ]; then + echo "WARNING: No coverage data was generated!" + echo "This might indicate:" + echo " 1. Tests didn't run successfully" + echo " 2. Binaries aren't instrumented" + echo " 3. LLVM_PROFILE_FILE environment variable not set correctly" + fi + + - name: Install LLVM tools for merging + if: always() + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 18 + sudo ln -sf /usr/bin/llvm-profdata-18 /usr/bin/llvm-profdata + + - name: Merge coverage data locally + if: always() + run: | + cd ${{ github.workspace }} + echo "Current directory: $(pwd)" + ls -la contrib/coverage/ || echo "contrib/coverage directory not found" + chmod +x contrib/coverage/collect-coverage.sh + mkdir -p coverage + CLN_COVERAGE_DIR="${{ github.workspace }}/coverage-raw" \ + ./contrib/coverage/collect-coverage.sh \ + coverage-raw \ + coverage/${{ matrix.name }}.profdata + echo "Merged profdata size:" + ls -lh coverage/${{ matrix.name }}.profdata || echo "No profdata file created" + # Clean up raw files to save disk space + rm -rf coverage-raw/*.profraw + echo "Cleaned up raw profraw files" - name: Upload coverage data uses: actions/upload-artifact@v4 if: always() with: - name: coverage-raw-${{ matrix.name }} - path: coverage-raw/*.profraw - if-no-files-found: error + name: coverage-merged-${{ matrix.name }} + path: coverage/${{ matrix.name }}.profdata + if-no-files-found: warn report: name: Generate Coverage Report @@ -132,24 +215,32 @@ jobs: - name: Download all coverage artifacts uses: actions/download-artifact@v4 with: - pattern: coverage-raw-* + pattern: coverage-merged-* path: coverage-artifacts - name: Merge coverage data run: | - mkdir -p coverage-raw coverage - find coverage-artifacts -name "*.profraw" -exec cp {} coverage-raw/ \; - PROFRAW_COUNT=$(ls -1 coverage-raw/*.profraw 2>/dev/null | wc -l) - echo "Found $PROFRAW_COUNT profile files" - if [ "$PROFRAW_COUNT" -eq 0 ]; then + mkdir -p coverage + # Find all .profdata files from test jobs + PROFDATA_FILES=($(find coverage-artifacts -name "*.profdata")) + echo "Found ${#PROFDATA_FILES[@]} profdata files from test jobs:" + printf '%s\n' "${PROFDATA_FILES[@]}" + + if [ ${#PROFDATA_FILES[@]} -eq 0 ]; then echo "ERROR: No coverage data found" exit 1 fi - chmod +x contrib/coverage/collect-coverage.sh - CLN_COVERAGE_DIR=coverage-raw ./contrib/coverage/collect-coverage.sh + + # Merge all pre-aggregated profdata files into final merged.profdata + echo "Merging ${#PROFDATA_FILES[@]} profdata files..." + llvm-profdata merge -sparse "${PROFDATA_FILES[@]}" -o coverage/merged.profdata + + echo "Final merged profdata size:" + ls -lh coverage/merged.profdata - name: Generate HTML report run: | + cd ${{ github.workspace }} chmod +x contrib/coverage/generate-coverage-report.sh ./contrib/coverage/generate-coverage-report.sh diff --git a/.gitignore b/.gitignore index c54cf77954ac..949960efa0da 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,7 @@ gen_*.c gen_*.h wire/gen_*_csv cli/lightning-cli -coverage +/coverage # Coverage profiling data files *.profraw *.profdata diff --git a/contrib/coverage/README.md b/contrib/coverage/README.md new file mode 100644 index 000000000000..84a8589f517c --- /dev/null +++ b/contrib/coverage/README.md @@ -0,0 +1,474 @@ +# Core Lightning Coverage Collection System + +This directory contains scripts and workflows for collecting, processing, and reporting code coverage data for Core Lightning (CLN) using LLVM source-based coverage instrumentation. + +## Table of Contents + +- [Overview](#overview) +- [How It Works](#how-it-works) +- [GitHub Actions Workflow](#github-actions-workflow) +- [Scripts Reference](#scripts-reference) +- [Local Usage](#local-usage) +- [Troubleshooting](#troubleshooting) + +## Overview + +CLN uses Clang's source-based code coverage to track which lines of code are executed during tests. The system is designed to: + +1. **Instrument binaries** during compilation with coverage tracking code +2. **Collect raw profile data** (`.profraw` files) as tests execute +3. **Aggregate and merge** profile data to reduce storage and transfer overhead +4. **Generate reports** showing line-by-line coverage statistics + +This approach provides accurate, detailed coverage information across the entire codebase, including main binaries, plugins, and test programs. + +## How It Works + +### Build Phase + +When building with `--enable-coverage`, CLN binaries are instrumented with LLVM coverage tracking: + +```bash +./configure --enable-coverage CC=clang +make +``` + +This adds instrumentation that writes `.profraw` files when binaries execute. + +### Collection Phase + +During test execution, the `LLVM_PROFILE_FILE` environment variable controls where `.profraw` files are written: + +```bash +export CLN_COVERAGE_DIR=/tmp/cln-coverage +export LLVM_PROFILE_FILE="${CLN_COVERAGE_DIR}/%p-%m.profraw" +``` + +- `%p` = process ID (ensures unique files per process) +- `%m` = instrumentation signature (ensures compatibility) + +Each test run can generate dozens to hundreds of `.profraw` files. + +### Aggregation Phase + +Raw `.profraw` files are merged into compressed `.profdata` files using `llvm-profdata`: + +```bash +llvm-profdata merge -sparse *.profraw -o merged.profdata +``` + +This dramatically reduces: +- **File count**: Thousands of files → 1 file +- **Disk usage**: ~50-80% reduction +- **Transfer time**: Significant improvement for CI/CD + +### Report Generation Phase + +The final `.profdata` file is used with `llvm-cov` to generate reports: + +```bash +llvm-cov show -instr-profile=merged.profdata -format=html +llvm-cov report -instr-profile=merged.profdata +``` + +## GitHub Actions Workflow + +The nightly coverage workflow (`.github/workflows/coverage-nightly.yaml`) implements a hierarchical merge strategy to handle disk space constraints and optimize artifact transfers. + +```mermaid +flowchart TD + Start([Workflow Trigger]) --> Compile[Job: compile
Build with Coverage] + + Compile --> Build[Configure --enable-coverage
Build all binaries] + Build --> Pack[Create testpack.tar.bz2] + Pack --> UploadBuild[Upload: cln-coverage-build] + + UploadBuild --> TestMatrix{Test Matrix
4 Runners} + + TestMatrix --> Test1[sqlite-1
group 1/2] + TestMatrix --> Test2[sqlite-2
group 2/2] + TestMatrix --> Test3[postgres-1
group 1/2] + TestMatrix --> Test4[postgres-2
group 2/2] + + Test1 --> Setup1[Download build
Unpack testpack] + Test2 --> Setup2[Download build
Unpack testpack] + Test3 --> Setup3[Download build
Unpack testpack] + Test4 --> Setup4[Download build
Unpack testpack] + + Setup1 --> Run1[Run pytest
--test-group 1
Generate .profraw files] + Setup2 --> Run2[Run pytest
--test-group 2
Generate .profraw files] + Setup3 --> Run3[Run pytest
--test-group 1
Generate .profraw files] + Setup4 --> Run4[Run pytest
--test-group 2
Generate .profraw files] + + Run1 --> Merge1[collect-coverage.sh
Merge to sqlite-1.profdata] + Run2 --> Merge2[collect-coverage.sh
Merge to sqlite-2.profdata] + Run3 --> Merge3[collect-coverage.sh
Merge to postgres-1.profdata] + Run4 --> Merge4[collect-coverage.sh
Merge to postgres-2.profdata] + + Merge1 --> Clean1[Delete .profraw files
Free disk space] + Merge2 --> Clean2[Delete .profraw files
Free disk space] + Merge3 --> Clean3[Delete .profraw files
Free disk space] + Merge4 --> Clean4[Delete .profraw files
Free disk space] + + Clean1 --> Upload1[Upload: coverage-merged-sqlite-1] + Clean2 --> Upload2[Upload: coverage-merged-sqlite-2] + Clean3 --> Upload3[Upload: coverage-merged-postgres-1] + Clean4 --> Upload4[Upload: coverage-merged-postgres-2] + + Upload1 --> Report[Job: report
Generate Final Report] + Upload2 --> Report + Upload3 --> Report + Upload4 --> Report + + Report --> Download[Download all
coverage-merged-* artifacts] + Download --> FinalMerge[llvm-profdata merge
4 .profdata → merged.profdata] + FinalMerge --> GenReport[generate-coverage-report.sh
Create HTML + text reports] + GenReport --> Codecov[Upload to Codecov] + GenReport --> UploadHTML[Upload HTML artifact] + GenReport --> Summary[Add to workflow summary] + + Codecov --> End([Complete]) + UploadHTML --> End + Summary --> End + + style Compile fill:#e1f5ff + style Test1 fill:#fff4e1 + style Test2 fill:#fff4e1 + style Test3 fill:#fff4e1 + style Test4 fill:#fff4e1 + style Report fill:#e8f5e9 + style Merge1 fill:#ffe1f5 + style Merge2 fill:#ffe1f5 + style Merge3 fill:#ffe1f5 + style Merge4 fill:#ffe1f5 + style FinalMerge fill:#ffe1f5 +``` + +### Workflow Phases + +#### 1. Compile Job +- Builds CLN with coverage instrumentation enabled +- Creates `testpack.tar.bz2` containing all binaries and test files +- Uploads as `cln-coverage-build` artifact (~2-3 GB) + +#### 2. Test Jobs (4 parallel runners) + +Each runner handles a subset of tests to stay within disk space limits: + +| Runner | Database | Test Group | Coverage Generated | +|--------|----------|------------|-------------------| +| sqlite-1 | SQLite | 1 of 2 | ~2-3 GB → ~500 MB | +| sqlite-2 | SQLite | 2 of 2 | ~2-3 GB → ~500 MB | +| postgres-1 | PostgreSQL | 1 of 2 | ~2-3 GB → ~500 MB | +| postgres-2 | PostgreSQL | 2 of 2 | ~2-3 GB → ~500 MB | + +**Per-runner process:** +1. Download and unpack build artifact +2. Run pytest with `--test-group-count 2 --test-group N` +3. Tests generate thousands of `.profraw` files (~5 GB) +4. Install LLVM tools (llvm-profdata) +5. Run `collect-coverage.sh` to merge into single `.profdata` (~500 MB) +6. Delete `.profraw` files (frees ~4.5 GB disk space) +7. Upload `.profdata` artifact + +**Key optimization:** Local aggregation before upload reduces: +- Artifact count: 4,000+ files → 4 files +- Artifact size: ~20 GB → ~2 GB +- Upload/download time: ~30 min → ~5 min +- Disk usage per runner: ~25 GB → ~15 GB + +#### 3. Report Job + +The final aggregation and reporting: + +1. Download all 4 `.profdata` artifacts +2. Merge them into final `merged.profdata` +3. Run `generate-coverage-report.sh` to create: + - HTML report (browsable coverage by file/function/line) + - Text summary (overall statistics) +4. Upload to Codecov for tracking +5. Store HTML report as workflow artifact (90 days retention) + +## Scripts Reference + +### collect-coverage.sh + +**Purpose:** Merge raw `.profraw` files into a single `.profdata` file + +**Usage:** +```bash +./collect-coverage.sh [COVERAGE_DIR] [OUTPUT_FILE] + +# Environment variables: +CLN_COVERAGE_DIR=/tmp/cln-coverage # Where to find .profraw files +``` + +**Features:** +- Validates each `.profraw` file before merging +- Filters out corrupt, empty, or incomplete files +- Handles "argument list too long" errors via batched merging +- Uses `-sparse` flag for efficient compression +- Parallelizes validation across CPU cores + +**Output:** +- `coverage/merged.profdata` (default) +- Validation statistics + +**Example:** +```bash +export CLN_COVERAGE_DIR=/tmp/cln-coverage +./contrib/coverage/collect-coverage.sh +# Output: coverage/merged.profdata +``` + +### generate-coverage-report.sh + +**Purpose:** Generate HTML and text coverage reports from merged profile data + +**Usage:** +```bash +./generate-coverage-report.sh [PROFDATA_FILE] [OUTPUT_DIR] +``` + +**Features:** +- Auto-discovers all instrumented binaries from Makefile +- Generates interactive HTML report with line-by-line coverage +- Generates text summary with file/function statistics +- Handles main binaries, plugins, and test programs + +**Output:** +- `coverage/html/index.html` - Browsable HTML report +- `coverage/summary.txt` - Text statistics + +**Example:** +```bash +./contrib/coverage/generate-coverage-report.sh coverage/merged.profdata +# Browse: coverage/html/index.html +``` + +### per-test-coverage.sh + +**Purpose:** Generate individual coverage reports for each test + +**Usage:** +```bash +./per-test-coverage.sh [COVERAGE_DIR] [OUTPUT_DIR] +``` + +**Requirements:** +- Tests must organize `.profraw` files into subdirectories by test name +- Requires `CLN_TEST_NAME` environment variable during test execution + +**Features:** +- Processes coverage data organized per test +- Creates `.profdata` and `.txt` summary for each test +- Helps identify which tests cover which code + +**Output:** +- `coverage/per-test/.profdata` +- `coverage/per-test/.txt` + +**Example:** +```bash +# During testing: +export CLN_COVERAGE_DIR=/tmp/cln-coverage +export CLN_TEST_NAME=test_connect # Creates subdir per test + +# After testing: +./contrib/coverage/per-test-coverage.sh +``` + +### per-test-coverage-html.sh + +**Purpose:** Generate HTML reports for each test's coverage + +**Usage:** +```bash +./per-test-coverage-html.sh [PROFDATA_DIR] [OUTPUT_DIR] +``` + +**Dependencies:** +- Requires `per-test-coverage.sh` to be run first + +**Output:** +- `coverage/per-test-html//index.html` for each test + +**Example:** +```bash +./contrib/coverage/per-test-coverage.sh +./contrib/coverage/per-test-coverage-html.sh +# Browse: coverage/per-test-html/*/index.html +``` + +### clang-coverage-report.sh + +**Purpose:** Simple script to generate HTML from a single `.profraw` file + +**Usage:** +```bash +./clang-coverage-report.sh BINARY RAW_PROFILE_FILE TARGET_HTML_FILE +``` + +**Use case:** Quick coverage check for a single test or binary + +**Example:** +```bash +./contrib/coverage/clang-coverage-report.sh \ + channeld/test/run-full_channel \ + full_channel.profraw \ + full_channel.html +``` + +### cleanup-corrupt-profraw.sh + +**Purpose:** Interactive utility to find and remove corrupt `.profraw` files + +**Usage:** +```bash +./cleanup-corrupt-profraw.sh [COVERAGE_DIR] +``` + +**Features:** +- Scans for corrupt/invalid `.profraw` files +- Uses `llvm-profdata show` to validate each file +- Asks for confirmation before deletion +- Useful when merging fails due to corrupt data + +**Example:** +```bash +./contrib/coverage/cleanup-corrupt-profraw.sh /tmp/cln-coverage +# Scans, lists corrupt files, prompts to delete +``` + +## Local Usage + +### Basic Local Coverage Collection + +1. **Build with coverage:** + ```bash + ./configure --enable-coverage CC=clang + make + ``` + +2. **Set coverage environment:** + ```bash + export CLN_COVERAGE_DIR=/tmp/cln-coverage + mkdir -p "$CLN_COVERAGE_DIR" + export LLVM_PROFILE_FILE="${CLN_COVERAGE_DIR}/%p-%m.profraw" + ``` + +3. **Run tests:** + ```bash + make check + # or + pytest tests/ + ``` + +4. **Collect and merge coverage:** + ```bash + ./contrib/coverage/collect-coverage.sh + ``` + +5. **Generate report:** + ```bash + ./contrib/coverage/generate-coverage-report.sh + open coverage/html/index.html + ``` + +### Per-Test Coverage + +To see which tests cover which code: + +1. **Enable test-name tracking during pytest:** + ```bash + # Modify pytest fixture to set CLN_TEST_NAME per test + export CLN_COVERAGE_DIR=/tmp/cln-coverage + pytest tests/ + ``` + +2. **Generate per-test reports:** + ```bash + ./contrib/coverage/per-test-coverage.sh + ./contrib/coverage/per-test-coverage-html.sh + ``` + +3. **Browse results:** + ```bash + open coverage/per-test-html/*/index.html + ``` + +## Troubleshooting + +### "No .profraw files found" + +**Cause:** Tests didn't generate coverage data + +**Solutions:** +- Verify binaries are instrumented: `nm lightningd/lightningd | grep __llvm_profile` +- Check `LLVM_PROFILE_FILE` environment variable is set +- Ensure `CLN_COVERAGE_DIR` directory exists and is writable + +### "Argument list too long" during merge + +**Cause:** Too many `.profraw` files to pass on command line + +**Solution:** `collect-coverage.sh` handles this automatically via batched merging + +### "Profile data is malformed" + +**Cause:** Corrupt or incomplete `.profraw` files (often from crashed tests) + +**Solutions:** +- Run `cleanup-corrupt-profraw.sh` to identify and remove corrupt files +- `collect-coverage.sh` automatically filters these out + +### High disk usage during CI + +**Cause:** Raw `.profraw` files are large and accumulate quickly + +**Solutions:** +- ✅ Split tests across more runners (reduce per-runner coverage generation) +- ✅ Merge locally and delete raw files (implemented in workflow) +- ✅ Use pytest-test-groups to partition tests evenly +- Free up disk space before tests: use GitHub Actions disk cleanup utilities + +### Binaries not found during report generation + +**Cause:** Scripts run from wrong directory, or binaries not built + +**Solutions:** +- Ensure you run scripts from repository root: `cd $GITHUB_WORKSPACE` +- Verify build completed: `ls lightningd/lightningd` +- Check configure used `--enable-coverage` + +## Technical Details + +### File Formats + +- **`.profraw`**: Raw binary profile data (uncompressed, ~1-5 MB per file) +- **`.profdata`**: Merged, indexed profile data (compressed, sparse format) + +### LLVM Tools + +- **`llvm-profdata`**: Merges, indexes, and validates profile data +- **`llvm-cov show`**: Generates HTML with source code and line coverage +- **`llvm-cov report`**: Generates text summary tables + +### GitHub Actions Constraints + +- **Disk space**: ~25-29 GB available per runner (ubuntu-22.04) +- **Artifact size**: Recommend keeping under 1 GB per artifact +- **Concurrency**: 4 parallel test runners balance speed vs. resource usage + +### Coverage Overhead + +- **Build time**: +10-15% (instrumentation) +- **Binary size**: +50-100% (instrumentation code) +- **Runtime**: +20-30% (profiling overhead) +- **Disk usage**: ~1-2 MB per test (raw profraw files) + +## References + +- [LLVM Source-Based Code Coverage](https://clang.llvm.org/docs/SourceBasedCodeCoverage.html) +- [GitHub Actions: Ubuntu Runner Specs](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners) +- [pytest-test-groups Plugin](https://github.com/mark-adams/pytest-test-groups) diff --git a/contrib/coverage/clang-coverage-report.sh b/contrib/coverage/clang-coverage-report.sh new file mode 100755 index 000000000000..246163ce55cb --- /dev/null +++ b/contrib/coverage/clang-coverage-report.sh @@ -0,0 +1,27 @@ +#!/bin/bash -eu +# +# Generates an HTML coverage report from a raw Clang coverage profile. See +# https://clang.llvm.org/docs/SourceBasedCodeCoverage.html for more details. +# +# Example usage to create full_channel.html from full_channel.profraw for the +# run-full_channel unit test: +# ./contrib/clang-coverage-report.sh channeld/test/run-full_channel \ +# full_channel.profraw full_channel.html + +if [[ "$#" -ne 3 ]]; then + echo "Usage: $0 BINARY RAW_PROFILE_FILE TARGET_HTML_FILE" + exit 1 +fi + +readonly BINARY="$1" +readonly RAW_PROFILE_FILE="$2" +readonly TARGET_HTML_FILE="$3" + +MERGED_PROFILE_FILE=$(mktemp) +readonly MERGED_PROFILE_FILE + +llvm-profdata merge -sparse "${RAW_PROFILE_FILE}" -o "${MERGED_PROFILE_FILE}" +llvm-cov show "${BINARY}" -instr-profile="${MERGED_PROFILE_FILE}" -format=html \ + > "${TARGET_HTML_FILE}" + +rm "${MERGED_PROFILE_FILE}" diff --git a/contrib/coverage/cleanup-corrupt-profraw.sh b/contrib/coverage/cleanup-corrupt-profraw.sh new file mode 100755 index 000000000000..13b104bdabbd --- /dev/null +++ b/contrib/coverage/cleanup-corrupt-profraw.sh @@ -0,0 +1,55 @@ +#!/bin/bash -eu +# Remove corrupt .profraw files from the coverage directory +# Usage: ./cleanup-corrupt-profraw.sh [COVERAGE_DIR] + +COVERAGE_DIR="${1:-${CLN_COVERAGE_DIR:-/tmp/cln-coverage}}" + +if [ ! -d "$COVERAGE_DIR" ]; then + echo "Coverage directory not found: $COVERAGE_DIR" + exit 1 +fi + +echo "Scanning for corrupt profraw files in: $COVERAGE_DIR" + +# Find all profraw files +mapfile -t PROFRAW_FILES < <(find "$COVERAGE_DIR" -name "*.profraw" 2>/dev/null || true) + +if [ ${#PROFRAW_FILES[@]} -eq 0 ]; then + echo "No .profraw files found" + exit 0 +fi + +echo "Found ${#PROFRAW_FILES[@]} profile files" + +CORRUPT_FILES=() +for profraw in "${PROFRAW_FILES[@]}"; do + # Try to validate the file + if ! llvm-profdata show "$profraw" >/dev/null 2>&1; then + CORRUPT_FILES+=("$profraw") + fi +done + +if [ ${#CORRUPT_FILES[@]} -eq 0 ]; then + echo "✓ No corrupt files found" + exit 0 +fi + +echo "" +echo "Found ${#CORRUPT_FILES[@]} corrupt file(s):" +for corrupt in "${CORRUPT_FILES[@]}"; do + echo " - $corrupt" +done + +echo "" +read -p "Delete these corrupt files? [y/N] " -n 1 -r +echo + +if [[ $REPLY =~ ^[Yy]$ ]]; then + for corrupt in "${CORRUPT_FILES[@]}"; do + rm -f "$corrupt" + echo " Deleted: $corrupt" + done + echo "✓ Removed ${#CORRUPT_FILES[@]} corrupt file(s)" +else + echo "No files were deleted" +fi diff --git a/contrib/coverage/collect-coverage.sh b/contrib/coverage/collect-coverage.sh new file mode 100755 index 000000000000..f6fa6eefda88 --- /dev/null +++ b/contrib/coverage/collect-coverage.sh @@ -0,0 +1,122 @@ +#!/bin/bash -eu +# Merge all .profraw files into a single .profdata file +# Usage: ./collect-coverage.sh [COVERAGE_DIR] [OUTPUT_FILE] + +COVERAGE_DIR="${1:-${CLN_COVERAGE_DIR:-/tmp/cln-coverage}}" +OUTPUT="${2:-coverage/merged.profdata}" + +echo "Collecting coverage from: $COVERAGE_DIR" + +# Find all profraw files +mapfile -t PROFRAW_FILES < <(find "$COVERAGE_DIR" -name "*.profraw" 2>/dev/null || true) + +if [ ${#PROFRAW_FILES[@]} -eq 0 ]; then + echo "ERROR: No .profraw files found in $COVERAGE_DIR" + exit 1 +fi + +echo "Found ${#PROFRAW_FILES[@]} profile files" + +# Validate each profraw file and filter out corrupt/incomplete ones +# Define validation function for parallel execution +validate_file() { + local profraw="$1" + + # Check if file is empty + if [ ! -s "$profraw" ]; then + return 1 # Empty + fi + + # Check if file is suspiciously small (likely incomplete write) + # Valid profraw files are typically > 1KB + filesize=$(stat -c%s "$profraw" 2>/dev/null || stat -f%z "$profraw" 2>/dev/null) + if [ "$filesize" -lt 1024 ]; then + return 2 # Too small + fi + + # Try to validate the file by checking if llvm-profdata can read it + if llvm-profdata show "$profraw" >/dev/null 2>&1; then + echo "$profraw" # Valid - output to stdout + return 0 + else + return 3 # Corrupt + fi +} + +# Export function for parallel execution +export -f validate_file + +TOTAL=${#PROFRAW_FILES[@]} +NPROC=$(nproc 2>/dev/null || echo 4) +echo "Validating ${TOTAL} files in parallel (using ${NPROC} cores)..." + +# Run validation in parallel and collect valid files +mapfile -t VALID_FILES < <( + printf '%s\n' "${PROFRAW_FILES[@]}" | \ + xargs -P "$NPROC" -I {} bash -c 'validate_file "$@"' _ {} +) + +# Calculate error counts +CORRUPT_COUNT=$((TOTAL - ${#VALID_FILES[@]})) + +if [ ${#VALID_FILES[@]} -eq 0 ]; then + echo "ERROR: No valid .profraw files found (all $CORRUPT_COUNT files were corrupt/incomplete)" + echo "" + echo "Sample validation error from first file:" + llvm-profdata show "${PROFRAW_FILES[0]}" 2>&1 | head -10 + echo "" + echo "File size: $(stat -c%s "${PROFRAW_FILES[0]}" 2>/dev/null || stat -f%z "${PROFRAW_FILES[0]}" 2>/dev/null) bytes" + echo "File: ${PROFRAW_FILES[0]}" + exit 1 +fi + +echo "Valid files: ${#VALID_FILES[@]}" +if [ $CORRUPT_COUNT -gt 0 ]; then + echo "Filtered out: $CORRUPT_COUNT files (empty/small/corrupt)" +fi +mkdir -p "$(dirname "$OUTPUT")" + +# Merge with -sparse flag for efficiency +# Use batched merging to avoid "Argument list too long" errors +BATCH_SIZE=500 +TOTAL_FILES=${#VALID_FILES[@]} + +if [ "$TOTAL_FILES" -le "$BATCH_SIZE" ]; then + # Small enough to merge in one go + echo "Merging ${TOTAL_FILES} files..." + llvm-profdata merge -sparse "${VALID_FILES[@]}" -o "$OUTPUT" +else + # Need to merge in batches + echo "Merging ${TOTAL_FILES} files in batches of ${BATCH_SIZE}..." + + # Create temp directory for intermediate files + TEMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/profdata-merge.XXXXXX") + trap 'rm -rf "$TEMP_DIR"' EXIT + + BATCH_NUM=0 + INTERMEDIATE_FILES=() + + # Merge files in batches + for ((i=0; i/dev/null | awk '/^ALL_PROGRAMS :=/ {$1=$2=""; print}' | tr ' ' '\n' | grep -v '^$') +mapfile -t TEST_BINARIES < <(make -qp 2>/dev/null | awk '/^ALL_TEST_PROGRAMS :=/ {$1=$2=""; print}' | tr ' ' '\n' | grep -v '^$') + +# Combine all binaries +ALL_BINARIES=("${BINARIES[@]}" "${TEST_BINARIES[@]}") + +# Build llvm-cov arguments +ARGS=() +for bin in "${ALL_BINARIES[@]}"; do + if [ -f "$bin" ]; then + if [ ${#ARGS[@]} -eq 0 ]; then + ARGS+=("$bin") # First binary is primary + else + ARGS+=("-object=$bin") # Others use -object= + fi + fi +done + +if [ ${#ARGS[@]} -eq 0 ]; then + echo "ERROR: No instrumented binaries found" + echo "Make sure you've built with --enable-coverage" + exit 1 +fi + +echo "Generating coverage report for ${#ARGS[@]} binaries..." + +# Generate HTML report +llvm-cov show "${ARGS[@]}" \ + -instr-profile="$PROFDATA" \ + -format=html \ + -output-dir="$OUTPUT_DIR" \ + -show-line-counts-or-regions \ + -show-instantiations=false + +echo "✓ HTML report: $OUTPUT_DIR/index.html" + +# Generate text summary +mkdir -p coverage +llvm-cov report "${ARGS[@]}" \ + -instr-profile="$PROFDATA" \ + | tee coverage/summary.txt + +echo "✓ Summary: coverage/summary.txt" diff --git a/contrib/coverage/per-test-coverage-html.sh b/contrib/coverage/per-test-coverage-html.sh new file mode 100755 index 000000000000..a4deb2320f41 --- /dev/null +++ b/contrib/coverage/per-test-coverage-html.sh @@ -0,0 +1,86 @@ +#!/bin/bash -eu +# Generate HTML coverage reports for each test +# Usage: ./per-test-coverage-html.sh [PROFDATA_DIR] [OUTPUT_DIR] + +PROFDATA_DIR="${1:-coverage/per-test}" +OUTPUT_DIR="${2:-coverage/per-test-html}" + +if [ ! -d "$PROFDATA_DIR" ]; then + echo "ERROR: Profdata directory not found: $PROFDATA_DIR" + echo "Run ./contrib/coverage/per-test-coverage.sh first" + exit 1 +fi + +# Get all binaries from Makefile (includes plugins, tools, test binaries) +echo "Discovering instrumented binaries from Makefile..." +mapfile -t BINARIES < <(make -qp 2>/dev/null | awk '/^ALL_PROGRAMS :=/ {$1=$2=""; print}' | tr ' ' '\n' | grep -v '^$') +mapfile -t TEST_BINARIES < <(make -qp 2>/dev/null | awk '/^ALL_TEST_PROGRAMS :=/ {$1=$2=""; print}' | tr ' ' '\n' | grep -v '^$') + +# Combine all binaries +ALL_BINARIES=("${BINARIES[@]}" "${TEST_BINARIES[@]}") + +# Build llvm-cov arguments +ARGS=() +for bin in "${ALL_BINARIES[@]}"; do + if [ -f "$bin" ]; then + if [ ${#ARGS[@]} -eq 0 ]; then + ARGS+=("$bin") # First binary is primary + else + ARGS+=("-object=$bin") # Others use -object= + fi + fi +done + +if [ ${#ARGS[@]} -eq 0 ]; then + echo "ERROR: No instrumented binaries found" + echo "Make sure you've built with --enable-coverage" + exit 1 +fi + +# Find all profdata files +mapfile -t PROFDATA_FILES < <(find "$PROFDATA_DIR" -name "*.profdata" 2>/dev/null | sort) + +if [ ${#PROFDATA_FILES[@]} -eq 0 ]; then + echo "ERROR: No .profdata files found in $PROFDATA_DIR" + echo "Run ./contrib/coverage/per-test-coverage.sh first" + exit 1 +fi + +mkdir -p "$OUTPUT_DIR" + +echo "Generating HTML reports for ${#PROFDATA_FILES[@]} tests..." +echo "Using ${#ARGS[@]} instrumented binaries" +echo "" + +# Generate HTML for each test +for profdata in "${PROFDATA_FILES[@]}"; do + test_name=$(basename "$profdata" .profdata) + html_dir="$OUTPUT_DIR/$test_name" + + echo -n " $test_name... " + + # Generate HTML report + if llvm-cov show "${ARGS[@]}" \ + -instr-profile="$profdata" \ + -format=html \ + -output-dir="$html_dir" \ + -show-line-counts-or-regions \ + -show-instantiations=false 2>/dev/null; then + echo "✓" + else + echo "✗ (failed)" + rm -rf "$html_dir" + fi +done + +echo "" +echo "HTML reports generated in: $OUTPUT_DIR" +echo "" +echo "Open reports:" +for profdata in "${PROFDATA_FILES[@]}"; do + test_name=$(basename "$profdata" .profdata) + html_dir="$OUTPUT_DIR/$test_name" + if [ -f "$html_dir/index.html" ]; then + echo " $test_name: $html_dir/index.html" + fi +done diff --git a/contrib/coverage/per-test-coverage.sh b/contrib/coverage/per-test-coverage.sh new file mode 100755 index 000000000000..29cb27ecb222 --- /dev/null +++ b/contrib/coverage/per-test-coverage.sh @@ -0,0 +1,108 @@ +#!/bin/bash -eu +# Generate per-test coverage reports +# Usage: ./per-test-coverage.sh [COVERAGE_DIR] [OUTPUT_DIR] + +COVERAGE_DIR="${1:-${CLN_COVERAGE_DIR:-/tmp/cln-coverage}}" +OUTPUT_DIR="${2:-coverage/per-test}" + +if [ ! -d "$COVERAGE_DIR" ]; then + echo "ERROR: Coverage directory not found: $COVERAGE_DIR" + exit 1 +fi + +# Get all binaries from Makefile (includes plugins, tools, test binaries) +echo "Discovering instrumented binaries from Makefile..." +mapfile -t BINARIES < <(make -qp 2>/dev/null | awk '/^ALL_PROGRAMS :=/ {$1=$2=""; print}' | tr ' ' '\n' | grep -v '^$') +mapfile -t TEST_BINARIES < <(make -qp 2>/dev/null | awk '/^ALL_TEST_PROGRAMS :=/ {$1=$2=""; print}' | tr ' ' '\n' | grep -v '^$') + +# Combine all binaries +ALL_BINARIES=("${BINARIES[@]}" "${TEST_BINARIES[@]}") + +# Build llvm-cov arguments +BINARY_ARGS=() +for bin in "${ALL_BINARIES[@]}"; do + if [ -f "$bin" ]; then + if [ ${#BINARY_ARGS[@]} -eq 0 ]; then + BINARY_ARGS+=("$bin") # First binary is primary + else + BINARY_ARGS+=("-object=$bin") # Others use -object= + fi + fi +done + +if [ ${#BINARY_ARGS[@]} -eq 0 ]; then + echo "ERROR: No instrumented binaries found" + echo "Make sure you've built with --enable-coverage" + exit 1 +fi + +mkdir -p "$OUTPUT_DIR" + +# Find all test subdirectories +mapfile -t TEST_DIRS < <(find "$COVERAGE_DIR" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort) + +if [ ${#TEST_DIRS[@]} -eq 0 ]; then + echo "ERROR: No test subdirectories found in $COVERAGE_DIR" + echo "Note: Test organization requires CLN_TEST_NAME to be set" + exit 1 +fi + +echo "Generating coverage for ${#TEST_DIRS[@]} tests..." + +# Process each test +for test_dir in "${TEST_DIRS[@]}"; do + test_name=$(basename "$test_dir") + echo -n " $test_name... " + + # Find profraw files for this test + mapfile -t PROFRAW_FILES < <(find "$test_dir" -name "*.profraw" 2>/dev/null || true) + + if [ ${#PROFRAW_FILES[@]} -eq 0 ]; then + echo "no profraw files" + continue + fi + + # Validate and filter profraw files + VALID_FILES=() + for profraw in "${PROFRAW_FILES[@]}"; do + if [ -s "$profraw" ]; then + filesize=$(stat -c%s "$profraw" 2>/dev/null || stat -f%z "$profraw" 2>/dev/null) + if [ "$filesize" -ge 1024 ]; then + if llvm-profdata show "$profraw" >/dev/null 2>&1; then + VALID_FILES+=("$profraw") + fi + fi + fi + done + + if [ ${#VALID_FILES[@]} -eq 0 ]; then + echo "no valid files" + continue + fi + + # Merge profraw files for this test + test_profdata="$OUTPUT_DIR/$test_name.profdata" + if ! llvm-profdata merge -sparse "${VALID_FILES[@]}" -o "$test_profdata" 2>/dev/null; then + echo "merge failed" + continue + fi + + # Generate text summary for this test + llvm-cov report "${BINARY_ARGS[@]}" \ + -instr-profile="$test_profdata" \ + > "$OUTPUT_DIR/$test_name.txt" 2>/dev/null || { + echo "report failed" + rm -f "$test_profdata" + continue + } + + echo "✓ (${#VALID_FILES[@]} files)" +done + +echo "" +echo "Per-test coverage reports in: $OUTPUT_DIR" +echo " - *.profdata - Merged profile data per test" +echo " - *.txt - Text coverage summary per test" +echo "" +echo "To generate HTML reports for all tests:" +echo " ./contrib/coverage/per-test-coverage-html.sh"