diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..98e9694 --- /dev/null +++ b/.flake8 @@ -0,0 +1,19 @@ +[flake8] +# Ignore Jupyter notebook files +exclude = + .git, + __pycache__, + .ipynb_checkpoints, + *.ipynb, + build, + dist, + site, + gh-pages + +# Maximum line length +max-line-length = 100 + +# Ignore specific error codes if needed +# E501: line too long (handled by max-line-length) +# W503: line break before binary operator (not a real issue) +ignore = W503 diff --git a/.github/workflows/blank.yml b/.github/workflows/blank.yml deleted file mode 100644 index 01502b1..0000000 --- a/.github/workflows/blank.yml +++ /dev/null @@ -1,36 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: CI - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on - runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4 - - # Runs a single command using the runners shell - - name: Run a one-line script - run: echo Hello, world! - - # Runs a set of commands using the runners shell - - name: Run a multi-line script - run: | - echo Add other actions to build, - echo test, and deploy your project. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f02c0a7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,53 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Lint (flake8) + run: | + flake8 . + + - name: Run unit tests (pytest) + run: | + pytest -q --maxfail=1 + + - name: Execute all notebooks (nbconvert) + run: | + python - <<'PY' + import glob, subprocess, sys + notebooks = glob.glob('**/*.ipynb', recursive=True) + if not notebooks: + print("No notebooks found.") + sys.exit(0) + for n in notebooks: + print("Executing", n) + subprocess.check_call([ + sys.executable, "-m", "jupyter", "nbconvert", + "--to", "notebook", "--execute", "--inplace", + "--ExecutePreprocessor.timeout=600", n + ]) + PY diff --git a/.github/workflows/deploy-notebooks-pages.yml b/.github/workflows/deploy-notebooks-pages.yml new file mode 100644 index 0000000..5e73843 --- /dev/null +++ b/.github/workflows/deploy-notebooks-pages.yml @@ -0,0 +1,90 @@ +on: + push: + branches: + - '**' + +name: Build and deploy notebooks to GitHub Pages (per-branch) + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + permissions: + contents: write # needed to push gh-pages + steps: + - name: Checkout repository (current branch) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Determine branch name + id: branch + run: | + echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" > $GITHUB_ENV + echo "branch=${GITHUB_REF#refs/heads/}" > $GITHUB_OUTPUT + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; else pip install jupyter nbconvert nbformat; fi + + - name: Build notebooks to HTML for this branch + run: | + mkdir -p site/${{ env.BRANCH_NAME }} + python scripts/build_notebooks.py site/${{ env.BRANCH_NAME }} + + - name: Prepare gh-pages branch workspace + run: | + # If gh-pages exists, clone it, otherwise init an empty gh-pages branch + REPO_URL="https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git" + if git ls-remote --exit-code origin gh-pages; then + git clone --depth 1 --branch gh-pages "$REPO_URL" gh-pages || git clone --single-branch --branch gh-pages "$REPO_URL" gh-pages + else + # create temporary folder, init gh-pages and push + mkdir gh-pages + cd gh-pages + git init + git remote add origin "$REPO_URL" + git checkout -b gh-pages || true + touch .nojekyll + git add .nojekyll + git commit -m "Initialize gh-pages" + git push origin gh-pages + cd .. + git clone --depth 1 --branch gh-pages "$REPO_URL" gh-pages + fi + + - name: Copy built site into gh-pages under branch folder + run: | + set -e + rsync -a --delete site/${{ env.BRANCH_NAME }}/ gh-pages/${{ env.BRANCH_NAME }}/ + # ensure a top-level index exists listing branches (optional) + python3 - <<'PY' +import os, json +root='gh-pages' +items = sorted([d for d in os.listdir(root) if os.path.isdir(os.path.join(root,d))]) +index_path = os.path.join(root, 'index.html') +with open(index_path, 'w') as f: + f.write("Published Branches") + f.write("

Published branches

") +PY + + - name: Commit and push changes to gh-pages + working-directory: gh-pages + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add --all + if git diff --quiet --cached; then + echo "No changes to deploy" + else + git commit -m "Deploy notebooks for branch '${{ env.BRANCH_NAME }}' [ci skip]" + git push origin gh-pages + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b3c1f27 --- /dev/null +++ b/.gitignore @@ -0,0 +1,50 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Jupyter Notebook +.ipynb_checkpoints + +# Testing +.pytest_cache/ +.coverage +htmlcov/ + +# Build outputs +site/ +gh-pages/ + +# Virtual environments +venv/ +ENV/ +env/ +.venv + +# IDEs +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..158c545 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,82 @@ +# Contributing + +## CI/CD Workflows + +This repository includes automated workflows for building, testing, and deploying Jupyter notebooks. + +### CI Workflow (`.github/workflows/ci.yml`) + +The CI workflow runs on every push and pull request to the `main` branch. It: + +1. **Lints** the code using `flake8` to ensure code quality +2. **Runs unit tests** using `pytest` to validate functionality +3. **Executes all notebooks** to ensure they run without errors + +To run these checks locally before pushing: + +```bash +# Install dependencies +pip install -r requirements.txt + +# Run linting +flake8 . + +# Run tests +pytest -q --maxfail=1 + +# Execute notebooks +python scripts/build_notebooks.py /tmp/test-output +``` + +### Notebook Deployment Workflow (`.github/workflows/deploy-notebooks-pages.yml`) + +The deployment workflow runs on every push to **any branch** and: + +1. Executes all notebooks in the repository +2. Converts them to HTML +3. Publishes them to GitHub Pages under a branch-specific folder + +The deployed notebooks are available at: +- `https://.github.io///` for branch-specific builds +- `https://.github.io//` for an index of all published branches + +### Building Notebooks Locally + +To build notebooks locally for testing: + +```bash +# Install dependencies +pip install -r requirements.txt + +# Build notebooks to HTML +python scripts/build_notebooks.py output/ + +# View the generated HTML +open output/index.html # macOS +xdg-open output/index.html # Linux +``` + +### Adding New Notebooks + +1. Create your `.ipynb` notebook file anywhere in the repository +2. Commit and push to your branch +3. The CI workflow will validate that the notebook executes successfully +4. The deployment workflow will automatically build and publish the notebook to GitHub Pages + +### Troubleshooting + +#### Notebook Execution Timeout + +If a notebook takes longer than 600 seconds (10 minutes) to execute, you may need to: +- Optimize the notebook code +- Split it into smaller notebooks +- Increase the timeout in the workflow files + +#### Flake8 Linting Errors + +If you have Python code in the repository (not in notebooks), ensure it follows PEP 8 style guidelines: +- Maximum line length: 79 characters (default) +- Proper indentation and spacing +- No unused imports + +To ignore certain flake8 errors, create a `.flake8` configuration file. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4c1d1b1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +# Core Jupyter dependencies +jupyter>=1.0.0 +nbconvert>=7.0.0 +nbformat>=5.0.0 + +# Testing and linting +pytest>=7.0.0 +flake8>=6.0.0 diff --git a/scripts/build_notebooks.py b/scripts/build_notebooks.py new file mode 100755 index 0000000..01f8723 --- /dev/null +++ b/scripts/build_notebooks.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +""" +Execute all notebooks in the repository (recursive) and export them to HTML into an output folder. + +Usage: + python scripts/build_notebooks.py + +Example: + python scripts/build_notebooks.py site/main +""" + +import sys +import os +import glob +import subprocess +import html + + +def main(): + if len(sys.argv) < 2: + print("Usage: python scripts/build_notebooks.py ") + sys.exit(1) + + output_dir = sys.argv[1] + os.makedirs(output_dir, exist_ok=True) + + # Find all notebooks in the repository + notebooks = glob.glob('**/*.ipynb', recursive=True) + if not notebooks: + print("No notebooks found.") + # Create a simple index page + index_path = os.path.join(output_dir, 'index.html') + with open(index_path, 'w') as f: + f.write("

No notebooks found

") + sys.exit(0) + + print(f"Found {len(notebooks)} notebook(s)") + + # Execute and convert each notebook to HTML + html_files = [] + for nb in notebooks: + print(f"Processing: {nb}") + try: + # Execute the notebook + subprocess.check_call([ + sys.executable, "-m", "jupyter", "nbconvert", + "--to", "html", + "--execute", + "--ExecutePreprocessor.timeout=600", + "--output-dir", output_dir, + nb + ]) + # Track the HTML file that was created + nb_basename = os.path.basename(nb).replace('.ipynb', '.html') + html_files.append(nb_basename) + print(f" āœ“ Successfully converted: {nb}") + except subprocess.CalledProcessError as e: + print(f" āœ— Failed to convert {nb}: {e}") + # Create an error page + error_html = os.path.join( + output_dir, os.path.basename(nb).replace('.ipynb', '.html')) + with open(error_html, 'w') as f: + error_msg = f"

Execution failed for {nb}

" + error_msg += f"
{html.escape(str(e))}
" + f.write(error_msg) + html_files.append(os.path.basename(error_html)) + + # Create an index.html listing all notebooks + index_path = os.path.join(output_dir, 'index.html') + with open(index_path, 'w') as f: + f.write("Notebook Index") + f.write("

Notebooks

") + f.write("
    ") + for html_file in sorted(html_files): + f.write(f'
  • {html_file}
  • ') + f.write("
") + f.write("") + + print(f"\nāœ“ Build complete. Output in: {output_dir}") + print(f" Generated index: {index_path}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_build_notebooks.py b/tests/test_build_notebooks.py new file mode 100644 index 0000000..806fe0c --- /dev/null +++ b/tests/test_build_notebooks.py @@ -0,0 +1,25 @@ +""" +Basic tests for the notebook build script. +""" +import os +from pathlib import Path + + +def test_build_script_exists(): + """Verify the build script exists and is executable.""" + script_path = Path(__file__).parent.parent / "scripts" / "build_notebooks.py" + assert script_path.exists(), "build_notebooks.py script should exist" + assert os.access(script_path, os.X_OK), "build_notebooks.py should be executable" + + +def test_requirements_file_exists(): + """Verify requirements.txt exists.""" + req_path = Path(__file__).parent.parent / "requirements.txt" + assert req_path.exists(), "requirements.txt should exist" + + # Check that it contains expected packages + content = req_path.read_text() + assert "jupyter" in content + assert "nbconvert" in content + assert "pytest" in content + assert "flake8" in content