diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c0a77f5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,51 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Lint (flake8) + run: | + flake8 . + + - name: Run unit tests (pytest) + run: | + pytest -q --maxfail=1 + + - name: Execute all notebooks (nbconvert) + run: | + python - <<'PY' + import glob, subprocess, sys + notebooks = glob.glob('**/*.ipynb', recursive=True) + if not notebooks: + print("No notebooks found.") + sys.exit(0) + for n in notebooks: + print("Executing", n) + subprocess.check_call([ + sys.executable, "-m", "jupyter", "nbconvert", + "--to", "notebook", "--execute", "--inplace", + "--ExecutePreprocessor.timeout=600", n + ]) + PY diff --git a/.github/workflows/deploy-notebooks-pages.yml b/.github/workflows/deploy-notebooks-pages.yml new file mode 100644 index 0000000..34795c2 --- /dev/null +++ b/.github/workflows/deploy-notebooks-pages.yml @@ -0,0 +1,91 @@ +on: + push: + branches: + - '**' + +name: Build and deploy notebooks to GitHub Pages (per-branch) + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + permissions: + contents: write # needed to push gh-pages + steps: + - name: Checkout repository (current branch) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Determine branch name + id: branch + run: | + echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; else pip install jupyter nbconvert nbformat; fi + + - name: Build notebooks to HTML for this branch + run: | + python -m pip install --upgrade pip + mkdir -p site/${{ env.BRANCH_NAME }} + python scripts/build_notebooks.py site/${{ env.BRANCH_NAME }} + + - name: Prepare gh-pages branch workspace + run: | + # If gh-pages exists, clone it, otherwise init an empty gh-pages branch + REPO_URL="https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git" + if git ls-remote --exit-code origin gh-pages; then + git clone --depth 1 --branch gh-pages "$REPO_URL" gh-pages || git clone "$REPO_URL" gh-pages + else + # create temporary folder, init gh-pages and push + mkdir gh-pages + cd gh-pages + git init + git remote add origin "$REPO_URL" + git checkout -b gh-pages || true + touch .nojekyll + git add .nojekyll + git commit -m "Initialize gh-pages" + git push origin gh-pages + cd .. + git clone --depth 1 --branch gh-pages "$REPO_URL" gh-pages + fi + + - name: Copy built site into gh-pages under branch folder + run: | + set -e + rsync -a --delete site/${{ env.BRANCH_NAME }}/ gh-pages/${{ env.BRANCH_NAME }}/ + # ensure a top-level index exists listing branches (optional) + python3 - <<'PY' +import os, json +root='gh-pages' +items = sorted([d for d in os.listdir(root) if os.path.isdir(os.path.join(root,d))]) +index_path = os.path.join(root, 'index.html') +with open(index_path, 'w') as f: + f.write("Branches") + f.write("

Published branches

") +PY + + - name: Commit and push changes to gh-pages + working-directory: gh-pages + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add --all + if git diff --quiet --cached; then + echo "No changes to deploy" + else + git commit -m "Deploy notebooks for branch '${{ env.BRANCH_NAME }}' [ci skip]" + git push origin gh-pages + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5edaeb2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.env +.venv +__pycache__/ +site/ +gh-pages/ +*.pyc +.ipynb_checkpoints/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..b7e3bc4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +repos: + - repo: https://github.com/psf/black + rev: 24.1.0 + hooks: + - id: black + language_version: python3.11 + + - repo: https://gitlab.com/pycqa/flake8 + rev: 6.0.0 + hooks: + - id: flake8 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6f4b574 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.11-slim + +ENV DEBIAN_FRONTEND=noninteractive +WORKDIR /app +COPY requirements.txt /app/requirements.txt +RUN python -m pip install --upgrade pip && \ + pip install --no-cache-dir -r /app/requirements.txt +COPY . /app +CMD ["bash"] diff --git a/README.md b/README.md index a161a49..217c7e2 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,17 @@ -# - -~ +# Repository: Notebook pages (branch-per-branch) + +This repository contains many Jupyter notebooks. This change adds CI and an automated build-and-deploy workflow that runs on pushes to any branch and publishes rendered HTML pages for that branch under gh-pages/. + +Files added: +- .github/workflows/ci.yml — runs lint/tests and executes notebooks on pushes/PRs to main. +- .github/workflows/deploy-notebooks-pages.yml — builds and deploys per-branch HTML pages to gh-pages when any branch is pushed. +- scripts/build_notebooks.py — script that executes notebooks and exports them to HTML. +- requirements.txt, Dockerfile, .pre-commit-config.yaml, .gitignore — tooling and environment files. + +How to run locally: +1. python -m venv .venv +2. source .venv/bin/activate +3. pip install -r requirements.txt +4. python scripts/build_notebooks.py site/local-branch + +Enable GitHub Pages to serve the gh-pages branch in repository Settings -> Pages. The per-branch pages will be available at https://.github.io/// . diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ed50aa2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +jupyter +nbconvert +nbformat +pytest +flake8 +black +pre-commit diff --git a/scripts/build_notebooks.py b/scripts/build_notebooks.py index 926d509..d2d3f31 100644 --- a/scripts/build_notebooks.py +++ b/scripts/build_notebooks.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 """ Execute all notebooks in the repository (recursive) and export them to HTML into an output folder. +Execute all notebooks in the repository (recursive) and export them to +HTML into an output folder. Usage: python scripts/build_notebooks.py [--timeout SECONDS] @@ -10,6 +12,16 @@ - export resulting notebook to HTML and place the HTML in preserving folder structure """ import sys +import os +import subprocess +from pathlib import Path + +By default this will search for all .ipynb files (excluding +.ipynb_checkpoints) and: +- execute them with a timeout +- export resulting notebook to HTML and place the HTML in preserving folder structure +""" +import sys import subprocess from pathlib import Path @@ -17,6 +29,9 @@ def find_notebooks(root="."): nbs = [] for p in Path(root).rglob("*.ipynb"): # skip checkpoints and files inside .git or site output + if ".ipynb_checkpoints" in p.parts or "site" in p.parts or "gh-pages" in p.parts: + if (".ipynb_checkpoints" in p.parts or "site" in p.parts or + "gh-pages" in p.parts): if (".ipynb_checkpoints" in p.parts or "site" in p.parts or "gh-pages" in p.parts): continue nbs.append(p) @@ -30,6 +45,7 @@ def main(): timeout = 600 if "--timeout" in sys.argv: try: + timeout = int(sys.argv[sys.argv.index("--timeout")+1]) timeout = int(sys.argv[sys.argv.index("--timeout") + 1]) except Exception: pass @@ -42,6 +58,7 @@ def main(): print(f"Found {len(notebooks)} notebooks. Exporting to {outdir} ...") for nb in notebooks: + rel = nb.relative_to(Path.cwd()) # Handle both absolute and relative paths try: rel = nb.relative_to(Path.cwd()) @@ -56,12 +73,21 @@ def main(): sys.executable, "-m", "jupyter", "nbconvert", "--to", "html", "--execute", + "--ExecutePreprocessor.timeout={}".format(timeout), f"--ExecutePreprocessor.timeout={timeout}", "--output-dir", str(target_dir), str(nb) ]) except subprocess.CalledProcessError as e: print(f"ERROR executing {nb}: {e}") + # Create a placeholder HTML with the failure message so CI pages report which notebooks failed + fail_html = target_dir.joinpath(nb.stem + ".html") + with open(fail_html, "w", encoding="utf-8") as fh: + fh.write(f"

Execution failed for {nb}

{e}
") + print("Done.") + + # Create a placeholder HTML with the failure message so + # CI pages report which notebooks failed fail_html = target_dir.joinpath(nb.stem + ".html") with open(fail_html, "w", encoding="utf-8") as fh: error_msg = (