From 5346312a0588b7c797f4f0d321bb5f3f0cdfd1b5 Mon Sep 17 00:00:00 2001 From: cbullinger Date: Thu, 9 Oct 2025 09:22:46 -0400 Subject: [PATCH 1/2] webhook secret logic --- examples-copier/.gitignore | 49 ++ examples-copier/CONFIG-LOADING-BEHAVIOR.md | 396 +++++++++++++++ examples-copier/DEPLOYMENT-CHECKLIST.md | 356 +++++++++++++ examples-copier/DEPLOYMENT-GUIDE.md | 471 ++++++++++++++++++ examples-copier/DEPLOYMENT-SUMMARY.md | 351 +++++++++++++ .../WEBHOOK-SECRET-MANAGER-GUIDE.md | 343 +++++++++++++ examples-copier/app.go | 6 + examples-copier/configs/environment.go | 9 +- examples-copier/deploy.sh | 258 ++++++++++ examples-copier/env.yaml.example | 123 +++++ examples-copier/services/github_auth.go | 43 ++ 11 files changed, 2404 insertions(+), 1 deletion(-) create mode 100644 examples-copier/.gitignore create mode 100644 examples-copier/CONFIG-LOADING-BEHAVIOR.md create mode 100644 examples-copier/DEPLOYMENT-CHECKLIST.md create mode 100644 examples-copier/DEPLOYMENT-GUIDE.md create mode 100644 examples-copier/DEPLOYMENT-SUMMARY.md create mode 100644 examples-copier/WEBHOOK-SECRET-MANAGER-GUIDE.md create mode 100755 examples-copier/deploy.sh create mode 100644 examples-copier/env.yaml.example diff --git a/examples-copier/.gitignore b/examples-copier/.gitignore new file mode 100644 index 0000000..18df324 --- /dev/null +++ b/examples-copier/.gitignore @@ -0,0 +1,49 @@ +# Binaries +examples-copier +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool +*.out + +# Dependency directories +vendor/ + +# Go workspace file +go.work + +# Environment files with secrets +env.yaml +.env +.env.local +.env.production +.env.*.local + +# Private keys +*.pem +*.key + +# IDE files +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# OS files +.DS_Store +Thumbs.db + +# Logs +*.log + +# Temporary files +tmp/ +temp/ + diff --git a/examples-copier/CONFIG-LOADING-BEHAVIOR.md b/examples-copier/CONFIG-LOADING-BEHAVIOR.md new file mode 100644 index 0000000..fa2140d --- /dev/null +++ b/examples-copier/CONFIG-LOADING-BEHAVIOR.md @@ -0,0 +1,396 @@ +# Config Loading Behavior - When Is Config Read? + +## Quick Answer + +**The config file is read from the SOURCE BRANCH (typically `main`), NOT from the merged PR.** + +This means: +- ❌ **Config changes in a PR are NOT used** for that same PR +- ✅ **Config changes take effect** for the NEXT PR after they're merged +- ⚠️ **You cannot update config and files in the same PR** and have the new config apply + +## How It Works + +### Webhook Flow + +When a PR is merged: + +``` +1. PR Merged → Webhook triggered +2. Authenticate with GitHub +3. Read config file from SOURCE BRANCH (main) ← IMPORTANT +4. Get changed files from the merged PR +5. Apply copy rules from config +6. Copy files to target repos +``` + +### Code Implementation + + +````go +func retrieveJsonFile(filePath string) string { + client := GetRestClient() + owner := os.Getenv(configs.RepoOwner) + repo := os.Getenv(configs.RepoName) + ctx := context.Background() + fileContent, _, _, err := + client.Repositories.GetContents(ctx, owner, repo, + filePath, &github.RepositoryContentGetOptions{ + Ref: os.Getenv(configs.SrcBranch), // ← Reads from SRC_BRANCH (default: "main") + }) + // ... +} +```` + + +**Key Point:** The `Ref` parameter is set to `os.Getenv(configs.SrcBranch)`, which defaults to `"main"`. + +### Environment Configuration + + +````go +func NewConfig() *Config { + return &Config{ + // ... + SrcBranch: "main", // ← Default branch to read config from + // ... + } +} +```` + + +**Default:** Config is always read from the `main` branch (or whatever `SRC_BRANCH` is set to). + +## Scenarios + +### Scenario 1: Update Config Only + +**PR Contents:** +- Modified `copier-config.yaml` (new copy rule added) + +**What Happens:** +1. PR merged +2. Webhook reads config from `main` (BEFORE this PR merged) +3. Old config is used (new rule NOT applied) +4. No files copied (because no other files changed) +5. Config changes are now in `main` for NEXT PR + +**Result:** ✅ Config updated for future PRs + +### Scenario 2: Update Config + Files in Same PR + +**PR Contents:** +- Modified `copier-config.yaml` (new rule: copy `examples/new/*.go`) +- Added `examples/new/example.go` + +**What Happens:** +1. PR merged +2. Webhook reads config from `main` (BEFORE this PR merged) +3. Old config is used (new rule NOT in effect yet) +4. `examples/new/example.go` does NOT match any rules +5. File is NOT copied ❌ +6. Config changes are now in `main` for NEXT PR + +**Result:** ⚠️ File NOT copied - need another PR to trigger copy + +### Scenario 3: Update Files After Config Merged + +**PR 1 Contents:** +- Modified `copier-config.yaml` (new rule: copy `examples/new/*.go`) + +**PR 1 Result:** +- Config merged to `main` +- No files copied (no other files changed) + +**PR 2 Contents:** +- Added `examples/new/example.go` + +**PR 2 What Happens:** +1. PR merged +2. Webhook reads config from `main` (includes new rule from PR 1) +3. New config is used ✅ +4. `examples/new/example.go` matches new rule +5. File is copied to target repo ✅ + +**Result:** ✅ File copied successfully + +### Scenario 4: Update Existing Rule + +**Current Config:** +```yaml +copy_rules: + - name: "Go examples" + source_pattern: + type: "prefix" + pattern: "examples/go" + targets: + - repo: "mongodb/docs" + branch: "main" + path_transform: "code/${relative_path}" +``` + +**PR Contents:** +- Modified `copier-config.yaml` (changed `path_transform` to `"docs/${relative_path}"`) +- Modified `examples/go/example.go` + +**What Happens:** +1. PR merged +2. Webhook reads config from `main` (BEFORE this PR merged) +3. Old config is used (old path_transform: `"code/${relative_path}"`) +4. `examples/go/example.go` is copied to `code/go/example.go` (OLD path) +5. Config changes are now in `main` for NEXT PR + +**Result:** ⚠️ File copied to OLD path - need another PR to copy to NEW path + +## Why This Design? + +### Stability + +**Reason:** Ensures config is stable and tested before being used. + +**Benefit:** +- Config changes are reviewed and merged first +- Next PR uses the reviewed config +- Reduces risk of broken config affecting file copying + +### Predictability + +**Reason:** Config state is known at webhook trigger time. + +**Benefit:** +- No race conditions between config and file changes +- Clear separation: config changes vs file changes +- Easier to debug and understand behavior + +### Simplicity + +**Reason:** Always reads from a known branch (`main`). + +**Benefit:** +- No need to check if config changed in PR +- No need to merge config changes before reading +- Simpler implementation + +## Workarounds + +### Option 1: Two-PR Workflow (Recommended) + +**Step 1:** Update config +```bash +# PR 1: Update config only +git checkout -b update-config +# Edit copier-config.yaml +git add copier-config.yaml +git commit -m "Add new copy rule for X" +git push origin update-config +# Create PR, get approval, merge +``` + +**Step 2:** Add/modify files +```bash +# PR 2: Add files that use new config +git checkout -b add-files +# Add/modify files +git add examples/new/ +git commit -m "Add new examples" +git push origin add-files +# Create PR, get approval, merge +# Files will be copied using new config ✅ +``` + +### Option 2: Manual Trigger (If Supported) + +If the tool supports manual triggering: +```bash +# After PR with config + files is merged +# Manually trigger webhook or re-run copier +# This will use the newly merged config +``` + +### Option 3: Empty Commit Trigger + +**After config is merged:** +```bash +# Create empty commit to trigger webhook +git checkout -b trigger-copy +git commit --allow-empty -m "Trigger copy with new config" +git push origin trigger-copy +# Create PR, merge +# This will trigger webhook with new config +``` + +### Option 4: Modify File Again + +**After config is merged:** +```bash +# Make a small change to the file +git checkout -b fix-copy +# Edit the file (add comment, fix typo, etc.) +git add examples/new/example.go +git commit -m "Trigger copy with updated config" +git push origin fix-copy +# Create PR, merge +# File will be copied with new config ✅ +``` + +## Best Practices + +### 1. Update Config First + +**Always update config in a separate PR before adding files that use it.** + +``` +PR 1: Update copier-config.yaml + ↓ (merge) +PR 2: Add files that match new rules + ↓ (merge, files copied ✅) +``` + +### 2. Test Config Changes + +**Use config-validator to test config before merging:** + +```bash +cd examples-copier +./tools/config-validator/config-validator -config copier-config.yaml +``` + +### 3. Document Config Changes + +**In PR description, note that files will be copied in NEXT PR:** + +```markdown +## Changes +- Added new copy rule for `examples/python/*.py` + +## Note +Files matching this rule will be copied starting with the NEXT PR after this is merged. +``` + +### 4. Plan Multi-Step Changes + +**For complex changes, plan the sequence:** + +``` +Step 1: Update config (PR #123) +Step 2: Add new files (PR #124) +Step 3: Update existing files (PR #125) +``` + +### 5. Use Dry-Run for Testing + +**Test config changes with dry-run mode:** + +```bash +DRY_RUN=true ./examples-copier +# Check logs to see what would be copied +``` + +## Common Mistakes + +### ❌ Mistake 1: Config + Files in Same PR + +**Problem:** +``` +PR: Update config + add files +Result: Files NOT copied (old config used) +``` + +**Solution:** +``` +PR 1: Update config +PR 2: Add files (after PR 1 merged) +``` + +### ❌ Mistake 2: Expecting Immediate Effect + +**Problem:** +``` +Merge PR with config changes +Expect next file change to use new config immediately +``` + +**Reality:** +``` +Config changes take effect for NEXT PR +Current PR uses OLD config +``` + +### ❌ Mistake 3: Not Testing Config + +**Problem:** +``` +Merge config with typo +Next PR fails to copy files +``` + +**Solution:** +``` +Use config-validator before merging +Test with dry-run mode +``` + +## Future Enhancements + +Potential improvements to config loading: + +### 1. Read Config from Merged PR + +**Idea:** Read config from the merged commit instead of `main`. + +**Benefit:** +- Config changes apply immediately +- Single PR can update config + files + +**Challenge:** +- More complex implementation +- Potential race conditions +- Harder to debug + +### 2. Config Validation on PR + +**Idea:** Validate config in PR before merge. + +**Benefit:** +- Catch config errors before merge +- Prevent broken config from reaching `main` + +**Implementation:** +- GitHub Action to validate config +- Block merge if validation fails + +### 3. Config Change Detection + +**Idea:** Detect if config changed in PR and warn user. + +**Benefit:** +- User knows config won't apply to current PR +- Suggests two-PR workflow + +**Implementation:** +- Check if `copier-config.yaml` in changed files +- Add comment to PR with warning + +## Summary + +**Current Behavior:** +- ✅ Config is read from `main` branch (or `SRC_BRANCH`) +- ❌ Config changes in PR do NOT apply to that PR +- ✅ Config changes apply to NEXT PR after merge + +**Recommended Workflow:** +1. Update config in separate PR +2. Merge config PR +3. Add/modify files in subsequent PR +4. Files copied with new config ✅ + +**Key Takeaway:** +> Config changes require a two-PR workflow: one to update config, another to use the new config. + +--- + +**See Also:** +- [Configuration Guide](docs/CONFIGURATION-GUIDE.md) - Complete config reference +- [Pattern Matching Guide](docs/PATTERN-MATCHING-GUIDE.md) - Pattern syntax +- [Troubleshooting](docs/TROUBLESHOOTING.md) - Common issues + diff --git a/examples-copier/DEPLOYMENT-CHECKLIST.md b/examples-copier/DEPLOYMENT-CHECKLIST.md new file mode 100644 index 0000000..5e957ad --- /dev/null +++ b/examples-copier/DEPLOYMENT-CHECKLIST.md @@ -0,0 +1,356 @@ +g# Deployment Checklist + +Quick checklist for deploying examples-copier to Google Cloud App Engine. + +## Pre-Deployment + +### ☐ 1. Install Prerequisites + +```bash +# Install Google Cloud SDK (if not installed) +brew install --cask google-cloud-sdk + +# Verify installation +gcloud --version +go version +``` + +### ☐ 2. Authenticate with Google Cloud + +```bash +# Login +gcloud auth login + +# Set application default credentials +gcloud auth application-default login +``` + +### ☐ 3. Set GCP Project + +```bash +# List projects +gcloud projects list + +# Set project +gcloud config set project YOUR_PROJECT_ID + +# Verify +gcloud config get-value project +``` + +### ☐ 4. Enable Required APIs + +```bash +# Enable App Engine +gcloud services enable appengine.googleapis.com + +# Enable Secret Manager +gcloud services enable secretmanager.googleapis.com + +# Enable Cloud Logging +gcloud services enable logging.googleapis.com + +# Verify +gcloud services list --enabled | grep -E "appengine|secretmanager|logging" +``` + +### ☐ 5. Store GitHub Private Key in Secret Manager + +```bash +# Create secret +gcloud secrets create CODE_COPIER_PEM \ + --data-file=/path/to/your/private-key.pem \ + --replication-policy="automatic" + +# Grant App Engine access +PROJECT_ID=$(gcloud config get-value project) +gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ + --member="serviceAccount:${PROJECT_ID}@appspot.gserviceaccount.com" \ + --role="roles/secretmanager.secretAccessor" + +# Get secret path (copy this for env.yaml) +echo "projects/${PROJECT_ID}/secrets/CODE_COPIER_PEM/versions/latest" +``` + +### ☐ 6. Create env.yaml + +```bash +# Copy example +cp env.yaml.example env.yaml + +# Edit with your values +nano env.yaml # or use your preferred editor +``` + +**Required values to update:** +- `GITHUB_APP_ID` +- `INSTALLATION_ID` +- `REPO_NAME` +- `REPO_OWNER` +- `GITHUB_APP_PRIVATE_KEY_SECRET_NAME` +- `WEBHOOK_SECRET` +- `GOOGLE_PROJECT_ID` + +### ☐ 7. Verify Configuration + +```bash +# Check env.yaml exists +ls -l env.yaml + +# Verify it's in .gitignore +grep "env.yaml" .gitignore +``` + +### ☐ 8. Build and Test + +```bash +# Build +go build -o examples-copier . + +# Run tests +go test ./... + +# Test locally (optional) +go run app.go -env ./configs/.env.test +``` + +## Deployment + +### ☐ 9. Deploy to App Engine + +**Option A: Using deployment script (recommended)** + +```bash +./deploy.sh +``` + +**Option B: Manual deployment** + +```bash +gcloud app deploy app.yaml --env-vars-file=env.yaml +``` + +**Option C: Deploy specific version without promoting** + +```bash +./deploy.sh --version=v2 --no-promote +``` + +### ☐ 10. Verify Deployment + +```bash +# Check deployment status +gcloud app versions list + +# Get app URL +gcloud app describe --format="value(defaultHostname)" + +# View logs +gcloud app logs tail -s default +``` + +## Post-Deployment + +### ☐ 11. Update GitHub Webhook + +1. Go to source repository on GitHub +2. Settings → Webhooks +3. Edit existing webhook or create new one +4. Update Payload URL: `https://YOUR_PROJECT_ID.appspot.com/events` +5. Content type: `application/json` +6. Secret: (same as `WEBHOOK_SECRET` in env.yaml) +7. Events: Select "Pull requests" +8. Active: ✓ Checked +9. Save webhook + +### ☐ 12. Test Webhook + +```bash +# Method 1: Merge a test PR in source repository +# Watch logs for webhook receipt +gcloud app logs tail -s default + +# Method 2: Send test webhook from GitHub +# Go to webhook settings → Recent Deliveries → Redeliver +``` + +### ☐ 13. Verify Application is Working + +**Check logs for:** +- ✓ Webhook received +- ✓ Config file loaded +- ✓ Files copied to target repos +- ✓ No errors + +```bash +# View recent logs +gcloud app logs read --limit=50 + +# Filter for errors +gcloud app logs read --limit=100 | grep ERROR + +# Filter for webhook events +gcloud app logs read --limit=100 | grep webhook +``` + +### ☐ 14. Monitor Application + +```bash +# Real-time logs +gcloud app logs tail -s default + +# Open Cloud Console +gcloud app open-console + +# View metrics +# Go to Cloud Console → App Engine → Dashboard +``` + +## Rollback (if needed) + +### ☐ If Deployment Has Issues + +```bash +# List versions +gcloud app versions list + +# Route traffic to previous version +gcloud app services set-traffic default --splits=PREVIOUS_VERSION=1 + +# Delete bad version +gcloud app versions delete BAD_VERSION +``` + +## Troubleshooting + +### Issue: Deployment fails + +```bash +# Check APIs are enabled +gcloud services list --enabled + +# Verify app.yaml is valid +cat app.yaml + +# Check for build errors +go build -o examples-copier . +``` + +### Issue: Webhooks not received + +```bash +# Check webhook URL is correct +gcloud app describe --format="value(defaultHostname)" + +# Verify webhook secret matches +# Compare GitHub webhook secret with WEBHOOK_SECRET in env.yaml + +# Check logs for errors +gcloud app logs tail -s default | grep webhook +``` + +### Issue: Cannot access secrets + +```bash +# Verify secret exists +gcloud secrets list + +# Check IAM permissions +gcloud secrets get-iam-policy CODE_COPIER_PEM + +# Grant access if needed +PROJECT_ID=$(gcloud config get-value project) +gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ + --member="serviceAccount:${PROJECT_ID}@appspot.gserviceaccount.com" \ + --role="roles/secretmanager.secretAccessor" +``` + +### Issue: Files not being copied + +```bash +# Check config file in source repo +# Verify copier-config.yaml exists and is valid + +# Check logs for pattern matching +gcloud app logs read --limit=100 | grep "file matched" + +# Verify GitHub App has write access to target repos +``` + +## Quick Commands Reference + +```bash +# Deploy +./deploy.sh + +# Deploy specific version +./deploy.sh --version=v2 + +# Deploy without promoting +./deploy.sh --version=v2 --no-promote + +# View logs +gcloud app logs tail -s default + +# List versions +gcloud app versions list + +# Set traffic split +gcloud app services set-traffic default --splits=v1=0.5,v2=0.5 + +# Delete version +gcloud app versions delete VERSION_ID + +# Open Cloud Console +gcloud app open-console + +# Get app URL +gcloud app describe --format="value(defaultHostname)" +``` + +## Environment Variables Quick Reference + +**Required:** +- `GITHUB_APP_ID` - GitHub App ID +- `INSTALLATION_ID` - Installation ID +- `REPO_NAME` - Source repo name +- `REPO_OWNER` - Source repo owner +- `GITHUB_APP_PRIVATE_KEY_SECRET_NAME` - Secret Manager path +- `WEBHOOK_SECRET` - Webhook secret + +**Optional:** +- `PORT` - Server port (default: 8080) +- `WEBSERVER_PATH` - Webhook path (default: /webhook) +- `CONFIG_FILE` - Config file (default: copier-config.yaml) +- `DEPRECATION_FILE` - Deprecation file (default: deprecated_examples.json) +- `COMMITTER_NAME` - Git committer name +- `COMMITTER_EMAIL` - Git committer email +- `GOOGLE_PROJECT_ID` - GCP project for logging +- `GOOGLE_LOG_NAME` - Log name + +## Success Criteria + +✅ Deployment completes without errors +✅ Application is accessible at App Engine URL +✅ Webhook receives PR events from GitHub +✅ Config file is loaded successfully +✅ Files are copied to target repositories +✅ Logs show no errors +✅ Deprecation tracking works (if enabled) + +## Next Steps After Deployment + +1. Monitor logs for first few PRs +2. Verify files are copied correctly +3. Check target repositories for commits +4. Set up monitoring/alerting (optional) +5. Document any custom configuration +6. Share webhook URL with team + +--- + +**See Also:** +- [DEPLOYMENT-GUIDE.md](DEPLOYMENT-GUIDE.md) - Detailed deployment guide +- [README.md](README.md) - Application overview +- [CONFIG-LOADING-BEHAVIOR.md](CONFIG-LOADING-BEHAVIOR.md) - Config details + diff --git a/examples-copier/DEPLOYMENT-GUIDE.md b/examples-copier/DEPLOYMENT-GUIDE.md new file mode 100644 index 0000000..75b750e --- /dev/null +++ b/examples-copier/DEPLOYMENT-GUIDE.md @@ -0,0 +1,471 @@ +# Deployment Guide - Google Cloud App Engine + +## Overview + +This guide covers deploying the examples-copier application to Google Cloud App Engine (Flexible Environment). + +## Prerequisites + +### 1. Google Cloud SDK + +Install the Google Cloud SDK if not already installed: + +```bash +# macOS (using Homebrew) +brew install --cask google-cloud-sdk + +# Or download from: https://cloud.google.com/sdk/docs/install +``` + +### 2. Authentication + +Authenticate with Google Cloud: + +```bash +# Login to your Google account +gcloud auth login + +# Set application default credentials +gcloud auth application-default login +``` + +### 3. Project Configuration + +Set your Google Cloud project: + +```bash +# List available projects +gcloud projects list + +# Set the project (replace with your project ID) +gcloud config set project YOUR_PROJECT_ID +``` + +### 4. Required APIs + +Enable required Google Cloud APIs: + +```bash +# Enable App Engine Admin API +gcloud services enable appengine.googleapis.com + +# Enable Secret Manager API (for GitHub private key) +gcloud services enable secretmanager.googleapis.com + +# Enable Cloud Logging API +gcloud services enable logging.googleapis.com +``` + +## Configuration + +### 1. Environment Variables + +The application uses environment variables for configuration. These are **NOT** stored in `app.yaml` for security reasons. + +**Required Environment Variables:** +- `GITHUB_APP_ID` - GitHub App ID (numeric) +- `INSTALLATION_ID` - GitHub App Installation ID +- `REPO_NAME` - Source repository name +- `REPO_OWNER` - Source repository owner +- `GITHUB_APP_PRIVATE_KEY_SECRET_NAME` - GCP Secret Manager path to private key +- `WEBHOOK_SECRET` - GitHub webhook secret for signature validation + +**Optional Environment Variables:** +- `PORT` - Server port (default: 8080) +- `WEBSERVER_PATH` - Webhook endpoint path (default: /webhook) +- `CONFIG_FILE` - Config file name (default: copier-config.yaml) +- `DEPRECATION_FILE` - Deprecation file name (default: deprecated_examples.json) +- `COMMITTER_NAME` - Git committer name (default: Copier Bot) +- `COMMITTER_EMAIL` - Git committer email (default: bot@example.com) +- `GOOGLE_PROJECT_ID` - GCP project ID for logging +- `GOOGLE_LOG_NAME` - Cloud Logging log name + +### 2. Store GitHub Private Key in Secret Manager + +The GitHub App private key must be stored in Google Cloud Secret Manager: + +```bash +# Create secret from file +gcloud secrets create CODE_COPIER_PEM \ + --data-file=/path/to/your/private-key.pem \ + --replication-policy="automatic" + +# Grant App Engine access to the secret +gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ + --member="serviceAccount:YOUR_PROJECT_ID@appspot.gserviceaccount.com" \ + --role="roles/secretmanager.secretAccessor" + +# Get the full secret path (use this in GITHUB_APP_PRIVATE_KEY_SECRET_NAME) +echo "projects/$(gcloud config get-value project)/secrets/CODE_COPIER_PEM/versions/latest" +``` + +### 3. Set Environment Variables in App Engine + +Environment variables are set during deployment using the `--env-vars-file` flag. + +Create `env.yaml`: + +```yaml +env_variables: + GITHUB_APP_ID: "1166559" + INSTALLATION_ID: "62138132" + REPO_NAME: "docs-code-examples" + REPO_OWNER: "mongodb" + GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/YOUR_PROJECT_ID/secrets/CODE_COPIER_PEM/versions/latest" + WEBHOOK_SECRET: "your-webhook-secret-here" + COMMITTER_NAME: "GitHub Copier App" + COMMITTER_EMAIL: "bot@mongodb.com" + CONFIG_FILE: "copier-config.yaml" + DEPRECATION_FILE: "deprecated_examples.json" + WEBSERVER_PATH: "/events" + GOOGLE_PROJECT_ID: "YOUR_PROJECT_ID" + GOOGLE_LOG_NAME: "code-copier-log" +``` + +**⚠️ IMPORTANT:** Add `env.yaml` to `.gitignore` to prevent committing secrets! + +## Deployment + +### Quick Deployment + +Use the provided deployment script: + +```bash +cd examples-copier +./deploy.sh +``` + +### Manual Deployment + +#### Step 1: Build and Test Locally + +```bash +cd examples-copier + +# Build the application +go build -o examples-copier . + +# Run tests +go test ./... + +# Test locally (optional) +go run app.go -env ./configs/.env.test +``` + +#### Step 2: Deploy to App Engine + +```bash +# Deploy with environment variables +gcloud app deploy app.yaml --env-vars-file=env.yaml + +# Or deploy without prompts +gcloud app deploy app.yaml --env-vars-file=env.yaml --quiet +``` + +#### Step 3: Verify Deployment + +```bash +# View deployment status +gcloud app versions list + +# View logs +gcloud app logs tail -s default + +# Open the application in browser +gcloud app browse +``` + +### Deployment Options + +**Deploy specific version:** +```bash +gcloud app deploy app.yaml --version=v1 --env-vars-file=env.yaml +``` + +**Deploy without promoting (traffic stays on current version):** +```bash +gcloud app deploy app.yaml --no-promote --env-vars-file=env.yaml +``` + +**Deploy and set traffic split:** +```bash +# Deploy new version +gcloud app deploy app.yaml --version=v2 --no-promote --env-vars-file=env.yaml + +# Split traffic (50% to v1, 50% to v2) +gcloud app services set-traffic default --splits=v1=0.5,v2=0.5 +``` + +## Post-Deployment + +### 1. Update GitHub Webhook URL + +After deployment, update the webhook URL in your source repository: + +``` +https://YOUR_PROJECT_ID.appspot.com/events +``` + +**Steps:** +1. Go to your source repository on GitHub +2. Settings → Webhooks +3. Edit the webhook +4. Update the Payload URL to your App Engine URL +5. Ensure Content type is `application/json` +6. Set the webhook secret (same as `WEBHOOK_SECRET` env var) +7. Select "Pull requests" event +8. Save webhook + +### 2. Verify Webhook + +Test the webhook: + +```bash +# Trigger a test PR merge in your source repo +# Check App Engine logs for webhook receipt + +gcloud app logs tail -s default +``` + +### 3. Monitor Application + +**View logs:** +```bash +# Tail logs in real-time +gcloud app logs tail -s default + +# View logs in Cloud Console +gcloud app logs read --limit=50 +``` + +**View metrics:** +```bash +# Open Cloud Console +gcloud app open-console +``` + +## Troubleshooting + +### Issue: Deployment Fails + +**Check:** +- All required APIs are enabled +- `app.yaml` is valid +- Go version matches runtime version +- Dependencies are up to date + +**Solution:** +```bash +# Update dependencies +go mod tidy + +# Verify app.yaml +cat app.yaml + +# Check enabled APIs +gcloud services list --enabled +``` + +### Issue: Application Not Receiving Webhooks + +**Check:** +- Webhook URL is correct +- Webhook secret matches `WEBHOOK_SECRET` env var +- GitHub App has correct permissions +- Firewall rules allow GitHub IPs + +**Solution:** +```bash +# Check logs for webhook errors +gcloud app logs tail -s default | grep webhook + +# Test webhook manually using curl +curl -X POST https://YOUR_PROJECT_ID.appspot.com/events \ + -H "Content-Type: application/json" \ + -d '{"test": "data"}' +``` + +### Issue: Cannot Access Secret Manager + +**Check:** +- Secret exists in Secret Manager +- App Engine service account has access +- Secret path is correct in env vars + +**Solution:** +```bash +# List secrets +gcloud secrets list + +# Check IAM permissions +gcloud secrets get-iam-policy CODE_COPIER_PEM + +# Grant access if needed +gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ + --member="serviceAccount:YOUR_PROJECT_ID@appspot.gserviceaccount.com" \ + --role="roles/secretmanager.secretAccessor" +``` + +### Issue: High Memory Usage + +**Check:** +- App Engine instance size +- Memory leaks in code +- Number of concurrent requests + +**Solution:** +Update `app.yaml`: +```yaml +runtime: go +runtime_config: + operating_system: "ubuntu22" + runtime_version: "1.23" +env: flex + +resources: + cpu: 1 + memory_gb: 2 + disk_size_gb: 10 + +automatic_scaling: + min_num_instances: 1 + max_num_instances: 5 + cool_down_period_sec: 120 + cpu_utilization: + target_utilization: 0.6 +``` + +## Rollback + +If deployment fails or has issues, rollback to previous version: + +```bash +# List versions +gcloud app versions list + +# Set traffic to previous version +gcloud app services set-traffic default --splits=PREVIOUS_VERSION=1 + +# Delete bad version (optional) +gcloud app versions delete BAD_VERSION +``` + +## Cost Optimization + +### 1. Use Minimum Instances + +For low-traffic applications: + +```yaml +automatic_scaling: + min_num_instances: 1 + max_num_instances: 2 +``` + +### 2. Use Standard Environment (if possible) + +App Engine Standard is cheaper than Flexible, but requires code changes. + +### 3. Monitor Costs + +```bash +# View billing +gcloud billing accounts list + +# Set budget alerts in Cloud Console +``` + +## Security Best Practices + +### 1. Never Commit Secrets + +Add to `.gitignore`: +``` +env.yaml +*.pem +.env.production +``` + +### 2. Rotate Secrets Regularly + +```bash +# Create new secret version +gcloud secrets versions add CODE_COPIER_PEM --data-file=/path/to/new-key.pem + +# Update env var to use new version +# Redeploy application +``` + +### 3. Use IAM Roles + +Grant minimum required permissions: +```bash +# App Engine service account should have: +# - Secret Manager Secret Accessor +# - Cloud Logging Writer +``` + +### 4. Enable VPC Service Controls (optional) + +For additional security in production. + +## Maintenance + +### Update Application + +```bash +# Pull latest code +git pull origin main + +# Build and test +go build -o examples-copier . +go test ./... + +# Deploy +gcloud app deploy app.yaml --env-vars-file=env.yaml +``` + +### Update Dependencies + +```bash +# Update Go modules +go get -u ./... +go mod tidy + +# Test +go test ./... + +# Deploy +gcloud app deploy app.yaml --env-vars-file=env.yaml +``` + +### View Application Info + +```bash +# App Engine info +gcloud app describe + +# List services +gcloud app services list + +# List versions +gcloud app versions list + +# View instance info +gcloud app instances list +``` + +## Additional Resources + +- [App Engine Go Documentation](https://cloud.google.com/appengine/docs/flexible/go) +- [Secret Manager Documentation](https://cloud.google.com/secret-manager/docs) +- [Cloud Logging Documentation](https://cloud.google.com/logging/docs) +- [GitHub Apps Documentation](https://docs.github.com/en/apps) + +--- + +**See Also:** +- [README.md](README.md) - Application overview +- [Configuration Guide](CONFIG-LOADING-BEHAVIOR.md) - Config file details +- [Troubleshooting](TROUBLESHOOTING.md) - Common issues + diff --git a/examples-copier/DEPLOYMENT-SUMMARY.md b/examples-copier/DEPLOYMENT-SUMMARY.md new file mode 100644 index 0000000..b6305ff --- /dev/null +++ b/examples-copier/DEPLOYMENT-SUMMARY.md @@ -0,0 +1,351 @@ +# Deployment Summary + +## Quick Start + +To deploy the latest version of examples-copier to Google Cloud App Engine: + +```bash +cd examples-copier + +# 1. Set up environment file +cp env.yaml.example env.yaml +# Edit env.yaml with your values + +# 2. Deploy +./deploy.sh +``` + +## What Was Created + +### 1. Deployment Documentation + +**DEPLOYMENT-GUIDE.md** (300+ lines) +- Complete deployment guide +- Prerequisites and setup +- Configuration instructions +- Deployment steps +- Post-deployment tasks +- Troubleshooting +- Security best practices +- Maintenance procedures + +**DEPLOYMENT-CHECKLIST.md** (300+ lines) +- Step-by-step checklist +- Pre-deployment tasks +- Deployment commands +- Post-deployment verification +- Rollback procedures +- Quick command reference +- Success criteria + +### 2. Deployment Script + +**deploy.sh** (executable) +- Automated deployment script +- Prerequisites checking +- Build and test +- Deployment to App Engine +- Post-deployment verification +- Options: + - `--project PROJECT_ID` - Set GCP project + - `--version VERSION` - Set version name + - `--no-promote` - Deploy without promoting + - `--quiet` - Skip prompts + - `--env-file FILE` - Custom env file path + - `--help` - Show help + +### 3. Configuration Files + +**env.yaml.example** +- Example environment variables +- All required and optional settings +- Documentation for each variable +- Security notes + +**.gitignore** +- Prevents committing secrets +- Excludes env.yaml, .env files +- Excludes private keys +- Standard Go ignores + +## Deployment Steps + +### Prerequisites + +1. **Install Google Cloud SDK** + ```bash + brew install --cask google-cloud-sdk + ``` + +2. **Authenticate** + ```bash + gcloud auth login + gcloud auth application-default login + ``` + +3. **Set Project** + ```bash + gcloud config set project YOUR_PROJECT_ID + ``` + +4. **Enable APIs** + ```bash + gcloud services enable appengine.googleapis.com + gcloud services enable secretmanager.googleapis.com + gcloud services enable logging.googleapis.com + ``` + +### Configuration + +1. **Store GitHub Private Key** + ```bash + gcloud secrets create CODE_COPIER_PEM \ + --data-file=/path/to/private-key.pem \ + --replication-policy="automatic" + + PROJECT_ID=$(gcloud config get-value project) + gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ + --member="serviceAccount:${PROJECT_ID}@appspot.gserviceaccount.com" \ + --role="roles/secretmanager.secretAccessor" + ``` + +2. **Create env.yaml** + ```bash + cp env.yaml.example env.yaml + # Edit with your values + ``` + + **Required values:** + - `GITHUB_APP_ID` + - `INSTALLATION_ID` + - `REPO_NAME` + - `REPO_OWNER` + - `GITHUB_APP_PRIVATE_KEY_SECRET_NAME` + - `WEBHOOK_SECRET` + - `GOOGLE_PROJECT_ID` + +### Deploy + +**Option 1: Using script (recommended)** +```bash +./deploy.sh +``` + +**Option 2: Manual** +```bash +gcloud app deploy app.yaml --env-vars-file=env.yaml +``` + +**Option 3: Specific version** +```bash +./deploy.sh --version=v2 --no-promote +``` + +### Post-Deployment + +1. **Get App URL** + ```bash + gcloud app describe --format="value(defaultHostname)" + ``` + +2. **Update GitHub Webhook** + - URL: `https://YOUR_PROJECT_ID.appspot.com/events` + - Content type: `application/json` + - Secret: (same as `WEBHOOK_SECRET`) + - Events: Pull requests + +3. **Verify** + ```bash + gcloud app logs tail -s default + ``` + +## Current Deployment Configuration + +### App Engine Settings (app.yaml) + +```yaml +runtime: go +runtime_config: + operating_system: "ubuntu22" + runtime_version: "1.23" +env: flex +``` + +**Environment:** Flexible Environment +**Runtime:** Go 1.23 +**OS:** Ubuntu 22.04 + +### Default Configuration + +**Port:** 8080 +**Webhook Path:** /events +**Config File:** copier-config.yaml +**Deprecation File:** deprecated_examples.json +**Source Branch:** main + +## Monitoring + +### View Logs + +```bash +# Real-time logs +gcloud app logs tail -s default + +# Recent logs +gcloud app logs read --limit=50 + +# Filter for errors +gcloud app logs read --limit=100 | grep ERROR + +# Filter for webhooks +gcloud app logs read --limit=100 | grep webhook +``` + +### View Metrics + +```bash +# Open Cloud Console +gcloud app open-console + +# View in browser +# Go to: App Engine → Dashboard +``` + +## Rollback + +If deployment has issues: + +```bash +# List versions +gcloud app versions list + +# Route traffic to previous version +gcloud app services set-traffic default --splits=PREVIOUS_VERSION=1 + +# Delete bad version +gcloud app versions delete BAD_VERSION +``` + +## Security Notes + +### ⚠️ Important + +1. **Never commit secrets** + - env.yaml is in .gitignore + - Never commit .env files + - Never commit .pem files + +2. **Use Secret Manager** + - GitHub private key stored in Secret Manager + - Not in environment variables + - Not in code + +3. **Rotate secrets regularly** + - Update GitHub App private key + - Update webhook secret + - Update in both GitHub and env.yaml + +4. **Minimum permissions** + - App Engine service account has minimal IAM roles + - Only Secret Manager Secret Accessor + - Only Cloud Logging Writer + +## Troubleshooting + +### Deployment Fails + +```bash +# Check APIs +gcloud services list --enabled + +# Verify build +go build -o examples-copier . + +# Check env.yaml +cat env.yaml +``` + +### Webhooks Not Received + +```bash +# Verify URL +gcloud app describe --format="value(defaultHostname)" + +# Check logs +gcloud app logs tail -s default | grep webhook + +# Test webhook from GitHub +# Settings → Webhooks → Recent Deliveries → Redeliver +``` + +### Cannot Access Secrets + +```bash +# List secrets +gcloud secrets list + +# Check permissions +gcloud secrets get-iam-policy CODE_COPIER_PEM + +# Grant access +PROJECT_ID=$(gcloud config get-value project) +gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ + --member="serviceAccount:${PROJECT_ID}@appspot.gserviceaccount.com" \ + --role="roles/secretmanager.secretAccessor" +``` + +## Files Created + +``` +examples-copier/ +├── DEPLOYMENT-GUIDE.md # Complete deployment guide +├── DEPLOYMENT-CHECKLIST.md # Step-by-step checklist +├── DEPLOYMENT-SUMMARY.md # This file +├── deploy.sh # Deployment script (executable) +├── env.yaml.example # Example environment variables +├── .gitignore # Git ignore file (includes env.yaml) +└── app.yaml # App Engine configuration (existing) +``` + +## Next Steps + +1. **Review Documentation** + - Read DEPLOYMENT-GUIDE.md for details + - Follow DEPLOYMENT-CHECKLIST.md step-by-step + +2. **Prepare Environment** + - Create env.yaml from example + - Store GitHub private key in Secret Manager + - Enable required APIs + +3. **Deploy** + - Run `./deploy.sh` + - Update GitHub webhook + - Test with a PR + +4. **Monitor** + - Watch logs for first few PRs + - Verify files are copied correctly + - Check for errors + +## Support + +**Documentation:** +- [DEPLOYMENT-GUIDE.md](DEPLOYMENT-GUIDE.md) - Detailed guide +- [DEPLOYMENT-CHECKLIST.md](DEPLOYMENT-CHECKLIST.md) - Checklist +- [README.md](README.md) - Application overview +- [CONFIG-LOADING-BEHAVIOR.md](CONFIG-LOADING-BEHAVIOR.md) - Config details + +**Google Cloud:** +- [App Engine Documentation](https://cloud.google.com/appengine/docs/flexible/go) +- [Secret Manager Documentation](https://cloud.google.com/secret-manager/docs) +- [Cloud Logging Documentation](https://cloud.google.com/logging/docs) + +**GitHub:** +- [GitHub Apps Documentation](https://docs.github.com/en/apps) +- [Webhooks Documentation](https://docs.github.com/en/webhooks) + +--- + +**Ready to deploy?** Run `./deploy.sh` to get started! + diff --git a/examples-copier/WEBHOOK-SECRET-MANAGER-GUIDE.md b/examples-copier/WEBHOOK-SECRET-MANAGER-GUIDE.md new file mode 100644 index 0000000..e7b1ea3 --- /dev/null +++ b/examples-copier/WEBHOOK-SECRET-MANAGER-GUIDE.md @@ -0,0 +1,343 @@ +# Using Webhook Secret from Google Cloud Secret Manager + +## Overview + +The examples-copier application now supports loading the webhook secret from Google Cloud Secret Manager instead of hardcoding it in environment variables. This is **more secure** and follows best practices for secret management. + +## Benefits + +✅ **Security**: Secrets are encrypted at rest and in transit +✅ **Audit Trail**: Secret Manager logs all access to secrets +✅ **Rotation**: Easy to rotate secrets without redeploying +✅ **Access Control**: Fine-grained IAM permissions +✅ **No Hardcoding**: Secrets never appear in config files or version control + +## Quick Start + +### 1. Store Webhook Secret in Secret Manager + +```bash +# Generate a secure random secret (if you don't have one) +WEBHOOK_SECRET=$(openssl rand -hex 32) +echo "Generated webhook secret: $WEBHOOK_SECRET" + +# Store in Secret Manager +echo -n "$WEBHOOK_SECRET" | gcloud secrets create webhook-secret \ + --data-file=- \ + --replication-policy="automatic" + +# Verify it was created +gcloud secrets describe webhook-secret +``` + +### 2. Grant App Engine Access + +```bash +# Get your project ID +PROJECT_ID=$(gcloud config get-value project) + +# Grant App Engine service account access to the secret +gcloud secrets add-iam-policy-binding webhook-secret \ + --member="serviceAccount:${PROJECT_ID}@appspot.gserviceaccount.com" \ + --role="roles/secretmanager.secretAccessor" + +# Verify permissions +gcloud secrets get-iam-policy webhook-secret +``` + +### 3. Configure env.yaml + +Use `WEBHOOK_SECRET_NAME` instead of `WEBHOOK_SECRET`: + +```yaml +env_variables: + # ... other config ... + + # Use Secret Manager (RECOMMENDED) + WEBHOOK_SECRET_NAME: "projects/1054147886816/secrets/webhook-secret/versions/latest" + + # DO NOT use direct secret in production + # WEBHOOK_SECRET: "hardcoded-secret-here" +``` + +### 4. Deploy + +```bash +./deploy.sh +``` + +## Configuration Options + +The application supports **two ways** to provide the webhook secret: + +### Option 1: Secret Manager (Recommended for Production) + +**Environment Variable:** `WEBHOOK_SECRET_NAME` + +**Format:** `projects/PROJECT_ID/secrets/SECRET_NAME/versions/VERSION` + +**Example:** +```yaml +WEBHOOK_SECRET_NAME: "projects/1054147886816/secrets/webhook-secret/versions/latest" +``` + +**Pros:** +- ✅ Secure - secret never in config files +- ✅ Auditable - all access logged +- ✅ Rotatable - update secret without redeploying +- ✅ Access controlled - IAM permissions + +**Cons:** +- ❌ Requires Secret Manager setup +- ❌ Requires IAM permissions + +### Option 2: Direct Environment Variable (For Testing Only) + +**Environment Variable:** `WEBHOOK_SECRET` + +**Format:** Plain text string + +**Example:** +```yaml +WEBHOOK_SECRET: "my-webhook-secret-123" +``` + +**Pros:** +- ✅ Simple - no Secret Manager needed +- ✅ Fast - no API calls + +**Cons:** +- ❌ Insecure - secret in config file +- ❌ No audit trail +- ❌ Hard to rotate +- ❌ Risk of committing to version control + +**⚠️ Use only for local development/testing!** + +## How It Works + +### Loading Priority + +1. **Check `WEBHOOK_SECRET`** - If set, use it directly (no Secret Manager call) +2. **Check `WEBHOOK_SECRET_NAME`** - If set, load from Secret Manager +3. **Use default** - `projects/1054147886816/secrets/webhook-secret/versions/latest` + +### Code Flow + +``` +app.go + ├─> configs.LoadEnvironment() + │ └─> Loads WEBHOOK_SECRET and WEBHOOK_SECRET_NAME from env + │ + └─> services.LoadWebhookSecret(config) + ├─> If config.WebhookSecret is set → use it + └─> Else → load from Secret Manager using config.WebhookSecretName + └─> Store in config.WebhookSecret +``` + +### Signature Verification + +``` +webhook_handler.go + └─> ParseWebhookDataWithConfig() + └─> verifySignatureFunc(sigHeader, payload, []byte(config.WebhookSecret)) + └─> HMAC-SHA256 verification +``` + +## Your Current Setup + +Based on your Secret Manager output: + +```yaml +name: projects/1054147886816/secrets/webhook-secret +createTime: '2025-10-06T17:56:10.467642Z' +replication: + automatic: {} +``` + +### Your env.yaml Should Use: + +```yaml +env_variables: + GITHUB_APP_ID: "1166559" + INSTALLATION_ID: "62138132" + REPO_NAME: "docs-code-examples" + REPO_OWNER: "mongodb" + + # GitHub App private key from Secret Manager + GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest" + + # Webhook secret from Secret Manager (RECOMMENDED) + WEBHOOK_SECRET_NAME: "projects/1054147886816/secrets/webhook-secret/versions/latest" + + # Other config... + COMMITTER_NAME: "GitHub Copier App" + COMMITTER_EMAIL: "bot@mongodb.com" + PORT: "8080" + WEBSERVER_PATH: "/events" + CONFIG_FILE: "copier-config.yaml" + DEPRECATION_FILE: "deprecated_examples.json" + GOOGLE_PROJECT_ID: "1054147886816" + GOOGLE_LOG_NAME: "code-copier-log" +``` + +## Testing Locally + +For local testing, you can use `SKIP_SECRET_MANAGER=true`: + +```bash +# In your .env.local file +SKIP_SECRET_MANAGER=true +WEBHOOK_SECRET="test-secret-123" +GITHUB_APP_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----" +``` + +Then run: +```bash +go run app.go -env .env.local +``` + +## Rotating the Webhook Secret + +### Step 1: Create New Secret Version + +```bash +# Generate new secret +NEW_SECRET=$(openssl rand -hex 32) + +# Add new version to Secret Manager +echo -n "$NEW_SECRET" | gcloud secrets versions add webhook-secret \ + --data-file=- +``` + +### Step 2: Update GitHub Webhook + +1. Go to your source repository on GitHub +2. Settings → Webhooks +3. Edit the webhook +4. Update the "Secret" field with the new secret +5. Save + +### Step 3: Verify + +The application will automatically use the latest version (no redeployment needed if using `versions/latest`). + +```bash +# Test webhook delivery +# GitHub → Settings → Webhooks → Recent Deliveries → Redeliver +``` + +### Step 4: Disable Old Version (Optional) + +```bash +# List versions +gcloud secrets versions list webhook-secret + +# Disable old version +gcloud secrets versions disable VERSION_NUMBER --secret=webhook-secret +``` + +## Troubleshooting + +### Error: "failed to load webhook secret" + +**Cause:** Secret Manager client can't access the secret + +**Solutions:** +1. Verify secret exists: + ```bash + gcloud secrets describe webhook-secret + ``` + +2. Check IAM permissions: + ```bash + gcloud secrets get-iam-policy webhook-secret + ``` + +3. Grant access: + ```bash + PROJECT_ID=$(gcloud config get-value project) + gcloud secrets add-iam-policy-binding webhook-secret \ + --member="serviceAccount:${PROJECT_ID}@appspot.gserviceaccount.com" \ + --role="roles/secretmanager.secretAccessor" + ``` + +### Error: "webhook signature verification failed" + +**Cause:** Secret in Secret Manager doesn't match GitHub webhook secret + +**Solutions:** +1. Get secret from Secret Manager: + ```bash + gcloud secrets versions access latest --secret=webhook-secret + ``` + +2. Compare with GitHub webhook secret: + - GitHub → Settings → Webhooks → Edit + - Check the "Secret" field + +3. Update one to match the other + +### Error: "SKIP_SECRET_MANAGER=true but no WEBHOOK_SECRET set" + +**Cause:** Testing locally without providing direct secret + +**Solution:** +```bash +export WEBHOOK_SECRET="test-secret-123" +``` + +## Security Best Practices + +### ✅ DO + +- ✅ Use Secret Manager in production +- ✅ Use `versions/latest` for automatic rotation +- ✅ Grant minimal IAM permissions +- ✅ Rotate secrets regularly (every 90 days) +- ✅ Use different secrets for different environments +- ✅ Monitor Secret Manager audit logs + +### ❌ DON'T + +- ❌ Hardcode secrets in env.yaml for production +- ❌ Commit env.yaml to version control +- ❌ Share secrets via email or chat +- ❌ Use the same secret across multiple apps +- ❌ Grant broad IAM permissions +- ❌ Use weak secrets (use `openssl rand -hex 32`) + +## Comparison with GitHub Private Key + +Both secrets are now loaded from Secret Manager: + +| Secret | Environment Variable | Secret Manager Path | +|--------|---------------------|---------------------| +| GitHub Private Key | `GITHUB_APP_PRIVATE_KEY_SECRET_NAME` | `projects/.../secrets/CODE_COPIER_PEM/versions/latest` | +| Webhook Secret | `WEBHOOK_SECRET_NAME` | `projects/.../secrets/webhook-secret/versions/latest` | + +**Consistency:** Both use the same pattern for security and maintainability. + +## Summary + +**Before (Insecure):** +```yaml +WEBHOOK_SECRET: "hardcoded-secret-in-config-file" +``` + +**After (Secure):** +```yaml +WEBHOOK_SECRET_NAME: "projects/1054147886816/secrets/webhook-secret/versions/latest" +``` + +**Result:** +- ✅ Secret stored securely in Secret Manager +- ✅ No secrets in config files +- ✅ Easy rotation without redeployment +- ✅ Audit trail of all access +- ✅ Fine-grained access control + +--- + +**Ready to deploy?** Your webhook secret is already in Secret Manager, so just update your env.yaml to use `WEBHOOK_SECRET_NAME` instead of `WEBHOOK_SECRET`! 🔒 + diff --git a/examples-copier/app.go b/examples-copier/app.go index 37b1d7d..e063d22 100644 --- a/examples-copier/app.go +++ b/examples-copier/app.go @@ -66,6 +66,12 @@ func main() { services.InitializeGoogleLogger() defer services.CloseGoogleLogger() + // Load webhook secret from Secret Manager if not directly provided + if err := services.LoadWebhookSecret(config); err != nil { + fmt.Printf("Error loading webhook secret: %v\n", err) + return + } + // Configure GitHub permissions services.ConfigurePermissions() diff --git a/examples-copier/configs/environment.go b/examples-copier/configs/environment.go index e65069a..b8d7eeb 100644 --- a/examples-copier/configs/environment.go +++ b/examples-copier/configs/environment.go @@ -24,6 +24,8 @@ type Config struct { WebserverPath string SrcBranch string PEMKeyName string + WebhookSecretName string + WebhookSecret string CopierLogName string GoogleCloudProjectId string DefaultRecursiveCopy bool @@ -62,6 +64,8 @@ const ( WebserverPath = "WEBSERVER_PATH" SrcBranch = "SRC_BRANCH" PEMKeyName = "PEM_NAME" + WebhookSecretName = "WEBHOOK_SECRET_NAME" + WebhookSecret = "WEBHOOK_SECRET" CopierLogName = "COPIER_LOG_NAME" GoogleCloudProjectId = "GOOGLE_CLOUD_PROJECT_ID" DefaultRecursiveCopy = "DEFAULT_RECURSIVE_COPY" @@ -92,6 +96,7 @@ func NewConfig() *Config { WebserverPath: "/webhook", SrcBranch: "main", // Default branch to copy from (NOTE: we are purposefully only allowing copying from `main` branch right now) PEMKeyName: "projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest", // default secret name for GCP Secret Manager + WebhookSecretName: "projects/1054147886816/secrets/webhook-secret/versions/latest", // default webhook secret name for GCP Secret Manager CopierLogName: "copy-copier-log", // default log name for logging to GCP GoogleCloudProjectId: "github-copy-code-examples", // default project ID for logging to GCP DefaultRecursiveCopy: true, // system-wide default for recursive copying that individual config entries can override. @@ -143,6 +148,8 @@ func LoadEnvironment(envFile string) (*Config, error) { config.WebserverPath = getEnvWithDefault(WebserverPath, config.WebserverPath) config.SrcBranch = getEnvWithDefault(SrcBranch, config.SrcBranch) config.PEMKeyName = getEnvWithDefault(PEMKeyName, config.PEMKeyName) + config.WebhookSecretName = getEnvWithDefault(WebhookSecretName, config.WebhookSecretName) + config.WebhookSecret = os.Getenv(WebhookSecret) config.DefaultRecursiveCopy = getBoolEnvWithDefault(DefaultRecursiveCopy, config.DefaultRecursiveCopy) config.DefaultPRMerge = getBoolEnvWithDefault(DefaultPRMerge, config.DefaultPRMerge) config.CopierLogName = getEnvWithDefault(CopierLogName, config.CopierLogName) @@ -217,7 +224,7 @@ func validateConfig(config *Config) error { requiredVars := map[string]string{ RepoName: config.RepoName, RepoOwner: config.RepoOwner, - AppId: config.AppId, + AppId: config.AppId, InstallationId: config.InstallationId, } diff --git a/examples-copier/deploy.sh b/examples-copier/deploy.sh new file mode 100755 index 0000000..12551e6 --- /dev/null +++ b/examples-copier/deploy.sh @@ -0,0 +1,258 @@ +#!/bin/bash + +# Deployment script for examples-copier to Google Cloud App Engine +# Usage: ./deploy.sh [options] +# +# Options: +# --project PROJECT_ID Set GCP project ID +# --version VERSION Set version name (default: auto-generated) +# --no-promote Deploy without promoting to receive traffic +# --quiet Skip confirmation prompts +# --env-file FILE Path to env.yaml file (default: env.yaml) +# --help Show this help message + +set -e # Exit on error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Default values +PROJECT_ID="" +VERSION="" +PROMOTE="true" +QUIET="false" +ENV_FILE="env.yaml" + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --project) + PROJECT_ID="$2" + shift 2 + ;; + --version) + VERSION="$2" + shift 2 + ;; + --no-promote) + PROMOTE="false" + shift + ;; + --quiet) + QUIET="true" + shift + ;; + --env-file) + ENV_FILE="$2" + shift 2 + ;; + --help) + echo "Usage: ./deploy.sh [options]" + echo "" + echo "Options:" + echo " --project PROJECT_ID Set GCP project ID" + echo " --version VERSION Set version name (default: auto-generated)" + echo " --no-promote Deploy without promoting to receive traffic" + echo " --quiet Skip confirmation prompts" + echo " --env-file FILE Path to env.yaml file (default: env.yaml)" + echo " --help Show this help message" + exit 0 + ;; + *) + echo -e "${RED}Unknown option: $1${NC}" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Function to print colored messages +print_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +print_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Function to check if command exists +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +# Check prerequisites +print_info "Checking prerequisites..." + +# Check if gcloud is installed +if ! command_exists gcloud; then + print_error "gcloud CLI is not installed" + echo "Install from: https://cloud.google.com/sdk/docs/install" + exit 1 +fi + +# Check if go is installed +if ! command_exists go; then + print_error "Go is not installed" + echo "Install from: https://golang.org/dl/" + exit 1 +fi + +print_success "Prerequisites check passed" + +# Get current project if not specified +if [ -z "$PROJECT_ID" ]; then + PROJECT_ID=$(gcloud config get-value project 2>/dev/null) + if [ -z "$PROJECT_ID" ]; then + print_error "No GCP project configured" + echo "Set project with: gcloud config set project PROJECT_ID" + echo "Or use: ./deploy.sh --project PROJECT_ID" + exit 1 + fi +fi + +print_info "Using GCP project: $PROJECT_ID" + +# Check if env.yaml exists +if [ ! -f "$ENV_FILE" ]; then + print_error "Environment file not found: $ENV_FILE" + echo "" + echo "Create $ENV_FILE with required environment variables:" + echo "" + cat << 'EOF' +env_variables: + GITHUB_APP_ID: "your-app-id" + INSTALLATION_ID: "your-installation-id" + REPO_NAME: "your-repo-name" + REPO_OWNER: "your-repo-owner" + GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/PROJECT_ID/secrets/SECRET_NAME/versions/latest" + WEBHOOK_SECRET: "your-webhook-secret" + COMMITTER_NAME: "GitHub Copier App" + COMMITTER_EMAIL: "bot@example.com" + CONFIG_FILE: "copier-config.yaml" + DEPRECATION_FILE: "deprecated_examples.json" + WEBSERVER_PATH: "/events" +EOF + echo "" + exit 1 +fi + +print_success "Environment file found: $ENV_FILE" + +# Check if app.yaml exists +if [ ! -f "app.yaml" ]; then + print_error "app.yaml not found in current directory" + echo "Run this script from the examples-copier directory" + exit 1 +fi + +# Build the application +print_info "Building application..." +if go build -o examples-copier .; then + print_success "Build successful" +else + print_error "Build failed" + exit 1 +fi + +# Run tests +print_info "Running tests..." +if go test ./... -v; then + print_success "All tests passed" +else + print_warning "Some tests failed - continuing anyway" +fi + +# Show deployment summary +echo "" +echo "=========================================" +echo "Deployment Summary" +echo "=========================================" +echo "Project: $PROJECT_ID" +echo "Version: ${VERSION:-auto-generated}" +echo "Promote: $PROMOTE" +echo "Env File: $ENV_FILE" +echo "=========================================" +echo "" + +# Confirm deployment +if [ "$QUIET" != "true" ]; then + read -p "Continue with deployment? (y/N) " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + print_info "Deployment cancelled" + exit 0 + fi +fi + +# Build gcloud command +DEPLOY_CMD="gcloud app deploy app.yaml --env-vars-file=$ENV_FILE --project=$PROJECT_ID" + +if [ -n "$VERSION" ]; then + DEPLOY_CMD="$DEPLOY_CMD --version=$VERSION" +fi + +if [ "$PROMOTE" != "true" ]; then + DEPLOY_CMD="$DEPLOY_CMD --no-promote" +fi + +if [ "$QUIET" = "true" ]; then + DEPLOY_CMD="$DEPLOY_CMD --quiet" +fi + +# Deploy to App Engine +print_info "Deploying to App Engine..." +echo "Command: $DEPLOY_CMD" +echo "" + +if eval "$DEPLOY_CMD"; then + print_success "Deployment successful!" + echo "" + + # Get app URL + APP_URL=$(gcloud app describe --project=$PROJECT_ID --format="value(defaultHostname)" 2>/dev/null) + if [ -n "$APP_URL" ]; then + print_info "Application URL: https://$APP_URL" + print_info "Webhook URL: https://$APP_URL/events" + fi + + echo "" + print_info "Next steps:" + echo " 1. Update GitHub webhook URL to: https://$APP_URL/events" + echo " 2. Verify webhook secret matches WEBHOOK_SECRET in env.yaml" + echo " 3. Test webhook by merging a PR in source repository" + echo " 4. Monitor logs: gcloud app logs tail -s default --project=$PROJECT_ID" + echo "" + + # Ask if user wants to view logs + if [ "$QUIET" != "true" ]; then + read -p "View application logs? (y/N) " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + gcloud app logs tail -s default --project=$PROJECT_ID + fi + fi +else + print_error "Deployment failed" + echo "" + echo "Troubleshooting:" + echo " 1. Check that all required APIs are enabled:" + echo " gcloud services enable appengine.googleapis.com --project=$PROJECT_ID" + echo " gcloud services enable secretmanager.googleapis.com --project=$PROJECT_ID" + echo " 2. Verify env.yaml contains all required variables" + echo " 3. Check deployment logs for specific errors" + echo " 4. See DEPLOYMENT-GUIDE.md for detailed troubleshooting" + exit 1 +fi + diff --git a/examples-copier/env.yaml.example b/examples-copier/env.yaml.example new file mode 100644 index 0000000..c0b8699 --- /dev/null +++ b/examples-copier/env.yaml.example @@ -0,0 +1,123 @@ +# Example environment variables for Google Cloud App Engine deployment +# Copy this file to env.yaml and fill in your actual values +# +# ⚠️ IMPORTANT: Add env.yaml to .gitignore to prevent committing secrets! + +env_variables: + # ============================================================================= + # REQUIRED CONFIGURATION + # ============================================================================= + + # GitHub App Configuration + GITHUB_APP_ID: "1166559" # Your GitHub App ID (numeric) + INSTALLATION_ID: "62138132" # GitHub App Installation ID + + # Source Repository Configuration + REPO_NAME: "docs-code-examples" # Source repository name + REPO_OWNER: "mongodb" # Source repository owner + + # Security Configuration + # Path to GitHub App private key in Google Secret Manager + GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/YOUR_PROJECT_ID/secrets/CODE_COPIER_PEM/versions/latest" + + # Webhook Secret Configuration (choose ONE of the following): + # Option 1: Store webhook secret in Secret Manager (RECOMMENDED for production) + WEBHOOK_SECRET_NAME: "projects/YOUR_PROJECT_ID/secrets/webhook-secret/versions/latest" + # Option 2: Provide webhook secret directly (NOT recommended for production) + # WEBHOOK_SECRET: "your-webhook-secret-here" + + # ============================================================================= + # OPTIONAL CONFIGURATION + # ============================================================================= + + # Committer Information + COMMITTER_NAME: "GitHub Copier App" + COMMITTER_EMAIL: "bot@mongodb.com" + + # Web Server Configuration + PORT: "8080" # Server port (App Engine uses 8080 by default) + WEBSERVER_PATH: "/events" # Webhook endpoint path + + # File Configuration + CONFIG_FILE: "copier-config.yaml" # Config file name in source repo + DEPRECATION_FILE: "deprecated_examples.json" # Deprecation tracking file name + + # Source Branch + SRC_BRANCH: "main" # Branch to monitor for changes + + # Google Cloud Configuration + GOOGLE_PROJECT_ID: "YOUR_PROJECT_ID" # GCP project ID for logging + GOOGLE_LOG_NAME: "code-copier-log" # Cloud Logging log name + + # Commit Strategy Defaults + COPIER_COMMIT_STRATEGY: "direct" # Default: "direct" or "pr" + DEFAULT_COMMIT_MESSAGE: "Update code examples from source repository" + DEFAULT_RECURSIVE_COPY: "true" # Default recursive copy behavior + DEFAULT_PR_MERGE: "false" # Default auto-merge PR behavior + + # Request Configuration + REQUEST_TIMEOUT_SECONDS: "300" # Webhook request timeout (5 minutes) + + # Logging Configuration + LOG_LEVEL: "info" # Log level: debug, info, warn, error + COPIER_DISABLE_CLOUD_LOGGING: "false" # Set to "true" to disable Cloud Logging + +# ============================================================================= +# NOTES +# ============================================================================= +# +# 1. GitHub App Private Key: +# Store your GitHub App private key in Google Secret Manager: +# +# gcloud secrets create CODE_COPIER_PEM \ +# --data-file=/path/to/private-key.pem \ +# --replication-policy="automatic" +# +# Grant App Engine access: +# gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ +# --member="serviceAccount:YOUR_PROJECT_ID@appspot.gserviceaccount.com" \ +# --role="roles/secretmanager.secretAccessor" +# +# 2. Webhook Secret: +# Option A: Store in Secret Manager (RECOMMENDED): +# +# # Generate a secure random string +# openssl rand -hex 32 +# +# # Store in Secret Manager +# echo -n "YOUR_GENERATED_SECRET" | gcloud secrets create webhook-secret \ +# --data-file=- \ +# --replication-policy="automatic" +# +# # Grant App Engine access +# gcloud secrets add-iam-policy-binding webhook-secret \ +# --member="serviceAccount:YOUR_PROJECT_ID@appspot.gserviceaccount.com" \ +# --role="roles/secretmanager.secretAccessor" +# +# # Use WEBHOOK_SECRET_NAME in env.yaml +# WEBHOOK_SECRET_NAME: "projects/YOUR_PROJECT_ID/secrets/webhook-secret/versions/latest" +# +# Option B: Provide directly (NOT recommended for production): +# +# # Use WEBHOOK_SECRET in env.yaml +# WEBHOOK_SECRET: "your-generated-secret-here" +# +# Use the same secret in your GitHub webhook configuration. +# +# 3. Deployment: +# Deploy using: +# +# gcloud app deploy app.yaml --env-vars-file=env.yaml +# +# Or use the deployment script: +# +# ./deploy.sh +# +# 4. Security: +# - Never commit env.yaml to version control +# - Add env.yaml to .gitignore +# - Rotate secrets regularly +# - Use least-privilege IAM roles +# +# ============================================================================= + diff --git a/examples-copier/services/github_auth.go b/examples-copier/services/github_auth.go index 046254e..174f5d0 100644 --- a/examples-copier/services/github_auth.go +++ b/examples-copier/services/github_auth.go @@ -118,6 +118,49 @@ func getPrivateKeyFromSecret() []byte { return result.Payload.Data } +// getWebhookSecretFromSecretManager retrieves the webhook secret from Google Cloud Secret Manager +func getWebhookSecretFromSecretManager(secretName string) (string, error) { + if os.Getenv("SKIP_SECRET_MANAGER") == "true" { + // For tests and local runs, use direct env var + if secret := os.Getenv(configs.WebhookSecret); secret != "" { + return secret, nil + } + return "", fmt.Errorf("SKIP_SECRET_MANAGER=true but no WEBHOOK_SECRET set") + } + + ctx := context.Background() + client, err := secretmanager.NewClient(ctx) + if err != nil { + return "", fmt.Errorf("failed to create Secret Manager client: %w", err) + } + defer client.Close() + + req := &secretmanagerpb.AccessSecretVersionRequest{ + Name: secretName, + } + result, err := client.AccessSecretVersion(ctx, req) + if err != nil { + return "", fmt.Errorf("failed to access secret version: %w", err) + } + return string(result.Payload.Data), nil +} + +// LoadWebhookSecret loads the webhook secret from Secret Manager or environment variable +func LoadWebhookSecret(config *configs.Config) error { + // If webhook secret is already set directly, use it + if config.WebhookSecret != "" { + return nil + } + + // Otherwise, load from Secret Manager + secret, err := getWebhookSecretFromSecretManager(config.WebhookSecretName) + if err != nil { + return fmt.Errorf("failed to load webhook secret: %w", err) + } + config.WebhookSecret = secret + return nil +} + // getInstallationAccessToken exchanges a JWT for a GitHub App installation access token. func getInstallationAccessToken(installationId, jwtToken string, hc *http.Client) (string, error) { if installationId == "" || installationId == configs.InstallationId { From 65f27e26a1c7aefff7d9822013c1bf2f64bc4310 Mon Sep 17 00:00:00 2001 From: cbullinger Date: Thu, 9 Oct 2025 11:30:07 -0400 Subject: [PATCH 2/2] Document missing env vars and update to use yaml in prod --- examples-copier/.gitignore | 1 + examples-copier/CONFIG-LOADING-BEHAVIOR.md | 396 ------------- examples-copier/DEPLOYMENT-CHECKLIST.md | 356 ------------ examples-copier/DEPLOYMENT-GUIDE.md | 471 ---------------- examples-copier/DEPLOYMENT-SUMMARY.md | 351 ------------ examples-copier/Makefile | 22 +- examples-copier/QUICK-REFERENCE.md | 27 +- examples-copier/README.md | 8 +- .../WEBHOOK-SECRET-MANAGER-GUIDE.md | 343 ------------ examples-copier/app.go | 17 +- examples-copier/app.yaml | 3 + examples-copier/cmd/test-webhook/main.go | 6 +- examples-copier/config.json | 14 - examples-copier/configs/.env.example | 54 -- examples-copier/configs/.env.example.new | 97 ---- .../{.env.local => .env.local.example} | 99 +++- examples-copier/configs/.env.test | 26 - examples-copier/configs/README.md | 272 +++++++++ examples-copier/configs/env.yaml.example | 181 ++++++ examples-copier/configs/env.yaml.production | 61 ++ examples-copier/configs/environment.go | 37 +- examples-copier/copier-config.yaml | 39 -- examples-copier/deploy.sh | 258 --------- examples-copier/docs/CONFIGURATION-GUIDE.md | 2 +- examples-copier/docs/DEBUG-LOGGING.md | 376 +++++++++++++ examples-copier/docs/DEPLOYMENT-CHECKLIST.md | 493 ++++++++++++++++ examples-copier/docs/DEPLOYMENT-GUIDE.md | 369 ------------ examples-copier/docs/DEPLOYMENT.md | 524 ++++++++++++++++++ examples-copier/docs/FAQ.md | 2 +- examples-copier/docs/LOCAL-TESTING.md | 54 +- examples-copier/docs/TROUBLESHOOTING.md | 3 + examples-copier/docs/WEBHOOK-TESTING.md | 2 +- examples-copier/env.yaml.example | 123 ---- .../scripts/convert-env-to-yaml.sh | 51 ++ .../scripts/grant-secret-access.sh | 42 ++ examples-copier/services/github_auth.go | 48 ++ .../services/github_write_to_source.go | 4 +- 37 files changed, 2259 insertions(+), 2973 deletions(-) delete mode 100644 examples-copier/CONFIG-LOADING-BEHAVIOR.md delete mode 100644 examples-copier/DEPLOYMENT-CHECKLIST.md delete mode 100644 examples-copier/DEPLOYMENT-GUIDE.md delete mode 100644 examples-copier/DEPLOYMENT-SUMMARY.md delete mode 100644 examples-copier/WEBHOOK-SECRET-MANAGER-GUIDE.md delete mode 100644 examples-copier/config.json delete mode 100644 examples-copier/configs/.env.example delete mode 100644 examples-copier/configs/.env.example.new rename examples-copier/configs/{.env.local => .env.local.example} (60%) delete mode 100644 examples-copier/configs/.env.test create mode 100644 examples-copier/configs/README.md create mode 100644 examples-copier/configs/env.yaml.example create mode 100644 examples-copier/configs/env.yaml.production delete mode 100644 examples-copier/copier-config.yaml delete mode 100755 examples-copier/deploy.sh create mode 100644 examples-copier/docs/DEBUG-LOGGING.md create mode 100644 examples-copier/docs/DEPLOYMENT-CHECKLIST.md delete mode 100644 examples-copier/docs/DEPLOYMENT-GUIDE.md create mode 100644 examples-copier/docs/DEPLOYMENT.md delete mode 100644 examples-copier/env.yaml.example create mode 100755 examples-copier/scripts/convert-env-to-yaml.sh create mode 100755 examples-copier/scripts/grant-secret-access.sh diff --git a/examples-copier/.gitignore b/examples-copier/.gitignore index 18df324..21aa038 100644 --- a/examples-copier/.gitignore +++ b/examples-copier/.gitignore @@ -47,3 +47,4 @@ Thumbs.db tmp/ temp/ +env.yaml diff --git a/examples-copier/CONFIG-LOADING-BEHAVIOR.md b/examples-copier/CONFIG-LOADING-BEHAVIOR.md deleted file mode 100644 index fa2140d..0000000 --- a/examples-copier/CONFIG-LOADING-BEHAVIOR.md +++ /dev/null @@ -1,396 +0,0 @@ -# Config Loading Behavior - When Is Config Read? - -## Quick Answer - -**The config file is read from the SOURCE BRANCH (typically `main`), NOT from the merged PR.** - -This means: -- ❌ **Config changes in a PR are NOT used** for that same PR -- ✅ **Config changes take effect** for the NEXT PR after they're merged -- ⚠️ **You cannot update config and files in the same PR** and have the new config apply - -## How It Works - -### Webhook Flow - -When a PR is merged: - -``` -1. PR Merged → Webhook triggered -2. Authenticate with GitHub -3. Read config file from SOURCE BRANCH (main) ← IMPORTANT -4. Get changed files from the merged PR -5. Apply copy rules from config -6. Copy files to target repos -``` - -### Code Implementation - - -````go -func retrieveJsonFile(filePath string) string { - client := GetRestClient() - owner := os.Getenv(configs.RepoOwner) - repo := os.Getenv(configs.RepoName) - ctx := context.Background() - fileContent, _, _, err := - client.Repositories.GetContents(ctx, owner, repo, - filePath, &github.RepositoryContentGetOptions{ - Ref: os.Getenv(configs.SrcBranch), // ← Reads from SRC_BRANCH (default: "main") - }) - // ... -} -```` - - -**Key Point:** The `Ref` parameter is set to `os.Getenv(configs.SrcBranch)`, which defaults to `"main"`. - -### Environment Configuration - - -````go -func NewConfig() *Config { - return &Config{ - // ... - SrcBranch: "main", // ← Default branch to read config from - // ... - } -} -```` - - -**Default:** Config is always read from the `main` branch (or whatever `SRC_BRANCH` is set to). - -## Scenarios - -### Scenario 1: Update Config Only - -**PR Contents:** -- Modified `copier-config.yaml` (new copy rule added) - -**What Happens:** -1. PR merged -2. Webhook reads config from `main` (BEFORE this PR merged) -3. Old config is used (new rule NOT applied) -4. No files copied (because no other files changed) -5. Config changes are now in `main` for NEXT PR - -**Result:** ✅ Config updated for future PRs - -### Scenario 2: Update Config + Files in Same PR - -**PR Contents:** -- Modified `copier-config.yaml` (new rule: copy `examples/new/*.go`) -- Added `examples/new/example.go` - -**What Happens:** -1. PR merged -2. Webhook reads config from `main` (BEFORE this PR merged) -3. Old config is used (new rule NOT in effect yet) -4. `examples/new/example.go` does NOT match any rules -5. File is NOT copied ❌ -6. Config changes are now in `main` for NEXT PR - -**Result:** ⚠️ File NOT copied - need another PR to trigger copy - -### Scenario 3: Update Files After Config Merged - -**PR 1 Contents:** -- Modified `copier-config.yaml` (new rule: copy `examples/new/*.go`) - -**PR 1 Result:** -- Config merged to `main` -- No files copied (no other files changed) - -**PR 2 Contents:** -- Added `examples/new/example.go` - -**PR 2 What Happens:** -1. PR merged -2. Webhook reads config from `main` (includes new rule from PR 1) -3. New config is used ✅ -4. `examples/new/example.go` matches new rule -5. File is copied to target repo ✅ - -**Result:** ✅ File copied successfully - -### Scenario 4: Update Existing Rule - -**Current Config:** -```yaml -copy_rules: - - name: "Go examples" - source_pattern: - type: "prefix" - pattern: "examples/go" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/${relative_path}" -``` - -**PR Contents:** -- Modified `copier-config.yaml` (changed `path_transform` to `"docs/${relative_path}"`) -- Modified `examples/go/example.go` - -**What Happens:** -1. PR merged -2. Webhook reads config from `main` (BEFORE this PR merged) -3. Old config is used (old path_transform: `"code/${relative_path}"`) -4. `examples/go/example.go` is copied to `code/go/example.go` (OLD path) -5. Config changes are now in `main` for NEXT PR - -**Result:** ⚠️ File copied to OLD path - need another PR to copy to NEW path - -## Why This Design? - -### Stability - -**Reason:** Ensures config is stable and tested before being used. - -**Benefit:** -- Config changes are reviewed and merged first -- Next PR uses the reviewed config -- Reduces risk of broken config affecting file copying - -### Predictability - -**Reason:** Config state is known at webhook trigger time. - -**Benefit:** -- No race conditions between config and file changes -- Clear separation: config changes vs file changes -- Easier to debug and understand behavior - -### Simplicity - -**Reason:** Always reads from a known branch (`main`). - -**Benefit:** -- No need to check if config changed in PR -- No need to merge config changes before reading -- Simpler implementation - -## Workarounds - -### Option 1: Two-PR Workflow (Recommended) - -**Step 1:** Update config -```bash -# PR 1: Update config only -git checkout -b update-config -# Edit copier-config.yaml -git add copier-config.yaml -git commit -m "Add new copy rule for X" -git push origin update-config -# Create PR, get approval, merge -``` - -**Step 2:** Add/modify files -```bash -# PR 2: Add files that use new config -git checkout -b add-files -# Add/modify files -git add examples/new/ -git commit -m "Add new examples" -git push origin add-files -# Create PR, get approval, merge -# Files will be copied using new config ✅ -``` - -### Option 2: Manual Trigger (If Supported) - -If the tool supports manual triggering: -```bash -# After PR with config + files is merged -# Manually trigger webhook or re-run copier -# This will use the newly merged config -``` - -### Option 3: Empty Commit Trigger - -**After config is merged:** -```bash -# Create empty commit to trigger webhook -git checkout -b trigger-copy -git commit --allow-empty -m "Trigger copy with new config" -git push origin trigger-copy -# Create PR, merge -# This will trigger webhook with new config -``` - -### Option 4: Modify File Again - -**After config is merged:** -```bash -# Make a small change to the file -git checkout -b fix-copy -# Edit the file (add comment, fix typo, etc.) -git add examples/new/example.go -git commit -m "Trigger copy with updated config" -git push origin fix-copy -# Create PR, merge -# File will be copied with new config ✅ -``` - -## Best Practices - -### 1. Update Config First - -**Always update config in a separate PR before adding files that use it.** - -``` -PR 1: Update copier-config.yaml - ↓ (merge) -PR 2: Add files that match new rules - ↓ (merge, files copied ✅) -``` - -### 2. Test Config Changes - -**Use config-validator to test config before merging:** - -```bash -cd examples-copier -./tools/config-validator/config-validator -config copier-config.yaml -``` - -### 3. Document Config Changes - -**In PR description, note that files will be copied in NEXT PR:** - -```markdown -## Changes -- Added new copy rule for `examples/python/*.py` - -## Note -Files matching this rule will be copied starting with the NEXT PR after this is merged. -``` - -### 4. Plan Multi-Step Changes - -**For complex changes, plan the sequence:** - -``` -Step 1: Update config (PR #123) -Step 2: Add new files (PR #124) -Step 3: Update existing files (PR #125) -``` - -### 5. Use Dry-Run for Testing - -**Test config changes with dry-run mode:** - -```bash -DRY_RUN=true ./examples-copier -# Check logs to see what would be copied -``` - -## Common Mistakes - -### ❌ Mistake 1: Config + Files in Same PR - -**Problem:** -``` -PR: Update config + add files -Result: Files NOT copied (old config used) -``` - -**Solution:** -``` -PR 1: Update config -PR 2: Add files (after PR 1 merged) -``` - -### ❌ Mistake 2: Expecting Immediate Effect - -**Problem:** -``` -Merge PR with config changes -Expect next file change to use new config immediately -``` - -**Reality:** -``` -Config changes take effect for NEXT PR -Current PR uses OLD config -``` - -### ❌ Mistake 3: Not Testing Config - -**Problem:** -``` -Merge config with typo -Next PR fails to copy files -``` - -**Solution:** -``` -Use config-validator before merging -Test with dry-run mode -``` - -## Future Enhancements - -Potential improvements to config loading: - -### 1. Read Config from Merged PR - -**Idea:** Read config from the merged commit instead of `main`. - -**Benefit:** -- Config changes apply immediately -- Single PR can update config + files - -**Challenge:** -- More complex implementation -- Potential race conditions -- Harder to debug - -### 2. Config Validation on PR - -**Idea:** Validate config in PR before merge. - -**Benefit:** -- Catch config errors before merge -- Prevent broken config from reaching `main` - -**Implementation:** -- GitHub Action to validate config -- Block merge if validation fails - -### 3. Config Change Detection - -**Idea:** Detect if config changed in PR and warn user. - -**Benefit:** -- User knows config won't apply to current PR -- Suggests two-PR workflow - -**Implementation:** -- Check if `copier-config.yaml` in changed files -- Add comment to PR with warning - -## Summary - -**Current Behavior:** -- ✅ Config is read from `main` branch (or `SRC_BRANCH`) -- ❌ Config changes in PR do NOT apply to that PR -- ✅ Config changes apply to NEXT PR after merge - -**Recommended Workflow:** -1. Update config in separate PR -2. Merge config PR -3. Add/modify files in subsequent PR -4. Files copied with new config ✅ - -**Key Takeaway:** -> Config changes require a two-PR workflow: one to update config, another to use the new config. - ---- - -**See Also:** -- [Configuration Guide](docs/CONFIGURATION-GUIDE.md) - Complete config reference -- [Pattern Matching Guide](docs/PATTERN-MATCHING-GUIDE.md) - Pattern syntax -- [Troubleshooting](docs/TROUBLESHOOTING.md) - Common issues - diff --git a/examples-copier/DEPLOYMENT-CHECKLIST.md b/examples-copier/DEPLOYMENT-CHECKLIST.md deleted file mode 100644 index 5e957ad..0000000 --- a/examples-copier/DEPLOYMENT-CHECKLIST.md +++ /dev/null @@ -1,356 +0,0 @@ -g# Deployment Checklist - -Quick checklist for deploying examples-copier to Google Cloud App Engine. - -## Pre-Deployment - -### ☐ 1. Install Prerequisites - -```bash -# Install Google Cloud SDK (if not installed) -brew install --cask google-cloud-sdk - -# Verify installation -gcloud --version -go version -``` - -### ☐ 2. Authenticate with Google Cloud - -```bash -# Login -gcloud auth login - -# Set application default credentials -gcloud auth application-default login -``` - -### ☐ 3. Set GCP Project - -```bash -# List projects -gcloud projects list - -# Set project -gcloud config set project YOUR_PROJECT_ID - -# Verify -gcloud config get-value project -``` - -### ☐ 4. Enable Required APIs - -```bash -# Enable App Engine -gcloud services enable appengine.googleapis.com - -# Enable Secret Manager -gcloud services enable secretmanager.googleapis.com - -# Enable Cloud Logging -gcloud services enable logging.googleapis.com - -# Verify -gcloud services list --enabled | grep -E "appengine|secretmanager|logging" -``` - -### ☐ 5. Store GitHub Private Key in Secret Manager - -```bash -# Create secret -gcloud secrets create CODE_COPIER_PEM \ - --data-file=/path/to/your/private-key.pem \ - --replication-policy="automatic" - -# Grant App Engine access -PROJECT_ID=$(gcloud config get-value project) -gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ - --member="serviceAccount:${PROJECT_ID}@appspot.gserviceaccount.com" \ - --role="roles/secretmanager.secretAccessor" - -# Get secret path (copy this for env.yaml) -echo "projects/${PROJECT_ID}/secrets/CODE_COPIER_PEM/versions/latest" -``` - -### ☐ 6. Create env.yaml - -```bash -# Copy example -cp env.yaml.example env.yaml - -# Edit with your values -nano env.yaml # or use your preferred editor -``` - -**Required values to update:** -- `GITHUB_APP_ID` -- `INSTALLATION_ID` -- `REPO_NAME` -- `REPO_OWNER` -- `GITHUB_APP_PRIVATE_KEY_SECRET_NAME` -- `WEBHOOK_SECRET` -- `GOOGLE_PROJECT_ID` - -### ☐ 7. Verify Configuration - -```bash -# Check env.yaml exists -ls -l env.yaml - -# Verify it's in .gitignore -grep "env.yaml" .gitignore -``` - -### ☐ 8. Build and Test - -```bash -# Build -go build -o examples-copier . - -# Run tests -go test ./... - -# Test locally (optional) -go run app.go -env ./configs/.env.test -``` - -## Deployment - -### ☐ 9. Deploy to App Engine - -**Option A: Using deployment script (recommended)** - -```bash -./deploy.sh -``` - -**Option B: Manual deployment** - -```bash -gcloud app deploy app.yaml --env-vars-file=env.yaml -``` - -**Option C: Deploy specific version without promoting** - -```bash -./deploy.sh --version=v2 --no-promote -``` - -### ☐ 10. Verify Deployment - -```bash -# Check deployment status -gcloud app versions list - -# Get app URL -gcloud app describe --format="value(defaultHostname)" - -# View logs -gcloud app logs tail -s default -``` - -## Post-Deployment - -### ☐ 11. Update GitHub Webhook - -1. Go to source repository on GitHub -2. Settings → Webhooks -3. Edit existing webhook or create new one -4. Update Payload URL: `https://YOUR_PROJECT_ID.appspot.com/events` -5. Content type: `application/json` -6. Secret: (same as `WEBHOOK_SECRET` in env.yaml) -7. Events: Select "Pull requests" -8. Active: ✓ Checked -9. Save webhook - -### ☐ 12. Test Webhook - -```bash -# Method 1: Merge a test PR in source repository -# Watch logs for webhook receipt -gcloud app logs tail -s default - -# Method 2: Send test webhook from GitHub -# Go to webhook settings → Recent Deliveries → Redeliver -``` - -### ☐ 13. Verify Application is Working - -**Check logs for:** -- ✓ Webhook received -- ✓ Config file loaded -- ✓ Files copied to target repos -- ✓ No errors - -```bash -# View recent logs -gcloud app logs read --limit=50 - -# Filter for errors -gcloud app logs read --limit=100 | grep ERROR - -# Filter for webhook events -gcloud app logs read --limit=100 | grep webhook -``` - -### ☐ 14. Monitor Application - -```bash -# Real-time logs -gcloud app logs tail -s default - -# Open Cloud Console -gcloud app open-console - -# View metrics -# Go to Cloud Console → App Engine → Dashboard -``` - -## Rollback (if needed) - -### ☐ If Deployment Has Issues - -```bash -# List versions -gcloud app versions list - -# Route traffic to previous version -gcloud app services set-traffic default --splits=PREVIOUS_VERSION=1 - -# Delete bad version -gcloud app versions delete BAD_VERSION -``` - -## Troubleshooting - -### Issue: Deployment fails - -```bash -# Check APIs are enabled -gcloud services list --enabled - -# Verify app.yaml is valid -cat app.yaml - -# Check for build errors -go build -o examples-copier . -``` - -### Issue: Webhooks not received - -```bash -# Check webhook URL is correct -gcloud app describe --format="value(defaultHostname)" - -# Verify webhook secret matches -# Compare GitHub webhook secret with WEBHOOK_SECRET in env.yaml - -# Check logs for errors -gcloud app logs tail -s default | grep webhook -``` - -### Issue: Cannot access secrets - -```bash -# Verify secret exists -gcloud secrets list - -# Check IAM permissions -gcloud secrets get-iam-policy CODE_COPIER_PEM - -# Grant access if needed -PROJECT_ID=$(gcloud config get-value project) -gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ - --member="serviceAccount:${PROJECT_ID}@appspot.gserviceaccount.com" \ - --role="roles/secretmanager.secretAccessor" -``` - -### Issue: Files not being copied - -```bash -# Check config file in source repo -# Verify copier-config.yaml exists and is valid - -# Check logs for pattern matching -gcloud app logs read --limit=100 | grep "file matched" - -# Verify GitHub App has write access to target repos -``` - -## Quick Commands Reference - -```bash -# Deploy -./deploy.sh - -# Deploy specific version -./deploy.sh --version=v2 - -# Deploy without promoting -./deploy.sh --version=v2 --no-promote - -# View logs -gcloud app logs tail -s default - -# List versions -gcloud app versions list - -# Set traffic split -gcloud app services set-traffic default --splits=v1=0.5,v2=0.5 - -# Delete version -gcloud app versions delete VERSION_ID - -# Open Cloud Console -gcloud app open-console - -# Get app URL -gcloud app describe --format="value(defaultHostname)" -``` - -## Environment Variables Quick Reference - -**Required:** -- `GITHUB_APP_ID` - GitHub App ID -- `INSTALLATION_ID` - Installation ID -- `REPO_NAME` - Source repo name -- `REPO_OWNER` - Source repo owner -- `GITHUB_APP_PRIVATE_KEY_SECRET_NAME` - Secret Manager path -- `WEBHOOK_SECRET` - Webhook secret - -**Optional:** -- `PORT` - Server port (default: 8080) -- `WEBSERVER_PATH` - Webhook path (default: /webhook) -- `CONFIG_FILE` - Config file (default: copier-config.yaml) -- `DEPRECATION_FILE` - Deprecation file (default: deprecated_examples.json) -- `COMMITTER_NAME` - Git committer name -- `COMMITTER_EMAIL` - Git committer email -- `GOOGLE_PROJECT_ID` - GCP project for logging -- `GOOGLE_LOG_NAME` - Log name - -## Success Criteria - -✅ Deployment completes without errors -✅ Application is accessible at App Engine URL -✅ Webhook receives PR events from GitHub -✅ Config file is loaded successfully -✅ Files are copied to target repositories -✅ Logs show no errors -✅ Deprecation tracking works (if enabled) - -## Next Steps After Deployment - -1. Monitor logs for first few PRs -2. Verify files are copied correctly -3. Check target repositories for commits -4. Set up monitoring/alerting (optional) -5. Document any custom configuration -6. Share webhook URL with team - ---- - -**See Also:** -- [DEPLOYMENT-GUIDE.md](DEPLOYMENT-GUIDE.md) - Detailed deployment guide -- [README.md](README.md) - Application overview -- [CONFIG-LOADING-BEHAVIOR.md](CONFIG-LOADING-BEHAVIOR.md) - Config details - diff --git a/examples-copier/DEPLOYMENT-GUIDE.md b/examples-copier/DEPLOYMENT-GUIDE.md deleted file mode 100644 index 75b750e..0000000 --- a/examples-copier/DEPLOYMENT-GUIDE.md +++ /dev/null @@ -1,471 +0,0 @@ -# Deployment Guide - Google Cloud App Engine - -## Overview - -This guide covers deploying the examples-copier application to Google Cloud App Engine (Flexible Environment). - -## Prerequisites - -### 1. Google Cloud SDK - -Install the Google Cloud SDK if not already installed: - -```bash -# macOS (using Homebrew) -brew install --cask google-cloud-sdk - -# Or download from: https://cloud.google.com/sdk/docs/install -``` - -### 2. Authentication - -Authenticate with Google Cloud: - -```bash -# Login to your Google account -gcloud auth login - -# Set application default credentials -gcloud auth application-default login -``` - -### 3. Project Configuration - -Set your Google Cloud project: - -```bash -# List available projects -gcloud projects list - -# Set the project (replace with your project ID) -gcloud config set project YOUR_PROJECT_ID -``` - -### 4. Required APIs - -Enable required Google Cloud APIs: - -```bash -# Enable App Engine Admin API -gcloud services enable appengine.googleapis.com - -# Enable Secret Manager API (for GitHub private key) -gcloud services enable secretmanager.googleapis.com - -# Enable Cloud Logging API -gcloud services enable logging.googleapis.com -``` - -## Configuration - -### 1. Environment Variables - -The application uses environment variables for configuration. These are **NOT** stored in `app.yaml` for security reasons. - -**Required Environment Variables:** -- `GITHUB_APP_ID` - GitHub App ID (numeric) -- `INSTALLATION_ID` - GitHub App Installation ID -- `REPO_NAME` - Source repository name -- `REPO_OWNER` - Source repository owner -- `GITHUB_APP_PRIVATE_KEY_SECRET_NAME` - GCP Secret Manager path to private key -- `WEBHOOK_SECRET` - GitHub webhook secret for signature validation - -**Optional Environment Variables:** -- `PORT` - Server port (default: 8080) -- `WEBSERVER_PATH` - Webhook endpoint path (default: /webhook) -- `CONFIG_FILE` - Config file name (default: copier-config.yaml) -- `DEPRECATION_FILE` - Deprecation file name (default: deprecated_examples.json) -- `COMMITTER_NAME` - Git committer name (default: Copier Bot) -- `COMMITTER_EMAIL` - Git committer email (default: bot@example.com) -- `GOOGLE_PROJECT_ID` - GCP project ID for logging -- `GOOGLE_LOG_NAME` - Cloud Logging log name - -### 2. Store GitHub Private Key in Secret Manager - -The GitHub App private key must be stored in Google Cloud Secret Manager: - -```bash -# Create secret from file -gcloud secrets create CODE_COPIER_PEM \ - --data-file=/path/to/your/private-key.pem \ - --replication-policy="automatic" - -# Grant App Engine access to the secret -gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ - --member="serviceAccount:YOUR_PROJECT_ID@appspot.gserviceaccount.com" \ - --role="roles/secretmanager.secretAccessor" - -# Get the full secret path (use this in GITHUB_APP_PRIVATE_KEY_SECRET_NAME) -echo "projects/$(gcloud config get-value project)/secrets/CODE_COPIER_PEM/versions/latest" -``` - -### 3. Set Environment Variables in App Engine - -Environment variables are set during deployment using the `--env-vars-file` flag. - -Create `env.yaml`: - -```yaml -env_variables: - GITHUB_APP_ID: "1166559" - INSTALLATION_ID: "62138132" - REPO_NAME: "docs-code-examples" - REPO_OWNER: "mongodb" - GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/YOUR_PROJECT_ID/secrets/CODE_COPIER_PEM/versions/latest" - WEBHOOK_SECRET: "your-webhook-secret-here" - COMMITTER_NAME: "GitHub Copier App" - COMMITTER_EMAIL: "bot@mongodb.com" - CONFIG_FILE: "copier-config.yaml" - DEPRECATION_FILE: "deprecated_examples.json" - WEBSERVER_PATH: "/events" - GOOGLE_PROJECT_ID: "YOUR_PROJECT_ID" - GOOGLE_LOG_NAME: "code-copier-log" -``` - -**⚠️ IMPORTANT:** Add `env.yaml` to `.gitignore` to prevent committing secrets! - -## Deployment - -### Quick Deployment - -Use the provided deployment script: - -```bash -cd examples-copier -./deploy.sh -``` - -### Manual Deployment - -#### Step 1: Build and Test Locally - -```bash -cd examples-copier - -# Build the application -go build -o examples-copier . - -# Run tests -go test ./... - -# Test locally (optional) -go run app.go -env ./configs/.env.test -``` - -#### Step 2: Deploy to App Engine - -```bash -# Deploy with environment variables -gcloud app deploy app.yaml --env-vars-file=env.yaml - -# Or deploy without prompts -gcloud app deploy app.yaml --env-vars-file=env.yaml --quiet -``` - -#### Step 3: Verify Deployment - -```bash -# View deployment status -gcloud app versions list - -# View logs -gcloud app logs tail -s default - -# Open the application in browser -gcloud app browse -``` - -### Deployment Options - -**Deploy specific version:** -```bash -gcloud app deploy app.yaml --version=v1 --env-vars-file=env.yaml -``` - -**Deploy without promoting (traffic stays on current version):** -```bash -gcloud app deploy app.yaml --no-promote --env-vars-file=env.yaml -``` - -**Deploy and set traffic split:** -```bash -# Deploy new version -gcloud app deploy app.yaml --version=v2 --no-promote --env-vars-file=env.yaml - -# Split traffic (50% to v1, 50% to v2) -gcloud app services set-traffic default --splits=v1=0.5,v2=0.5 -``` - -## Post-Deployment - -### 1. Update GitHub Webhook URL - -After deployment, update the webhook URL in your source repository: - -``` -https://YOUR_PROJECT_ID.appspot.com/events -``` - -**Steps:** -1. Go to your source repository on GitHub -2. Settings → Webhooks -3. Edit the webhook -4. Update the Payload URL to your App Engine URL -5. Ensure Content type is `application/json` -6. Set the webhook secret (same as `WEBHOOK_SECRET` env var) -7. Select "Pull requests" event -8. Save webhook - -### 2. Verify Webhook - -Test the webhook: - -```bash -# Trigger a test PR merge in your source repo -# Check App Engine logs for webhook receipt - -gcloud app logs tail -s default -``` - -### 3. Monitor Application - -**View logs:** -```bash -# Tail logs in real-time -gcloud app logs tail -s default - -# View logs in Cloud Console -gcloud app logs read --limit=50 -``` - -**View metrics:** -```bash -# Open Cloud Console -gcloud app open-console -``` - -## Troubleshooting - -### Issue: Deployment Fails - -**Check:** -- All required APIs are enabled -- `app.yaml` is valid -- Go version matches runtime version -- Dependencies are up to date - -**Solution:** -```bash -# Update dependencies -go mod tidy - -# Verify app.yaml -cat app.yaml - -# Check enabled APIs -gcloud services list --enabled -``` - -### Issue: Application Not Receiving Webhooks - -**Check:** -- Webhook URL is correct -- Webhook secret matches `WEBHOOK_SECRET` env var -- GitHub App has correct permissions -- Firewall rules allow GitHub IPs - -**Solution:** -```bash -# Check logs for webhook errors -gcloud app logs tail -s default | grep webhook - -# Test webhook manually using curl -curl -X POST https://YOUR_PROJECT_ID.appspot.com/events \ - -H "Content-Type: application/json" \ - -d '{"test": "data"}' -``` - -### Issue: Cannot Access Secret Manager - -**Check:** -- Secret exists in Secret Manager -- App Engine service account has access -- Secret path is correct in env vars - -**Solution:** -```bash -# List secrets -gcloud secrets list - -# Check IAM permissions -gcloud secrets get-iam-policy CODE_COPIER_PEM - -# Grant access if needed -gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ - --member="serviceAccount:YOUR_PROJECT_ID@appspot.gserviceaccount.com" \ - --role="roles/secretmanager.secretAccessor" -``` - -### Issue: High Memory Usage - -**Check:** -- App Engine instance size -- Memory leaks in code -- Number of concurrent requests - -**Solution:** -Update `app.yaml`: -```yaml -runtime: go -runtime_config: - operating_system: "ubuntu22" - runtime_version: "1.23" -env: flex - -resources: - cpu: 1 - memory_gb: 2 - disk_size_gb: 10 - -automatic_scaling: - min_num_instances: 1 - max_num_instances: 5 - cool_down_period_sec: 120 - cpu_utilization: - target_utilization: 0.6 -``` - -## Rollback - -If deployment fails or has issues, rollback to previous version: - -```bash -# List versions -gcloud app versions list - -# Set traffic to previous version -gcloud app services set-traffic default --splits=PREVIOUS_VERSION=1 - -# Delete bad version (optional) -gcloud app versions delete BAD_VERSION -``` - -## Cost Optimization - -### 1. Use Minimum Instances - -For low-traffic applications: - -```yaml -automatic_scaling: - min_num_instances: 1 - max_num_instances: 2 -``` - -### 2. Use Standard Environment (if possible) - -App Engine Standard is cheaper than Flexible, but requires code changes. - -### 3. Monitor Costs - -```bash -# View billing -gcloud billing accounts list - -# Set budget alerts in Cloud Console -``` - -## Security Best Practices - -### 1. Never Commit Secrets - -Add to `.gitignore`: -``` -env.yaml -*.pem -.env.production -``` - -### 2. Rotate Secrets Regularly - -```bash -# Create new secret version -gcloud secrets versions add CODE_COPIER_PEM --data-file=/path/to/new-key.pem - -# Update env var to use new version -# Redeploy application -``` - -### 3. Use IAM Roles - -Grant minimum required permissions: -```bash -# App Engine service account should have: -# - Secret Manager Secret Accessor -# - Cloud Logging Writer -``` - -### 4. Enable VPC Service Controls (optional) - -For additional security in production. - -## Maintenance - -### Update Application - -```bash -# Pull latest code -git pull origin main - -# Build and test -go build -o examples-copier . -go test ./... - -# Deploy -gcloud app deploy app.yaml --env-vars-file=env.yaml -``` - -### Update Dependencies - -```bash -# Update Go modules -go get -u ./... -go mod tidy - -# Test -go test ./... - -# Deploy -gcloud app deploy app.yaml --env-vars-file=env.yaml -``` - -### View Application Info - -```bash -# App Engine info -gcloud app describe - -# List services -gcloud app services list - -# List versions -gcloud app versions list - -# View instance info -gcloud app instances list -``` - -## Additional Resources - -- [App Engine Go Documentation](https://cloud.google.com/appengine/docs/flexible/go) -- [Secret Manager Documentation](https://cloud.google.com/secret-manager/docs) -- [Cloud Logging Documentation](https://cloud.google.com/logging/docs) -- [GitHub Apps Documentation](https://docs.github.com/en/apps) - ---- - -**See Also:** -- [README.md](README.md) - Application overview -- [Configuration Guide](CONFIG-LOADING-BEHAVIOR.md) - Config file details -- [Troubleshooting](TROUBLESHOOTING.md) - Common issues - diff --git a/examples-copier/DEPLOYMENT-SUMMARY.md b/examples-copier/DEPLOYMENT-SUMMARY.md deleted file mode 100644 index b6305ff..0000000 --- a/examples-copier/DEPLOYMENT-SUMMARY.md +++ /dev/null @@ -1,351 +0,0 @@ -# Deployment Summary - -## Quick Start - -To deploy the latest version of examples-copier to Google Cloud App Engine: - -```bash -cd examples-copier - -# 1. Set up environment file -cp env.yaml.example env.yaml -# Edit env.yaml with your values - -# 2. Deploy -./deploy.sh -``` - -## What Was Created - -### 1. Deployment Documentation - -**DEPLOYMENT-GUIDE.md** (300+ lines) -- Complete deployment guide -- Prerequisites and setup -- Configuration instructions -- Deployment steps -- Post-deployment tasks -- Troubleshooting -- Security best practices -- Maintenance procedures - -**DEPLOYMENT-CHECKLIST.md** (300+ lines) -- Step-by-step checklist -- Pre-deployment tasks -- Deployment commands -- Post-deployment verification -- Rollback procedures -- Quick command reference -- Success criteria - -### 2. Deployment Script - -**deploy.sh** (executable) -- Automated deployment script -- Prerequisites checking -- Build and test -- Deployment to App Engine -- Post-deployment verification -- Options: - - `--project PROJECT_ID` - Set GCP project - - `--version VERSION` - Set version name - - `--no-promote` - Deploy without promoting - - `--quiet` - Skip prompts - - `--env-file FILE` - Custom env file path - - `--help` - Show help - -### 3. Configuration Files - -**env.yaml.example** -- Example environment variables -- All required and optional settings -- Documentation for each variable -- Security notes - -**.gitignore** -- Prevents committing secrets -- Excludes env.yaml, .env files -- Excludes private keys -- Standard Go ignores - -## Deployment Steps - -### Prerequisites - -1. **Install Google Cloud SDK** - ```bash - brew install --cask google-cloud-sdk - ``` - -2. **Authenticate** - ```bash - gcloud auth login - gcloud auth application-default login - ``` - -3. **Set Project** - ```bash - gcloud config set project YOUR_PROJECT_ID - ``` - -4. **Enable APIs** - ```bash - gcloud services enable appengine.googleapis.com - gcloud services enable secretmanager.googleapis.com - gcloud services enable logging.googleapis.com - ``` - -### Configuration - -1. **Store GitHub Private Key** - ```bash - gcloud secrets create CODE_COPIER_PEM \ - --data-file=/path/to/private-key.pem \ - --replication-policy="automatic" - - PROJECT_ID=$(gcloud config get-value project) - gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ - --member="serviceAccount:${PROJECT_ID}@appspot.gserviceaccount.com" \ - --role="roles/secretmanager.secretAccessor" - ``` - -2. **Create env.yaml** - ```bash - cp env.yaml.example env.yaml - # Edit with your values - ``` - - **Required values:** - - `GITHUB_APP_ID` - - `INSTALLATION_ID` - - `REPO_NAME` - - `REPO_OWNER` - - `GITHUB_APP_PRIVATE_KEY_SECRET_NAME` - - `WEBHOOK_SECRET` - - `GOOGLE_PROJECT_ID` - -### Deploy - -**Option 1: Using script (recommended)** -```bash -./deploy.sh -``` - -**Option 2: Manual** -```bash -gcloud app deploy app.yaml --env-vars-file=env.yaml -``` - -**Option 3: Specific version** -```bash -./deploy.sh --version=v2 --no-promote -``` - -### Post-Deployment - -1. **Get App URL** - ```bash - gcloud app describe --format="value(defaultHostname)" - ``` - -2. **Update GitHub Webhook** - - URL: `https://YOUR_PROJECT_ID.appspot.com/events` - - Content type: `application/json` - - Secret: (same as `WEBHOOK_SECRET`) - - Events: Pull requests - -3. **Verify** - ```bash - gcloud app logs tail -s default - ``` - -## Current Deployment Configuration - -### App Engine Settings (app.yaml) - -```yaml -runtime: go -runtime_config: - operating_system: "ubuntu22" - runtime_version: "1.23" -env: flex -``` - -**Environment:** Flexible Environment -**Runtime:** Go 1.23 -**OS:** Ubuntu 22.04 - -### Default Configuration - -**Port:** 8080 -**Webhook Path:** /events -**Config File:** copier-config.yaml -**Deprecation File:** deprecated_examples.json -**Source Branch:** main - -## Monitoring - -### View Logs - -```bash -# Real-time logs -gcloud app logs tail -s default - -# Recent logs -gcloud app logs read --limit=50 - -# Filter for errors -gcloud app logs read --limit=100 | grep ERROR - -# Filter for webhooks -gcloud app logs read --limit=100 | grep webhook -``` - -### View Metrics - -```bash -# Open Cloud Console -gcloud app open-console - -# View in browser -# Go to: App Engine → Dashboard -``` - -## Rollback - -If deployment has issues: - -```bash -# List versions -gcloud app versions list - -# Route traffic to previous version -gcloud app services set-traffic default --splits=PREVIOUS_VERSION=1 - -# Delete bad version -gcloud app versions delete BAD_VERSION -``` - -## Security Notes - -### ⚠️ Important - -1. **Never commit secrets** - - env.yaml is in .gitignore - - Never commit .env files - - Never commit .pem files - -2. **Use Secret Manager** - - GitHub private key stored in Secret Manager - - Not in environment variables - - Not in code - -3. **Rotate secrets regularly** - - Update GitHub App private key - - Update webhook secret - - Update in both GitHub and env.yaml - -4. **Minimum permissions** - - App Engine service account has minimal IAM roles - - Only Secret Manager Secret Accessor - - Only Cloud Logging Writer - -## Troubleshooting - -### Deployment Fails - -```bash -# Check APIs -gcloud services list --enabled - -# Verify build -go build -o examples-copier . - -# Check env.yaml -cat env.yaml -``` - -### Webhooks Not Received - -```bash -# Verify URL -gcloud app describe --format="value(defaultHostname)" - -# Check logs -gcloud app logs tail -s default | grep webhook - -# Test webhook from GitHub -# Settings → Webhooks → Recent Deliveries → Redeliver -``` - -### Cannot Access Secrets - -```bash -# List secrets -gcloud secrets list - -# Check permissions -gcloud secrets get-iam-policy CODE_COPIER_PEM - -# Grant access -PROJECT_ID=$(gcloud config get-value project) -gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ - --member="serviceAccount:${PROJECT_ID}@appspot.gserviceaccount.com" \ - --role="roles/secretmanager.secretAccessor" -``` - -## Files Created - -``` -examples-copier/ -├── DEPLOYMENT-GUIDE.md # Complete deployment guide -├── DEPLOYMENT-CHECKLIST.md # Step-by-step checklist -├── DEPLOYMENT-SUMMARY.md # This file -├── deploy.sh # Deployment script (executable) -├── env.yaml.example # Example environment variables -├── .gitignore # Git ignore file (includes env.yaml) -└── app.yaml # App Engine configuration (existing) -``` - -## Next Steps - -1. **Review Documentation** - - Read DEPLOYMENT-GUIDE.md for details - - Follow DEPLOYMENT-CHECKLIST.md step-by-step - -2. **Prepare Environment** - - Create env.yaml from example - - Store GitHub private key in Secret Manager - - Enable required APIs - -3. **Deploy** - - Run `./deploy.sh` - - Update GitHub webhook - - Test with a PR - -4. **Monitor** - - Watch logs for first few PRs - - Verify files are copied correctly - - Check for errors - -## Support - -**Documentation:** -- [DEPLOYMENT-GUIDE.md](DEPLOYMENT-GUIDE.md) - Detailed guide -- [DEPLOYMENT-CHECKLIST.md](DEPLOYMENT-CHECKLIST.md) - Checklist -- [README.md](README.md) - Application overview -- [CONFIG-LOADING-BEHAVIOR.md](CONFIG-LOADING-BEHAVIOR.md) - Config details - -**Google Cloud:** -- [App Engine Documentation](https://cloud.google.com/appengine/docs/flexible/go) -- [Secret Manager Documentation](https://cloud.google.com/secret-manager/docs) -- [Cloud Logging Documentation](https://cloud.google.com/logging/docs) - -**GitHub:** -- [GitHub Apps Documentation](https://docs.github.com/en/apps) -- [Webhooks Documentation](https://docs.github.com/en/webhooks) - ---- - -**Ready to deploy?** Run `./deploy.sh` to get started! - diff --git a/examples-copier/Makefile b/examples-copier/Makefile index 3ea26a1..b586e4c 100644 --- a/examples-copier/Makefile +++ b/examples-copier/Makefile @@ -60,7 +60,16 @@ test-webhook: # Test with example payload test-webhook-example: test-webhook @echo "Testing with example payload..." - @./test-webhook -payload test-payloads/example-pr-merged.json + @if [ -z "$$WEBHOOK_SECRET" ]; then \ + echo "Fetching webhook secret from Secret Manager..."; \ + export WEBHOOK_SECRET=$$(gcloud secrets versions access latest --secret=webhook-secret 2>/dev/null); \ + fi; \ + if [ -n "$$WEBHOOK_SECRET" ]; then \ + ./test-webhook -payload test-payloads/example-pr-merged.json -secret "$$WEBHOOK_SECRET"; \ + else \ + echo "Warning: WEBHOOK_SECRET not set, sending without signature"; \ + ./test-webhook -payload test-payloads/example-pr-merged.json; \ + fi # Test with real PR (requires PR, OWNER, REPO variables) test-webhook-pr: test-webhook @@ -69,7 +78,16 @@ test-webhook-pr: test-webhook echo "Usage: make test-webhook-pr PR=123 OWNER=myorg REPO=myrepo"; \ exit 1; \ fi - @./test-webhook -pr $(PR) -owner $(OWNER) -repo $(REPO) + @if [ -z "$$WEBHOOK_SECRET" ]; then \ + echo "Fetching webhook secret from Secret Manager..."; \ + export WEBHOOK_SECRET=$$(gcloud secrets versions access latest --secret=webhook-secret 2>/dev/null); \ + fi; \ + if [ -n "$$WEBHOOK_SECRET" ]; then \ + ./test-webhook -pr $(PR) -owner $(OWNER) -repo $(REPO) -secret "$$WEBHOOK_SECRET"; \ + else \ + echo "Warning: WEBHOOK_SECRET not set, sending without signature"; \ + ./test-webhook -pr $(PR) -owner $(OWNER) -repo $(REPO); \ + fi # Test with real PR using helper script test-pr: diff --git a/examples-copier/QUICK-REFERENCE.md b/examples-copier/QUICK-REFERENCE.md index 783b80c..3d83912 100644 --- a/examples-copier/QUICK-REFERENCE.md +++ b/examples-copier/QUICK-REFERENCE.md @@ -382,6 +382,29 @@ curl http://localhost:8080/health curl http://localhost:8080/metrics | jq ``` +## Deployment + +### Google Cloud Quick Commands + +```bash +# Deploy (env.yaml is included via 'includes' directive in app.yaml) +gcloud app deploy app.yaml + +# View logs +gcloud app logs tail -s default + +# Check health +curl https://github-copy-code-examples.appspot.com/health + +# List secrets +gcloud secrets list + +# Grant access +./grant-secret-access.sh +``` + + + ## File Locations ``` @@ -390,7 +413,9 @@ examples-copier/ ├── MIGRATION-GUIDE.md # Migration from legacy ├── QUICK-REFERENCE.md # This file ├── REFACTORING-SUMMARY.md # Feature details -├── DEPLOYMENT-GUIDE.md # Deployment instructions +├── docs/ +│ ├── DEPLOYMENT.md # Deployment guide +│ └── DEPLOYMENT-CHECKLIST.md # Deployment checklist ├── TESTING-SUMMARY.md # Test documentation ├── configs/ │ ├── .env # Environment config diff --git a/examples-copier/README.md b/examples-copier/README.md index 54b0ae9..1dbb97c 100644 --- a/examples-copier/README.md +++ b/examples-copier/README.md @@ -66,7 +66,8 @@ GITHUB_INSTALLATION_ID=789012 # Google Cloud GCP_PROJECT_ID=your-project -PEM_KEY_NAME=projects/123/secrets/CODE_COPIER_PEM/versions/latest +PEM_KEY_NAME=projects/123/secrets//versions/latest +WEBHOOK_SECRET_NAME=projects/123/secrets/webhook-secret # Application Settings PORT=8080 @@ -415,7 +416,7 @@ container := NewServiceContainer(config) ## Deployment -See [DEPLOYMENT-GUIDE.md](DEPLOYMENT-GUIDE.md) for complete deployment instructions. +See [DEPLOYMENT.md](./docs/DEPLOYMENT.md) for complete deployment guide and [DEPLOYMENT-CHECKLIST.md](./docs/DEPLOYMENT-CHECKLIST.md) for step-by-step checklist. ### Google Cloud App Engine @@ -444,7 +445,8 @@ docker run -p 8080:8080 --env-file .env examples-copier - **[Configuration Guide](docs/CONFIGURATION-GUIDE.md)** - Complete configuration reference ⭐ NEW - **[Pattern Matching Guide](docs/PATTERN-MATCHING-GUIDE.md)** - Pattern matching with examples - **[Local Testing](docs/LOCAL-TESTING.md)** - Test locally before deploying -- **[Deployment Guide](docs/DEPLOYMENT-GUIDE.md)** - Deploy to production +- **[Deployment Guide](docs/DEPLOYMENT.md)** - Deploy to production +- **[Deployment Checklist](docs/DEPLOYMENT-CHECKLIST.md)** - Step-by-step deployment checklist ### Reference diff --git a/examples-copier/WEBHOOK-SECRET-MANAGER-GUIDE.md b/examples-copier/WEBHOOK-SECRET-MANAGER-GUIDE.md deleted file mode 100644 index e7b1ea3..0000000 --- a/examples-copier/WEBHOOK-SECRET-MANAGER-GUIDE.md +++ /dev/null @@ -1,343 +0,0 @@ -# Using Webhook Secret from Google Cloud Secret Manager - -## Overview - -The examples-copier application now supports loading the webhook secret from Google Cloud Secret Manager instead of hardcoding it in environment variables. This is **more secure** and follows best practices for secret management. - -## Benefits - -✅ **Security**: Secrets are encrypted at rest and in transit -✅ **Audit Trail**: Secret Manager logs all access to secrets -✅ **Rotation**: Easy to rotate secrets without redeploying -✅ **Access Control**: Fine-grained IAM permissions -✅ **No Hardcoding**: Secrets never appear in config files or version control - -## Quick Start - -### 1. Store Webhook Secret in Secret Manager - -```bash -# Generate a secure random secret (if you don't have one) -WEBHOOK_SECRET=$(openssl rand -hex 32) -echo "Generated webhook secret: $WEBHOOK_SECRET" - -# Store in Secret Manager -echo -n "$WEBHOOK_SECRET" | gcloud secrets create webhook-secret \ - --data-file=- \ - --replication-policy="automatic" - -# Verify it was created -gcloud secrets describe webhook-secret -``` - -### 2. Grant App Engine Access - -```bash -# Get your project ID -PROJECT_ID=$(gcloud config get-value project) - -# Grant App Engine service account access to the secret -gcloud secrets add-iam-policy-binding webhook-secret \ - --member="serviceAccount:${PROJECT_ID}@appspot.gserviceaccount.com" \ - --role="roles/secretmanager.secretAccessor" - -# Verify permissions -gcloud secrets get-iam-policy webhook-secret -``` - -### 3. Configure env.yaml - -Use `WEBHOOK_SECRET_NAME` instead of `WEBHOOK_SECRET`: - -```yaml -env_variables: - # ... other config ... - - # Use Secret Manager (RECOMMENDED) - WEBHOOK_SECRET_NAME: "projects/1054147886816/secrets/webhook-secret/versions/latest" - - # DO NOT use direct secret in production - # WEBHOOK_SECRET: "hardcoded-secret-here" -``` - -### 4. Deploy - -```bash -./deploy.sh -``` - -## Configuration Options - -The application supports **two ways** to provide the webhook secret: - -### Option 1: Secret Manager (Recommended for Production) - -**Environment Variable:** `WEBHOOK_SECRET_NAME` - -**Format:** `projects/PROJECT_ID/secrets/SECRET_NAME/versions/VERSION` - -**Example:** -```yaml -WEBHOOK_SECRET_NAME: "projects/1054147886816/secrets/webhook-secret/versions/latest" -``` - -**Pros:** -- ✅ Secure - secret never in config files -- ✅ Auditable - all access logged -- ✅ Rotatable - update secret without redeploying -- ✅ Access controlled - IAM permissions - -**Cons:** -- ❌ Requires Secret Manager setup -- ❌ Requires IAM permissions - -### Option 2: Direct Environment Variable (For Testing Only) - -**Environment Variable:** `WEBHOOK_SECRET` - -**Format:** Plain text string - -**Example:** -```yaml -WEBHOOK_SECRET: "my-webhook-secret-123" -``` - -**Pros:** -- ✅ Simple - no Secret Manager needed -- ✅ Fast - no API calls - -**Cons:** -- ❌ Insecure - secret in config file -- ❌ No audit trail -- ❌ Hard to rotate -- ❌ Risk of committing to version control - -**⚠️ Use only for local development/testing!** - -## How It Works - -### Loading Priority - -1. **Check `WEBHOOK_SECRET`** - If set, use it directly (no Secret Manager call) -2. **Check `WEBHOOK_SECRET_NAME`** - If set, load from Secret Manager -3. **Use default** - `projects/1054147886816/secrets/webhook-secret/versions/latest` - -### Code Flow - -``` -app.go - ├─> configs.LoadEnvironment() - │ └─> Loads WEBHOOK_SECRET and WEBHOOK_SECRET_NAME from env - │ - └─> services.LoadWebhookSecret(config) - ├─> If config.WebhookSecret is set → use it - └─> Else → load from Secret Manager using config.WebhookSecretName - └─> Store in config.WebhookSecret -``` - -### Signature Verification - -``` -webhook_handler.go - └─> ParseWebhookDataWithConfig() - └─> verifySignatureFunc(sigHeader, payload, []byte(config.WebhookSecret)) - └─> HMAC-SHA256 verification -``` - -## Your Current Setup - -Based on your Secret Manager output: - -```yaml -name: projects/1054147886816/secrets/webhook-secret -createTime: '2025-10-06T17:56:10.467642Z' -replication: - automatic: {} -``` - -### Your env.yaml Should Use: - -```yaml -env_variables: - GITHUB_APP_ID: "1166559" - INSTALLATION_ID: "62138132" - REPO_NAME: "docs-code-examples" - REPO_OWNER: "mongodb" - - # GitHub App private key from Secret Manager - GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest" - - # Webhook secret from Secret Manager (RECOMMENDED) - WEBHOOK_SECRET_NAME: "projects/1054147886816/secrets/webhook-secret/versions/latest" - - # Other config... - COMMITTER_NAME: "GitHub Copier App" - COMMITTER_EMAIL: "bot@mongodb.com" - PORT: "8080" - WEBSERVER_PATH: "/events" - CONFIG_FILE: "copier-config.yaml" - DEPRECATION_FILE: "deprecated_examples.json" - GOOGLE_PROJECT_ID: "1054147886816" - GOOGLE_LOG_NAME: "code-copier-log" -``` - -## Testing Locally - -For local testing, you can use `SKIP_SECRET_MANAGER=true`: - -```bash -# In your .env.local file -SKIP_SECRET_MANAGER=true -WEBHOOK_SECRET="test-secret-123" -GITHUB_APP_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----" -``` - -Then run: -```bash -go run app.go -env .env.local -``` - -## Rotating the Webhook Secret - -### Step 1: Create New Secret Version - -```bash -# Generate new secret -NEW_SECRET=$(openssl rand -hex 32) - -# Add new version to Secret Manager -echo -n "$NEW_SECRET" | gcloud secrets versions add webhook-secret \ - --data-file=- -``` - -### Step 2: Update GitHub Webhook - -1. Go to your source repository on GitHub -2. Settings → Webhooks -3. Edit the webhook -4. Update the "Secret" field with the new secret -5. Save - -### Step 3: Verify - -The application will automatically use the latest version (no redeployment needed if using `versions/latest`). - -```bash -# Test webhook delivery -# GitHub → Settings → Webhooks → Recent Deliveries → Redeliver -``` - -### Step 4: Disable Old Version (Optional) - -```bash -# List versions -gcloud secrets versions list webhook-secret - -# Disable old version -gcloud secrets versions disable VERSION_NUMBER --secret=webhook-secret -``` - -## Troubleshooting - -### Error: "failed to load webhook secret" - -**Cause:** Secret Manager client can't access the secret - -**Solutions:** -1. Verify secret exists: - ```bash - gcloud secrets describe webhook-secret - ``` - -2. Check IAM permissions: - ```bash - gcloud secrets get-iam-policy webhook-secret - ``` - -3. Grant access: - ```bash - PROJECT_ID=$(gcloud config get-value project) - gcloud secrets add-iam-policy-binding webhook-secret \ - --member="serviceAccount:${PROJECT_ID}@appspot.gserviceaccount.com" \ - --role="roles/secretmanager.secretAccessor" - ``` - -### Error: "webhook signature verification failed" - -**Cause:** Secret in Secret Manager doesn't match GitHub webhook secret - -**Solutions:** -1. Get secret from Secret Manager: - ```bash - gcloud secrets versions access latest --secret=webhook-secret - ``` - -2. Compare with GitHub webhook secret: - - GitHub → Settings → Webhooks → Edit - - Check the "Secret" field - -3. Update one to match the other - -### Error: "SKIP_SECRET_MANAGER=true but no WEBHOOK_SECRET set" - -**Cause:** Testing locally without providing direct secret - -**Solution:** -```bash -export WEBHOOK_SECRET="test-secret-123" -``` - -## Security Best Practices - -### ✅ DO - -- ✅ Use Secret Manager in production -- ✅ Use `versions/latest` for automatic rotation -- ✅ Grant minimal IAM permissions -- ✅ Rotate secrets regularly (every 90 days) -- ✅ Use different secrets for different environments -- ✅ Monitor Secret Manager audit logs - -### ❌ DON'T - -- ❌ Hardcode secrets in env.yaml for production -- ❌ Commit env.yaml to version control -- ❌ Share secrets via email or chat -- ❌ Use the same secret across multiple apps -- ❌ Grant broad IAM permissions -- ❌ Use weak secrets (use `openssl rand -hex 32`) - -## Comparison with GitHub Private Key - -Both secrets are now loaded from Secret Manager: - -| Secret | Environment Variable | Secret Manager Path | -|--------|---------------------|---------------------| -| GitHub Private Key | `GITHUB_APP_PRIVATE_KEY_SECRET_NAME` | `projects/.../secrets/CODE_COPIER_PEM/versions/latest` | -| Webhook Secret | `WEBHOOK_SECRET_NAME` | `projects/.../secrets/webhook-secret/versions/latest` | - -**Consistency:** Both use the same pattern for security and maintainability. - -## Summary - -**Before (Insecure):** -```yaml -WEBHOOK_SECRET: "hardcoded-secret-in-config-file" -``` - -**After (Secure):** -```yaml -WEBHOOK_SECRET_NAME: "projects/1054147886816/secrets/webhook-secret/versions/latest" -``` - -**Result:** -- ✅ Secret stored securely in Secret Manager -- ✅ No secrets in config files -- ✅ Easy rotation without redeployment -- ✅ Audit trail of all access -- ✅ Fine-grained access control - ---- - -**Ready to deploy?** Your webhook secret is already in Secret Manager, so just update your env.yaml to use `WEBHOOK_SECRET_NAME` instead of `WEBHOOK_SECRET`! 🔒 - diff --git a/examples-copier/app.go b/examples-copier/app.go index e063d22..ac99a28 100644 --- a/examples-copier/app.go +++ b/examples-copier/app.go @@ -39,6 +39,17 @@ func main() { os.Exit(1) } + // Load secrets from Secret Manager if not directly provided + if err := services.LoadWebhookSecret(config); err != nil { + fmt.Printf("❌ Error loading webhook secret: %v\n", err) + os.Exit(1) + } + + if err := services.LoadMongoURI(config); err != nil { + fmt.Printf("❌ Error loading MongoDB URI: %v\n", err) + os.Exit(1) + } + // Override dry-run from command line if dryRun { config.DryRun = true @@ -66,12 +77,6 @@ func main() { services.InitializeGoogleLogger() defer services.CloseGoogleLogger() - // Load webhook secret from Secret Manager if not directly provided - if err := services.LoadWebhookSecret(config); err != nil { - fmt.Printf("Error loading webhook secret: %v\n", err) - return - } - // Configure GitHub permissions services.ConfigurePermissions() diff --git a/examples-copier/app.yaml b/examples-copier/app.yaml index 9e4885f..4b2ee75 100644 --- a/examples-copier/app.yaml +++ b/examples-copier/app.yaml @@ -3,3 +3,6 @@ runtime_config: operating_system: "ubuntu22" runtime_version: "1.23" env: flex + +includes: + - env.yaml diff --git a/examples-copier/cmd/test-webhook/main.go b/examples-copier/cmd/test-webhook/main.go index c3754be..e3bbeb2 100644 --- a/examples-copier/cmd/test-webhook/main.go +++ b/examples-copier/cmd/test-webhook/main.go @@ -20,7 +20,7 @@ func main() { prNumber := flag.Int("pr", 0, "PR number to fetch from GitHub") owner := flag.String("owner", "", "Repository owner") repo := flag.String("repo", "", "Repository name") - webhookURL := flag.String("url", "http://localhost:8080/webhook", "Webhook URL") + webhookURL := flag.String("url", "http://localhost:8080/events", "Webhook URL") secret := flag.String("secret", "", "Webhook secret for signature") payloadFile := flag.String("payload", "", "Path to custom payload JSON file") dryRun := flag.Bool("dry-run", false, "Print payload without sending") @@ -95,7 +95,7 @@ Options: -pr int PR number to fetch from GitHub -owner string Repository owner (required with -pr) -repo string Repository name (required with -pr) - -url string Webhook URL (default: http://localhost:8080/webhook) + -url string Webhook URL (default: http://localhost:8080/events) -secret string Webhook secret for HMAC signature -payload string Path to custom payload JSON file -dry-run Print payload without sending @@ -117,7 +117,7 @@ Examples: # Send to production with secret test-webhook -pr 123 -owner myorg -repo myrepo \ - -url https://myapp.appspot.com/webhook \ + -url https://myapp.appspot.com/events \ -secret "my-webhook-secret" Environment Variables: diff --git a/examples-copier/config.json b/examples-copier/config.json deleted file mode 100644 index f66e3a5..0000000 --- a/examples-copier/config.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "source_directory": "examples", - "target_repo": "mongodb/target-repo", - "target_branch": "main", - "target_directory": "docs/code-examples", - "recursive_copy": true, - "copier_commit_strategy": "pr", - "pr_title": "Update code examples", - "commit_message": "Sync examples from source repository", - "merge_without_review": false - } -] - diff --git a/examples-copier/configs/.env.example b/examples-copier/configs/.env.example deleted file mode 100644 index 19b62cb..0000000 --- a/examples-copier/configs/.env.example +++ /dev/null @@ -1,54 +0,0 @@ -# Sample .env for the Docs Examples Copier -# Required identifiers for the GitHub App installation -GITHUB_APP_ID="" -INSTALLATION_ID="" -# Optional (not used for JWT auth): OAuth client ID of your GitHub App -# GITHUB_APP_CLIENT_ID="" - -# Source repository (where config.json and deprecated_examples.json live) -REPO_OWNER="mongodb" -REPO_NAME="docs-code-examples" -SRC_BRANCH="main" - -# Author information for commits to the source repo -COMMITER_NAME="Copier Bot" -COMMITER_EMAIL="bot@example.com" - -# Web server configuration -PORT="8080" # omit or empty to use default 8080 -WEBSERVER_PATH="/webhook" # webhook route path - -# File names/paths in the source repo -CONFIG_FILE="config.json" -DEPRECATION_FILE="deprecated_examples.json" - -# Logging / GCP (optional) -GOOGLE_CLOUD_PROJECT_ID="github-copy-code-examples" -COPIER_LOG_NAME="copy-copier-log" -# Disable GCP logging in local/dev if desired (string boolean) -# COPIER_DISABLE_CLOUD_LOGGING="true" -# Enable debug logging either by LOG_LEVEL=debug or COPIER_DEBUG=true -# LOG_LEVEL="debug" -# COPIER_DEBUG="true" - -# Authentication: Secret Manager vs local -# If running locally/tests, you can skip GCP Secret Manager and provide the key via env -# SKIP_SECRET_MANAGER="true" -# GITHUB_APP_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----\n" -# or base64-encoded: -# GITHUB_APP_PRIVATE_KEY_B64="" -# Otherwise, when using GCP Secret Manager, set the fully-qualified secret name: -# PEM_NAME="projects/1234567890/secrets/CODE_COPIER_PEM/versions/latest" - -# BEHAVIOR DEFAULTS -# Global default commit message if not set at config level -DEFAULT_COMMIT_MESSAGE="Automatic copy of updated code examples" -# Default copy behavior if not set at config level (string booleans: "true" or "false") -# If "true", all files in source repo are recursively copied from path, otherwise only the top-level examples. -DEFAULT_RECURSIVE_COPY="true" -# Default write strategy if not set at config level: "direct" (commit directly) or "pr" (PR created) -# If "pr", the DEFAULT_PR_MERGE setting applies. -COPIER_COMMIT_STRATEGY="pr" -# When COPIER_COMMIT_STRATEGY is set to "pr", default PR merge behavior if not set at config level (string booleans: "true" or "false") -# If "true", PRs merge automatically if no conflicts, otherwise PR created but not merged. -DEFAULT_PR_MERGE="false" \ No newline at end of file diff --git a/examples-copier/configs/.env.example.new b/examples-copier/configs/.env.example.new deleted file mode 100644 index 5f6fa02..0000000 --- a/examples-copier/configs/.env.example.new +++ /dev/null @@ -1,97 +0,0 @@ -# ============================================================================= -# REQUIRED CONFIGURATION -# ============================================================================= - -# GitHub App Configuration (Required) -GITHUB_APP_CLIENT_ID="" -GITHUB_APP_ID="" -INSTALLATION_ID="" - -# Source Repository Configuration (Required) -REPO_NAME="" -REPO_OWNER="" - -# Security Configuration (Required) -PEM_NAME="projects//secrets//versions/latest" -WEBHOOK_SECRET="" - -# ============================================================================= -# OPTIONAL CONFIGURATION -# ============================================================================= - -# Committer Information (Optional - defaults provided) -COMMITER_EMAIL="copier@example.com" -COMMITER_NAME="GitHub Copier App" - -# Web Server Configuration (Optional - defaults provided) -PORT="3000" -WEBSERVER_PATH="/events" - -# File Configuration (Optional - defaults provided) -# Supports both JSON and YAML formats -CONFIG_FILE="copier-config.yaml" -DEPRECATION_FILE="deprecated_examples.json" - -# Source Branch (Optional - default: main) -SRC_BRANCH="main" - -# Google Cloud Logging (Optional) -GOOGLE_CLOUD_PROJECT_ID="" -COPIER_LOG_NAME="copy-copier-log" - -# ============================================================================= -# NEW FEATURES CONFIGURATION -# ============================================================================= - -# Dry Run Mode (Optional - default: false) -# When enabled, no actual changes are made to target repositories -DRY_RUN="false" - -# Audit Logging (Optional - default: false) -# Enable MongoDB audit logging for tracking all copy operations -AUDIT_ENABLED="false" -MONGO_URI="mongodb://localhost:27017" -AUDIT_DATABASE="copier_audit" -AUDIT_COLLECTION="events" - -# Metrics Collection (Optional - default: true) -# Enable metrics collection for /metrics endpoint -METRICS_ENABLED="true" - -# ============================================================================= -# LEGACY/DEPRECATED CONFIGURATION -# These are maintained for backward compatibility with JSON configs -# ============================================================================= - -# Default Recursive Copy (Optional - default: true) -DEFAULT_RECURSIVE_COPY="true" - -# Default PR Merge (Optional - default: false) -DEFAULT_PR_MERGE="false" - -# Default Commit Message (Optional) -DEFAULT_COMMIT_MESSAGE="Automated PR with updated examples" - -# ============================================================================= -# EXAMPLE CONFIGURATIONS -# ============================================================================= - -# Example 1: Local Development with Dry Run -# DRY_RUN="true" -# AUDIT_ENABLED="false" -# METRICS_ENABLED="true" -# CONFIG_FILE="copier-config.example.yaml" - -# Example 2: Production with Audit Logging -# DRY_RUN="false" -# AUDIT_ENABLED="true" -# MONGO_URI="mongodb+srv://user:pass@cluster.mongodb.net" -# AUDIT_DATABASE="copier_audit" -# METRICS_ENABLED="true" - -# Example 3: Staging Environment -# CONFIG_FILE="copier-config.staging.yaml" -# DRY_RUN="false" -# AUDIT_ENABLED="true" -# METRICS_ENABLED="true" - diff --git a/examples-copier/configs/.env.local b/examples-copier/configs/.env.local.example similarity index 60% rename from examples-copier/configs/.env.local rename to examples-copier/configs/.env.local.example index 387e19c..4affbac 100644 --- a/examples-copier/configs/.env.local +++ b/examples-copier/configs/.env.local.example @@ -1,14 +1,25 @@ # Local Development Environment Configuration -# Copy this file to .env for local testing + +# To use this file, copy it to .env and edit with your values: +# cp configs/.env.local configs/.env +# source configs/.env +# ./examples-copier + +# Or use with make: +# make run-dry + +# Or run directly: +# COPIER_DISABLE_CLOUD_LOGGING=true DRY_RUN=true ./examples-copier # ============================================================================ # REQUIRED FOR LOCAL TESTING # ============================================================================ -# Repository Configuration -REPO_OWNER=mongodb -REPO_NAME=docs-realm -SRC_BRANCH=main +# Source Repository Configuration (Required) +REPO_NAME="" +REPO_OWNER="" +# Source Branch (Optional - default: main) +SRC_BRANCH="test" # Configuration Files CONFIG_FILE=copier-config.yaml @@ -104,19 +115,67 @@ SLACK_ICON_EMOJI=:robot_face: # Enable/disable Slack notifications (default: true if webhook URL is set) SLACK_ENABLED=false -# ============================================================================ -# NOTES -# ============================================================================ - -# To use this file: -# cp configs/.env.local configs/.env -# # Edit .env with your values -# source configs/.env -# ./examples-copier - -# Or use with make: -# make run-dry - -# Or run directly: -# COPIER_DISABLE_CLOUD_LOGGING=true DRY_RUN=true ./examples-copier +# ============================================================================= +# EXAMPLE CONFIGURATIONS +# ============================================================================= + +# Example 1: Local Development with Dry Run +# DRY_RUN="true" +# AUDIT_ENABLED="false" +# METRICS_ENABLED="true" +# CONFIG_FILE="copier-config.example.yaml" +# WEBHOOK_SECRET="test-secret-123" + +# Example 2: Local Development with MongoDB Audit Logging +# DRY_RUN="false" +# AUDIT_ENABLED="true" +# MONGO_URI="mongodb://localhost:27017" +# AUDIT_DATABASE="copier_audit" +# METRICS_ENABLED="true" + +# Example 3: Local Development with Slack Notifications +# DRY_RUN="true" +# SLACK_WEBHOOK_URL="https://hooks.slack.com/services/YOUR/WEBHOOK/URL" +# SLACK_CHANNEL="#code-examples-dev" +# SLACK_ENABLED="true" + +# ============================================================================= +# NOTES FOR LOCAL DEVELOPMENT +# ============================================================================= + +# 1. DIRECT SECRETS (Recommended for local dev): +# - Use WEBHOOK_SECRET instead of WEBHOOK_SECRET_NAME +# - Use MONGO_URI instead of MONGO_URI_SECRET_NAME +# - Use PEM_NAME pointing to Secret Manager (still needed for GitHub auth) +# +# 2. DRY RUN MODE: +# - Set DRY_RUN="true" to test without making actual commits/PRs +# - Webhooks are processed, files matched, but no changes made +# +# 3. MONGODB: +# - For local testing, use mongodb://localhost:27017 +# - Or disable audit logging: AUDIT_ENABLED="false" +# +# 4. SLACK: +# - Optional for local dev +# - Useful for testing error notifications +# +# 5. GOOGLE CLOUD: +# - You still need PEM_NAME for GitHub App authentication +# - Set up Secret Manager even for local dev +# - Or use a local PEM file (not recommended) +# +# 6. TESTING: +# - Use CONFIG_FILE="copier-config.example.yaml" for testing +# - Set WEBSERVER_PATH="/events" to match GitHub webhook +# - Use PORT="3000" or any available port + +# ============================================================================= +# SEE ALSO +# ============================================================================= + +# - env.yaml.example - Complete reference for all variables +# - env.yaml.production - Production deployment template +# - ../configs/README.md - Comparison of all env files +# - ../docs/LOCAL-TESTING.md - Local development guide diff --git a/examples-copier/configs/.env.test b/examples-copier/configs/.env.test deleted file mode 100644 index dcedb1b..0000000 --- a/examples-copier/configs/.env.test +++ /dev/null @@ -1,26 +0,0 @@ -GITHUB_APP_CLIENT_ID="Iv23licLjdtAHKK4QBuc" -INSTALLATION_ID="62138132" -GITHUB_APP_ID="1166559" - -REPO_OWNER="mongodb" -REPO_NAME="docs-code-examples" -SRC_BRANCH="main" - -COMMITER_NAME="GitHub Copier App" -COMMITER_EMAIL="bot@mongodb.com" - -PORT:"3000" -WEBSERVER_PATH="/" - -CONFIG_FILE="config.json" -DEPRECATION_FILE="deprecated_examples.json" - -COPIER_DISABLE_CLOUD_LOGGING="true" -LOG_LEVEL="debug" - -SKIP_SECRET_MANAGER="true" -GITHUB_APP_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQ...\n-----END PRIVATE KEY-----\n" - -DEFAULT_COMMIT_MESSAGE="Test commit message for code copier" -DEFAULT_RECURSIVE_COPY="false" -COPIER_COMMIT_STRATEGY="direct" \ No newline at end of file diff --git a/examples-copier/configs/README.md b/examples-copier/configs/README.md new file mode 100644 index 0000000..d6e6562 --- /dev/null +++ b/examples-copier/configs/README.md @@ -0,0 +1,272 @@ +# Environment Files Comparison + +Overview of the different environment configuration files and when to use each. + +## Files Overview + +| File | Purpose | Use Case | +|-----------------------|---------------------------------------|---------------------------------| +| `env.yaml.example` | Complete reference with all variables | First-time setup, documentation | +| `env.yaml.production` | Production-ready template | Quick deployment to production | +| `.env.example` | Local development template | Local testing and development | + +--- + +## env.yaml.example + +**Location:** `configs/env.yaml.example` + +**Purpose:** Comprehensive reference showing ALL possible environment variables + +**Contents:** +- ✅ All 30+ supported variables +- ✅ Detailed comments for each variable +- ✅ Default values shown +- ✅ Security best practices +- ✅ Deployment notes +- ✅ Local development tips + +**Use this when:** +- Setting up for the first time +- Need to understand all available options +- Want to see what features are available +- Need reference documentation + +--- + +## env.yaml.production + +**Location:** `configs/env.yaml.production` + +**Purpose:** Production-ready template with sensible defaults + +**Contents:** +- ✅ Required variables pre-filled with MongoDB values +- ✅ Secret Manager references (recommended approach) +- ✅ Essential settings only +- ✅ Audit logging enabled +- ✅ Metrics enabled +- ✅ Production-optimized + +**Use this when:** +- Deploying to production quickly +- Want a minimal, clean configuration +- Using Secret Manager (recommended) +- Don't need advanced features + +**NOT included:** +- Slack notifications (optional) +- MongoDB direct URI (use Secret Manager instead) +- Advanced options (rarely needed) + +--- + +## .env.example.new + +**Location:** `configs/.env.example.new` + +**Purpose:** Local development template (traditional .env format) + +**Contents:** +- ✅ .env format (KEY=value) +- ✅ Suitable for local testing +- ✅ Works with godotenv +- ✅ Can use direct secrets (not Secret Manager) + +**Use this when:** +- Developing locally +- Testing without Google Cloud +- Using `go run` or local server +- Don't want to set up Secret Manager + +**Format difference:** +```bash +# .env format (for local development) +GITHUB_APP_ID=123456 +REPO_OWNER=mongodb +REPO_NAME=docs-code-examples +``` + +vs + +```yaml +# env.yaml format (for App Engine deployment) +env_variables: + GITHUB_APP_ID: "123456" + REPO_OWNER: "mongodb" + REPO_NAME: "docs-code-examples" +``` + +--- + +## Usage Scenarios + +### Scenario 1: First-Time Production Deployment + +**Recommended:** `env.yaml.production` + +```bash +# Quick start +cp configs/env.yaml.production env.yaml +nano env.yaml # Update PROJECT_NUMBER and values +./scripts/grant-secret-access.sh +gcloud app deploy app.yaml # env.yaml is included via 'includes' directive +``` + +**Why:** Pre-configured with production best practices, minimal setup required. + +--- + +### Scenario 2: Need to Understand All Options + +**Recommended:** `env.yaml.example` + +```bash +# Reference all options +cat configs/env.yaml.example + +# Copy and customize +cp configs/env.yaml.example env.yaml +nano env.yaml # Enable features you need +``` + +**Why:** Shows all available features with detailed explanations. + +--- + +### Scenario 3: Local Development + +**Recommended:** `.env.example.new` + +```bash +# Local development +cp configs/.env.example.new configs/.env +nano configs/.env # Add your values + +# Run locally +go run app.go -env configs/.env +``` + +**Why:** Simpler format for local testing, no Secret Manager required. + +--- + +### Scenario 4: Custom Production Setup + +**Recommended:** Start with `env.yaml.example`, customize + +```bash +# Start with full reference +cp configs/env.yaml.example env.yaml + +# Enable features you need +nano env.yaml +# - Enable Slack notifications +# - Configure custom MongoDB settings +# - Set custom defaults + +# Deploy +gcloud app deploy app.yaml # env.yaml is included via 'includes' directive +``` + +**Why:** Need advanced features not in production template. + +--- + +## Migration Guide + +### From .env to env.yaml + +Use the conversion script: + +```bash +./scripts/convert-env-to-yaml.sh configs/.env env.yaml +``` + +Or manually convert: + +```bash +# .env format: +GITHUB_APP_ID=123456 +REPO_OWNER=mongodb + +# env.yaml format: +env_variables: + GITHUB_APP_ID: "123456" + REPO_OWNER: "mongodb" +``` + +### From env.yaml.production to env.yaml.example + +```bash +# Start with production template +cp configs/env.yaml.production env.yaml + +# Add optional features from example +# Compare files and add what you need: +diff configs/env.yaml.production configs/env.yaml.example +``` + +--- + +## Best Practices + +### ✅ DO + +- **Use `env.yaml.production` for quick production deployment** +- **Use `env.yaml.example` as reference documentation** +- **Use `.env.example.new` for local development** +- **Add `env.yaml` and `.env` to `.gitignore`** +- **Use Secret Manager for production secrets** +- **Keep comments in your env.yaml for team documentation** + +### ❌ DON'T + +- **Don't commit `env.yaml` or `.env` with actual secrets** +- **Don't use direct secrets in production (use Secret Manager)** +- **Don't mix .env and env.yaml formats** +- **Don't remove all comments (they help future you)** +- **Don't use `env.yaml.production` as-is (update values first)** + +--- + +## File Locations + +``` +examples-copier/ +├── configs/ +│ ├── env.yaml.example # ← Complete reference (all variables) +│ ├── env.yaml.production # ← Production template (essential only) +│ └── .env.example # ← Local development template +├── env.yaml # ← Your actual config (gitignored) +└── .env # ← Your local config (gitignored) +``` + +--- + +## Quick Reference + +**Need to deploy quickly?** +→ Use `env.yaml.production` + +**Need to understand all options?** +→ Read `env.yaml.example` + +**Need to develop locally?** +→ Use `.env.example.new` + +**Need advanced features?** +→ Start with `env.yaml.example`, customize + +**Need to convert formats?** +→ Use `./scripts/convert-env-to-yaml.sh` + +--- + +## See Also + +- [CONFIGURATION-GUIDE.md](../docs/CONFIGURATION-GUIDE.md) - Variable validation and reference +- [DEPLOYMENT.md](../docs/DEPLOYMENT.md) - Complete deployment guide +- [DEPLOYMENT-CHECKLIST.md](../docs/DEPLOYMENT-CHECKLIST.md) - Step-by-step checklist +- [LOCAL-TESTING.md](../docs/LOCAL-TESTING.md) - Local development guide + diff --git a/examples-copier/configs/env.yaml.example b/examples-copier/configs/env.yaml.example new file mode 100644 index 0000000..59b7584 --- /dev/null +++ b/examples-copier/configs/env.yaml.example @@ -0,0 +1,181 @@ +env_variables: + # ============================================================================= + # REQUIRED VARIABLES + # ============================================================================= + + # GitHub App Configuration + GITHUB_APP_ID: "YOUR_GITHUB_APP_ID" # Your GitHub App ID (required) + INSTALLATION_ID: "YOUR_INSTALLATION_ID" # GitHub App Installation ID (required) + REPO_OWNER: "your-org" # Source repository owner (required) + REPO_NAME: "your-repo" # Source repository name (required) + + # ============================================================================= + # SECRET MANAGER REFERENCES (RECOMMENDED - Most Secure) + # ============================================================================= + + # GitHub App Private Key - loaded from Secret Manager + # Format: projects/PROJECT_NUMBER/secrets/SECRET_NAME/versions/VERSION + GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/YOUR_PROJECT_NUMBER/secrets/CODE_COPIER_PEM/versions/latest" + + # Webhook Secret - loaded from Secret Manager (for signature verification) + WEBHOOK_SECRET_NAME: "projects/YOUR_PROJECT_NUMBER/secrets/webhook-secret/versions/latest" + + # MongoDB URI - loaded from Secret Manager (for audit logging - OPTIONAL) + MONGO_URI_SECRET_NAME: "projects/YOUR_PROJECT_NUMBER/secrets/mongo-uri/versions/latest" + + # ============================================================================= + # ALTERNATIVE: DIRECT SECRETS (NOT RECOMMENDED - Less Secure) + # ============================================================================= + # Only use these if you're NOT using Secret Manager + # WARNING: Never commit actual secrets to version control! + + # PEM_NAME: "projects/YOUR_PROJECT_NUMBER/secrets/CODE_COPIER_PEM/versions/latest" + # WEBHOOK_SECRET: "your-webhook-secret-here" + # MONGO_URI: "mongodb+srv://user:pass@cluster.mongodb.net/dbname" + + # ============================================================================= + # APPLICATION SETTINGS + # ============================================================================= + + # Web Server Configuration + # PORT: "8080" # DO NOT SET - App Engine Flexible sets this automatically + WEBSERVER_PATH: "/events" # Webhook endpoint path (default: /webhook) + + # Configuration Files + CONFIG_FILE: "copier-config.yaml" # Config file name in source repo (default: copier-config.yaml) + DEPRECATION_FILE: "deprecated_examples.json" # Deprecation tracking file (default: deprecated_examples.json) + + # Source Branch + SRC_BRANCH: "main" # Branch to copy from (default: main) + + # ============================================================================= + # COMMITTER INFORMATION + # ============================================================================= + # Used when committing deprecation file updates to source repo + + COMMITTER_NAME: "GitHub Copier App" # Git committer name (default: Copier Bot) + COMMITTER_EMAIL: "bot@example.com" # Git committer email (default: bot@example.com) + + # ============================================================================= + # GOOGLE CLOUD CONFIGURATION + # ============================================================================= + # For Google Cloud Logging integration + + GOOGLE_CLOUD_PROJECT_ID: "your-project-id" # GCP Project ID (default: github-copy-code-examples) + COPIER_LOG_NAME: "code-copier-log" # Cloud Logging log name (default: copy-copier-log) + + # ============================================================================= + # LOGGING CONFIGURATION + # ============================================================================= + # Control logging verbosity and output + + # LOG_LEVEL: "debug" # Set to "debug" for verbose logging (default: info) + # COPIER_DEBUG: "true" # Alternative way to enable debug mode (default: false) + # COPIER_DISABLE_CLOUD_LOGGING: "true" # Disable Google Cloud Logging (useful for local dev) + + # ============================================================================= + # FEATURE FLAGS + # ============================================================================= + + # Dry Run Mode - process webhooks but don't make actual commits/PRs + DRY_RUN: "false" # Enable dry-run mode (default: false) + + # Audit Logging - log all operations to MongoDB + AUDIT_ENABLED: "true" # Enable audit logging (default: false) + + # Metrics - expose /metrics endpoint + METRICS_ENABLED: "true" # Enable metrics endpoint (default: true) + + # ============================================================================= + # MONGODB AUDIT LOGGING (OPTIONAL) + # ============================================================================= + # Only needed if AUDIT_ENABLED is true + + # MongoDB Configuration + # MONGO_URI: "mongodb+srv://user:pass@cluster.mongodb.net/dbname" # Use MONGO_URI_SECRET_NAME instead + AUDIT_DATABASE: "copier_audit" # MongoDB database name (default: copier_audit) + AUDIT_COLLECTION: "events" # MongoDB collection name (default: events) + + # ============================================================================= + # SLACK NOTIFICATIONS (OPTIONAL) + # ============================================================================= + # Enable Slack notifications for errors and important events + + # SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/YOUR/WEBHOOK/URL" # Slack webhook URL (enables Slack) + # SLACK_CHANNEL: "#code-examples" # Slack channel (default: #code-examples) + # SLACK_USERNAME: "Examples Copier" # Slack bot username (default: Examples Copier) + # SLACK_ICON_EMOJI: ":robot_face:" # Slack bot icon (default: :robot_face:) + # SLACK_ENABLED: "true" # Explicitly enable/disable Slack (default: auto-enabled if URL set) + + # ============================================================================= + # DEFAULT BEHAVIORS (OPTIONAL) + # ============================================================================= + # System-wide defaults that individual config rules can override + + DEFAULT_RECURSIVE_COPY: "true" # Default recursive copy behavior (default: true) + DEFAULT_PR_MERGE: "false" # Default auto-merge PRs without review (default: false) + DEFAULT_COMMIT_MESSAGE: "Automated PR with updated examples" # Default commit message (default: shown) + + # ============================================================================= + # TESTING / DEVELOPMENT OVERRIDES (DO NOT USE IN PRODUCTION) + # ============================================================================= + + # Skip Secret Manager - for local testing/CI only + # When enabled, uses direct environment variables instead of Secret Manager + # WARNING: Never enable in production! + # SKIP_SECRET_MANAGER: "true" + + # Direct GitHub App Private Key (only when SKIP_SECRET_MANAGER=true) + # Use one of these formats: + # GITHUB_APP_PRIVATE_KEY: "-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" + # GITHUB_APP_PRIVATE_KEY_B64: "base64-encoded-private-key" + + # Environment file path (alternative to --env-vars-file flag) + # ENV_FILE: "./configs/.env.test" + +# ============================================================================= +# DEPLOYMENT NOTES +# ============================================================================= +# +# 1. SECURITY BEST PRACTICES: +# - Use Secret Manager for all sensitive data (recommended) +# - Never commit env.yaml with actual secrets to version control +# - Add env.yaml to .gitignore +# +# 2. SECRET MANAGER SETUP: +# - Create secrets: gcloud secrets create SECRET_NAME --data-file=FILE +# - Grant access: ./scripts/grant-secret-access.sh +# - Use format: projects/PROJECT_NUMBER/secrets/SECRET_NAME/versions/latest +# +# 3. REQUIRED VARIABLES: +# - GITHUB_APP_ID +# - INSTALLATION_ID +# - REPO_OWNER +# - REPO_NAME +# - GITHUB_APP_PRIVATE_KEY_SECRET_NAME (or PEM_NAME) +# +# 4. RECOMMENDED SETUP: +# - Use Secret Manager for: GitHub private key, webhook secret, MongoDB URI +# - Enable audit logging for production +# - Enable metrics for monitoring +# - Configure Slack for error notifications +# +# 5. LOCAL DEVELOPMENT: +# - Set DRY_RUN: "true" to test without making actual changes +# - Disable audit logging if you don't have MongoDB +# - Use direct secrets (not Secret Manager) for easier local testing +# +# 6. DEPLOYMENT: +# - Copy this file: cp configs/env.yaml.example env.yaml +# - Fill in your values +# - Deploy: gcloud app deploy app.yaml --env-vars-file=env.yaml +# +# ============================================================================= +# VARIABLE REFERENCE +# ============================================================================= +# +# See CONFIG-VALIDATION-REPORT.md for complete variable documentation +# See docs/DEPLOYMENT.md for deployment guide +# See docs/DEPLOYMENT-CHECKLIST.md for step-by-step checklist +# + diff --git a/examples-copier/configs/env.yaml.production b/examples-copier/configs/env.yaml.production new file mode 100644 index 0000000..9a75ff6 --- /dev/null +++ b/examples-copier/configs/env.yaml.production @@ -0,0 +1,61 @@ +env_variables: + # ============================================================================= + # GitHub Configuration (Non-sensitive) + # ============================================================================= + GITHUB_APP_ID: "1166559" + INSTALLATION_ID: "62138132" + REPO_OWNER: "mongodb" + REPO_NAME: "docs-mongodb-internal" + SRC_BRANCH: "main" + + # ============================================================================= + # Secret Manager References (Sensitive Data - SECURE!) + # ============================================================================= + # GitHub App private key - loaded from Secret Manager + GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest" + + # Webhook secret - loaded from Secret Manager + WEBHOOK_SECRET_NAME: "projects/1054147886816/secrets/webhook-secret/versions/latest" + + # MongoDB URI - loaded from Secret Manager (for audit logging - OPTIONAL) + MONGO_URI_SECRET_NAME: "projects/1054147886816/secrets/mongo-uri/versions/latest" + + # ============================================================================= + # Application Settings (Non-sensitive) + # ============================================================================= + # PORT is automatically set by App Engine Flexible (do not override) + WEBSERVER_PATH: "/events" + CONFIG_FILE: "copier-config.yaml" + DEPRECATION_FILE: "deprecated_examples.json" + + # ============================================================================= + # Committer Information (Non-sensitive) + # ============================================================================= + COMMITTER_NAME: "GitHub Copier App" + COMMITTER_EMAIL: "bot@mongodb.com" + + # ============================================================================= + # Google Cloud Configuration (Non-sensitive) + # ============================================================================= + GOOGLE_CLOUD_PROJECT_ID: "github-copy-code-examples" + COPIER_LOG_NAME: "code-copier-log" + + # Logging Configuration (Optional - uncomment for debugging) + # LOG_LEVEL: "debug" # Enable verbose debug logging + # COPIER_DEBUG: "true" # Alternative debug flag + # COPIER_DISABLE_CLOUD_LOGGING: "true" # Disable GCP logging + + # ============================================================================= + # Feature Flags (Optional) + # ============================================================================= + AUDIT_ENABLED: "true" + METRICS_ENABLED: "true" + # DRY_RUN: "false" + + # ============================================================================= + # Default Behaviors (Optional) + # ============================================================================= + # DEFAULT_RECURSIVE_COPY: "true" + # DEFAULT_PR_MERGE: "false" + # DEFAULT_COMMIT_MESSAGE: "Automated PR with updated examples" + diff --git a/examples-copier/configs/environment.go b/examples-copier/configs/environment.go index b8d7eeb..0693777 100644 --- a/examples-copier/configs/environment.go +++ b/examples-copier/configs/environment.go @@ -17,8 +17,8 @@ type Config struct { AppId string AppClientId string InstallationId string - CommiterName string - CommiterEmail string + CommitterName string + CommitterEmail string ConfigFile string DeprecationFile string WebserverPath string @@ -33,13 +33,13 @@ type Config struct { DefaultCommitMessage string // New features - DryRun bool - AuditEnabled bool - MongoURI string - AuditDatabase string - AuditCollection string - MetricsEnabled bool - WebhookSecret string + DryRun bool + AuditEnabled bool + MongoURI string + MongoURISecretName string + AuditDatabase string + AuditCollection string + MetricsEnabled bool // Slack notifications SlackWebhookURL string @@ -57,8 +57,8 @@ const ( AppId = "GITHUB_APP_ID" AppClientId = "GITHUB_APP_CLIENT_ID" InstallationId = "INSTALLATION_ID" - CommiterName = "COMMITER_NAME" - CommiterEmail = "COMMITER_EMAIL" + CommitterName = "COMMITTER_NAME" + CommitterEmail = "COMMITTER_EMAIL" ConfigFile = "CONFIG_FILE" DeprecationFile = "DEPRECATION_FILE" WebserverPath = "WEBSERVER_PATH" @@ -74,10 +74,10 @@ const ( DryRun = "DRY_RUN" AuditEnabled = "AUDIT_ENABLED" MongoURI = "MONGO_URI" + MongoURISecretName = "MONGO_URI_SECRET_NAME" AuditDatabase = "AUDIT_DATABASE" AuditCollection = "AUDIT_COLLECTION" MetricsEnabled = "METRICS_ENABLED" - WebhookSecret = "WEBHOOK_SECRET" SlackWebhookURL = "SLACK_WEBHOOK_URL" SlackChannel = "SLACK_CHANNEL" SlackUsername = "SLACK_USERNAME" @@ -89,8 +89,8 @@ const ( func NewConfig() *Config { return &Config{ Port: "8080", - CommiterName: "Copier Bot", - CommiterEmail: "bot@example.com", + CommitterName: "Copier Bot", + CommitterEmail: "bot@example.com", ConfigFile: "copier-config.yaml", DeprecationFile: "deprecated_examples.json", WebserverPath: "/webhook", @@ -141,8 +141,8 @@ func LoadEnvironment(envFile string) (*Config, error) { config.AppId = os.Getenv(AppId) config.AppClientId = os.Getenv(AppClientId) config.InstallationId = os.Getenv(InstallationId) - config.CommiterName = getEnvWithDefault(CommiterName, config.CommiterName) - config.CommiterEmail = getEnvWithDefault(CommiterEmail, config.CommiterEmail) + config.CommitterName = getEnvWithDefault(CommitterName, config.CommitterName) + config.CommitterEmail = getEnvWithDefault(CommitterEmail, config.CommitterEmail) config.ConfigFile = getEnvWithDefault(ConfigFile, config.ConfigFile) config.DeprecationFile = getEnvWithDefault(DeprecationFile, config.DeprecationFile) config.WebserverPath = getEnvWithDefault(WebserverPath, config.WebserverPath) @@ -160,6 +160,7 @@ func LoadEnvironment(envFile string) (*Config, error) { config.DryRun = getBoolEnvWithDefault(DryRun, false) config.AuditEnabled = getBoolEnvWithDefault(AuditEnabled, false) config.MongoURI = os.Getenv(MongoURI) + config.MongoURISecretName = os.Getenv(MongoURISecretName) config.AuditDatabase = getEnvWithDefault(AuditDatabase, "copier_audit") config.AuditCollection = getEnvWithDefault(AuditCollection, "events") config.MetricsEnabled = getBoolEnvWithDefault(MetricsEnabled, true) @@ -179,8 +180,8 @@ func LoadEnvironment(envFile string) (*Config, error) { _ = os.Setenv(AppId, config.AppId) _ = os.Setenv(AppClientId, config.AppClientId) _ = os.Setenv(InstallationId, config.InstallationId) - _ = os.Setenv(CommiterName, config.CommiterName) - _ = os.Setenv(CommiterEmail, config.CommiterEmail) + _ = os.Setenv(CommitterName, config.CommitterName) + _ = os.Setenv(CommitterEmail, config.CommitterEmail) _ = os.Setenv(ConfigFile, config.ConfigFile) _ = os.Setenv(DeprecationFile, config.DeprecationFile) _ = os.Setenv(WebserverPath, config.WebserverPath) diff --git a/examples-copier/copier-config.yaml b/examples-copier/copier-config.yaml deleted file mode 100644 index 717a5af..0000000 --- a/examples-copier/copier-config.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Examples Copier Configuration -# This is a sample configuration - customize for your needs - -source_repo: "mongodb/docs-realm" -source_branch: "main" - -copy_rules: - # Rule 1: Copy generated examples (matches actual API file paths) - - name: "Copy generated examples" - source_pattern: - type: "regex" - pattern: "^generated-examples/(?P[^/]+)/(?P.+)$" - targets: - - repo: "mongodb/target-repo" - branch: "main" - path_transform: "examples/${project}/${rest}" - commit_strategy: - type: "pull_request" - commit_message: "Update ${project} examples" - pr_title: "Update ${project} examples" - pr_body: "Automated update from ${source_repo}" - auto_merge: false - deprecation_check: - enabled: true - file: "deprecated_examples.json" - - # Rule 2: Copy all generated-examples files (fallback) - - name: "Copy all generated examples" - source_pattern: - type: "prefix" - pattern: "generated-examples/" - targets: - - repo: "mongodb/target-repo" - branch: "main" - path_transform: "docs/${path}" - commit_strategy: - type: "direct" - commit_message: "Update examples from ${source_repo}" - diff --git a/examples-copier/deploy.sh b/examples-copier/deploy.sh deleted file mode 100755 index 12551e6..0000000 --- a/examples-copier/deploy.sh +++ /dev/null @@ -1,258 +0,0 @@ -#!/bin/bash - -# Deployment script for examples-copier to Google Cloud App Engine -# Usage: ./deploy.sh [options] -# -# Options: -# --project PROJECT_ID Set GCP project ID -# --version VERSION Set version name (default: auto-generated) -# --no-promote Deploy without promoting to receive traffic -# --quiet Skip confirmation prompts -# --env-file FILE Path to env.yaml file (default: env.yaml) -# --help Show this help message - -set -e # Exit on error - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Default values -PROJECT_ID="" -VERSION="" -PROMOTE="true" -QUIET="false" -ENV_FILE="env.yaml" - -# Parse command line arguments -while [[ $# -gt 0 ]]; do - case $1 in - --project) - PROJECT_ID="$2" - shift 2 - ;; - --version) - VERSION="$2" - shift 2 - ;; - --no-promote) - PROMOTE="false" - shift - ;; - --quiet) - QUIET="true" - shift - ;; - --env-file) - ENV_FILE="$2" - shift 2 - ;; - --help) - echo "Usage: ./deploy.sh [options]" - echo "" - echo "Options:" - echo " --project PROJECT_ID Set GCP project ID" - echo " --version VERSION Set version name (default: auto-generated)" - echo " --no-promote Deploy without promoting to receive traffic" - echo " --quiet Skip confirmation prompts" - echo " --env-file FILE Path to env.yaml file (default: env.yaml)" - echo " --help Show this help message" - exit 0 - ;; - *) - echo -e "${RED}Unknown option: $1${NC}" - echo "Use --help for usage information" - exit 1 - ;; - esac -done - -# Function to print colored messages -print_info() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -print_success() { - echo -e "${GREEN}[SUCCESS]${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}[WARNING]${NC} $1" -} - -print_error() { - echo -e "${RED}[ERROR]${NC} $1" -} - -# Function to check if command exists -command_exists() { - command -v "$1" >/dev/null 2>&1 -} - -# Check prerequisites -print_info "Checking prerequisites..." - -# Check if gcloud is installed -if ! command_exists gcloud; then - print_error "gcloud CLI is not installed" - echo "Install from: https://cloud.google.com/sdk/docs/install" - exit 1 -fi - -# Check if go is installed -if ! command_exists go; then - print_error "Go is not installed" - echo "Install from: https://golang.org/dl/" - exit 1 -fi - -print_success "Prerequisites check passed" - -# Get current project if not specified -if [ -z "$PROJECT_ID" ]; then - PROJECT_ID=$(gcloud config get-value project 2>/dev/null) - if [ -z "$PROJECT_ID" ]; then - print_error "No GCP project configured" - echo "Set project with: gcloud config set project PROJECT_ID" - echo "Or use: ./deploy.sh --project PROJECT_ID" - exit 1 - fi -fi - -print_info "Using GCP project: $PROJECT_ID" - -# Check if env.yaml exists -if [ ! -f "$ENV_FILE" ]; then - print_error "Environment file not found: $ENV_FILE" - echo "" - echo "Create $ENV_FILE with required environment variables:" - echo "" - cat << 'EOF' -env_variables: - GITHUB_APP_ID: "your-app-id" - INSTALLATION_ID: "your-installation-id" - REPO_NAME: "your-repo-name" - REPO_OWNER: "your-repo-owner" - GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/PROJECT_ID/secrets/SECRET_NAME/versions/latest" - WEBHOOK_SECRET: "your-webhook-secret" - COMMITTER_NAME: "GitHub Copier App" - COMMITTER_EMAIL: "bot@example.com" - CONFIG_FILE: "copier-config.yaml" - DEPRECATION_FILE: "deprecated_examples.json" - WEBSERVER_PATH: "/events" -EOF - echo "" - exit 1 -fi - -print_success "Environment file found: $ENV_FILE" - -# Check if app.yaml exists -if [ ! -f "app.yaml" ]; then - print_error "app.yaml not found in current directory" - echo "Run this script from the examples-copier directory" - exit 1 -fi - -# Build the application -print_info "Building application..." -if go build -o examples-copier .; then - print_success "Build successful" -else - print_error "Build failed" - exit 1 -fi - -# Run tests -print_info "Running tests..." -if go test ./... -v; then - print_success "All tests passed" -else - print_warning "Some tests failed - continuing anyway" -fi - -# Show deployment summary -echo "" -echo "=========================================" -echo "Deployment Summary" -echo "=========================================" -echo "Project: $PROJECT_ID" -echo "Version: ${VERSION:-auto-generated}" -echo "Promote: $PROMOTE" -echo "Env File: $ENV_FILE" -echo "=========================================" -echo "" - -# Confirm deployment -if [ "$QUIET" != "true" ]; then - read -p "Continue with deployment? (y/N) " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - print_info "Deployment cancelled" - exit 0 - fi -fi - -# Build gcloud command -DEPLOY_CMD="gcloud app deploy app.yaml --env-vars-file=$ENV_FILE --project=$PROJECT_ID" - -if [ -n "$VERSION" ]; then - DEPLOY_CMD="$DEPLOY_CMD --version=$VERSION" -fi - -if [ "$PROMOTE" != "true" ]; then - DEPLOY_CMD="$DEPLOY_CMD --no-promote" -fi - -if [ "$QUIET" = "true" ]; then - DEPLOY_CMD="$DEPLOY_CMD --quiet" -fi - -# Deploy to App Engine -print_info "Deploying to App Engine..." -echo "Command: $DEPLOY_CMD" -echo "" - -if eval "$DEPLOY_CMD"; then - print_success "Deployment successful!" - echo "" - - # Get app URL - APP_URL=$(gcloud app describe --project=$PROJECT_ID --format="value(defaultHostname)" 2>/dev/null) - if [ -n "$APP_URL" ]; then - print_info "Application URL: https://$APP_URL" - print_info "Webhook URL: https://$APP_URL/events" - fi - - echo "" - print_info "Next steps:" - echo " 1. Update GitHub webhook URL to: https://$APP_URL/events" - echo " 2. Verify webhook secret matches WEBHOOK_SECRET in env.yaml" - echo " 3. Test webhook by merging a PR in source repository" - echo " 4. Monitor logs: gcloud app logs tail -s default --project=$PROJECT_ID" - echo "" - - # Ask if user wants to view logs - if [ "$QUIET" != "true" ]; then - read -p "View application logs? (y/N) " -n 1 -r - echo - if [[ $REPLY =~ ^[Yy]$ ]]; then - gcloud app logs tail -s default --project=$PROJECT_ID - fi - fi -else - print_error "Deployment failed" - echo "" - echo "Troubleshooting:" - echo " 1. Check that all required APIs are enabled:" - echo " gcloud services enable appengine.googleapis.com --project=$PROJECT_ID" - echo " gcloud services enable secretmanager.googleapis.com --project=$PROJECT_ID" - echo " 2. Verify env.yaml contains all required variables" - echo " 3. Check deployment logs for specific errors" - echo " 4. See DEPLOYMENT-GUIDE.md for detailed troubleshooting" - exit 1 -fi - diff --git a/examples-copier/docs/CONFIGURATION-GUIDE.md b/examples-copier/docs/CONFIGURATION-GUIDE.md index 81d377d..510aa0a 100644 --- a/examples-copier/docs/CONFIGURATION-GUIDE.md +++ b/examples-copier/docs/CONFIGURATION-GUIDE.md @@ -885,7 +885,7 @@ Error: copy_rules[0]: name is required - [Pattern Matching Cheat Sheet](PATTERN-MATCHING-CHEATSHEET.md) - Quick reference - [Migration Guide](MIGRATION-GUIDE.md) - Migrating from legacy JSON config - [Quick Reference](../QUICK-REFERENCE.md) - Command reference -- [Deployment Guide](DEPLOYMENT-GUIDE.md) - Deploying the application +- [Deployment Guide](DEPLOYMENT.md) - Deploying the application --- diff --git a/examples-copier/docs/DEBUG-LOGGING.md b/examples-copier/docs/DEBUG-LOGGING.md new file mode 100644 index 0000000..05f3682 --- /dev/null +++ b/examples-copier/docs/DEBUG-LOGGING.md @@ -0,0 +1,376 @@ +# Debug Logging Guide + +This guide explains how to enable and use debug logging in the Examples Copier application. + +## Overview + +The Examples Copier supports configurable logging levels to help with development, troubleshooting, and debugging. By default, the application logs at the INFO level, but you can enable DEBUG logging for more verbose output. + +## Environment Variables + +### LOG_LEVEL + +**Purpose:** Set the logging level for the application + +**Values:** +- `info` (default) - Standard operational logs +- `debug` - Verbose debug logs with detailed operation information + +**Example:** +```bash +LOG_LEVEL="debug" +``` + +### COPIER_DEBUG + +**Purpose:** Alternative way to enable debug mode + +**Values:** +- `true` - Enable debug logging +- `false` (default) - Standard logging + +**Example:** +```bash +COPIER_DEBUG="true" +``` + +**Note:** Either `LOG_LEVEL="debug"` OR `COPIER_DEBUG="true"` will enable debug logging. You only need to set one. + +### COPIER_DISABLE_CLOUD_LOGGING + +**Purpose:** Disable Google Cloud Logging (useful for local development) + +**Values:** +- `true` - Disable GCP logging, only log to stdout +- `false` (default) - Enable GCP logging if configured + +**Example:** +```bash +COPIER_DISABLE_CLOUD_LOGGING="true" +``` + +**Use case:** When developing locally, you may not want logs sent to Google Cloud. This flag keeps all logs local. + +--- + +## How It Works + +### Code Implementation + +The logging system is implemented in `services/logger.go`: + +```go +// LogDebug writes debug logs only when LOG_LEVEL=debug or COPIER_DEBUG=true. +func LogDebug(message string) { + if !isDebugEnabled() { + return + } + // Mirror to GCP as info if available, plus prefix to stdout + if googleInfoLogger != nil && gcpLoggingEnabled { + googleInfoLogger.Println("[DEBUG] " + message) + } + log.Println("[DEBUG] " + message) +} + +func isDebugEnabled() bool { + if strings.EqualFold(os.Getenv("LOG_LEVEL"), "debug") { + return true + } + return strings.EqualFold(os.Getenv("COPIER_DEBUG"), "true") +} + +func isCloudLoggingDisabled() bool { + return strings.EqualFold(os.Getenv("COPIER_DISABLE_CLOUD_LOGGING"), "true") +} +``` + +### Log Levels + +The application supports the following log levels: + +| Level | Function | When to Use | Example | +|-------|----------|-------------|---------| +| **DEBUG** | `LogDebug()` | Detailed operation logs, file matching, API calls | `[DEBUG] Matched file: src/example.js` | +| **INFO** | `LogInfo()` | Standard operational logs | `[INFO] Processing webhook event` | +| **WARN** | `LogWarning()` | Warning conditions | `[WARN] File not found, skipping` | +| **ERROR** | `LogError()` | Error conditions | `[ERROR] Failed to create PR` | +| **CRITICAL** | `LogCritical()` | Critical failures | `[CRITICAL] Database connection failed` | + +--- + +## Usage Examples + +### Local Development with Debug Logging + +**Using .env file:** +```bash +# configs/.env +LOG_LEVEL="debug" +COPIER_DISABLE_CLOUD_LOGGING="true" +DRY_RUN="true" +``` + +**Using environment variables:** +```bash +export LOG_LEVEL=debug +export COPIER_DISABLE_CLOUD_LOGGING=true +export DRY_RUN=true +go run app.go +``` + +### Production with Debug Logging (Temporary) + +**env.yaml:** +```yaml +env_variables: + LOG_LEVEL: "debug" + # ... other variables +``` + +**Deploy:** +```bash +gcloud app deploy app.yaml # env.yaml is included via 'includes' directive +``` + +**Important:** Remember to disable debug logging after troubleshooting to reduce log volume and costs. + +### Local Development without Cloud Logging + +```bash +# configs/.env +COPIER_DISABLE_CLOUD_LOGGING="true" +``` + +This keeps all logs local (stdout only), which is faster and doesn't require GCP credentials. + +--- + +## What Gets Logged at DEBUG Level? + +When debug logging is enabled, you'll see additional information about: + +### 1. **File Matching Operations** +``` +[DEBUG] Checking pattern: src/**/*.js +[DEBUG] Matched file: src/examples/example1.js +[DEBUG] Excluded file: src/tests/test.js (matches exclude pattern) +``` + +### 2. **GitHub API Calls** +``` +[DEBUG] Fetching file from GitHub: src/example.js +[DEBUG] Creating PR for target repo: mongodb/docs-code-examples +[DEBUG] GitHub API response: 200 OK +``` + +### 3. **Configuration Loading** +``` +[DEBUG] Loading config file: copier-config.yaml +[DEBUG] Found 5 copy rules +[DEBUG] Rule 1: Copy src/**/*.js to examples/ +``` + +### 4. **Webhook Processing** +``` +[DEBUG] Received webhook event: pull_request +[DEBUG] PR action: closed +[DEBUG] PR merged: true +[DEBUG] Processing 3 changed files +``` + +### 5. **Pattern Matching** +``` +[DEBUG] Testing pattern: src/**/*.{js,ts} +[DEBUG] File matches: true +[DEBUG] Applying transformations: 2 +``` + +--- + +## Best Practices + +### ✅ DO + +- **Enable debug logging when troubleshooting issues** + ```bash + LOG_LEVEL="debug" + ``` + +- **Disable cloud logging for local development** + ```bash + COPIER_DISABLE_CLOUD_LOGGING="true" + ``` + +- **Use debug logging with dry run mode for testing** + ```bash + LOG_LEVEL="debug" + DRY_RUN="true" + ``` + +- **Disable debug logging in production after troubleshooting** + - High log volume can increase costs + - May expose sensitive information + +### ❌ DON'T + +- **Don't leave debug logging enabled in production long-term** + - Increases log volume and storage costs + - May impact performance + - Can expose internal implementation details + +- **Don't rely on debug logs for critical monitoring** + - Use INFO/WARN/ERROR levels for operational monitoring + - Debug logs may be disabled in production + +- **Don't log sensitive data even in debug mode** + - The code already avoids logging secrets + - Be careful when adding new debug logs + +--- + +## Troubleshooting + +### Debug Logs Not Appearing + +**Problem:** Set `LOG_LEVEL="debug"` but not seeing debug logs + +**Solutions:** + +1. **Check the variable is set correctly:** + ```bash + echo $LOG_LEVEL + # Should output: debug + ``` + +2. **Try the alternative flag:** + ```bash + COPIER_DEBUG="true" + ``` + +3. **Check case sensitivity:** + ```bash + # Both work (case-insensitive): + LOG_LEVEL="debug" + LOG_LEVEL="DEBUG" + ``` + +4. **Verify the code is calling LogDebug():** + - Not all operations have debug logs + - Check `services/logger.go` for `LogDebug()` calls + +### Logs Not Going to Google Cloud + +**Problem:** Logs appear in stdout but not in Google Cloud Logging + +**Solutions:** + +1. **Check if cloud logging is disabled:** + ```bash + # Remove or set to false: + # COPIER_DISABLE_CLOUD_LOGGING="true" + ``` + +2. **Verify GCP credentials:** + ```bash + gcloud auth application-default login + ``` + +3. **Check project ID is set:** + ```bash + GOOGLE_CLOUD_PROJECT_ID="your-project-id" + ``` + +4. **Check log name is set:** + ```bash + COPIER_LOG_NAME="code-copier-log" + ``` + +### Too Many Logs + +**Problem:** Debug logging produces too much output + +**Solutions:** + +1. **Disable debug logging:** + ```bash + # Remove or comment out: + # LOG_LEVEL="debug" + # COPIER_DEBUG="true" + ``` + +2. **Use grep to filter:** + ```bash + # Show only errors: + go run app.go 2>&1 | grep ERROR + + # Show only specific operations: + go run app.go 2>&1 | grep "pattern matching" + ``` + +3. **Redirect to file:** + ```bash + go run app.go > debug.log 2>&1 + ``` + +--- + +## Configuration Examples + +### Example 1: Local Development (Recommended) + +```bash +# configs/.env +LOG_LEVEL="debug" +COPIER_DISABLE_CLOUD_LOGGING="true" +DRY_RUN="true" +AUDIT_ENABLED="false" +METRICS_ENABLED="true" +``` + +**Why:** +- Debug logs help understand what's happening +- No cloud logging keeps it fast and local +- Dry run prevents accidental changes +- No audit logging (simpler setup) + +### Example 2: Production Troubleshooting + +```yaml +# env.yaml +env_variables: + LOG_LEVEL: "debug" + GOOGLE_CLOUD_PROJECT_ID: "your-project-id" + COPIER_LOG_NAME: "code-copier-log" + # ... other variables +``` + +**Why:** +- Temporarily enable debug for troubleshooting +- Logs go to Cloud Logging for analysis +- Remember to disable after fixing issue + +### Example 3: Local with Cloud Logging + +```bash +# configs/.env +LOG_LEVEL="debug" +GOOGLE_CLOUD_PROJECT_ID="your-project-id" +COPIER_LOG_NAME="code-copier-log-dev" +# COPIER_DISABLE_CLOUD_LOGGING not set (defaults to false) +``` + +**Why:** +- Test cloud logging integration locally +- Separate log name for dev environment +- Useful for testing logging infrastructure + +--- + +## See Also + +- [LOCAL-TESTING.md](LOCAL-TESTING.md) - Local development guide +- [TROUBLESHOOTING.md](TROUBLESHOOTING.md) - General troubleshooting +- [CONFIGURATION-GUIDE.md](CONFIGURATION-GUIDE.md) - Complete configuration reference +- [../configs/env.yaml.example](../configs/env.yaml.example) - All environment variables +- [../configs/.env.example](../configs/.env.example) - Local development template + diff --git a/examples-copier/docs/DEPLOYMENT-CHECKLIST.md b/examples-copier/docs/DEPLOYMENT-CHECKLIST.md new file mode 100644 index 0000000..fd9bb9a --- /dev/null +++ b/examples-copier/docs/DEPLOYMENT-CHECKLIST.md @@ -0,0 +1,493 @@ +# Deployment Checklist + +Quick reference checklist for deploying the GitHub Code Example Copier to Google Cloud App Engine. + +## 📋 Pre-Deployment + +### ☐ 1. Prerequisites Installed + +```bash +# Verify Go +go version # Should be 1.23+ + +# Verify gcloud +gcloud --version + +# Verify authentication +gcloud auth list +``` + +### ☐ 2. Google Cloud Project Setup + +```bash +# Set project +gcloud config set project YOUR_PROJECT_ID + +# Verify +gcloud config get-value project + +# Enable required APIs +gcloud services enable secretmanager.googleapis.com +gcloud services enable appengine.googleapis.com +``` + +### ☐ 3. Secrets in Secret Manager + +```bash +# List secrets +gcloud secrets list + +# Expected secrets: +# ✅ CODE_COPIER_PEM - GitHub App private key +# ✅ webhook-secret - Webhook signature validation +# ✅ mongo-uri - MongoDB connection (optional) +``` + +**If secrets don't exist, create them:** + +```bash +# GitHub private key +gcloud secrets create CODE_COPIER_PEM \ + --data-file=/path/to/private-key.pem \ + --replication-policy="automatic" + +# Webhook secret +WEBHOOK_SECRET=$(openssl rand -hex 32) +echo -n "$WEBHOOK_SECRET" | gcloud secrets create webhook-secret \ + --data-file=- \ + --replication-policy="automatic" +echo "Save this: $WEBHOOK_SECRET" + +# MongoDB URI (optional) +echo -n "mongodb+srv://..." | gcloud secrets create mongo-uri \ + --data-file=- \ + --replication-policy="automatic" +``` + +### ☐ 4. Grant IAM Permissions + +```bash +# Run the grant script +cd examples-copier +./scripts/grant-secret-access.sh +``` + +**Or manually:** + +```bash +PROJECT_NUMBER=$(gcloud projects describe $(gcloud config get-value project) --format="value(projectNumber)") +SERVICE_ACCOUNT="${PROJECT_NUMBER}@appspot.gserviceaccount.com" + +gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="roles/secretmanager.secretAccessor" + +gcloud secrets add-iam-policy-binding webhook-secret \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="roles/secretmanager.secretAccessor" + +gcloud secrets add-iam-policy-binding mongo-uri \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="roles/secretmanager.secretAccessor" +``` + +**Verify:** +```bash +gcloud secrets get-iam-policy CODE_COPIER_PEM | grep @appspot +gcloud secrets get-iam-policy webhook-secret | grep @appspot +gcloud secrets get-iam-policy mongo-uri | grep @appspot +``` + +### ☐ 5. Create env.yaml + +```bash +cd examples-copier + +# Copy from template +cp configs/env.yaml.production env.yaml + +# Or convert from .env +./scripts/convert-env-to-yaml.sh configs/.env env.yaml + +# Edit with your values if needed +nano env.yaml +``` + +**Required changes in env.yaml:** +- `GITHUB_APP_ID` - Your GitHub App ID +- `INSTALLATION_ID` - Your installation ID +- `REPO_OWNER` - Source repository owner +- `REPO_NAME` - Source repository name +- `GITHUB_APP_PRIVATE_KEY_SECRET_NAME` - Update project number +- `WEBHOOK_SECRET_NAME` - Update project number +- `MONGO_URI_SECRET_NAME` - Update project number (if using audit logging) +- `GOOGLE_PROJECT_ID` - Your Google Cloud project ID + +### ☐ 6. Verify env.yaml in .gitignore + +```bash +# Check +grep "env.yaml" .gitignore + +# If not found, add it +echo "env.yaml" >> .gitignore +``` + +### ☐ 7. Verify app.yaml Configuration + +```bash +cat app.yaml +``` + +**Should contain:** +```yaml +runtime: go +runtime_config: + operating_system: "ubuntu22" + runtime_version: "1.23" +env: flex +``` + +**Should NOT contain:** +- ❌ `env_variables:` section (those go in env.yaml) + +--- + +## 🚀 Deployment + +### ☐ 8. Deploy to App Engine + +```bash +cd examples-copier + +# Deploy (env.yaml is included via 'includes' directive in app.yaml) +gcloud app deploy app.yaml +``` + +**Expected output:** +``` +Updating service [default]...done. +Setting traffic split for service [default]...done. +Deployed service [default] to [https://YOUR_APP.appspot.com] +``` + +### ☐ 9. Verify Deployment + +```bash +# Check versions +gcloud app versions list + +# Get app URL +APP_URL=$(gcloud app describe --format="value(defaultHostname)") +echo "App URL: https://${APP_URL}" +``` + +### ☐ 10. Check Logs + +```bash +# View real-time logs +gcloud app logs tail -s default +``` + +**Look for:** +- ✅ "Starting web server on port :8080" +- ✅ No errors about secrets +- ✅ No "failed to load webhook secret" +- ✅ No "failed to load MongoDB URI" + +**Should NOT see:** +- ❌ "failed to load webhook secret" +- ❌ "failed to load MongoDB URI" +- ❌ "SKIP_SECRET_MANAGER=true" + +### ☐ 11. Test Health Endpoint + +```bash +# Get app URL +APP_URL=$(gcloud app describe --format="value(defaultHostname)") + +# Test health +curl https://${APP_URL}/health +``` + +**Expected response:** +```json +{ + "status": "healthy", + "started": true, + "github": { + "status": "healthy", + "authenticated": true + }, + "queues": { + "upload_count": 0, + "deprecation_count": 0 + }, + "uptime": "1m23s" +} +``` + +--- + +## 🔗 GitHub Webhook Configuration + +### ☐ 12. Get Webhook Secret + +```bash +# Get the webhook secret value +gcloud secrets versions access latest --secret=webhook-secret +``` + +**Save this value** - you'll need it for GitHub webhook configuration. + +### ☐ 13. Configure GitHub Webhook + +1. **Go to repository settings** + - URL: `https://github.com/YOUR_ORG/YOUR_REPO/settings/hooks` + +2. **Add or edit webhook** + - **Payload URL:** `https://YOUR_APP.appspot.com/events` + - **Content type:** `application/json` + - **Secret:** (paste the value from step 12) + - **SSL verification:** Enable SSL verification + - **Events:** Select "Pull requests" + - **Active:** ✓ Checked + +3. **Save webhook** + +### ☐ 14. Test Webhook + +**Option A: Redeliver existing webhook** +1. Go to webhook settings +2. Click "Recent Deliveries" +3. Click on a delivery +4. Click "Redeliver" + +**Option B: Create test PR** +1. Create a test PR in your source repository +2. Merge it +3. Watch logs for webhook receipt + +```bash +# Watch logs +gcloud app logs tail -s default | grep webhook +``` + +--- + +## ✅ Post-Deployment Verification + +### ☐ 15. Verify Secrets Loaded + +```bash +# Check logs for secret loading +gcloud app logs read --limit=100 | grep -i "secret" +``` + +**Should NOT see:** +- ❌ "failed to load webhook secret" +- ❌ "failed to load MongoDB URI" + +### ☐ 16. Verify Webhook Signature Validation + +```bash +# Watch logs during webhook delivery +gcloud app logs tail -s default +``` + +**Look for:** +- ✅ "webhook received" +- ✅ "signature verified" +- ✅ "processing webhook" + +**Should NOT see:** +- ❌ "webhook signature verification failed" +- ❌ "invalid signature" + +### ☐ 17. Verify File Copying + +```bash +# Watch logs during PR merge +gcloud app logs tail -s default +``` + +**Look for:** +- ✅ "Config file loaded successfully" +- ✅ "file matched pattern" +- ✅ "Copied file to target repo" + +### ☐ 18. Verify Audit Logging (if enabled) + +```bash +# Connect to MongoDB +mongosh "YOUR_MONGO_URI" + +# Check for recent events +db.audit_events.find().sort({timestamp: -1}).limit(5) +``` + +### ☐ 19. Verify Metrics (if enabled) + +```bash +# Check metrics endpoint +curl https://YOUR_APP.appspot.com/metrics +``` + +**Expected response:** +```json +{ + "webhooks": { + "received": 1, + "processed": 1, + "failed": 0 + }, + "files": { + "matched": 5, + "uploaded": 5, + "failed": 0 + } +} +``` + +### ☐ 20. Security Verification + +```bash +# Verify env.yaml doesn't contain actual secrets +cat env.yaml | grep -E "BEGIN|mongodb\+srv|ghp_" +# Should return NOTHING (only Secret Manager paths) + +# Verify env.yaml is not committed +git status | grep env.yaml +# Should show: nothing to commit (or untracked) + +# Verify IAM permissions +gcloud secrets get-iam-policy CODE_COPIER_PEM | grep @appspot +gcloud secrets get-iam-policy webhook-secret | grep @appspot +# Should see the service account +``` + +--- + +## 🐛 Troubleshooting + +### Error: "failed to load webhook secret" + +**Cause:** Secret Manager access denied + +**Fix:** +```bash +./scripts/grant-secret-access.sh +``` + +### Error: "webhook signature verification failed" + +**Cause:** Secret in Secret Manager doesn't match GitHub webhook secret + +**Fix:** +```bash +# Get secret from Secret Manager +gcloud secrets versions access latest --secret=webhook-secret + +# Update GitHub webhook with this value +# OR update Secret Manager with GitHub's value +``` + +### Error: "MONGO_URI is required when audit logging is enabled" + +**Cause:** Audit logging enabled but MongoDB URI not loaded + +**Fix:** +```bash +# Option 1: Disable audit logging +# In env.yaml: AUDIT_ENABLED: "false" + +# Option 2: Ensure MONGO_URI_SECRET_NAME is set +# In env.yaml: MONGO_URI_SECRET_NAME: "projects/.../secrets/mongo-uri/versions/latest" + +# Redeploy +gcloud app deploy app.yaml +``` + +### Error: "Config file not found" + +**Cause:** `copier-config.yaml` missing from source repository + +**Fix:** +```bash +# Add copier-config.yaml to your source repository +# See documentation for config file format +``` + +--- + +## 📊 Success Criteria + +All items should be ✅: + +- ✅ Deployment completes without errors +- ✅ App Engine is running +- ✅ Health endpoint returns 200 OK +- ✅ Logs show no secret loading errors +- ✅ Webhook receives PR events +- ✅ Webhook signature validation works +- ✅ Files are copied to target repos +- ✅ Audit events logged (if enabled) +- ✅ Metrics available (if enabled) +- ✅ No secrets in config files +- ✅ env.yaml not in version control + +--- + +## 🎉 You're Done! + +Your application is deployed with: +- ✅ All secrets in Secret Manager (secure!) +- ✅ No hardcoded secrets in config files +- ✅ Easy secret rotation (just update in Secret Manager) +- ✅ Audit trail of secret access +- ✅ Fine-grained IAM permissions + +**Next steps:** +1. Monitor logs for first few PRs +2. Verify files are copied correctly +3. Set up alerts (optional) +4. Document any custom configuration + +--- + +## 📚 Quick Reference + +```bash +# Deploy +gcloud app deploy app.yaml + +# View logs +gcloud app logs tail -s default + +# Check health +curl https://YOUR_APP.appspot.com/health + +# Check metrics +curl https://YOUR_APP.appspot.com/metrics + +# List secrets +gcloud secrets list + +# Get secret value +gcloud secrets versions access latest --secret=SECRET_NAME + +# Grant access +./scripts/grant-secret-access.sh + +# Rollback +gcloud app versions list +gcloud app services set-traffic default --splits=PREVIOUS_VERSION=1 +``` + +--- + +**See also:** +- [DEPLOYMENT.md](DEPLOYMENT.md) - Complete deployment guide +- [../WEBHOOK-SECRET-MANAGER-GUIDE.md](../WEBHOOK-SECRET-MANAGER-GUIDE.md) - Secret Manager details +- [../ENV-FILES-EXPLAINED.md](../ENV-FILES-EXPLAINED.md) - Environment file explanation + diff --git a/examples-copier/docs/DEPLOYMENT-GUIDE.md b/examples-copier/docs/DEPLOYMENT-GUIDE.md deleted file mode 100644 index c880cda..0000000 --- a/examples-copier/docs/DEPLOYMENT-GUIDE.md +++ /dev/null @@ -1,369 +0,0 @@ -# Deployment Guide - -This guide walks you through deploying the examples-copier application. - -## Prerequisites - -1. **Go 1.23.4+** installed -2. **MongoDB Atlas** account (for audit logging) -3. **GitHub App** credentials -4. **Google Cloud** project (for Secret Manager and logging) - -## Step 1: Build the Application - -```bash -cd examples-copier - -# Build main application -go build -o examples-copier . - -# Build CLI validator -go build -o config-validator ./cmd/config-validator - -# Verify builds -./examples-copier -help -./config-validator -help -``` - -## Step 2: Configure Environment - -Create or update your `.env` file: - -```bash -# Copy example -cp configs/.env.example.new configs/.env - -# Edit with your values -vim configs/.env -``` - -### Required Environment Variables - -```bash -# GitHub Configuration -REPO_OWNER=your-org -REPO_NAME=your-repo -SRC_BRANCH=main -GITHUB_APP_ID=123456 -GITHUB_INSTALLATION_ID=789012 - -# Google Cloud -GCP_PROJECT_ID=your-project -PEM_KEY_NAME=projects/123/secrets/CODE_COPIER_PEM/versions/latest - -# Application Settings -PORT=8080 -WEBSERVER_PATH=/webhook -CONFIG_FILE=copier-config.yaml -DEPRECATION_FILE=deprecated_examples.json - -# New Features -DRY_RUN=false -AUDIT_ENABLED=true -METRICS_ENABLED=true -WEBHOOK_SECRET=your-webhook-secret - -# MongoDB (for audit logging) -MONGO_URI=mongodb+srv://user:pass@cluster.mongodb.net -AUDIT_DATABASE=code_copier -AUDIT_COLLECTION=audit_events -``` - -## Step 3: Create YAML Configuration - -Create `copier-config.yaml` in your repository: - -```yaml -source_repo: "your-org/source-repo" -source_branch: "main" - -copy_rules: - - name: "Copy Go examples" - source_pattern: - type: "regex" - pattern: "^examples/go/(?P[^/]+)/(?P.+)$" - targets: - - repo: "your-org/target-repo" - branch: "main" - path_transform: "docs/examples/${category}/${file}" - commit_strategy: - type: "pull_request" - commit_message: "Update ${category} examples from source" - pr_title: "Update ${category} examples" - auto_merge: false - deprecation_check: - enabled: true - file: "deprecated_examples.json" -``` - -### Validate Configuration - -```bash -# Validate config file -./config-validator validate -config copier-config.yaml -v - -# Test pattern matching -./config-validator test-pattern \ - -type regex \ - -pattern "^examples/go/(?P[^/]+)/(?P.+)$" \ - -file "examples/go/database/connect.go" - -# Test path transformation -./config-validator test-transform \ - -template "docs/examples/${category}/${file}" \ - -file "examples/go/database/connect.go" \ - -pattern "^examples/go/(?P[^/]+)/(?P.+)$" -``` - -## Step 4: Test with Dry-Run Mode - -Before deploying to production, test with dry-run mode: - -```bash -# Enable dry-run in .env -DRY_RUN=true ./examples-copier -env ./configs/.env -``` - -In dry-run mode: -- Webhooks are processed -- Files are matched and transformed -- Audit events are logged -- **NO actual commits or PRs are created** - -## Step 5: Deploy to Google Cloud App Engine - -### Update `app.yaml` - -```yaml -runtime: go123 -env: standard - -env_variables: - REPO_OWNER: "your-org" - REPO_NAME: "your-repo" - CONFIG_FILE: "copier-config.yaml" - AUDIT_ENABLED: "true" - METRICS_ENABLED: "true" - MONGO_URI: "mongodb+srv://..." - # ... other variables - -handlers: - - url: /.* - script: auto - secure: always -``` - -### Deploy - -```bash -# Deploy to App Engine -gcloud app deploy - -# View logs -gcloud app logs tail -s default - -# Check health -curl https://your-app.appspot.com/health -``` - -## Step 6: Configure GitHub Webhook - -1. Go to your repository settings -2. Navigate to **Webhooks** → **Add webhook** -3. Set **Payload URL**: `https://your-app.appspot.com/webhook` -4. Set **Content type**: `application/json` -5. Set **Secret**: (your WEBHOOK_SECRET value) -6. Select events: **Pull requests** -7. Click **Add webhook** - -## Step 7: Monitor and Verify - -### Check Health Endpoint - -```bash -curl https://your-app.appspot.com/health -``` - -Expected response: -```json -{ - "status": "healthy", - "started": true, - "github": { - "status": "healthy", - "authenticated": true - }, - "queues": { - "upload_count": 0, - "deprecation_count": 0 - }, - "uptime": "1h23m45s" -} -``` - -### Check Metrics Endpoint - -```bash -curl https://your-app.appspot.com/metrics -``` - -Expected response: -```json -{ - "webhooks": { - "received": 42, - "processed": 40, - "failed": 2, - "success_rate": 95.24 - }, - "files": { - "matched": 150, - "uploaded": 145, - "failed": 5, - "deprecated": 3 - }, - "processing_time": { - "p50": 234, - "p95": 567, - "p99": 890 - } -} -``` - -### Query Audit Logs - -Connect to MongoDB and query audit events: - -```javascript -// Recent successful copies -db.audit_events.find({ - event_type: "copy", - success: true -}).sort({timestamp: -1}).limit(10) - -// Failed operations -db.audit_events.find({ - success: false -}).sort({timestamp: -1}) - -// Statistics by rule -db.audit_events.aggregate([ - {$match: {event_type: "copy"}}, - {$group: { - _id: "$rule_name", - count: {$sum: 1}, - avg_duration: {$avg: "$duration_ms"} - }} -]) -``` - -## Step 8: Gradual Rollout - -### Phase 1: Test with One Rule - -Start with a single, simple copy rule: - -```yaml -copy_rules: - - name: "Test rule" - source_pattern: - type: "prefix" - pattern: "test/examples/" - targets: - - repo: "your-org/test-repo" - branch: "test-branch" - path_transform: "${path}" -``` - -### Phase 2: Add More Rules - -Gradually add more complex rules with regex patterns and transformations. - -### Phase 3: Enable Auto-Merge - -Once confident, enable auto-merge for specific rules: - -```yaml -commit_strategy: - type: "pull_request" - auto_merge: true -``` - -## Troubleshooting - -### Issue: Config validation fails - -```bash -# Check config syntax -./config-validator validate -config copier-config.yaml -v - -# Test specific patterns -./config-validator test-pattern -type regex -pattern "..." -file "..." -``` - -### Issue: Files not matching - -Check the audit logs for match attempts: - -```javascript -db.audit_events.find({ - source_path: "your/file/path.go" -}) -``` - -### Issue: MongoDB connection fails - -```bash -# Test connection -mongosh "mongodb+srv://user:pass@cluster.mongodb.net/code_copier" - -# Check environment variable -echo $MONGO_URI -``` - -### Issue: Webhook signature verification fails - -```bash -# Verify webhook secret matches -echo $WEBHOOK_SECRET - -# Check GitHub webhook delivery logs -# Go to Settings → Webhooks → Recent Deliveries -``` - -## Rollback Plan - -If issues arise: - -1. **Disable webhook** in GitHub repository settings -2. **Revert to previous version** using `gcloud app versions list` and `gcloud app services set-traffic` -3. **Check audit logs** to identify what was changed -4. **Fix configuration** and redeploy - -## Performance Tuning - -### Optimize Pattern Matching - -- Use **prefix** patterns for simple directory matching (fastest) -- Use **glob** patterns for wildcard matching (medium) -- Use **regex** patterns only when necessary (slowest) - -### Batch Operations - -Group multiple file changes into single commits/PRs: - -```yaml -commit_strategy: - type: "pull_request" - # All files matching this rule will be in one PR -``` - -### MongoDB Indexing - -Ensure indexes exist for common queries: - -```javascript -db.audit_events.createIndex({timestamp: -1}) -db.audit_events.createIndex({rule_name: 1, timestamp: -1}) -db.audit_events.createIndex({success: 1, timestamp: -1}) -``` diff --git a/examples-copier/docs/DEPLOYMENT.md b/examples-copier/docs/DEPLOYMENT.md new file mode 100644 index 0000000..e9d1777 --- /dev/null +++ b/examples-copier/docs/DEPLOYMENT.md @@ -0,0 +1,524 @@ +# Deployment Guide + +Complete guide for deploying the GitHub Code Example Copier to Google Cloud App Engine with Secret Manager. + +## Table of Contents + +- [Prerequisites](#prerequisites) +- [Architecture Overview](#architecture-overview) +- [Secret Manager Setup](#secret-manager-setup) +- [Configuration](#configuration) +- [Deployment](#deployment) +- [Post-Deployment](#post-deployment) +- [Monitoring](#monitoring) +- [Troubleshooting](#troubleshooting) + +## Prerequisites + +### Required Tools + +- **Go 1.23+** - For local development and testing +- **Google Cloud SDK** - For deployment +- **GitHub App** - With appropriate permissions +- **MongoDB Atlas** (optional) - For audit logging + +### Required Accounts & Access + +- Google Cloud project with billing enabled +- GitHub organization admin access (to create/configure GitHub App) +- MongoDB Atlas account (if using audit logging) + +### Install Google Cloud SDK + +```bash +# macOS +brew install --cask google-cloud-sdk + +# Verify installation +gcloud --version +``` + +### Authenticate with Google Cloud + +```bash +# Login to Google Cloud +gcloud auth login + +# Set application default credentials +gcloud auth application-default login + +# Set your project +gcloud config set project YOUR_PROJECT_ID + +# Verify +gcloud config get-value project +``` + +## Architecture Overview + +### Components + +``` +┌─────────────────────────────────────────────────────────────┐ +│ GitHub Repository │ +│ (docs-code-examples) │ +└────────────────────┬────────────────────────────────────────┘ + │ Webhook (PR merged) + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ Google Cloud App Engine │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ examples-copier Application │ │ +│ │ - Receives webhook │ │ +│ │ - Validates signature │ │ +│ │ - Loads config from source repo │ │ +│ │ - Matches files against patterns │ │ +│ │ - Copies to target repos │ │ +│ └──────────────────────────────────────────────────────┘ │ +└────────────┬────────────────────────────┬───────────────────┘ + │ │ + ↓ ↓ +┌────────────────────────┐ ┌───────────────────────────────┐ +│ Secret Manager │ │ Cloud Logging │ +│ - GitHub private key │ │ - Application logs │ +│ - Webhook secret │ │ - Webhook events │ +│ - MongoDB URI │ │ - Error tracking │ +└────────────────────────┘ └───────────────────────────────┘ + │ + ↓ +┌────────────────────────┐ +│ MongoDB Atlas │ +│ - Audit events │ +│ - Metrics │ +└────────────────────────┘ +``` + +### Environment: Flexible vs Standard + +This application uses **App Engine Flexible Environment**: + +**app.yaml:** +```yaml +runtime: go +runtime_config: + operating_system: "ubuntu22" + runtime_version: "1.23" +env: flex # ← Flexible Environment +``` + +**Key differences:** +- Environment variables in **separate file** (`env.yaml`) included via `includes` directive +- Deployment: `gcloud app deploy app.yaml` +- Better Secret Manager integration +- More flexible runtime configuration + +## Secret Manager Setup + +### Why Secret Manager? + +✅ **Security**: Secrets encrypted at rest and in transit +✅ **Audit Trail**: All access logged +✅ **Rotation**: Update secrets without redeployment +✅ **Access Control**: Fine-grained IAM permissions +✅ **No Hardcoding**: Secrets never in config files or version control + +### Enable Secret Manager API + +```bash +gcloud services enable secretmanager.googleapis.com +``` + +### Store Secrets + +#### 1. GitHub App Private Key + +```bash +# Store your GitHub App private key +gcloud secrets create CODE_COPIER_PEM \ + --data-file=/path/to/your/private-key.pem \ + --replication-policy="automatic" +``` + +#### 2. Webhook Secret + +```bash +# Generate a secure webhook secret +WEBHOOK_SECRET=$(openssl rand -hex 32) +echo "Generated: $WEBHOOK_SECRET" + +# Store in Secret Manager +echo -n "$WEBHOOK_SECRET" | gcloud secrets create webhook-secret \ + --data-file=- \ + --replication-policy="automatic" + +# Save this value - you'll need it for GitHub webhook configuration +``` + +#### 3. MongoDB URI (Optional - for audit logging) + +```bash +# Store MongoDB connection string +echo -n "mongodb+srv://user:pass@cluster.mongodb.net/dbname" | \ + gcloud secrets create mongo-uri \ + --data-file=- \ + --replication-policy="automatic" +``` + +### Grant App Engine Access + +```bash +# Get your project number +PROJECT_NUMBER=$(gcloud projects describe $(gcloud config get-value project) --format="value(projectNumber)") + +# App Engine service account +SERVICE_ACCOUNT="${PROJECT_NUMBER}@appspot.gserviceaccount.com" + +# Grant access to each secret +gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="roles/secretmanager.secretAccessor" + +gcloud secrets add-iam-policy-binding webhook-secret \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="roles/secretmanager.secretAccessor" + +gcloud secrets add-iam-policy-binding mongo-uri \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="roles/secretmanager.secretAccessor" +``` + +**Or use the provided script:** +```bash +cd examples-copier +./scripts/grant-secret-access.sh +``` + +### Verify Secrets + +```bash +# List all secrets +gcloud secrets list + +# View secret metadata +gcloud secrets describe CODE_COPIER_PEM + +# Verify IAM permissions +gcloud secrets get-iam-policy CODE_COPIER_PEM +``` + +## Configuration + +### Create env.yaml + +The `env.yaml` file contains environment variables for App Engine deployment. + +```bash +cd examples-copier + +# Copy from production template +cp configs/env.yaml.production env.yaml + +# Or convert from .env file +./scripts/convert-env-to-yaml.sh configs/.env env.yaml + +# Edit if needed +nano env.yaml +``` + +### env.yaml Structure + +**Important Notes:** +- Do NOT set `PORT` in `env.yaml` - App Engine Flexible automatically sets this +- The application defaults to port 8080 for local development +- Secret Manager references must include `/versions/latest` or a specific version number + +```yaml +env_variables: + # ============================================================================= + # GitHub Configuration (Non-sensitive) + # ============================================================================= + GITHUB_APP_ID: "YOUR_APP_ID" + INSTALLATION_ID: "YOUR_INSTALLATION_ID" + REPO_OWNER: "your-org" + REPO_NAME: "your-repo" + SRC_BRANCH: "main" + + # ============================================================================= + # Secret Manager References (Sensitive - SECURE!) + # ============================================================================= + GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/PROJECT_NUMBER/secrets/CODE_COPIER_PEM/versions/latest" + WEBHOOK_SECRET_NAME: "projects/PROJECT_NUMBER/secrets/webhook-secret/versions/latest" + MONGO_URI_SECRET_NAME: "projects/PROJECT_NUMBER/secrets/mongo-uri/versions/latest" + + # ============================================================================= + # Application Settings + # ============================================================================= + # PORT: "8080" # DO NOT SET - App Engine sets this automatically + WEBSERVER_PATH: "/events" + CONFIG_FILE: "copier-config.yaml" + DEPRECATION_FILE: "deprecated_examples.json" + + # ============================================================================= + # Committer Information + # ============================================================================= + COMMITTER_NAME: "GitHub Copier App" + COMMITTER_EMAIL: "bot@example.com" + + # ============================================================================= + # Google Cloud Configuration + # ============================================================================= + GOOGLE_PROJECT_ID: "your-project-id" + GOOGLE_LOG_NAME: "code-copier-log" + + # ============================================================================= + # Feature Flags + # ============================================================================= + AUDIT_ENABLED: "true" + METRICS_ENABLED: "true" + # DRY_RUN: "false" +``` + +### Important Notes + +**✅ DO:** +- Use Secret Manager references (`*_SECRET_NAME` variables) +- Keep `env.yaml` in `.gitignore` +- Use `env.yaml.production` as template + +**❌ DON'T:** +- Put actual secrets in `env.yaml` (use `*_SECRET_NAME` instead) +- Commit `env.yaml` to version control +- Share `env.yaml` via email/chat + +### How Secrets Are Loaded + +``` +Application Startup: +1. Load env.yaml → environment variables +2. Read WEBHOOK_SECRET_NAME from env +3. Call Secret Manager API to get actual secret +4. Store in config.WebhookSecret +5. Use for webhook signature validation +``` + +**Code flow:** +```go +// app.go +config, _ := configs.LoadEnvironment(envFile) +services.LoadWebhookSecret(config) // Loads from Secret Manager +services.LoadMongoURI(config) // Loads from Secret Manager +``` + +## Deployment + +### Pre-Deployment Checklist + +- [ ] Secrets created in Secret Manager +- [ ] IAM permissions granted to App Engine +- [ ] `env.yaml` created and configured +- [ ] `env.yaml` in `.gitignore` +- [ ] `app.yaml` uses Flexible Environment + +### Deploy to App Engine + +```bash +cd examples-copier + +# Deploy (env.yaml is included via 'includes' directive in app.yaml) +gcloud app deploy app.yaml + +# Or specify project +gcloud app deploy app.yaml --project=your-project-id +``` + +### Verify Deployment + +```bash +# Check deployment status +gcloud app versions list + +# Get app URL +APP_URL=$(gcloud app describe --format="value(defaultHostname)") +echo "App URL: https://${APP_URL}" + +# View logs +gcloud app logs tail -s default +``` + +### Test Health Endpoint + +```bash +# Test health +curl https://${APP_URL}/health + +# Expected response: +# { +# "status": "healthy", +# "started": true, +# "github": { +# "status": "healthy", +# "authenticated": true +# }, +# "queues": { +# "upload_count": 0, +# "deprecation_count": 0 +# }, +# "uptime": "5m30s" +# } +``` + +## Post-Deployment + +### Configure GitHub Webhook + +1. **Navigate to repository settings** + - Go to: `https://github.com/YOUR_ORG/YOUR_REPO/settings/hooks` + +2. **Add or edit webhook** + - **Payload URL:** `https://YOUR_APP.appspot.com/events` + - **Content type:** `application/json` + - **Secret:** (the webhook secret from Secret Manager) + - **Events:** Select "Pull requests" + - **Active:** ✓ Checked + +3. **Get webhook secret from Secret Manager** + ```bash + gcloud secrets versions access latest --secret=webhook-secret + ``` + +4. **Save webhook** + +### Test Webhook + +**Option A: Merge a test PR** +```bash +# Create and merge a test PR +# Watch logs for webhook receipt +gcloud app logs tail -s default | grep webhook +``` + +**Option B: Redeliver from GitHub** +1. Go to webhook settings +2. Click "Recent Deliveries" +3. Click on a delivery +4. Click "Redeliver" +5. Watch logs + +### Verify Functionality + +```bash +# Check logs for successful processing +gcloud app logs read --limit=50 + +# Look for: +# ✅ "Starting web server on port :8080" +# ✅ "webhook received" +# ✅ "Config file loaded successfully" +# ✅ "file matched pattern" +# ✅ "Copied file to target repo" + +# Should NOT see: +# ❌ "failed to load webhook secret" +# ❌ "failed to load MongoDB URI" +# ❌ "webhook signature verification failed" +``` + +## Monitoring + +### View Logs + +```bash +# Real-time logs +gcloud app logs tail -s default + +# Recent logs +gcloud app logs read --limit=100 + +# Filter for errors +gcloud app logs read --limit=100 | grep ERROR + +# Filter for webhooks +gcloud app logs read --limit=100 | grep webhook +``` + +### Check Metrics + +```bash +# Metrics endpoint +curl https://YOUR_APP.appspot.com/metrics + +# Response includes: +# - webhooks_received +# - webhooks_processed +# - files_matched +# - files_uploaded +# - processing_time (p50, p95, p99) +``` + +### Audit Logging (if enabled) + +Query MongoDB for audit events: + +```javascript +// Connect to MongoDB +mongosh "mongodb+srv://..." + +// Recent events +db.audit_events.find().sort({timestamp: -1}).limit(10) + +// Failed operations +db.audit_events.find({success: false}) + +// Statistics by rule +db.audit_events.aggregate([ + {$match: {event_type: "copy"}}, + {$group: { + _id: "$rule_name", + count: {$sum: 1} + }} +]) +``` + +## Troubleshooting + +See [DEPLOYMENT-CHECKLIST.md](DEPLOYMENT-CHECKLIST.md) for detailed troubleshooting steps. + +### Common Issues + +| Error | Cause | Solution | +|-------|-------|----------| +| "failed to load webhook secret" | Secret Manager access denied | Run `./grant-secret-access.sh` | +| "webhook signature verification failed" | Secret mismatch | Verify secret matches GitHub webhook | +| "MONGO_URI is required" | Audit enabled but no URI | Set `MONGO_URI_SECRET_NAME` or disable audit | +| "Config file not found" | Missing copier-config.yaml | Add config file to source repo | + +### Quick Fixes + +```bash +# Grant secret access +./scripts/grant-secret-access.sh + +# View secret value +gcloud secrets versions access latest --secret=webhook-secret + +# Disable audit logging +# In env.yaml: AUDIT_ENABLED: "false" + +# Redeploy +gcloud app deploy app.yaml +``` + +## Next Steps + +1. **Monitor first few PRs** - Watch logs to ensure files are copied correctly +2. **Set up alerts** (optional) - Configure Cloud Monitoring alerts +3. **Document custom config** - Add notes about your specific setup +4. **Plan secret rotation** - Schedule regular secret updates + +--- + +**See also:** +- [DEPLOYMENT-CHECKLIST.md](DEPLOYMENT-CHECKLIST.md) - Step-by-step checklist +- [../WEBHOOK-SECRET-MANAGER-GUIDE.md](../WEBHOOK-SECRET-MANAGER-GUIDE.md) - Secret Manager details +- [../ENV-FILES-EXPLAINED.md](../ENV-FILES-EXPLAINED.md) - Environment file explanation + diff --git a/examples-copier/docs/FAQ.md b/examples-copier/docs/FAQ.md index 5074bc0..d04f378 100644 --- a/examples-copier/docs/FAQ.md +++ b/examples-copier/docs/FAQ.md @@ -176,7 +176,7 @@ Yes! See [Local Testing](LOCAL-TESTING.md) for instructions. ### How do I deploy to Google Cloud? -See [Deployment Guide](DEPLOYMENT-GUIDE.md) for step-by-step instructions. +See [Deployment Guide](DEPLOYMENT.md) for complete guide and [Deployment Checklist](DEPLOYMENT-CHECKLIST.md) for step-by-step instructions. ### Do I need MongoDB? diff --git a/examples-copier/docs/LOCAL-TESTING.md b/examples-copier/docs/LOCAL-TESTING.md index 875cb31..ac4b9d1 100644 --- a/examples-copier/docs/LOCAL-TESTING.md +++ b/examples-copier/docs/LOCAL-TESTING.md @@ -69,15 +69,15 @@ GITHUB_TOKEN=ghp_your_token_here ```bash # Terminal 1: Start the app -make run-local +make run-local-quick # You should see: # ╔════════════════════════════════════════════════════════════════╗ # ║ GitHub Code Example Copier ║ # ╠════════════════════════════════════════════════════════════════╣ # ║ Port: 8080 ║ -# ║ Webhook Path: /webhook ║ -# ║ Config File: config.json ║ +# ║ Webhook Path: /events ║ +# ║ Config File: copier-config.example.yaml ║ # ║ Dry Run: true ║ # ║ Audit Log: false ║ # ║ Metrics: true ║ @@ -87,12 +87,17 @@ make run-local ### Test with Webhook ```bash -# Terminal 2: Send test webhook -./test-webhook -payload test-payloads/example-pr-merged.json +# Terminal 2: Send test webhook (automatically fetches webhook secret) +make test-webhook-example + +# Or send webhook manually with secret +export WEBHOOK_SECRET=$(gcloud secrets versions access latest --secret=webhook-secret) +./test-webhook -payload test-payloads/example-pr-merged.json -secret "$WEBHOOK_SECRET" # Or test with real PR export GITHUB_TOKEN=ghp_... -./test-webhook -pr 456 -owner mongodb -repo docs-realm +export WEBHOOK_SECRET=$(gcloud secrets versions access latest --secret=webhook-secret) +./test-webhook -pr 456 -owner mongodb -repo docs-realm -secret "$WEBHOOK_SECRET" ``` ## What Happens in Local Mode @@ -309,17 +314,24 @@ COPIER_DISABLE_CLOUD_LOGGING=true ./examples-copier ### Error: "connection refused" when sending webhook -**Problem:** Application is not running +**Problem:** Application is not running, or you're trying to run both in the same terminal **Solution:** ```bash -# Make sure app is running in Terminal 1 -make run-local +# Terminal 1: Start the app (this blocks the terminal) +make run-local-quick -# Then send webhook in Terminal 2 -./test-webhook -payload test-payloads/example-pr-merged.json +# Terminal 2: In a NEW terminal window, send the webhook +cd examples-copier +make test-webhook-example + +# Or manually: +export WEBHOOK_SECRET=$(gcloud secrets versions access latest --secret=webhook-secret) +./test-webhook -payload test-payloads/example-pr-merged.json -secret "$WEBHOOK_SECRET" ``` +**Note:** The `make test-webhook-example` command requires the server to be running in a separate terminal. You cannot run both commands in the same terminal unless you background the server process. + ### Error: "GITHUB_TOKEN environment variable not set" **Problem:** Trying to fetch real PR without token @@ -411,19 +423,25 @@ Then you can: 3. Monitor metrics and audit logs 4. Deploy to production -See [DEPLOYMENT-GUIDE.md](DEPLOYMENT-GUIDE.md) for deployment instructions. +See [DEPLOYMENT.md](DEPLOYMENT.md) for deployment instructions. ## Quick Reference ```bash -# Start app locally -make run-local +# Terminal 1: Start app locally +make run-local-quick -# Test with example -./test-webhook -payload test-payloads/example-pr-merged.json +# Terminal 2: Test with example (auto-fetches webhook secret) +make test-webhook-example + +# Or test manually with webhook secret +export WEBHOOK_SECRET=$(gcloud secrets versions access latest --secret=webhook-secret) +./test-webhook -payload test-payloads/example-pr-merged.json -secret "$WEBHOOK_SECRET" # Test with real PR -./test-webhook -pr 456 -owner mongodb -repo docs-realm +export GITHUB_TOKEN=ghp_... +export WEBHOOK_SECRET=$(gcloud secrets versions access latest --secret=webhook-secret) +./test-webhook -pr 456 -owner mongodb -repo docs-realm -secret "$WEBHOOK_SECRET" # Check metrics curl http://localhost:8080/metrics | jq @@ -432,6 +450,6 @@ curl http://localhost:8080/metrics | jq curl http://localhost:8080/health | jq # Validate config -./config-validator validate -config config.json -v +./config-validator validate -config copier-config.yaml -v ``` diff --git a/examples-copier/docs/TROUBLESHOOTING.md b/examples-copier/docs/TROUBLESHOOTING.md index 59486a2..254f703 100644 --- a/examples-copier/docs/TROUBLESHOOTING.md +++ b/examples-copier/docs/TROUBLESHOOTING.md @@ -283,6 +283,9 @@ export COPIER_DISABLE_CLOUD_LOGGING=true export AUDIT_ENABLED=false ``` + + + ### GitHub API Rate Limit **Error:** diff --git a/examples-copier/docs/WEBHOOK-TESTING.md b/examples-copier/docs/WEBHOOK-TESTING.md index 2cf25d7..00a0d7e 100644 --- a/examples-copier/docs/WEBHOOK-TESTING.md +++ b/examples-copier/docs/WEBHOOK-TESTING.md @@ -449,5 +449,5 @@ After successful webhook testing: 5. Deploy to production 6. Set up alerts for failures -See [DEPLOYMENT-GUIDE.md](DEPLOYMENT-GUIDE.md) for deployment instructions. +See [DEPLOYMENT.md](DEPLOYMENT.md) for deployment instructions. diff --git a/examples-copier/env.yaml.example b/examples-copier/env.yaml.example deleted file mode 100644 index c0b8699..0000000 --- a/examples-copier/env.yaml.example +++ /dev/null @@ -1,123 +0,0 @@ -# Example environment variables for Google Cloud App Engine deployment -# Copy this file to env.yaml and fill in your actual values -# -# ⚠️ IMPORTANT: Add env.yaml to .gitignore to prevent committing secrets! - -env_variables: - # ============================================================================= - # REQUIRED CONFIGURATION - # ============================================================================= - - # GitHub App Configuration - GITHUB_APP_ID: "1166559" # Your GitHub App ID (numeric) - INSTALLATION_ID: "62138132" # GitHub App Installation ID - - # Source Repository Configuration - REPO_NAME: "docs-code-examples" # Source repository name - REPO_OWNER: "mongodb" # Source repository owner - - # Security Configuration - # Path to GitHub App private key in Google Secret Manager - GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/YOUR_PROJECT_ID/secrets/CODE_COPIER_PEM/versions/latest" - - # Webhook Secret Configuration (choose ONE of the following): - # Option 1: Store webhook secret in Secret Manager (RECOMMENDED for production) - WEBHOOK_SECRET_NAME: "projects/YOUR_PROJECT_ID/secrets/webhook-secret/versions/latest" - # Option 2: Provide webhook secret directly (NOT recommended for production) - # WEBHOOK_SECRET: "your-webhook-secret-here" - - # ============================================================================= - # OPTIONAL CONFIGURATION - # ============================================================================= - - # Committer Information - COMMITTER_NAME: "GitHub Copier App" - COMMITTER_EMAIL: "bot@mongodb.com" - - # Web Server Configuration - PORT: "8080" # Server port (App Engine uses 8080 by default) - WEBSERVER_PATH: "/events" # Webhook endpoint path - - # File Configuration - CONFIG_FILE: "copier-config.yaml" # Config file name in source repo - DEPRECATION_FILE: "deprecated_examples.json" # Deprecation tracking file name - - # Source Branch - SRC_BRANCH: "main" # Branch to monitor for changes - - # Google Cloud Configuration - GOOGLE_PROJECT_ID: "YOUR_PROJECT_ID" # GCP project ID for logging - GOOGLE_LOG_NAME: "code-copier-log" # Cloud Logging log name - - # Commit Strategy Defaults - COPIER_COMMIT_STRATEGY: "direct" # Default: "direct" or "pr" - DEFAULT_COMMIT_MESSAGE: "Update code examples from source repository" - DEFAULT_RECURSIVE_COPY: "true" # Default recursive copy behavior - DEFAULT_PR_MERGE: "false" # Default auto-merge PR behavior - - # Request Configuration - REQUEST_TIMEOUT_SECONDS: "300" # Webhook request timeout (5 minutes) - - # Logging Configuration - LOG_LEVEL: "info" # Log level: debug, info, warn, error - COPIER_DISABLE_CLOUD_LOGGING: "false" # Set to "true" to disable Cloud Logging - -# ============================================================================= -# NOTES -# ============================================================================= -# -# 1. GitHub App Private Key: -# Store your GitHub App private key in Google Secret Manager: -# -# gcloud secrets create CODE_COPIER_PEM \ -# --data-file=/path/to/private-key.pem \ -# --replication-policy="automatic" -# -# Grant App Engine access: -# gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ -# --member="serviceAccount:YOUR_PROJECT_ID@appspot.gserviceaccount.com" \ -# --role="roles/secretmanager.secretAccessor" -# -# 2. Webhook Secret: -# Option A: Store in Secret Manager (RECOMMENDED): -# -# # Generate a secure random string -# openssl rand -hex 32 -# -# # Store in Secret Manager -# echo -n "YOUR_GENERATED_SECRET" | gcloud secrets create webhook-secret \ -# --data-file=- \ -# --replication-policy="automatic" -# -# # Grant App Engine access -# gcloud secrets add-iam-policy-binding webhook-secret \ -# --member="serviceAccount:YOUR_PROJECT_ID@appspot.gserviceaccount.com" \ -# --role="roles/secretmanager.secretAccessor" -# -# # Use WEBHOOK_SECRET_NAME in env.yaml -# WEBHOOK_SECRET_NAME: "projects/YOUR_PROJECT_ID/secrets/webhook-secret/versions/latest" -# -# Option B: Provide directly (NOT recommended for production): -# -# # Use WEBHOOK_SECRET in env.yaml -# WEBHOOK_SECRET: "your-generated-secret-here" -# -# Use the same secret in your GitHub webhook configuration. -# -# 3. Deployment: -# Deploy using: -# -# gcloud app deploy app.yaml --env-vars-file=env.yaml -# -# Or use the deployment script: -# -# ./deploy.sh -# -# 4. Security: -# - Never commit env.yaml to version control -# - Add env.yaml to .gitignore -# - Rotate secrets regularly -# - Use least-privilege IAM roles -# -# ============================================================================= - diff --git a/examples-copier/scripts/convert-env-to-yaml.sh b/examples-copier/scripts/convert-env-to-yaml.sh new file mode 100755 index 0000000..3a41ace --- /dev/null +++ b/examples-copier/scripts/convert-env-to-yaml.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Convert .env file to env.yaml format for Google Cloud App Engine + +set -e + +# Default input/output files +INPUT_FILE="${1:-.env}" +OUTPUT_FILE="${2:-env.yaml}" + +if [ ! -f "$INPUT_FILE" ]; then + echo "Error: Input file '$INPUT_FILE' not found" + echo "Usage: $0 [input-file] [output-file]" + echo "Example: $0 .env.production env.yaml" + exit 1 +fi + +echo "Converting $INPUT_FILE to $OUTPUT_FILE..." + +# Start the YAML file +echo "env_variables:" > "$OUTPUT_FILE" + +# Read the .env file and convert to YAML +while IFS= read -r line || [ -n "$line" ]; do + # Skip empty lines and comments + if [[ -z "$line" ]] || [[ "$line" =~ ^[[:space:]]*# ]]; then + continue + fi + + # Extract key and value + if [[ "$line" =~ ^([^=]+)=(.*)$ ]]; then + key="${BASH_REMATCH[1]}" + value="${BASH_REMATCH[2]}" + + # Remove leading/trailing whitespace from key + key=$(echo "$key" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + # Remove quotes from value if present + value=$(echo "$value" | sed 's/^["'\'']\(.*\)["'\'']$/\1/') + + # Write to YAML file with proper indentation + echo " $key: \"$value\"" >> "$OUTPUT_FILE" + fi +done < "$INPUT_FILE" + +echo "✅ Conversion complete: $OUTPUT_FILE" +echo "" +echo "⚠️ IMPORTANT: Review $OUTPUT_FILE before deploying!" +echo " - Verify all values are correct" +echo " - Check for sensitive data" +echo " - Ensure $OUTPUT_FILE is in .gitignore" + diff --git a/examples-copier/scripts/grant-secret-access.sh b/examples-copier/scripts/grant-secret-access.sh new file mode 100755 index 0000000..023832c --- /dev/null +++ b/examples-copier/scripts/grant-secret-access.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Grant App Engine access to all secrets + +set -e + +PROJECT_ID="github-copy-code-examples" +PROJECT_NUMBER="1054147886816" +SERVICE_ACCOUNT="${PROJECT_NUMBER}@appspot.gserviceaccount.com" + +echo "Granting App Engine service account access to secrets..." +echo "Service Account: ${SERVICE_ACCOUNT}" +echo "" + +# Array of secrets to grant access to +SECRETS=( + "CODE_COPIER_PEM" + "webhook-secret" + "mongo-uri" +) + +for SECRET in "${SECRETS[@]}"; do + echo "Granting access to: ${SECRET}" + gcloud secrets add-iam-policy-binding "${SECRET}" \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="roles/secretmanager.secretAccessor" \ + --project="${PROJECT_ID}" 2>&1 | grep -E "Updated|bindings" || echo " Already has access" + echo "" +done + +echo "✅ Done! Verifying permissions..." +echo "" + +for SECRET in "${SECRETS[@]}"; do + echo "Permissions for ${SECRET}:" + gcloud secrets get-iam-policy "${SECRET}" \ + --project="${PROJECT_ID}" \ + --format="table(bindings.members)" 2>&1 | grep -A 5 "serviceAccount:${SERVICE_ACCOUNT}" || echo " Not found" + echo "" +done + +echo "✅ All secrets are now accessible by App Engine!" + diff --git a/examples-copier/services/github_auth.go b/examples-copier/services/github_auth.go index 174f5d0..6b28189 100644 --- a/examples-copier/services/github_auth.go +++ b/examples-copier/services/github_auth.go @@ -161,6 +161,54 @@ func LoadWebhookSecret(config *configs.Config) error { return nil } +// LoadMongoURI loads the MongoDB URI from Secret Manager or environment variable +func LoadMongoURI(config *configs.Config) error { + // If MongoDB URI is already set directly, use it + if config.MongoURI != "" { + return nil + } + + // If no secret name is configured, skip (audit logging is optional) + if config.MongoURISecretName == "" { + return nil + } + + // Load from Secret Manager + uri, err := getSecretFromSecretManager(config.MongoURISecretName, "MONGO_URI") + if err != nil { + return fmt.Errorf("failed to load MongoDB URI: %w", err) + } + config.MongoURI = uri + return nil +} + +// getSecretFromSecretManager is a generic function to retrieve any secret from Secret Manager +func getSecretFromSecretManager(secretName, envVarName string) (string, error) { + if os.Getenv("SKIP_SECRET_MANAGER") == "true" { + // For tests and local runs, use direct env var + if secret := os.Getenv(envVarName); secret != "" { + return secret, nil + } + return "", fmt.Errorf("SKIP_SECRET_MANAGER=true but no %s set", envVarName) + } + + ctx := context.Background() + client, err := secretmanager.NewClient(ctx) + if err != nil { + return "", fmt.Errorf("failed to create Secret Manager client: %w", err) + } + defer client.Close() + + req := &secretmanagerpb.AccessSecretVersionRequest{ + Name: secretName, + } + result, err := client.AccessSecretVersion(ctx, req) + if err != nil { + return "", fmt.Errorf("failed to access secret version: %w", err) + } + return string(result.Payload.Data), nil +} + // getInstallationAccessToken exchanges a JWT for a GitHub App installation access token. func getInstallationAccessToken(installationId, jwtToken string, hc *http.Client) (string, error) { if installationId == "" || installationId == configs.InstallationId { diff --git a/examples-copier/services/github_write_to_source.go b/examples-copier/services/github_write_to_source.go index 54a6b8d..2f336b6 100644 --- a/examples-copier/services/github_write_to_source.go +++ b/examples-copier/services/github_write_to_source.go @@ -63,8 +63,8 @@ func uploadDeprecationFileChanges(message string, newDeprecationFileContents str Message: github.String(message), Content: []byte(newDeprecationFileContents), Branch: github.String(os.Getenv(configs.SrcBranch)), - Committer: &github.CommitAuthor{Name: github.String(os.Getenv(configs.CommiterName)), - Email: github.String(os.Getenv(configs.CommiterEmail))}, + Committer: &github.CommitAuthor{Name: github.String(os.Getenv(configs.CommitterName)), + Email: github.String(os.Getenv(configs.CommitterEmail))}, } options.SHA = targetFileContent.SHA