diff --git a/github-metrics/.drone.yml b/github-metrics/.drone.yml new file mode 100644 index 0000000..71525a5 --- /dev/null +++ b/github-metrics/.drone.yml @@ -0,0 +1,94 @@ +--- +kind: pipeline +type: kubernetes +name: github-metrics-test + +trigger: + branch: + - main + event: + - push # Runs when PR is merged to main + - pull_request # Also runs on PRs for early feedback + +steps: + - name: check-changes + image: alpine/git + commands: + - | + # Check if any files in github-metrics/ directory changed + if [ "$DRONE_BUILD_EVENT" = "pull_request" ]; then + # For PRs, compare against target branch + git diff --name-only origin/$DRONE_TARGET_BRANCH...HEAD | grep -q "^github-metrics/" && echo "Changes detected" || (echo "No changes in github-metrics/, skipping" && exit 78) + else + # For pushes, compare against previous commit + git diff --name-only $DRONE_COMMIT_BEFORE $DRONE_COMMIT_AFTER | grep -q "^github-metrics/" && echo "Changes detected" || (echo "No changes in github-metrics/, skipping" && exit 78) + fi + + - name: test + image: node:20-alpine + commands: + - cd github-metrics + - npm ci + - node --version + - npm --version + - echo "Validating package.json and dependencies..." + +--- +depends_on: ['github-metrics-test'] +kind: pipeline +type: kubernetes +name: github-metrics-build + +trigger: + branch: + - main + event: + - push + - tag + +steps: + # Builds and publishes Docker image for production + - name: publish-production + image: plugins/kaniko-ecr + settings: + create_repository: true + registry: 795250896452.dkr.ecr.us-east-1.amazonaws.com + repo: docs/github-metrics + tags: + - git-${DRONE_COMMIT_SHA:0:7} + - latest + access_key: + from_secret: ecr_access_key + secret_key: + from_secret: ecr_secret_key + context: github-metrics + dockerfile: github-metrics/Dockerfile + +--- +depends_on: ['github-metrics-build'] +kind: pipeline +type: kubernetes +name: github-metrics-deploy + +trigger: + branch: + - main + event: + - push + - tag + +steps: + # Deploys cronjob to production using Helm + - name: deploy-production + image: quay.io/mongodb/drone-helm:v3 + settings: + chart: mongodb/cronjobs + chart_version: 1.21.2 + add_repos: [mongodb=https://10gen.github.io/helm-charts] + namespace: docs + release: github-metrics + values: image.tag=git-${DRONE_COMMIT_SHA:0:7},image.repository=795250896452.dkr.ecr.us-east-1.amazonaws.com/docs/github-metrics + values_files: ['github-metrics/cronjobs.yml'] + api_server: https://api.prod.corp.mongodb.com + kubernetes_token: + from_secret: kubernetes_token \ No newline at end of file diff --git a/github-metrics/Dockerfile b/github-metrics/Dockerfile new file mode 100644 index 0000000..932aa4e --- /dev/null +++ b/github-metrics/Dockerfile @@ -0,0 +1,28 @@ +FROM node:20-alpine + +# Set working directory +WORKDIR /app + +# Copy package files first (for better Docker layer caching) +COPY package.json package-lock.json ./ + +# Install dependencies (use ci for reproducible builds) +RUN npm ci --only=production + +# Copy the rest of the application files +COPY . . + +# Create a non-root user for security best practices +RUN addgroup -g 1001 -S nodejs && \ + adduser -S nodejs -u 1001 && \ + chown -R nodejs:nodejs /app + +# Switch to non-root user +USER nodejs + +# Set NODE_ENV to production +ENV NODE_ENV=production + +# Command to run the application +# This will be executed by the Kubernetes CronJob +CMD ["node", "index.js"] \ No newline at end of file diff --git a/github-metrics/SCHEDULING.md b/github-metrics/SCHEDULING.md new file mode 100644 index 0000000..03912fd --- /dev/null +++ b/github-metrics/SCHEDULING.md @@ -0,0 +1,186 @@ +# GitHub Metrics Collection Scheduling + +## Overview + +The GitHub metrics collection job is designed to run **every 14 days** to collect repository metrics from GitHub. This interval ensures we capture metrics within GitHub's 14-day data retention window while avoiding unnecessary API calls. + +## How It Works + +### File-Based Tracking + +The system uses a file-based approach to track when the job last ran successfully: + +1. **Persistent Storage**: A Kubernetes persistent volume is mounted at `/data` to store the last run timestamp +2. **Last Run File**: The file `/data/last-run.json` contains the timestamp of the last successful run +3. **Automatic Checking**: Each time the cronjob executes, it checks if 14 days have passed since the last run +4. **Skip Logic**: If less than 14 days have passed, the job exits successfully without collecting metrics + +### Cronjob Schedule + +The Kubernetes cronjob is configured to run **every Sunday at 8am UTC** (`0 8 * * 0`). + +- The job runs weekly, but the application logic determines whether to actually collect metrics +- This approach is more reliable than trying to schedule exactly every 14 days with cron syntax +- If a run is missed (e.g., due to maintenance), the next weekly run will catch it + +### Example Timeline + +``` +Week 1, Sunday: Job runs → Collects metrics → Records timestamp +Week 2, Sunday: Job runs → Checks timestamp → Skips (only 7 days) +Week 3, Sunday: Job runs → Checks timestamp → Collects metrics (14+ days) → Records timestamp +Week 4, Sunday: Job runs → Checks timestamp → Skips (only 7 days) +Week 5, Sunday: Job runs → Checks timestamp → Collects metrics (14+ days) → Records timestamp +``` + +## Implementation Details + +### Last Run Tracker Module + +The `last-run-tracker.js` module provides three main functions: + +#### `shouldRunMetricsCollection()` +Checks if 14 days have passed since the last run. + +**Returns:** +```javascript +{ + shouldRun: boolean, // true if 14+ days have passed + lastRun: Date|null, // timestamp of last run + daysSinceLastRun: number|null // days since last run +} +``` + +#### `recordSuccessfulRun()` +Records the current timestamp as the last successful run. + +#### `getLastRunInfo()` +Gets information about the last run without checking if we should run (useful for debugging). + +### Last Run File Format + +The `/data/last-run.json` file contains: + +```json +{ + "lastRun": "2025-12-03T08:00:00.000Z", + "timestamp": 1733212800000 +} +``` + +## Configuration + +### Cronjob Configuration (`cronjobs.yml`) + +```yaml +persistence: + enabled: true + storageClass: "standard" + accessMode: ReadWriteOnce + size: 1Gi + mountPath: /data + +cronJobs: + - name: github-metrics-collection + schedule: "0 8 * * 0" # Every Sunday at 8am UTC + command: + - node + - index.js +``` + +### Key Configuration Points + +- **Persistent Volume**: Required to maintain state between cronjob executions +- **Mount Path**: `/data` - where the last run file is stored +- **Schedule**: Weekly execution allows the application to decide when to run +- **Exit Code**: The job exits with code 0 (success) even when skipping, so Kubernetes doesn't mark it as failed + +## Monitoring + +### Checking Last Run Status + +To check when the job last ran, you can: + +1. **View the last-run file** in the persistent volume: + ```bash + kubectl exec -it -n docs -- cat /data/last-run.json + ``` + +2. **Check job logs** for skip messages: + ```bash + kubectl logs -n docs -l job-name=github-metrics-collection --tail=50 + ``` + +### Expected Log Output + +**When running:** +``` +No previous run detected. This is the first run. +Starting metrics collection... +✓ Metrics collection completed successfully +✓ Recorded successful run at 2025-12-03T08:00:00.000Z +``` + +**When skipping:** +``` +Last run: 2025-12-03T08:00:00.000Z +Days since last run: 7 +✗ Only 7 days have passed. Skipping metrics collection. + Next run should occur in approximately 7 days. +Skipping metrics collection - not enough time has passed since last run. +Last run was 7 days ago on 2025-12-03T08:00:00.000Z +``` + +## Troubleshooting + +### Job Never Runs + +If the job keeps skipping even though 14+ days have passed: + +1. Check the last-run file timestamp: + ```bash + kubectl exec -it -n docs -- cat /data/last-run.json + ``` + +2. Manually delete the file to force a run: + ```bash + kubectl exec -it -n docs -- rm /data/last-run.json + ``` + +### Persistent Volume Issues + +If the persistent volume isn't working: + +1. Check if the PVC is bound: + ```bash + kubectl get pvc -n docs + ``` + +2. Check pod events for volume mount errors: + ```bash + kubectl describe pod -n docs + ``` + +### Force a Run + +To force the job to run immediately regardless of the last run time: + +1. Delete the last-run file: + ```bash + kubectl exec -it -n docs -- rm /data/last-run.json + ``` + +2. Manually trigger the cronjob: + ```bash + kubectl create job --from=cronjob/github-metrics-collection manual-run-$(date +%s) -n docs + ``` + +## Benefits of This Approach + +1. **Reliable 14-day interval**: Ensures metrics are collected every 14 days without complex cron syntax +2. **Resilient to missed runs**: If a run is missed, the next execution will catch it +3. **Simple to monitor**: Clear log messages indicate whether the job ran or skipped +4. **Easy to override**: Can force a run by deleting the last-run file +5. **Kubernetes-native**: Uses persistent volumes for state management +6. **No external dependencies**: Doesn't require a database or external service to track state + diff --git a/github-metrics/cronjobs.yml b/github-metrics/cronjobs.yml new file mode 100644 index 0000000..76d48c7 --- /dev/null +++ b/github-metrics/cronjobs.yml @@ -0,0 +1,44 @@ +--- +# `image` can be skipped if the values are being set in your .drone.yml file +image: + repository: 795250896452.dkr.ecr.us-east-1.amazonaws.com/docs/code-example-tooling + tag: latest + +# Service account configuration for IRSA (IAM Roles for Service Accounts) +serviceAccount: + enabled: true + irsa: + accountId: "216656347858" + roleName: devDocsCodeToolingServiceRole + +# global secrets are references to k8s Secrets +globalEnvSecrets: + GITHUB_TOKEN: github-token + ATLAS_CONNECTION_STRING: atlas-connection-string + +# Persistent volume for storing last run timestamp +# This allows the cronjob to track when it last ran successfully +persistence: + enabled: true + storageClass: "standard" + accessMode: ReadWriteOnce + size: 1Gi + mountPath: /data + +cronJobs: + - name: github-metrics-collection + # Run every Sunday at 8am UTC + # The job will check if 14 days have passed since the last run + # and skip execution if not enough time has elapsed + schedule: "0 8 * * 0" + command: + - node + - index.js + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + diff --git a/github-metrics/index.js b/github-metrics/index.js index 0616e92..4d7b081 100644 --- a/github-metrics/index.js +++ b/github-metrics/index.js @@ -2,6 +2,7 @@ import { readFile } from 'fs/promises'; import { getGitHubMetrics } from "./get-github-metrics.js"; import { addMetricsToAtlas } from "./write-to-db.js"; import { RepoDetails } from './RepoDetails.js'; // Import the RepoDetails class +import { shouldRunMetricsCollection, recordSuccessfulRun } from './last-run-tracker.js'; /* To change which repos to track metrics for, update the `repo-details.json` file. To track metrics for a new repo, add a new entry with the owner and repo name. @@ -16,6 +17,17 @@ NOTE: The GitHub token used to retrieve the info from a repo MUST have repo admi // processRepos reads the JSON config file and iterates through the repos specified, converting each to an instance of the RepoDetails class. async function processRepos() { try { + // Check if we should run based on last run time + const { shouldRun, lastRun, daysSinceLastRun } = await shouldRunMetricsCollection(); + + if (!shouldRun) { + console.log('Skipping metrics collection - not enough time has passed since last run.'); + console.log(`Last run was ${daysSinceLastRun} days ago on ${lastRun?.toISOString()}`); + process.exit(0); // Exit successfully without running + } + + console.log('Starting metrics collection...'); + // Read the JSON file const data = await readFile('repo-details.json', 'utf8'); @@ -36,8 +48,13 @@ async function processRepos() { } await addMetricsToAtlas(metricsDocs); + + // Record successful run + await recordSuccessfulRun(); + console.log('✓ Metrics collection completed successfully'); } catch (error) { console.error('Error processing repos:', error); + throw error; // Re-throw to ensure the job fails } } @@ -46,3 +63,4 @@ processRepos().catch(error => { console.error('Fatal error:', error); process.exit(1); }); + diff --git a/github-metrics/last-run-tracker.js b/github-metrics/last-run-tracker.js new file mode 100644 index 0000000..d2f58d8 --- /dev/null +++ b/github-metrics/last-run-tracker.js @@ -0,0 +1,128 @@ +import { readFile, writeFile, mkdir } from 'fs/promises'; +import { existsSync } from 'fs'; +import { join, dirname } from 'path'; + +const DATA_DIR = '/data'; +const LAST_RUN_FILE = join(DATA_DIR, 'last-run.json'); +const FOURTEEN_DAYS_MS = 14 * 24 * 60 * 60 * 1000; // 14 days in milliseconds + +/** + * Checks if enough time has passed since the last run. + * @returns {Promise<{shouldRun: boolean, lastRun: Date|null, daysSinceLastRun: number|null}>} + */ +export async function shouldRunMetricsCollection() { + try { + // Ensure data directory exists + if (!existsSync(DATA_DIR)) { + await mkdir(DATA_DIR, { recursive: true }); + } + + // Check if last run file exists + if (!existsSync(LAST_RUN_FILE)) { + console.log('No previous run detected. This is the first run.'); + return { + shouldRun: true, + lastRun: null, + daysSinceLastRun: null + }; + } + + // Read last run timestamp + const data = await readFile(LAST_RUN_FILE, 'utf8'); + const { lastRun } = JSON.parse(data); + const lastRunDate = new Date(lastRun); + const now = new Date(); + const timeSinceLastRun = now - lastRunDate; + const daysSinceLastRun = Math.floor(timeSinceLastRun / (24 * 60 * 60 * 1000)); + + console.log(`Last run: ${lastRunDate.toISOString()}`); + console.log(`Days since last run: ${daysSinceLastRun}`); + + if (timeSinceLastRun >= FOURTEEN_DAYS_MS) { + console.log(`✓ 14 days have passed. Proceeding with metrics collection.`); + return { + shouldRun: true, + lastRun: lastRunDate, + daysSinceLastRun + }; + } else { + const daysRemaining = Math.ceil((FOURTEEN_DAYS_MS - timeSinceLastRun) / (24 * 60 * 60 * 1000)); + console.log(`✗ Only ${daysSinceLastRun} days have passed. Skipping metrics collection.`); + console.log(` Next run should occur in approximately ${daysRemaining} days.`); + return { + shouldRun: false, + lastRun: lastRunDate, + daysSinceLastRun + }; + } + } catch (error) { + console.error('Error checking last run time:', error); + // If there's an error reading the file, assume we should run + console.log('Error reading last run file. Proceeding with metrics collection.'); + return { + shouldRun: true, + lastRun: null, + daysSinceLastRun: null + }; + } +} + +/** + * Records the current timestamp as the last successful run. + * @returns {Promise} + */ +export async function recordSuccessfulRun() { + try { + // Ensure data directory exists + if (!existsSync(DATA_DIR)) { + await mkdir(DATA_DIR, { recursive: true }); + } + + const now = new Date(); + const data = { + lastRun: now.toISOString(), + timestamp: now.getTime() + }; + + await writeFile(LAST_RUN_FILE, JSON.stringify(data, null, 2), 'utf8'); + console.log(`✓ Recorded successful run at ${now.toISOString()}`); + } catch (error) { + console.error('Error recording last run time:', error); + // Don't throw - we don't want to fail the entire job just because we couldn't write the file + } +} + +/** + * Gets information about the last run without checking if we should run. + * Useful for debugging and monitoring. + * @returns {Promise<{lastRun: Date|null, daysSinceLastRun: number|null}>} + */ +export async function getLastRunInfo() { + try { + if (!existsSync(LAST_RUN_FILE)) { + return { + lastRun: null, + daysSinceLastRun: null + }; + } + + const data = await readFile(LAST_RUN_FILE, 'utf8'); + const { lastRun } = JSON.parse(data); + const lastRunDate = new Date(lastRun); + const now = new Date(); + const timeSinceLastRun = now - lastRunDate; + const daysSinceLastRun = Math.floor(timeSinceLastRun / (24 * 60 * 60 * 1000)); + + return { + lastRun: lastRunDate, + daysSinceLastRun + }; + } catch (error) { + console.error('Error getting last run info:', error); + return { + lastRun: null, + daysSinceLastRun: null + }; + } +} +