diff --git a/.env b/.env index 64d1c05..ce52c9d 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ # DEBUG=true -POSTGRES_URL=postgres://postgres:123@localhost:5432/umami_dev +POSTGRES_URL=postgres://xetera@localhost:5432/postgres LOG_PATH=/tmp/postgres_logs/postgres.log # STATISTICS_PATH=test/umami_test.json STATISTICS_PATH=statistics.json diff --git a/.github/workflows/build-action.yaml b/.github/workflows/build-action.yaml index af33bb7..b7a39be 100644 --- a/.github/workflows/build-action.yaml +++ b/.github/workflows/build-action.yaml @@ -4,8 +4,6 @@ name: Release Action on: workflow_dispatch: push: - branches: - - main jobs: release: @@ -27,8 +25,12 @@ jobs: - name: Typecheck run: deno check + - name: Run tests + run: deno run test + - name: Upload files to release uses: softprops/action-gh-release@v2 + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} with: tag_name: v${{ env.DENO_VERSION }} env: diff --git a/.github/workflows/custom.yaml b/.github/workflows/custom.yaml index e207ee2..a5e9735 100644 --- a/.github/workflows/custom.yaml +++ b/.github/workflows/custom.yaml @@ -45,5 +45,5 @@ jobs: uses: ./ env: GITHUB_TOKEN: ${{ github.token }} - POSTGRES_URL: http://query_doctor@localhost:5432/testing + POSTGRES_URL: postgres://query_doctor@localhost:5432/testing LOG_PATH: /var/log/postgresql/postgres.log diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 7f0b5f5..efe4046 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -50,7 +50,7 @@ jobs: with: context: . file: Dockerfile - push: ${{ github.event_name == 'push' }} + push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} # outputs: type=docker platforms: linux/amd64,linux/arm64 tags: | @@ -69,7 +69,7 @@ jobs: mv /tmp/.buildx-cache-new /tmp/.buildx-cache - name: Attest uses: actions/attest-build-provenance@v2 - if: ${{ github.event_name == 'push' }} + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} with: subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} subject-digest: ${{ steps.build.outputs.digest }} diff --git a/.gitignore b/.gitignore index 5525035..72ad066 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ devenv.local.yaml # pre-commit .pre-commit-config.yaml .zed + +node_modules diff --git a/action.yaml b/action.yaml index d8a65b0..e1c71df 100644 --- a/action.yaml +++ b/action.yaml @@ -51,7 +51,6 @@ runs: # Run the compiled application - name: Run Analyzer shell: bash - working-directory: ${{ github.action_path }} run: deno run start env: PG_DUMP_BINARY: /usr/bin/pg_dump diff --git a/deno.json b/deno.json index 417bf7a..4a77419 100644 --- a/deno.json +++ b/deno.json @@ -1,9 +1,8 @@ { "$schema": "https://raw.githubusercontent.com/denoland/deno/refs/heads/main/cli/schemas/config-file.v1.json", "version": "0.1.1", - "unstable": [ - "raw-imports" - ], + "unstable": ["raw-imports"], + "nodeModulesDir": "auto", "permissions": { "run": { "sys": true, @@ -17,22 +16,26 @@ "tasks": { "start": "deno run --permission-set=run src/main.ts", "start:dev": "deno run --env-file=.env -A src/main.ts", + "test": "deno test --permission-set=run src", "dev": "deno run --env-file=.env --permission-set=run --watch src/main.ts" }, "imports": { "@actions/core": "npm:@actions/core@^1.11.1", "@actions/github": "npm:@actions/github@^6.0.1", - "@libpg-query/parser": "npm:@libpg-query/parser@^17.7.0", + "@libpg-query/parser": "npm:@libpg-query/parser@^17.6.3", "@opentelemetry/api": "jsr:@opentelemetry/api@^1.9.0", "@pgsql/types": "npm:@pgsql/types@^17.6.1", - "@query-doctor/core": "npm:@query-doctor/core@^0.0.3", + "@query-doctor/core": "npm:@query-doctor/core@^0.1.0", "@rabbit-company/rate-limiter": "jsr:@rabbit-company/rate-limiter@^3.0.0", "@std/assert": "jsr:@std/assert@^1.0.14", "@std/collections": "jsr:@std/collections@^1.1.3", "@std/data-structures": "jsr:@std/data-structures@^1.0.9", "@std/fmt": "jsr:@std/fmt@^1.0.8", + "@std/testing": "jsr:@std/testing@^1.0.16", + "@testcontainers/postgresql": "npm:@testcontainers/postgresql@^11.9.0", "@types/node": "npm:@types/node@^24.9.1", "@types/nunjucks": "npm:@types/nunjucks@^3.2.6", + "async-sema": "npm:async-sema@^3.1.1", "chokidar": "npm:chokidar@^4.0.3", "dedent": "npm:dedent@^1.6.0", "fast-csv": "npm:fast-csv@^5.0.5", diff --git a/deno.lock b/deno.lock index 50955c6..62bd55b 100644 --- a/deno.lock +++ b/deno.lock @@ -5,28 +5,35 @@ "jsr:@rabbit-company/rate-limiter@3": "3.0.0", "jsr:@std/assert@^1.0.13": "1.0.14", "jsr:@std/assert@^1.0.14": "1.0.14", + "jsr:@std/assert@^1.0.15": "1.0.16", + "jsr:@std/async@^1.0.15": "1.0.15", "jsr:@std/collections@^1.1.3": "1.1.3", "jsr:@std/data-structures@^1.0.9": "1.0.9", "jsr:@std/fmt@^1.0.8": "1.0.8", + "jsr:@std/fs@^1.0.19": "1.0.20", "jsr:@std/internal@^1.0.10": "1.0.10", - "npm:@actions/core@*": "1.11.1", + "jsr:@std/internal@^1.0.12": "1.0.12", + "jsr:@std/path@^1.1.2": "1.1.3", + "jsr:@std/path@^1.1.3": "1.1.3", + "jsr:@std/testing@^1.0.16": "1.0.16", "npm:@actions/core@^1.11.1": "1.11.1", - "npm:@actions/github@*": "6.0.1_@octokit+core@5.2.1", - "npm:@actions/github@^6.0.1": "6.0.1_@octokit+core@5.2.1", - "npm:@libpg-query/parser@^17.7.0": "17.7.0", + "npm:@actions/github@^6.0.1": "6.0.1_@octokit+core@5.2.2", + "npm:@libpg-query/parser@^17.6.3": "17.6.3", "npm:@pgsql/types@^17.6.1": "17.6.1", - "npm:@query-doctor/core@^0.0.3": "0.0.3", - "npm:@types/node@^24.9.1": "24.9.1", + "npm:@query-doctor/core@0.1": "0.1.0", + "npm:@testcontainers/postgresql@^11.9.0": "11.9.0", + "npm:@types/node@^24.9.1": "24.10.1", "npm:@types/nunjucks@^3.2.6": "3.2.6", + "npm:async-sema@^3.1.1": "3.1.1", "npm:chokidar@^4.0.3": "4.0.3", - "npm:dedent@^1.6.0": "1.6.0", + "npm:dedent@^1.6.0": "1.7.0", "npm:fast-csv@^5.0.5": "5.0.5", "npm:jsondiffpatch@~0.7.3": "0.7.3", - "npm:nunjucks@^3.2.4": "3.2.4", - "npm:pgsql-deparser@^17.11.1": "17.11.1", + "npm:nunjucks@^3.2.4": "3.2.4_chokidar@4.0.3", + "npm:pgsql-deparser@^17.11.1": "17.12.1", "npm:sql-formatter@^15.6.6": "15.6.6", "npm:sql-highlight@^6.1.0": "6.1.0", - "npm:zod@^4.1.12": "4.1.12" + "npm:zod@^4.1.12": "4.1.13" }, "jsr": { "@opentelemetry/api@1.9.0": { @@ -38,9 +45,18 @@ "@std/assert@1.0.14": { "integrity": "68d0d4a43b365abc927f45a9b85c639ea18a9fab96ad92281e493e4ed84abaa4", "dependencies": [ - "jsr:@std/internal" + "jsr:@std/internal@^1.0.10" ] }, + "@std/assert@1.0.16": { + "integrity": "6a7272ed1eaa77defe76e5ff63ca705d9c495077e2d5fd0126d2b53fc5bd6532", + "dependencies": [ + "jsr:@std/internal@^1.0.12" + ] + }, + "@std/async@1.0.15": { + "integrity": "55d1d9d04f99403fe5730ab16bdcc3c47f658a6bf054cafb38a50f046238116e" + }, "@std/collections@1.1.3": { "integrity": "bf8b0818886df6a32b64c7d3b037a425111f28278d69fd0995aeb62777c986b0" }, @@ -53,8 +69,34 @@ "@std/fmt@1.0.8": { "integrity": "71e1fc498787e4434d213647a6e43e794af4fd393ef8f52062246e06f7e372b7" }, + "@std/fs@1.0.20": { + "integrity": "e953206aae48d46ee65e8783ded459f23bec7dd1f3879512911c35e5484ea187", + "dependencies": [ + "jsr:@std/path@^1.1.3" + ] + }, "@std/internal@1.0.10": { "integrity": "e3be62ce42cab0e177c27698e5d9800122f67b766a0bea6ca4867886cbde8cf7" + }, + "@std/internal@1.0.12": { + "integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027" + }, + "@std/path@1.1.3": { + "integrity": "b015962d82a5e6daea980c32b82d2c40142149639968549c649031a230b1afb3", + "dependencies": [ + "jsr:@std/internal@^1.0.12" + ] + }, + "@std/testing@1.0.16": { + "integrity": "a917ffdeb5924c9be436dc78bc32e511760e14d3a96e49c607fc5ecca86d0092", + "dependencies": [ + "jsr:@std/assert@^1.0.15", + "jsr:@std/async", + "jsr:@std/data-structures", + "jsr:@std/fs", + "jsr:@std/internal@^1.0.12", + "jsr:@std/path@^1.1.2" + ] } }, "npm": { @@ -71,7 +113,7 @@ "@actions/io" ] }, - "@actions/github@6.0.1_@octokit+core@5.2.1": { + "@actions/github@6.0.1_@octokit+core@5.2.2": { "integrity": "sha512-xbZVcaqD4XnQAe35qSQqskb3SqIAfRyLBrHMd/8TuL7hJSz2QtbDwnNM8zWx4zO5l2fnGtseNE3MbEvD7BxVMw==", "dependencies": [ "@actions/http-client", @@ -80,147 +122,25 @@ "@octokit/plugin-rest-endpoint-methods", "@octokit/request", "@octokit/request-error", - "undici" + "undici@5.29.0" ] }, "@actions/http-client@2.2.3": { "integrity": "sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA==", "dependencies": [ "tunnel", - "undici" + "undici@5.29.0" ] }, "@actions/io@1.1.3": { "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==" }, + "@balena/dockerignore@1.0.2": { + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==" + }, "@dmsnell/diff-match-patch@1.1.0": { "integrity": "sha512-yejLPmM5pjsGvxS9gXablUSbInW7H976c/FJ4iQxWIm7/38xBySRemTPDe34lhg1gVLbJntX0+sH0jYfU+PN9A==" }, - "@esbuild/aix-ppc64@0.25.5": { - "integrity": "sha512-9o3TMmpmftaCMepOdA5k/yDw8SfInyzWWTjYTFCX3kPSDJMROQTb8jg+h9Cnwnmm1vOzvxN7gIfB5V2ewpjtGA==", - "os": ["aix"], - "cpu": ["ppc64"] - }, - "@esbuild/android-arm64@0.25.5": { - "integrity": "sha512-VGzGhj4lJO+TVGV1v8ntCZWJktV7SGCs3Pn1GRWI1SBFtRALoomm8k5E9Pmwg3HOAal2VDc2F9+PM/rEY6oIDg==", - "os": ["android"], - "cpu": ["arm64"] - }, - "@esbuild/android-arm@0.25.5": { - "integrity": "sha512-AdJKSPeEHgi7/ZhuIPtcQKr5RQdo6OO2IL87JkianiMYMPbCtot9fxPbrMiBADOWWm3T2si9stAiVsGbTQFkbA==", - "os": ["android"], - "cpu": ["arm"] - }, - "@esbuild/android-x64@0.25.5": { - "integrity": "sha512-D2GyJT1kjvO//drbRT3Hib9XPwQeWd9vZoBJn+bu/lVsOZ13cqNdDeqIF/xQ5/VmWvMduP6AmXvylO/PIc2isw==", - "os": ["android"], - "cpu": ["x64"] - }, - "@esbuild/darwin-arm64@0.25.5": { - "integrity": "sha512-GtaBgammVvdF7aPIgH2jxMDdivezgFu6iKpmT+48+F8Hhg5J/sfnDieg0aeG/jfSvkYQU2/pceFPDKlqZzwnfQ==", - "os": ["darwin"], - "cpu": ["arm64"] - }, - "@esbuild/darwin-x64@0.25.5": { - "integrity": "sha512-1iT4FVL0dJ76/q1wd7XDsXrSW+oLoquptvh4CLR4kITDtqi2e/xwXwdCVH8hVHU43wgJdsq7Gxuzcs6Iq/7bxQ==", - "os": ["darwin"], - "cpu": ["x64"] - }, - "@esbuild/freebsd-arm64@0.25.5": { - "integrity": "sha512-nk4tGP3JThz4La38Uy/gzyXtpkPW8zSAmoUhK9xKKXdBCzKODMc2adkB2+8om9BDYugz+uGV7sLmpTYzvmz6Sw==", - "os": ["freebsd"], - "cpu": ["arm64"] - }, - "@esbuild/freebsd-x64@0.25.5": { - "integrity": "sha512-PrikaNjiXdR2laW6OIjlbeuCPrPaAl0IwPIaRv+SMV8CiM8i2LqVUHFC1+8eORgWyY7yhQY+2U2fA55mBzReaw==", - "os": ["freebsd"], - "cpu": ["x64"] - }, - "@esbuild/linux-arm64@0.25.5": { - "integrity": "sha512-Z9kfb1v6ZlGbWj8EJk9T6czVEjjq2ntSYLY2cw6pAZl4oKtfgQuS4HOq41M/BcoLPzrUbNd+R4BXFyH//nHxVg==", - "os": ["linux"], - "cpu": ["arm64"] - }, - "@esbuild/linux-arm@0.25.5": { - "integrity": "sha512-cPzojwW2okgh7ZlRpcBEtsX7WBuqbLrNXqLU89GxWbNt6uIg78ET82qifUy3W6OVww6ZWobWub5oqZOVtwolfw==", - "os": ["linux"], - "cpu": ["arm"] - }, - "@esbuild/linux-ia32@0.25.5": { - "integrity": "sha512-sQ7l00M8bSv36GLV95BVAdhJ2QsIbCuCjh/uYrWiMQSUuV+LpXwIqhgJDcvMTj+VsQmqAHL2yYaasENvJ7CDKA==", - "os": ["linux"], - "cpu": ["ia32"] - }, - "@esbuild/linux-loong64@0.25.5": { - "integrity": "sha512-0ur7ae16hDUC4OL5iEnDb0tZHDxYmuQyhKhsPBV8f99f6Z9KQM02g33f93rNH5A30agMS46u2HP6qTdEt6Q1kg==", - "os": ["linux"], - "cpu": ["loong64"] - }, - "@esbuild/linux-mips64el@0.25.5": { - "integrity": "sha512-kB/66P1OsHO5zLz0i6X0RxlQ+3cu0mkxS3TKFvkb5lin6uwZ/ttOkP3Z8lfR9mJOBk14ZwZ9182SIIWFGNmqmg==", - "os": ["linux"], - "cpu": ["mips64el"] - }, - "@esbuild/linux-ppc64@0.25.5": { - "integrity": "sha512-UZCmJ7r9X2fe2D6jBmkLBMQetXPXIsZjQJCjgwpVDz+YMcS6oFR27alkgGv3Oqkv07bxdvw7fyB71/olceJhkQ==", - "os": ["linux"], - "cpu": ["ppc64"] - }, - "@esbuild/linux-riscv64@0.25.5": { - "integrity": "sha512-kTxwu4mLyeOlsVIFPfQo+fQJAV9mh24xL+y+Bm6ej067sYANjyEw1dNHmvoqxJUCMnkBdKpvOn0Ahql6+4VyeA==", - "os": ["linux"], - "cpu": ["riscv64"] - }, - "@esbuild/linux-s390x@0.25.5": { - "integrity": "sha512-K2dSKTKfmdh78uJ3NcWFiqyRrimfdinS5ErLSn3vluHNeHVnBAFWC8a4X5N+7FgVE1EjXS1QDZbpqZBjfrqMTQ==", - "os": ["linux"], - "cpu": ["s390x"] - }, - "@esbuild/linux-x64@0.25.5": { - "integrity": "sha512-uhj8N2obKTE6pSZ+aMUbqq+1nXxNjZIIjCjGLfsWvVpy7gKCOL6rsY1MhRh9zLtUtAI7vpgLMK6DxjO8Qm9lJw==", - "os": ["linux"], - "cpu": ["x64"] - }, - "@esbuild/netbsd-arm64@0.25.5": { - "integrity": "sha512-pwHtMP9viAy1oHPvgxtOv+OkduK5ugofNTVDilIzBLpoWAM16r7b/mxBvfpuQDpRQFMfuVr5aLcn4yveGvBZvw==", - "os": ["netbsd"], - "cpu": ["arm64"] - }, - "@esbuild/netbsd-x64@0.25.5": { - "integrity": "sha512-WOb5fKrvVTRMfWFNCroYWWklbnXH0Q5rZppjq0vQIdlsQKuw6mdSihwSo4RV/YdQ5UCKKvBy7/0ZZYLBZKIbwQ==", - "os": ["netbsd"], - "cpu": ["x64"] - }, - "@esbuild/openbsd-arm64@0.25.5": { - "integrity": "sha512-7A208+uQKgTxHd0G0uqZO8UjK2R0DDb4fDmERtARjSHWxqMTye4Erz4zZafx7Di9Cv+lNHYuncAkiGFySoD+Mw==", - "os": ["openbsd"], - "cpu": ["arm64"] - }, - "@esbuild/openbsd-x64@0.25.5": { - "integrity": "sha512-G4hE405ErTWraiZ8UiSoesH8DaCsMm0Cay4fsFWOOUcz8b8rC6uCvnagr+gnioEjWn0wC+o1/TAHt+It+MpIMg==", - "os": ["openbsd"], - "cpu": ["x64"] - }, - "@esbuild/sunos-x64@0.25.5": { - "integrity": "sha512-l+azKShMy7FxzY0Rj4RCt5VD/q8mG/e+mDivgspo+yL8zW7qEwctQ6YqKX34DTEleFAvCIUviCFX1SDZRSyMQA==", - "os": ["sunos"], - "cpu": ["x64"] - }, - "@esbuild/win32-arm64@0.25.5": { - "integrity": "sha512-O2S7SNZzdcFG7eFKgvwUEZ2VG9D/sn/eIiz8XRZ1Q/DO5a3s76Xv0mdBzVM5j5R639lXQmPmSo0iRpHqUUrsxw==", - "os": ["win32"], - "cpu": ["arm64"] - }, - "@esbuild/win32-ia32@0.25.5": { - "integrity": "sha512-onOJ02pqs9h1iMJ1PQphR+VZv8qBMQ77Klcsqv9CNW2w6yLqoURLcgERAIurY6QE63bbLuqgP9ATqajFLK5AMQ==", - "os": ["win32"], - "cpu": ["ia32"] - }, - "@esbuild/win32-x64@0.25.5": { - "integrity": "sha512-TXv6YnJ8ZMVdX+SXWVBo/0p8LTcrUYngpWjvm91TMjjBQii7Oz11Lw5lbDV5Y0TzuhSJHwiH4hEtC1I42mMS0g==", - "os": ["win32"], - "cpu": ["x64"] - }, "@fast-csv/format@5.0.5": { "integrity": "sha512-0P9SJXXnqKdmuWlLaTelqbrfdgN37Mvrb369J6eNmqL41IEIZQmV4sNM4GgAK2Dz3aH04J0HKGDMJFkYObThTw==", "dependencies": [ @@ -244,6 +164,47 @@ "@fastify/busboy@2.1.1": { "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==" }, + "@grpc/grpc-js@1.14.2": { + "integrity": "sha512-QzVUtEFyu05UNx2xr0fCQmStUO17uVQhGNowtxs00IgTZT6/W2PBLfUkj30s0FKJ29VtTa3ArVNIhNP6akQhqA==", + "dependencies": [ + "@grpc/proto-loader@0.8.0", + "@js-sdsl/ordered-map" + ] + }, + "@grpc/proto-loader@0.7.15": { + "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", + "dependencies": [ + "lodash.camelcase", + "long", + "protobufjs", + "yargs" + ], + "bin": true + }, + "@grpc/proto-loader@0.8.0": { + "integrity": "sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ==", + "dependencies": [ + "lodash.camelcase", + "long", + "protobufjs", + "yargs" + ], + "bin": true + }, + "@isaacs/cliui@8.0.2": { + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dependencies": [ + "string-width@5.1.2", + "string-width-cjs@npm:string-width@4.2.3", + "strip-ansi@7.1.2", + "strip-ansi-cjs@npm:strip-ansi@6.0.1", + "wrap-ansi@8.1.0", + "wrap-ansi-cjs@npm:wrap-ansi@7.0.0" + ] + }, + "@js-sdsl/ordered-map@4.4.2": { + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==" + }, "@launchql/protobufjs@7.2.6": { "integrity": "sha512-vwi1nG2/heVFsIMHQU1KxTjUp5c757CTtRAZn/jutApCkFlle1iv8tzM/DHlSZJKDldxaYqnNYTg0pTyp8Bbtg==", "dependencies": [ @@ -257,13 +218,13 @@ "@protobufjs/path", "@protobufjs/pool", "@protobufjs/utf8", - "@types/node", + "@types/node@24.10.1", "long" ], "scripts": true }, - "@libpg-query/parser@17.7.0": { - "integrity": "sha512-G+DA8enxveO3ESbLDhU7sAz02CPI16dKqXxPAvXbssHZnuBsaTpdHjm9CHC14ownvdujJ4CGwbs7kDUi3Lin9Q==", + "@libpg-query/parser@17.6.3": { + "integrity": "sha512-AvbS7b9GJZfCzqt4tLMqTaYK7Css9pJRTA2dKWLoTlob/XO2VNc30Q3g9DmxNBmokVTrmibkn/dy9bw8hfosQQ==", "dependencies": [ "@launchql/protobufjs", "@pgsql/types" @@ -272,8 +233,8 @@ "@octokit/auth-token@4.0.0": { "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==" }, - "@octokit/core@5.2.1": { - "integrity": "sha512-dKYCMuPO1bmrpuogcjQ8z7ICCH3FP6WmxpwC03yjzGfZhj9fTJg6+bS1+UAplekbN2C+M61UNllGOOoAfGCrdQ==", + "@octokit/core@5.2.2": { + "integrity": "sha512-/g2d4sW9nUDJOMz3mabVQvOGhVa4e/BN/Um7yca9Bb2XTzPPnfTWHWQg+IsEYO7M3Vx+EXvaM/I2pJWIMun1bg==", "dependencies": [ "@octokit/auth-token", "@octokit/graphql", @@ -305,14 +266,14 @@ "@octokit/openapi-types@24.2.0": { "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==" }, - "@octokit/plugin-paginate-rest@9.2.2_@octokit+core@5.2.1": { + "@octokit/plugin-paginate-rest@9.2.2_@octokit+core@5.2.2": { "integrity": "sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==", "dependencies": [ "@octokit/core", "@octokit/types@12.6.0" ] }, - "@octokit/plugin-rest-endpoint-methods@10.4.1_@octokit+core@5.2.1": { + "@octokit/plugin-rest-endpoint-methods@10.4.1_@octokit+core@5.2.2": { "integrity": "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==", "dependencies": [ "@octokit/core", @@ -351,6 +312,9 @@ "@pgsql/types@17.6.1": { "integrity": "sha512-Hk51+nyOxS7Dy5oySWywyNZxo5HndX1VDXT4ZEBD+p+vvMFM2Vc+sKSuByCiI8banou4edbgdnOC251IOuq7QQ==" }, + "@pkgjs/parseargs@0.11.0": { + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==" + }, "@protobufjs/aspromise@1.1.2": { "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" }, @@ -385,43 +349,253 @@ "@protobufjs/utf8@1.1.0": { "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, - "@query-doctor/core@0.0.3": { - "integrity": "sha512-krDTeLQNnBD3ZtM5cMLoxZhqPwFQMZaOgtJ3po4Tpq7O//ZPwAz4f/qIScpInFl29W35DlE6E6UrvO3Ud/4JGA==", + "@query-doctor/core@0.1.0": { + "integrity": "sha512-+rUn5IAXA8BRPm6GbJqdi6hfDj8Ei9kO7I7jCSvMJCILY/uSpPY4xQhkNXobNnO6EDVpcDb/D5dp9/lCpI2lGA==", "dependencies": [ "@pgsql/types", "colorette", - "dedent@1.7.0", + "dedent", "pgsql-deparser", "zod" ] }, - "@types/node@24.9.1": { - "integrity": "sha512-QoiaXANRkSXK6p0Duvt56W208du4P9Uye9hWLWgGMDTEoKPhuenzNcC4vGUmrNkiOKTlIrBoyNQYNpSwfEZXSg==", + "@testcontainers/postgresql@11.9.0": { + "integrity": "sha512-beLyLdLygFllktviM132Xd6tQ4i5FnuyZP+4BQEjUb5sJYHYnIrV/ZBzRRflIlF8gugt1GXgudkmr/HxM9vtKw==", + "dependencies": [ + "testcontainers" + ] + }, + "@types/docker-modem@3.0.6": { + "integrity": "sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==", + "dependencies": [ + "@types/node@24.2.0", + "@types/ssh2" + ] + }, + "@types/dockerode@3.3.47": { + "integrity": "sha512-ShM1mz7rCjdssXt7Xz0u1/R2BJC7piWa3SJpUBiVjCf2A3XNn4cP6pUVaD8bLanpPVVn4IKzJuw3dOvkJ8IbYw==", + "dependencies": [ + "@types/docker-modem", + "@types/node@24.2.0", + "@types/ssh2" + ] + }, + "@types/node@24.10.1": { + "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", "dependencies": [ - "undici-types" + "undici-types@7.16.0" + ] + }, + "@types/node@24.2.0": { + "integrity": "sha512-3xyG3pMCq3oYCNg7/ZP+E1ooTaGB4cG8JWRsqqOYQdbWNY4zbaV0Ennrd7stjiJEFZCaybcIgpTjJWHRfBSIDw==", + "dependencies": [ + "undici-types@7.10.0" ] }, "@types/nunjucks@3.2.6": { "integrity": "sha512-pHiGtf83na1nCzliuAdq8GowYiXvH5l931xZ0YEHaLMNFgynpEqx+IPStlu7UaDkehfvl01e4x/9Tpwhy7Ue3w==" }, + "@types/ssh2-streams@0.1.13": { + "integrity": "sha512-faHyY3brO9oLEA0QlcO8N2wT7R0+1sHWZvQ+y3rMLwdY1ZyS1z0W3t65j9PqT4HmQ6ALzNe7RZlNuCNE0wBSWA==", + "dependencies": [ + "@types/node@24.2.0" + ] + }, + "@types/ssh2@0.5.52": { + "integrity": "sha512-lbLLlXxdCZOSJMCInKH2+9V/77ET2J6NPQHpFI0kda61Dd1KglJs+fPQBchizmzYSOJBgdTajhPqBO1xxLywvg==", + "dependencies": [ + "@types/node@24.2.0", + "@types/ssh2-streams" + ] + }, "a-sync-waterfall@1.0.1": { "integrity": "sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA==" }, + "abort-controller@3.0.0": { + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": [ + "event-target-shim" + ] + }, + "ansi-regex@5.0.1": { + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "ansi-regex@6.2.2": { + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==" + }, + "ansi-styles@4.3.0": { + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": [ + "color-convert" + ] + }, + "ansi-styles@6.2.3": { + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==" + }, + "archiver-utils@5.0.2": { + "integrity": "sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==", + "dependencies": [ + "glob", + "graceful-fs", + "is-stream", + "lazystream", + "lodash", + "normalize-path", + "readable-stream@4.7.0" + ] + }, + "archiver@7.0.1": { + "integrity": "sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==", + "dependencies": [ + "archiver-utils", + "async", + "buffer-crc32", + "readable-stream@4.7.0", + "readdir-glob", + "tar-stream@3.1.7", + "zip-stream" + ] + }, "argparse@2.0.1": { "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, "asap@2.0.6": { "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" }, + "asn1@0.2.6": { + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "dependencies": [ + "safer-buffer" + ] + }, + "async-lock@1.4.1": { + "integrity": "sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==" + }, + "async-sema@3.1.1": { + "integrity": "sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg==" + }, + "async@3.2.6": { + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==" + }, + "b4a@1.7.3": { + "integrity": "sha512-5Q2mfq2WfGuFp3uS//0s6baOJLMoVduPYVeNmDYxu5OUA1/cBfvr2RIS7vi62LdNj/urk1hfmj867I3qt6uZ7Q==" + }, + "balanced-match@1.0.2": { + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "bare-events@2.8.2": { + "integrity": "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==" + }, + "bare-fs@4.5.2_bare-events@2.8.2": { + "integrity": "sha512-veTnRzkb6aPHOvSKIOy60KzURfBdUflr5VReI+NSaPL6xf+XLdONQgZgpYvUuZLVQ8dCqxpBAudaOM1+KpAUxw==", + "dependencies": [ + "bare-events", + "bare-path", + "bare-stream", + "bare-url", + "fast-fifo" + ] + }, + "bare-os@3.6.2": { + "integrity": "sha512-T+V1+1srU2qYNBmJCXZkUY5vQ0B4FSlL3QDROnKQYOqeiQR8UbjNHlPa+TIbM4cuidiN9GaTaOZgSEgsvPbh5A==" + }, + "bare-path@3.0.0": { + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "dependencies": [ + "bare-os" + ] + }, + "bare-stream@2.7.0_bare-events@2.8.2": { + "integrity": "sha512-oyXQNicV1y8nc2aKffH+BUHFRXmx6VrPzlnaEvMhram0nPBrKcEdcyBg5r08D0i8VxngHFAiVyn1QKXpSG0B8A==", + "dependencies": [ + "bare-events", + "streamx" + ], + "optionalPeers": [ + "bare-events" + ] + }, + "bare-url@2.3.2": { + "integrity": "sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw==", + "dependencies": [ + "bare-path" + ] + }, + "base64-js@1.5.1": { + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "bcrypt-pbkdf@1.0.2": { + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "dependencies": [ + "tweetnacl" + ] + }, "before-after-hook@2.2.3": { "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==" }, + "bl@4.1.0": { + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": [ + "buffer@5.7.1", + "inherits", + "readable-stream@3.6.2" + ] + }, + "brace-expansion@2.0.2": { + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dependencies": [ + "balanced-match" + ] + }, + "buffer-crc32@1.0.0": { + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==" + }, + "buffer@5.7.1": { + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dependencies": [ + "base64-js", + "ieee754" + ] + }, + "buffer@6.0.3": { + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "dependencies": [ + "base64-js", + "ieee754" + ] + }, + "buildcheck@0.0.7": { + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==" + }, + "byline@5.0.0": { + "integrity": "sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==" + }, "chokidar@4.0.3": { "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", "dependencies": [ "readdirp" ] }, + "chownr@1.1.4": { + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "cliui@8.0.1": { + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": [ + "string-width@4.2.3", + "strip-ansi@6.0.1", + "wrap-ansi@7.0.0" + ] + }, + "color-convert@2.0.1": { + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": [ + "color-name" + ] + }, + "color-name@1.1.4": { + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, "colorette@2.0.20": { "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==" }, @@ -431,8 +605,51 @@ "commander@5.1.0": { "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==" }, - "dedent@1.6.0": { - "integrity": "sha512-F1Z+5UCFpmQUzJa11agbyPVMbpgT/qA3/SKyJ1jyBgm7dUcUEa8v9JwDkerSQXfakBwFljIxhOJqGkjUwZ9FSA==" + "compress-commons@6.0.2": { + "integrity": "sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==", + "dependencies": [ + "crc-32", + "crc32-stream", + "is-stream", + "normalize-path", + "readable-stream@4.7.0" + ] + }, + "core-util-is@1.0.3": { + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "cpu-features@0.0.10": { + "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==", + "dependencies": [ + "buildcheck", + "nan" + ], + "scripts": true + }, + "crc-32@1.2.2": { + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "bin": true + }, + "crc32-stream@6.0.0": { + "integrity": "sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==", + "dependencies": [ + "crc-32", + "readable-stream@4.7.0" + ] + }, + "cross-spawn@7.0.6": { + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dependencies": [ + "path-key", + "shebang-command", + "which" + ] + }, + "debug@4.4.3": { + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dependencies": [ + "ms" + ] }, "dedent@1.7.0": { "integrity": "sha512-HGFtf8yhuhGhqO07SV79tRp+br4MnbdjeVxotpn1QBl30pcLLCQjX5b2295ll0fv8RKDKsmWYrl05usHM9CewQ==" @@ -443,12 +660,63 @@ "discontinuous-range@1.0.0": { "integrity": "sha512-c68LpLbO+7kP/b1Hr1qs8/BJ09F5khZGTxqxZuhzxpmwJKOgRFHJWIb9/KmqnqHhLdO55aOxFH/EGBvUQbL/RQ==" }, - "encoding@0.1.13": { - "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "docker-compose@1.3.0": { + "integrity": "sha512-7Gevk/5eGD50+eMD+XDnFnOrruFkL0kSd7jEG4cjmqweDSUhB7i0g8is/nBdVpl+Bx338SqIB2GLKm32M+Vs6g==", + "dependencies": [ + "yaml" + ] + }, + "docker-modem@5.0.6": { + "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==", + "dependencies": [ + "debug", + "readable-stream@3.6.2", + "split-ca", + "ssh2" + ] + }, + "dockerode@4.0.9": { + "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==", + "dependencies": [ + "@balena/dockerignore", + "@grpc/grpc-js", + "@grpc/proto-loader@0.7.15", + "docker-modem", + "protobufjs", + "tar-fs@2.1.4", + "uuid" + ] + }, + "eastasianwidth@0.2.0": { + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" + }, + "emoji-regex@8.0.0": { + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "emoji-regex@9.2.2": { + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "end-of-stream@1.4.5": { + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dependencies": [ + "once" + ] + }, + "escalade@3.2.0": { + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==" + }, + "event-target-shim@5.0.1": { + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" + }, + "events-universal@1.0.1": { + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", "dependencies": [ - "iconv-lite" + "bare-events" ] }, + "events@3.3.0": { + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" + }, "fast-csv@5.0.5": { "integrity": "sha512-9//QpogDIPln5Dc8e3Q3vbSSLXlTeU7z1JqsUOXZYOln8EIn/OOO8+NS2c3ukR6oYngDd3+P1HXSkby3kNV9KA==", "dependencies": [ @@ -456,15 +724,65 @@ "@fast-csv/parse" ] }, - "fsevents@2.3.3": { - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "os": ["darwin"], - "scripts": true + "fast-fifo@1.3.2": { + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" }, - "iconv-lite@0.6.3": { - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "foreground-child@3.3.1": { + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", "dependencies": [ - "safer-buffer" + "cross-spawn", + "signal-exit@4.1.0" + ] + }, + "fs-constants@1.0.0": { + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, + "get-caller-file@2.0.5": { + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + }, + "get-port@7.1.0": { + "integrity": "sha512-QB9NKEeDg3xxVwCCwJQ9+xycaz6pBB6iQ76wiWMl1927n0Kir6alPiP+yuiICLLU4jpMe08dXfpebuQppFA2zw==" + }, + "glob@10.5.0": { + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "dependencies": [ + "foreground-child", + "jackspeak", + "minimatch@9.0.5", + "minipass", + "package-json-from-dist", + "path-scurry" + ], + "bin": true + }, + "graceful-fs@4.2.11": { + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" + }, + "ieee754@1.2.1": { + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, + "inherits@2.0.4": { + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "is-fullwidth-code-point@3.0.0": { + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "is-stream@2.0.1": { + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" + }, + "isarray@1.0.0": { + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "isexe@2.0.0": { + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "jackspeak@3.4.3": { + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dependencies": [ + "@isaacs/cliui" + ], + "optionalDependencies": [ + "@pkgjs/parseargs" ] }, "jsondiffpatch@0.7.3": { @@ -474,6 +792,15 @@ ], "bin": true }, + "lazystream@1.0.1": { + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "dependencies": [ + "readable-stream@2.3.8" + ] + }, + "lodash.camelcase@4.3.0": { + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" + }, "lodash.escaperegexp@4.1.2": { "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==" }, @@ -495,12 +822,46 @@ "lodash.uniq@4.5.0": { "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==" }, + "lodash@4.17.21": { + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, "long@5.3.2": { "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==" }, + "lru-cache@10.4.3": { + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" + }, + "minimatch@5.1.6": { + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": [ + "brace-expansion" + ] + }, + "minimatch@9.0.5": { + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dependencies": [ + "brace-expansion" + ] + }, + "minipass@7.1.2": { + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==" + }, + "mkdirp-classic@0.5.3": { + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, + "mkdirp@1.0.4": { + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "bin": true + }, "moo@0.5.2": { "integrity": "sha512-iSAJLHYKnX41mKcJKjqvnAN9sf0LMDTXDEvFv+ffuRR9a1MIuXLjMNL6EsnDHSkKLTWNqQQ5uo61P4EbU4NU+Q==" }, + "ms@2.1.3": { + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "nan@2.24.0": { + "integrity": "sha512-Vpf9qnVW1RaDkoNKFUvfxqAbtI8ncb8OJlqZ9wwpXzWPEsvsB1nvdUi6oYrHIkQ1Y/tMDnr1h4nczS0VB9Xykg==" + }, "nearley@2.20.1": { "integrity": "sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ==", "dependencies": [ @@ -511,13 +872,20 @@ ], "bin": true }, - "nunjucks@3.2.4": { + "normalize-path@3.0.0": { + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" + }, + "nunjucks@3.2.4_chokidar@4.0.3": { "integrity": "sha512-26XRV6BhkgK0VOxfbU5cQI+ICFUtMLixv1noZn1tGU38kQH5A5nmmbk/O45xdyBhD1esk47nKrY0mvQpZIhRjQ==", "dependencies": [ "a-sync-waterfall", "asap", + "chokidar", "commander@5.1.0" ], + "optionalPeers": [ + "chokidar" + ], "bin": true }, "once@1.4.0": { @@ -526,12 +894,70 @@ "wrappy" ] }, - "pgsql-deparser@17.11.1": { - "integrity": "sha512-BGKgwC4qs+FPcG8Ai989LO6i4E8KF5HEvlTnI8uhS4qUyu6P1xCyP9pJDky95ZL8DolaGUDFAJtxteDBw33OCg==", + "package-json-from-dist@1.0.1": { + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==" + }, + "path-key@3.1.1": { + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" + }, + "path-scurry@1.11.1": { + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dependencies": [ + "lru-cache", + "minipass" + ] + }, + "pgsql-deparser@17.12.1": { + "integrity": "sha512-G27wb4rhXNwaV8+/ni3RlGV+CMk753ErX341c4rY98hI8xPbPI07dYGxT3asur0HW4SyyNZ4cS1vhfwLfxPLaA==", "dependencies": [ "@pgsql/types" ] }, + "process-nextick-args@2.0.1": { + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "process@0.11.10": { + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==" + }, + "proper-lockfile@4.1.2": { + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "dependencies": [ + "graceful-fs", + "retry", + "signal-exit@3.0.7" + ] + }, + "properties-reader@2.3.0": { + "integrity": "sha512-z597WicA7nDZxK12kZqHr2TcvwNU1GCfA5UwfDY/HDp3hXPoPlb5rlEx9bwGTiJnc0OqbBTkU975jDToth8Gxw==", + "dependencies": [ + "mkdirp" + ] + }, + "protobufjs@7.5.4": { + "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "dependencies": [ + "@protobufjs/aspromise", + "@protobufjs/base64", + "@protobufjs/codegen", + "@protobufjs/eventemitter", + "@protobufjs/fetch", + "@protobufjs/float", + "@protobufjs/inquire", + "@protobufjs/path", + "@protobufjs/pool", + "@protobufjs/utf8", + "@types/node@24.10.1", + "long" + ], + "scripts": true + }, + "pump@3.0.3": { + "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", + "dependencies": [ + "end-of-stream", + "once" + ] + }, "railroad-diagrams@1.0.0": { "integrity": "sha512-cz93DjNeLY0idrCNOH6PviZGRN9GJhsdm9hpn1YCS879fj4W+x5IFJhhkRZcwVgMmFF7R82UA/7Oh+R8lLZg6A==" }, @@ -542,15 +968,81 @@ "ret" ] }, + "readable-stream@2.3.8": { + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": [ + "core-util-is", + "inherits", + "isarray", + "process-nextick-args", + "safe-buffer@5.1.2", + "string_decoder@1.1.1", + "util-deprecate" + ] + }, + "readable-stream@3.6.2": { + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": [ + "inherits", + "string_decoder@1.3.0", + "util-deprecate" + ] + }, + "readable-stream@4.7.0": { + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "dependencies": [ + "abort-controller", + "buffer@6.0.3", + "events", + "process", + "string_decoder@1.3.0" + ] + }, + "readdir-glob@1.1.3": { + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "dependencies": [ + "minimatch@5.1.6" + ] + }, "readdirp@4.1.2": { "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==" }, + "require-directory@2.1.1": { + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" + }, "ret@0.1.15": { "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==" }, + "retry@0.12.0": { + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==" + }, + "safe-buffer@5.1.2": { + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "safe-buffer@5.2.1": { + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, "safer-buffer@2.1.2": { "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "shebang-command@2.0.0": { + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dependencies": [ + "shebang-regex" + ] + }, + "shebang-regex@3.0.0": { + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" + }, + "signal-exit@3.0.7": { + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" + }, + "signal-exit@4.1.0": { + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==" + }, + "split-ca@1.0.1": { + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==" + }, "sql-formatter@15.6.6": { "integrity": "sha512-bZydXEXhaNDQBr8xYHC3a8thwcaMuTBp0CkKGjwGYDsIB26tnlWeWPwJtSQ0TEwiJcz9iJJON5mFPkx7XroHcg==", "dependencies": [ @@ -562,9 +1054,149 @@ "sql-highlight@6.1.0": { "integrity": "sha512-ed7OK4e9ywpE7pgRMkMQmZDPKSVdm0oX5IEtZiKnFucSF0zu6c80GZBe38UqHuVhTWJ9xsKgSMjCG2bml86KvA==" }, + "ssh-remote-port-forward@1.0.4": { + "integrity": "sha512-x0LV1eVDwjf1gmG7TTnfqIzf+3VPRz7vrNIjX6oYLbeCrf/PeVY6hkT68Mg+q02qXxQhrLjB0jfgvhevoCRmLQ==", + "dependencies": [ + "@types/ssh2", + "ssh2" + ] + }, + "ssh2@1.17.0": { + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", + "dependencies": [ + "asn1", + "bcrypt-pbkdf" + ], + "optionalDependencies": [ + "cpu-features", + "nan" + ], + "scripts": true + }, + "streamx@2.23.0": { + "integrity": "sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg==", + "dependencies": [ + "events-universal", + "fast-fifo", + "text-decoder" + ] + }, + "string-width@4.2.3": { + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": [ + "emoji-regex@8.0.0", + "is-fullwidth-code-point", + "strip-ansi@6.0.1" + ] + }, + "string-width@5.1.2": { + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dependencies": [ + "eastasianwidth", + "emoji-regex@9.2.2", + "strip-ansi@7.1.2" + ] + }, + "string_decoder@1.1.1": { + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": [ + "safe-buffer@5.1.2" + ] + }, + "string_decoder@1.3.0": { + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": [ + "safe-buffer@5.2.1" + ] + }, + "strip-ansi@6.0.1": { + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": [ + "ansi-regex@5.0.1" + ] + }, + "strip-ansi@7.1.2": { + "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", + "dependencies": [ + "ansi-regex@6.2.2" + ] + }, + "tar-fs@2.1.4": { + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "dependencies": [ + "chownr", + "mkdirp-classic", + "pump", + "tar-stream@2.2.0" + ] + }, + "tar-fs@3.1.1": { + "integrity": "sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==", + "dependencies": [ + "pump", + "tar-stream@3.1.7" + ], + "optionalDependencies": [ + "bare-fs", + "bare-path" + ] + }, + "tar-stream@2.2.0": { + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": [ + "bl", + "end-of-stream", + "fs-constants", + "inherits", + "readable-stream@3.6.2" + ] + }, + "tar-stream@3.1.7": { + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "dependencies": [ + "b4a", + "fast-fifo", + "streamx" + ] + }, + "testcontainers@11.9.0": { + "integrity": "sha512-SQ6OqQUig7HcGVF72i+ZVIMvxPSpEz8cgC/B63ekqMzgf98DnveoBbOmqux/Wa5wQAQCt4mEPNMa/Jz7vMg9fQ==", + "dependencies": [ + "@balena/dockerignore", + "@types/dockerode", + "archiver", + "async-lock", + "byline", + "debug", + "docker-compose", + "dockerode", + "get-port", + "proper-lockfile", + "properties-reader", + "ssh-remote-port-forward", + "tar-fs@3.1.1", + "tmp", + "undici@7.16.0" + ] + }, + "text-decoder@1.2.3": { + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "dependencies": [ + "b4a" + ] + }, + "tmp@0.2.5": { + "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==" + }, "tunnel@0.0.6": { "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" }, + "tweetnacl@0.14.5": { + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" + }, + "undici-types@7.10.0": { + "integrity": "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag==" + }, "undici-types@7.16.0": { "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==" }, @@ -574,14 +1206,77 @@ "@fastify/busboy" ] }, + "undici@7.16.0": { + "integrity": "sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g==" + }, "universal-user-agent@6.0.1": { "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==" }, + "util-deprecate@1.0.2": { + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "uuid@10.0.0": { + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "bin": true + }, + "which@2.0.2": { + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dependencies": [ + "isexe" + ], + "bin": true + }, + "wrap-ansi@7.0.0": { + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": [ + "ansi-styles@4.3.0", + "string-width@4.2.3", + "strip-ansi@6.0.1" + ] + }, + "wrap-ansi@8.1.0": { + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dependencies": [ + "ansi-styles@6.2.3", + "string-width@5.1.2", + "strip-ansi@7.1.2" + ] + }, "wrappy@1.0.2": { "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, - "zod@4.1.12": { - "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==" + "y18n@5.0.8": { + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" + }, + "yaml@2.8.2": { + "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", + "bin": true + }, + "yargs-parser@21.1.1": { + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==" + }, + "yargs@17.7.2": { + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dependencies": [ + "cliui", + "escalade", + "get-caller-file", + "require-directory", + "string-width@4.2.3", + "y18n", + "yargs-parser" + ] + }, + "zip-stream@6.0.1": { + "integrity": "sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==", + "dependencies": [ + "archiver-utils", + "compress-commons", + "readable-stream@4.7.0" + ] + }, + "zod@4.1.13": { + "integrity": "sha512-AvvthqfqrAhNH9dnfmrfKzX5upOdjUVJYFqNSlkmGf64gRaTzlPwz99IHYnVs28qYAybvAlBV+H7pn0saFY4Ig==" } }, "redirects": { @@ -955,13 +1650,16 @@ "jsr:@std/collections@^1.1.3", "jsr:@std/data-structures@^1.0.9", "jsr:@std/fmt@^1.0.8", + "jsr:@std/testing@^1.0.16", "npm:@actions/core@^1.11.1", "npm:@actions/github@^6.0.1", - "npm:@libpg-query/parser@^17.7.0", + "npm:@libpg-query/parser@^17.6.3", "npm:@pgsql/types@^17.6.1", - "npm:@query-doctor/core@^0.0.3", + "npm:@query-doctor/core@0.1", + "npm:@testcontainers/postgresql@^11.9.0", "npm:@types/node@^24.9.1", "npm:@types/nunjucks@^3.2.6", + "npm:async-sema@^3.1.1", "npm:chokidar@^4.0.3", "npm:dedent@^1.6.0", "npm:fast-csv@^5.0.5", diff --git a/devenv.lock b/devenv.lock index 49e179c..2d377d4 100644 --- a/devenv.lock +++ b/devenv.lock @@ -3,10 +3,10 @@ "devenv": { "locked": { "dir": "src/modules", - "lastModified": 1764227073, + "lastModified": 1765279746, "owner": "cachix", "repo": "devenv", - "rev": "ee868b9986b84b82fee40ecc2524340d4a154961", + "rev": "8241b5bedae223d439edc17eb8b2a6e31b40386d", "type": "github" }, "original": { @@ -19,10 +19,10 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1761588595, + "lastModified": 1765121682, "owner": "edolstra", "repo": "flake-compat", - "rev": "f387cd2afec9419c8ee37694406ca490c3f34ee5", + "rev": "65f23138d8d09a92e30f1e5c87611b23ef451bf3", "type": "github" }, "original": { @@ -40,10 +40,10 @@ ] }, "locked": { - "lastModified": 1763988335, + "lastModified": 1765016596, "owner": "cachix", "repo": "git-hooks.nix", - "rev": "50b9238891e388c9fdc6a5c49e49c42533a1b5ce", + "rev": "548fc44fca28a5e81c5d6b846e555e6b9c2a5a3c", "type": "github" }, "original": { @@ -74,10 +74,10 @@ }, "nixpkgs": { "locked": { - "lastModified": 1761313199, + "lastModified": 1764580874, "owner": "cachix", "repo": "devenv-nixpkgs", - "rev": "d1c30452ebecfc55185ae6d1c983c09da0c274ff", + "rev": "dcf61356c3ab25f1362b4a4428a6d871e84f1d1d", "type": "github" }, "original": { diff --git a/devenv.nix b/devenv.nix index 06705a4..10c5386 100644 --- a/devenv.nix +++ b/devenv.nix @@ -1,7 +1,6 @@ { pkgs, ... }: { - env.CI = "false"; dotenv.enable = true; packages = with pkgs; [ git diff --git a/src/env.ts b/src/env.ts index 160f7fb..f570937 100644 --- a/src/env.ts +++ b/src/env.ts @@ -5,6 +5,7 @@ const envSchema = z.object({ CI: z.stringbool().default(false), // sync PG_DUMP_BINARY: z.string().optional(), + PG_RESTORE_BINARY: z.string().optional(), HOSTED: z.stringbool().default(false), HOST: z.string().default("0.0.0.0"), PORT: z.coerce.number().min(1024).max(65535).default(2345), diff --git a/src/main.ts b/src/main.ts index 80b6ea7..1574ea3 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,12 +2,12 @@ import * as core from "@actions/core"; import { Runner } from "./runner.ts"; import { env } from "./env.ts"; import { log } from "./log.ts"; -import { PostgresSchemaLink } from "./sync/schema-link.ts"; import { createServer } from "./server/http.ts"; import { shutdown } from "./shutdown.ts"; +import { Connectable } from "./sync/connectable.ts"; async function runInCI( - postgresUrl: string, + postgresUrl: Connectable, logPath: string, statisticsPath?: string, maxCost?: number, @@ -29,11 +29,11 @@ function runOutsideCI() { `Starting server (${os}-${arch}) on ${env.HOST}:${env.PORT}`, "main", ); - log.info( - `Using pg_dump binary: ${PostgresSchemaLink.pgDumpBinaryPath}`, - "main", - ); - createServer(env.HOST, env.PORT); + if (!env.POSTGRES_URL) { + core.setFailed("POSTGRES_URL environment variable is not set"); + Deno.exit(1); + } + createServer(env.HOST, env.PORT, Connectable.fromString(env.POSTGRES_URL)); } async function main() { @@ -49,7 +49,7 @@ async function main() { Deno.exit(1); } await runInCI( - env.POSTGRES_URL, + Connectable.fromString(env.POSTGRES_URL), env.LOG_PATH, env.STATISTICS_PATH, typeof env.MAX_COST === "number" ? env.MAX_COST : undefined, diff --git a/src/remote/optimization.ts b/src/remote/optimization.ts new file mode 100644 index 0000000..054269d --- /dev/null +++ b/src/remote/optimization.ts @@ -0,0 +1,26 @@ +import z from "zod"; + +export const LiveQueryOptimization = z.discriminatedUnion("state", [ + z.object({ + state: z.literal("waiting"), + }), + z.object({ state: z.literal("optimizing") }), + z.object({ state: z.literal("not_supported"), reason: z.string() }), + z.object({ + state: z.literal("improvements_available"), + cost: z.number(), + optimizedCost: z.number(), + costReductionPercentage: z.number(), + indexRecommendations: z.array(z.string()), + indexesUsed: z.array(z.string()), + }), + z.object({ + state: z.literal("no_improvement_found"), + cost: z.number(), + indexesUsed: z.array(z.string()), + }), + z.object({ state: z.literal("timeout") }), + z.object({ state: z.literal("error"), error: z.instanceof(Error) }), +]); + +export type LiveQueryOptimization = z.infer; diff --git a/src/remote/query-optimizer.test.ts b/src/remote/query-optimizer.test.ts new file mode 100644 index 0000000..03d6cdf --- /dev/null +++ b/src/remote/query-optimizer.test.ts @@ -0,0 +1,150 @@ +import { PostgreSqlContainer } from "@testcontainers/postgresql"; +import { QueryOptimizer } from "./query-optimizer.ts"; +import { ConnectionManager } from "../sync/connection-manager.ts"; +import { Connectable } from "../sync/connectable.ts"; +import { setTimeout } from "node:timers/promises"; +import { assertArrayIncludes } from "@std/assert/array-includes"; +import { assert } from "@std/assert"; + +Deno.test({ + name: "controller syncs correctly", + sanitizeOps: false, + sanitizeResources: false, + fn: async () => { + const pg = await new PostgreSqlContainer("postgres:17") + .withCopyContentToContainer([ + { + content: ` + create table testing(a int, b text); + insert into testing (a, b) values (1, 'hello'); + create index "testing_index" on testing(b); + + -- normally should be in another db but + -- this makes testing much faster + create extension pg_stat_statements; + select * from testing where a = 10; + select * from testing where b = 'c'; + select * from testing where b > 'a'; + select * from testing where b < 'b'; + select * from pg_index where 1 = 1; + select * from pg_class where relname > 'example' /* @qd_introspection */; + `, + target: "/docker-entrypoint-initdb.d/init.sql", + }, + ]) + .withCommand([ + "-c", + "shared_preload_libraries=pg_stat_statements", + "-c", + "autovacuum=off", + "-c", + "track_counts=off", + "-c", + "track_io_timing=off", + "-c", + "track_activities=off", + ]) + .start(); + + const manager = ConnectionManager.forLocalDatabase(); + const optimizer = new QueryOptimizer(manager); + + const expectedImprovements = ["select * from testing where a = $1"]; + const expectedNoImprovements = [ + "select * from testing where b = $1", + "select * from testing where b > $1", + "select * from testing where b < $1", + ]; + + let improvements: string[] = []; + let noImprovements: string[] = []; + + optimizer.addListener("error", (query, error) => { + console.error("error when running query", query); + throw error; + }); + optimizer.addListener("improvementsAvailable", (query) => { + improvements.push(query.query); + }); + optimizer.addListener("noImprovements", (query) => { + noImprovements.push(query.query); + }); + + const conn = Connectable.fromString(pg.getConnectionUri()); + const connector = manager.getConnectorFor(conn); + try { + const recentQueries = await connector.getRecentQueries(); + const includedQueries = await optimizer.start(conn, recentQueries, { + kind: "fromStatisticsExport", + source: { kind: "inline" }, + stats: [{ + tableName: "testing", + schemaName: "public", + relpages: 56, + reltuples: 100_000, + relallvisible: 1, + columns: [{ + columnName: "a", + stats: null, + }, { + columnName: "b", + stats: null, + }], + indexes: [{ + indexName: "testing_index", + relpages: 2, + reltuples: 10000, + relallvisible: 1, + }], + }], + }); + // should ignore the query with + assert( + includedQueries.every((q) => + !q.query.startsWith("select * from pg_class where relname > $1") + ), + "Optimizer did not ignore a query with @qd_introspection", + ); + assert( + includedQueries.every((q) => + !q.query.startsWith("select * from pg_index where $1 = $2") + ), + "Optimizer did not ignore a system query", + ); + await setTimeout(1_000); + assertArrayIncludes(expectedImprovements, improvements); + assertArrayIncludes(expectedNoImprovements, noImprovements); + improvements = []; + noImprovements = []; + await optimizer.start(conn, recentQueries, { + kind: "fromStatisticsExport", + source: { kind: "inline" }, + stats: [{ + tableName: "testing", + schemaName: "public", + relpages: 1, + reltuples: 100, + relallvisible: 1, + columns: [{ + columnName: "a", + stats: null, + }, { + columnName: "b", + stats: null, + }], + indexes: [{ + indexName: "testing_index", + relpages: 2, + reltuples: 10000, + relallvisible: 1, + }], + }], + }); + await setTimeout(1_000); + console.log(improvements); + console.log(noImprovements); + } finally { + await pg.stop(); + } + }, +}); diff --git a/src/remote/query-optimizer.ts b/src/remote/query-optimizer.ts new file mode 100644 index 0000000..b75f41d --- /dev/null +++ b/src/remote/query-optimizer.ts @@ -0,0 +1,422 @@ +import EventEmitter from "node:events"; +import { OptimizedQuery, QueryHash, RecentQuery } from "../sql/recent-query.ts"; +import type { LiveQueryOptimization } from "./optimization.ts"; +import { ConnectionManager } from "../sync/connection-manager.ts"; +import { Sema } from "async-sema"; +import { + Analyzer, + IndexOptimizer, + OptimizeResult, + PostgresQueryBuilder, + PostgresVersion, + Statistics, + StatisticsMode, +} from "@query-doctor/core"; +import { Connectable } from "../sync/connectable.ts"; +import { parse } from "@libpg-query/parser"; + +const MINIMUM_COST_CHANGE_PERCENTAGE = 5; +const QUERY_TIMEOUT_MS = 10000; + +type EventMap = { + error: [Error, OptimizedQuery]; + timeout: [OptimizedQuery, number]; + zeroCostPlan: [OptimizedQuery]; + noImprovements: [OptimizedQuery]; + improvementsAvailable: [OptimizedQuery]; +}; + +type Target = { + connectable: Connectable; + optimizer: IndexOptimizer; + statistics: Statistics; +}; + +export class QueryOptimizer extends EventEmitter { + private static readonly MAX_CONCURRENCY = 1; + private static readonly defaultStatistics: StatisticsMode = { + kind: "fromAssumption", + relpages: 1, + reltuples: 10_000, + }; + private readonly queries = new Map(); + private target?: Target; + private semaphore = new Sema(QueryOptimizer.MAX_CONCURRENCY); + private _finish = Promise.withResolvers(); + + private _validQueriesProcessed = 0; + private _invalidQueries = 0; + private _allQueries = 0; + + constructor( + private readonly manager: ConnectionManager, + ) { + super(); + } + + get validQueriesProcessed() { + return this._validQueriesProcessed; + } + + get invalidQueries() { + return this._invalidQueries; + } + + get allQueries() { + return this._allQueries; + } + + get finish() { + return this._finish.promise; + } + + /** + * Start optimizing a new set of queries + * @returns Promise of array of queries that were considered for optimization. + * Resolves when all queries are optimized + */ + async start( + conn: Connectable, + allRecentQueries: RecentQuery[], + statsMode: StatisticsMode = QueryOptimizer.defaultStatistics, + ): Promise { + this.stop(); + const validQueries: OptimizedQuery[] = []; + for (const query of allRecentQueries) { + let optimization: LiveQueryOptimization; + const status = this.checkQueryUnsupported(query); + switch (status.type) { + case "ok": + optimization = { state: "waiting" }; + break; + case "not_supported": + optimization = this.onQueryUnsupported(status.reason); + break; + case "ignored": + continue; + } + const optimized = query.withOptimization(optimization); + + validQueries.push(optimized); + this.queries.set(query.hash, optimized); + } + const version = PostgresVersion.parse("17"); + const pg = this.manager.getOrCreateConnection(conn); + const ownStats = await Statistics.dumpStats(pg, version, "full"); + const statistics = new Statistics( + pg, + version, + ownStats, + statsMode, + ); + const existingIndexes = await statistics.getExistingIndexes(); + const optimizer = new IndexOptimizer(pg, statistics, existingIndexes, { + // we're not running on our pg fork (yet) + // so traces have to be disabled + trace: false, + }); + this.target = { connectable: conn, optimizer, statistics }; + + this._allQueries = this.queries.size; + await this.work(); + return validQueries; + } + + stop() { + this.semaphore = new Sema(QueryOptimizer.MAX_CONCURRENCY); + this.queries.clear(); + this.target = undefined; + this._allQueries = 0; + this._finish = Promise.withResolvers(); + this._invalidQueries = 0; + this._validQueriesProcessed = 0; + } + + private async work() { + if (!this.target) { + return; + } + + while (true) { + let optimized: OptimizedQuery | undefined; + const token = await this.semaphore.acquire(); + try { + for (const [hash, entry] of this.queries.entries()) { + if (entry.optimization.state !== "waiting") { + continue; + } + this.queries.set( + hash, + entry.withOptimization({ state: "optimizing" }), + ); + optimized = entry; + break; + } + } finally { + this.semaphore.release(token); + } + if (!optimized) { + this._finish.resolve(0); + break; + } + this._validQueriesProcessed++; + const optimization = await this.optimizeQuery( + optimized, + this.target, + ); + + this.queries.set( + optimized.hash, + optimized.withOptimization(optimization), + ); + } + } + + getQueries(): OptimizedQuery[] { + return Array.from(this.queries.values()); + } + + private checkQueryUnsupported( + query: RecentQuery, + ): { type: "ok" } | { type: "ignored" } | { + type: "not_supported"; + reason: string; + } { + if ( + query.isIntrospection || query.isSystemQuery || + query.isTargetlessSelectQuery + ) { + return { type: "ignored" }; + } + if (!query.isSelectQuery) { + return { + type: "not_supported", + reason: + "Only select statements are currently eligible for optimization", + }; + } + return { type: "ok" }; + } + + private async optimizeQuery( + recent: OptimizedQuery, + target: Target, + timeoutMs = QUERY_TIMEOUT_MS, + ): Promise { + const builder = new PostgresQueryBuilder(recent.query); + let cost: number; + try { + const explain = await withTimeout( + target.optimizer.testQueryWithStats(builder), + timeoutMs, + ); + cost = explain.Plan["Total Cost"]; + } catch (error) { + if (error instanceof TimeoutError) { + return this.onTimeout(recent, timeoutMs); + } else if (error instanceof Error) { + return this.onError(recent, error.message); + } else { + return this.onError(recent, "Internal error"); + } + } + if (cost === 0) { + return this.onZeroCostPlan(recent); + } + const indexes = this.getPotentialIndexCandidates( + target.statistics, + recent, + ); + let result: OptimizeResult; + try { + result = await withTimeout( + target.optimizer.run(builder, indexes), + QUERY_TIMEOUT_MS, + ); + } catch (error) { + if (error instanceof TimeoutError) { + return this.onTimeout(recent, QUERY_TIMEOUT_MS); + } else if (error instanceof Error) { + return this.onError(recent, error.message); + } else { + return this.onError(recent, "Internal error"); + } + } + + return this.onOptimizeReady(result, recent); + } + + private onOptimizeReady( + result: OptimizeResult, + recent: OptimizedQuery, + ): LiveQueryOptimization { + switch (result.kind) { + case "ok": { + const indexRecommendations = mapIndexRecommandations(result); + const percentageReduction = costDifferencePercentage( + result.baseCost, + result.finalCost, + ); + const indexesUsed = Array.from(result.existingIndexes); + const costReductionPercentage = Math.trunc( + Math.abs(percentageReduction), + ); + if (costReductionPercentage < MINIMUM_COST_CHANGE_PERCENTAGE) { + this.onNoImprovements(recent, result.baseCost, indexesUsed); + return { + state: "no_improvement_found", + cost: result.baseCost, + indexesUsed, + }; + } else { + this.onImprovementsAvailable(recent, result); + return { + state: "improvements_available", + cost: result.baseCost, + optimizedCost: result.finalCost, + costReductionPercentage, + indexRecommendations, + indexesUsed, + }; + } + } + // unlikely to hit if we've already checked the base plan for zero cost + case "zero_cost_plan": + return this.onZeroCostPlan(recent); + } + } + + private onNoImprovements( + recent: OptimizedQuery, + cost: number, + indexesUsed: string[], + ) { + this.emit( + "noImprovements", + recent.withOptimization({ + state: "no_improvement_found", + cost, + indexesUsed, + }), + ); + } + + private getPotentialIndexCandidates( + statistics: Statistics, + recent: OptimizedQuery, + ) { + const analyzer = new Analyzer(parse); + return analyzer.deriveIndexes( + statistics.ownMetadata, + recent.columnReferences, + ); + } + + private onQueryUnsupported(reason: string): LiveQueryOptimization { + this._invalidQueries++; + return { + state: "not_supported", + reason, + }; + } + + private onImprovementsAvailable( + recent: OptimizedQuery, + result: Extract, + ) { + const optimized = recent.withOptimization( + this.resultToImprovementsAvailable(result), + ); + this.emit("improvementsAvailable", optimized); + this.queries.set( + optimized.hash, + optimized, + ); + } + + private resultToImprovementsAvailable( + result: Extract, + ): LiveQueryOptimization { + const indexesUsed = Array.from(result.existingIndexes); + const indexRecommendations = Array.from(result.newIndexes) + .map((n) => result.triedIndexes.get(n)?.definition) + .filter((n) => n !== undefined); + return { + state: "improvements_available", + cost: result.baseCost, + optimizedCost: result.finalCost, + costReductionPercentage: 0, + indexRecommendations, + indexesUsed, + }; + } + + private onZeroCostPlan(recent: OptimizedQuery): LiveQueryOptimization { + this.emit("zeroCostPlan", recent); + return { + state: "error", + error: new Error( + "Query plan had zero cost. This should not happen on a patched postgres instance", + ), + }; + } + + private onError( + recent: OptimizedQuery, + errorMessage: string, + ): LiveQueryOptimization { + const error = new Error(errorMessage); + this.emit("error", error, recent); + return { state: "error", error }; + } + + private onTimeout( + recent: OptimizedQuery, + waitedMs: number, + ): LiveQueryOptimization { + this.emit("timeout", recent, waitedMs); + return { state: "timeout" }; + } +} + +export class TimeoutError extends Error { + constructor() { + super("Timeout"); + this.name = "TimeoutError"; + } +} + +export const withTimeout = ( + promise: Promise, + timeout: number, +): Promise => { + return Promise.race([ + promise, + new Promise((_, reject) => + setTimeout(() => reject(new TimeoutError()), timeout) + ), + ]); +}; + +function mapIndexRecommandations( + result: Extract, +): string[] { + return Array.from(result.newIndexes.keys(), (definition) => { + const index = result.triedIndexes.get(definition); + if (!index) { + throw new Error( + `Index ${definition} not found in tried indexes. This shouldn't happen.`, + ); + } + return definition; + }); +} + +type PercentageDifference = number; + +export function costDifferencePercentage( + oldVal: number, + newVal: number, +): PercentageDifference { + return ((newVal - oldVal) / oldVal) * 100; +} diff --git a/src/remote/remote-controller.test.ts b/src/remote/remote-controller.test.ts new file mode 100644 index 0000000..663559e --- /dev/null +++ b/src/remote/remote-controller.test.ts @@ -0,0 +1,101 @@ +import { PostgreSqlContainer } from "@testcontainers/postgresql"; +import { Connectable } from "../sync/connectable.ts"; +import { Remote } from "./remote.ts"; +import postgres from "postgresjs"; +import { assertEquals } from "@std/assert/equals"; +import { RemoteController } from "./remote-controller.ts"; +import { ConnectionManager } from "../sync/connection-manager.ts"; +import { RemoteSyncRequest } from "./remote.dto.ts"; +import { assertSpyCalls, spy } from "@std/testing/mock"; +import { setTimeout } from "node:timers/promises"; + +Deno.test({ + name: "controller syncs correctly", + sanitizeOps: false, + // deno is weird... the sync seems like it might be leaking resources? + sanitizeResources: false, + fn: async () => { + const [sourceDb, targetDb] = await Promise.all([ + new PostgreSqlContainer("postgres:17") + .withCopyContentToContainer([ + { + content: ` + create table testing(a int, b text); + insert into testing values (1); + create index on testing(b); + create extension pg_stat_statements; + select * from testing where a = 1; + `, + target: "/docker-entrypoint-initdb.d/init.sql", + }, + ]) + .withCommand(["-c", "shared_preload_libraries=pg_stat_statements"]) + .start(), + new PostgreSqlContainer("postgres:17").start(), + ]); + const controller = new AbortController(); + + const target = Connectable.fromString( + targetDb.getConnectionUri(), + ); + const source = Connectable.fromString( + sourceDb.getConnectionUri(), + ); + + const sourceOptimizer = ConnectionManager.forLocalDatabase(); + + const remote = new RemoteController( + new Remote(target, sourceOptimizer), + ); + + const server = Deno.serve( + { port: 0, signal: controller.signal }, + async (req: Request): Promise => { + const result = await remote.execute(req); + if (!result) { + throw new Error(); + } + return result; + }, + ); + try { + const ws = new WebSocket(`ws://localhost:${server.addr.port}/postgres`); + const messageFunction = spy(); + ws.addEventListener("error", console.log); + ws.addEventListener("message", messageFunction); + + const response = await fetch( + new Request( + `http://localhost:${server.addr.port}/postgres`, + { + method: "POST", + body: RemoteSyncRequest.encode({ + db: source, + }), + }, + ), + ); + + assertEquals(response?.status, 200); + await setTimeout(1000); + + console.log("a", await response.json()); + const sql = postgres( + target.withDatabaseName(Remote.optimizingDbName).toString(), + ); + const tablesAfter = + await sql`select tablename from pg_tables where schemaname = 'public'`; + assertEquals(tablesAfter.count, 1); + const indexesAfter = + await sql`select * from pg_indexes where schemaname = 'public'`; + assertEquals(indexesAfter.count, 1); + assertEquals(tablesAfter[0], { tablename: "testing" }); + const rows = await sql`select * from testing`; + assertEquals(rows.length, 0); + + assertSpyCalls(messageFunction, 1); + } finally { + await Promise.all([sourceDb.stop(), targetDb.stop(), server.shutdown()]); + } + }, +}); diff --git a/src/remote/remote-controller.ts b/src/remote/remote-controller.ts new file mode 100644 index 0000000..9f249fe --- /dev/null +++ b/src/remote/remote-controller.ts @@ -0,0 +1,103 @@ +import { env } from "../env.ts"; +import { OptimizedQuery } from "../sql/recent-query.ts"; +import { QueryOptimizer } from "./query-optimizer.ts"; +import { RemoteSyncRequest } from "./remote.dto.ts"; +import { Remote } from "./remote.ts"; + +export class RemoteController { + /** + * Only a single socket can be active at the same time. + * Multi-tab support not currently available + */ + private socket?: WebSocket; + private syncResponse?: ReturnType; + + constructor( + private readonly remote: Remote, + ) { + this.hookUpWebsockets(remote.optimizer); + } + + async execute( + request: Request, + ): Promise { + const url = new URL(request.url); + if (url.pathname === "/postgres") { + const isWebsocket = request.headers.get("upgrade") === "websocket"; + if (isWebsocket) { + return this.onWebsocketRequest(request); + } else if (request.method === "POST") { + return await this.onFullSync(request); + } + } + } + + private hookUpWebsockets(optimizer: QueryOptimizer) { + const onQueryProcessed = this.eventOnQueryProcessed.bind(this); + const onError = this.eventError.bind(this); + optimizer.on("noImprovements", onQueryProcessed); + optimizer.on("improvementsAvailable", onQueryProcessed); + optimizer.on("error", onError); + optimizer.on("timeout", onQueryProcessed); + optimizer.on("zeroCostPlan", onQueryProcessed); + } + + private async onFullSync(request: Request): Promise { + const body = RemoteSyncRequest.safeDecode(await request.text()); + if (!body.success) { + return new Response(JSON.stringify(body.error), { status: 400 }); + } + + const { db } = body.data; + try { + if (!this.syncResponse) { + this.syncResponse = this.remote.syncFrom(db); + } + const { schema } = await this.syncResponse; + const queries = this.remote.optimizer.getQueries(); + + return Response.json({ schema, queries: { type: "ok", value: queries } }); + } catch (error) { + console.error(error); + return Response.json({ + error: env.HOSTED ? "Internal Server Error" : error, + message: "Failed to sync database", + }, { + status: 500, + }); + } + } + + private onWebsocketRequest(request: Request): Response { + const { socket, response } = Deno.upgradeWebSocket(request); + console.log({ socket }); + this.socket = socket; + + socket.addEventListener("open", () => { + this.syncResponse = undefined; + }); + + socket.addEventListener("close", () => { + this.socket = undefined; + }); + + return response; + } + + private eventOnQueryProcessed(query: OptimizedQuery) { + this.socket?.send(JSON.stringify({ + type: "queryProcessed", + query, + })); + } + + private eventError(error: Error, query: OptimizedQuery) { + this.socket?.send( + JSON.stringify({ + type: "error", + query, + error: error.message, + }), + ); + } +} diff --git a/src/remote/remote.dto.ts b/src/remote/remote.dto.ts new file mode 100644 index 0000000..305a259 --- /dev/null +++ b/src/remote/remote.dto.ts @@ -0,0 +1,42 @@ +import { z } from "zod"; +import { Connectable } from "../sync/connectable.ts"; +import { FullSchema } from "../sync/schema_differ.ts"; +import { OptimizedQuery } from "../sql/recent-query.ts"; + +export const RemoteSyncRequest = z.codec( + z.string(), + z.object({ + db: z.custom(), + }), + { + encode: (value) => JSON.stringify({ db: value.db.toString() }), + decode: (value) => { + const parsed = JSON.parse(value); + return { db: Connectable.fromString(parsed.db) }; + }, + }, +); + +export const RemoteSyncFullSchemaResponse = z.discriminatedUnion("type", [ + z.object({ type: z.literal("ok"), value: FullSchema }), + z.object({ type: z.literal("error"), error: z.string() }), +]); + +export type RemoteSyncFullSchemaResponse = z.infer< + typeof RemoteSyncFullSchemaResponse +>; + +export const RemoteSyncQueriesResponse = z.discriminatedUnion("type", [ + z.object({ + type: z.literal("ok"), + value: z.array(z.custom()), + }), + z.object({ type: z.literal("error"), error: z.string() }), +]); + +export const RemoteSyncResponse = z.object({ + // queries: RemoteSyncQueriesResponse, + schema: RemoteSyncFullSchemaResponse, +}); + +export type RemoteSyncResponse = z.infer; diff --git a/src/remote/remote.test.ts b/src/remote/remote.test.ts new file mode 100644 index 0000000..7071de3 --- /dev/null +++ b/src/remote/remote.test.ts @@ -0,0 +1,267 @@ +import { PostgreSqlContainer } from "@testcontainers/postgresql"; +import { Connectable } from "../sync/connectable.ts"; +import { Remote } from "./remote.ts"; +import postgres from "postgresjs"; +import { assertEquals } from "@std/assert/equals"; +import { ConnectionManager } from "../sync/connection-manager.ts"; +import { assertArrayIncludes } from "@std/assert"; +import { PgIdentifier } from "@query-doctor/core"; + +const TEST_TARGET_CONTAINER_NAME = "postgres:17"; +const TEST_TARGET_CONTAINER_TIMESCALEDB_NAME = + "timescale/timescaledb:latest-pg17"; + +export function testSpawnTarget( + options: { content?: string; containerName?: string } = { + containerName: TEST_TARGET_CONTAINER_NAME, + }, +) { + let pg = new PostgreSqlContainer( + options.containerName ?? TEST_TARGET_CONTAINER_NAME, + ); + if (options.content) { + pg = pg.withCopyContentToContainer([ + { + content: options.content, + target: "/docker-entrypoint-initdb.d/init.sql", + }, + ]); + } + return pg.start(); +} + +function assertOk( + result: { type: string; value?: T }, +): asserts result is { type: "ok"; value: T } { + assertEquals(result.type, "ok"); +} + +Deno.test({ + name: "syncs correctly", + sanitizeOps: false, + // deno is weird... the sync seems like it might be leaking resources? + sanitizeResources: false, + fn: async () => { + const [sourceDb, targetDb] = await Promise.all([ + new PostgreSqlContainer("postgres:17") + .withCopyContentToContainer([ + { + content: ` + create extension pg_stat_statements; + create table testing(a int, b text); + insert into testing values (1); + create index "testing_1234" on testing(b); + select * from testing where a = 1; + `, + target: "/docker-entrypoint-initdb.d/init.sql", + }, + ]) + .withCommand(["-c", "shared_preload_libraries=pg_stat_statements"]) + .start(), + testSpawnTarget( + { content: "create table testing(a int); create index on testing(a)" }, + ), + ]); + + try { + const target = Connectable.fromString(targetDb.getConnectionUri()); + const source = Connectable.fromString(sourceDb.getConnectionUri()); + + const remote = new Remote( + target, + ConnectionManager.forLocalDatabase(), + ); + + const result = await remote.syncFrom(source); + const optimizedQueries = remote.optimizer.getQueries(); + + const queries = optimizedQueries.map((f) => f.query); + assertArrayIncludes(queries, [ + "create table testing(a int, b text)", + "select * from testing where a = $1", + ]); + + assertOk(result.schema); + + const tableNames = result.schema.value.tables.map((table) => + table.tableName + ); + + assertArrayIncludes(tableNames, ["testing"]); + + const indexNames = result.schema.value.indexes.map((index) => + index.indexName + ); + assertArrayIncludes(indexNames, ["testing_1234"]); + + const sql = postgres( + target.withDatabaseName(Remote.optimizingDbName).toString(), + ); + + const indexesAfter = + await sql`select indexname from pg_indexes where schemaname = 'public'`; + assertEquals( + indexesAfter.count, + 1, + "Indexes were not copied over correctly from the source db", + ); + + assertEquals(indexesAfter[0], { indexname: "testing_1234" }); + + const tablesAfter = + await sql`select tablename from pg_tables where schemaname = 'public'`; + assertEquals( + tablesAfter.count, + 1, + "Tables were not copied over correctly from the source db", + ); + assertEquals( + tablesAfter[0], + { tablename: "testing" }, + "Table name mismatch", + ); + const rows = await sql`select * from testing`; + // expect no rows to have been synced + assertEquals(rows.length, 0, "Table in target db not empty"); + } finally { + await Promise.all([sourceDb.stop(), targetDb.stop()]); + } + }, +}); + +Deno.test({ + name: "raw timescaledb syncs correctly", + sanitizeOps: false, + sanitizeResources: false, + fn: async () => { + const [source, target] = await Promise.all([ + new PostgreSqlContainer( + "timescale/timescaledb:latest-pg17", + ) + .withEnvironment({ + POSTGRES_HOST_AUTH_METHOD: "trust", + }) + .withCopyContentToContainer([ + { + content: ` + create table testing(a int, b text); + insert into testing values (1); + create index "testing_1234" on testing(b); + `, + target: "/docker-entrypoint-initdb.d/init.sql", + }, + ]) + .start(), + testSpawnTarget({ + containerName: TEST_TARGET_CONTAINER_TIMESCALEDB_NAME, + }), + ]); + + const sourceConn = Connectable.fromString(source.getConnectionUri()); + const targetConn = Connectable.fromString(target.getConnectionUri()); + const manager = ConnectionManager.forLocalDatabase(); + const remote = new Remote(targetConn, manager); + + try { + const t = manager.getOrCreateConnection( + targetConn.withDatabaseName(PgIdentifier.fromString("optimizing_db")), + ); + await remote.syncFrom(sourceConn); + const indexesAfter = await t.exec( + "select indexname from pg_indexes where schemaname = 'public'", + ); + assertEquals( + indexesAfter.length, + 1, + "Indexes were not copied over correctly from the source db", + ); + + assertEquals(indexesAfter[0], { indexname: "testing_1234" }); + } finally { + await Promise.all([source.stop(), target.stop()]); + } + }, +}); + +Deno.test({ + name: "timescaledb with continuous aggregates sync correctly", + sanitizeOps: false, + sanitizeResources: false, + fn: async () => { + const [source, target] = await Promise.all([ + new PostgreSqlContainer( + "timescale/timescaledb:latest-pg17", + ) + .withEnvironment({ + POSTGRES_HOST_AUTH_METHOD: "trust", + }) + .withCommand([ + "-c", + "shared_preload_libraries=pg_stat_statements,timescaledb", + ]) + .withLogConsumer((a) => a.pipe(process.stdout)) + .withCopyContentToContainer([ + { + content: ` + create extension if not exists pg_stat_statements; + create table conditions( + "time" timestamptz not null, + device_id integer, + temperature float + ) + with( + timescaledb.hypertable, + timescaledb.partition_column = 'time' + ); + create materialized view conditions_summary_daily + with (timescaledb.continuous) as + select device_id, + time_bucket(interval '1 day', time) as bucket, + avg(temperature), + max(temperature), + min(temperature) + from conditions + group by device_id, bucket; + select * from conditions where time < now(); + `, + target: "/docker-entrypoint-initdb.d/init.sql", + }, + ]) + .start(), + testSpawnTarget({ + containerName: TEST_TARGET_CONTAINER_TIMESCALEDB_NAME, + }), + ]); + + const sourceConn = Connectable.fromString(source.getConnectionUri()); + const targetConn = Connectable.fromString(target.getConnectionUri()); + const manager = ConnectionManager.forLocalDatabase(); + const remote = new Remote(targetConn, manager); + + try { + const t = manager.getOrCreateConnection( + targetConn.withDatabaseName(PgIdentifier.fromString("optimizing_db")), + ); + await remote.syncFrom(sourceConn); + const queries = remote.optimizer.getQueries(); + const queryStrings = queries.map((q) => q.query); + + assertArrayIncludes(queryStrings, [ + "select * from conditions where time < $1", + ]); + const indexesAfter = await t.exec( + "select indexname from pg_indexes where schemaname = 'public'", + ); + assertEquals( + indexesAfter.length, + 1, + "Indexes were not copied over correctly from the source db", + ); + + assertEquals(indexesAfter[0], { indexname: "conditions_time_idx" }); + } finally { + await manager.closeAll(); + await Promise.all([source.stop(), target.stop()]); + } + }, +}); diff --git a/src/remote/remote.ts b/src/remote/remote.ts new file mode 100644 index 0000000..fdd33a4 --- /dev/null +++ b/src/remote/remote.ts @@ -0,0 +1,210 @@ +import { + PgIdentifier, + type Postgres, + PostgresVersion, + Statistics, + StatisticsMode, +} from "@query-doctor/core"; +import { type Connectable } from "../sync/connectable.ts"; +import { DumpCommand, RestoreCommand } from "../sync/schema-link.ts"; +import { ConnectionManager } from "../sync/connection-manager.ts"; +import { type RecentQuery } from "../sql/recent-query.ts"; +import { type FullSchema, SchemaDiffer } from "../sync/schema_differ.ts"; +import { type RemoteSyncFullSchemaResponse } from "./remote.dto.ts"; +import { QueryOptimizer } from "./query-optimizer.ts"; + +/** + * Represents a db for doing optimization work. + * We only maintain one instance of this class as we only do + * optimization against one physical postgres database. + * But potentially more logical databases in the future. + * + * `Remote` only concerns itself with the remote it's doing optimization + * against. It does not deal with the source in any way aside from running sync + */ +export class Remote { + static readonly baseDbName = PgIdentifier.fromString("postgres"); + static readonly optimizingDbName = PgIdentifier.fromString( + "optimizing_db", + ); + + private readonly differ = new SchemaDiffer(); + readonly optimizer: QueryOptimizer; + + /** + * We have to juggle 2 different connections to the Remote + * + * 1 -> connection to `/postgres` where we manage other databases. + * this pool stays connected long-term. That's this variable + * + * 2 -> connections to {@link Remote.databaseName}. This connection pool is + * destroyed and re-created on each successful sync along with the db itself + */ + private baseDbURL: Connectable; + /** The URL of the optimizing db */ + private readonly optimizingDbUDRL: Connectable; + + constructor( + /** This has to be a local url. Very bad things will happen if this is a remote URL */ + targetURL: Connectable, + private readonly manager: ConnectionManager, + ) { + this.baseDbURL = targetURL.withDatabaseName(Remote.baseDbName); + this.optimizingDbUDRL = targetURL.withDatabaseName(Remote.optimizingDbName); + this.optimizer = new QueryOptimizer(manager); + } + + async syncFrom( + source: Connectable, + statsStrategy: StatisticsStrategy = { type: "pullFromSource" }, + ): Promise<{ schema: RemoteSyncFullSchemaResponse }> { + await this.resetDatabase(); + const [_restoreResult, recentQueries, fullSchema, pulledStats] = + await Promise + .allSettled([ + // This potentially creates a lot of connections to the source + this.pipeSchema(this.optimizingDbUDRL, source), + this.getRecentQueries(source), + this.getFullSchema(source), + this.dumpSourceStats(source), + this.resolveStatisticsStrategy(source, statsStrategy), + ]); + + if (fullSchema.status === "fulfilled") { + this.differ.put(source, fullSchema.value); + } + + const pg = this.manager.getOrCreateConnection( + this.optimizingDbUDRL, + ); + + let queries: RecentQuery[] = []; + if (recentQueries.status === "fulfilled") { + queries = recentQueries.value; + } + + let stats: StatisticsMode | undefined; + if (pulledStats.status === "fulfilled") { + stats = pulledStats.value; + } + + await this.onSuccessfulSync( + pg, + source, + queries, + stats, + ); + + return { + schema: fullSchema.status === "fulfilled" + ? { type: "ok", value: fullSchema.value } + : { + type: "error", + error: fullSchema.reason instanceof Error + ? fullSchema.reason.message + : "Unknown error", + }, + }; + } + + /** + * Drops and recreates the {@link Remote.optimizingDbName} db. + * + * TODO: allow juggling multiple databases in the future + */ + private async resetDatabase(): Promise { + const databaseName = Remote.optimizingDbName; + const baseDb = this.manager.getOrCreateConnection(this.baseDbURL); + // these cannot be run in the same `exec` block as that implicitly creates transactions + await baseDb.exec( + // drop database does not allow parameterization + `drop database if exists ${databaseName} with (force);`, + ); + await baseDb.exec(`create database ${databaseName};`); + } + + private async pipeSchema( + target: Connectable, + source: Connectable, + ): Promise { + const dump = DumpCommand.spawn(source, "native-postgres"); + // TODO: handle event emitter events + // dump.on("dump", (data) => { + // console.log("got dump data", data); + // }); + // dump.on("restore", (data) => { + // console.log("got restore data", data); + // }); + const restore = RestoreCommand.spawn(target); + const { dump: dumpResult, restore: restoreResult } = await dump.pipeTo( + restore, + ); + if (!dumpResult.status.success) { + throw new Error( + `Dump failed with status ${dumpResult.status.code}`, + ); + } + if (restoreResult && !restoreResult.status.success) { + throw new Error( + `Restore failed with status ${restoreResult.status.code}`, + ); + } + } + + private resolveStatisticsStrategy( + source: Connectable, + strategy: StatisticsStrategy, + ): Promise { + switch (strategy.type) { + case "static": + return Promise.resolve(strategy.stats); + case "pullFromSource": + return this.dumpSourceStats(source); + } + } + + private async dumpSourceStats(source: Connectable): Promise { + const pg = this.manager.getOrCreateConnection(source); + const stats = await Statistics.dumpStats( + pg, + PostgresVersion.parse("17"), + "full", + ); + return { kind: "fromStatisticsExport", source: { kind: "inline" }, stats }; + } + + private async getRecentQueries( + source: Connectable, + ): Promise { + const connector = this.manager.getConnectorFor(source); + return await connector.getRecentQueries(); + } + + private getFullSchema(source: Connectable): Promise { + const connector = this.manager.getConnectorFor(source); + return connector.getSchema(); + } + + /** + * Process a successful sync and run any potential cleanup functions + */ + private async onSuccessfulSync( + postgres: Postgres, + source: Connectable, + recentQueries: RecentQuery[], + stats?: StatisticsMode, + ): Promise { + if (source.isSupabase()) { + // https://gist.github.com/Xetera/067c613580320468e8367d9d6c0e06ad + await postgres.exec("drop schema if exists extensions cascade"); + } + this.optimizer.start(this.optimizingDbUDRL, recentQueries, stats); + } +} + +export type StatisticsStrategy = { + type: "pullFromSource"; +} | { + type: "static"; + stats: StatisticsMode; +}; diff --git a/src/runner.ts b/src/runner.ts index 23c759a..0fb5b3f 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -25,8 +25,9 @@ import { } from "./reporters/reporter.ts"; import { bgBrightMagenta, blue, yellow } from "@std/fmt/colors"; import { env } from "./env.ts"; -import { wrapGenericPostgresInterface } from "./sql/postgresjs.ts"; +import { connectToSource } from "./sql/postgresjs.ts"; import { parse } from "@libpg-query/parser"; +import { Connectable } from "./sync/connectable.ts"; export class Runner { private readonly seenQueries = new Set(); @@ -45,12 +46,12 @@ export class Runner { ) {} static async build(options: { - postgresUrl: string; + postgresUrl: Connectable; statisticsPath?: string; maxCost?: number; logPath: string; }) { - const db = wrapGenericPostgresInterface({ url: options.postgresUrl }); + const db = connectToSource(options.postgresUrl); const statisticsMode = Runner.decideStatisticsMode(options.statisticsPath); const stats = await Statistics.fromPostgres(db, statisticsMode); const existingIndexes = await stats.getExistingIndexes(); diff --git a/src/server/http.ts b/src/server/http.ts index b3f35d4..c85b47d 100644 --- a/src/server/http.ts +++ b/src/server/http.ts @@ -7,11 +7,17 @@ import { ZodError } from "zod"; import { shutdownController } from "../shutdown.ts"; import { env } from "../env.ts"; import { SyncResult } from "../sync/syncer.ts"; -import { wrapGenericPostgresInterface } from "../sql/postgresjs.ts"; +import { connectToOptimizer, connectToSource } from "../sql/postgresjs.ts"; import type { RateLimitResult } from "@rabbit-company/rate-limiter"; import * as errors from "../sync/errors.ts"; +import { RemoteController } from "../remote/remote-controller.ts"; +import { Connectable } from "../sync/connectable.ts"; +import { ConnectionManager } from "../sync/connection-manager.ts"; +import { Remote } from "../remote/remote.ts"; -const syncer = new PostgresSyncer(wrapGenericPostgresInterface); +const sourceConnectionManager = new ConnectionManager(connectToSource); + +const syncer = new PostgresSyncer(sourceConnectionManager); async function onSync(req: Request) { const startTime = Date.now(); @@ -148,7 +154,20 @@ async function onReset(req: Request) { } } -export function createServer(hostname: string, port: number) { +export function createServer( + hostname: string, + port: number, + targetDb?: Connectable, +) { + const optimizingDbConnectionManager = new ConnectionManager( + connectToOptimizer, + ); + + const remoteController = targetDb + ? new RemoteController( + new Remote(targetDb, optimizingDbConnectionManager), + ) + : undefined; return Deno.serve( { hostname, port, signal: shutdownController.signal }, async (req, info) => { @@ -193,6 +212,14 @@ export function createServer(hostname: string, port: number) { const res = await onReset(req); return transformResponse(res, limit); } + const remoteResponse = await remoteController?.execute(req); + if (remoteResponse) { + // WebSocket upgrade responses have immutable headers, skip transform + if (req.headers.get("upgrade") === "websocket") { + return remoteResponse; + } + return transformResponse(remoteResponse, limit); + } return new Response("Not found", { status: 404 }); }, ); diff --git a/src/server/sync.dto.ts b/src/server/sync.dto.ts index a8fcf1e..1aaac4d 100644 --- a/src/server/sync.dto.ts +++ b/src/server/sync.dto.ts @@ -1,14 +1,14 @@ -import { z } from "zod/v4"; -import { Connectable } from "../sync/connectable.ts"; +import { z } from "zod"; +import { ConnectableParser } from "../sync/connectable.ts"; export const LiveQueryRequest = z.object({ - db: z.string().transform(Connectable.transform), + db: ConnectableParser, }); export type LiveQueryRequest = z.infer; export const SyncRequest = z.object({ - db: z.string().transform(Connectable.transform), + db: ConnectableParser, seed: z.coerce.number().min(0).max(1).default(0), schema: z.coerce.string().default("public").meta({ deprecated: true, diff --git a/src/sql/postgresjs.ts b/src/sql/postgresjs.ts index 4d29825..ad6c7e1 100644 --- a/src/sql/postgresjs.ts +++ b/src/sql/postgresjs.ts @@ -1,10 +1,10 @@ import postgres from "postgresjs"; import { type Postgres, - type PostgresConnectionInput, type PostgresTransaction, PostgresVersion, } from "@query-doctor/core"; +import { Connectable } from "../sync/connectable.ts"; type PgConnectionOptions = postgres.Options< Record @@ -17,23 +17,43 @@ const DEFAULT_IDLE_TIMEOUT_SECONDS = 15; // it's ok to recycle connections frequently if needed const DEFAULT_MAX_LIFETIME_SECONDS = 60 * 5; -const connectionOptions: PgConnectionOptions = { - max: 20, - max_lifetime: DEFAULT_MAX_LIFETIME_SECONDS, - idle_timeout: DEFAULT_IDLE_TIMEOUT_SECONDS, -}; +/** + * Connecting to the local optimizer + */ +export function connectToOptimizer(connectable: Connectable) { + const connectionOptions: PgConnectionOptions = { + max: 100, + }; + + return connect(connectable, connectionOptions); +} + +/** + * Connect to the source database to pull data out. + * We have to be a lot more conservative here + * and make sure the connections drop asap to prevent + * exhausting them + */ +export function connectToSource( + connectable: Connectable, +) { + const connectionOptions: PgConnectionOptions = { + max: 20, + max_lifetime: DEFAULT_MAX_LIFETIME_SECONDS, + idle_timeout: DEFAULT_IDLE_TIMEOUT_SECONDS, + }; -export function wrapGenericPostgresInterface( - input: PostgresConnectionInput, -): Postgres { - let pg: postgres.Sql; - if ("url" in input) { - pg = postgres(input.url, connectionOptions); - } else { - throw new Error("Invalid input"); - } + return connect(connectable, connectionOptions); +} + +function connect(connectable: Connectable, options: PgConnectionOptions) { + const pg = postgres(connectable.toString(), options); + return wrapGenericPostgresInterface(pg); +} + +export function wrapGenericPostgresInterface(pg: postgres.Sql): Postgres { return { - exec: async (query, params) => { + exec: (query, params) => { return pg.unsafe(query, params as postgres.ParameterOrJSON[]); }, serverNum: async () => @@ -65,5 +85,9 @@ export function wrapGenericPostgresInterface( yield* row as T[]; } }, + // @ts-expect-error | this will be added to the pg interface later + close() { + return pg.end(); + }, }; } diff --git a/src/sql/recent-query.ts b/src/sql/recent-query.ts new file mode 100644 index 0000000..dfd520c --- /dev/null +++ b/src/sql/recent-query.ts @@ -0,0 +1,120 @@ +// import { format } from "sql-formatter"; +// deno-lint-ignore no-unused-vars +import type { SegmentedQueryCache } from "../sync/seen-cache.ts"; +import { + Analyzer, + DiscoveredColumnReference, + Nudge, + SQLCommenterTag, +} from "@query-doctor/core"; +import { parse } from "@libpg-query/parser"; +import z from "zod"; +import type { LiveQueryOptimization } from "../remote/optimization.ts"; + +/** + * Constructed by syncing with {@link SegmentedQueryCache.sync} + * and supplying the date the query was last seen + */ +export class RecentQuery { + readonly formattedQuery: string; + readonly username: string; + readonly query: string; + readonly meanTime: number; + readonly calls: string; + readonly rows: string; + readonly topLevel: boolean; + + readonly isSystemQuery: boolean; + readonly isSelectQuery: boolean; + readonly isIntrospection: boolean; + readonly isTargetlessSelectQuery: boolean; + + /** Use {@link RecentQuery.analyze} instead */ + constructor( + data: RawRecentQuery, + readonly tableReferences: string[], + readonly columnReferences: DiscoveredColumnReference[], + readonly tags: SQLCommenterTag[], + readonly nudges: Nudge[], + readonly hash: QueryHash, + readonly seenAt: number, + ) { + this.username = data.username; + this.query = data.query; + this.formattedQuery = data.query; + this.meanTime = data.meanTime; + this.calls = data.calls; + this.rows = data.rows; + this.topLevel = data.topLevel; + + this.isSystemQuery = RecentQuery.isSystemQuery(tableReferences); + this.isSelectQuery = RecentQuery.isSelectQuery(data); + this.isIntrospection = RecentQuery.isIntrospection(data); + this.isTargetlessSelectQuery = this.isSelectQuery + ? RecentQuery.isTargetlessSelectQuery(tableReferences) + : false; + } + + withOptimization( + optimization: LiveQueryOptimization, + ): OptimizedQuery { + return Object.assign(this, { optimization }); + } + + static async analyze( + data: RawRecentQuery, + hash: QueryHash, + seenAt: number, + ) { + const analyzer = new Analyzer(parse); + const analysis = await analyzer.analyze(data.query); + return new RecentQuery( + { ...data, query: analysis.queryWithoutTags }, + analysis.referencedTables, + analysis.indexesToCheck, + analysis.tags, + analysis.nudges, + hash, + seenAt, + ); + } + + static isSelectQuery(data: RawRecentQuery): boolean { + return /^select/i.test(data.query); + } + + static isSystemQuery(referencedTables: string[]): boolean { + return referencedTables.some((table) => + table.startsWith("pg_") || + /* timescaledb jobs */ + table.startsWith("bgw_job_stat_") + ); + } + + static isIntrospection(data: RawRecentQuery): boolean { + return data.query.match("@qd_introspection") !== null; + } + + static isTargetlessSelectQuery( + referencedTables: string[], + ): boolean { + return referencedTables.length === 0; + } +} + +export type RawRecentQuery = { + username: string; + query: string; + formattedQuery: string; + meanTime: number; + calls: string; + rows: string; + topLevel: boolean; +}; + +export type OptimizedQuery = RecentQuery & { + optimization: LiveQueryOptimization; +}; + +export const QueryHash = z.string().brand<"QueryHash">(); +export type QueryHash = z.infer; diff --git a/src/sync/connectable.test.ts b/src/sync/connectable.test.ts new file mode 100644 index 0000000..ce69571 --- /dev/null +++ b/src/sync/connectable.test.ts @@ -0,0 +1,16 @@ +import { assertEquals } from "@std/assert"; +import { Connectable } from "./connectable.ts"; +import { PgIdentifier } from "@query-doctor/core"; + +Deno.test("connectable", () => { + const connectable = Connectable.fromString( + "postgres://user:password@localhost:5432/dbname?a=b&c=d", + ); + const newConnectable = connectable.withDatabaseName( + PgIdentifier.fromString("testing"), + ); + assertEquals( + newConnectable.toString(), + "postgres://user:password@localhost:5432/testing?a=b&c=d", + ); +}); diff --git a/src/sync/connectable.ts b/src/sync/connectable.ts index 2642dff..e6ef648 100644 --- a/src/sync/connectable.ts +++ b/src/sync/connectable.ts @@ -1,5 +1,6 @@ -import { z } from "zod/v4"; +import { z } from "zod"; import { env } from "../env.ts"; +import { PgIdentifier } from "@query-doctor/core"; /** * Represents a valid connection to a database. @@ -13,8 +14,14 @@ export class Connectable { return this.url.hostname.endsWith("supabase.com"); } + withDatabaseName(databaseName: PgIdentifier) { + const newUrl = new URL(this.url); + newUrl.pathname = `/${databaseName}`; + return new Connectable(newUrl); + } + /** - * Custom logic for parsing a string into a Connectable through zod. + * Custom logic for parsing a string into a {@link Connectable} through zod. */ static transform( urlString: string, @@ -80,6 +87,10 @@ export class Connectable { return new Connectable(url); } + static fromString(url: string): Connectable { + return ConnectableParser.parse(url); + } + private static extractSupabaseAccount(url: URL): string | undefined { const match = url.toString().match(/db\.(\w+)\.supabase\.co/); if (!match) { @@ -99,3 +110,5 @@ export class Connectable { return this.url.toString(); } } + +export const ConnectableParser = z.string().transform(Connectable.transform); diff --git a/src/sync/connection-manager.ts b/src/sync/connection-manager.ts new file mode 100644 index 0000000..e1bea94 --- /dev/null +++ b/src/sync/connection-manager.ts @@ -0,0 +1,62 @@ +import type { Postgres } from "@query-doctor/core"; +import { SegmentedQueryCache } from "./seen-cache.ts"; +import { Connectable } from "./connectable.ts"; +import { PostgresConnector } from "./pg-connector.ts"; +import { connectToOptimizer, connectToSource } from "../sql/postgresjs.ts"; + +/** + * Manages connections and query caches for each connection + */ +export class ConnectionManager { + readonly segmentedQueryCache = new SegmentedQueryCache(); + + // This prevents connections being garbage collected. + // ConnectionMap should be responsible for closing connections + private readonly connections = new Map(); + + constructor( + private readonly factory: (connectable: Connectable) => Postgres, + ) {} + + /** + * Create a connection manager with default settings + * optimized for connecting to local dbs (used for optimizing) + */ + static forLocalDatabase() { + return new ConnectionManager(connectToOptimizer); + } + + /** + * Create a connection manager with default settings + * optimized for connecting to remote dbs (given by users) + */ + static forRemoteDatabase() { + return new ConnectionManager(connectToSource); + } + + getOrCreateConnection(connectable: Connectable): Postgres { + const urlString = connectable.toString(); + let sql = this.connections.get(urlString); + if (!sql) { + sql = this.factory(connectable); + this.connections.set(urlString, sql); + } + return sql; + } + + getConnectorFor(input: Connectable | Postgres): PostgresConnector { + const sql = input instanceof Connectable + ? this.getOrCreateConnection(input) + : input; + return new PostgresConnector(sql, this.segmentedQueryCache); + } + + async closeAll(): Promise { + const closePromises = Array.from(this.connections.values()).map((conn) => + // @ts-expect-error | this will exist later + conn.close() + ); + this.connections.clear(); + await Promise.all(closePromises); + } +} diff --git a/src/sync/executable.ts b/src/sync/executable.ts new file mode 100644 index 0000000..2aabe6d --- /dev/null +++ b/src/sync/executable.ts @@ -0,0 +1,76 @@ +import { env } from "../env.ts"; +import { log } from "../log.ts"; + +const decoder = new TextDecoder(); + +// Is there a way to get this working for windows? +function lookupBinary( + os: typeof Deno.build.os, + name: string, +): string | undefined { + try { + if (os === "linux" || os === "darwin") { + const output = new Deno.Command("which", { args: [name] }).outputSync(); + return decoder.decode(output.stdout) || undefined; + } + } catch (_error) { + // it's not in path. No problem + } +} + +export function findPgRestoreBinary(version: string): string { + const forcePath = env.PG_RESTORE_BINARY; + if (forcePath) { + log.info( + `Using pg_restore binary from env(PG_RESTORE_BINARY): ${forcePath}`, + "schema:setup", + ); + return forcePath; + } + const os = Deno.build.os; + const arch = Deno.build.arch; + const existing = lookupBinary(os, "pg_restore")?.trim(); + if (existing) { + log.info( + `Using pg_restore binary from PATH: ${existing.trim()}`, + "schema:setup", + ); + return existing; + } + const shippedPath = `./bin/pg_restore-${version}/pg_restore.${os}-${arch}`; + if (!Deno.statSync(shippedPath).isFile) { + throw new Error(`pg_restore binary not found at ${shippedPath}`); + } + log.info( + `Using built-in "pg_restore" binary: ${shippedPath}`, + "schema:setup", + ); + return shippedPath; +} + +export function findPgDumpBinary(version: string): string { + const forcePath = env.PG_DUMP_BINARY; + if (forcePath) { + log.info( + `Using pg_dump binary from env(PG_DUMP_BINARY): ${forcePath}`, + "schema:setup", + ); + return forcePath; + } + const os = Deno.build.os; + const arch = Deno.build.arch; + const existing = lookupBinary(os, "pg_dump")?.trim(); + if (existing) { + log.info( + `Using pg_dump binary from PATH: ${existing}`, + "schema:setup", + ); + return existing; + } + const shippedPath = `./bin/pg_dump-${version}/pg_dump.${os}-${arch}`; + if (!Deno.statSync(shippedPath).isFile) { + throw new Error(`pg_dump binary not found at ${shippedPath}`); + } + log.info(`Using built-in "pg_dump" binary: ${shippedPath}`, "schema:setup"); + return shippedPath; +} diff --git a/src/sync/pg-connector.ts b/src/sync/pg-connector.ts index 1801b0c..7f894d4 100644 --- a/src/sync/pg-connector.ts +++ b/src/sync/pg-connector.ts @@ -1,4 +1,3 @@ -import { format } from "sql-formatter"; import schemaDumpSql from "./schema_dump.sql" with { type: "text" }; import type { CursorOptions, @@ -17,6 +16,9 @@ import { Postgres } from "@query-doctor/core"; import { SegmentedQueryCache } from "./seen-cache.ts"; import { FullSchema, FullSchemaColumn } from "./schema_differ.ts"; import { ExtensionNotInstalledError, PostgresError } from "./errors.ts"; +import { RawRecentQuery, RecentQuery } from "../sql/recent-query.ts"; +// deno-lint-ignore no-unused-vars +import { ConnectionManager } from "./connection-manager.ts"; const ctidSymbol = Symbol("ctid"); type Row = NonNullable & { @@ -42,20 +44,6 @@ export type SerializeResult = { sampledRecords: Record; }; -export type RawRecentQuery = { - username: string; - query: string; - formattedQuery: string; - meanTime: number; - calls: string; - rows: string; - topLevel: boolean; -}; - -export type RecentQuery = { - firstSeen: number; -}; - export type RecentQueriesError = { kind: "error"; type: "extension_not_installed"; @@ -87,6 +75,9 @@ export type ResetPgStatStatementsResult = } | ResetPgStatStatementsError; +/** + * Use {@link ConnectionManager.getConnectorFor} to grab an instance of this class + */ export class PostgresConnector implements DatabaseConnector { private static readonly QUERY_DOCTOR_USER = "query_doctor_db_link"; private readonly tupleEstimates = new Map(); @@ -460,36 +451,22 @@ ORDER BY try { const results = await this.db.exec(` SELECT - pg_user.usename as "username", + 'unknown_user' as "username", query, mean_exec_time as "meanTime", calls, rows, toplevel as "topLevel" FROM pg_stat_statements - JOIN pg_user ON pg_user.usesysid = pg_stat_statements.userid WHERE query not like '%pg_stat_statements%' - and dbid = (select oid from pg_database where datname = current_database()) + -- and dbid = (select oid from pg_database where datname = current_database()) and query not like '%@qd_introspection%' - and pg_user.usename not in (/* supabase */ 'supabase_admin', 'supabase_auth_admin', /* neon */ 'cloud_admin'); -- @qd_introspection + -- and pg_user.usename not in (/* supabase */ 'supabase_admin', 'supabase_auth_admin', /* neon */ 'cloud_admin'); -- @qd_introspection `); // we're excluding `pg_stat_statements` from the results since it's almost certainly unrelated - // this is a horrible place for this code to live, it doesn't belong here - // the alternatives are equally as bad: repeat the logic 3 times in the various endpoints - const resultsWithFormattedQueries = results.map((r) => { - return { - ...r, - formattedQuery: format(r.query, { - language: "postgresql", - keywordCase: "upper", - linesBetweenQueries: 2, - }), - }; - }); - - return this.segmentedQueryCache.sync( + return await this.segmentedQueryCache.sync( this.db, - resultsWithFormattedQueries, + results, ); } catch (err) { if ( diff --git a/src/sync/schema-link.ts b/src/sync/schema-link.ts index 55b53f8..2a3bab8 100644 --- a/src/sync/schema-link.ts +++ b/src/sync/schema-link.ts @@ -1,33 +1,64 @@ import { SpanStatusCode, trace } from "@opentelemetry/api"; import { log } from "../log.ts"; import { shutdownController } from "../shutdown.ts"; -import { env } from "../env.ts"; import { withSpan } from "../otel.ts"; import { Connectable } from "./connectable.ts"; +import { findPgDumpBinary, findPgRestoreBinary } from "./executable.ts"; +import { EventEmitter } from "node:events"; export type TableStats = { name: string; }; -export type DumpFormat = "as-text" | "as-binary"; +export type DumpTargetType = "pglite" | "native-postgres"; export class PostgresSchemaLink { - private static readonly PG_DUMP_VERSION = "17.2"; - public static readonly pgDumpBinaryPath = PostgresSchemaLink - .findPgDumpBinary(); - constructor( public readonly connectable: Connectable, - public readonly format: DumpFormat, + public readonly targetType: DumpTargetType, ) {} + excludedSchemas(): string[] { + return DumpCommand.excludedSchemas(this.connectable); + } + + /** + * Dump schema to be consumed exclusively by pglite. + */ + async dumpAsText(): Promise { + const command = DumpCommand.spawn(this.connectable, this.targetType); + const { stdout } = await command.collectOutput(); + return this.sanitizeSchemaForPglite(stdout); + } + + /** + * Used to prepare data being returned to pglite. + * Not necessary for when the target is another pg database + */ + private sanitizeSchemaForPglite(schema: string): string { + // strip CREATE SCHEMA statements and a little bit of extra whitespace. + // Important: we ONLY want to remove the public schema directive. + // If the user wants to dump a different schema it still needs to be created + // we should also remove the comments describing the schema above but meh + return schema.replace(/^CREATE SCHEMA public.*\n\n?/m, "") + // strip unrestrict and restrict statements. They're only valid for psql + // and will break things if imported by pglite + // added in pg_dump 17.6+ + .replace(/^\\(un)?restrict\s+.*\n?/gm, ""); + } +} + +export class DumpCommand + extends EventEmitter<{ restore: [string]; dump: [string] }> { + public static readonly binaryPath = findPgDumpBinary("17.2"); // we're intentionally NOT excluding the "extensions" schema // because supabase has triggers on that schema that cannot be // omitted without manual schema finagling which we want to keep // to a minimum to reduce complexity. // Everything else is safe to exclude // https://gist.github.com/Xetera/067c613580320468e8367d9d6c0e06ad - public static readonly supabaseExcludedSchemas = [ + private static readonly supabaseExcludedSchemas = [ + "extensions", "graphql", "auth", "graphql_public", @@ -38,7 +69,7 @@ export class PostgresSchemaLink { "vault", ]; - public static readonly supabaseExcludedExtensions = [ + private static readonly supabaseExcludedExtensions = [ "pgsodium", "pg_graphql", "supabase_vault", @@ -51,52 +82,23 @@ export class PostgresSchemaLink { "uuid-ossp", ]; - static findPgDumpBinary(): string { - const forcePath = env.PG_DUMP_BINARY; - if (forcePath) { - log.info( - `Using pg_dump binary from env(PG_DUMP_BINARY): ${forcePath}`, - "schema:setup", - ); - return forcePath; - } - const os = Deno.build.os; - const arch = Deno.build.arch; - const shippedPath = - `./bin/pg_dump-${this.PG_DUMP_VERSION}/pg_dump.${os}-${arch}`; - if (!Deno.statSync(shippedPath).isFile) { - throw new Error(`pg_dump binary not found at ${shippedPath}`); - } - log.info(`Using built-in "pg_dump" binary: ${shippedPath}`, "schema:setup"); - return shippedPath; - } - - spawnDumpCommand(): DumpCommand { - const command = this.pgDumpCommand(); - return new DumpCommand(command.spawn()); - } - - async dumpAsText(): Promise { - const command = this.spawnDumpCommand(); - const { stdout } = await command.collectOutput(); - return this.sanitizeSchemaForPglite(stdout); - } - - excludedSchemas() { - if (this.connectable.isSupabase()) { - return PostgresSchemaLink.supabaseExcludedSchemas; - } - return []; + // we don't want to allow callers to construct an instance + // with any arbitrary child process. Use the static method instead + private constructor(private readonly process: Deno.ChildProcess) { + super(); } - excludedExtensions() { - if (this.connectable.isSupabase()) { - return PostgresSchemaLink.supabaseExcludedExtensions; + static excludedSchemas(connectable: Connectable): string[] { + if (connectable.isSupabase()) { + return this.supabaseExcludedSchemas; } return []; } - private pgDumpCommand(): Deno.Command { + static spawn( + connectable: Connectable, + targetType: DumpTargetType, + ): DumpCommand { const args = [ // the owner doesn't exist "--no-owner", @@ -109,66 +111,23 @@ export class PostgresSchemaLink { // not sure if this is 100% necessary but we don't want triggers anyway "--disable-triggers", "--schema-only", - ...this.formatFlags(), - ...this.extraFlags(), - this.connectable.toString(), + ...DumpCommand.formatFlags(targetType), + ...DumpCommand.extraFlags(connectable, targetType), + connectable.toString(), ]; - return new Deno.Command(PostgresSchemaLink.pgDumpBinaryPath, { + const command = new Deno.Command(DumpCommand.binaryPath, { args, + stdin: "null", stdout: "piped", stderr: "piped", signal: shutdownController.signal, }); - } - /** - * Text format is used when the dump is restored by pglite - * we use the binary format when piping the command to pg_restore - * or any other locally running postgres instance - */ - private formatFlags(): string[] { - if (this.format === "as-binary") { - return ["--format", "custom"]; - } - return ["--format", "plain"]; - } + const process = command.spawn(); - private extraFlags(): string[] { - // creating an array twice just for flags is super inefficient - return [ - ...this.excludedSchemas().flatMap((schema) => [ - "--exclude-schema", - schema, - ]), - ...this.excludedExtensions().flatMap( - (extension) => [ - "--exclude-extension", - extension, - ], - ), - ]; + return new DumpCommand(process); } - /** - * Used to prepare data being returned to pglite. - * Not necessary for when the target is another pg database - */ - private sanitizeSchemaForPglite(schema: string): string { - // strip CREATE SCHEMA statements and a little bit of extra whitespace. - // Important: we ONLY want to remove the public schema directive. - // If the user wants to dump a different schema it still needs to be created - // we should also remove the comments describing the schema above but meh - return schema.replace(/^CREATE SCHEMA public.*\n\n?/m, "") - // strip unrestrict and restrict statements. They're only valid for psql - // and will break things if imported by pglite - // added in pg_dump 17.6+ - .replace(/^\\(un)?restrict\s+.*\n?/gm, ""); - } -} - -class DumpCommand { - constructor(private readonly process: Deno.ChildProcess) {} - async collectOutput(): Promise { const span = trace.getActiveSpan(); const decoder = new TextDecoder(); @@ -194,6 +153,105 @@ class DumpCommand { return { stdout, stderr }; })(); } + + async pipeTo(restore: RestoreCommand): Promise { + // Start consuming stderr in the background to prevent resource leaks + // const stderrPromise = this.process.stderr.text(); + + const decoder = new TextDecoder(); + this.process.stderr.pipeTo( + new WritableStream({ + write: (chunk) => { + this.emit("dump", decoder.decode(chunk)); + }, + }), + ); + + restore.stderr.pipeTo( + new WritableStream({ + write: (chunk) => { + this.emit("dump", decoder.decode(chunk)); + }, + }), + ); + restore.stdout.pipeTo( + new WritableStream({ + write: (chunk) => { + this.emit("dump", decoder.decode(chunk)); + }, + }), + ); + + try { + await this.process.stdout.pipeTo(restore.stdin); + } catch (error) { + return { + dump: { + status: await this.process.status, + }, + }; + } + + const dumpStatus = await this.process.status; + // this only fails if the command is non-zero + const restoreStatus = await restore.status; + const out = { + dump: { + status: dumpStatus, + }, + restore: { + status: restoreStatus, + }, + }; + await restore.cleanup(); + return out; + } + + /** + * Text format is used when the dump is restored by pglite + * we use the binary format when piping the command to pg_restore + * or any other locally running postgres instance + */ + private static formatFlags(format: DumpTargetType): string[] { + if (format === "native-postgres") { + return ["--format", "custom"]; + } + return ["--format", "plain"]; + } + + private static excludedExtensions(connectable: Connectable): string[] { + if (connectable.isSupabase()) { + return this.supabaseExcludedExtensions; + } + return []; + } + + private static extraFlags( + connectable: Connectable, + format: DumpTargetType, + ): string[] { + // creating an array twice just for flags is super inefficient + const flags = [ + ...this.excludedSchemas(connectable).flatMap((schema) => [ + "--exclude-schema", + schema, + ]), + ...this.excludedExtensions(connectable).flatMap( + (extension) => [ + "--exclude-extension", + extension, + ], + ), + ]; + // we want to drop existing objects when syncing + // to regular postgres. Not needed for pglite + // since we always create a new db anyway + if (format === "native-postgres") { + flags.push("--clean", "--if-exists"); + } + + return flags; + } } export type DumpCommandOutput = { @@ -201,5 +259,76 @@ export type DumpCommandOutput = { stderr?: string; }; -// Don't allow class construction outside the file -export type { DumpCommand }; +/** + * Represents a `pg_restore` command. + * This class does NOT perform cleanup for the target database like is needed when syncing from supabase. + * + * Commands like `drop schema if exists extensions cascade;` need to be run independently after the restore. + */ +export class RestoreCommand { + public static readonly binaryPath = findPgRestoreBinary("17.2"); + private constructor(private process: Deno.ChildProcess) {} + + static spawn(connectable: Connectable): RestoreCommand { + const args = [ + "--no-owner", + "--no-acl", + "--clean", + "--if-exists", + "--verbose", + ...RestoreCommand.formatFlags(), + "--dbname", + connectable.toString(), + ]; + + const command = new Deno.Command(RestoreCommand.binaryPath, { + args, + stdin: "piped", + stdout: "piped", + stderr: "piped", + signal: shutdownController.signal, + }); + + const process = command.spawn(); + + return new RestoreCommand(process); + } + + get stdin() { + return this.process.stdin; + } + + get stdout() { + return this.process.stdout; + } + + get stderr() { + return this.process.stderr; + } + + get status() { + return this.process.status; + } + + async cleanup() { + if (!this.process.stdout.locked) { + await this.process.stdout.cancel(); + } + if (!this.process.stderr.locked) { + await this.process.stderr.cancel(); + } + } + + private static formatFlags(): string[] { + return ["--format", "custom"]; + } +} + +export type RestoreCommandResult = { + dump: { + status: Deno.CommandStatus; + }; + restore?: { + status: Deno.CommandStatus; + }; +}; diff --git a/src/sync/schema_differ.ts b/src/sync/schema_differ.ts index fd55727..22f5337 100644 --- a/src/sync/schema_differ.ts +++ b/src/sync/schema_differ.ts @@ -1,11 +1,11 @@ -import type { Postgres } from "@query-doctor/core"; import { create } from "jsondiffpatch"; import { format, type Op } from "jsondiffpatch/formatters/jsonpatch"; import { z } from "zod"; +import { Connectable } from "./connectable.ts"; export class SchemaDiffer { - private differ = create({ - arrays: { detectMove: true, }, + private readonly differ = create({ + arrays: { detectMove: true }, objectHash(obj, index) { // shouldn't happen but we don't want to throw an error for this if (!("type" in obj)) { @@ -28,9 +28,9 @@ export class SchemaDiffer { }, }); - private stats: Map = new Map(); + private readonly stats = new WeakMap(); - put(postgres: Postgres, schema: FullSchema): Op[] | undefined { + put(postgres: Connectable, schema: FullSchema): Op[] | undefined { const old = this.stats.get(postgres); if (!old) { this.stats.set(postgres, schema); diff --git a/src/sync/seen-cache.ts b/src/sync/seen-cache.ts index b6857c1..f7704cb 100644 --- a/src/sync/seen-cache.ts +++ b/src/sync/seen-cache.ts @@ -1,21 +1,21 @@ -import type { RawRecentQuery, RecentQuery } from "./pg-connector.ts"; import type { Postgres } from "@query-doctor/core"; +import { QueryHash, RawRecentQuery, RecentQuery } from "../sql/recent-query.ts"; +import { fingerprint } from "@libpg-query/parser"; + interface CacheEntry { firstSeen: number; lastSeen: number; } -type Query = string; - export class QueryCache { - list: Record = {}; + private list: Record = {}; private readonly createdAt: number; constructor() { this.createdAt = Date.now(); } - isCached(key: string): boolean { + isCached(key: QueryHash): boolean { const entry = this.list[key]; if (!entry) { return false; @@ -23,7 +23,7 @@ export class QueryCache { return true; } - isNew(key: string): boolean { + isNew(key: QueryHash): boolean { const entry = this.list[key]; if (!entry) { return true; @@ -31,9 +31,8 @@ export class QueryCache { return entry.firstSeen >= this.createdAt; } - store(query: string) { - // TODO: use fingerprint from @libpg-query/parser instead of the full query string - const key = query; + async store(recentQuery: RawRecentQuery): Promise { + const key = await this.hash(recentQuery.query); const now = Date.now(); if (this.list[key]) { this.list[key].lastSeen = now; @@ -43,23 +42,25 @@ export class QueryCache { return key; } - getFirstSeen(key: string): number { + getFirstSeen(key: QueryHash): number { return this.list[key]?.firstSeen || Date.now(); } - sync(queries: RawRecentQuery[]): RecentQuery[] { - return queries.map((query) => { - const key = this.store(query.query); - return { - ...query, - firstSeen: this.getFirstSeen(key), - }; - }); + async sync(rawQueries: RawRecentQuery[]): Promise { + // TODO: bound the concurrency + return await Promise.all(rawQueries.map(async (rawQuery) => { + const key = await this.store(rawQuery); + return RecentQuery.analyze(rawQuery, key, this.getFirstSeen(key)); + })); } - reset() { + reset(): void { this.list = {}; } + + private async hash(query: string): Promise { + return QueryHash.parse(await fingerprint(query)); + } } /** @@ -69,14 +70,14 @@ export class SegmentedQueryCache { // weak reference to the db instance to allow cache to be garbage collected // when the connection to the database is closed. // Can be relevant for - dbs: WeakMap = new WeakMap(); + private readonly dbs: WeakMap = new WeakMap(); - sync(db: Postgres, queries: RawRecentQuery[]): RecentQuery[] { + sync(db: Postgres, queries: RawRecentQuery[]): Promise { const cache = this.getOrCreateCache(db); return cache.sync(queries); } - store(db: Postgres, query: string) { + store(db: Postgres, query: RawRecentQuery) { const cache = this.getOrCreateCache(db); return cache.store(query); } diff --git a/src/sync/sync_test.ts b/src/sync/sync.test.ts similarity index 79% rename from src/sync/sync_test.ts rename to src/sync/sync.test.ts index b4c3198..2deb81b 100644 --- a/src/sync/sync_test.ts +++ b/src/sync/sync.test.ts @@ -10,8 +10,8 @@ function testDb(): DatabaseConnector<{ table: string; }> { const db = { - users: [{ id: 0 }, { id: 1 }, { id: 2 }], - posts: [ + "public.users": [{ id: 0 }, { id: 1 }, { id: 2 }], + "public.posts": [ { id: 3, poster_id: 0 }, { id: 4, poster_id: 1 }, ], @@ -45,11 +45,11 @@ function testDb(): DatabaseConnector<{ get(table, values) { const found = db[table as keyof typeof db].find((row) => { for (const [key, value] of Object.entries(values)) { - if (String(row[key as keyof typeof row]) !== value) { - return false; + if (String(row[key as keyof typeof row]) === String(value)) { + return true; } } - return Promise.resolve(true); + return false; }); return Promise.resolve(found ? { data: found, table } : undefined); }, @@ -66,15 +66,15 @@ Deno.test(async function addTest() { maxRows: 8, seed: 0, }); - const result = await da.findAllDependencies(new Map()); + const graph = await da.buildGraph( + await dbSimple.dependencies({ excludedSchemas: [] }), + ); + const result = await da.findAllDependencies(graph); assertEquals(result.items, { - posts: [ + "public.posts": [ { id: 3, poster_id: 0 }, { id: 4, poster_id: 1 }, ], - users: [{ id: 0 }, { id: 1 }], + "public.users": [{ id: 0 }, { id: 1 }], }); - // const mockConnector = { - // } - // assertEquals(add(2, 3), 5); }); diff --git a/src/sync/syncer.ts b/src/sync/syncer.ts index aabd664..d9ea142 100644 --- a/src/sync/syncer.ts +++ b/src/sync/syncer.ts @@ -3,20 +3,14 @@ import { type DependencyAnalyzerOptions, DependencyResolutionNotice, } from "./dependency-tree.ts"; -import { PostgresConnector, RecentQueriesResult } from "./pg-connector.ts"; +import { RecentQueriesResult } from "./pg-connector.ts"; import { PostgresSchemaLink } from "./schema-link.ts"; import { withSpan } from "../otel.ts"; import { Connectable } from "./connectable.ts"; -import { - ExportedStats, - type Postgres, - type PostgresFactory, - PostgresVersion, - Statistics, -} from "@query-doctor/core"; -import { SegmentedQueryCache } from "./seen-cache.ts"; +import { ExportedStats, PostgresVersion, Statistics } from "@query-doctor/core"; import { SchemaDiffer } from "./schema_differ.ts"; import { ExtensionNotInstalledError } from "./errors.ts"; +import { ConnectionManager } from "./connection-manager.ts"; type SyncOptions = DependencyAnalyzerOptions; @@ -44,11 +38,11 @@ export type SyncResult = { }; export class PostgresSyncer { - private readonly connections = new Map(); - private readonly segmentedQueryCache = new SegmentedQueryCache(); private readonly differ = new SchemaDiffer(); - constructor(private readonly factory: PostgresFactory) {} + constructor( + private readonly manager: ConnectionManager, + ) {} /** * @throws {ExtensionNotInstalledError} @@ -58,9 +52,9 @@ export class PostgresSyncer { connectable: Connectable, options: SyncOptions, ): Promise { - const sql = this.getConnection(connectable); - const connector = new PostgresConnector(sql, this.segmentedQueryCache); - const link = new PostgresSchemaLink(connectable, "as-text"); + const sql = this.manager.getOrCreateConnection(connectable); + const connector = this.manager.getConnectorFor(sql); + const link = new PostgresSchemaLink(connectable, "pglite"); const analyzer = new DependencyAnalyzer(connector, options); const [ stats, @@ -119,7 +113,7 @@ export class PostgresSyncer { }); } - this.differ.put(sql, serializedResult.schema); + this.differ.put(connectable, serializedResult.schema); const wrapped = schema + serializedResult.serialized; @@ -139,13 +133,12 @@ export class PostgresSyncer { * @throws {PostgresError} */ async liveQuery(connectable: Connectable) { - const sql = this.getConnection(connectable); - const connector = new PostgresConnector(sql, this.segmentedQueryCache); + const connector = this.manager.getConnectorFor(connectable); const [queries, schema] = await Promise.all([ connector.getRecentQueries(), connector.getSchema(), ]); - const deltas = this.differ.put(sql, schema); + const deltas = this.differ.put(connectable, schema); return { queries, deltas }; } @@ -156,18 +149,7 @@ export class PostgresSyncer { async reset( connectable: Connectable, ): Promise { - const sql = this.getConnection(connectable); - const connector = new PostgresConnector(sql, this.segmentedQueryCache); + const connector = this.manager.getConnectorFor(connectable); await connector.resetPgStatStatements(); } - - private getConnection(connectable: Connectable) { - const urlString = connectable.toString(); - let sql = this.connections.get(urlString); - if (!sql) { - sql = this.factory({ url: urlString }); - this.connections.set(urlString, sql); - } - return sql; - } }