From db4b7d0542b2b98822b31737085a5cf806da7db5 Mon Sep 17 00:00:00 2001 From: Xetera Date: Sat, 29 Nov 2025 03:44:30 +0300 Subject: [PATCH 01/29] feat: restoring state from an upstream db --- deno.json | 1 + deno.lock | 845 +++++++++++++++++++++++- devenv.lock | 4 +- src/env.ts | 1 + src/main.ts | 5 - src/remote/remote.test.ts | 53 ++ src/remote/remote.ts | 37 ++ src/sync/connectable.test.ts | 13 + src/sync/connectable.ts | 12 + src/sync/executable.ts | 76 +++ src/sync/schema-link.ts | 288 +++++--- src/sync/{sync_test.ts => sync.test.ts} | 22 +- src/sync/syncer.ts | 2 +- 13 files changed, 1219 insertions(+), 140 deletions(-) create mode 100644 src/remote/remote.test.ts create mode 100644 src/remote/remote.ts create mode 100644 src/sync/connectable.test.ts create mode 100644 src/sync/executable.ts rename src/sync/{sync_test.ts => sync.test.ts} (79%) diff --git a/deno.json b/deno.json index 417bf7a..8f6a246 100644 --- a/deno.json +++ b/deno.json @@ -31,6 +31,7 @@ "@std/collections": "jsr:@std/collections@^1.1.3", "@std/data-structures": "jsr:@std/data-structures@^1.0.9", "@std/fmt": "jsr:@std/fmt@^1.0.8", + "@testcontainers/postgresql": "npm:@testcontainers/postgresql@^11.9.0", "@types/node": "npm:@types/node@^24.9.1", "@types/nunjucks": "npm:@types/nunjucks@^3.2.6", "chokidar": "npm:chokidar@^4.0.3", diff --git a/deno.lock b/deno.lock index 50955c6..b6b0496 100644 --- a/deno.lock +++ b/deno.lock @@ -9,20 +9,19 @@ "jsr:@std/data-structures@^1.0.9": "1.0.9", "jsr:@std/fmt@^1.0.8": "1.0.8", "jsr:@std/internal@^1.0.10": "1.0.10", - "npm:@actions/core@*": "1.11.1", "npm:@actions/core@^1.11.1": "1.11.1", - "npm:@actions/github@*": "6.0.1_@octokit+core@5.2.1", - "npm:@actions/github@^6.0.1": "6.0.1_@octokit+core@5.2.1", + "npm:@actions/github@^6.0.1": "6.0.1_@octokit+core@5.2.2", "npm:@libpg-query/parser@^17.7.0": "17.7.0", "npm:@pgsql/types@^17.6.1": "17.6.1", "npm:@query-doctor/core@^0.0.3": "0.0.3", + "npm:@testcontainers/postgresql@^11.9.0": "11.9.0", "npm:@types/node@^24.9.1": "24.9.1", "npm:@types/nunjucks@^3.2.6": "3.2.6", "npm:chokidar@^4.0.3": "4.0.3", "npm:dedent@^1.6.0": "1.6.0", "npm:fast-csv@^5.0.5": "5.0.5", "npm:jsondiffpatch@~0.7.3": "0.7.3", - "npm:nunjucks@^3.2.4": "3.2.4", + "npm:nunjucks@^3.2.4": "3.2.4_chokidar@4.0.3", "npm:pgsql-deparser@^17.11.1": "17.11.1", "npm:sql-formatter@^15.6.6": "15.6.6", "npm:sql-highlight@^6.1.0": "6.1.0", @@ -71,7 +70,7 @@ "@actions/io" ] }, - "@actions/github@6.0.1_@octokit+core@5.2.1": { + "@actions/github@6.0.1_@octokit+core@5.2.2": { "integrity": "sha512-xbZVcaqD4XnQAe35qSQqskb3SqIAfRyLBrHMd/8TuL7hJSz2QtbDwnNM8zWx4zO5l2fnGtseNE3MbEvD7BxVMw==", "dependencies": [ "@actions/http-client", @@ -80,19 +79,22 @@ "@octokit/plugin-rest-endpoint-methods", "@octokit/request", "@octokit/request-error", - "undici" + "undici@5.29.0" ] }, "@actions/http-client@2.2.3": { "integrity": "sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA==", "dependencies": [ "tunnel", - "undici" + "undici@5.29.0" ] }, "@actions/io@1.1.3": { "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==" }, + "@balena/dockerignore@1.0.2": { + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==" + }, "@dmsnell/diff-match-patch@1.1.0": { "integrity": "sha512-yejLPmM5pjsGvxS9gXablUSbInW7H976c/FJ4iQxWIm7/38xBySRemTPDe34lhg1gVLbJntX0+sH0jYfU+PN9A==" }, @@ -244,6 +246,47 @@ "@fastify/busboy@2.1.1": { "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==" }, + "@grpc/grpc-js@1.14.1": { + "integrity": "sha512-sPxgEWtPUR3EnRJCEtbGZG2iX8LQDUls2wUS3o27jg07KqJFMq6YDeWvMo1wfpmy3rqRdS0rivpLwhqQtEyCuQ==", + "dependencies": [ + "@grpc/proto-loader@0.8.0", + "@js-sdsl/ordered-map" + ] + }, + "@grpc/proto-loader@0.7.15": { + "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", + "dependencies": [ + "lodash.camelcase", + "long", + "protobufjs", + "yargs" + ], + "bin": true + }, + "@grpc/proto-loader@0.8.0": { + "integrity": "sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ==", + "dependencies": [ + "lodash.camelcase", + "long", + "protobufjs", + "yargs" + ], + "bin": true + }, + "@isaacs/cliui@8.0.2": { + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dependencies": [ + "string-width@5.1.2", + "string-width-cjs@npm:string-width@4.2.3", + "strip-ansi@7.1.2", + "strip-ansi-cjs@npm:strip-ansi@6.0.1", + "wrap-ansi@8.1.0", + "wrap-ansi-cjs@npm:wrap-ansi@7.0.0" + ] + }, + "@js-sdsl/ordered-map@4.4.2": { + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==" + }, "@launchql/protobufjs@7.2.6": { "integrity": "sha512-vwi1nG2/heVFsIMHQU1KxTjUp5c757CTtRAZn/jutApCkFlle1iv8tzM/DHlSZJKDldxaYqnNYTg0pTyp8Bbtg==", "dependencies": [ @@ -257,7 +300,7 @@ "@protobufjs/path", "@protobufjs/pool", "@protobufjs/utf8", - "@types/node", + "@types/node@24.9.1", "long" ], "scripts": true @@ -272,8 +315,8 @@ "@octokit/auth-token@4.0.0": { "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==" }, - "@octokit/core@5.2.1": { - "integrity": "sha512-dKYCMuPO1bmrpuogcjQ8z7ICCH3FP6WmxpwC03yjzGfZhj9fTJg6+bS1+UAplekbN2C+M61UNllGOOoAfGCrdQ==", + "@octokit/core@5.2.2": { + "integrity": "sha512-/g2d4sW9nUDJOMz3mabVQvOGhVa4e/BN/Um7yca9Bb2XTzPPnfTWHWQg+IsEYO7M3Vx+EXvaM/I2pJWIMun1bg==", "dependencies": [ "@octokit/auth-token", "@octokit/graphql", @@ -305,14 +348,14 @@ "@octokit/openapi-types@24.2.0": { "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==" }, - "@octokit/plugin-paginate-rest@9.2.2_@octokit+core@5.2.1": { + "@octokit/plugin-paginate-rest@9.2.2_@octokit+core@5.2.2": { "integrity": "sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==", "dependencies": [ "@octokit/core", "@octokit/types@12.6.0" ] }, - "@octokit/plugin-rest-endpoint-methods@10.4.1_@octokit+core@5.2.1": { + "@octokit/plugin-rest-endpoint-methods@10.4.1_@octokit+core@5.2.2": { "integrity": "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==", "dependencies": [ "@octokit/core", @@ -351,6 +394,9 @@ "@pgsql/types@17.6.1": { "integrity": "sha512-Hk51+nyOxS7Dy5oySWywyNZxo5HndX1VDXT4ZEBD+p+vvMFM2Vc+sKSuByCiI8banou4edbgdnOC251IOuq7QQ==" }, + "@pkgjs/parseargs@0.11.0": { + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==" + }, "@protobufjs/aspromise@1.1.2": { "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" }, @@ -395,33 +441,252 @@ "zod" ] }, + "@testcontainers/postgresql@11.9.0": { + "integrity": "sha512-beLyLdLygFllktviM132Xd6tQ4i5FnuyZP+4BQEjUb5sJYHYnIrV/ZBzRRflIlF8gugt1GXgudkmr/HxM9vtKw==", + "dependencies": [ + "testcontainers" + ] + }, + "@types/docker-modem@3.0.6": { + "integrity": "sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==", + "dependencies": [ + "@types/node@24.2.0", + "@types/ssh2@1.15.5" + ] + }, + "@types/dockerode@3.3.47": { + "integrity": "sha512-ShM1mz7rCjdssXt7Xz0u1/R2BJC7piWa3SJpUBiVjCf2A3XNn4cP6pUVaD8bLanpPVVn4IKzJuw3dOvkJ8IbYw==", + "dependencies": [ + "@types/docker-modem", + "@types/node@24.2.0", + "@types/ssh2@1.15.5" + ] + }, + "@types/node@18.19.130": { + "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "dependencies": [ + "undici-types@5.26.5" + ] + }, + "@types/node@24.2.0": { + "integrity": "sha512-3xyG3pMCq3oYCNg7/ZP+E1ooTaGB4cG8JWRsqqOYQdbWNY4zbaV0Ennrd7stjiJEFZCaybcIgpTjJWHRfBSIDw==", + "dependencies": [ + "undici-types@7.10.0" + ] + }, "@types/node@24.9.1": { "integrity": "sha512-QoiaXANRkSXK6p0Duvt56W208du4P9Uye9hWLWgGMDTEoKPhuenzNcC4vGUmrNkiOKTlIrBoyNQYNpSwfEZXSg==", "dependencies": [ - "undici-types" + "undici-types@7.16.0" ] }, "@types/nunjucks@3.2.6": { "integrity": "sha512-pHiGtf83na1nCzliuAdq8GowYiXvH5l931xZ0YEHaLMNFgynpEqx+IPStlu7UaDkehfvl01e4x/9Tpwhy7Ue3w==" }, + "@types/ssh2-streams@0.1.13": { + "integrity": "sha512-faHyY3brO9oLEA0QlcO8N2wT7R0+1sHWZvQ+y3rMLwdY1ZyS1z0W3t65j9PqT4HmQ6ALzNe7RZlNuCNE0wBSWA==", + "dependencies": [ + "@types/node@24.2.0" + ] + }, + "@types/ssh2@0.5.52": { + "integrity": "sha512-lbLLlXxdCZOSJMCInKH2+9V/77ET2J6NPQHpFI0kda61Dd1KglJs+fPQBchizmzYSOJBgdTajhPqBO1xxLywvg==", + "dependencies": [ + "@types/node@24.2.0", + "@types/ssh2-streams" + ] + }, + "@types/ssh2@1.15.5": { + "integrity": "sha512-N1ASjp/nXH3ovBHddRJpli4ozpk6UdDYIX4RJWFa9L1YKnzdhTlVmiGHm4DZnj/jLbqZpes4aeR30EFGQtvhQQ==", + "dependencies": [ + "@types/node@18.19.130" + ] + }, "a-sync-waterfall@1.0.1": { "integrity": "sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA==" }, + "abort-controller@3.0.0": { + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": [ + "event-target-shim" + ] + }, + "ansi-regex@5.0.1": { + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "ansi-regex@6.2.2": { + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==" + }, + "ansi-styles@4.3.0": { + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": [ + "color-convert" + ] + }, + "ansi-styles@6.2.3": { + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==" + }, + "archiver-utils@5.0.2": { + "integrity": "sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==", + "dependencies": [ + "glob", + "graceful-fs", + "is-stream", + "lazystream", + "lodash", + "normalize-path", + "readable-stream@4.7.0" + ] + }, + "archiver@7.0.1": { + "integrity": "sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==", + "dependencies": [ + "archiver-utils", + "async", + "buffer-crc32", + "readable-stream@4.7.0", + "readdir-glob", + "tar-stream@3.1.7", + "zip-stream" + ] + }, "argparse@2.0.1": { "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, "asap@2.0.6": { "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" }, + "asn1@0.2.6": { + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "dependencies": [ + "safer-buffer" + ] + }, + "async-lock@1.4.1": { + "integrity": "sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==" + }, + "async@3.2.6": { + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==" + }, + "b4a@1.7.3": { + "integrity": "sha512-5Q2mfq2WfGuFp3uS//0s6baOJLMoVduPYVeNmDYxu5OUA1/cBfvr2RIS7vi62LdNj/urk1hfmj867I3qt6uZ7Q==" + }, + "balanced-match@1.0.2": { + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "bare-events@2.8.2": { + "integrity": "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==" + }, + "bare-fs@4.5.2_bare-events@2.8.2": { + "integrity": "sha512-veTnRzkb6aPHOvSKIOy60KzURfBdUflr5VReI+NSaPL6xf+XLdONQgZgpYvUuZLVQ8dCqxpBAudaOM1+KpAUxw==", + "dependencies": [ + "bare-events", + "bare-path", + "bare-stream", + "bare-url", + "fast-fifo" + ] + }, + "bare-os@3.6.2": { + "integrity": "sha512-T+V1+1srU2qYNBmJCXZkUY5vQ0B4FSlL3QDROnKQYOqeiQR8UbjNHlPa+TIbM4cuidiN9GaTaOZgSEgsvPbh5A==" + }, + "bare-path@3.0.0": { + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "dependencies": [ + "bare-os" + ] + }, + "bare-stream@2.7.0_bare-events@2.8.2": { + "integrity": "sha512-oyXQNicV1y8nc2aKffH+BUHFRXmx6VrPzlnaEvMhram0nPBrKcEdcyBg5r08D0i8VxngHFAiVyn1QKXpSG0B8A==", + "dependencies": [ + "bare-events", + "streamx" + ], + "optionalPeers": [ + "bare-events" + ] + }, + "bare-url@2.3.2": { + "integrity": "sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw==", + "dependencies": [ + "bare-path" + ] + }, + "base64-js@1.5.1": { + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "bcrypt-pbkdf@1.0.2": { + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "dependencies": [ + "tweetnacl" + ] + }, "before-after-hook@2.2.3": { "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==" }, + "bl@4.1.0": { + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": [ + "buffer@5.7.1", + "inherits", + "readable-stream@3.6.2" + ] + }, + "brace-expansion@2.0.2": { + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dependencies": [ + "balanced-match" + ] + }, + "buffer-crc32@1.0.0": { + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==" + }, + "buffer@5.7.1": { + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dependencies": [ + "base64-js", + "ieee754" + ] + }, + "buffer@6.0.3": { + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "dependencies": [ + "base64-js", + "ieee754" + ] + }, + "buildcheck@0.0.7": { + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==" + }, + "byline@5.0.0": { + "integrity": "sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==" + }, "chokidar@4.0.3": { "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", "dependencies": [ "readdirp" ] }, + "chownr@1.1.4": { + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "cliui@8.0.1": { + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": [ + "string-width@4.2.3", + "strip-ansi@6.0.1", + "wrap-ansi@7.0.0" + ] + }, + "color-convert@2.0.1": { + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": [ + "color-name" + ] + }, + "color-name@1.1.4": { + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, "colorette@2.0.20": { "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==" }, @@ -431,6 +696,52 @@ "commander@5.1.0": { "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==" }, + "compress-commons@6.0.2": { + "integrity": "sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==", + "dependencies": [ + "crc-32", + "crc32-stream", + "is-stream", + "normalize-path", + "readable-stream@4.7.0" + ] + }, + "core-util-is@1.0.3": { + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "cpu-features@0.0.10": { + "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==", + "dependencies": [ + "buildcheck", + "nan" + ], + "scripts": true + }, + "crc-32@1.2.2": { + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "bin": true + }, + "crc32-stream@6.0.0": { + "integrity": "sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==", + "dependencies": [ + "crc-32", + "readable-stream@4.7.0" + ] + }, + "cross-spawn@7.0.6": { + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dependencies": [ + "path-key", + "shebang-command", + "which" + ] + }, + "debug@4.4.3": { + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dependencies": [ + "ms" + ] + }, "dedent@1.6.0": { "integrity": "sha512-F1Z+5UCFpmQUzJa11agbyPVMbpgT/qA3/SKyJ1jyBgm7dUcUEa8v9JwDkerSQXfakBwFljIxhOJqGkjUwZ9FSA==" }, @@ -443,12 +754,63 @@ "discontinuous-range@1.0.0": { "integrity": "sha512-c68LpLbO+7kP/b1Hr1qs8/BJ09F5khZGTxqxZuhzxpmwJKOgRFHJWIb9/KmqnqHhLdO55aOxFH/EGBvUQbL/RQ==" }, - "encoding@0.1.13": { - "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "docker-compose@1.3.0": { + "integrity": "sha512-7Gevk/5eGD50+eMD+XDnFnOrruFkL0kSd7jEG4cjmqweDSUhB7i0g8is/nBdVpl+Bx338SqIB2GLKm32M+Vs6g==", + "dependencies": [ + "yaml" + ] + }, + "docker-modem@5.0.6": { + "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==", + "dependencies": [ + "debug", + "readable-stream@3.6.2", + "split-ca", + "ssh2" + ] + }, + "dockerode@4.0.9": { + "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==", + "dependencies": [ + "@balena/dockerignore", + "@grpc/grpc-js", + "@grpc/proto-loader@0.7.15", + "docker-modem", + "protobufjs", + "tar-fs@2.1.4", + "uuid" + ] + }, + "eastasianwidth@0.2.0": { + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" + }, + "emoji-regex@8.0.0": { + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "emoji-regex@9.2.2": { + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "end-of-stream@1.4.5": { + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dependencies": [ + "once" + ] + }, + "escalade@3.2.0": { + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==" + }, + "event-target-shim@5.0.1": { + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" + }, + "events-universal@1.0.1": { + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", "dependencies": [ - "iconv-lite" + "bare-events" ] }, + "events@3.3.0": { + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" + }, "fast-csv@5.0.5": { "integrity": "sha512-9//QpogDIPln5Dc8e3Q3vbSSLXlTeU7z1JqsUOXZYOln8EIn/OOO8+NS2c3ukR6oYngDd3+P1HXSkby3kNV9KA==", "dependencies": [ @@ -456,15 +818,70 @@ "@fast-csv/parse" ] }, + "fast-fifo@1.3.2": { + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" + }, + "foreground-child@3.3.1": { + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dependencies": [ + "cross-spawn", + "signal-exit@4.1.0" + ] + }, + "fs-constants@1.0.0": { + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "fsevents@2.3.3": { "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", "os": ["darwin"], "scripts": true }, - "iconv-lite@0.6.3": { - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "get-caller-file@2.0.5": { + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + }, + "get-port@7.1.0": { + "integrity": "sha512-QB9NKEeDg3xxVwCCwJQ9+xycaz6pBB6iQ76wiWMl1927n0Kir6alPiP+yuiICLLU4jpMe08dXfpebuQppFA2zw==" + }, + "glob@10.5.0": { + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", "dependencies": [ - "safer-buffer" + "foreground-child", + "jackspeak", + "minimatch@9.0.5", + "minipass", + "package-json-from-dist", + "path-scurry" + ], + "bin": true + }, + "graceful-fs@4.2.11": { + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" + }, + "ieee754@1.2.1": { + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, + "inherits@2.0.4": { + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "is-fullwidth-code-point@3.0.0": { + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "is-stream@2.0.1": { + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" + }, + "isarray@1.0.0": { + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "isexe@2.0.0": { + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "jackspeak@3.4.3": { + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dependencies": [ + "@isaacs/cliui" + ], + "optionalDependencies": [ + "@pkgjs/parseargs" ] }, "jsondiffpatch@0.7.3": { @@ -474,6 +891,15 @@ ], "bin": true }, + "lazystream@1.0.1": { + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "dependencies": [ + "readable-stream@2.3.8" + ] + }, + "lodash.camelcase@4.3.0": { + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" + }, "lodash.escaperegexp@4.1.2": { "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==" }, @@ -495,12 +921,46 @@ "lodash.uniq@4.5.0": { "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==" }, + "lodash@4.17.21": { + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, "long@5.3.2": { "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==" }, + "lru-cache@10.4.3": { + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" + }, + "minimatch@5.1.6": { + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": [ + "brace-expansion" + ] + }, + "minimatch@9.0.5": { + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dependencies": [ + "brace-expansion" + ] + }, + "minipass@7.1.2": { + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==" + }, + "mkdirp-classic@0.5.3": { + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, + "mkdirp@1.0.4": { + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "bin": true + }, "moo@0.5.2": { "integrity": "sha512-iSAJLHYKnX41mKcJKjqvnAN9sf0LMDTXDEvFv+ffuRR9a1MIuXLjMNL6EsnDHSkKLTWNqQQ5uo61P4EbU4NU+Q==" }, + "ms@2.1.3": { + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "nan@2.23.1": { + "integrity": "sha512-r7bBUGKzlqk8oPBDYxt6Z0aEdF1G1rwlMcLk8LCOMbOzf0mG+JUfUzG4fIMWwHWP0iyaLWEQZJmtB7nOHEm/qw==" + }, "nearley@2.20.1": { "integrity": "sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ==", "dependencies": [ @@ -511,6 +971,9 @@ ], "bin": true }, + "normalize-path@3.0.0": { + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" + }, "nunjucks@3.2.4": { "integrity": "sha512-26XRV6BhkgK0VOxfbU5cQI+ICFUtMLixv1noZn1tGU38kQH5A5nmmbk/O45xdyBhD1esk47nKrY0mvQpZIhRjQ==", "dependencies": [ @@ -520,18 +983,89 @@ ], "bin": true }, + "nunjucks@3.2.4_chokidar@4.0.3": { + "integrity": "sha512-26XRV6BhkgK0VOxfbU5cQI+ICFUtMLixv1noZn1tGU38kQH5A5nmmbk/O45xdyBhD1esk47nKrY0mvQpZIhRjQ==", + "dependencies": [ + "a-sync-waterfall", + "asap", + "chokidar", + "commander@5.1.0" + ], + "optionalPeers": [ + "chokidar" + ], + "bin": true + }, "once@1.4.0": { "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "dependencies": [ "wrappy" ] }, + "package-json-from-dist@1.0.1": { + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==" + }, + "path-key@3.1.1": { + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" + }, + "path-scurry@1.11.1": { + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dependencies": [ + "lru-cache", + "minipass" + ] + }, "pgsql-deparser@17.11.1": { "integrity": "sha512-BGKgwC4qs+FPcG8Ai989LO6i4E8KF5HEvlTnI8uhS4qUyu6P1xCyP9pJDky95ZL8DolaGUDFAJtxteDBw33OCg==", "dependencies": [ "@pgsql/types" ] }, + "process-nextick-args@2.0.1": { + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "process@0.11.10": { + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==" + }, + "proper-lockfile@4.1.2": { + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "dependencies": [ + "graceful-fs", + "retry", + "signal-exit@3.0.7" + ] + }, + "properties-reader@2.3.0": { + "integrity": "sha512-z597WicA7nDZxK12kZqHr2TcvwNU1GCfA5UwfDY/HDp3hXPoPlb5rlEx9bwGTiJnc0OqbBTkU975jDToth8Gxw==", + "dependencies": [ + "mkdirp" + ] + }, + "protobufjs@7.5.4": { + "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "dependencies": [ + "@protobufjs/aspromise", + "@protobufjs/base64", + "@protobufjs/codegen", + "@protobufjs/eventemitter", + "@protobufjs/fetch", + "@protobufjs/float", + "@protobufjs/inquire", + "@protobufjs/path", + "@protobufjs/pool", + "@protobufjs/utf8", + "@types/node@24.9.1", + "long" + ], + "scripts": true + }, + "pump@3.0.3": { + "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", + "dependencies": [ + "end-of-stream", + "once" + ] + }, "railroad-diagrams@1.0.0": { "integrity": "sha512-cz93DjNeLY0idrCNOH6PviZGRN9GJhsdm9hpn1YCS879fj4W+x5IFJhhkRZcwVgMmFF7R82UA/7Oh+R8lLZg6A==" }, @@ -542,15 +1076,81 @@ "ret" ] }, + "readable-stream@2.3.8": { + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": [ + "core-util-is", + "inherits", + "isarray", + "process-nextick-args", + "safe-buffer@5.1.2", + "string_decoder@1.1.1", + "util-deprecate" + ] + }, + "readable-stream@3.6.2": { + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": [ + "inherits", + "string_decoder@1.3.0", + "util-deprecate" + ] + }, + "readable-stream@4.7.0": { + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "dependencies": [ + "abort-controller", + "buffer@6.0.3", + "events", + "process", + "string_decoder@1.3.0" + ] + }, + "readdir-glob@1.1.3": { + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "dependencies": [ + "minimatch@5.1.6" + ] + }, "readdirp@4.1.2": { "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==" }, + "require-directory@2.1.1": { + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" + }, "ret@0.1.15": { "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==" }, + "retry@0.12.0": { + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==" + }, + "safe-buffer@5.1.2": { + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "safe-buffer@5.2.1": { + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, "safer-buffer@2.1.2": { "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "shebang-command@2.0.0": { + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dependencies": [ + "shebang-regex" + ] + }, + "shebang-regex@3.0.0": { + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" + }, + "signal-exit@3.0.7": { + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" + }, + "signal-exit@4.1.0": { + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==" + }, + "split-ca@1.0.1": { + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==" + }, "sql-formatter@15.6.6": { "integrity": "sha512-bZydXEXhaNDQBr8xYHC3a8thwcaMuTBp0CkKGjwGYDsIB26tnlWeWPwJtSQ0TEwiJcz9iJJON5mFPkx7XroHcg==", "dependencies": [ @@ -562,9 +1162,152 @@ "sql-highlight@6.1.0": { "integrity": "sha512-ed7OK4e9ywpE7pgRMkMQmZDPKSVdm0oX5IEtZiKnFucSF0zu6c80GZBe38UqHuVhTWJ9xsKgSMjCG2bml86KvA==" }, + "ssh-remote-port-forward@1.0.4": { + "integrity": "sha512-x0LV1eVDwjf1gmG7TTnfqIzf+3VPRz7vrNIjX6oYLbeCrf/PeVY6hkT68Mg+q02qXxQhrLjB0jfgvhevoCRmLQ==", + "dependencies": [ + "@types/ssh2@0.5.52", + "ssh2" + ] + }, + "ssh2@1.17.0": { + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", + "dependencies": [ + "asn1", + "bcrypt-pbkdf" + ], + "optionalDependencies": [ + "cpu-features", + "nan" + ], + "scripts": true + }, + "streamx@2.23.0": { + "integrity": "sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg==", + "dependencies": [ + "events-universal", + "fast-fifo", + "text-decoder" + ] + }, + "string-width@4.2.3": { + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": [ + "emoji-regex@8.0.0", + "is-fullwidth-code-point", + "strip-ansi@6.0.1" + ] + }, + "string-width@5.1.2": { + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dependencies": [ + "eastasianwidth", + "emoji-regex@9.2.2", + "strip-ansi@7.1.2" + ] + }, + "string_decoder@1.1.1": { + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": [ + "safe-buffer@5.1.2" + ] + }, + "string_decoder@1.3.0": { + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": [ + "safe-buffer@5.2.1" + ] + }, + "strip-ansi@6.0.1": { + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": [ + "ansi-regex@5.0.1" + ] + }, + "strip-ansi@7.1.2": { + "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", + "dependencies": [ + "ansi-regex@6.2.2" + ] + }, + "tar-fs@2.1.4": { + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "dependencies": [ + "chownr", + "mkdirp-classic", + "pump", + "tar-stream@2.2.0" + ] + }, + "tar-fs@3.1.1": { + "integrity": "sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==", + "dependencies": [ + "pump", + "tar-stream@3.1.7" + ], + "optionalDependencies": [ + "bare-fs", + "bare-path" + ] + }, + "tar-stream@2.2.0": { + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": [ + "bl", + "end-of-stream", + "fs-constants", + "inherits", + "readable-stream@3.6.2" + ] + }, + "tar-stream@3.1.7": { + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "dependencies": [ + "b4a", + "fast-fifo", + "streamx" + ] + }, + "testcontainers@11.9.0": { + "integrity": "sha512-SQ6OqQUig7HcGVF72i+ZVIMvxPSpEz8cgC/B63ekqMzgf98DnveoBbOmqux/Wa5wQAQCt4mEPNMa/Jz7vMg9fQ==", + "dependencies": [ + "@balena/dockerignore", + "@types/dockerode", + "archiver", + "async-lock", + "byline", + "debug", + "docker-compose", + "dockerode", + "get-port", + "proper-lockfile", + "properties-reader", + "ssh-remote-port-forward", + "tar-fs@3.1.1", + "tmp", + "undici@7.16.0" + ] + }, + "text-decoder@1.2.3": { + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "dependencies": [ + "b4a" + ] + }, + "tmp@0.2.5": { + "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==" + }, "tunnel@0.0.6": { "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" }, + "tweetnacl@0.14.5": { + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" + }, + "undici-types@5.26.5": { + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, + "undici-types@7.10.0": { + "integrity": "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag==" + }, "undici-types@7.16.0": { "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==" }, @@ -574,12 +1317,75 @@ "@fastify/busboy" ] }, + "undici@7.16.0": { + "integrity": "sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g==" + }, "universal-user-agent@6.0.1": { "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==" }, + "util-deprecate@1.0.2": { + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "uuid@10.0.0": { + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "bin": true + }, + "which@2.0.2": { + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dependencies": [ + "isexe" + ], + "bin": true + }, + "wrap-ansi@7.0.0": { + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": [ + "ansi-styles@4.3.0", + "string-width@4.2.3", + "strip-ansi@6.0.1" + ] + }, + "wrap-ansi@8.1.0": { + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dependencies": [ + "ansi-styles@6.2.3", + "string-width@5.1.2", + "strip-ansi@7.1.2" + ] + }, "wrappy@1.0.2": { "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, + "y18n@5.0.8": { + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" + }, + "yaml@2.8.1": { + "integrity": "sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==", + "bin": true + }, + "yargs-parser@21.1.1": { + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==" + }, + "yargs@17.7.2": { + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dependencies": [ + "cliui", + "escalade", + "get-caller-file", + "require-directory", + "string-width@4.2.3", + "y18n", + "yargs-parser" + ] + }, + "zip-stream@6.0.1": { + "integrity": "sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==", + "dependencies": [ + "archiver-utils", + "compress-commons", + "readable-stream@4.7.0" + ] + }, "zod@4.1.12": { "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==" } @@ -960,6 +1766,7 @@ "npm:@libpg-query/parser@^17.7.0", "npm:@pgsql/types@^17.6.1", "npm:@query-doctor/core@^0.0.3", + "npm:@testcontainers/postgresql@^11.9.0", "npm:@types/node@^24.9.1", "npm:@types/nunjucks@^3.2.6", "npm:chokidar@^4.0.3", diff --git a/devenv.lock b/devenv.lock index 49e179c..33dca3d 100644 --- a/devenv.lock +++ b/devenv.lock @@ -3,10 +3,10 @@ "devenv": { "locked": { "dir": "src/modules", - "lastModified": 1764227073, + "lastModified": 1764368166, "owner": "cachix", "repo": "devenv", - "rev": "ee868b9986b84b82fee40ecc2524340d4a154961", + "rev": "47a243b97499bfe5d5783d1fc86d9fe776b2497f", "type": "github" }, "original": { diff --git a/src/env.ts b/src/env.ts index 160f7fb..f570937 100644 --- a/src/env.ts +++ b/src/env.ts @@ -5,6 +5,7 @@ const envSchema = z.object({ CI: z.stringbool().default(false), // sync PG_DUMP_BINARY: z.string().optional(), + PG_RESTORE_BINARY: z.string().optional(), HOSTED: z.stringbool().default(false), HOST: z.string().default("0.0.0.0"), PORT: z.coerce.number().min(1024).max(65535).default(2345), diff --git a/src/main.ts b/src/main.ts index 80b6ea7..f3d7b6e 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,7 +2,6 @@ import * as core from "@actions/core"; import { Runner } from "./runner.ts"; import { env } from "./env.ts"; import { log } from "./log.ts"; -import { PostgresSchemaLink } from "./sync/schema-link.ts"; import { createServer } from "./server/http.ts"; import { shutdown } from "./shutdown.ts"; @@ -29,10 +28,6 @@ function runOutsideCI() { `Starting server (${os}-${arch}) on ${env.HOST}:${env.PORT}`, "main", ); - log.info( - `Using pg_dump binary: ${PostgresSchemaLink.pgDumpBinaryPath}`, - "main", - ); createServer(env.HOST, env.PORT); } diff --git a/src/remote/remote.test.ts b/src/remote/remote.test.ts new file mode 100644 index 0000000..dcfbf9a --- /dev/null +++ b/src/remote/remote.test.ts @@ -0,0 +1,53 @@ +import z from "zod"; +import { PostgreSqlContainer } from "@testcontainers/postgresql"; +import { Connectable } from "../sync/connectable.ts"; +import { Remote } from "./remote.ts"; +import postgres from "postgresjs"; +import { assertEquals } from "@std/assert/equals"; +import { wrapGenericPostgresInterface } from "../sql/postgresjs.ts"; + +const connectable = z.string().transform(Connectable.transform); +Deno.test({ + name: "syncs correctly", + sanitizeOps: false, + fn: async () => { + const [sourceDb, targetDb] = await Promise.all([ + new PostgreSqlContainer("postgres:17") + .withCopyContentToContainer([ + { + content: + "create table testing(a int); insert into testing values (1);", + target: "/docker-entrypoint-initdb.d/init.sql", + }, + ]).start(), + new PostgreSqlContainer("postgres:17") + .start(), + ]); + + try { + const target = connectable.parse( + targetDb.getConnectionUri(), + ); + const source = connectable.parse( + sourceDb.getConnectionUri(), + ); + const sql = postgres(target.toString()); + const tablesBefore = + await sql`select tablename from pg_tables where schemaname = 'public'`; + assertEquals(tablesBefore.count, 0); + + const remote = new Remote(target, wrapGenericPostgresInterface); + await remote.syncFrom(source); + + const tablesAfter = + await sql`select tablename from pg_tables where schemaname = 'public'`; + assertEquals(tablesAfter.count, 1); + assertEquals(tablesAfter[0], { tablename: "testing" }); + const rows = await sql`select * from testing`; + // expect no rows to have been synced + assertEquals(rows.length, 0); + } finally { + await Promise.all([sourceDb.stop(), targetDb.stop()]); + } + }, +}); diff --git a/src/remote/remote.ts b/src/remote/remote.ts new file mode 100644 index 0000000..6c551dd --- /dev/null +++ b/src/remote/remote.ts @@ -0,0 +1,37 @@ +import { Postgres, PostgresFactory } from "@query-doctor/core"; +import { Connectable } from "../sync/connectable.ts"; +import { DumpCommand, RestoreCommand } from "../sync/schema-link.ts"; + +export class Remote { + constructor( + private readonly targetDb: Connectable, + private createPool: PostgresFactory, + ) {} + + async syncFrom( + source: Connectable, + databaseName?: string, + ): Promise { + const target = databaseName + ? this.targetDb.withDatabaseName(databaseName) + : this.targetDb; + const dump = DumpCommand.spawn(source, "native-postgres"); + const restore = RestoreCommand.spawn(target); + await dump.pipeTo(restore); + return this.onSuccessfulSync(target); + } + + /** + * Process a successful sync and run any potential cleanup functions + */ + private async onSuccessfulSync( + newConnection: Connectable, + ): Promise { + const postgres = this.createPool({ url: newConnection.toString() }); + if (this.targetDb.isSupabase()) { + // https://gist.github.com/Xetera/067c613580320468e8367d9d6c0e06ad + await postgres.exec("drop schema if exists extensions cascade"); + } + return postgres; + } +} diff --git a/src/sync/connectable.test.ts b/src/sync/connectable.test.ts new file mode 100644 index 0000000..66b7665 --- /dev/null +++ b/src/sync/connectable.test.ts @@ -0,0 +1,13 @@ +import { assertEquals } from "@std/assert"; +import { Connectable } from "./connectable.ts"; + +Deno.test("connectable", () => { + const connectable = Connectable.fromString( + "postgres://user:password@localhost:5432/dbname?a=b&c=d", + ); + const newConnectable = connectable.withDatabaseName("testing"); + assertEquals( + newConnectable.toString(), + "postgres://user:password@localhost:5432/testing?a=b&c=d", + ); +}); diff --git a/src/sync/connectable.ts b/src/sync/connectable.ts index 2642dff..959b650 100644 --- a/src/sync/connectable.ts +++ b/src/sync/connectable.ts @@ -13,6 +13,12 @@ export class Connectable { return this.url.hostname.endsWith("supabase.com"); } + withDatabaseName(databaseName: string) { + const newUrl = new URL(this.url); + newUrl.pathname = `/${databaseName}`; + return new Connectable(newUrl); + } + /** * Custom logic for parsing a string into a Connectable through zod. */ @@ -80,6 +86,10 @@ export class Connectable { return new Connectable(url); } + static fromString(url: string): Connectable { + return ConnectableParser.parse(url); + } + private static extractSupabaseAccount(url: URL): string | undefined { const match = url.toString().match(/db\.(\w+)\.supabase\.co/); if (!match) { @@ -99,3 +109,5 @@ export class Connectable { return this.url.toString(); } } + +const ConnectableParser = z.string().transform(Connectable.transform); diff --git a/src/sync/executable.ts b/src/sync/executable.ts new file mode 100644 index 0000000..2aabe6d --- /dev/null +++ b/src/sync/executable.ts @@ -0,0 +1,76 @@ +import { env } from "../env.ts"; +import { log } from "../log.ts"; + +const decoder = new TextDecoder(); + +// Is there a way to get this working for windows? +function lookupBinary( + os: typeof Deno.build.os, + name: string, +): string | undefined { + try { + if (os === "linux" || os === "darwin") { + const output = new Deno.Command("which", { args: [name] }).outputSync(); + return decoder.decode(output.stdout) || undefined; + } + } catch (_error) { + // it's not in path. No problem + } +} + +export function findPgRestoreBinary(version: string): string { + const forcePath = env.PG_RESTORE_BINARY; + if (forcePath) { + log.info( + `Using pg_restore binary from env(PG_RESTORE_BINARY): ${forcePath}`, + "schema:setup", + ); + return forcePath; + } + const os = Deno.build.os; + const arch = Deno.build.arch; + const existing = lookupBinary(os, "pg_restore")?.trim(); + if (existing) { + log.info( + `Using pg_restore binary from PATH: ${existing.trim()}`, + "schema:setup", + ); + return existing; + } + const shippedPath = `./bin/pg_restore-${version}/pg_restore.${os}-${arch}`; + if (!Deno.statSync(shippedPath).isFile) { + throw new Error(`pg_restore binary not found at ${shippedPath}`); + } + log.info( + `Using built-in "pg_restore" binary: ${shippedPath}`, + "schema:setup", + ); + return shippedPath; +} + +export function findPgDumpBinary(version: string): string { + const forcePath = env.PG_DUMP_BINARY; + if (forcePath) { + log.info( + `Using pg_dump binary from env(PG_DUMP_BINARY): ${forcePath}`, + "schema:setup", + ); + return forcePath; + } + const os = Deno.build.os; + const arch = Deno.build.arch; + const existing = lookupBinary(os, "pg_dump")?.trim(); + if (existing) { + log.info( + `Using pg_dump binary from PATH: ${existing}`, + "schema:setup", + ); + return existing; + } + const shippedPath = `./bin/pg_dump-${version}/pg_dump.${os}-${arch}`; + if (!Deno.statSync(shippedPath).isFile) { + throw new Error(`pg_dump binary not found at ${shippedPath}`); + } + log.info(`Using built-in "pg_dump" binary: ${shippedPath}`, "schema:setup"); + return shippedPath; +} diff --git a/src/sync/schema-link.ts b/src/sync/schema-link.ts index 55b53f8..05e007c 100644 --- a/src/sync/schema-link.ts +++ b/src/sync/schema-link.ts @@ -1,33 +1,62 @@ import { SpanStatusCode, trace } from "@opentelemetry/api"; import { log } from "../log.ts"; import { shutdownController } from "../shutdown.ts"; -import { env } from "../env.ts"; import { withSpan } from "../otel.ts"; import { Connectable } from "./connectable.ts"; +import { findPgDumpBinary, findPgRestoreBinary } from "./executable.ts"; export type TableStats = { name: string; }; -export type DumpFormat = "as-text" | "as-binary"; +export type DumpTargetType = "pglite" | "native-postgres"; export class PostgresSchemaLink { - private static readonly PG_DUMP_VERSION = "17.2"; - public static readonly pgDumpBinaryPath = PostgresSchemaLink - .findPgDumpBinary(); - constructor( public readonly connectable: Connectable, - public readonly format: DumpFormat, + public readonly targetType: DumpTargetType, ) {} + excludedSchemas(): string[] { + return DumpCommand.excludedSchemas(this.connectable); + } + + /** + * Dump schema to be consumed exclusively by pglite. + */ + async dumpAsText(): Promise { + const command = DumpCommand.spawn(this.connectable, this.targetType); + const { stdout } = await command.collectOutput(); + return this.sanitizeSchemaForPglite(stdout); + } + + /** + * Used to prepare data being returned to pglite. + * Not necessary for when the target is another pg database + */ + private sanitizeSchemaForPglite(schema: string): string { + // strip CREATE SCHEMA statements and a little bit of extra whitespace. + // Important: we ONLY want to remove the public schema directive. + // If the user wants to dump a different schema it still needs to be created + // we should also remove the comments describing the schema above but meh + return schema.replace(/^CREATE SCHEMA public.*\n\n?/m, "") + // strip unrestrict and restrict statements. They're only valid for psql + // and will break things if imported by pglite + // added in pg_dump 17.6+ + .replace(/^\\(un)?restrict\s+.*\n?/gm, ""); + } +} + +export class DumpCommand { + public static readonly binaryPath = findPgDumpBinary("17.2"); // we're intentionally NOT excluding the "extensions" schema // because supabase has triggers on that schema that cannot be // omitted without manual schema finagling which we want to keep // to a minimum to reduce complexity. // Everything else is safe to exclude // https://gist.github.com/Xetera/067c613580320468e8367d9d6c0e06ad - public static readonly supabaseExcludedSchemas = [ + private static readonly supabaseExcludedSchemas = [ + "extensions", "graphql", "auth", "graphql_public", @@ -38,7 +67,7 @@ export class PostgresSchemaLink { "vault", ]; - public static readonly supabaseExcludedExtensions = [ + private static readonly supabaseExcludedExtensions = [ "pgsodium", "pg_graphql", "supabase_vault", @@ -51,52 +80,21 @@ export class PostgresSchemaLink { "uuid-ossp", ]; - static findPgDumpBinary(): string { - const forcePath = env.PG_DUMP_BINARY; - if (forcePath) { - log.info( - `Using pg_dump binary from env(PG_DUMP_BINARY): ${forcePath}`, - "schema:setup", - ); - return forcePath; - } - const os = Deno.build.os; - const arch = Deno.build.arch; - const shippedPath = - `./bin/pg_dump-${this.PG_DUMP_VERSION}/pg_dump.${os}-${arch}`; - if (!Deno.statSync(shippedPath).isFile) { - throw new Error(`pg_dump binary not found at ${shippedPath}`); - } - log.info(`Using built-in "pg_dump" binary: ${shippedPath}`, "schema:setup"); - return shippedPath; - } + // we don't want to allow callers to construct an instance + // with any arbitrary child process. Use the static method instead + private constructor(private readonly process: Deno.ChildProcess) {} - spawnDumpCommand(): DumpCommand { - const command = this.pgDumpCommand(); - return new DumpCommand(command.spawn()); - } - - async dumpAsText(): Promise { - const command = this.spawnDumpCommand(); - const { stdout } = await command.collectOutput(); - return this.sanitizeSchemaForPglite(stdout); - } - - excludedSchemas() { - if (this.connectable.isSupabase()) { - return PostgresSchemaLink.supabaseExcludedSchemas; + static excludedSchemas(connectable: Connectable): string[] { + if (connectable.isSupabase()) { + return this.supabaseExcludedSchemas; } return []; } - excludedExtensions() { - if (this.connectable.isSupabase()) { - return PostgresSchemaLink.supabaseExcludedExtensions; - } - return []; - } - - private pgDumpCommand(): Deno.Command { + static spawn( + connectable: Connectable, + targetType: DumpTargetType, + ): DumpCommand { const args = [ // the owner doesn't exist "--no-owner", @@ -109,65 +107,22 @@ export class PostgresSchemaLink { // not sure if this is 100% necessary but we don't want triggers anyway "--disable-triggers", "--schema-only", - ...this.formatFlags(), - ...this.extraFlags(), - this.connectable.toString(), + ...DumpCommand.formatFlags(targetType), + ...DumpCommand.extraFlags(connectable, targetType), + connectable.toString(), ]; - return new Deno.Command(PostgresSchemaLink.pgDumpBinaryPath, { + const command = new Deno.Command(DumpCommand.binaryPath, { args, + stdin: "null", stdout: "piped", stderr: "piped", signal: shutdownController.signal, }); - } - - /** - * Text format is used when the dump is restored by pglite - * we use the binary format when piping the command to pg_restore - * or any other locally running postgres instance - */ - private formatFlags(): string[] { - if (this.format === "as-binary") { - return ["--format", "custom"]; - } - return ["--format", "plain"]; - } - private extraFlags(): string[] { - // creating an array twice just for flags is super inefficient - return [ - ...this.excludedSchemas().flatMap((schema) => [ - "--exclude-schema", - schema, - ]), - ...this.excludedExtensions().flatMap( - (extension) => [ - "--exclude-extension", - extension, - ], - ), - ]; - } + const process = command.spawn(); - /** - * Used to prepare data being returned to pglite. - * Not necessary for when the target is another pg database - */ - private sanitizeSchemaForPglite(schema: string): string { - // strip CREATE SCHEMA statements and a little bit of extra whitespace. - // Important: we ONLY want to remove the public schema directive. - // If the user wants to dump a different schema it still needs to be created - // we should also remove the comments describing the schema above but meh - return schema.replace(/^CREATE SCHEMA public.*\n\n?/m, "") - // strip unrestrict and restrict statements. They're only valid for psql - // and will break things if imported by pglite - // added in pg_dump 17.6+ - .replace(/^\\(un)?restrict\s+.*\n?/gm, ""); + return new DumpCommand(process); } -} - -class DumpCommand { - constructor(private readonly process: Deno.ChildProcess) {} async collectOutput(): Promise { const span = trace.getActiveSpan(); @@ -194,6 +149,80 @@ class DumpCommand { return { stdout, stderr }; })(); } + + async pipeTo(restore: RestoreCommand) { + // Start consuming stderr in the background to prevent resource leaks + const stderrPromise = this.process.stderr.text(); + + try { + await this.process.stdout.pipeTo(restore.stdin); + } catch (error) { + const stderr = await stderrPromise; + if (stderr) { + throw new Error(`pg_dump failed: ${stderr}`, { cause: error }); + } + throw error; + } + + const dumpStatus = await this.process.status; + if (dumpStatus.code !== 0) { + const stderr = await stderrPromise; + if (stderr) { + throw new Error(`pg_dump failed: ${stderr}`); + } else { + throw new Error(`pg_dump failed with code ${dumpStatus.code}`); + } + } + await stderrPromise; + await restore.status; + await restore.cleanup(); + } + + /** + * Text format is used when the dump is restored by pglite + * we use the binary format when piping the command to pg_restore + * or any other locally running postgres instance + */ + private static formatFlags(format: DumpTargetType): string[] { + if (format === "native-postgres") { + return ["--format", "custom"]; + } + return ["--format", "plain"]; + } + + private static excludedExtensions(connectable: Connectable): string[] { + if (connectable.isSupabase()) { + return this.supabaseExcludedExtensions; + } + return []; + } + + private static extraFlags( + connectable: Connectable, + format: DumpTargetType, + ): string[] { + // creating an array twice just for flags is super inefficient + const flags = [ + ...this.excludedSchemas(connectable).flatMap((schema) => [ + "--exclude-schema", + schema, + ]), + ...this.excludedExtensions(connectable).flatMap( + (extension) => [ + "--exclude-extension", + extension, + ], + ), + ]; + // we want to drop existing objects when syncing + // to regular postgres. Not needed for pglite + // since we always create a new db anyway + if (format === "native-postgres") { + flags.push("--clean", "--if-exists"); + } + + return flags; + } } export type DumpCommandOutput = { @@ -201,5 +230,60 @@ export type DumpCommandOutput = { stderr?: string; }; -// Don't allow class construction outside the file -export type { DumpCommand }; +/** + * Represents a `pg_restore` command. + * This class does NOT perform cleanup for the target database like is needed when syncing from supabase. + * + * Commands like `drop schema if exists extensions cascade;` need to be run independently after the restore. + */ +export class RestoreCommand { + public static readonly binaryPath = findPgRestoreBinary("17.2"); + private constructor(private process: Deno.ChildProcess) {} + + static spawn(connectable: Connectable): RestoreCommand { + const args = [ + "--no-owner", + "--no-acl", + ...RestoreCommand.formatFlags(), + "--dbname", + connectable.toString(), + ]; + + const command = new Deno.Command(RestoreCommand.binaryPath, { + args, + stdin: "piped", + stdout: "piped", + stderr: "piped", + signal: shutdownController.signal, + }); + + const process = command.spawn(); + + return new RestoreCommand(process); + } + + get stdin() { + return this.process.stdin; + } + + get stderr() { + return this.process.stderr; + } + + get status() { + return this.process.status; + } + + async cleanup() { + if (!this.process.stdout.locked) { + await this.process.stdout.cancel(); + } + if (!this.process.stderr.locked) { + await this.process.stderr.cancel(); + } + } + + private static formatFlags(): string[] { + return ["--format", "custom"]; + } +} diff --git a/src/sync/sync_test.ts b/src/sync/sync.test.ts similarity index 79% rename from src/sync/sync_test.ts rename to src/sync/sync.test.ts index b4c3198..2deb81b 100644 --- a/src/sync/sync_test.ts +++ b/src/sync/sync.test.ts @@ -10,8 +10,8 @@ function testDb(): DatabaseConnector<{ table: string; }> { const db = { - users: [{ id: 0 }, { id: 1 }, { id: 2 }], - posts: [ + "public.users": [{ id: 0 }, { id: 1 }, { id: 2 }], + "public.posts": [ { id: 3, poster_id: 0 }, { id: 4, poster_id: 1 }, ], @@ -45,11 +45,11 @@ function testDb(): DatabaseConnector<{ get(table, values) { const found = db[table as keyof typeof db].find((row) => { for (const [key, value] of Object.entries(values)) { - if (String(row[key as keyof typeof row]) !== value) { - return false; + if (String(row[key as keyof typeof row]) === String(value)) { + return true; } } - return Promise.resolve(true); + return false; }); return Promise.resolve(found ? { data: found, table } : undefined); }, @@ -66,15 +66,15 @@ Deno.test(async function addTest() { maxRows: 8, seed: 0, }); - const result = await da.findAllDependencies(new Map()); + const graph = await da.buildGraph( + await dbSimple.dependencies({ excludedSchemas: [] }), + ); + const result = await da.findAllDependencies(graph); assertEquals(result.items, { - posts: [ + "public.posts": [ { id: 3, poster_id: 0 }, { id: 4, poster_id: 1 }, ], - users: [{ id: 0 }, { id: 1 }], + "public.users": [{ id: 0 }, { id: 1 }], }); - // const mockConnector = { - // } - // assertEquals(add(2, 3), 5); }); diff --git a/src/sync/syncer.ts b/src/sync/syncer.ts index aabd664..cf615a7 100644 --- a/src/sync/syncer.ts +++ b/src/sync/syncer.ts @@ -60,7 +60,7 @@ export class PostgresSyncer { ): Promise { const sql = this.getConnection(connectable); const connector = new PostgresConnector(sql, this.segmentedQueryCache); - const link = new PostgresSchemaLink(connectable, "as-text"); + const link = new PostgresSchemaLink(connectable, "pglite"); const analyzer = new DependencyAnalyzer(connector, options); const [ stats, From fd53f350cff0aee5b78cf1056878676dc6dfaa79 Mon Sep 17 00:00:00 2001 From: Xetera Date: Tue, 2 Dec 2025 17:19:33 +0300 Subject: [PATCH 02/29] feat: add remote controller, syncing queries --- deno.json | 6 +- deno.lock | 42 ++++----- src/remote/remote-controller.test.ts | 82 +++++++++++++++++ src/remote/remote-controller.ts | 38 ++++++++ src/remote/remote.dto.ts | 13 +++ src/remote/remote.test.ts | 27 ++++-- src/remote/remote.ts | 128 +++++++++++++++++++++++---- src/server/http.ts | 21 ++++- src/sql/postgresjs.ts | 2 +- src/sql/recent-query.ts | 44 +++++++++ src/sync/connectable.test.ts | 5 +- src/sync/connectable.ts | 5 +- src/sync/connection-manager.ts | 34 +++++++ src/sync/pg-connector.ts | 36 ++------ src/sync/schema-link.ts | 50 +++++++---- src/sync/schema_differ.ts | 6 +- src/sync/seen-cache.ts | 21 ++--- src/sync/syncer.ts | 39 +++----- 18 files changed, 454 insertions(+), 145 deletions(-) create mode 100644 src/remote/remote-controller.test.ts create mode 100644 src/remote/remote-controller.ts create mode 100644 src/remote/remote.dto.ts create mode 100644 src/sql/recent-query.ts create mode 100644 src/sync/connection-manager.ts diff --git a/deno.json b/deno.json index 8f6a246..6531de1 100644 --- a/deno.json +++ b/deno.json @@ -1,9 +1,7 @@ { "$schema": "https://raw.githubusercontent.com/denoland/deno/refs/heads/main/cli/schemas/config-file.v1.json", "version": "0.1.1", - "unstable": [ - "raw-imports" - ], + "unstable": ["raw-imports"], "permissions": { "run": { "sys": true, @@ -25,7 +23,7 @@ "@libpg-query/parser": "npm:@libpg-query/parser@^17.7.0", "@opentelemetry/api": "jsr:@opentelemetry/api@^1.9.0", "@pgsql/types": "npm:@pgsql/types@^17.6.1", - "@query-doctor/core": "npm:@query-doctor/core@^0.0.3", + "@query-doctor/core": "npm:@query-doctor/core@^0.0.4", "@rabbit-company/rate-limiter": "jsr:@rabbit-company/rate-limiter@^3.0.0", "@std/assert": "jsr:@std/assert@^1.0.14", "@std/collections": "jsr:@std/collections@^1.1.3", diff --git a/deno.lock b/deno.lock index b6b0496..1db2b02 100644 --- a/deno.lock +++ b/deno.lock @@ -13,19 +13,19 @@ "npm:@actions/github@^6.0.1": "6.0.1_@octokit+core@5.2.2", "npm:@libpg-query/parser@^17.7.0": "17.7.0", "npm:@pgsql/types@^17.6.1": "17.6.1", - "npm:@query-doctor/core@^0.0.3": "0.0.3", + "npm:@query-doctor/core@^0.0.4": "0.0.4", "npm:@testcontainers/postgresql@^11.9.0": "11.9.0", - "npm:@types/node@^24.9.1": "24.9.1", + "npm:@types/node@^24.9.1": "24.10.1", "npm:@types/nunjucks@^3.2.6": "3.2.6", "npm:chokidar@^4.0.3": "4.0.3", "npm:dedent@^1.6.0": "1.6.0", "npm:fast-csv@^5.0.5": "5.0.5", "npm:jsondiffpatch@~0.7.3": "0.7.3", "npm:nunjucks@^3.2.4": "3.2.4_chokidar@4.0.3", - "npm:pgsql-deparser@^17.11.1": "17.11.1", + "npm:pgsql-deparser@^17.11.1": "17.12.1", "npm:sql-formatter@^15.6.6": "15.6.6", "npm:sql-highlight@^6.1.0": "6.1.0", - "npm:zod@^4.1.12": "4.1.12" + "npm:zod@^4.1.12": "4.1.13" }, "jsr": { "@opentelemetry/api@1.9.0": { @@ -300,7 +300,7 @@ "@protobufjs/path", "@protobufjs/pool", "@protobufjs/utf8", - "@types/node@24.9.1", + "@types/node@24.10.1", "long" ], "scripts": true @@ -431,8 +431,8 @@ "@protobufjs/utf8@1.1.0": { "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, - "@query-doctor/core@0.0.3": { - "integrity": "sha512-krDTeLQNnBD3ZtM5cMLoxZhqPwFQMZaOgtJ3po4Tpq7O//ZPwAz4f/qIScpInFl29W35DlE6E6UrvO3Ud/4JGA==", + "@query-doctor/core@0.0.4": { + "integrity": "sha512-kv7tERboZeOr3ebIxaaHkxFbYS+IPEIKAsZRbEz8UCYFpm8ZkT4C2+4gOQ86SksSRaLNSFkwF7DeELtcpMbKpw==", "dependencies": [ "@pgsql/types", "colorette", @@ -468,16 +468,16 @@ "undici-types@5.26.5" ] }, - "@types/node@24.2.0": { - "integrity": "sha512-3xyG3pMCq3oYCNg7/ZP+E1ooTaGB4cG8JWRsqqOYQdbWNY4zbaV0Ennrd7stjiJEFZCaybcIgpTjJWHRfBSIDw==", + "@types/node@24.10.1": { + "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", "dependencies": [ - "undici-types@7.10.0" + "undici-types@7.16.0" ] }, - "@types/node@24.9.1": { - "integrity": "sha512-QoiaXANRkSXK6p0Duvt56W208du4P9Uye9hWLWgGMDTEoKPhuenzNcC4vGUmrNkiOKTlIrBoyNQYNpSwfEZXSg==", + "@types/node@24.2.0": { + "integrity": "sha512-3xyG3pMCq3oYCNg7/ZP+E1ooTaGB4cG8JWRsqqOYQdbWNY4zbaV0Ennrd7stjiJEFZCaybcIgpTjJWHRfBSIDw==", "dependencies": [ - "undici-types@7.16.0" + "undici-types@7.10.0" ] }, "@types/nunjucks@3.2.6": { @@ -1015,8 +1015,8 @@ "minipass" ] }, - "pgsql-deparser@17.11.1": { - "integrity": "sha512-BGKgwC4qs+FPcG8Ai989LO6i4E8KF5HEvlTnI8uhS4qUyu6P1xCyP9pJDky95ZL8DolaGUDFAJtxteDBw33OCg==", + "pgsql-deparser@17.12.1": { + "integrity": "sha512-G27wb4rhXNwaV8+/ni3RlGV+CMk753ErX341c4rY98hI8xPbPI07dYGxT3asur0HW4SyyNZ4cS1vhfwLfxPLaA==", "dependencies": [ "@pgsql/types" ] @@ -1054,7 +1054,7 @@ "@protobufjs/path", "@protobufjs/pool", "@protobufjs/utf8", - "@types/node@24.9.1", + "@types/node@24.10.1", "long" ], "scripts": true @@ -1359,8 +1359,8 @@ "y18n@5.0.8": { "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" }, - "yaml@2.8.1": { - "integrity": "sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==", + "yaml@2.8.2": { + "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", "bin": true }, "yargs-parser@21.1.1": { @@ -1386,8 +1386,8 @@ "readable-stream@4.7.0" ] }, - "zod@4.1.12": { - "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==" + "zod@4.1.13": { + "integrity": "sha512-AvvthqfqrAhNH9dnfmrfKzX5upOdjUVJYFqNSlkmGf64gRaTzlPwz99IHYnVs28qYAybvAlBV+H7pn0saFY4Ig==" } }, "redirects": { @@ -1765,7 +1765,7 @@ "npm:@actions/github@^6.0.1", "npm:@libpg-query/parser@^17.7.0", "npm:@pgsql/types@^17.6.1", - "npm:@query-doctor/core@^0.0.3", + "npm:@query-doctor/core@^0.0.4", "npm:@testcontainers/postgresql@^11.9.0", "npm:@types/node@^24.9.1", "npm:@types/nunjucks@^3.2.6", diff --git a/src/remote/remote-controller.test.ts b/src/remote/remote-controller.test.ts new file mode 100644 index 0000000..b976417 --- /dev/null +++ b/src/remote/remote-controller.test.ts @@ -0,0 +1,82 @@ +import z from "zod"; +import { PostgreSqlContainer } from "@testcontainers/postgresql"; +import { Connectable } from "../sync/connectable.ts"; +import { Remote } from "./remote.ts"; +import postgres from "postgresjs"; +import { assertEquals } from "@std/assert/equals"; +import { wrapGenericPostgresInterface } from "../sql/postgresjs.ts"; +import { RemoteController } from "./remote-controller.ts"; +import { ConnectionManager } from "../sync/connection-manager.ts"; + +const connectable = z.string().transform(Connectable.transform); +Deno.test({ + name: "controller syncs correctly", + sanitizeOps: false, + // deno is weird... the sync seems like it might be leaking resources? + sanitizeResources: false, + fn: async () => { + const [sourceDb, targetDb] = await Promise.all([ + new PostgreSqlContainer("postgres:17") + .withCopyContentToContainer([ + { + content: ` + create table testing(a int, b text); + insert into testing values (1); + create index on testing(b); + create extension pg_stat_statements; + select * from testing where a = 1; + `, + target: "/docker-entrypoint-initdb.d/init.sql", + }, + ]) + .withCommand(["-c", "shared_preload_libraries=pg_stat_statements"]) + .start(), + new PostgreSqlContainer("postgres:17").start(), + ]); + + try { + const target = connectable.parse( + targetDb.getConnectionUri(), + ); + const source = connectable.parse( + sourceDb.getConnectionUri(), + ); + + const man = new ConnectionManager(wrapGenericPostgresInterface); + const remote = new RemoteController( + new Remote(target, man), + ); + + const response = await remote.execute( + new Request( + "http://testing.local/postgres", + { + method: "POST", + body: JSON.stringify({ + db: source.toString(), + }), + }, + ), + ); + + console.log(await response?.json()); + assertEquals(response?.status, 200); + + const sql = postgres( + target.withDatabaseName(Remote.optimizingDbName).toString(), + ); + const tablesAfter = + await sql`select tablename from pg_tables where schemaname = 'public'`; + assertEquals(tablesAfter.count, 1); + const indexesAfter = + await sql`select * from pg_indexes where schemaname = 'public'`; + assertEquals(indexesAfter.count, 1); + assertEquals(tablesAfter[0], { tablename: "testing" }); + const rows = await sql`select * from testing`; + // expect no rows to have been synced + assertEquals(rows.length, 0); + } finally { + await Promise.all([sourceDb.stop(), targetDb.stop()]); + } + }, +}); diff --git a/src/remote/remote-controller.ts b/src/remote/remote-controller.ts new file mode 100644 index 0000000..b4ab390 --- /dev/null +++ b/src/remote/remote-controller.ts @@ -0,0 +1,38 @@ +import { env } from "../env.ts"; +import { RemoteSyncRequest } from "./remote.dto.ts"; +import { Remote } from "./remote.ts"; + +export class RemoteController { + constructor( + private readonly remote: Remote, + ) {} + + async execute( + request: Request, + ): Promise { + const url = new URL(request.url); + if (url.pathname === "/postgres" && request.method === "POST") { + return await this.onFullSync(request); + } + } + + async onFullSync(request: Request): Promise { + const body = RemoteSyncRequest.safeParse(await request.json()); + if (!body.success) { + return new Response(JSON.stringify(body.error), { status: 400 }); + } + const { db } = body.data; + try { + const sync = await this.remote.syncFrom(db); + return Response.json(sync); + } catch (error) { + console.error(error); + return Response.json({ + error: env.HOSTED ? "Internal Server Error" : error, + message: "Failed to sync database", + }, { + status: 500, + }); + } + } +} diff --git a/src/remote/remote.dto.ts b/src/remote/remote.dto.ts new file mode 100644 index 0000000..8305018 --- /dev/null +++ b/src/remote/remote.dto.ts @@ -0,0 +1,13 @@ +import { z } from "zod"; +import { Connectable } from "../sync/connectable.ts"; +import { RecentQuery } from "../sql/recent-query.ts"; + +export const RemoteSyncRequest = z.object({ + db: z.string().transform(Connectable.transform), +}); + +export const RemoteSyncResponse = z.object({ + queries: z.array(z.instanceof(RecentQuery)), +}); + +export type RemoteSyncResponse = z.infer; diff --git a/src/remote/remote.test.ts b/src/remote/remote.test.ts index dcfbf9a..aaf8c1d 100644 --- a/src/remote/remote.test.ts +++ b/src/remote/remote.test.ts @@ -5,23 +5,31 @@ import { Remote } from "./remote.ts"; import postgres from "postgresjs"; import { assertEquals } from "@std/assert/equals"; import { wrapGenericPostgresInterface } from "../sql/postgresjs.ts"; +import { ConnectionManager } from "../sync/connection-manager.ts"; const connectable = z.string().transform(Connectable.transform); Deno.test({ name: "syncs correctly", sanitizeOps: false, + // deno is weird... the sync seems like it might be leaking resources? + sanitizeResources: false, fn: async () => { const [sourceDb, targetDb] = await Promise.all([ new PostgreSqlContainer("postgres:17") .withCopyContentToContainer([ { content: - "create table testing(a int); insert into testing values (1);", + "create table testing(a int, b text); insert into testing values (1); create index on testing(b)", target: "/docker-entrypoint-initdb.d/init.sql", }, ]).start(), new PostgreSqlContainer("postgres:17") - .start(), + .withCopyContentToContainer([ + { + content: "create table testing(a int); create index on testing(a)", + target: "/docker-entrypoint-initdb.d/init.sql", + }, + ]).start(), ]); try { @@ -31,16 +39,21 @@ Deno.test({ const source = connectable.parse( sourceDb.getConnectionUri(), ); - const sql = postgres(target.toString()); - const tablesBefore = - await sql`select tablename from pg_tables where schemaname = 'public'`; - assertEquals(tablesBefore.count, 0); - const remote = new Remote(target, wrapGenericPostgresInterface); + const remote = new Remote( + target, + new ConnectionManager(wrapGenericPostgresInterface), + ); await remote.syncFrom(source); + const sql = postgres( + target.withDatabaseName(Remote.optimizingDbName).toString(), + ); const tablesAfter = await sql`select tablename from pg_tables where schemaname = 'public'`; + const indexesAfter = + await sql`select * from pg_indexes where schemaname = 'public'`; + assertEquals(indexesAfter.count, 1); assertEquals(tablesAfter.count, 1); assertEquals(tablesAfter[0], { tablename: "testing" }); const rows = await sql`select * from testing`; diff --git a/src/remote/remote.ts b/src/remote/remote.ts index 6c551dd..aca9a1c 100644 --- a/src/remote/remote.ts +++ b/src/remote/remote.ts @@ -1,37 +1,127 @@ -import { Postgres, PostgresFactory } from "@query-doctor/core"; +import { PgIdentifier, Postgres } from "@query-doctor/core"; import { Connectable } from "../sync/connectable.ts"; import { DumpCommand, RestoreCommand } from "../sync/schema-link.ts"; +import { ConnectionManager } from "../sync/connection-manager.ts"; +import { RecentQuery } from "../sql/recent-query.ts"; +import { SchemaDiffer } from "../sync/schema_differ.ts"; +import { RemoteSyncResponse } from "./remote.dto.ts"; +/** + * Represents a db for doing optimization work. + * We only maintain one instance of this class as we only do + * optimization against one physical postgres database. + * But potentially more logical databases in the future. + */ export class Remote { + public static readonly baseDbName = PgIdentifier.fromString("postgres"); + public static readonly optimizingDbName = PgIdentifier.fromString( + "optimizing_db", + ); + + private readonly differ = new SchemaDiffer(); + + /** + * We have to juggle 2 different connections to the Remote + * + * 1 -> connection to `/postgres` where we manage other databases. + * this pool stays connected long-term. That's this variable + * + * 2 -> connections to {@link Remote.databaseName}. This connection pool is + * destroyed and re-created on each successful sync along with the db itself + */ + private readonly baseDb: Postgres; + constructor( - private readonly targetDb: Connectable, - private createPool: PostgresFactory, - ) {} + /** This has to be a local url. Very bad things will happen if this is a remote URL */ + private readonly targetURL: Connectable, + private readonly manager: ConnectionManager, + ) { + const baseUrl = targetURL.withDatabaseName(Remote.baseDbName); + this.baseDb = this.manager.getOrCreateConnection(baseUrl); + } - async syncFrom( - source: Connectable, - databaseName?: string, - ): Promise { - const target = databaseName - ? this.targetDb.withDatabaseName(databaseName) - : this.targetDb; + async syncFrom(source: Connectable): Promise { + await this.resetDatabase(); + const target = this.targetURL.withDatabaseName(Remote.optimizingDbName); + const sql = this.manager.getOrCreateConnection(source); + const [restoreResult, recentQueries, fullSchema] = await Promise.allSettled( + [ + this.pipeSchema(target, source), + this.getRecentConnections(source), + this.getFullSchema(source), + ], + ); + + if (fullSchema.status === "fulfilled") { + this.differ.put(sql, fullSchema.value); + } + + const pg = this.manager.getOrCreateConnection(this.targetURL); + await this.onSuccessfulSync(pg); + return { + queries: recentQueries.status === "fulfilled" ? recentQueries.value : [], + }; + } + + /** + * Drops and recreates the {@link Remote.optimizingDbName} db. + * + * TODO: allow juggling multiple databases in the future + */ + private async resetDatabase() { + const databaseName = Remote.optimizingDbName; + // these cannot be run in the same `exec` block as that implicitly creates transactions + await this.baseDb.exec( + // drop database does not allow parameterization + `drop database if exists ${databaseName} with (force);`, + ); + await this.baseDb.exec(`create database ${databaseName};`); + } + + private async pipeSchema(target: Connectable, source: Connectable) { const dump = DumpCommand.spawn(source, "native-postgres"); const restore = RestoreCommand.spawn(target); - await dump.pipeTo(restore); - return this.onSuccessfulSync(target); + const { dump: dumpResult, restore: restoreResult } = await dump.pipeTo( + restore, + ); + if (dumpResult.error) { + console.error(dumpResult.error); + } + if (!dumpResult.status.success) { + throw new Error( + `Dump failed with status ${dumpResult.status.code}\n${dumpResult.error}`, + ); + } + if (restoreResult?.error) { + console.error(restoreResult.error); + } + if (restoreResult && !restoreResult.status.success) { + console.log(restoreResult.error); + throw new Error( + `Restore failed with status ${restoreResult.status.code}\n${restoreResult.error}`, + ); + } + } + + private getRecentConnections( + source: Connectable, + ): Promise { + const connector = this.manager.getConnectorFor(source); + return connector.getRecentQueries(); + } + + private getFullSchema(source: Connectable) { + const connector = this.manager.getConnectorFor(source); + return connector.getSchema(); } /** * Process a successful sync and run any potential cleanup functions */ - private async onSuccessfulSync( - newConnection: Connectable, - ): Promise { - const postgres = this.createPool({ url: newConnection.toString() }); - if (this.targetDb.isSupabase()) { + private async onSuccessfulSync(postgres: Postgres): Promise { + if (this.targetURL.isSupabase()) { // https://gist.github.com/Xetera/067c613580320468e8367d9d6c0e06ad await postgres.exec("drop schema if exists extensions cascade"); } - return postgres; } } diff --git a/src/server/http.ts b/src/server/http.ts index b3f35d4..1ce54e9 100644 --- a/src/server/http.ts +++ b/src/server/http.ts @@ -10,8 +10,13 @@ import { SyncResult } from "../sync/syncer.ts"; import { wrapGenericPostgresInterface } from "../sql/postgresjs.ts"; import type { RateLimitResult } from "@rabbit-company/rate-limiter"; import * as errors from "../sync/errors.ts"; +import { RemoteController } from "../remote/remote-controller.ts"; +import { Connectable } from "../sync/connectable.ts"; +import { ConnectionManager } from "../sync/connection-manager.ts"; +import { Remote } from "../remote/remote.ts"; -const syncer = new PostgresSyncer(wrapGenericPostgresInterface); +const manager = new ConnectionManager(wrapGenericPostgresInterface); +const syncer = new PostgresSyncer(manager); async function onSync(req: Request) { const startTime = Date.now(); @@ -148,7 +153,15 @@ async function onReset(req: Request) { } } -export function createServer(hostname: string, port: number) { +export function createServer( + hostname: string, + port: number, + targetDb?: Connectable, +) { + const manager = new ConnectionManager(wrapGenericPostgresInterface); + const remoteController = targetDb + ? new RemoteController(new Remote(targetDb, manager)) + : undefined; return Deno.serve( { hostname, port, signal: shutdownController.signal }, async (req, info) => { @@ -193,6 +206,10 @@ export function createServer(hostname: string, port: number) { const res = await onReset(req); return transformResponse(res, limit); } + const remoteResponse = await remoteController?.execute(req); + if (remoteResponse) { + return transformResponse(remoteResponse, limit); + } return new Response("Not found", { status: 404 }); }, ); diff --git a/src/sql/postgresjs.ts b/src/sql/postgresjs.ts index 4d29825..3d71b27 100644 --- a/src/sql/postgresjs.ts +++ b/src/sql/postgresjs.ts @@ -33,7 +33,7 @@ export function wrapGenericPostgresInterface( throw new Error("Invalid input"); } return { - exec: async (query, params) => { + exec: (query, params) => { return pg.unsafe(query, params as postgres.ParameterOrJSON[]); }, serverNum: async () => diff --git a/src/sql/recent-query.ts b/src/sql/recent-query.ts new file mode 100644 index 0000000..c5b7eca --- /dev/null +++ b/src/sql/recent-query.ts @@ -0,0 +1,44 @@ +import { format } from "sql-formatter"; +// deno-lint-ignore no-unused-vars +import type { SegmentedQueryCache } from "../sync/seen-cache.ts"; + +/** + * Constructed by {@link SegmentedQueryCache} by supplying the + * date the query was last seen + */ +export class RecentQuery { + public readonly formattedQuery: string; + public readonly username: string; + public readonly query: string; + public readonly meanTime: number; + public readonly calls: string; + public readonly rows: string; + public readonly topLevel: boolean; + + constructor( + data: RawRecentQuery, + public readonly seenAt: number, + ) { + this.username = data.username; + this.query = data.query; + this.formattedQuery = format(data.query, { + language: "postgresql", + keywordCase: "lower", + linesBetweenQueries: 2, + }); + this.meanTime = data.meanTime; + this.calls = data.calls; + this.rows = data.rows; + this.topLevel = data.topLevel; + } +} + +export type RawRecentQuery = { + username: string; + query: string; + formattedQuery: string; + meanTime: number; + calls: string; + rows: string; + topLevel: boolean; +}; diff --git a/src/sync/connectable.test.ts b/src/sync/connectable.test.ts index 66b7665..ce69571 100644 --- a/src/sync/connectable.test.ts +++ b/src/sync/connectable.test.ts @@ -1,11 +1,14 @@ import { assertEquals } from "@std/assert"; import { Connectable } from "./connectable.ts"; +import { PgIdentifier } from "@query-doctor/core"; Deno.test("connectable", () => { const connectable = Connectable.fromString( "postgres://user:password@localhost:5432/dbname?a=b&c=d", ); - const newConnectable = connectable.withDatabaseName("testing"); + const newConnectable = connectable.withDatabaseName( + PgIdentifier.fromString("testing"), + ); assertEquals( newConnectable.toString(), "postgres://user:password@localhost:5432/testing?a=b&c=d", diff --git a/src/sync/connectable.ts b/src/sync/connectable.ts index 959b650..5ace159 100644 --- a/src/sync/connectable.ts +++ b/src/sync/connectable.ts @@ -1,5 +1,6 @@ import { z } from "zod/v4"; import { env } from "../env.ts"; +import { PgIdentifier } from "@query-doctor/core"; /** * Represents a valid connection to a database. @@ -13,14 +14,14 @@ export class Connectable { return this.url.hostname.endsWith("supabase.com"); } - withDatabaseName(databaseName: string) { + withDatabaseName(databaseName: PgIdentifier) { const newUrl = new URL(this.url); newUrl.pathname = `/${databaseName}`; return new Connectable(newUrl); } /** - * Custom logic for parsing a string into a Connectable through zod. + * Custom logic for parsing a string into a {@link Connectable} through zod. */ static transform( urlString: string, diff --git a/src/sync/connection-manager.ts b/src/sync/connection-manager.ts new file mode 100644 index 0000000..50dd4ee --- /dev/null +++ b/src/sync/connection-manager.ts @@ -0,0 +1,34 @@ +import { Postgres, PostgresFactory } from "@query-doctor/core"; +import { SegmentedQueryCache } from "./seen-cache.ts"; +import { Connectable } from "./connectable.ts"; +import { PostgresConnector } from "./pg-connector.ts"; + +/** + * Manages connections and query caches for each connection + */ +export class ConnectionManager { + public readonly segmentedQueryCache = new SegmentedQueryCache(); + + // This prevents connections being garbage collected. + // ConnectionMap should be responsible for closing connections + private readonly connections = new Map(); + + constructor(private readonly factory: PostgresFactory) {} + + getOrCreateConnection(connectable: Connectable): Postgres { + const urlString = connectable.toString(); + let sql = this.connections.get(urlString); + if (!sql) { + sql = this.factory({ url: urlString }); + this.connections.set(urlString, sql); + } + return sql; + } + + getConnectorFor(input: Connectable | Postgres): PostgresConnector { + const sql = input instanceof Connectable + ? this.getOrCreateConnection(input) + : input; + return new PostgresConnector(sql, this.segmentedQueryCache); + } +} diff --git a/src/sync/pg-connector.ts b/src/sync/pg-connector.ts index 1801b0c..69fa387 100644 --- a/src/sync/pg-connector.ts +++ b/src/sync/pg-connector.ts @@ -1,4 +1,3 @@ -import { format } from "sql-formatter"; import schemaDumpSql from "./schema_dump.sql" with { type: "text" }; import type { CursorOptions, @@ -17,6 +16,9 @@ import { Postgres } from "@query-doctor/core"; import { SegmentedQueryCache } from "./seen-cache.ts"; import { FullSchema, FullSchemaColumn } from "./schema_differ.ts"; import { ExtensionNotInstalledError, PostgresError } from "./errors.ts"; +import { RawRecentQuery, RecentQuery } from "../sql/recent-query.ts"; +// deno-lint-ignore no-unused-vars +import { ConnectionManager } from "./connection-manager.ts"; const ctidSymbol = Symbol("ctid"); type Row = NonNullable & { @@ -42,20 +44,6 @@ export type SerializeResult = { sampledRecords: Record; }; -export type RawRecentQuery = { - username: string; - query: string; - formattedQuery: string; - meanTime: number; - calls: string; - rows: string; - topLevel: boolean; -}; - -export type RecentQuery = { - firstSeen: number; -}; - export type RecentQueriesError = { kind: "error"; type: "extension_not_installed"; @@ -87,6 +75,9 @@ export type ResetPgStatStatementsResult = } | ResetPgStatStatementsError; +/** + * Use {@link ConnectionManager.getConnectorFor} to grab an instance of this class + */ export class PostgresConnector implements DatabaseConnector { private static readonly QUERY_DOCTOR_USER = "query_doctor_db_link"; private readonly tupleEstimates = new Map(); @@ -474,22 +465,9 @@ ORDER BY and pg_user.usename not in (/* supabase */ 'supabase_admin', 'supabase_auth_admin', /* neon */ 'cloud_admin'); -- @qd_introspection `); // we're excluding `pg_stat_statements` from the results since it's almost certainly unrelated - // this is a horrible place for this code to live, it doesn't belong here - // the alternatives are equally as bad: repeat the logic 3 times in the various endpoints - const resultsWithFormattedQueries = results.map((r) => { - return { - ...r, - formattedQuery: format(r.query, { - language: "postgresql", - keywordCase: "upper", - linesBetweenQueries: 2, - }), - }; - }); - return this.segmentedQueryCache.sync( this.db, - resultsWithFormattedQueries, + results, ); } catch (err) { if ( diff --git a/src/sync/schema-link.ts b/src/sync/schema-link.ts index 05e007c..f725ffa 100644 --- a/src/sync/schema-link.ts +++ b/src/sync/schema-link.ts @@ -150,32 +150,37 @@ export class DumpCommand { })(); } - async pipeTo(restore: RestoreCommand) { + async pipeTo(restore: RestoreCommand): Promise { // Start consuming stderr in the background to prevent resource leaks const stderrPromise = this.process.stderr.text(); try { await this.process.stdout.pipeTo(restore.stdin); } catch (error) { - const stderr = await stderrPromise; - if (stderr) { - throw new Error(`pg_dump failed: ${stderr}`, { cause: error }); - } - throw error; + return { + dump: { + error: error instanceof Error ? error.message : await stderrPromise, + status: await this.process.status, + }, + }; } const dumpStatus = await this.process.status; - if (dumpStatus.code !== 0) { - const stderr = await stderrPromise; - if (stderr) { - throw new Error(`pg_dump failed: ${stderr}`); - } else { - throw new Error(`pg_dump failed with code ${dumpStatus.code}`); - } - } - await stderrPromise; - await restore.status; + // this only fails if the command is non-zero + const error = (await stderrPromise).trim(); + const restoreStatus = await restore.status; + const out = { + dump: { + error, + status: dumpStatus, + }, + restore: { + error: (await restore.stderr.text()).trim(), + status: restoreStatus, + }, + }; await restore.cleanup(); + return out; } /** @@ -244,6 +249,8 @@ export class RestoreCommand { const args = [ "--no-owner", "--no-acl", + "--clean", + "--if-exists", ...RestoreCommand.formatFlags(), "--dbname", connectable.toString(), @@ -287,3 +294,14 @@ export class RestoreCommand { return ["--format", "custom"]; } } + +export type RestoreCommandResult = { + dump: { + error: string; + status: Deno.CommandStatus; + }; + restore?: { + error: string; + status: Deno.CommandStatus; + }; +}; diff --git a/src/sync/schema_differ.ts b/src/sync/schema_differ.ts index fd55727..ac9c69d 100644 --- a/src/sync/schema_differ.ts +++ b/src/sync/schema_differ.ts @@ -4,8 +4,8 @@ import { format, type Op } from "jsondiffpatch/formatters/jsonpatch"; import { z } from "zod"; export class SchemaDiffer { - private differ = create({ - arrays: { detectMove: true, }, + private readonly differ = create({ + arrays: { detectMove: true }, objectHash(obj, index) { // shouldn't happen but we don't want to throw an error for this if (!("type" in obj)) { @@ -28,7 +28,7 @@ export class SchemaDiffer { }, }); - private stats: Map = new Map(); + private readonly stats = new WeakMap(); put(postgres: Postgres, schema: FullSchema): Op[] | undefined { const old = this.stats.get(postgres); diff --git a/src/sync/seen-cache.ts b/src/sync/seen-cache.ts index b6857c1..a14e493 100644 --- a/src/sync/seen-cache.ts +++ b/src/sync/seen-cache.ts @@ -1,5 +1,5 @@ -import type { RawRecentQuery, RecentQuery } from "./pg-connector.ts"; import type { Postgres } from "@query-doctor/core"; +import { RawRecentQuery, RecentQuery } from "../sql/recent-query.ts"; interface CacheEntry { firstSeen: number; lastSeen: number; @@ -31,9 +31,9 @@ export class QueryCache { return entry.firstSeen >= this.createdAt; } - store(query: string) { + store(recentQuery: RawRecentQuery): string { // TODO: use fingerprint from @libpg-query/parser instead of the full query string - const key = query; + const key = recentQuery.query; const now = Date.now(); if (this.list[key]) { this.list[key].lastSeen = now; @@ -47,17 +47,14 @@ export class QueryCache { return this.list[key]?.firstSeen || Date.now(); } - sync(queries: RawRecentQuery[]): RecentQuery[] { - return queries.map((query) => { - const key = this.store(query.query); - return { - ...query, - firstSeen: this.getFirstSeen(key), - }; + sync(rawQueries: RawRecentQuery[]): RecentQuery[] { + return rawQueries.map((rawQuery) => { + const key = this.store(rawQuery); + return new RecentQuery(rawQuery, this.getFirstSeen(key)); }); } - reset() { + reset(): void { this.list = {}; } } @@ -76,7 +73,7 @@ export class SegmentedQueryCache { return cache.sync(queries); } - store(db: Postgres, query: string) { + store(db: Postgres, query: RawRecentQuery) { const cache = this.getOrCreateCache(db); return cache.store(query); } diff --git a/src/sync/syncer.ts b/src/sync/syncer.ts index cf615a7..9e0b6b9 100644 --- a/src/sync/syncer.ts +++ b/src/sync/syncer.ts @@ -3,20 +3,14 @@ import { type DependencyAnalyzerOptions, DependencyResolutionNotice, } from "./dependency-tree.ts"; -import { PostgresConnector, RecentQueriesResult } from "./pg-connector.ts"; +import { RecentQueriesResult } from "./pg-connector.ts"; import { PostgresSchemaLink } from "./schema-link.ts"; import { withSpan } from "../otel.ts"; import { Connectable } from "./connectable.ts"; -import { - ExportedStats, - type Postgres, - type PostgresFactory, - PostgresVersion, - Statistics, -} from "@query-doctor/core"; -import { SegmentedQueryCache } from "./seen-cache.ts"; +import { ExportedStats, PostgresVersion, Statistics } from "@query-doctor/core"; import { SchemaDiffer } from "./schema_differ.ts"; import { ExtensionNotInstalledError } from "./errors.ts"; +import { ConnectionManager } from "./connection-manager.ts"; type SyncOptions = DependencyAnalyzerOptions; @@ -44,11 +38,11 @@ export type SyncResult = { }; export class PostgresSyncer { - private readonly connections = new Map(); - private readonly segmentedQueryCache = new SegmentedQueryCache(); private readonly differ = new SchemaDiffer(); - constructor(private readonly factory: PostgresFactory) {} + constructor( + private readonly manager: ConnectionManager, + ) {} /** * @throws {ExtensionNotInstalledError} @@ -58,8 +52,8 @@ export class PostgresSyncer { connectable: Connectable, options: SyncOptions, ): Promise { - const sql = this.getConnection(connectable); - const connector = new PostgresConnector(sql, this.segmentedQueryCache); + const sql = this.manager.getOrCreateConnection(connectable); + const connector = this.manager.getConnectorFor(sql); const link = new PostgresSchemaLink(connectable, "pglite"); const analyzer = new DependencyAnalyzer(connector, options); const [ @@ -139,8 +133,8 @@ export class PostgresSyncer { * @throws {PostgresError} */ async liveQuery(connectable: Connectable) { - const sql = this.getConnection(connectable); - const connector = new PostgresConnector(sql, this.segmentedQueryCache); + const sql = this.manager.getOrCreateConnection(connectable); + const connector = this.manager.getConnectorFor(sql); const [queries, schema] = await Promise.all([ connector.getRecentQueries(), connector.getSchema(), @@ -156,18 +150,7 @@ export class PostgresSyncer { async reset( connectable: Connectable, ): Promise { - const sql = this.getConnection(connectable); - const connector = new PostgresConnector(sql, this.segmentedQueryCache); + const connector = this.manager.getConnectorFor(connectable); await connector.resetPgStatStatements(); } - - private getConnection(connectable: Connectable) { - const urlString = connectable.toString(); - let sql = this.connections.get(urlString); - if (!sql) { - sql = this.factory({ url: urlString }); - this.connections.set(urlString, sql); - } - return sql; - } } From 2a550c5aac4dc6e95e52ceb76036eef8af643d80 Mon Sep 17 00:00:00 2001 From: Xetera Date: Tue, 2 Dec 2025 17:36:44 +0300 Subject: [PATCH 03/29] feat: return full schema on initial sync --- deno.json | 2 +- deno.lock | 8 ++++---- src/remote/remote.dto.ts | 7 +++++++ src/remote/remote.ts | 26 ++++++++++++++++++-------- src/sync/pg-connector.ts | 2 +- src/sync/seen-cache.ts | 36 ++++++++++++++++++++++-------------- 6 files changed, 53 insertions(+), 28 deletions(-) diff --git a/deno.json b/deno.json index 6531de1..16c55b9 100644 --- a/deno.json +++ b/deno.json @@ -20,7 +20,7 @@ "imports": { "@actions/core": "npm:@actions/core@^1.11.1", "@actions/github": "npm:@actions/github@^6.0.1", - "@libpg-query/parser": "npm:@libpg-query/parser@^17.7.0", + "@libpg-query/parser": "npm:@libpg-query/parser@^17.6.3", "@opentelemetry/api": "jsr:@opentelemetry/api@^1.9.0", "@pgsql/types": "npm:@pgsql/types@^17.6.1", "@query-doctor/core": "npm:@query-doctor/core@^0.0.4", diff --git a/deno.lock b/deno.lock index 1db2b02..1bf6eb8 100644 --- a/deno.lock +++ b/deno.lock @@ -11,7 +11,7 @@ "jsr:@std/internal@^1.0.10": "1.0.10", "npm:@actions/core@^1.11.1": "1.11.1", "npm:@actions/github@^6.0.1": "6.0.1_@octokit+core@5.2.2", - "npm:@libpg-query/parser@^17.7.0": "17.7.0", + "npm:@libpg-query/parser@^17.6.3": "17.6.3", "npm:@pgsql/types@^17.6.1": "17.6.1", "npm:@query-doctor/core@^0.0.4": "0.0.4", "npm:@testcontainers/postgresql@^11.9.0": "11.9.0", @@ -305,8 +305,8 @@ ], "scripts": true }, - "@libpg-query/parser@17.7.0": { - "integrity": "sha512-G+DA8enxveO3ESbLDhU7sAz02CPI16dKqXxPAvXbssHZnuBsaTpdHjm9CHC14ownvdujJ4CGwbs7kDUi3Lin9Q==", + "@libpg-query/parser@17.6.3": { + "integrity": "sha512-AvbS7b9GJZfCzqt4tLMqTaYK7Css9pJRTA2dKWLoTlob/XO2VNc30Q3g9DmxNBmokVTrmibkn/dy9bw8hfosQQ==", "dependencies": [ "@launchql/protobufjs", "@pgsql/types" @@ -1763,7 +1763,7 @@ "jsr:@std/fmt@^1.0.8", "npm:@actions/core@^1.11.1", "npm:@actions/github@^6.0.1", - "npm:@libpg-query/parser@^17.7.0", + "npm:@libpg-query/parser@^17.6.3", "npm:@pgsql/types@^17.6.1", "npm:@query-doctor/core@^0.0.4", "npm:@testcontainers/postgresql@^11.9.0", diff --git a/src/remote/remote.dto.ts b/src/remote/remote.dto.ts index 8305018..ac09387 100644 --- a/src/remote/remote.dto.ts +++ b/src/remote/remote.dto.ts @@ -1,13 +1,20 @@ import { z } from "zod"; import { Connectable } from "../sync/connectable.ts"; import { RecentQuery } from "../sql/recent-query.ts"; +import { FullSchema } from "../sync/schema_differ.ts"; export const RemoteSyncRequest = z.object({ db: z.string().transform(Connectable.transform), }); +export const RemoteSyncFullSchemaResponse = z.discriminatedUnion("type", [ + z.object({ type: z.literal("ok"), value: FullSchema }), + z.object({ type: z.literal("error"), error: z.string() }), +]); + export const RemoteSyncResponse = z.object({ queries: z.array(z.instanceof(RecentQuery)), + schema: RemoteSyncFullSchemaResponse, }); export type RemoteSyncResponse = z.infer; diff --git a/src/remote/remote.ts b/src/remote/remote.ts index aca9a1c..f07c05f 100644 --- a/src/remote/remote.ts +++ b/src/remote/remote.ts @@ -44,13 +44,15 @@ export class Remote { await this.resetDatabase(); const target = this.targetURL.withDatabaseName(Remote.optimizingDbName); const sql = this.manager.getOrCreateConnection(source); - const [restoreResult, recentQueries, fullSchema] = await Promise.allSettled( - [ - this.pipeSchema(target, source), - this.getRecentConnections(source), - this.getFullSchema(source), - ], - ); + const [_restoreResult, recentQueries, fullSchema] = await Promise + .allSettled( + [ + // This potentially creates a lot of connections to the source + this.pipeSchema(target, source), + this.getRecentQueries(source), + this.getFullSchema(source), + ], + ); if (fullSchema.status === "fulfilled") { this.differ.put(sql, fullSchema.value); @@ -60,6 +62,14 @@ export class Remote { await this.onSuccessfulSync(pg); return { queries: recentQueries.status === "fulfilled" ? recentQueries.value : [], + schema: fullSchema.status === "fulfilled" + ? { type: "ok", value: fullSchema.value } + : { + type: "error", + error: fullSchema.reason instanceof Error + ? fullSchema.reason.message + : "Unknown error", + }, }; } @@ -103,7 +113,7 @@ export class Remote { } } - private getRecentConnections( + private getRecentQueries( source: Connectable, ): Promise { const connector = this.manager.getConnectorFor(source); diff --git a/src/sync/pg-connector.ts b/src/sync/pg-connector.ts index 69fa387..d11eb53 100644 --- a/src/sync/pg-connector.ts +++ b/src/sync/pg-connector.ts @@ -465,7 +465,7 @@ ORDER BY and pg_user.usename not in (/* supabase */ 'supabase_admin', 'supabase_auth_admin', /* neon */ 'cloud_admin'); -- @qd_introspection `); // we're excluding `pg_stat_statements` from the results since it's almost certainly unrelated - return this.segmentedQueryCache.sync( + return await this.segmentedQueryCache.sync( this.db, results, ); diff --git a/src/sync/seen-cache.ts b/src/sync/seen-cache.ts index a14e493..852c51c 100644 --- a/src/sync/seen-cache.ts +++ b/src/sync/seen-cache.ts @@ -1,21 +1,25 @@ import type { Postgres } from "@query-doctor/core"; import { RawRecentQuery, RecentQuery } from "../sql/recent-query.ts"; +import { fingerprint } from "@libpg-query/parser"; +import z from "zod"; + interface CacheEntry { firstSeen: number; lastSeen: number; } -type Query = string; +const QueryHash = z.string().brand<"QueryHash">(); +type QueryHash = z.infer; export class QueryCache { - list: Record = {}; + private list: Record = {}; private readonly createdAt: number; constructor() { this.createdAt = Date.now(); } - isCached(key: string): boolean { + isCached(key: QueryHash): boolean { const entry = this.list[key]; if (!entry) { return false; @@ -23,7 +27,7 @@ export class QueryCache { return true; } - isNew(key: string): boolean { + isNew(key: QueryHash): boolean { const entry = this.list[key]; if (!entry) { return true; @@ -31,9 +35,8 @@ export class QueryCache { return entry.firstSeen >= this.createdAt; } - store(recentQuery: RawRecentQuery): string { - // TODO: use fingerprint from @libpg-query/parser instead of the full query string - const key = recentQuery.query; + async store(recentQuery: RawRecentQuery): Promise { + const key = await this.hash(recentQuery.query); const now = Date.now(); if (this.list[key]) { this.list[key].lastSeen = now; @@ -43,20 +46,25 @@ export class QueryCache { return key; } - getFirstSeen(key: string): number { + getFirstSeen(key: QueryHash): number { return this.list[key]?.firstSeen || Date.now(); } - sync(rawQueries: RawRecentQuery[]): RecentQuery[] { - return rawQueries.map((rawQuery) => { - const key = this.store(rawQuery); + async sync(rawQueries: RawRecentQuery[]): Promise { + // TODO: bound the concurrency + return await Promise.all(rawQueries.map(async (rawQuery) => { + const key = await this.store(rawQuery); return new RecentQuery(rawQuery, this.getFirstSeen(key)); - }); + })); } reset(): void { this.list = {}; } + + private async hash(query: string): Promise { + return QueryHash.parse(await fingerprint(query)); + } } /** @@ -66,9 +74,9 @@ export class SegmentedQueryCache { // weak reference to the db instance to allow cache to be garbage collected // when the connection to the database is closed. // Can be relevant for - dbs: WeakMap = new WeakMap(); + private readonly dbs: WeakMap = new WeakMap(); - sync(db: Postgres, queries: RawRecentQuery[]): RecentQuery[] { + sync(db: Postgres, queries: RawRecentQuery[]): Promise { const cache = this.getOrCreateCache(db); return cache.sync(queries); } From 5c9f363ad5ca2841f46a00b173cddf73f2a9938c Mon Sep 17 00:00:00 2001 From: Xetera Date: Tue, 2 Dec 2025 19:15:30 +0300 Subject: [PATCH 04/29] fix: return an error-able result for queries --- src/remote/remote-controller.test.ts | 1 - src/remote/remote.dto.ts | 10 ++++- src/remote/remote.test.ts | 55 +++++++++++++++++++++----- src/remote/remote.ts | 50 ++++++++++++++--------- src/sql/recent-query.ts | 59 +++++++++++++++++++++++----- src/sync/connection-manager.ts | 2 +- src/sync/seen-cache.ts | 2 +- 7 files changed, 136 insertions(+), 43 deletions(-) diff --git a/src/remote/remote-controller.test.ts b/src/remote/remote-controller.test.ts index b976417..190a567 100644 --- a/src/remote/remote-controller.test.ts +++ b/src/remote/remote-controller.test.ts @@ -59,7 +59,6 @@ Deno.test({ ), ); - console.log(await response?.json()); assertEquals(response?.status, 200); const sql = postgres( diff --git a/src/remote/remote.dto.ts b/src/remote/remote.dto.ts index ac09387..d5a1f58 100644 --- a/src/remote/remote.dto.ts +++ b/src/remote/remote.dto.ts @@ -12,8 +12,16 @@ export const RemoteSyncFullSchemaResponse = z.discriminatedUnion("type", [ z.object({ type: z.literal("error"), error: z.string() }), ]); +export const RemoteSyncQueriesResponse = z.discriminatedUnion("type", [ + z.object({ + type: z.literal("ok"), + value: z.array(z.instanceof(RecentQuery)), + }), + z.object({ type: z.literal("error"), error: z.string() }), +]); + export const RemoteSyncResponse = z.object({ - queries: z.array(z.instanceof(RecentQuery)), + queries: RemoteSyncQueriesResponse, schema: RemoteSyncFullSchemaResponse, }); diff --git a/src/remote/remote.test.ts b/src/remote/remote.test.ts index aaf8c1d..837ea77 100644 --- a/src/remote/remote.test.ts +++ b/src/remote/remote.test.ts @@ -6,6 +6,13 @@ import postgres from "postgresjs"; import { assertEquals } from "@std/assert/equals"; import { wrapGenericPostgresInterface } from "../sql/postgresjs.ts"; import { ConnectionManager } from "../sync/connection-manager.ts"; +import { assertArrayIncludes } from "@std/assert"; + +function assertOk( + result: { type: string; value?: T }, +): asserts result is { type: "ok"; value: T } { + assertEquals(result.type, "ok"); +} const connectable = z.string().transform(Connectable.transform); Deno.test({ @@ -18,11 +25,18 @@ Deno.test({ new PostgreSqlContainer("postgres:17") .withCopyContentToContainer([ { - content: - "create table testing(a int, b text); insert into testing values (1); create index on testing(b)", + content: ` + create extension pg_stat_statements; + create table testing(a int, b text); + insert into testing values (1); + create index on testing(b); + select * from testing where a = 1; + `, target: "/docker-entrypoint-initdb.d/init.sql", }, - ]).start(), + ]) + .withCommand(["-c", "shared_preload_libraries=pg_stat_statements"]) + .start(), new PostgreSqlContainer("postgres:17") .withCopyContentToContainer([ { @@ -44,21 +58,42 @@ Deno.test({ target, new ConnectionManager(wrapGenericPostgresInterface), ); - await remote.syncFrom(source); + const result = await remote.syncFrom(source); + assertOk(result.queries); + + const queries = result.queries.value.map((f) => f.query); + assertArrayIncludes(queries, [ + "create table testing(a int, b text)", + "select * from testing where a = $1", + ]); const sql = postgres( target.withDatabaseName(Remote.optimizingDbName).toString(), ); - const tablesAfter = - await sql`select tablename from pg_tables where schemaname = 'public'`; + const indexesAfter = await sql`select * from pg_indexes where schemaname = 'public'`; - assertEquals(indexesAfter.count, 1); - assertEquals(tablesAfter.count, 1); - assertEquals(tablesAfter[0], { tablename: "testing" }); + assertEquals( + indexesAfter.count, + 1, + "Indexes were not copied over correctly from the source db", + ); + + const tablesAfter = + await sql`select tablename from pg_tables where schemaname = 'public'`; + assertEquals( + tablesAfter.count, + 1, + "Tables were not copied over correctly from the source db", + ); + assertEquals( + tablesAfter[0], + { tablename: "testing" }, + "Table name mismatch", + ); const rows = await sql`select * from testing`; // expect no rows to have been synced - assertEquals(rows.length, 0); + assertEquals(rows.length, 0, "Table in target db not empty"); } finally { await Promise.all([sourceDb.stop(), targetDb.stop()]); } diff --git a/src/remote/remote.ts b/src/remote/remote.ts index f07c05f..b658b1f 100644 --- a/src/remote/remote.ts +++ b/src/remote/remote.ts @@ -1,10 +1,10 @@ -import { PgIdentifier, Postgres } from "@query-doctor/core"; -import { Connectable } from "../sync/connectable.ts"; +import { PgIdentifier, type Postgres } from "@query-doctor/core"; +import { type Connectable } from "../sync/connectable.ts"; import { DumpCommand, RestoreCommand } from "../sync/schema-link.ts"; import { ConnectionManager } from "../sync/connection-manager.ts"; -import { RecentQuery } from "../sql/recent-query.ts"; -import { SchemaDiffer } from "../sync/schema_differ.ts"; -import { RemoteSyncResponse } from "./remote.dto.ts"; +import { type RecentQuery } from "../sql/recent-query.ts"; +import { type FullSchema, SchemaDiffer } from "../sync/schema_differ.ts"; +import { type RemoteSyncResponse } from "./remote.dto.ts"; /** * Represents a db for doing optimization work. @@ -13,8 +13,8 @@ import { RemoteSyncResponse } from "./remote.dto.ts"; * But potentially more logical databases in the future. */ export class Remote { - public static readonly baseDbName = PgIdentifier.fromString("postgres"); - public static readonly optimizingDbName = PgIdentifier.fromString( + static readonly baseDbName = PgIdentifier.fromString("postgres"); + static readonly optimizingDbName = PgIdentifier.fromString( "optimizing_db", ); @@ -45,14 +45,12 @@ export class Remote { const target = this.targetURL.withDatabaseName(Remote.optimizingDbName); const sql = this.manager.getOrCreateConnection(source); const [_restoreResult, recentQueries, fullSchema] = await Promise - .allSettled( - [ - // This potentially creates a lot of connections to the source - this.pipeSchema(target, source), - this.getRecentQueries(source), - this.getFullSchema(source), - ], - ); + .allSettled([ + // This potentially creates a lot of connections to the source + this.pipeSchema(target, source), + this.getRecentQueries(source), + this.getFullSchema(source), + ]); if (fullSchema.status === "fulfilled") { this.differ.put(sql, fullSchema.value); @@ -60,8 +58,19 @@ export class Remote { const pg = this.manager.getOrCreateConnection(this.targetURL); await this.onSuccessfulSync(pg); + return { - queries: recentQueries.status === "fulfilled" ? recentQueries.value : [], + queries: recentQueries.status === "fulfilled" + ? { + type: "ok", + value: recentQueries.value, + } + : { + type: "error", + error: recentQueries.reason instanceof Error + ? recentQueries.reason.message + : "Unknown error", + }, schema: fullSchema.status === "fulfilled" ? { type: "ok", value: fullSchema.value } : { @@ -78,7 +87,7 @@ export class Remote { * * TODO: allow juggling multiple databases in the future */ - private async resetDatabase() { + private async resetDatabase(): Promise { const databaseName = Remote.optimizingDbName; // these cannot be run in the same `exec` block as that implicitly creates transactions await this.baseDb.exec( @@ -88,7 +97,10 @@ export class Remote { await this.baseDb.exec(`create database ${databaseName};`); } - private async pipeSchema(target: Connectable, source: Connectable) { + private async pipeSchema( + target: Connectable, + source: Connectable, + ): Promise { const dump = DumpCommand.spawn(source, "native-postgres"); const restore = RestoreCommand.spawn(target); const { dump: dumpResult, restore: restoreResult } = await dump.pipeTo( @@ -120,7 +132,7 @@ export class Remote { return connector.getRecentQueries(); } - private getFullSchema(source: Connectable) { + private getFullSchema(source: Connectable): Promise { const connector = this.manager.getConnectorFor(source); return connector.getSchema(); } diff --git a/src/sql/recent-query.ts b/src/sql/recent-query.ts index c5b7eca..95cc25a 100644 --- a/src/sql/recent-query.ts +++ b/src/sql/recent-query.ts @@ -1,23 +1,36 @@ import { format } from "sql-formatter"; // deno-lint-ignore no-unused-vars import type { SegmentedQueryCache } from "../sync/seen-cache.ts"; +import { + Analyzer, + DiscoveredColumnReference, + SQLCommenterTag, +} from "@query-doctor/core"; +import { parse } from "@libpg-query/parser"; /** - * Constructed by {@link SegmentedQueryCache} by supplying the - * date the query was last seen + * Constructed by syncing with {@link SegmentedQueryCache.sync} + * and supplying the date the query was last seen */ export class RecentQuery { - public readonly formattedQuery: string; - public readonly username: string; - public readonly query: string; - public readonly meanTime: number; - public readonly calls: string; - public readonly rows: string; - public readonly topLevel: boolean; + readonly formattedQuery: string; + readonly username: string; + readonly query: string; + readonly meanTime: number; + readonly calls: string; + readonly rows: string; + readonly topLevel: boolean; + readonly isSystemQuery: boolean; + readonly isSelectQuery: boolean; + + /** Use {@link RecentQuery.analyze} instead */ constructor( data: RawRecentQuery, - public readonly seenAt: number, + readonly tableReferences: string[], + readonly columnReferences: DiscoveredColumnReference[], + readonly tags: SQLCommenterTag[], + readonly seenAt: number, ) { this.username = data.username; this.query = data.query; @@ -30,6 +43,32 @@ export class RecentQuery { this.calls = data.calls; this.rows = data.rows; this.topLevel = data.topLevel; + + this.isSystemQuery = RecentQuery.isSystemQuery(tableReferences); + this.isSelectQuery = RecentQuery.isSelectQuery(data); + } + + static async analyze( + data: RawRecentQuery, + seenAt: number, + ) { + const analyzer = new Analyzer(parse); + const analysis = await analyzer.analyze(data.query); + return new RecentQuery( + { ...data, query: analysis.queryWithoutTags }, + analysis.referencedTables, + analysis.indexesToCheck, + analysis.tags, + seenAt, + ); + } + + static isSelectQuery(data: RawRecentQuery): boolean { + return /^select/i.test(data.query); + } + + static isSystemQuery(referencedTables: string[]): boolean { + return referencedTables.some((table) => table.startsWith("pg_")); } } diff --git a/src/sync/connection-manager.ts b/src/sync/connection-manager.ts index 50dd4ee..9c8c6e7 100644 --- a/src/sync/connection-manager.ts +++ b/src/sync/connection-manager.ts @@ -7,7 +7,7 @@ import { PostgresConnector } from "./pg-connector.ts"; * Manages connections and query caches for each connection */ export class ConnectionManager { - public readonly segmentedQueryCache = new SegmentedQueryCache(); + readonly segmentedQueryCache = new SegmentedQueryCache(); // This prevents connections being garbage collected. // ConnectionMap should be responsible for closing connections diff --git a/src/sync/seen-cache.ts b/src/sync/seen-cache.ts index 852c51c..8502af7 100644 --- a/src/sync/seen-cache.ts +++ b/src/sync/seen-cache.ts @@ -54,7 +54,7 @@ export class QueryCache { // TODO: bound the concurrency return await Promise.all(rawQueries.map(async (rawQuery) => { const key = await this.store(rawQuery); - return new RecentQuery(rawQuery, this.getFirstSeen(key)); + return RecentQuery.analyze(rawQuery, this.getFirstSeen(key)); })); } From 13ca24500db521fc209d2f743bc964af69b359ec Mon Sep 17 00:00:00 2001 From: Xetera Date: Thu, 4 Dec 2025 23:24:26 +0300 Subject: [PATCH 05/29] feat: query optimization --- deno.json | 4 +- deno.lock | 21 +- src/main.ts | 5 +- src/remote/query-optimizer.test.ts | 54 ++++ src/remote/query-optimizer.ts | 356 +++++++++++++++++++++++++++ src/remote/remote-controller.test.ts | 19 +- src/remote/remote-controller.ts | 2 +- src/remote/remote.dto.ts | 16 +- src/remote/remote.test.ts | 33 ++- src/remote/remote.ts | 54 +++- src/runner.ts | 7 +- src/server/http.ts | 16 +- src/server/sync.dto.ts | 8 +- src/sql/postgresjs.ts | 50 ++-- src/sql/recent-query.ts | 7 + src/sync/connectable.ts | 4 +- src/sync/connection-manager.ts | 25 +- src/sync/schema_differ.ts | 6 +- src/sync/seen-cache.ts | 8 +- 19 files changed, 605 insertions(+), 90 deletions(-) create mode 100644 src/remote/query-optimizer.test.ts create mode 100644 src/remote/query-optimizer.ts diff --git a/deno.json b/deno.json index 16c55b9..313c953 100644 --- a/deno.json +++ b/deno.json @@ -15,6 +15,7 @@ "tasks": { "start": "deno run --permission-set=run src/main.ts", "start:dev": "deno run --env-file=.env -A src/main.ts", + "test": "deno test --permission-set=run src", "dev": "deno run --env-file=.env --permission-set=run --watch src/main.ts" }, "imports": { @@ -23,7 +24,7 @@ "@libpg-query/parser": "npm:@libpg-query/parser@^17.6.3", "@opentelemetry/api": "jsr:@opentelemetry/api@^1.9.0", "@pgsql/types": "npm:@pgsql/types@^17.6.1", - "@query-doctor/core": "npm:@query-doctor/core@^0.0.4", + "@query-doctor/core": "npm:@query-doctor/core@^0.0.5", "@rabbit-company/rate-limiter": "jsr:@rabbit-company/rate-limiter@^3.0.0", "@std/assert": "jsr:@std/assert@^1.0.14", "@std/collections": "jsr:@std/collections@^1.1.3", @@ -32,6 +33,7 @@ "@testcontainers/postgresql": "npm:@testcontainers/postgresql@^11.9.0", "@types/node": "npm:@types/node@^24.9.1", "@types/nunjucks": "npm:@types/nunjucks@^3.2.6", + "async-sema": "npm:async-sema@^3.1.1", "chokidar": "npm:chokidar@^4.0.3", "dedent": "npm:dedent@^1.6.0", "fast-csv": "npm:fast-csv@^5.0.5", diff --git a/deno.lock b/deno.lock index 1bf6eb8..c092bbb 100644 --- a/deno.lock +++ b/deno.lock @@ -13,10 +13,11 @@ "npm:@actions/github@^6.0.1": "6.0.1_@octokit+core@5.2.2", "npm:@libpg-query/parser@^17.6.3": "17.6.3", "npm:@pgsql/types@^17.6.1": "17.6.1", - "npm:@query-doctor/core@^0.0.4": "0.0.4", + "npm:@query-doctor/core@^0.0.5": "0.0.5", "npm:@testcontainers/postgresql@^11.9.0": "11.9.0", "npm:@types/node@^24.9.1": "24.10.1", "npm:@types/nunjucks@^3.2.6": "3.2.6", + "npm:async-sema@^3.1.1": "3.1.1", "npm:chokidar@^4.0.3": "4.0.3", "npm:dedent@^1.6.0": "1.6.0", "npm:fast-csv@^5.0.5": "5.0.5", @@ -246,8 +247,8 @@ "@fastify/busboy@2.1.1": { "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==" }, - "@grpc/grpc-js@1.14.1": { - "integrity": "sha512-sPxgEWtPUR3EnRJCEtbGZG2iX8LQDUls2wUS3o27jg07KqJFMq6YDeWvMo1wfpmy3rqRdS0rivpLwhqQtEyCuQ==", + "@grpc/grpc-js@1.14.2": { + "integrity": "sha512-QzVUtEFyu05UNx2xr0fCQmStUO17uVQhGNowtxs00IgTZT6/W2PBLfUkj30s0FKJ29VtTa3ArVNIhNP6akQhqA==", "dependencies": [ "@grpc/proto-loader@0.8.0", "@js-sdsl/ordered-map" @@ -431,8 +432,8 @@ "@protobufjs/utf8@1.1.0": { "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, - "@query-doctor/core@0.0.4": { - "integrity": "sha512-kv7tERboZeOr3ebIxaaHkxFbYS+IPEIKAsZRbEz8UCYFpm8ZkT4C2+4gOQ86SksSRaLNSFkwF7DeELtcpMbKpw==", + "@query-doctor/core@0.0.5": { + "integrity": "sha512-+PJs/0oAqrvorWDDXoSO8a7QU0JYSj9KrsmvNRpSuclPDgBEgqLNL2V8a6ozaW15dOguRtwt+UlMdGYBVcNgFw==", "dependencies": [ "@pgsql/types", "colorette", @@ -565,6 +566,9 @@ "async-lock@1.4.1": { "integrity": "sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==" }, + "async-sema@3.1.1": { + "integrity": "sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg==" + }, "async@3.2.6": { "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==" }, @@ -958,8 +962,8 @@ "ms@2.1.3": { "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, - "nan@2.23.1": { - "integrity": "sha512-r7bBUGKzlqk8oPBDYxt6Z0aEdF1G1rwlMcLk8LCOMbOzf0mG+JUfUzG4fIMWwHWP0iyaLWEQZJmtB7nOHEm/qw==" + "nan@2.24.0": { + "integrity": "sha512-Vpf9qnVW1RaDkoNKFUvfxqAbtI8ncb8OJlqZ9wwpXzWPEsvsB1nvdUi6oYrHIkQ1Y/tMDnr1h4nczS0VB9Xykg==" }, "nearley@2.20.1": { "integrity": "sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ==", @@ -1765,10 +1769,11 @@ "npm:@actions/github@^6.0.1", "npm:@libpg-query/parser@^17.6.3", "npm:@pgsql/types@^17.6.1", - "npm:@query-doctor/core@^0.0.4", + "npm:@query-doctor/core@^0.0.5", "npm:@testcontainers/postgresql@^11.9.0", "npm:@types/node@^24.9.1", "npm:@types/nunjucks@^3.2.6", + "npm:async-sema@^3.1.1", "npm:chokidar@^4.0.3", "npm:dedent@^1.6.0", "npm:fast-csv@^5.0.5", diff --git a/src/main.ts b/src/main.ts index f3d7b6e..92b7021 100644 --- a/src/main.ts +++ b/src/main.ts @@ -4,9 +4,10 @@ import { env } from "./env.ts"; import { log } from "./log.ts"; import { createServer } from "./server/http.ts"; import { shutdown } from "./shutdown.ts"; +import { Connectable } from "./sync/connectable.ts"; async function runInCI( - postgresUrl: string, + postgresUrl: Connectable, logPath: string, statisticsPath?: string, maxCost?: number, @@ -44,7 +45,7 @@ async function main() { Deno.exit(1); } await runInCI( - env.POSTGRES_URL, + Connectable.fromString(env.POSTGRES_URL), env.LOG_PATH, env.STATISTICS_PATH, typeof env.MAX_COST === "number" ? env.MAX_COST : undefined, diff --git a/src/remote/query-optimizer.test.ts b/src/remote/query-optimizer.test.ts new file mode 100644 index 0000000..f0a65e2 --- /dev/null +++ b/src/remote/query-optimizer.test.ts @@ -0,0 +1,54 @@ +import { PostgreSqlContainer } from "@testcontainers/postgresql"; +import { QueryOptimizer } from "./query-optimizer.ts"; +import { ConnectionManager } from "../sync/connection-manager.ts"; +import { Connectable } from "../sync/connectable.ts"; +import { setTimeout } from "node:timers/promises"; + +Deno.test({ + name: "controller syncs correctly", + sanitizeOps: false, + sanitizeResources: false, + fn: async () => { + const pg = await new PostgreSqlContainer("postgres:17") + .withCopyContentToContainer([ + { + content: ` + create table testing(a int, b text); + insert into testing values (1); + create index on testing(b); + create extension pg_stat_statements; + select * from testing where a = 1; + `, + target: "/docker-entrypoint-initdb.d/init.sql", + }, + ]) + .withCommand([ + "-c", + "shared_preload_libraries=pg_stat_statements", + "-c", + "log_statement=all", + ]) + .start(); + + const manager = ConnectionManager.forLocalDatabase(); + const optimizer = new QueryOptimizer(manager); + optimizer.addListener("improvementsAvailable", (query) => { + console.log("optimized!", query.query); + }); + optimizer.addListener("error", (query, error) => { + console.error("error!", query, error); + }); + optimizer.addListener("zeroCostPlan", (query) => { + console.log("zero cost plan!", query.query); + }); + const conn = Connectable.fromString(pg.getConnectionUri()); + const connector = manager.getConnectorFor(conn); + try { + const recentQueries = await connector.getRecentQueries(); + await optimizer.start(conn, recentQueries); + await setTimeout(10_000); + } finally { + await pg.stop(); + } + }, +}); diff --git a/src/remote/query-optimizer.ts b/src/remote/query-optimizer.ts new file mode 100644 index 0000000..8f6614b --- /dev/null +++ b/src/remote/query-optimizer.ts @@ -0,0 +1,356 @@ +import EventEmitter from "node:events"; +import { QueryHash, RecentQuery } from "../sql/recent-query.ts"; +import { ConnectionManager } from "../sync/connection-manager.ts"; +import { Sema } from "async-sema"; +import { + Analyzer, + IndexOptimizer, + OptimizeResult, + PostgresQueryBuilder, + PostgresVersion, + Statistics, + StatisticsMode, +} from "@query-doctor/core"; +import { Connectable } from "../sync/connectable.ts"; +import { parse } from "@libpg-query/parser"; + +const MINIMUM_COST_CHANGE_PERCENTAGE = 5; +const QUERY_TIMEOUT_MS = 10000; + +type EventMap = { + optimize: [RecentQuery]; + error: [RecentQuery, string]; + timeout: [RecentQuery]; + zeroCostPlan: [RecentQuery]; + noImprovements: [RecentQuery]; + improvementsAvailable: [RecentQuery]; +}; + +type OptimizedQuery = { + recentQuery: RecentQuery; + optimization: LiveQueryOptimization; +}; + +export class QueryOptimizer extends EventEmitter { + private static readonly MAX_CONCURRENCY = 1; + private static readonly defaultStatistics: StatisticsMode = { + kind: "fromAssumption", + relpages: 1, + reltuples: 10_000, + }; + private readonly queries = new Map(); + private target?: { + optimizer: IndexOptimizer; + statistics: Statistics; + }; + private semaphore = new Sema(QueryOptimizer.MAX_CONCURRENCY); + + private readonly analyzer = new Analyzer(parse); + + constructor( + private readonly manager: ConnectionManager, + ) { + super(); + } + + async start( + conn: Connectable, + recentQueries?: RecentQuery[], + statsMode: StatisticsMode = QueryOptimizer.defaultStatistics, + ) { + this.stop(); + const version = PostgresVersion.parse("17"); + const pg = this.manager.getOrCreateConnection(conn); + const ownStats = await Statistics.dumpStats(pg, version, "full"); + const statistics = new Statistics( + pg, + version, + ownStats, + statsMode, + ); + const existingIndexes = await statistics.getExistingIndexes(); + const optimizer = new IndexOptimizer(pg, statistics, existingIndexes, { + // we're not running on the que + // so traces have to be disabled + trace: false, + }); + this.target = { + optimizer, + statistics, + }; + if (recentQueries) { + for (const query of recentQueries) { + this.queries.set(query.hash, { + recentQuery: query, + optimization: { state: "waiting" }, + }); + } + } + for (let i = 0; i < QueryOptimizer.MAX_CONCURRENCY; i++) { + this.work(); + } + } + + stop() { + this.semaphore = new Sema(QueryOptimizer.MAX_CONCURRENCY); + this.queries.clear(); + this.target = undefined; + } + + private async work() { + // don't enter if there isn't enough space in the semaphore + const token = await this.semaphore.acquire(); + try { + if (!this.target) { + return; + } + let recentQuery: RecentQuery | undefined; + for (const [_hash, query] of this.queries.entries()) { + if (query.optimization.state !== "waiting") { + continue; + } + recentQuery = query.recentQuery; + } + if (recentQuery) { + if (!this.isQuerySupported(recentQuery)) { + this.onQueryUnsupported(recentQuery); + return; + } + this.queries.set(recentQuery.hash, { + recentQuery, + optimization: { state: "optimizing" }, + }); + await this.optimizeQuery(recentQuery); + } + } finally { + this.semaphore.release(token); + setTimeout(() => this.work(), 100); + } + } + + private isQuerySupported(q: RecentQuery) { + return !q.isSystemQuery && q.isSelectQuery; + } + + private async optimizeQuery(recent: RecentQuery) { + if (!this.target) { + return; + } + const builder = new PostgresQueryBuilder(recent.query); + let cost: number; + try { + const explain = await withTimeout( + this.target.optimizer.runWithoutIndexes(builder), + QUERY_TIMEOUT_MS, + ); + cost = explain.Plan["Total Cost"]; + } catch (error) { + if (error instanceof TimeoutError) { + this.onTimeout(recent); + } else if (error instanceof Error) { + this.onError(recent, error.message); + } else { + this.onError(recent, "Internal error"); + } + return; + } + if (cost === 0) { + this.onZeroCostPlan(recent); + return; + } + const indexes = this.getPotentialIndexCandidates( + this.target.statistics, + recent, + ); + let result: OptimizeResult; + try { + result = await withTimeout( + this.target.optimizer.run(builder, indexes), + QUERY_TIMEOUT_MS, + ); + } catch (error) { + if (error instanceof TimeoutError) { + this.onTimeout(recent); + } else if (error instanceof Error) { + this.onError(recent, error.message); + } else { + this.onError(recent, "Internal error"); + } + return; + } + + return this.onOptimizeReady(result, recent); + } + + private onOptimizeReady(result: OptimizeResult, recent: RecentQuery) { + switch (result.kind) { + case "ok": { + const indexRecommendations = mapIndexRecommandations(result); + const percentageReduction = costDifferencePercentage( + result.baseCost, + result.finalCost, + ); + const indexesUsed = Array.from(result.existingIndexes); + const costReductionPercentage = Math.trunc( + Math.abs(percentageReduction), + ); + if (costReductionPercentage < MINIMUM_COST_CHANGE_PERCENTAGE) { + this.emit("noImprovements", recent); + return { + state: "no_improvement_found", + cost: result.baseCost, + indexesUsed, + }; + } else { + this.emit("improvementsAvailable", recent); + return { + state: "improvements_available", + cost: result.baseCost, + optimizedCost: result.finalCost, + costReductionPercentage, + indexRecommendations, + indexesUsed, + }; + } + } + // unlikely to hit if we've already checked the base plan for zero cost + case "zero_cost_plan": + return this.onZeroCostPlan(recent); + } + } + + private getPotentialIndexCandidates( + statistics: Statistics, + recent: RecentQuery, + ) { + return this.analyzer.deriveIndexes( + statistics.ownMetadata, + recent.columnReferences, + ); + } + + private onQueryUnsupported(recent: RecentQuery) { + this.queries.set(recent.hash, { + recentQuery: recent, + optimization: { + state: "not_supported", + reason: "Query is not supported", + }, + }); + } + + private onImprovementsAvailable( + recent: RecentQuery, + result: Extract, + ) { + this.queries.set(recent.hash, { + recentQuery: recent, + optimization: { + state: "improvements_available", + cost: result.baseCost, + optimizedCost: result.finalCost, + costReductionPercentage: 0, + indexRecommendations: [], + indexesUsed: [], + // costReductionPercentage, + // indexRecommendations, + // indexesUsed, + }, + }); + this.emit("improvementsAvailable", recent); + } + + private onZeroCostPlan(recent: RecentQuery) { + this.queries.set(recent.hash, { + recentQuery: recent, + optimization: { + state: "error", + error: + "Query plan had zero cost. This should not happen on a patched postgres instance", + }, + }); + this.emit("zeroCostPlan", recent); + } + + private onError(recent: RecentQuery, errorMessage: string) { + this.queries.set(recent.hash, { + recentQuery: recent, + optimization: { state: "error", error: errorMessage }, + }); + this.emit("error", recent, errorMessage); + } + + private onTimeout(recent: RecentQuery) { + this.queries.set(recent.hash, { + recentQuery: recent, + optimization: { state: "timeout" }, + }); + this.emit("timeout", recent); + } +} + +export class TimeoutError extends Error { + constructor() { + super("Timeout"); + this.name = "TimeoutError"; + } +} + +export const withTimeout = ( + promise: Promise, + timeout: number, +): Promise => { + return Promise.race([ + promise, + new Promise((_, reject) => + setTimeout(() => reject(new TimeoutError()), timeout) + ), + ]); +}; + +function mapIndexRecommandations( + result: Extract, +): string[] { + return Array.from(result.newIndexes.keys(), (definition) => { + const index = result.triedIndexes.get(definition); + if (!index) { + throw new Error( + `Index ${definition} not found in tried indexes. This shouldn't happen.`, + ); + } + return definition; + }); +} + +type PercentageDifference = number; + +export function costDifferencePercentage( + oldVal: number, + newVal: number, +): PercentageDifference { + return ((newVal - oldVal) / oldVal) * 100; +} + +export type LiveQueryOptimization = + | { state: "waiting" } + | { state: "optimizing" } + // system queries and certain other queries are exempt from optimization + | { state: "not_supported"; reason: string } + | { + state: "improvements_available"; + cost: number; + optimizedCost: number; + costReductionPercentage: number; + indexRecommendations: string[]; + // indexRecommendations: TraceFoundIndex[]; + indexesUsed: string[]; + } + | { + state: "no_improvement_found"; + cost: number; + indexesUsed: string[]; + } + // Cost is nullable in case the timeout was caused by the initial query + // before we even add any indexes to it (usually unlikely) + | { state: "timeout"; cost?: number } + | { state: "error"; error: string }; diff --git a/src/remote/remote-controller.test.ts b/src/remote/remote-controller.test.ts index 190a567..6f4be41 100644 --- a/src/remote/remote-controller.test.ts +++ b/src/remote/remote-controller.test.ts @@ -1,14 +1,12 @@ -import z from "zod"; import { PostgreSqlContainer } from "@testcontainers/postgresql"; import { Connectable } from "../sync/connectable.ts"; import { Remote } from "./remote.ts"; import postgres from "postgresjs"; import { assertEquals } from "@std/assert/equals"; -import { wrapGenericPostgresInterface } from "../sql/postgresjs.ts"; import { RemoteController } from "./remote-controller.ts"; import { ConnectionManager } from "../sync/connection-manager.ts"; +import { RemoteSyncRequest } from "./remote.dto.ts"; -const connectable = z.string().transform(Connectable.transform); Deno.test({ name: "controller syncs correctly", sanitizeOps: false, @@ -35,25 +33,26 @@ Deno.test({ ]); try { - const target = connectable.parse( + const target = Connectable.fromString( targetDb.getConnectionUri(), ); - const source = connectable.parse( + const source = Connectable.fromString( sourceDb.getConnectionUri(), ); - const man = new ConnectionManager(wrapGenericPostgresInterface); + const sourceOptimizer = ConnectionManager.forLocalDatabase(); + const remote = new RemoteController( - new Remote(target, man), + new Remote(target, sourceOptimizer), ); const response = await remote.execute( new Request( - "http://testing.local/postgres", + "https://anything.whatever/postgres", { method: "POST", - body: JSON.stringify({ - db: source.toString(), + body: RemoteSyncRequest.encode({ + db: source, }), }, ), diff --git a/src/remote/remote-controller.ts b/src/remote/remote-controller.ts index b4ab390..f50eecf 100644 --- a/src/remote/remote-controller.ts +++ b/src/remote/remote-controller.ts @@ -17,7 +17,7 @@ export class RemoteController { } async onFullSync(request: Request): Promise { - const body = RemoteSyncRequest.safeParse(await request.json()); + const body = RemoteSyncRequest.safeDecode(await request.text()); if (!body.success) { return new Response(JSON.stringify(body.error), { status: 400 }); } diff --git a/src/remote/remote.dto.ts b/src/remote/remote.dto.ts index d5a1f58..8d39ec5 100644 --- a/src/remote/remote.dto.ts +++ b/src/remote/remote.dto.ts @@ -3,9 +3,19 @@ import { Connectable } from "../sync/connectable.ts"; import { RecentQuery } from "../sql/recent-query.ts"; import { FullSchema } from "../sync/schema_differ.ts"; -export const RemoteSyncRequest = z.object({ - db: z.string().transform(Connectable.transform), -}); +export const RemoteSyncRequest = z.codec( + z.string(), + z.object({ + db: z.custom(), + }), + { + encode: (value) => JSON.stringify({ db: value.db.toString() }), + decode: (value) => { + const parsed = JSON.parse(value); + return { db: Connectable.fromString(parsed.db) }; + }, + }, +); export const RemoteSyncFullSchemaResponse = z.discriminatedUnion("type", [ z.object({ type: z.literal("ok"), value: FullSchema }), diff --git a/src/remote/remote.test.ts b/src/remote/remote.test.ts index 837ea77..1f016b4 100644 --- a/src/remote/remote.test.ts +++ b/src/remote/remote.test.ts @@ -1,10 +1,8 @@ -import z from "zod"; import { PostgreSqlContainer } from "@testcontainers/postgresql"; import { Connectable } from "../sync/connectable.ts"; import { Remote } from "./remote.ts"; import postgres from "postgresjs"; import { assertEquals } from "@std/assert/equals"; -import { wrapGenericPostgresInterface } from "../sql/postgresjs.ts"; import { ConnectionManager } from "../sync/connection-manager.ts"; import { assertArrayIncludes } from "@std/assert"; @@ -14,7 +12,6 @@ function assertOk( assertEquals(result.type, "ok"); } -const connectable = z.string().transform(Connectable.transform); Deno.test({ name: "syncs correctly", sanitizeOps: false, @@ -29,7 +26,7 @@ Deno.test({ create extension pg_stat_statements; create table testing(a int, b text); insert into testing values (1); - create index on testing(b); + create index "testing_1234" on testing(b); select * from testing where a = 1; `, target: "/docker-entrypoint-initdb.d/init.sql", @@ -47,17 +44,14 @@ Deno.test({ ]); try { - const target = connectable.parse( - targetDb.getConnectionUri(), - ); - const source = connectable.parse( - sourceDb.getConnectionUri(), - ); + const target = Connectable.fromString(targetDb.getConnectionUri()); + const source = Connectable.fromString(sourceDb.getConnectionUri()); const remote = new Remote( target, - new ConnectionManager(wrapGenericPostgresInterface), + ConnectionManager.forLocalDatabase(), ); + const result = await remote.syncFrom(source); assertOk(result.queries); @@ -67,18 +61,33 @@ Deno.test({ "select * from testing where a = $1", ]); + assertOk(result.schema); + + const tableNames = result.schema.value.tables.map((table) => + table.tableName + ); + + assertArrayIncludes(tableNames, ["testing"]); + + const indexNames = result.schema.value.indexes.map((index) => + index.indexName + ); + assertArrayIncludes(indexNames, ["testing_1234"]); + const sql = postgres( target.withDatabaseName(Remote.optimizingDbName).toString(), ); const indexesAfter = - await sql`select * from pg_indexes where schemaname = 'public'`; + await sql`select indexname from pg_indexes where schemaname = 'public'`; assertEquals( indexesAfter.count, 1, "Indexes were not copied over correctly from the source db", ); + assertEquals(indexesAfter[0], { indexname: "testing_1234" }); + const tablesAfter = await sql`select tablename from pg_tables where schemaname = 'public'`; assertEquals( diff --git a/src/remote/remote.ts b/src/remote/remote.ts index b658b1f..96230a8 100644 --- a/src/remote/remote.ts +++ b/src/remote/remote.ts @@ -1,16 +1,24 @@ -import { PgIdentifier, type Postgres } from "@query-doctor/core"; +import { + PgIdentifier, + type Postgres, + StatisticsMode, +} from "@query-doctor/core"; import { type Connectable } from "../sync/connectable.ts"; import { DumpCommand, RestoreCommand } from "../sync/schema-link.ts"; import { ConnectionManager } from "../sync/connection-manager.ts"; import { type RecentQuery } from "../sql/recent-query.ts"; import { type FullSchema, SchemaDiffer } from "../sync/schema_differ.ts"; import { type RemoteSyncResponse } from "./remote.dto.ts"; +import { QueryOptimizer } from "./query-optimizer.ts"; /** * Represents a db for doing optimization work. * We only maintain one instance of this class as we only do * optimization against one physical postgres database. * But potentially more logical databases in the future. + * + * `Remote` only concerns itself with the remote it's doing optimization + * against. It does not deal with the source in any way aside from running sync */ export class Remote { static readonly baseDbName = PgIdentifier.fromString("postgres"); @@ -29,21 +37,25 @@ export class Remote { * 2 -> connections to {@link Remote.databaseName}. This connection pool is * destroyed and re-created on each successful sync along with the db itself */ - private readonly baseDb: Postgres; + private baseDbURL: Connectable; + private readonly queryOptimizer: QueryOptimizer; constructor( /** This has to be a local url. Very bad things will happen if this is a remote URL */ private readonly targetURL: Connectable, private readonly manager: ConnectionManager, ) { - const baseUrl = targetURL.withDatabaseName(Remote.baseDbName); - this.baseDb = this.manager.getOrCreateConnection(baseUrl); + this.baseDbURL = targetURL.withDatabaseName(Remote.baseDbName); + // this.baseDb = this.targetManager.getOrCreateConnection(baseUrl); + this.queryOptimizer = new QueryOptimizer(this.manager); } - async syncFrom(source: Connectable): Promise { + async syncFrom( + source: Connectable, + stats?: StatisticsMode, + ): Promise { await this.resetDatabase(); const target = this.targetURL.withDatabaseName(Remote.optimizingDbName); - const sql = this.manager.getOrCreateConnection(source); const [_restoreResult, recentQueries, fullSchema] = await Promise .allSettled([ // This potentially creates a lot of connections to the source @@ -53,11 +65,23 @@ export class Remote { ]); if (fullSchema.status === "fulfilled") { - this.differ.put(sql, fullSchema.value); + this.differ.put(source, fullSchema.value); } - const pg = this.manager.getOrCreateConnection(this.targetURL); - await this.onSuccessfulSync(pg); + const pg = this.manager.getOrCreateConnection( + this.targetURL, + ); + + let queries: RecentQuery[] = []; + if (recentQueries.status === "fulfilled") { + queries = recentQueries.value; + } + + await this.onSuccessfulSync( + pg, + queries, + stats, + ); return { queries: recentQueries.status === "fulfilled" @@ -89,12 +113,13 @@ export class Remote { */ private async resetDatabase(): Promise { const databaseName = Remote.optimizingDbName; + const baseDb = this.manager.getOrCreateConnection(this.baseDbURL); // these cannot be run in the same `exec` block as that implicitly creates transactions - await this.baseDb.exec( + await baseDb.exec( // drop database does not allow parameterization `drop database if exists ${databaseName} with (force);`, ); - await this.baseDb.exec(`create database ${databaseName};`); + await baseDb.exec(`create database ${databaseName};`); } private async pipeSchema( @@ -140,10 +165,15 @@ export class Remote { /** * Process a successful sync and run any potential cleanup functions */ - private async onSuccessfulSync(postgres: Postgres): Promise { + private async onSuccessfulSync( + postgres: Postgres, + recentQueries?: RecentQuery[], + stats?: StatisticsMode, + ): Promise { if (this.targetURL.isSupabase()) { // https://gist.github.com/Xetera/067c613580320468e8367d9d6c0e06ad await postgres.exec("drop schema if exists extensions cascade"); } + await this.queryOptimizer.start(this.targetURL, recentQueries, stats); } } diff --git a/src/runner.ts b/src/runner.ts index 23c759a..0fb5b3f 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -25,8 +25,9 @@ import { } from "./reporters/reporter.ts"; import { bgBrightMagenta, blue, yellow } from "@std/fmt/colors"; import { env } from "./env.ts"; -import { wrapGenericPostgresInterface } from "./sql/postgresjs.ts"; +import { connectToSource } from "./sql/postgresjs.ts"; import { parse } from "@libpg-query/parser"; +import { Connectable } from "./sync/connectable.ts"; export class Runner { private readonly seenQueries = new Set(); @@ -45,12 +46,12 @@ export class Runner { ) {} static async build(options: { - postgresUrl: string; + postgresUrl: Connectable; statisticsPath?: string; maxCost?: number; logPath: string; }) { - const db = wrapGenericPostgresInterface({ url: options.postgresUrl }); + const db = connectToSource(options.postgresUrl); const statisticsMode = Runner.decideStatisticsMode(options.statisticsPath); const stats = await Statistics.fromPostgres(db, statisticsMode); const existingIndexes = await stats.getExistingIndexes(); diff --git a/src/server/http.ts b/src/server/http.ts index 1ce54e9..bec834a 100644 --- a/src/server/http.ts +++ b/src/server/http.ts @@ -7,7 +7,7 @@ import { ZodError } from "zod"; import { shutdownController } from "../shutdown.ts"; import { env } from "../env.ts"; import { SyncResult } from "../sync/syncer.ts"; -import { wrapGenericPostgresInterface } from "../sql/postgresjs.ts"; +import { connectToOptimizer, connectToSource } from "../sql/postgresjs.ts"; import type { RateLimitResult } from "@rabbit-company/rate-limiter"; import * as errors from "../sync/errors.ts"; import { RemoteController } from "../remote/remote-controller.ts"; @@ -15,8 +15,9 @@ import { Connectable } from "../sync/connectable.ts"; import { ConnectionManager } from "../sync/connection-manager.ts"; import { Remote } from "../remote/remote.ts"; -const manager = new ConnectionManager(wrapGenericPostgresInterface); -const syncer = new PostgresSyncer(manager); +const sourceConnectionManager = new ConnectionManager(connectToSource); + +const syncer = new PostgresSyncer(sourceConnectionManager); async function onSync(req: Request) { const startTime = Date.now(); @@ -158,9 +159,14 @@ export function createServer( port: number, targetDb?: Connectable, ) { - const manager = new ConnectionManager(wrapGenericPostgresInterface); + const optimizingDbConnectionManager = new ConnectionManager( + connectToOptimizer, + ); + const remoteController = targetDb - ? new RemoteController(new Remote(targetDb, manager)) + ? new RemoteController( + new Remote(targetDb, optimizingDbConnectionManager), + ) : undefined; return Deno.serve( { hostname, port, signal: shutdownController.signal }, diff --git a/src/server/sync.dto.ts b/src/server/sync.dto.ts index a8fcf1e..1aaac4d 100644 --- a/src/server/sync.dto.ts +++ b/src/server/sync.dto.ts @@ -1,14 +1,14 @@ -import { z } from "zod/v4"; -import { Connectable } from "../sync/connectable.ts"; +import { z } from "zod"; +import { ConnectableParser } from "../sync/connectable.ts"; export const LiveQueryRequest = z.object({ - db: z.string().transform(Connectable.transform), + db: ConnectableParser, }); export type LiveQueryRequest = z.infer; export const SyncRequest = z.object({ - db: z.string().transform(Connectable.transform), + db: ConnectableParser, seed: z.coerce.number().min(0).max(1).default(0), schema: z.coerce.string().default("public").meta({ deprecated: true, diff --git a/src/sql/postgresjs.ts b/src/sql/postgresjs.ts index 3d71b27..082055d 100644 --- a/src/sql/postgresjs.ts +++ b/src/sql/postgresjs.ts @@ -1,10 +1,10 @@ import postgres from "postgresjs"; import { type Postgres, - type PostgresConnectionInput, type PostgresTransaction, PostgresVersion, } from "@query-doctor/core"; +import { Connectable } from "../sync/connectable.ts"; type PgConnectionOptions = postgres.Options< Record @@ -17,21 +17,41 @@ const DEFAULT_IDLE_TIMEOUT_SECONDS = 15; // it's ok to recycle connections frequently if needed const DEFAULT_MAX_LIFETIME_SECONDS = 60 * 5; -const connectionOptions: PgConnectionOptions = { - max: 20, - max_lifetime: DEFAULT_MAX_LIFETIME_SECONDS, - idle_timeout: DEFAULT_IDLE_TIMEOUT_SECONDS, -}; +/** + * Connecting to the local optimizer + */ +export function connectToOptimizer(connectable: Connectable) { + const connectionOptions: PgConnectionOptions = { + max: 100, + }; + + return connect(connectable, connectionOptions); +} + +/** + * Connect to the source database to pull data out. + * We have to be a lot more conservative here + * and make sure the connections drop asap to prevent + * exhausting them + */ +export function connectToSource( + connectable: Connectable, +) { + const connectionOptions: PgConnectionOptions = { + max: 20, + max_lifetime: DEFAULT_MAX_LIFETIME_SECONDS, + idle_timeout: DEFAULT_IDLE_TIMEOUT_SECONDS, + }; + + return connect(connectable, connectionOptions); +} + +function connect(connectable: Connectable, options: PgConnectionOptions) { + const pg = postgres(connectable.toString(), options); + return wrapGenericPostgresInterface(pg); +} -export function wrapGenericPostgresInterface( - input: PostgresConnectionInput, -): Postgres { - let pg: postgres.Sql; - if ("url" in input) { - pg = postgres(input.url, connectionOptions); - } else { - throw new Error("Invalid input"); - } +export function wrapGenericPostgresInterface(pg: postgres.Sql): Postgres { return { exec: (query, params) => { return pg.unsafe(query, params as postgres.ParameterOrJSON[]); diff --git a/src/sql/recent-query.ts b/src/sql/recent-query.ts index 95cc25a..7f521ab 100644 --- a/src/sql/recent-query.ts +++ b/src/sql/recent-query.ts @@ -7,6 +7,7 @@ import { SQLCommenterTag, } from "@query-doctor/core"; import { parse } from "@libpg-query/parser"; +import z from "zod"; /** * Constructed by syncing with {@link SegmentedQueryCache.sync} @@ -30,6 +31,7 @@ export class RecentQuery { readonly tableReferences: string[], readonly columnReferences: DiscoveredColumnReference[], readonly tags: SQLCommenterTag[], + readonly hash: QueryHash, readonly seenAt: number, ) { this.username = data.username; @@ -50,6 +52,7 @@ export class RecentQuery { static async analyze( data: RawRecentQuery, + hash: QueryHash, seenAt: number, ) { const analyzer = new Analyzer(parse); @@ -59,6 +62,7 @@ export class RecentQuery { analysis.referencedTables, analysis.indexesToCheck, analysis.tags, + hash, seenAt, ); } @@ -81,3 +85,6 @@ export type RawRecentQuery = { rows: string; topLevel: boolean; }; + +export const QueryHash = z.string().brand<"QueryHash">(); +export type QueryHash = z.infer; diff --git a/src/sync/connectable.ts b/src/sync/connectable.ts index 5ace159..e6ef648 100644 --- a/src/sync/connectable.ts +++ b/src/sync/connectable.ts @@ -1,4 +1,4 @@ -import { z } from "zod/v4"; +import { z } from "zod"; import { env } from "../env.ts"; import { PgIdentifier } from "@query-doctor/core"; @@ -111,4 +111,4 @@ export class Connectable { } } -const ConnectableParser = z.string().transform(Connectable.transform); +export const ConnectableParser = z.string().transform(Connectable.transform); diff --git a/src/sync/connection-manager.ts b/src/sync/connection-manager.ts index 9c8c6e7..f547d10 100644 --- a/src/sync/connection-manager.ts +++ b/src/sync/connection-manager.ts @@ -1,7 +1,8 @@ -import { Postgres, PostgresFactory } from "@query-doctor/core"; +import type { Postgres } from "@query-doctor/core"; import { SegmentedQueryCache } from "./seen-cache.ts"; import { Connectable } from "./connectable.ts"; import { PostgresConnector } from "./pg-connector.ts"; +import { connectToOptimizer, connectToSource } from "../sql/postgresjs.ts"; /** * Manages connections and query caches for each connection @@ -13,13 +14,31 @@ export class ConnectionManager { // ConnectionMap should be responsible for closing connections private readonly connections = new Map(); - constructor(private readonly factory: PostgresFactory) {} + constructor( + private readonly factory: (connectable: Connectable) => Postgres, + ) {} + + /** + * Create a connection manager with default settings + * optimized for connecting to local dbs (used for optimizing) + */ + static forLocalDatabase() { + return new ConnectionManager(connectToOptimizer); + } + + /** + * Create a connection manager with default settings + * optimized for connecting to remote dbs (given by users) + */ + static forRemoteDatabase() { + return new ConnectionManager(connectToSource); + } getOrCreateConnection(connectable: Connectable): Postgres { const urlString = connectable.toString(); let sql = this.connections.get(urlString); if (!sql) { - sql = this.factory({ url: urlString }); + sql = this.factory(connectable); this.connections.set(urlString, sql); } return sql; diff --git a/src/sync/schema_differ.ts b/src/sync/schema_differ.ts index ac9c69d..22f5337 100644 --- a/src/sync/schema_differ.ts +++ b/src/sync/schema_differ.ts @@ -1,7 +1,7 @@ -import type { Postgres } from "@query-doctor/core"; import { create } from "jsondiffpatch"; import { format, type Op } from "jsondiffpatch/formatters/jsonpatch"; import { z } from "zod"; +import { Connectable } from "./connectable.ts"; export class SchemaDiffer { private readonly differ = create({ @@ -28,9 +28,9 @@ export class SchemaDiffer { }, }); - private readonly stats = new WeakMap(); + private readonly stats = new WeakMap(); - put(postgres: Postgres, schema: FullSchema): Op[] | undefined { + put(postgres: Connectable, schema: FullSchema): Op[] | undefined { const old = this.stats.get(postgres); if (!old) { this.stats.set(postgres, schema); diff --git a/src/sync/seen-cache.ts b/src/sync/seen-cache.ts index 8502af7..f7704cb 100644 --- a/src/sync/seen-cache.ts +++ b/src/sync/seen-cache.ts @@ -1,16 +1,12 @@ import type { Postgres } from "@query-doctor/core"; -import { RawRecentQuery, RecentQuery } from "../sql/recent-query.ts"; +import { QueryHash, RawRecentQuery, RecentQuery } from "../sql/recent-query.ts"; import { fingerprint } from "@libpg-query/parser"; -import z from "zod"; interface CacheEntry { firstSeen: number; lastSeen: number; } -const QueryHash = z.string().brand<"QueryHash">(); -type QueryHash = z.infer; - export class QueryCache { private list: Record = {}; private readonly createdAt: number; @@ -54,7 +50,7 @@ export class QueryCache { // TODO: bound the concurrency return await Promise.all(rawQueries.map(async (rawQuery) => { const key = await this.store(rawQuery); - return RecentQuery.analyze(rawQuery, this.getFirstSeen(key)); + return RecentQuery.analyze(rawQuery, key, this.getFirstSeen(key)); })); } From d0496ee6d8a74748612d20b124680e329374cc02 Mon Sep 17 00:00:00 2001 From: Xetera Date: Thu, 4 Dec 2025 23:50:27 +0300 Subject: [PATCH 06/29] feat: add tests for optimizer --- src/remote/query-optimizer.test.ts | 33 ++++++++++++++++++++++-------- src/remote/query-optimizer.ts | 3 +-- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/remote/query-optimizer.test.ts b/src/remote/query-optimizer.test.ts index f0a65e2..3540697 100644 --- a/src/remote/query-optimizer.test.ts +++ b/src/remote/query-optimizer.test.ts @@ -3,6 +3,9 @@ import { QueryOptimizer } from "./query-optimizer.ts"; import { ConnectionManager } from "../sync/connection-manager.ts"; import { Connectable } from "../sync/connectable.ts"; import { setTimeout } from "node:timers/promises"; +import { assertStrictEquals } from "@std/assert"; +import { assertEquals } from "@std/assert/equals"; +import { assertArrayIncludes } from "@std/assert/array-includes"; Deno.test({ name: "controller syncs correctly", @@ -17,7 +20,10 @@ Deno.test({ insert into testing values (1); create index on testing(b); create extension pg_stat_statements; - select * from testing where a = 1; + select * from testing where a = 10; + select * from testing where b = 'c'; + select * from testing where b > 'a'; + select * from testing where b < 'b'; `, target: "/docker-entrypoint-initdb.d/init.sql", }, @@ -32,21 +38,32 @@ Deno.test({ const manager = ConnectionManager.forLocalDatabase(); const optimizer = new QueryOptimizer(manager); + + const expectedImprovements = ["select * from testing where a = $1"]; + const expectedNoImprovements = [ + "select * from testing where b = $1", + "select * from testing where b > $1", + "select * from testing where b < $1", + ]; + + const improvements: string[] = []; + const noImprovements: string[] = []; + optimizer.addListener("improvementsAvailable", (query) => { - console.log("optimized!", query.query); + improvements.push(query.query); }); - optimizer.addListener("error", (query, error) => { - console.error("error!", query, error); - }); - optimizer.addListener("zeroCostPlan", (query) => { - console.log("zero cost plan!", query.query); + optimizer.addListener("noImprovements", (query) => { + noImprovements.push(query.query); }); + const conn = Connectable.fromString(pg.getConnectionUri()); const connector = manager.getConnectorFor(conn); try { const recentQueries = await connector.getRecentQueries(); await optimizer.start(conn, recentQueries); - await setTimeout(10_000); + await setTimeout(1_000); + assertArrayIncludes(expectedImprovements, improvements); + assertArrayIncludes(expectedNoImprovements, noImprovements); } finally { await pg.stop(); } diff --git a/src/remote/query-optimizer.ts b/src/remote/query-optimizer.ts index 8f6614b..2f18c62 100644 --- a/src/remote/query-optimizer.ts +++ b/src/remote/query-optimizer.ts @@ -18,7 +18,6 @@ const MINIMUM_COST_CHANGE_PERCENTAGE = 5; const QUERY_TIMEOUT_MS = 10000; type EventMap = { - optimize: [RecentQuery]; error: [RecentQuery, string]; timeout: [RecentQuery]; zeroCostPlan: [RecentQuery]; @@ -202,7 +201,7 @@ export class QueryOptimizer extends EventEmitter { indexesUsed, }; } else { - this.emit("improvementsAvailable", recent); + this.onImprovementsAvailable(recent, result); return { state: "improvements_available", cost: result.baseCost, From 1b3ec46ab7a58337da6e17522063737a769920ae Mon Sep 17 00:00:00 2001 From: Xetera Date: Wed, 10 Dec 2025 21:49:33 +0300 Subject: [PATCH 07/29] feat: query optimizer + websockets --- .gitignore | 2 + deno.json | 3 + deno.lock | 725 +++++++++++++++++++++------ devenv.lock | 16 +- src/remote/query-optimizer.test.ts | 95 +++- src/remote/query-optimizer.ts | 352 ++++++++----- src/remote/remote-controller.test.ts | 51 +- src/remote/remote-controller.ts | 95 +++- src/remote/remote.ts | 47 +- src/sql/recent-query.ts | 16 + src/sync/pg-connector.ts | 7 +- src/sync/schema-link.ts | 45 +- src/sync/syncer.ts | 7 +- 13 files changed, 1128 insertions(+), 333 deletions(-) diff --git a/.gitignore b/.gitignore index 5525035..72ad066 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ devenv.local.yaml # pre-commit .pre-commit-config.yaml .zed + +node_modules diff --git a/deno.json b/deno.json index 313c953..6488556 100644 --- a/deno.json +++ b/deno.json @@ -2,6 +2,7 @@ "$schema": "https://raw.githubusercontent.com/denoland/deno/refs/heads/main/cli/schemas/config-file.v1.json", "version": "0.1.1", "unstable": ["raw-imports"], + "nodeModulesDir": "auto", "permissions": { "run": { "sys": true, @@ -30,6 +31,7 @@ "@std/collections": "jsr:@std/collections@^1.1.3", "@std/data-structures": "jsr:@std/data-structures@^1.0.9", "@std/fmt": "jsr:@std/fmt@^1.0.8", + "@std/testing": "jsr:@std/testing@^1.0.16", "@testcontainers/postgresql": "npm:@testcontainers/postgresql@^11.9.0", "@types/node": "npm:@types/node@^24.9.1", "@types/nunjucks": "npm:@types/nunjucks@^3.2.6", @@ -39,6 +41,7 @@ "fast-csv": "npm:fast-csv@^5.0.5", "jsondiffpatch": "npm:jsondiffpatch@^0.7.3", "nunjucks": "npm:nunjucks@^3.2.4", + "patch-package": "npm:patch-package@^8.0.1", "pgsql-deparser": "npm:pgsql-deparser@^17.11.1", "sql-formatter": "npm:sql-formatter@^15.6.6", "sql-highlight": "npm:sql-highlight@^6.1.0", diff --git a/deno.lock b/deno.lock index c092bbb..cb6199a 100644 --- a/deno.lock +++ b/deno.lock @@ -5,10 +5,17 @@ "jsr:@rabbit-company/rate-limiter@3": "3.0.0", "jsr:@std/assert@^1.0.13": "1.0.14", "jsr:@std/assert@^1.0.14": "1.0.14", + "jsr:@std/assert@^1.0.15": "1.0.16", + "jsr:@std/async@^1.0.15": "1.0.15", "jsr:@std/collections@^1.1.3": "1.1.3", "jsr:@std/data-structures@^1.0.9": "1.0.9", "jsr:@std/fmt@^1.0.8": "1.0.8", + "jsr:@std/fs@^1.0.19": "1.0.20", "jsr:@std/internal@^1.0.10": "1.0.10", + "jsr:@std/internal@^1.0.12": "1.0.12", + "jsr:@std/path@^1.1.2": "1.1.3", + "jsr:@std/path@^1.1.3": "1.1.3", + "jsr:@std/testing@^1.0.16": "1.0.16", "npm:@actions/core@^1.11.1": "1.11.1", "npm:@actions/github@^6.0.1": "6.0.1_@octokit+core@5.2.2", "npm:@libpg-query/parser@^17.6.3": "17.6.3", @@ -23,6 +30,7 @@ "npm:fast-csv@^5.0.5": "5.0.5", "npm:jsondiffpatch@~0.7.3": "0.7.3", "npm:nunjucks@^3.2.4": "3.2.4_chokidar@4.0.3", + "npm:patch-package@^8.0.1": "8.0.1", "npm:pgsql-deparser@^17.11.1": "17.12.1", "npm:sql-formatter@^15.6.6": "15.6.6", "npm:sql-highlight@^6.1.0": "6.1.0", @@ -38,9 +46,18 @@ "@std/assert@1.0.14": { "integrity": "68d0d4a43b365abc927f45a9b85c639ea18a9fab96ad92281e493e4ed84abaa4", "dependencies": [ - "jsr:@std/internal" + "jsr:@std/internal@^1.0.10" ] }, + "@std/assert@1.0.16": { + "integrity": "6a7272ed1eaa77defe76e5ff63ca705d9c495077e2d5fd0126d2b53fc5bd6532", + "dependencies": [ + "jsr:@std/internal@^1.0.12" + ] + }, + "@std/async@1.0.15": { + "integrity": "55d1d9d04f99403fe5730ab16bdcc3c47f658a6bf054cafb38a50f046238116e" + }, "@std/collections@1.1.3": { "integrity": "bf8b0818886df6a32b64c7d3b037a425111f28278d69fd0995aeb62777c986b0" }, @@ -53,8 +70,34 @@ "@std/fmt@1.0.8": { "integrity": "71e1fc498787e4434d213647a6e43e794af4fd393ef8f52062246e06f7e372b7" }, + "@std/fs@1.0.20": { + "integrity": "e953206aae48d46ee65e8783ded459f23bec7dd1f3879512911c35e5484ea187", + "dependencies": [ + "jsr:@std/path@^1.1.3" + ] + }, "@std/internal@1.0.10": { "integrity": "e3be62ce42cab0e177c27698e5d9800122f67b766a0bea6ca4867886cbde8cf7" + }, + "@std/internal@1.0.12": { + "integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027" + }, + "@std/path@1.1.3": { + "integrity": "b015962d82a5e6daea980c32b82d2c40142149639968549c649031a230b1afb3", + "dependencies": [ + "jsr:@std/internal@^1.0.12" + ] + }, + "@std/testing@1.0.16": { + "integrity": "a917ffdeb5924c9be436dc78bc32e511760e14d3a96e49c607fc5ecca86d0092", + "dependencies": [ + "jsr:@std/assert@^1.0.15", + "jsr:@std/async", + "jsr:@std/data-structures", + "jsr:@std/fs", + "jsr:@std/internal@^1.0.12", + "jsr:@std/path@^1.1.2" + ] } }, "npm": { @@ -99,131 +142,6 @@ "@dmsnell/diff-match-patch@1.1.0": { "integrity": "sha512-yejLPmM5pjsGvxS9gXablUSbInW7H976c/FJ4iQxWIm7/38xBySRemTPDe34lhg1gVLbJntX0+sH0jYfU+PN9A==" }, - "@esbuild/aix-ppc64@0.25.5": { - "integrity": "sha512-9o3TMmpmftaCMepOdA5k/yDw8SfInyzWWTjYTFCX3kPSDJMROQTb8jg+h9Cnwnmm1vOzvxN7gIfB5V2ewpjtGA==", - "os": ["aix"], - "cpu": ["ppc64"] - }, - "@esbuild/android-arm64@0.25.5": { - "integrity": "sha512-VGzGhj4lJO+TVGV1v8ntCZWJktV7SGCs3Pn1GRWI1SBFtRALoomm8k5E9Pmwg3HOAal2VDc2F9+PM/rEY6oIDg==", - "os": ["android"], - "cpu": ["arm64"] - }, - "@esbuild/android-arm@0.25.5": { - "integrity": "sha512-AdJKSPeEHgi7/ZhuIPtcQKr5RQdo6OO2IL87JkianiMYMPbCtot9fxPbrMiBADOWWm3T2si9stAiVsGbTQFkbA==", - "os": ["android"], - "cpu": ["arm"] - }, - "@esbuild/android-x64@0.25.5": { - "integrity": "sha512-D2GyJT1kjvO//drbRT3Hib9XPwQeWd9vZoBJn+bu/lVsOZ13cqNdDeqIF/xQ5/VmWvMduP6AmXvylO/PIc2isw==", - "os": ["android"], - "cpu": ["x64"] - }, - "@esbuild/darwin-arm64@0.25.5": { - "integrity": "sha512-GtaBgammVvdF7aPIgH2jxMDdivezgFu6iKpmT+48+F8Hhg5J/sfnDieg0aeG/jfSvkYQU2/pceFPDKlqZzwnfQ==", - "os": ["darwin"], - "cpu": ["arm64"] - }, - "@esbuild/darwin-x64@0.25.5": { - "integrity": "sha512-1iT4FVL0dJ76/q1wd7XDsXrSW+oLoquptvh4CLR4kITDtqi2e/xwXwdCVH8hVHU43wgJdsq7Gxuzcs6Iq/7bxQ==", - "os": ["darwin"], - "cpu": ["x64"] - }, - "@esbuild/freebsd-arm64@0.25.5": { - "integrity": "sha512-nk4tGP3JThz4La38Uy/gzyXtpkPW8zSAmoUhK9xKKXdBCzKODMc2adkB2+8om9BDYugz+uGV7sLmpTYzvmz6Sw==", - "os": ["freebsd"], - "cpu": ["arm64"] - }, - "@esbuild/freebsd-x64@0.25.5": { - "integrity": "sha512-PrikaNjiXdR2laW6OIjlbeuCPrPaAl0IwPIaRv+SMV8CiM8i2LqVUHFC1+8eORgWyY7yhQY+2U2fA55mBzReaw==", - "os": ["freebsd"], - "cpu": ["x64"] - }, - "@esbuild/linux-arm64@0.25.5": { - "integrity": "sha512-Z9kfb1v6ZlGbWj8EJk9T6czVEjjq2ntSYLY2cw6pAZl4oKtfgQuS4HOq41M/BcoLPzrUbNd+R4BXFyH//nHxVg==", - "os": ["linux"], - "cpu": ["arm64"] - }, - "@esbuild/linux-arm@0.25.5": { - "integrity": "sha512-cPzojwW2okgh7ZlRpcBEtsX7WBuqbLrNXqLU89GxWbNt6uIg78ET82qifUy3W6OVww6ZWobWub5oqZOVtwolfw==", - "os": ["linux"], - "cpu": ["arm"] - }, - "@esbuild/linux-ia32@0.25.5": { - "integrity": "sha512-sQ7l00M8bSv36GLV95BVAdhJ2QsIbCuCjh/uYrWiMQSUuV+LpXwIqhgJDcvMTj+VsQmqAHL2yYaasENvJ7CDKA==", - "os": ["linux"], - "cpu": ["ia32"] - }, - "@esbuild/linux-loong64@0.25.5": { - "integrity": "sha512-0ur7ae16hDUC4OL5iEnDb0tZHDxYmuQyhKhsPBV8f99f6Z9KQM02g33f93rNH5A30agMS46u2HP6qTdEt6Q1kg==", - "os": ["linux"], - "cpu": ["loong64"] - }, - "@esbuild/linux-mips64el@0.25.5": { - "integrity": "sha512-kB/66P1OsHO5zLz0i6X0RxlQ+3cu0mkxS3TKFvkb5lin6uwZ/ttOkP3Z8lfR9mJOBk14ZwZ9182SIIWFGNmqmg==", - "os": ["linux"], - "cpu": ["mips64el"] - }, - "@esbuild/linux-ppc64@0.25.5": { - "integrity": "sha512-UZCmJ7r9X2fe2D6jBmkLBMQetXPXIsZjQJCjgwpVDz+YMcS6oFR27alkgGv3Oqkv07bxdvw7fyB71/olceJhkQ==", - "os": ["linux"], - "cpu": ["ppc64"] - }, - "@esbuild/linux-riscv64@0.25.5": { - "integrity": "sha512-kTxwu4mLyeOlsVIFPfQo+fQJAV9mh24xL+y+Bm6ej067sYANjyEw1dNHmvoqxJUCMnkBdKpvOn0Ahql6+4VyeA==", - "os": ["linux"], - "cpu": ["riscv64"] - }, - "@esbuild/linux-s390x@0.25.5": { - "integrity": "sha512-K2dSKTKfmdh78uJ3NcWFiqyRrimfdinS5ErLSn3vluHNeHVnBAFWC8a4X5N+7FgVE1EjXS1QDZbpqZBjfrqMTQ==", - "os": ["linux"], - "cpu": ["s390x"] - }, - "@esbuild/linux-x64@0.25.5": { - "integrity": "sha512-uhj8N2obKTE6pSZ+aMUbqq+1nXxNjZIIjCjGLfsWvVpy7gKCOL6rsY1MhRh9zLtUtAI7vpgLMK6DxjO8Qm9lJw==", - "os": ["linux"], - "cpu": ["x64"] - }, - "@esbuild/netbsd-arm64@0.25.5": { - "integrity": "sha512-pwHtMP9viAy1oHPvgxtOv+OkduK5ugofNTVDilIzBLpoWAM16r7b/mxBvfpuQDpRQFMfuVr5aLcn4yveGvBZvw==", - "os": ["netbsd"], - "cpu": ["arm64"] - }, - "@esbuild/netbsd-x64@0.25.5": { - "integrity": "sha512-WOb5fKrvVTRMfWFNCroYWWklbnXH0Q5rZppjq0vQIdlsQKuw6mdSihwSo4RV/YdQ5UCKKvBy7/0ZZYLBZKIbwQ==", - "os": ["netbsd"], - "cpu": ["x64"] - }, - "@esbuild/openbsd-arm64@0.25.5": { - "integrity": "sha512-7A208+uQKgTxHd0G0uqZO8UjK2R0DDb4fDmERtARjSHWxqMTye4Erz4zZafx7Di9Cv+lNHYuncAkiGFySoD+Mw==", - "os": ["openbsd"], - "cpu": ["arm64"] - }, - "@esbuild/openbsd-x64@0.25.5": { - "integrity": "sha512-G4hE405ErTWraiZ8UiSoesH8DaCsMm0Cay4fsFWOOUcz8b8rC6uCvnagr+gnioEjWn0wC+o1/TAHt+It+MpIMg==", - "os": ["openbsd"], - "cpu": ["x64"] - }, - "@esbuild/sunos-x64@0.25.5": { - "integrity": "sha512-l+azKShMy7FxzY0Rj4RCt5VD/q8mG/e+mDivgspo+yL8zW7qEwctQ6YqKX34DTEleFAvCIUviCFX1SDZRSyMQA==", - "os": ["sunos"], - "cpu": ["x64"] - }, - "@esbuild/win32-arm64@0.25.5": { - "integrity": "sha512-O2S7SNZzdcFG7eFKgvwUEZ2VG9D/sn/eIiz8XRZ1Q/DO5a3s76Xv0mdBzVM5j5R639lXQmPmSo0iRpHqUUrsxw==", - "os": ["win32"], - "cpu": ["arm64"] - }, - "@esbuild/win32-ia32@0.25.5": { - "integrity": "sha512-onOJ02pqs9h1iMJ1PQphR+VZv8qBMQ77Klcsqv9CNW2w6yLqoURLcgERAIurY6QE63bbLuqgP9ATqajFLK5AMQ==", - "os": ["win32"], - "cpu": ["ia32"] - }, - "@esbuild/win32-x64@0.25.5": { - "integrity": "sha512-TXv6YnJ8ZMVdX+SXWVBo/0p8LTcrUYngpWjvm91TMjjBQii7Oz11Lw5lbDV5Y0TzuhSJHwiH4hEtC1I42mMS0g==", - "os": ["win32"], - "cpu": ["x64"] - }, "@fast-csv/format@5.0.5": { "integrity": "sha512-0P9SJXXnqKdmuWlLaTelqbrfdgN37Mvrb369J6eNmqL41IEIZQmV4sNM4GgAK2Dz3aH04J0HKGDMJFkYObThTw==", "dependencies": [ @@ -433,7 +351,6 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, "@query-doctor/core@0.0.5": { - "integrity": "sha512-+PJs/0oAqrvorWDDXoSO8a7QU0JYSj9KrsmvNRpSuclPDgBEgqLNL2V8a6ozaW15dOguRtwt+UlMdGYBVcNgFw==", "dependencies": [ "@pgsql/types", "colorette", @@ -503,6 +420,9 @@ "@types/node@18.19.130" ] }, + "@yarnpkg/lockfile@1.1.0": { + "integrity": "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==" + }, "a-sync-waterfall@1.0.1": { "integrity": "sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA==" }, @@ -642,6 +562,12 @@ "balanced-match" ] }, + "braces@3.0.3": { + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dependencies": [ + "fill-range" + ] + }, "buffer-crc32@1.0.0": { "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==" }, @@ -665,6 +591,36 @@ "byline@5.0.0": { "integrity": "sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==" }, + "call-bind-apply-helpers@1.0.2": { + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dependencies": [ + "es-errors", + "function-bind" + ] + }, + "call-bind@1.0.8": { + "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", + "dependencies": [ + "call-bind-apply-helpers", + "es-define-property", + "get-intrinsic", + "set-function-length" + ] + }, + "call-bound@1.0.4": { + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "dependencies": [ + "call-bind-apply-helpers", + "get-intrinsic" + ] + }, + "chalk@4.1.2": { + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dependencies": [ + "ansi-styles@4.3.0", + "supports-color" + ] + }, "chokidar@4.0.3": { "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", "dependencies": [ @@ -674,6 +630,9 @@ "chownr@1.1.4": { "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, + "ci-info@3.9.0": { + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==" + }, "cliui@8.0.1": { "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", "dependencies": [ @@ -752,6 +711,14 @@ "dedent@1.7.0": { "integrity": "sha512-HGFtf8yhuhGhqO07SV79tRp+br4MnbdjeVxotpn1QBl30pcLLCQjX5b2295ll0fv8RKDKsmWYrl05usHM9CewQ==" }, + "define-data-property@1.1.4": { + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "dependencies": [ + "es-define-property", + "es-errors", + "gopd" + ] + }, "deprecation@2.3.1": { "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" }, @@ -785,6 +752,14 @@ "uuid" ] }, + "dunder-proto@1.0.1": { + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dependencies": [ + "call-bind-apply-helpers", + "es-errors", + "gopd" + ] + }, "eastasianwidth@0.2.0": { "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" }, @@ -800,6 +775,18 @@ "once" ] }, + "es-define-property@1.0.1": { + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==" + }, + "es-errors@1.3.0": { + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==" + }, + "es-object-atoms@1.1.1": { + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dependencies": [ + "es-errors" + ] + }, "escalade@3.2.0": { "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==" }, @@ -825,6 +812,18 @@ "fast-fifo@1.3.2": { "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" }, + "fill-range@7.1.1": { + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dependencies": [ + "to-regex-range" + ] + }, + "find-yarn-workspace-root@2.0.0": { + "integrity": "sha512-1IMnbjt4KzsQfnhnzNd8wUEgXZ44IzZaZmnLYx7D5FZlaHt2gW20Cri8Q+E/t5tIj4+epTBub+2Zxu/vNILzqQ==", + "dependencies": [ + "micromatch" + ] + }, "foreground-child@3.3.1": { "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", "dependencies": [ @@ -835,17 +834,45 @@ "fs-constants@1.0.0": { "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, - "fsevents@2.3.3": { - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "os": ["darwin"], - "scripts": true + "fs-extra@10.1.0": { + "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "dependencies": [ + "graceful-fs", + "jsonfile", + "universalify" + ] + }, + "function-bind@1.1.2": { + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" }, "get-caller-file@2.0.5": { "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, + "get-intrinsic@1.3.0": { + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dependencies": [ + "call-bind-apply-helpers", + "es-define-property", + "es-errors", + "es-object-atoms", + "function-bind", + "get-proto", + "gopd", + "has-symbols", + "hasown", + "math-intrinsics" + ] + }, "get-port@7.1.0": { "integrity": "sha512-QB9NKEeDg3xxVwCCwJQ9+xycaz6pBB6iQ76wiWMl1927n0Kir6alPiP+yuiICLLU4jpMe08dXfpebuQppFA2zw==" }, + "get-proto@1.0.1": { + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dependencies": [ + "dunder-proto", + "es-object-atoms" + ] + }, "glob@10.5.0": { "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", "dependencies": [ @@ -858,24 +885,61 @@ ], "bin": true }, + "gopd@1.2.0": { + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==" + }, "graceful-fs@4.2.11": { "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" }, + "has-flag@4.0.0": { + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + }, + "has-property-descriptors@1.0.2": { + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "dependencies": [ + "es-define-property" + ] + }, + "has-symbols@1.1.0": { + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==" + }, + "hasown@2.0.2": { + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dependencies": [ + "function-bind" + ] + }, "ieee754@1.2.1": { "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" }, "inherits@2.0.4": { "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "is-docker@2.2.1": { + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "bin": true + }, "is-fullwidth-code-point@3.0.0": { "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, + "is-number@7.0.0": { + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==" + }, "is-stream@2.0.1": { "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" }, + "is-wsl@2.2.0": { + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "dependencies": [ + "is-docker" + ] + }, "isarray@1.0.0": { "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" }, + "isarray@2.0.5": { + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" + }, "isexe@2.0.0": { "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" }, @@ -888,6 +952,16 @@ "@pkgjs/parseargs" ] }, + "json-stable-stringify@1.3.0": { + "integrity": "sha512-qtYiSSFlwot9XHtF9bD9c7rwKjr+RecWT//ZnPvSmEjpV5mmPOCN4j8UjY5hbjNkOwZ/jQv3J6R1/pL7RwgMsg==", + "dependencies": [ + "call-bind", + "call-bound", + "isarray@2.0.5", + "jsonify", + "object-keys" + ] + }, "jsondiffpatch@0.7.3": { "integrity": "sha512-zd4dqFiXSYyant2WgSXAZ9+yYqilNVvragVNkNRn2IFZKgjyULNrKRznqN4Zon0MkLueCg+3QaPVCnDAVP20OQ==", "dependencies": [ @@ -895,6 +969,24 @@ ], "bin": true }, + "jsonfile@6.2.0": { + "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", + "dependencies": [ + "universalify" + ], + "optionalDependencies": [ + "graceful-fs" + ] + }, + "jsonify@0.0.1": { + "integrity": "sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg==" + }, + "klaw-sync@6.0.0": { + "integrity": "sha512-nIeuVSzdCCs6TDPTqI8w1Yre34sSq7AkZ4B3sfOBbI2CgVSB4Du4aLQijFU2+lhAFCwt9+42Hel6lQNIv6AntQ==", + "dependencies": [ + "graceful-fs" + ] + }, "lazystream@1.0.1": { "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", "dependencies": [ @@ -934,6 +1026,16 @@ "lru-cache@10.4.3": { "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" }, + "math-intrinsics@1.1.0": { + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==" + }, + "micromatch@4.0.8": { + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dependencies": [ + "braces", + "picomatch" + ] + }, "minimatch@5.1.6": { "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", "dependencies": [ @@ -946,6 +1048,9 @@ "brace-expansion" ] }, + "minimist@1.2.8": { + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==" + }, "minipass@7.1.2": { "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==" }, @@ -978,15 +1083,6 @@ "normalize-path@3.0.0": { "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" }, - "nunjucks@3.2.4": { - "integrity": "sha512-26XRV6BhkgK0VOxfbU5cQI+ICFUtMLixv1noZn1tGU38kQH5A5nmmbk/O45xdyBhD1esk47nKrY0mvQpZIhRjQ==", - "dependencies": [ - "a-sync-waterfall", - "asap", - "commander@5.1.0" - ], - "bin": true - }, "nunjucks@3.2.4_chokidar@4.0.3": { "integrity": "sha512-26XRV6BhkgK0VOxfbU5cQI+ICFUtMLixv1noZn1tGU38kQH5A5nmmbk/O45xdyBhD1esk47nKrY0mvQpZIhRjQ==", "dependencies": [ @@ -1000,15 +1096,45 @@ ], "bin": true }, + "object-keys@1.1.1": { + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" + }, "once@1.4.0": { "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "dependencies": [ "wrappy" ] }, + "open@7.4.2": { + "integrity": "sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==", + "dependencies": [ + "is-docker", + "is-wsl" + ] + }, "package-json-from-dist@1.0.1": { "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==" }, + "patch-package@8.0.1": { + "integrity": "sha512-VsKRIA8f5uqHQ7NGhwIna6Bx6D9s/1iXlA1hthBVBEbkq+t4kXD0HHt+rJhf/Z+Ci0F/HCB2hvn0qLdLG+Qxlw==", + "dependencies": [ + "@yarnpkg/lockfile", + "chalk", + "ci-info", + "cross-spawn", + "find-yarn-workspace-root", + "fs-extra", + "json-stable-stringify", + "klaw-sync", + "minimist", + "open", + "semver", + "slash", + "tmp", + "yaml" + ], + "bin": true + }, "path-key@3.1.1": { "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" }, @@ -1025,6 +1151,9 @@ "@pgsql/types" ] }, + "picomatch@2.3.1": { + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" + }, "process-nextick-args@2.0.1": { "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, @@ -1085,7 +1214,7 @@ "dependencies": [ "core-util-is", "inherits", - "isarray", + "isarray@1.0.0", "process-nextick-args", "safe-buffer@5.1.2", "string_decoder@1.1.1", @@ -1137,6 +1266,21 @@ "safer-buffer@2.1.2": { "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "semver@7.7.3": { + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "bin": true + }, + "set-function-length@1.2.2": { + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "dependencies": [ + "define-data-property", + "es-errors", + "function-bind", + "get-intrinsic", + "gopd", + "has-property-descriptors" + ] + }, "shebang-command@2.0.0": { "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dependencies": [ @@ -1152,6 +1296,9 @@ "signal-exit@4.1.0": { "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==" }, + "slash@2.0.0": { + "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==" + }, "split-ca@1.0.1": { "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==" }, @@ -1233,6 +1380,12 @@ "ansi-regex@6.2.2" ] }, + "supports-color@7.2.0": { + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dependencies": [ + "has-flag" + ] + }, "tar-fs@2.1.4": { "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", "dependencies": [ @@ -1300,6 +1453,12 @@ "tmp@0.2.5": { "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==" }, + "to-regex-range@5.0.1": { + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dependencies": [ + "is-number" + ] + }, "tunnel@0.0.6": { "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" }, @@ -1327,6 +1486,9 @@ "universal-user-agent@6.0.1": { "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==" }, + "universalify@2.0.1": { + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==" + }, "util-deprecate@1.0.2": { "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, @@ -1765,6 +1927,7 @@ "jsr:@std/collections@^1.1.3", "jsr:@std/data-structures@^1.0.9", "jsr:@std/fmt@^1.0.8", + "jsr:@std/testing@^1.0.16", "npm:@actions/core@^1.11.1", "npm:@actions/github@^6.0.1", "npm:@libpg-query/parser@^17.6.3", @@ -1779,10 +1942,288 @@ "npm:fast-csv@^5.0.5", "npm:jsondiffpatch@~0.7.3", "npm:nunjucks@^3.2.4", + "npm:patch-package@^8.0.1", "npm:pgsql-deparser@^17.11.1", "npm:sql-formatter@^15.6.6", "npm:sql-highlight@^6.1.0", "npm:zod@^4.1.12" - ] + ], + "links": { + "npm:@query-doctor/common@0.0.0": { + "peerDependencies": [ + "npm:react-dom@^18.2.0", + "npm:react@^18.2.0" + ] + }, + "npm:@query-doctor/core@0.0.5": { + "dependencies": [ + "npm:@pgsql/types@^17.6.1", + "npm:colorette@^2.0.20", + "npm:dedent@^1.7.0", + "npm:pgsql-deparser@^17.11.1", + "npm:zod@^4.1.13" + ] + }, + "npm:@query-doctor/db-engine@0.0.0": { + "dependencies": [ + "npm:@query-doctor/common@*", + "npm:@query-doctor/pglite@1.0.7", + "npm:lodash@^4.17.21", + "npm:pgsql-ast-parser@^12.0.1" + ] + }, + "npm:@query-doctor/eslint-config-custom-server@0.0.0": { + "dependencies": [ + "npm:eslint-config-turbo@^1.12.4", + "npm:eslint-plugin-import@^2.31.0" + ], + "peerDependencies": [ + "npm:eslint@^8.43.0" + ] + }, + "npm:@query-doctor/eslint-config-custom@0.0.0": { + "dependencies": [ + "npm:eslint-config-prettier@^9.1.0", + "npm:eslint-config-turbo@^1.12.4", + "npm:eslint-plugin-prettier@^5.2.3", + "npm:eslint-plugin-react@7.37.5" + ], + "peerDependencies": [ + "npm:eslint@^8.43.0", + "npm:prettier@^2.8.8", + "npm:react-dom@^18.2.0", + "npm:react@^18.2.0", + "npm:typescript@^5.6.3" + ] + }, + "npm:@query-doctor/hooks@0.0.0": { + "dependencies": [ + "npm:@pgsql/parser@^17.5.0", + "npm:@query-doctor/common@*", + "npm:@query-doctor/db-engine@*", + "npm:@query-doctor/pglite@1.0.7", + "npm:cookie@^1.0.2", + "npm:dayjs@^1.11.19", + "npm:react-fast-compare@^3.2.2", + "npm:zod@^4.1.13" + ], + "peerDependencies": [ + "npm:react-dom@^18.2.0", + "npm:react@^18.2.0" + ] + }, + "npm:@query-doctor/sql-editor@0.0.0": { + "dependencies": [ + "npm:@codemirror/commands@^6.8.1", + "npm:@codemirror/lang-json@^6.0.2", + "npm:@codemirror/lang-sql@^6.10.0", + "npm:@codemirror/view@^6.38.8", + "npm:@query-doctor/db-engine@*", + "npm:@query-doctor/pglite@1.0.7", + "npm:@stitches/react@^1.2.8", + "npm:@uiw/codemirror-themes@^4.25.3", + "npm:@uiw/react-codemirror@^4.25.3", + "npm:clsx@^2.1.1", + "npm:json5@^2.2.3" + ], + "peerDependencies": [ + "npm:react-dom@^18.2.0", + "npm:react@^18.2.0" + ] + }, + "npm:@query-doctor/tailwind-config@0.0.0": {}, + "npm:@query-doctor/types@0.0.0": { + "dependencies": [ + "npm:zod@^4.1.13" + ], + "peerDependencies": [ + "npm:react-dom@^18.2.0", + "npm:react@^18.2.0" + ] + }, + "npm:@query-doctor/typescript-config@0.0.0": {}, + "npm:@query-doctor/ui@0.0.0": { + "dependencies": [ + "npm:@dagrejs/dagre@^1.1.5", + "npm:@query-doctor/common@*", + "npm:@query-doctor/hooks@*", + "npm:@query-doctor/pglite@1.0.7", + "npm:@stitches/react@^1.2.8", + "npm:@xyflow/react@^12.4.4", + "npm:clsx@^2.1.1", + "npm:lodash@^4.17.21", + "npm:polished@^4.3.1", + "npm:react-aria@3.28.0", + "npm:react-fast-compare@^3.2.2", + "npm:react-icons@^5.5.0", + "npm:react-stately@^3.26.0", + "npm:react-syntax-highlighter@^15.6.6" + ], + "peerDependencies": [ + "npm:react-dom@^18.2.0", + "npm:react@^18.2.0" + ] + }, + "npm:api@0.0.1": { + "dependencies": [ + "npm:@mrleebo/prisma-ast@~0.12.1", + "npm:@nestjs/cli@^10.4.9", + "npm:@nestjs/common@^10.4.20", + "npm:@nestjs/config@^3.3.0", + "npm:@nestjs/core@^10.4.20", + "npm:@nestjs/platform-express@^10.4.20", + "npm:@nestjs/schedule@^5.0.1", + "npm:@nestjs/throttler@^5.2.0", + "npm:@prisma/internals@6.16.3", + "npm:@sendgrid/mail@^8.1.6", + "npm:@sentry/cli@^2.55.0", + "npm:@sentry/nestjs@^10.14.0", + "npm:@total-typescript/ts-reset@~0.6.1", + "npm:antlr4@^4.13.2", + "npm:antlr4ng@^3.0.16", + "npm:bcrypt@6", + "npm:connect-mongodb-session@^3.1.1", + "npm:dedent@^1.7.0", + "npm:express-session@^1.18.2", + "npm:fast-xml-parser@^5.2.5", + "npm:lodash@^4.17.21", + "npm:mongodb@^5.8.1", + "npm:nanoid@^5.1.6", + "npm:raw-body@^3.0.2", + "npm:reflect-metadata@~0.2.2", + "npm:rxjs@^7.8.2", + "npm:stripe@^17.7.0", + "npm:tsconfig-paths@^4.2.0", + "npm:vite-tsconfig-paths@^5.1.4", + "npm:zod@^4.1.13" + ] + }, + "npm:app@0.0.0": { + "dependencies": [ + "npm:@query-doctor/common@*", + "npm:@query-doctor/core@*", + "npm:@query-doctor/hooks@*", + "npm:@query-doctor/pglite@1.0.7", + "npm:@query-doctor/sql-editor@*", + "npm:@query-doctor/ui@*", + "npm:@radix-ui/react-dialog@^1.1.15", + "npm:@radix-ui/react-menubar@^1.1.16", + "npm:@radix-ui/react-portal@^1.1.9", + "npm:@reduxjs/toolkit@^1.9.5", + "npm:@sentry/react@^10.14.0", + "npm:@tanstack/react-form@^1.23.8", + "npm:@tanstack/react-query@^5.90.11", + "npm:@tanstack/react-router@^1.132.37", + "npm:@tanstack/react-table@^8.21.3", + "npm:@xyflow/react@^12.4.4", + "npm:clarinet@~0.12.6", + "npm:clsx@^2.1.1", + "npm:date-fns@^3.6.0", + "npm:framer-motion@^12.23.25", + "npm:js-cookie@^3.0.5", + "npm:lodash@^4.17.21", + "npm:pgsql-ast-parser@^12.0.1", + "npm:query-string@^9.3.1", + "npm:react-aria@^3.28.0", + "npm:react-dom@^18.3.1", + "npm:react-icons@^5.5.0", + "npm:react-loading-skeleton@^3.5.0", + "npm:react-redux@^8.1.2", + "npm:react-verification-input@^4.2.2", + "npm:react@^18.3.1", + "npm:tailwind-merge@^2.6.0", + "npm:tailwindcss-animate@^1.0.7", + "npm:tailwindcss-radix-colors@^1.4.1", + "npm:uuid@13", + "npm:zod@^4.1.13" + ] + }, + "npm:blog@0.0.0": { + "dependencies": [ + "npm:@astrojs/mdx@^4.3.12", + "npm:@astrojs/react@^4.4.2", + "npm:@astrojs/rss@^4.0.14", + "npm:@astrojs/sitemap@^3.6.0", + "npm:@astrojs/tailwind@^6.0.2", + "npm:@query-doctor/common@*", + "npm:@query-doctor/db-engine@*", + "npm:@query-doctor/hooks@*", + "npm:@query-doctor/pglite@1.0.7", + "npm:@query-doctor/sql-editor@*", + "npm:@query-doctor/types@*", + "npm:@query-doctor/ui@*", + "npm:@tanstack/react-query@^5.90.11", + "npm:astro@^5.16.3", + "npm:framer-motion@^12.23.25", + "npm:fuse.js@^6.6.2", + "npm:react-dom@^18.3.1", + "npm:react-icons@^5.5.0", + "npm:react-rough-notation@^1.0.8", + "npm:react@^18.3.1", + "npm:sharp@~0.34.5", + "npm:zod@^4.1.13" + ] + }, + "npm:course@0.0.1": { + "dependencies": [ + "npm:@astrojs/check@~0.9.6", + "npm:@astrojs/mdx@^4.3.12", + "npm:@astrojs/node@^9.5.1", + "npm:@astrojs/prefetch@~0.4.1", + "npm:@astrojs/react@^4.4.2", + "npm:@astrojs/tailwind@^6.0.2", + "npm:@nanostores/react@~0.8.4", + "npm:@query-doctor/common@*", + "npm:@query-doctor/hooks@*", + "npm:@query-doctor/types@*", + "npm:@query-doctor/ui@*", + "npm:@radix-ui/react-tooltip@^1.0.7", + "npm:@tailwindcss/typography@~0.5.19", + "npm:astro@^5.16.3", + "npm:clsx@^2.1.1", + "npm:cookie@^1.0.2", + "npm:framer-motion@^12.23.25", + "npm:github-slugger@2", + "npm:nanostores@~0.11.4", + "npm:quick-score@0.2", + "npm:react-dom@^18.3.1", + "npm:react-fast-compare@^3.2.2", + "npm:react-icons@^5.5.0", + "npm:react-rough-notation@^1.0.8", + "npm:react-sortablejs@^6.1.4", + "npm:react@^18.3.1", + "npm:remark-mdx@^3.1.1", + "npm:remark@^15.0.1", + "npm:shiki@^1.4.0", + "npm:sortablejs@^1.15.6", + "npm:tailwind-scrollbar@^3.1.0", + "npm:tailwindcss-radix-colors@^1.4.1", + "npm:tailwindcss@^3.4.17", + "npm:typescript@^5.6.3" + ] + }, + "npm:docs@0.0.1": { + "dependencies": [ + "npm:@astrojs/starlight@0.37", + "npm:astro@^5.16.3", + "npm:sharp@~0.34.5" + ] + }, + "npm:query-doctor-site@0.0.1": { + "dependencies": [ + "npm:patch-package@^8.0.1", + "npm:turbo@^1.13.3", + "npm:typescript@^5.6.3" + ] + }, + "npm:ui-playground@0.0.0": { + "dependencies": [ + "npm:@ladle/react@^5.0.3", + "npm:@query-doctor/ui@*", + "npm:react-dom@^18.3.1", + "npm:react@^18.3.1" + ] + } + } } } diff --git a/devenv.lock b/devenv.lock index 33dca3d..2d377d4 100644 --- a/devenv.lock +++ b/devenv.lock @@ -3,10 +3,10 @@ "devenv": { "locked": { "dir": "src/modules", - "lastModified": 1764368166, + "lastModified": 1765279746, "owner": "cachix", "repo": "devenv", - "rev": "47a243b97499bfe5d5783d1fc86d9fe776b2497f", + "rev": "8241b5bedae223d439edc17eb8b2a6e31b40386d", "type": "github" }, "original": { @@ -19,10 +19,10 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1761588595, + "lastModified": 1765121682, "owner": "edolstra", "repo": "flake-compat", - "rev": "f387cd2afec9419c8ee37694406ca490c3f34ee5", + "rev": "65f23138d8d09a92e30f1e5c87611b23ef451bf3", "type": "github" }, "original": { @@ -40,10 +40,10 @@ ] }, "locked": { - "lastModified": 1763988335, + "lastModified": 1765016596, "owner": "cachix", "repo": "git-hooks.nix", - "rev": "50b9238891e388c9fdc6a5c49e49c42533a1b5ce", + "rev": "548fc44fca28a5e81c5d6b846e555e6b9c2a5a3c", "type": "github" }, "original": { @@ -74,10 +74,10 @@ }, "nixpkgs": { "locked": { - "lastModified": 1761313199, + "lastModified": 1764580874, "owner": "cachix", "repo": "devenv-nixpkgs", - "rev": "d1c30452ebecfc55185ae6d1c983c09da0c274ff", + "rev": "dcf61356c3ab25f1362b4a4428a6d871e84f1d1d", "type": "github" }, "original": { diff --git a/src/remote/query-optimizer.test.ts b/src/remote/query-optimizer.test.ts index 3540697..03d6cdf 100644 --- a/src/remote/query-optimizer.test.ts +++ b/src/remote/query-optimizer.test.ts @@ -3,9 +3,8 @@ import { QueryOptimizer } from "./query-optimizer.ts"; import { ConnectionManager } from "../sync/connection-manager.ts"; import { Connectable } from "../sync/connectable.ts"; import { setTimeout } from "node:timers/promises"; -import { assertStrictEquals } from "@std/assert"; -import { assertEquals } from "@std/assert/equals"; import { assertArrayIncludes } from "@std/assert/array-includes"; +import { assert } from "@std/assert"; Deno.test({ name: "controller syncs correctly", @@ -17,13 +16,18 @@ Deno.test({ { content: ` create table testing(a int, b text); - insert into testing values (1); - create index on testing(b); + insert into testing (a, b) values (1, 'hello'); + create index "testing_index" on testing(b); + + -- normally should be in another db but + -- this makes testing much faster create extension pg_stat_statements; select * from testing where a = 10; select * from testing where b = 'c'; select * from testing where b > 'a'; select * from testing where b < 'b'; + select * from pg_index where 1 = 1; + select * from pg_class where relname > 'example' /* @qd_introspection */; `, target: "/docker-entrypoint-initdb.d/init.sql", }, @@ -32,7 +36,13 @@ Deno.test({ "-c", "shared_preload_libraries=pg_stat_statements", "-c", - "log_statement=all", + "autovacuum=off", + "-c", + "track_counts=off", + "-c", + "track_io_timing=off", + "-c", + "track_activities=off", ]) .start(); @@ -46,9 +56,13 @@ Deno.test({ "select * from testing where b < $1", ]; - const improvements: string[] = []; - const noImprovements: string[] = []; + let improvements: string[] = []; + let noImprovements: string[] = []; + optimizer.addListener("error", (query, error) => { + console.error("error when running query", query); + throw error; + }); optimizer.addListener("improvementsAvailable", (query) => { improvements.push(query.query); }); @@ -60,10 +74,75 @@ Deno.test({ const connector = manager.getConnectorFor(conn); try { const recentQueries = await connector.getRecentQueries(); - await optimizer.start(conn, recentQueries); + const includedQueries = await optimizer.start(conn, recentQueries, { + kind: "fromStatisticsExport", + source: { kind: "inline" }, + stats: [{ + tableName: "testing", + schemaName: "public", + relpages: 56, + reltuples: 100_000, + relallvisible: 1, + columns: [{ + columnName: "a", + stats: null, + }, { + columnName: "b", + stats: null, + }], + indexes: [{ + indexName: "testing_index", + relpages: 2, + reltuples: 10000, + relallvisible: 1, + }], + }], + }); + // should ignore the query with + assert( + includedQueries.every((q) => + !q.query.startsWith("select * from pg_class where relname > $1") + ), + "Optimizer did not ignore a query with @qd_introspection", + ); + assert( + includedQueries.every((q) => + !q.query.startsWith("select * from pg_index where $1 = $2") + ), + "Optimizer did not ignore a system query", + ); await setTimeout(1_000); assertArrayIncludes(expectedImprovements, improvements); assertArrayIncludes(expectedNoImprovements, noImprovements); + improvements = []; + noImprovements = []; + await optimizer.start(conn, recentQueries, { + kind: "fromStatisticsExport", + source: { kind: "inline" }, + stats: [{ + tableName: "testing", + schemaName: "public", + relpages: 1, + reltuples: 100, + relallvisible: 1, + columns: [{ + columnName: "a", + stats: null, + }, { + columnName: "b", + stats: null, + }], + indexes: [{ + indexName: "testing_index", + relpages: 2, + reltuples: 10000, + relallvisible: 1, + }], + }], + }); + await setTimeout(1_000); + console.log(improvements); + console.log(noImprovements); } finally { await pg.stop(); } diff --git a/src/remote/query-optimizer.ts b/src/remote/query-optimizer.ts index 2f18c62..86165ea 100644 --- a/src/remote/query-optimizer.ts +++ b/src/remote/query-optimizer.ts @@ -13,21 +13,24 @@ import { } from "@query-doctor/core"; import { Connectable } from "../sync/connectable.ts"; import { parse } from "@libpg-query/parser"; +import z from "zod"; const MINIMUM_COST_CHANGE_PERCENTAGE = 5; const QUERY_TIMEOUT_MS = 10000; type EventMap = { - error: [RecentQuery, string]; - timeout: [RecentQuery]; + error: [RecentQuery, Error]; + timeout: [RecentQuery, number]; zeroCostPlan: [RecentQuery]; + queryUnsupported: [RecentQuery]; noImprovements: [RecentQuery]; improvementsAvailable: [RecentQuery]; }; -type OptimizedQuery = { - recentQuery: RecentQuery; - optimization: LiveQueryOptimization; +type Target = { + connectable: Connectable; + optimizer: IndexOptimizer; + statistics: Statistics; }; export class QueryOptimizer extends EventEmitter { @@ -38,13 +41,13 @@ export class QueryOptimizer extends EventEmitter { reltuples: 10_000, }; private readonly queries = new Map(); - private target?: { - optimizer: IndexOptimizer; - statistics: Statistics; - }; + private target?: Target; private semaphore = new Sema(QueryOptimizer.MAX_CONCURRENCY); + private _finish = Promise.withResolvers(); - private readonly analyzer = new Analyzer(parse); + private _validQueriesProcessed = 0; + private _invalidQueries = 0; + private _allQueries = 0; constructor( private readonly manager: ConnectionManager, @@ -52,11 +55,31 @@ export class QueryOptimizer extends EventEmitter { super(); } + get validQueriesProcessed() { + return this._validQueriesProcessed; + } + + get invalidQueries() { + return this._invalidQueries; + } + + get allQueries() { + return this._allQueries; + } + + get finish() { + return this._finish.promise; + } + + /** + * Start optimizing a new set of queries. + * @returns the array of queries that will be considered + */ async start( conn: Connectable, - recentQueries?: RecentQuery[], + allRecentQueries: RecentQuery[], statsMode: StatisticsMode = QueryOptimizer.defaultStatistics, - ) { + ): Promise { this.stop(); const version = PostgresVersion.parse("17"); const pg = this.manager.getOrCreateConnection(conn); @@ -69,119 +92,195 @@ export class QueryOptimizer extends EventEmitter { ); const existingIndexes = await statistics.getExistingIndexes(); const optimizer = new IndexOptimizer(pg, statistics, existingIndexes, { - // we're not running on the que + // we're not running on our pg fork (yet) // so traces have to be disabled trace: false, }); - this.target = { - optimizer, - statistics, - }; - if (recentQueries) { - for (const query of recentQueries) { - this.queries.set(query.hash, { - recentQuery: query, - optimization: { state: "waiting" }, - }); + this.target = { connectable: conn, optimizer, statistics }; + + const validQueries: RecentQuery[] = []; + for (const query of allRecentQueries) { + let optimization: LiveQueryOptimization; + const status = this.checkQueryUnsupported(query); + switch (status.type) { + case "ok": + optimization = { state: "waiting" }; + break; + case "not_supported": + optimization = this.onQueryUnsupported(); + break; + case "ignored": + continue; } + validQueries.push(query); + this.queries.set(query.hash, { query, optimization }); } - for (let i = 0; i < QueryOptimizer.MAX_CONCURRENCY; i++) { - this.work(); - } + this._allQueries = this.queries.size; + while (await this.work()); + return validQueries; } stop() { this.semaphore = new Sema(QueryOptimizer.MAX_CONCURRENCY); this.queries.clear(); this.target = undefined; + this._allQueries = 0; + this._finish = Promise.withResolvers(); + this._invalidQueries = 0; + this._validQueriesProcessed = 0; } private async work() { - // don't enter if there isn't enough space in the semaphore + if (!this.target) { + return; + } + let recentQuery: RecentQuery | undefined; const token = await this.semaphore.acquire(); try { - if (!this.target) { - return; - } - let recentQuery: RecentQuery | undefined; - for (const [_hash, query] of this.queries.entries()) { - if (query.optimization.state !== "waiting") { + for (const [hash, entry] of this.queries.entries()) { + if (entry.optimization.state !== "waiting") { continue; } - recentQuery = query.recentQuery; - } - if (recentQuery) { - if (!this.isQuerySupported(recentQuery)) { - this.onQueryUnsupported(recentQuery); - return; - } - this.queries.set(recentQuery.hash, { - recentQuery, + this.queries.set(hash, { + query: entry.query, optimization: { state: "optimizing" }, }); - await this.optimizeQuery(recentQuery); + recentQuery = entry.query; + break; } } finally { this.semaphore.release(token); - setTimeout(() => this.work(), 100); } - } + if (recentQuery) { + this._validQueriesProcessed++; + const optimization = await this.optimizeQuery( + recentQuery, + this.target, + ); - private isQuerySupported(q: RecentQuery) { - return !q.isSystemQuery && q.isSelectQuery; + this.queries.set(recentQuery.hash, { + query: recentQuery, + optimization, + }); + return true; + } else { + this._finish.resolve(0); + return false; + } } - private async optimizeQuery(recent: RecentQuery) { - if (!this.target) { - return; + // private summarizeQueue() { + // let waitingQueries = 0; + // let optimizingQueries = 0; + // let improvementsAvailableQueries = 0; + // let noImprovementFoundQueries = 0; + // let timeoutQueries = 0; + // let errorQueries = 0; + // let notSupportedQueries = 0; + + // for (const [_hash, query] of this.queries.entries()) { + // if (query.optimization.state === "waiting") { + // waitingQueries++; + // } else if (query.optimization.state === "optimizing") { + // optimizingQueries++; + // } else if (query.optimization.state === "improvements_available") { + // improvementsAvailableQueries++; + // } else if (query.optimization.state === "no_improvement_found") { + // noImprovementFoundQueries++; + // } else if (query.optimization.state === "timeout") { + // timeoutQueries++; + // } else if (query.optimization.state === "error") { + // errorQueries++; + // } else if (query.optimization.state === "not_supported") { + // notSupportedQueries++; + // } + // } + // console.log("============"); + // console.log(`waiting: ${waitingQueries}`); + // console.log(`optimizing: ${optimizingQueries}`); + // console.log(`timeout: ${timeoutQueries}`); + // console.log(`error: ${errorQueries}`); + // console.log( + // `improvements: ${improvementsAvailableQueries}`, + // ); + // console.log(`no improvements: ${noImprovementFoundQueries}`); + // console.log("============"); + // } + + private checkQueryUnsupported( + query: RecentQuery, + ): { type: "ok" } | { type: "ignored" } | { + type: "not_supported"; + reason: string; + } { + if ( + query.isIntrospection || query.isSystemQuery || + query.isTargetlessSelectQuery + ) { + return { type: "ignored" }; } + if (!query.isSelectQuery) { + return { + type: "not_supported", + reason: + "Only select statements are currently eligible for optimization", + }; + } + return { type: "ok" }; + } + + private async optimizeQuery( + recent: RecentQuery, + target: Target, + timeoutMs = QUERY_TIMEOUT_MS, + ): Promise { const builder = new PostgresQueryBuilder(recent.query); let cost: number; try { const explain = await withTimeout( - this.target.optimizer.runWithoutIndexes(builder), - QUERY_TIMEOUT_MS, + target.optimizer.runWithoutIndexes(builder), + timeoutMs, ); cost = explain.Plan["Total Cost"]; } catch (error) { if (error instanceof TimeoutError) { - this.onTimeout(recent); + return this.onTimeout(recent, timeoutMs); } else if (error instanceof Error) { - this.onError(recent, error.message); + return this.onError(recent, error.message); } else { - this.onError(recent, "Internal error"); + return this.onError(recent, "Internal error"); } - return; } if (cost === 0) { - this.onZeroCostPlan(recent); - return; + return this.onZeroCostPlan(recent); } const indexes = this.getPotentialIndexCandidates( - this.target.statistics, + target.statistics, recent, ); let result: OptimizeResult; try { result = await withTimeout( - this.target.optimizer.run(builder, indexes), + target.optimizer.run(builder, indexes), QUERY_TIMEOUT_MS, ); } catch (error) { if (error instanceof TimeoutError) { - this.onTimeout(recent); + return this.onTimeout(recent, QUERY_TIMEOUT_MS); } else if (error instanceof Error) { - this.onError(recent, error.message); + return this.onError(recent, error.message); } else { - this.onError(recent, "Internal error"); + return this.onError(recent, "Internal error"); } - return; } return this.onOptimizeReady(result, recent); } - private onOptimizeReady(result: OptimizeResult, recent: RecentQuery) { + private onOptimizeReady( + result: OptimizeResult, + recent: RecentQuery, + ): LiveQueryOptimization { switch (result.kind) { case "ok": { const indexRecommendations = mapIndexRecommandations(result); @@ -194,7 +293,7 @@ export class QueryOptimizer extends EventEmitter { Math.abs(percentageReduction), ); if (costReductionPercentage < MINIMUM_COST_CHANGE_PERCENTAGE) { - this.emit("noImprovements", recent); + this.onNoImprovements(recent); return { state: "no_improvement_found", cost: result.baseCost, @@ -218,32 +317,37 @@ export class QueryOptimizer extends EventEmitter { } } + private onNoImprovements(recent: RecentQuery) { + this.emit("noImprovements", recent); + } + private getPotentialIndexCandidates( statistics: Statistics, recent: RecentQuery, ) { - return this.analyzer.deriveIndexes( + const analyzer = new Analyzer(parse); + return analyzer.deriveIndexes( statistics.ownMetadata, recent.columnReferences, ); } - private onQueryUnsupported(recent: RecentQuery) { - this.queries.set(recent.hash, { - recentQuery: recent, - optimization: { - state: "not_supported", - reason: "Query is not supported", - }, - }); + private onQueryUnsupported(): LiveQueryOptimization { + // this.emit("queryUnsupported", recent); + this._invalidQueries++; + return { + state: "not_supported", + reason: "Query is not supported", + }; } private onImprovementsAvailable( recent: RecentQuery, result: Extract, ) { + this.emit("improvementsAvailable", recent); this.queries.set(recent.hash, { - recentQuery: recent, + query: recent, optimization: { state: "improvements_available", cost: result.baseCost, @@ -256,35 +360,33 @@ export class QueryOptimizer extends EventEmitter { // indexesUsed, }, }); - this.emit("improvementsAvailable", recent); } - private onZeroCostPlan(recent: RecentQuery) { - this.queries.set(recent.hash, { - recentQuery: recent, - optimization: { - state: "error", - error: - "Query plan had zero cost. This should not happen on a patched postgres instance", - }, - }); + private onZeroCostPlan(recent: RecentQuery): LiveQueryOptimization { this.emit("zeroCostPlan", recent); + return { + state: "error", + error: new Error( + "Query plan had zero cost. This should not happen on a patched postgres instance", + ), + }; } - private onError(recent: RecentQuery, errorMessage: string) { - this.queries.set(recent.hash, { - recentQuery: recent, - optimization: { state: "error", error: errorMessage }, - }); - this.emit("error", recent, errorMessage); + private onError( + recent: RecentQuery, + errorMessage: string, + ): LiveQueryOptimization { + const error = new Error(errorMessage); + this.emit("error", recent, error); + return { state: "error", error }; } - private onTimeout(recent: RecentQuery) { - this.queries.set(recent.hash, { - recentQuery: recent, - optimization: { state: "timeout" }, - }); - this.emit("timeout", recent); + private onTimeout( + recent: RecentQuery, + waitedMs: number, + ): LiveQueryOptimization { + this.emit("timeout", recent, waitedMs); + return { state: "timeout" }; } } @@ -330,26 +432,34 @@ export function costDifferencePercentage( return ((newVal - oldVal) / oldVal) * 100; } -export type LiveQueryOptimization = - | { state: "waiting" } - | { state: "optimizing" } - // system queries and certain other queries are exempt from optimization - | { state: "not_supported"; reason: string } - | { - state: "improvements_available"; - cost: number; - optimizedCost: number; - costReductionPercentage: number; - indexRecommendations: string[]; - // indexRecommendations: TraceFoundIndex[]; - indexesUsed: string[]; - } - | { - state: "no_improvement_found"; - cost: number; - indexesUsed: string[]; - } - // Cost is nullable in case the timeout was caused by the initial query - // before we even add any indexes to it (usually unlikely) - | { state: "timeout"; cost?: number } - | { state: "error"; error: string }; +export const LiveQueryOptimization = z.discriminatedUnion("state", [ + z.object({ + state: z.literal("waiting"), + }), + z.object({ state: z.literal("optimizing") }), + z.object({ state: z.literal("not_supported"), reason: z.string() }), + z.object({ + state: z.literal("improvements_available"), + cost: z.number(), + optimizedCost: z.number(), + costReductionPercentage: z.number(), + indexRecommendations: z.array(z.string()), + indexesUsed: z.array(z.string()), + }), + z.object({ + state: z.literal("no_improvement_found"), + cost: z.number(), + indexesUsed: z.array(z.string()), + }), + z.object({ state: z.literal("timeout") }), + z.object({ state: z.literal("error"), error: z.instanceof(Error) }), +]); + +export type LiveQueryOptimization = z.infer; + +export const OptimizedQuery = z.object({ + query: z.instanceof(RecentQuery), + optimization: LiveQueryOptimization, +}); + +export type OptimizedQuery = z.infer; diff --git a/src/remote/remote-controller.test.ts b/src/remote/remote-controller.test.ts index 6f4be41..e4a6563 100644 --- a/src/remote/remote-controller.test.ts +++ b/src/remote/remote-controller.test.ts @@ -6,6 +6,7 @@ import { assertEquals } from "@std/assert/equals"; import { RemoteController } from "./remote-controller.ts"; import { ConnectionManager } from "../sync/connection-manager.ts"; import { RemoteSyncRequest } from "./remote.dto.ts"; +import { assertSpyCalls, spy } from "@std/testing/mock"; Deno.test({ name: "controller syncs correctly", @@ -31,24 +32,43 @@ Deno.test({ .start(), new PostgreSqlContainer("postgres:17").start(), ]); + const controller = new AbortController(); - try { - const target = Connectable.fromString( - targetDb.getConnectionUri(), - ); - const source = Connectable.fromString( - sourceDb.getConnectionUri(), - ); + const target = Connectable.fromString( + targetDb.getConnectionUri(), + ); + const source = Connectable.fromString( + sourceDb.getConnectionUri(), + ); - const sourceOptimizer = ConnectionManager.forLocalDatabase(); + const sourceOptimizer = ConnectionManager.forLocalDatabase(); - const remote = new RemoteController( - new Remote(target, sourceOptimizer), - ); + const remote = new RemoteController( + new Remote(target, sourceOptimizer), + ); - const response = await remote.execute( + const server = Deno.serve( + { port: 0, signal: controller.signal }, + async (req: Request): Promise => { + const result = await remote.execute(req); + if (!result) { + throw new Error(); + } + return result; + }, + ); + try { + const ws = new WebSocket(`ws://localhost:${server.addr.port}/postgres`); + const messageFunction = spy(); + ws.addEventListener("open", (event) => { + console.log("OPENED", event); + }); + ws.addEventListener("error", console.log); + ws.addEventListener("message", messageFunction); + + const response = await fetch( new Request( - "https://anything.whatever/postgres", + `http://localhost:${server.addr.port}/postgres`, { method: "POST", body: RemoteSyncRequest.encode({ @@ -73,8 +93,11 @@ Deno.test({ const rows = await sql`select * from testing`; // expect no rows to have been synced assertEquals(rows.length, 0); + + // exactly one query must have been processed + assertSpyCalls(messageFunction, 1); } finally { - await Promise.all([sourceDb.stop(), targetDb.stop()]); + await Promise.all([sourceDb.stop(), targetDb.stop(), server.shutdown()]); } }, }); diff --git a/src/remote/remote-controller.ts b/src/remote/remote-controller.ts index f50eecf..eb1b34c 100644 --- a/src/remote/remote-controller.ts +++ b/src/remote/remote-controller.ts @@ -1,26 +1,57 @@ import { env } from "../env.ts"; +import { RecentQuery } from "../sql/recent-query.ts"; +import { QueryOptimizer } from "./query-optimizer.ts"; import { RemoteSyncRequest } from "./remote.dto.ts"; import { Remote } from "./remote.ts"; export class RemoteController { + /** + * Only a single socket can be active at the same time. + * Multi-tab support not currently available + */ + private socket?: WebSocket; + constructor( private readonly remote: Remote, - ) {} + ) { + this.hookUpWebsockets(remote.optimizer); + } async execute( request: Request, ): Promise { const url = new URL(request.url); - if (url.pathname === "/postgres" && request.method === "POST") { - return await this.onFullSync(request); + if (url.pathname === "/postgres") { + const isWebsocket = request.headers.get("upgrade") === "websocket"; + if (isWebsocket) { + return this.onWebsocketRequest(request); + } else if (request.method === "POST") { + return await this.onFullSync(request); + } } } - async onFullSync(request: Request): Promise { + private hookUpWebsockets(optimizer: QueryOptimizer) { + optimizer.on( + "noImprovements", + this.eventNoImprovementsAvailable.bind(this), + ); + optimizer.on( + "improvementsAvailable", + this.eventImprovementsAvailable.bind(this), + ); + optimizer.on("error", this.eventError.bind(this)); + optimizer.on("timeout", this.eventTimeout.bind(this)); + optimizer.on("zeroCostPlan", this.eventZeroCostPlan.bind(this)); + optimizer.on("queryUnsupported", this.eventQueryUnsupported.bind(this)); + } + + private async onFullSync(request: Request): Promise { const body = RemoteSyncRequest.safeDecode(await request.text()); if (!body.success) { return new Response(JSON.stringify(body.error), { status: 400 }); } + const { db } = body.data; try { const sync = await this.remote.syncFrom(db); @@ -35,4 +66,60 @@ export class RemoteController { }); } } + + private onWebsocketRequest(request: Request): Response { + const { socket, response } = Deno.upgradeWebSocket(request); + console.log({ socket }); + this.socket = socket; + + socket.addEventListener("close", () => { + this.socket = undefined; + }); + + return response; + } + + private eventNoImprovementsAvailable(query: RecentQuery) { + this.socket?.send( + JSON.stringify({ type: "noImprovements", query }), + ); + } + + private eventImprovementsAvailable(query: RecentQuery) { + this.socket?.send( + JSON.stringify({ type: "improvementsAvailable", query }), + ); + } + + private eventError(recentQuery: RecentQuery, error: Error) { + this.socket?.send( + JSON.stringify({ + type: "error", + query: recentQuery, + error: error.message, + }), + ); + } + + private eventTimeout(recentQuery: RecentQuery, waitedMs: number) { + this.socket?.send( + JSON.stringify({ + type: "timeout", + query: recentQuery, + waitTimeMs: waitedMs, + }), + ); + } + + private eventZeroCostPlan(recentQuery: RecentQuery) { + this.socket?.send( + JSON.stringify({ type: "zeroCostPlan", query: recentQuery }), + ); + } + + private eventQueryUnsupported(query: RecentQuery) { + this.socket?.send( + JSON.stringify({ type: "queryUnsupported", query }), + ); + } } diff --git a/src/remote/remote.ts b/src/remote/remote.ts index 96230a8..4ef1889 100644 --- a/src/remote/remote.ts +++ b/src/remote/remote.ts @@ -27,6 +27,7 @@ export class Remote { ); private readonly differ = new SchemaDiffer(); + readonly optimizer: QueryOptimizer; /** * We have to juggle 2 different connections to the Remote @@ -38,16 +39,17 @@ export class Remote { * destroyed and re-created on each successful sync along with the db itself */ private baseDbURL: Connectable; - private readonly queryOptimizer: QueryOptimizer; + /** The URL of the optimizing db */ + private readonly optimizingDbUDRL: Connectable; constructor( /** This has to be a local url. Very bad things will happen if this is a remote URL */ - private readonly targetURL: Connectable, + targetURL: Connectable, private readonly manager: ConnectionManager, ) { this.baseDbURL = targetURL.withDatabaseName(Remote.baseDbName); - // this.baseDb = this.targetManager.getOrCreateConnection(baseUrl); - this.queryOptimizer = new QueryOptimizer(this.manager); + this.optimizingDbUDRL = targetURL.withDatabaseName(Remote.optimizingDbName); + this.optimizer = new QueryOptimizer(manager); } async syncFrom( @@ -55,11 +57,10 @@ export class Remote { stats?: StatisticsMode, ): Promise { await this.resetDatabase(); - const target = this.targetURL.withDatabaseName(Remote.optimizingDbName); const [_restoreResult, recentQueries, fullSchema] = await Promise .allSettled([ // This potentially creates a lot of connections to the source - this.pipeSchema(target, source), + this.pipeSchema(this.optimizingDbUDRL, source), this.getRecentQueries(source), this.getFullSchema(source), ]); @@ -69,7 +70,7 @@ export class Remote { } const pg = this.manager.getOrCreateConnection( - this.targetURL, + this.optimizingDbUDRL, ); let queries: RecentQuery[] = []; @@ -79,6 +80,7 @@ export class Remote { await this.onSuccessfulSync( pg, + source, queries, stats, ); @@ -127,25 +129,25 @@ export class Remote { source: Connectable, ): Promise { const dump = DumpCommand.spawn(source, "native-postgres"); + // TODO: handle event emitter events + // dump.on("dump", (data) => { + // console.log("got dump data", data); + // }); + // dump.on("restore", (data) => { + // console.log("got restore data", data); + // }); const restore = RestoreCommand.spawn(target); const { dump: dumpResult, restore: restoreResult } = await dump.pipeTo( restore, ); - if (dumpResult.error) { - console.error(dumpResult.error); - } if (!dumpResult.status.success) { throw new Error( - `Dump failed with status ${dumpResult.status.code}\n${dumpResult.error}`, + `Dump failed with status ${dumpResult.status.code}`, ); } - if (restoreResult?.error) { - console.error(restoreResult.error); - } if (restoreResult && !restoreResult.status.success) { - console.log(restoreResult.error); throw new Error( - `Restore failed with status ${restoreResult.status.code}\n${restoreResult.error}`, + `Restore failed with status ${restoreResult.status.code}`, ); } } @@ -167,13 +169,20 @@ export class Remote { */ private async onSuccessfulSync( postgres: Postgres, - recentQueries?: RecentQuery[], + source: Connectable, + recentQueries: RecentQuery[], stats?: StatisticsMode, ): Promise { - if (this.targetURL.isSupabase()) { + if (source.isSupabase()) { // https://gist.github.com/Xetera/067c613580320468e8367d9d6c0e06ad await postgres.exec("drop schema if exists extensions cascade"); } - await this.queryOptimizer.start(this.targetURL, recentQueries, stats); + // await postgres.exec( + // `insert into uploaded_photos (original_filename, format, bytes, is_olo_menu, aim_available, created_at, updated_at) values ('', '', 2, true, true, now(), now())`, + // ); + // await postgres.exec( + // "vacuum analyze", + // ); + await this.optimizer.start(this.optimizingDbUDRL, recentQueries, stats); } } diff --git a/src/sql/recent-query.ts b/src/sql/recent-query.ts index 7f521ab..2e61cf3 100644 --- a/src/sql/recent-query.ts +++ b/src/sql/recent-query.ts @@ -24,6 +24,8 @@ export class RecentQuery { readonly isSystemQuery: boolean; readonly isSelectQuery: boolean; + readonly isIntrospection: boolean; + readonly isTargetlessSelectQuery: boolean; /** Use {@link RecentQuery.analyze} instead */ constructor( @@ -48,6 +50,10 @@ export class RecentQuery { this.isSystemQuery = RecentQuery.isSystemQuery(tableReferences); this.isSelectQuery = RecentQuery.isSelectQuery(data); + this.isIntrospection = RecentQuery.isIntrospection(data); + this.isTargetlessSelectQuery = this.isSelectQuery + ? RecentQuery.isTargetlessSelectQuery(tableReferences) + : false; } static async analyze( @@ -74,6 +80,16 @@ export class RecentQuery { static isSystemQuery(referencedTables: string[]): boolean { return referencedTables.some((table) => table.startsWith("pg_")); } + + static isIntrospection(data: RawRecentQuery): boolean { + return data.query.match("@qd_introspection") !== null; + } + + static isTargetlessSelectQuery( + referencedTables: string[], + ): boolean { + return referencedTables.length === 0; + } } export type RawRecentQuery = { diff --git a/src/sync/pg-connector.ts b/src/sync/pg-connector.ts index d11eb53..7f894d4 100644 --- a/src/sync/pg-connector.ts +++ b/src/sync/pg-connector.ts @@ -451,18 +451,17 @@ ORDER BY try { const results = await this.db.exec(` SELECT - pg_user.usename as "username", + 'unknown_user' as "username", query, mean_exec_time as "meanTime", calls, rows, toplevel as "topLevel" FROM pg_stat_statements - JOIN pg_user ON pg_user.usesysid = pg_stat_statements.userid WHERE query not like '%pg_stat_statements%' - and dbid = (select oid from pg_database where datname = current_database()) + -- and dbid = (select oid from pg_database where datname = current_database()) and query not like '%@qd_introspection%' - and pg_user.usename not in (/* supabase */ 'supabase_admin', 'supabase_auth_admin', /* neon */ 'cloud_admin'); -- @qd_introspection + -- and pg_user.usename not in (/* supabase */ 'supabase_admin', 'supabase_auth_admin', /* neon */ 'cloud_admin'); -- @qd_introspection `); // we're excluding `pg_stat_statements` from the results since it's almost certainly unrelated return await this.segmentedQueryCache.sync( diff --git a/src/sync/schema-link.ts b/src/sync/schema-link.ts index f725ffa..2a3bab8 100644 --- a/src/sync/schema-link.ts +++ b/src/sync/schema-link.ts @@ -4,6 +4,7 @@ import { shutdownController } from "../shutdown.ts"; import { withSpan } from "../otel.ts"; import { Connectable } from "./connectable.ts"; import { findPgDumpBinary, findPgRestoreBinary } from "./executable.ts"; +import { EventEmitter } from "node:events"; export type TableStats = { name: string; @@ -47,7 +48,8 @@ export class PostgresSchemaLink { } } -export class DumpCommand { +export class DumpCommand + extends EventEmitter<{ restore: [string]; dump: [string] }> { public static readonly binaryPath = findPgDumpBinary("17.2"); // we're intentionally NOT excluding the "extensions" schema // because supabase has triggers on that schema that cannot be @@ -82,7 +84,9 @@ export class DumpCommand { // we don't want to allow callers to construct an instance // with any arbitrary child process. Use the static method instead - private constructor(private readonly process: Deno.ChildProcess) {} + private constructor(private readonly process: Deno.ChildProcess) { + super(); + } static excludedSchemas(connectable: Connectable): string[] { if (connectable.isSupabase()) { @@ -152,14 +156,37 @@ export class DumpCommand { async pipeTo(restore: RestoreCommand): Promise { // Start consuming stderr in the background to prevent resource leaks - const stderrPromise = this.process.stderr.text(); + // const stderrPromise = this.process.stderr.text(); + + const decoder = new TextDecoder(); + this.process.stderr.pipeTo( + new WritableStream({ + write: (chunk) => { + this.emit("dump", decoder.decode(chunk)); + }, + }), + ); + + restore.stderr.pipeTo( + new WritableStream({ + write: (chunk) => { + this.emit("dump", decoder.decode(chunk)); + }, + }), + ); + restore.stdout.pipeTo( + new WritableStream({ + write: (chunk) => { + this.emit("dump", decoder.decode(chunk)); + }, + }), + ); try { await this.process.stdout.pipeTo(restore.stdin); } catch (error) { return { dump: { - error: error instanceof Error ? error.message : await stderrPromise, status: await this.process.status, }, }; @@ -167,15 +194,12 @@ export class DumpCommand { const dumpStatus = await this.process.status; // this only fails if the command is non-zero - const error = (await stderrPromise).trim(); const restoreStatus = await restore.status; const out = { dump: { - error, status: dumpStatus, }, restore: { - error: (await restore.stderr.text()).trim(), status: restoreStatus, }, }; @@ -251,6 +275,7 @@ export class RestoreCommand { "--no-acl", "--clean", "--if-exists", + "--verbose", ...RestoreCommand.formatFlags(), "--dbname", connectable.toString(), @@ -273,6 +298,10 @@ export class RestoreCommand { return this.process.stdin; } + get stdout() { + return this.process.stdout; + } + get stderr() { return this.process.stderr; } @@ -297,11 +326,9 @@ export class RestoreCommand { export type RestoreCommandResult = { dump: { - error: string; status: Deno.CommandStatus; }; restore?: { - error: string; status: Deno.CommandStatus; }; }; diff --git a/src/sync/syncer.ts b/src/sync/syncer.ts index 9e0b6b9..d9ea142 100644 --- a/src/sync/syncer.ts +++ b/src/sync/syncer.ts @@ -113,7 +113,7 @@ export class PostgresSyncer { }); } - this.differ.put(sql, serializedResult.schema); + this.differ.put(connectable, serializedResult.schema); const wrapped = schema + serializedResult.serialized; @@ -133,13 +133,12 @@ export class PostgresSyncer { * @throws {PostgresError} */ async liveQuery(connectable: Connectable) { - const sql = this.manager.getOrCreateConnection(connectable); - const connector = this.manager.getConnectorFor(sql); + const connector = this.manager.getConnectorFor(connectable); const [queries, schema] = await Promise.all([ connector.getRecentQueries(), connector.getSchema(), ]); - const deltas = this.differ.put(sql, schema); + const deltas = this.differ.put(connectable, schema); return { queries, deltas }; } From f9fe275ea0ec4e7ae9a65c7c780d0ef3d4c1cf2e Mon Sep 17 00:00:00 2001 From: Xetera Date: Wed, 10 Dec 2025 21:58:09 +0300 Subject: [PATCH 08/29] fix: lockfile --- deno.lock | 280 +----------------------------------------------------- 1 file changed, 2 insertions(+), 278 deletions(-) diff --git a/deno.lock b/deno.lock index cb6199a..ba8aa38 100644 --- a/deno.lock +++ b/deno.lock @@ -351,6 +351,7 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, "@query-doctor/core@0.0.5": { + "integrity": "sha512-+PJs/0oAqrvorWDDXoSO8a7QU0JYSj9KrsmvNRpSuclPDgBEgqLNL2V8a6ozaW15dOguRtwt+UlMdGYBVcNgFw==", "dependencies": [ "@pgsql/types", "colorette", @@ -1947,283 +1948,6 @@ "npm:sql-formatter@^15.6.6", "npm:sql-highlight@^6.1.0", "npm:zod@^4.1.12" - ], - "links": { - "npm:@query-doctor/common@0.0.0": { - "peerDependencies": [ - "npm:react-dom@^18.2.0", - "npm:react@^18.2.0" - ] - }, - "npm:@query-doctor/core@0.0.5": { - "dependencies": [ - "npm:@pgsql/types@^17.6.1", - "npm:colorette@^2.0.20", - "npm:dedent@^1.7.0", - "npm:pgsql-deparser@^17.11.1", - "npm:zod@^4.1.13" - ] - }, - "npm:@query-doctor/db-engine@0.0.0": { - "dependencies": [ - "npm:@query-doctor/common@*", - "npm:@query-doctor/pglite@1.0.7", - "npm:lodash@^4.17.21", - "npm:pgsql-ast-parser@^12.0.1" - ] - }, - "npm:@query-doctor/eslint-config-custom-server@0.0.0": { - "dependencies": [ - "npm:eslint-config-turbo@^1.12.4", - "npm:eslint-plugin-import@^2.31.0" - ], - "peerDependencies": [ - "npm:eslint@^8.43.0" - ] - }, - "npm:@query-doctor/eslint-config-custom@0.0.0": { - "dependencies": [ - "npm:eslint-config-prettier@^9.1.0", - "npm:eslint-config-turbo@^1.12.4", - "npm:eslint-plugin-prettier@^5.2.3", - "npm:eslint-plugin-react@7.37.5" - ], - "peerDependencies": [ - "npm:eslint@^8.43.0", - "npm:prettier@^2.8.8", - "npm:react-dom@^18.2.0", - "npm:react@^18.2.0", - "npm:typescript@^5.6.3" - ] - }, - "npm:@query-doctor/hooks@0.0.0": { - "dependencies": [ - "npm:@pgsql/parser@^17.5.0", - "npm:@query-doctor/common@*", - "npm:@query-doctor/db-engine@*", - "npm:@query-doctor/pglite@1.0.7", - "npm:cookie@^1.0.2", - "npm:dayjs@^1.11.19", - "npm:react-fast-compare@^3.2.2", - "npm:zod@^4.1.13" - ], - "peerDependencies": [ - "npm:react-dom@^18.2.0", - "npm:react@^18.2.0" - ] - }, - "npm:@query-doctor/sql-editor@0.0.0": { - "dependencies": [ - "npm:@codemirror/commands@^6.8.1", - "npm:@codemirror/lang-json@^6.0.2", - "npm:@codemirror/lang-sql@^6.10.0", - "npm:@codemirror/view@^6.38.8", - "npm:@query-doctor/db-engine@*", - "npm:@query-doctor/pglite@1.0.7", - "npm:@stitches/react@^1.2.8", - "npm:@uiw/codemirror-themes@^4.25.3", - "npm:@uiw/react-codemirror@^4.25.3", - "npm:clsx@^2.1.1", - "npm:json5@^2.2.3" - ], - "peerDependencies": [ - "npm:react-dom@^18.2.0", - "npm:react@^18.2.0" - ] - }, - "npm:@query-doctor/tailwind-config@0.0.0": {}, - "npm:@query-doctor/types@0.0.0": { - "dependencies": [ - "npm:zod@^4.1.13" - ], - "peerDependencies": [ - "npm:react-dom@^18.2.0", - "npm:react@^18.2.0" - ] - }, - "npm:@query-doctor/typescript-config@0.0.0": {}, - "npm:@query-doctor/ui@0.0.0": { - "dependencies": [ - "npm:@dagrejs/dagre@^1.1.5", - "npm:@query-doctor/common@*", - "npm:@query-doctor/hooks@*", - "npm:@query-doctor/pglite@1.0.7", - "npm:@stitches/react@^1.2.8", - "npm:@xyflow/react@^12.4.4", - "npm:clsx@^2.1.1", - "npm:lodash@^4.17.21", - "npm:polished@^4.3.1", - "npm:react-aria@3.28.0", - "npm:react-fast-compare@^3.2.2", - "npm:react-icons@^5.5.0", - "npm:react-stately@^3.26.0", - "npm:react-syntax-highlighter@^15.6.6" - ], - "peerDependencies": [ - "npm:react-dom@^18.2.0", - "npm:react@^18.2.0" - ] - }, - "npm:api@0.0.1": { - "dependencies": [ - "npm:@mrleebo/prisma-ast@~0.12.1", - "npm:@nestjs/cli@^10.4.9", - "npm:@nestjs/common@^10.4.20", - "npm:@nestjs/config@^3.3.0", - "npm:@nestjs/core@^10.4.20", - "npm:@nestjs/platform-express@^10.4.20", - "npm:@nestjs/schedule@^5.0.1", - "npm:@nestjs/throttler@^5.2.0", - "npm:@prisma/internals@6.16.3", - "npm:@sendgrid/mail@^8.1.6", - "npm:@sentry/cli@^2.55.0", - "npm:@sentry/nestjs@^10.14.0", - "npm:@total-typescript/ts-reset@~0.6.1", - "npm:antlr4@^4.13.2", - "npm:antlr4ng@^3.0.16", - "npm:bcrypt@6", - "npm:connect-mongodb-session@^3.1.1", - "npm:dedent@^1.7.0", - "npm:express-session@^1.18.2", - "npm:fast-xml-parser@^5.2.5", - "npm:lodash@^4.17.21", - "npm:mongodb@^5.8.1", - "npm:nanoid@^5.1.6", - "npm:raw-body@^3.0.2", - "npm:reflect-metadata@~0.2.2", - "npm:rxjs@^7.8.2", - "npm:stripe@^17.7.0", - "npm:tsconfig-paths@^4.2.0", - "npm:vite-tsconfig-paths@^5.1.4", - "npm:zod@^4.1.13" - ] - }, - "npm:app@0.0.0": { - "dependencies": [ - "npm:@query-doctor/common@*", - "npm:@query-doctor/core@*", - "npm:@query-doctor/hooks@*", - "npm:@query-doctor/pglite@1.0.7", - "npm:@query-doctor/sql-editor@*", - "npm:@query-doctor/ui@*", - "npm:@radix-ui/react-dialog@^1.1.15", - "npm:@radix-ui/react-menubar@^1.1.16", - "npm:@radix-ui/react-portal@^1.1.9", - "npm:@reduxjs/toolkit@^1.9.5", - "npm:@sentry/react@^10.14.0", - "npm:@tanstack/react-form@^1.23.8", - "npm:@tanstack/react-query@^5.90.11", - "npm:@tanstack/react-router@^1.132.37", - "npm:@tanstack/react-table@^8.21.3", - "npm:@xyflow/react@^12.4.4", - "npm:clarinet@~0.12.6", - "npm:clsx@^2.1.1", - "npm:date-fns@^3.6.0", - "npm:framer-motion@^12.23.25", - "npm:js-cookie@^3.0.5", - "npm:lodash@^4.17.21", - "npm:pgsql-ast-parser@^12.0.1", - "npm:query-string@^9.3.1", - "npm:react-aria@^3.28.0", - "npm:react-dom@^18.3.1", - "npm:react-icons@^5.5.0", - "npm:react-loading-skeleton@^3.5.0", - "npm:react-redux@^8.1.2", - "npm:react-verification-input@^4.2.2", - "npm:react@^18.3.1", - "npm:tailwind-merge@^2.6.0", - "npm:tailwindcss-animate@^1.0.7", - "npm:tailwindcss-radix-colors@^1.4.1", - "npm:uuid@13", - "npm:zod@^4.1.13" - ] - }, - "npm:blog@0.0.0": { - "dependencies": [ - "npm:@astrojs/mdx@^4.3.12", - "npm:@astrojs/react@^4.4.2", - "npm:@astrojs/rss@^4.0.14", - "npm:@astrojs/sitemap@^3.6.0", - "npm:@astrojs/tailwind@^6.0.2", - "npm:@query-doctor/common@*", - "npm:@query-doctor/db-engine@*", - "npm:@query-doctor/hooks@*", - "npm:@query-doctor/pglite@1.0.7", - "npm:@query-doctor/sql-editor@*", - "npm:@query-doctor/types@*", - "npm:@query-doctor/ui@*", - "npm:@tanstack/react-query@^5.90.11", - "npm:astro@^5.16.3", - "npm:framer-motion@^12.23.25", - "npm:fuse.js@^6.6.2", - "npm:react-dom@^18.3.1", - "npm:react-icons@^5.5.0", - "npm:react-rough-notation@^1.0.8", - "npm:react@^18.3.1", - "npm:sharp@~0.34.5", - "npm:zod@^4.1.13" - ] - }, - "npm:course@0.0.1": { - "dependencies": [ - "npm:@astrojs/check@~0.9.6", - "npm:@astrojs/mdx@^4.3.12", - "npm:@astrojs/node@^9.5.1", - "npm:@astrojs/prefetch@~0.4.1", - "npm:@astrojs/react@^4.4.2", - "npm:@astrojs/tailwind@^6.0.2", - "npm:@nanostores/react@~0.8.4", - "npm:@query-doctor/common@*", - "npm:@query-doctor/hooks@*", - "npm:@query-doctor/types@*", - "npm:@query-doctor/ui@*", - "npm:@radix-ui/react-tooltip@^1.0.7", - "npm:@tailwindcss/typography@~0.5.19", - "npm:astro@^5.16.3", - "npm:clsx@^2.1.1", - "npm:cookie@^1.0.2", - "npm:framer-motion@^12.23.25", - "npm:github-slugger@2", - "npm:nanostores@~0.11.4", - "npm:quick-score@0.2", - "npm:react-dom@^18.3.1", - "npm:react-fast-compare@^3.2.2", - "npm:react-icons@^5.5.0", - "npm:react-rough-notation@^1.0.8", - "npm:react-sortablejs@^6.1.4", - "npm:react@^18.3.1", - "npm:remark-mdx@^3.1.1", - "npm:remark@^15.0.1", - "npm:shiki@^1.4.0", - "npm:sortablejs@^1.15.6", - "npm:tailwind-scrollbar@^3.1.0", - "npm:tailwindcss-radix-colors@^1.4.1", - "npm:tailwindcss@^3.4.17", - "npm:typescript@^5.6.3" - ] - }, - "npm:docs@0.0.1": { - "dependencies": [ - "npm:@astrojs/starlight@0.37", - "npm:astro@^5.16.3", - "npm:sharp@~0.34.5" - ] - }, - "npm:query-doctor-site@0.0.1": { - "dependencies": [ - "npm:patch-package@^8.0.1", - "npm:turbo@^1.13.3", - "npm:typescript@^5.6.3" - ] - }, - "npm:ui-playground@0.0.0": { - "dependencies": [ - "npm:@ladle/react@^5.0.3", - "npm:@query-doctor/ui@*", - "npm:react-dom@^18.3.1", - "npm:react@^18.3.1" - ] - } - } + ] } } From f22876d9d35bea1ce1295b9dc5cb4c40f358f637 Mon Sep 17 00:00:00 2001 From: Xetera Date: Wed, 10 Dec 2025 22:01:43 +0300 Subject: [PATCH 09/29] fix: actions --- .github/workflows/build-action.yaml | 3 +++ action.yaml | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-action.yaml b/.github/workflows/build-action.yaml index af33bb7..843b190 100644 --- a/.github/workflows/build-action.yaml +++ b/.github/workflows/build-action.yaml @@ -27,6 +27,9 @@ jobs: - name: Typecheck run: deno check + - name: Run tests + run: deno run test + - name: Upload files to release uses: softprops/action-gh-release@v2 with: diff --git a/action.yaml b/action.yaml index d8a65b0..e1c71df 100644 --- a/action.yaml +++ b/action.yaml @@ -51,7 +51,6 @@ runs: # Run the compiled application - name: Run Analyzer shell: bash - working-directory: ${{ github.action_path }} run: deno run start env: PG_DUMP_BINARY: /usr/bin/pg_dump From 08a59e54102501edb39e7784b0d0800ded79a7f7 Mon Sep 17 00:00:00 2001 From: Xetera Date: Wed, 10 Dec 2025 22:03:30 +0300 Subject: [PATCH 10/29] chore: remove unused patch-package --- deno.json | 1 - deno.lock | 263 +----------------------------------------------------- 2 files changed, 1 insertion(+), 263 deletions(-) diff --git a/deno.json b/deno.json index 6488556..f31830a 100644 --- a/deno.json +++ b/deno.json @@ -41,7 +41,6 @@ "fast-csv": "npm:fast-csv@^5.0.5", "jsondiffpatch": "npm:jsondiffpatch@^0.7.3", "nunjucks": "npm:nunjucks@^3.2.4", - "patch-package": "npm:patch-package@^8.0.1", "pgsql-deparser": "npm:pgsql-deparser@^17.11.1", "sql-formatter": "npm:sql-formatter@^15.6.6", "sql-highlight": "npm:sql-highlight@^6.1.0", diff --git a/deno.lock b/deno.lock index ba8aa38..58f76c9 100644 --- a/deno.lock +++ b/deno.lock @@ -30,7 +30,6 @@ "npm:fast-csv@^5.0.5": "5.0.5", "npm:jsondiffpatch@~0.7.3": "0.7.3", "npm:nunjucks@^3.2.4": "3.2.4_chokidar@4.0.3", - "npm:patch-package@^8.0.1": "8.0.1", "npm:pgsql-deparser@^17.11.1": "17.12.1", "npm:sql-formatter@^15.6.6": "15.6.6", "npm:sql-highlight@^6.1.0": "6.1.0", @@ -421,9 +420,6 @@ "@types/node@18.19.130" ] }, - "@yarnpkg/lockfile@1.1.0": { - "integrity": "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==" - }, "a-sync-waterfall@1.0.1": { "integrity": "sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA==" }, @@ -563,12 +559,6 @@ "balanced-match" ] }, - "braces@3.0.3": { - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dependencies": [ - "fill-range" - ] - }, "buffer-crc32@1.0.0": { "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==" }, @@ -592,36 +582,6 @@ "byline@5.0.0": { "integrity": "sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==" }, - "call-bind-apply-helpers@1.0.2": { - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "dependencies": [ - "es-errors", - "function-bind" - ] - }, - "call-bind@1.0.8": { - "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", - "dependencies": [ - "call-bind-apply-helpers", - "es-define-property", - "get-intrinsic", - "set-function-length" - ] - }, - "call-bound@1.0.4": { - "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", - "dependencies": [ - "call-bind-apply-helpers", - "get-intrinsic" - ] - }, - "chalk@4.1.2": { - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dependencies": [ - "ansi-styles@4.3.0", - "supports-color" - ] - }, "chokidar@4.0.3": { "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", "dependencies": [ @@ -631,9 +591,6 @@ "chownr@1.1.4": { "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, - "ci-info@3.9.0": { - "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==" - }, "cliui@8.0.1": { "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", "dependencies": [ @@ -712,14 +669,6 @@ "dedent@1.7.0": { "integrity": "sha512-HGFtf8yhuhGhqO07SV79tRp+br4MnbdjeVxotpn1QBl30pcLLCQjX5b2295ll0fv8RKDKsmWYrl05usHM9CewQ==" }, - "define-data-property@1.1.4": { - "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "dependencies": [ - "es-define-property", - "es-errors", - "gopd" - ] - }, "deprecation@2.3.1": { "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" }, @@ -753,14 +702,6 @@ "uuid" ] }, - "dunder-proto@1.0.1": { - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "dependencies": [ - "call-bind-apply-helpers", - "es-errors", - "gopd" - ] - }, "eastasianwidth@0.2.0": { "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" }, @@ -776,18 +717,6 @@ "once" ] }, - "es-define-property@1.0.1": { - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==" - }, - "es-errors@1.3.0": { - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==" - }, - "es-object-atoms@1.1.1": { - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "dependencies": [ - "es-errors" - ] - }, "escalade@3.2.0": { "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==" }, @@ -813,18 +742,6 @@ "fast-fifo@1.3.2": { "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" }, - "fill-range@7.1.1": { - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dependencies": [ - "to-regex-range" - ] - }, - "find-yarn-workspace-root@2.0.0": { - "integrity": "sha512-1IMnbjt4KzsQfnhnzNd8wUEgXZ44IzZaZmnLYx7D5FZlaHt2gW20Cri8Q+E/t5tIj4+epTBub+2Zxu/vNILzqQ==", - "dependencies": [ - "micromatch" - ] - }, "foreground-child@3.3.1": { "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", "dependencies": [ @@ -835,45 +752,12 @@ "fs-constants@1.0.0": { "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, - "fs-extra@10.1.0": { - "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", - "dependencies": [ - "graceful-fs", - "jsonfile", - "universalify" - ] - }, - "function-bind@1.1.2": { - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" - }, "get-caller-file@2.0.5": { "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, - "get-intrinsic@1.3.0": { - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "dependencies": [ - "call-bind-apply-helpers", - "es-define-property", - "es-errors", - "es-object-atoms", - "function-bind", - "get-proto", - "gopd", - "has-symbols", - "hasown", - "math-intrinsics" - ] - }, "get-port@7.1.0": { "integrity": "sha512-QB9NKEeDg3xxVwCCwJQ9+xycaz6pBB6iQ76wiWMl1927n0Kir6alPiP+yuiICLLU4jpMe08dXfpebuQppFA2zw==" }, - "get-proto@1.0.1": { - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "dependencies": [ - "dunder-proto", - "es-object-atoms" - ] - }, "glob@10.5.0": { "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", "dependencies": [ @@ -886,61 +770,24 @@ ], "bin": true }, - "gopd@1.2.0": { - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==" - }, "graceful-fs@4.2.11": { "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" }, - "has-flag@4.0.0": { - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" - }, - "has-property-descriptors@1.0.2": { - "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "dependencies": [ - "es-define-property" - ] - }, - "has-symbols@1.1.0": { - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==" - }, - "hasown@2.0.2": { - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "dependencies": [ - "function-bind" - ] - }, "ieee754@1.2.1": { "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" }, "inherits@2.0.4": { "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, - "is-docker@2.2.1": { - "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", - "bin": true - }, "is-fullwidth-code-point@3.0.0": { "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, - "is-number@7.0.0": { - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==" - }, "is-stream@2.0.1": { "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" }, - "is-wsl@2.2.0": { - "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", - "dependencies": [ - "is-docker" - ] - }, "isarray@1.0.0": { "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" }, - "isarray@2.0.5": { - "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" - }, "isexe@2.0.0": { "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" }, @@ -953,16 +800,6 @@ "@pkgjs/parseargs" ] }, - "json-stable-stringify@1.3.0": { - "integrity": "sha512-qtYiSSFlwot9XHtF9bD9c7rwKjr+RecWT//ZnPvSmEjpV5mmPOCN4j8UjY5hbjNkOwZ/jQv3J6R1/pL7RwgMsg==", - "dependencies": [ - "call-bind", - "call-bound", - "isarray@2.0.5", - "jsonify", - "object-keys" - ] - }, "jsondiffpatch@0.7.3": { "integrity": "sha512-zd4dqFiXSYyant2WgSXAZ9+yYqilNVvragVNkNRn2IFZKgjyULNrKRznqN4Zon0MkLueCg+3QaPVCnDAVP20OQ==", "dependencies": [ @@ -970,24 +807,6 @@ ], "bin": true }, - "jsonfile@6.2.0": { - "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", - "dependencies": [ - "universalify" - ], - "optionalDependencies": [ - "graceful-fs" - ] - }, - "jsonify@0.0.1": { - "integrity": "sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg==" - }, - "klaw-sync@6.0.0": { - "integrity": "sha512-nIeuVSzdCCs6TDPTqI8w1Yre34sSq7AkZ4B3sfOBbI2CgVSB4Du4aLQijFU2+lhAFCwt9+42Hel6lQNIv6AntQ==", - "dependencies": [ - "graceful-fs" - ] - }, "lazystream@1.0.1": { "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", "dependencies": [ @@ -1027,16 +846,6 @@ "lru-cache@10.4.3": { "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" }, - "math-intrinsics@1.1.0": { - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==" - }, - "micromatch@4.0.8": { - "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dependencies": [ - "braces", - "picomatch" - ] - }, "minimatch@5.1.6": { "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", "dependencies": [ @@ -1049,9 +858,6 @@ "brace-expansion" ] }, - "minimist@1.2.8": { - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==" - }, "minipass@7.1.2": { "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==" }, @@ -1097,45 +903,15 @@ ], "bin": true }, - "object-keys@1.1.1": { - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" - }, "once@1.4.0": { "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "dependencies": [ "wrappy" ] }, - "open@7.4.2": { - "integrity": "sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==", - "dependencies": [ - "is-docker", - "is-wsl" - ] - }, "package-json-from-dist@1.0.1": { "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==" }, - "patch-package@8.0.1": { - "integrity": "sha512-VsKRIA8f5uqHQ7NGhwIna6Bx6D9s/1iXlA1hthBVBEbkq+t4kXD0HHt+rJhf/Z+Ci0F/HCB2hvn0qLdLG+Qxlw==", - "dependencies": [ - "@yarnpkg/lockfile", - "chalk", - "ci-info", - "cross-spawn", - "find-yarn-workspace-root", - "fs-extra", - "json-stable-stringify", - "klaw-sync", - "minimist", - "open", - "semver", - "slash", - "tmp", - "yaml" - ], - "bin": true - }, "path-key@3.1.1": { "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" }, @@ -1152,9 +928,6 @@ "@pgsql/types" ] }, - "picomatch@2.3.1": { - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" - }, "process-nextick-args@2.0.1": { "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, @@ -1215,7 +988,7 @@ "dependencies": [ "core-util-is", "inherits", - "isarray@1.0.0", + "isarray", "process-nextick-args", "safe-buffer@5.1.2", "string_decoder@1.1.1", @@ -1267,21 +1040,6 @@ "safer-buffer@2.1.2": { "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "semver@7.7.3": { - "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", - "bin": true - }, - "set-function-length@1.2.2": { - "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", - "dependencies": [ - "define-data-property", - "es-errors", - "function-bind", - "get-intrinsic", - "gopd", - "has-property-descriptors" - ] - }, "shebang-command@2.0.0": { "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dependencies": [ @@ -1297,9 +1055,6 @@ "signal-exit@4.1.0": { "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==" }, - "slash@2.0.0": { - "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==" - }, "split-ca@1.0.1": { "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==" }, @@ -1381,12 +1136,6 @@ "ansi-regex@6.2.2" ] }, - "supports-color@7.2.0": { - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dependencies": [ - "has-flag" - ] - }, "tar-fs@2.1.4": { "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", "dependencies": [ @@ -1454,12 +1203,6 @@ "tmp@0.2.5": { "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==" }, - "to-regex-range@5.0.1": { - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dependencies": [ - "is-number" - ] - }, "tunnel@0.0.6": { "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" }, @@ -1487,9 +1230,6 @@ "universal-user-agent@6.0.1": { "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==" }, - "universalify@2.0.1": { - "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==" - }, "util-deprecate@1.0.2": { "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, @@ -1943,7 +1683,6 @@ "npm:fast-csv@^5.0.5", "npm:jsondiffpatch@~0.7.3", "npm:nunjucks@^3.2.4", - "npm:patch-package@^8.0.1", "npm:pgsql-deparser@^17.11.1", "npm:sql-formatter@^15.6.6", "npm:sql-highlight@^6.1.0", From 04a6d5d02c3a8fd2cc631e3d7688001ae86b560b Mon Sep 17 00:00:00 2001 From: Xetera Date: Wed, 10 Dec 2025 22:11:24 +0300 Subject: [PATCH 11/29] fix: only publish when pushing to main --- .github/workflows/publish.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 7f0b5f5..efe4046 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -50,7 +50,7 @@ jobs: with: context: . file: Dockerfile - push: ${{ github.event_name == 'push' }} + push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} # outputs: type=docker platforms: linux/amd64,linux/arm64 tags: | @@ -69,7 +69,7 @@ jobs: mv /tmp/.buildx-cache-new /tmp/.buildx-cache - name: Attest uses: actions/attest-build-provenance@v2 - if: ${{ github.event_name == 'push' }} + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} with: subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} subject-digest: ${{ steps.build.outputs.digest }} From f9efd13b615aef05bc91cc687fe9f6f838bd2d68 Mon Sep 17 00:00:00 2001 From: Xetera Date: Wed, 10 Dec 2025 22:11:28 +0300 Subject: [PATCH 12/29] fix: postgres url --- .github/workflows/custom.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/custom.yaml b/.github/workflows/custom.yaml index e207ee2..a5e9735 100644 --- a/.github/workflows/custom.yaml +++ b/.github/workflows/custom.yaml @@ -45,5 +45,5 @@ jobs: uses: ./ env: GITHUB_TOKEN: ${{ github.token }} - POSTGRES_URL: http://query_doctor@localhost:5432/testing + POSTGRES_URL: postgres://query_doctor@localhost:5432/testing LOG_PATH: /var/log/postgresql/postgres.log From ccc01aae0d9e8ecbc34407bc225c226bd068ac43 Mon Sep 17 00:00:00 2001 From: Xetera Date: Wed, 10 Dec 2025 22:18:01 +0300 Subject: [PATCH 13/29] fix: run tests and typechecks even when not pushing to main --- .github/workflows/build-action.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-action.yaml b/.github/workflows/build-action.yaml index 843b190..b7a39be 100644 --- a/.github/workflows/build-action.yaml +++ b/.github/workflows/build-action.yaml @@ -4,8 +4,6 @@ name: Release Action on: workflow_dispatch: push: - branches: - - main jobs: release: @@ -32,6 +30,7 @@ jobs: - name: Upload files to release uses: softprops/action-gh-release@v2 + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} with: tag_name: v${{ env.DENO_VERSION }} env: From 10651b3080ad033c026856e0c247f3017cd94218 Mon Sep 17 00:00:00 2001 From: Xetera Date: Wed, 10 Dec 2025 22:45:50 +0300 Subject: [PATCH 14/29] chore: slight cleanup of query-optimizer logic --- src/remote/query-optimizer.ts | 46 ++++++++++++++++++----------------- src/remote/remote.ts | 6 ----- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/src/remote/query-optimizer.ts b/src/remote/query-optimizer.ts index 86165ea..2b3773a 100644 --- a/src/remote/query-optimizer.ts +++ b/src/remote/query-optimizer.ts @@ -72,8 +72,9 @@ export class QueryOptimizer extends EventEmitter { } /** - * Start optimizing a new set of queries. - * @returns the array of queries that will be considered + * Start optimizing a new set of queries + * @returns Promise of array of queries that were considered for optimization. + * Resolves when all queries are optimized */ async start( conn: Connectable, @@ -116,7 +117,7 @@ export class QueryOptimizer extends EventEmitter { this.queries.set(query.hash, { query, optimization }); } this._allQueries = this.queries.size; - while (await this.work()); + await this.work(); return validQueries; } @@ -134,24 +135,29 @@ export class QueryOptimizer extends EventEmitter { if (!this.target) { return; } - let recentQuery: RecentQuery | undefined; - const token = await this.semaphore.acquire(); - try { - for (const [hash, entry] of this.queries.entries()) { - if (entry.optimization.state !== "waiting") { - continue; + + while (true) { + let recentQuery: RecentQuery | undefined; + const token = await this.semaphore.acquire(); + try { + for (const [hash, entry] of this.queries.entries()) { + if (entry.optimization.state !== "waiting") { + continue; + } + this.queries.set(hash, { + query: entry.query, + optimization: { state: "optimizing" }, + }); + recentQuery = entry.query; + break; } - this.queries.set(hash, { - query: entry.query, - optimization: { state: "optimizing" }, - }); - recentQuery = entry.query; + } finally { + this.semaphore.release(token); + } + if (!recentQuery) { + this._finish.resolve(0); break; } - } finally { - this.semaphore.release(token); - } - if (recentQuery) { this._validQueriesProcessed++; const optimization = await this.optimizeQuery( recentQuery, @@ -162,10 +168,6 @@ export class QueryOptimizer extends EventEmitter { query: recentQuery, optimization, }); - return true; - } else { - this._finish.resolve(0); - return false; } } diff --git a/src/remote/remote.ts b/src/remote/remote.ts index 4ef1889..d1381f2 100644 --- a/src/remote/remote.ts +++ b/src/remote/remote.ts @@ -177,12 +177,6 @@ export class Remote { // https://gist.github.com/Xetera/067c613580320468e8367d9d6c0e06ad await postgres.exec("drop schema if exists extensions cascade"); } - // await postgres.exec( - // `insert into uploaded_photos (original_filename, format, bytes, is_olo_menu, aim_available, created_at, updated_at) values ('', '', 2, true, true, now(), now())`, - // ); - // await postgres.exec( - // "vacuum analyze", - // ); await this.optimizer.start(this.optimizingDbUDRL, recentQueries, stats); } } From e943423935ba427f047c9406be99c556f4c12afc Mon Sep 17 00:00:00 2001 From: Xetera Date: Tue, 16 Dec 2025 17:53:34 +0300 Subject: [PATCH 15/29] chore: update core library --- deno.json | 2 +- deno.lock | 36 +++++++++--------------------------- 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/deno.json b/deno.json index f31830a..4a77419 100644 --- a/deno.json +++ b/deno.json @@ -25,7 +25,7 @@ "@libpg-query/parser": "npm:@libpg-query/parser@^17.6.3", "@opentelemetry/api": "jsr:@opentelemetry/api@^1.9.0", "@pgsql/types": "npm:@pgsql/types@^17.6.1", - "@query-doctor/core": "npm:@query-doctor/core@^0.0.5", + "@query-doctor/core": "npm:@query-doctor/core@^0.1.0", "@rabbit-company/rate-limiter": "jsr:@rabbit-company/rate-limiter@^3.0.0", "@std/assert": "jsr:@std/assert@^1.0.14", "@std/collections": "jsr:@std/collections@^1.1.3", diff --git a/deno.lock b/deno.lock index 58f76c9..62bd55b 100644 --- a/deno.lock +++ b/deno.lock @@ -20,13 +20,13 @@ "npm:@actions/github@^6.0.1": "6.0.1_@octokit+core@5.2.2", "npm:@libpg-query/parser@^17.6.3": "17.6.3", "npm:@pgsql/types@^17.6.1": "17.6.1", - "npm:@query-doctor/core@^0.0.5": "0.0.5", + "npm:@query-doctor/core@0.1": "0.1.0", "npm:@testcontainers/postgresql@^11.9.0": "11.9.0", "npm:@types/node@^24.9.1": "24.10.1", "npm:@types/nunjucks@^3.2.6": "3.2.6", "npm:async-sema@^3.1.1": "3.1.1", "npm:chokidar@^4.0.3": "4.0.3", - "npm:dedent@^1.6.0": "1.6.0", + "npm:dedent@^1.6.0": "1.7.0", "npm:fast-csv@^5.0.5": "5.0.5", "npm:jsondiffpatch@~0.7.3": "0.7.3", "npm:nunjucks@^3.2.4": "3.2.4_chokidar@4.0.3", @@ -349,12 +349,12 @@ "@protobufjs/utf8@1.1.0": { "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, - "@query-doctor/core@0.0.5": { - "integrity": "sha512-+PJs/0oAqrvorWDDXoSO8a7QU0JYSj9KrsmvNRpSuclPDgBEgqLNL2V8a6ozaW15dOguRtwt+UlMdGYBVcNgFw==", + "@query-doctor/core@0.1.0": { + "integrity": "sha512-+rUn5IAXA8BRPm6GbJqdi6hfDj8Ei9kO7I7jCSvMJCILY/uSpPY4xQhkNXobNnO6EDVpcDb/D5dp9/lCpI2lGA==", "dependencies": [ "@pgsql/types", "colorette", - "dedent@1.7.0", + "dedent", "pgsql-deparser", "zod" ] @@ -369,7 +369,7 @@ "integrity": "sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==", "dependencies": [ "@types/node@24.2.0", - "@types/ssh2@1.15.5" + "@types/ssh2" ] }, "@types/dockerode@3.3.47": { @@ -377,13 +377,7 @@ "dependencies": [ "@types/docker-modem", "@types/node@24.2.0", - "@types/ssh2@1.15.5" - ] - }, - "@types/node@18.19.130": { - "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", - "dependencies": [ - "undici-types@5.26.5" + "@types/ssh2" ] }, "@types/node@24.10.1": { @@ -414,12 +408,6 @@ "@types/ssh2-streams" ] }, - "@types/ssh2@1.15.5": { - "integrity": "sha512-N1ASjp/nXH3ovBHddRJpli4ozpk6UdDYIX4RJWFa9L1YKnzdhTlVmiGHm4DZnj/jLbqZpes4aeR30EFGQtvhQQ==", - "dependencies": [ - "@types/node@18.19.130" - ] - }, "a-sync-waterfall@1.0.1": { "integrity": "sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA==" }, @@ -663,9 +651,6 @@ "ms" ] }, - "dedent@1.6.0": { - "integrity": "sha512-F1Z+5UCFpmQUzJa11agbyPVMbpgT/qA3/SKyJ1jyBgm7dUcUEa8v9JwDkerSQXfakBwFljIxhOJqGkjUwZ9FSA==" - }, "dedent@1.7.0": { "integrity": "sha512-HGFtf8yhuhGhqO07SV79tRp+br4MnbdjeVxotpn1QBl30pcLLCQjX5b2295ll0fv8RKDKsmWYrl05usHM9CewQ==" }, @@ -1072,7 +1057,7 @@ "ssh-remote-port-forward@1.0.4": { "integrity": "sha512-x0LV1eVDwjf1gmG7TTnfqIzf+3VPRz7vrNIjX6oYLbeCrf/PeVY6hkT68Mg+q02qXxQhrLjB0jfgvhevoCRmLQ==", "dependencies": [ - "@types/ssh2@0.5.52", + "@types/ssh2", "ssh2" ] }, @@ -1209,9 +1194,6 @@ "tweetnacl@0.14.5": { "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, - "undici-types@5.26.5": { - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" - }, "undici-types@7.10.0": { "integrity": "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag==" }, @@ -1673,7 +1655,7 @@ "npm:@actions/github@^6.0.1", "npm:@libpg-query/parser@^17.6.3", "npm:@pgsql/types@^17.6.1", - "npm:@query-doctor/core@^0.0.5", + "npm:@query-doctor/core@0.1", "npm:@testcontainers/postgresql@^11.9.0", "npm:@types/node@^24.9.1", "npm:@types/nunjucks@^3.2.6", From 69e3312b0d41cd049939b4f57d3319fb9f911666 Mon Sep 17 00:00:00 2001 From: Xetera Date: Tue, 16 Dec 2025 17:53:49 +0300 Subject: [PATCH 16/29] fix: don't drop indexes when optimizing --- src/remote/query-optimizer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/remote/query-optimizer.ts b/src/remote/query-optimizer.ts index 2b3773a..8ccb3bd 100644 --- a/src/remote/query-optimizer.ts +++ b/src/remote/query-optimizer.ts @@ -240,7 +240,7 @@ export class QueryOptimizer extends EventEmitter { let cost: number; try { const explain = await withTimeout( - target.optimizer.runWithoutIndexes(builder), + target.optimizer.testQueryWithStats(builder), timeoutMs, ); cost = explain.Plan["Total Cost"]; From 8a2f08c6d473227d59ba955b72ef571b11137fdc Mon Sep 17 00:00:00 2001 From: Xetera Date: Tue, 16 Dec 2025 17:54:34 +0300 Subject: [PATCH 17/29] fix: pass `POSTGRES_URL` to the webserver --- src/main.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main.ts b/src/main.ts index 92b7021..1574ea3 100644 --- a/src/main.ts +++ b/src/main.ts @@ -29,7 +29,11 @@ function runOutsideCI() { `Starting server (${os}-${arch}) on ${env.HOST}:${env.PORT}`, "main", ); - createServer(env.HOST, env.PORT); + if (!env.POSTGRES_URL) { + core.setFailed("POSTGRES_URL environment variable is not set"); + Deno.exit(1); + } + createServer(env.HOST, env.PORT, Connectable.fromString(env.POSTGRES_URL)); } async function main() { From b6a0b4a5a01b663ad17a938bf56333ee79daca74 Mon Sep 17 00:00:00 2001 From: Xetera Date: Wed, 17 Dec 2025 17:07:09 +0300 Subject: [PATCH 18/29] wip timescaledb --- src/remote/remote.test.ts | 158 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 151 insertions(+), 7 deletions(-) diff --git a/src/remote/remote.test.ts b/src/remote/remote.test.ts index 1f016b4..765a87a 100644 --- a/src/remote/remote.test.ts +++ b/src/remote/remote.test.ts @@ -5,6 +5,30 @@ import postgres from "postgresjs"; import { assertEquals } from "@std/assert/equals"; import { ConnectionManager } from "../sync/connection-manager.ts"; import { assertArrayIncludes } from "@std/assert"; +import { PgIdentifier } from "@query-doctor/core"; + +const TEST_TARGET_CONTAINER_NAME = "postgres:17"; +const TEST_TARGET_CONTAINER_TIMESCALEDB_NAME = + "timescale/timescaledb:latest-pg17"; + +export function testSpawnTarget( + options: { content?: string; containerName?: string } = { + containerName: TEST_TARGET_CONTAINER_NAME, + }, +) { + let pg = new PostgreSqlContainer( + options.containerName ?? TEST_TARGET_CONTAINER_NAME, + ); + if (options.content) { + pg = pg.withCopyContentToContainer([ + { + content: options.content, + target: "/docker-entrypoint-initdb.d/init.sql", + }, + ]); + } + return pg.start(); +} function assertOk( result: { type: string; value?: T }, @@ -34,13 +58,9 @@ Deno.test({ ]) .withCommand(["-c", "shared_preload_libraries=pg_stat_statements"]) .start(), - new PostgreSqlContainer("postgres:17") - .withCopyContentToContainer([ - { - content: "create table testing(a int); create index on testing(a)", - target: "/docker-entrypoint-initdb.d/init.sql", - }, - ]).start(), + testSpawnTarget( + { content: "create table testing(a int); create index on testing(a)" }, + ), ]); try { @@ -108,3 +128,127 @@ Deno.test({ } }, }); + +Deno.test({ + name: "raw timescaledb syncs correctly", + sanitizeOps: false, + sanitizeResources: false, + fn: async () => { + const [source, target] = await Promise.all([ + new PostgreSqlContainer( + "timescale/timescaledb:latest-pg17", + ) + .withEnvironment({ + POSTGRES_HOST_AUTH_METHOD: "trust", + }) + .withCopyContentToContainer([ + { + content: ` + create table testing(a int, b text); + insert into testing values (1); + create index "testing_1234" on testing(b); + `, + target: "/docker-entrypoint-initdb.d/init.sql", + }, + ]) + .start(), + testSpawnTarget({ + containerName: TEST_TARGET_CONTAINER_TIMESCALEDB_NAME, + }), + ]); + + const sourceConn = Connectable.fromString(source.getConnectionUri()); + const targetConn = Connectable.fromString(target.getConnectionUri()); + const manager = ConnectionManager.forLocalDatabase(); + const remote = new Remote(targetConn, manager); + + try { + const t = manager.getOrCreateConnection( + targetConn.withDatabaseName(PgIdentifier.fromString("optimizing_db")), + ); + await remote.syncFrom(sourceConn); + const indexesAfter = await t.exec( + "select indexname from pg_indexes where schemaname = 'public'", + ); + assertEquals( + indexesAfter.length, + 1, + "Indexes were not copied over correctly from the source db", + ); + + assertEquals(indexesAfter[0], { indexname: "testing_1234" }); + } finally { + await Promise.all([source.stop(), target.stop()]); + } + }, +}); + +Deno.test({ + name: "timescaledb with continuous aggregates sync correctly", + sanitizeOps: false, + sanitizeResources: false, + fn: async () => { + const [source, target] = await Promise.all([ + new PostgreSqlContainer( + "timescale/timescaledb:latest-pg17", + ) + .withEnvironment({ + POSTGRES_HOST_AUTH_METHOD: "trust", + }) + .withLogConsumer((a) => a.pipe(process.stdout)) + .withCopyContentToContainer([ + { + content: ` + create table conditions( + "time" timestamptz not null, + device_id integer, + temperature float + ) + with( + timescaledb.hypertable, + timescaledb.partition_column = 'time' + ); + create materialized view conditions_summary_daily + with (timescaledb.continuous) as + select device_id, + time_bucket(interval '1 day', time) as bucket, + avg(temperature), + max(temperature), + min(temperature) + from conditions + group by device_id, bucket; + `, + target: "/docker-entrypoint-initdb.d/init.sql", + }, + ]) + .start(), + testSpawnTarget({ + containerName: TEST_TARGET_CONTAINER_TIMESCALEDB_NAME, + }), + ]); + + const sourceConn = Connectable.fromString(source.getConnectionUri()); + const targetConn = Connectable.fromString(target.getConnectionUri()); + const manager = ConnectionManager.forLocalDatabase(); + const remote = new Remote(targetConn, manager); + + try { + const t = manager.getOrCreateConnection( + targetConn.withDatabaseName(PgIdentifier.fromString("optimizing_db")), + ); + await remote.syncFrom(sourceConn); + const indexesAfter = await t.exec( + "select indexname from pg_indexes where schemaname = 'public'", + ); + assertEquals( + indexesAfter.length, + 1, + "Indexes were not copied over correctly from the source db", + ); + + assertEquals(indexesAfter[0], { indexname: "testing_1234" }); + } finally { + await Promise.all([source.stop(), target.stop()]); + } + }, +}); From 57d5f7a8a6e8ab893c3a82bd9c79e8dd8feaa1f8 Mon Sep 17 00:00:00 2001 From: Xetera Date: Wed, 17 Dec 2025 18:30:29 +0300 Subject: [PATCH 19/29] feat: pull stats from source by default --- src/remote/remote.ts | 55 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/src/remote/remote.ts b/src/remote/remote.ts index d1381f2..bb89996 100644 --- a/src/remote/remote.ts +++ b/src/remote/remote.ts @@ -1,6 +1,8 @@ import { PgIdentifier, type Postgres, + PostgresVersion, + Statistics, StatisticsMode, } from "@query-doctor/core"; import { type Connectable } from "../sync/connectable.ts"; @@ -54,16 +56,19 @@ export class Remote { async syncFrom( source: Connectable, - stats?: StatisticsMode, + statsStrategy: StatisticsStrategy = { type: "pullFromSource" }, ): Promise { await this.resetDatabase(); - const [_restoreResult, recentQueries, fullSchema] = await Promise - .allSettled([ - // This potentially creates a lot of connections to the source - this.pipeSchema(this.optimizingDbUDRL, source), - this.getRecentQueries(source), - this.getFullSchema(source), - ]); + const [_restoreResult, recentQueries, fullSchema, pulledStats] = + await Promise + .allSettled([ + // This potentially creates a lot of connections to the source + this.pipeSchema(this.optimizingDbUDRL, source), + this.getRecentQueries(source), + this.getFullSchema(source), + this.dumpSourceStats(source), + this.resolveStatisticsStrategy(source, statsStrategy), + ]); if (fullSchema.status === "fulfilled") { this.differ.put(source, fullSchema.value); @@ -78,6 +83,11 @@ export class Remote { queries = recentQueries.value; } + let stats: StatisticsMode | undefined; + if (pulledStats.status === "fulfilled") { + stats = pulledStats.value; + } + await this.onSuccessfulSync( pg, source, @@ -152,6 +162,28 @@ export class Remote { } } + private resolveStatisticsStrategy( + source: Connectable, + strategy: StatisticsStrategy, + ): Promise { + switch (strategy.type) { + case "static": + return Promise.resolve(strategy.stats); + case "pullFromSource": + return this.dumpSourceStats(source); + } + } + + private async dumpSourceStats(source: Connectable): Promise { + const pg = this.manager.getOrCreateConnection(source); + const stats = await Statistics.dumpStats( + pg, + PostgresVersion.parse("17"), + "full", + ); + return { kind: "fromStatisticsExport", source: { kind: "inline" }, stats }; + } + private getRecentQueries( source: Connectable, ): Promise { @@ -180,3 +212,10 @@ export class Remote { await this.optimizer.start(this.optimizingDbUDRL, recentQueries, stats); } } + +export type StatisticsStrategy = { + type: "pullFromSource"; +} | { + type: "static"; + stats: StatisticsMode; +}; From 3f20940434ce2c97459c72633c19327a679d6c0a Mon Sep 17 00:00:00 2001 From: Jean-Philippe Sirois Date: Fri, 19 Dec 2025 18:28:34 +0400 Subject: [PATCH 20/29] fix: avoid mutating websocket headers --- src/server/http.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/server/http.ts b/src/server/http.ts index bec834a..c85b47d 100644 --- a/src/server/http.ts +++ b/src/server/http.ts @@ -214,6 +214,10 @@ export function createServer( } const remoteResponse = await remoteController?.execute(req); if (remoteResponse) { + // WebSocket upgrade responses have immutable headers, skip transform + if (req.headers.get("upgrade") === "websocket") { + return remoteResponse; + } return transformResponse(remoteResponse, limit); } return new Response("Not found", { status: 404 }); From 514a63b212f8a06cedd580fc7b1a4f1213fd28ae Mon Sep 17 00:00:00 2001 From: Jean-Philippe Sirois Date: Fri, 19 Dec 2025 18:17:03 +0400 Subject: [PATCH 21/29] fix: avoid blocking API while optimizer is running --- src/remote/remote.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/remote/remote.ts b/src/remote/remote.ts index bb89996..a05a936 100644 --- a/src/remote/remote.ts +++ b/src/remote/remote.ts @@ -209,7 +209,7 @@ export class Remote { // https://gist.github.com/Xetera/067c613580320468e8367d9d6c0e06ad await postgres.exec("drop schema if exists extensions cascade"); } - await this.optimizer.start(this.optimizingDbUDRL, recentQueries, stats); + this.optimizer.start(this.optimizingDbUDRL, recentQueries, stats); } } From 87aecb1ff092d9d81a2f3e66880eecbb19446fa8 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Sirois Date: Fri, 19 Dec 2025 18:00:08 +0400 Subject: [PATCH 22/29] feat: restore query metadata for new api --- src/remote/query-optimizer.ts | 12 ++++++------ src/remote/remote-controller.ts | 21 +++++++++++++++++---- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/remote/query-optimizer.ts b/src/remote/query-optimizer.ts index 8ccb3bd..a50eab9 100644 --- a/src/remote/query-optimizer.ts +++ b/src/remote/query-optimizer.ts @@ -23,8 +23,8 @@ type EventMap = { timeout: [RecentQuery, number]; zeroCostPlan: [RecentQuery]; queryUnsupported: [RecentQuery]; - noImprovements: [RecentQuery]; - improvementsAvailable: [RecentQuery]; + noImprovements: [RecentQuery, Extract]; + improvementsAvailable: [RecentQuery, Extract]; }; type Target = { @@ -295,7 +295,7 @@ export class QueryOptimizer extends EventEmitter { Math.abs(percentageReduction), ); if (costReductionPercentage < MINIMUM_COST_CHANGE_PERCENTAGE) { - this.onNoImprovements(recent); + this.onNoImprovements(recent, result); return { state: "no_improvement_found", cost: result.baseCost, @@ -319,8 +319,8 @@ export class QueryOptimizer extends EventEmitter { } } - private onNoImprovements(recent: RecentQuery) { - this.emit("noImprovements", recent); + private onNoImprovements(recent: RecentQuery, result: Extract) { + this.emit("noImprovements", recent, result); } private getPotentialIndexCandidates( @@ -347,7 +347,7 @@ export class QueryOptimizer extends EventEmitter { recent: RecentQuery, result: Extract, ) { - this.emit("improvementsAvailable", recent); + this.emit("improvementsAvailable", recent, result); this.queries.set(recent.hash, { query: recent, optimization: { diff --git a/src/remote/remote-controller.ts b/src/remote/remote-controller.ts index eb1b34c..a1962e3 100644 --- a/src/remote/remote-controller.ts +++ b/src/remote/remote-controller.ts @@ -3,6 +3,7 @@ import { RecentQuery } from "../sql/recent-query.ts"; import { QueryOptimizer } from "./query-optimizer.ts"; import { RemoteSyncRequest } from "./remote.dto.ts"; import { Remote } from "./remote.ts"; +import { OptimizeResult } from "@query-doctor/core"; export class RemoteController { /** @@ -79,15 +80,27 @@ export class RemoteController { return response; } - private eventNoImprovementsAvailable(query: RecentQuery) { + private eventNoImprovementsAvailable(query: RecentQuery, result: Extract) { + const indexesUsed = Array.from(result.existingIndexes); this.socket?.send( - JSON.stringify({ type: "noImprovements", query }), + JSON.stringify({ type: "noImprovements", query, cost: result.baseCost, indexesUsed }), ); } - private eventImprovementsAvailable(query: RecentQuery) { + private eventImprovementsAvailable(query: RecentQuery, result: Extract) { + const indexesUsed = Array.from(result.existingIndexes); + const recommendedIndexes = Array.from(result.newIndexes) + .map((n) => result.triedIndexes.get(n)?.definition) + .filter((n) => n !== undefined); this.socket?.send( - JSON.stringify({ type: "improvementsAvailable", query }), + JSON.stringify({ + type: "improvementsAvailable", + query, + cost: result.baseCost, + optimizedCost: result.finalCost, + indexesUsed, + recommendedIndexes, + }), ); } From a7c531f9527f02846d5407ee257f7b5ea8e9cb3f Mon Sep 17 00:00:00 2001 From: Jean-Philippe Sirois Date: Fri, 19 Dec 2025 19:05:13 +0400 Subject: [PATCH 23/29] fix: reuse existing promise on rerun --- src/remote/remote-controller.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/remote/remote-controller.ts b/src/remote/remote-controller.ts index a1962e3..7cb17f4 100644 --- a/src/remote/remote-controller.ts +++ b/src/remote/remote-controller.ts @@ -11,6 +11,7 @@ export class RemoteController { * Multi-tab support not currently available */ private socket?: WebSocket; + private syncResponse?: ReturnType; constructor( private readonly remote: Remote, @@ -55,8 +56,11 @@ export class RemoteController { const { db } = body.data; try { - const sync = await this.remote.syncFrom(db); - return Response.json(sync); + if (!this.syncResponse) { + this.syncResponse = this.remote.syncFrom(db); + } + + return Response.json(await this.syncResponse); } catch (error) { console.error(error); return Response.json({ From a5c45a7c924d5ed6878d17ea0e368e7f4ffed416 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Sirois Date: Fri, 19 Dec 2025 19:10:02 +0400 Subject: [PATCH 24/29] fix: disable formatter for now --- src/sql/recent-query.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/sql/recent-query.ts b/src/sql/recent-query.ts index 2e61cf3..76575db 100644 --- a/src/sql/recent-query.ts +++ b/src/sql/recent-query.ts @@ -1,4 +1,4 @@ -import { format } from "sql-formatter"; +// import { format } from "sql-formatter"; // deno-lint-ignore no-unused-vars import type { SegmentedQueryCache } from "../sync/seen-cache.ts"; import { @@ -38,11 +38,12 @@ export class RecentQuery { ) { this.username = data.username; this.query = data.query; - this.formattedQuery = format(data.query, { - language: "postgresql", - keywordCase: "lower", - linesBetweenQueries: 2, - }); + // this.formattedQuery = format(data.query, { + // language: "postgresql", + // keywordCase: "lower", + // linesBetweenQueries: 2, + // }); + this.formattedQuery = data.query; this.meanTime = data.meanTime; this.calls = data.calls; this.rows = data.rows; From 5e523d6d6aeb31e72bbc2bd8741238035c9571a1 Mon Sep 17 00:00:00 2001 From: Xetera Date: Mon, 22 Dec 2025 17:18:17 +0300 Subject: [PATCH 25/29] fix: always return query status even if response is cached --- src/remote/query-optimizer.ts | 51 ++++++++++++++++------------ src/remote/remote-controller.test.ts | 8 ++--- src/remote/remote-controller.ts | 23 ++++++++++--- src/remote/remote.dto.ts | 10 ++++-- src/remote/remote.test.ts | 19 +++++++++-- src/remote/remote.ts | 19 +++-------- src/sql/postgresjs.ts | 4 +++ src/sql/recent-query.ts | 6 +++- src/sync/connection-manager.ts | 9 +++++ 9 files changed, 95 insertions(+), 54 deletions(-) diff --git a/src/remote/query-optimizer.ts b/src/remote/query-optimizer.ts index a50eab9..a5bfdf5 100644 --- a/src/remote/query-optimizer.ts +++ b/src/remote/query-optimizer.ts @@ -19,7 +19,7 @@ const MINIMUM_COST_CHANGE_PERCENTAGE = 5; const QUERY_TIMEOUT_MS = 10000; type EventMap = { - error: [RecentQuery, Error]; + error: [Error, RecentQuery]; timeout: [RecentQuery, number]; zeroCostPlan: [RecentQuery]; queryUnsupported: [RecentQuery]; @@ -82,6 +82,23 @@ export class QueryOptimizer extends EventEmitter { statsMode: StatisticsMode = QueryOptimizer.defaultStatistics, ): Promise { this.stop(); + const validQueries: RecentQuery[] = []; + for (const query of allRecentQueries) { + let optimization: LiveQueryOptimization; + const status = this.checkQueryUnsupported(query); + switch (status.type) { + case "ok": + optimization = { state: "waiting" }; + break; + case "not_supported": + optimization = this.onQueryUnsupported(status.reason); + break; + case "ignored": + continue; + } + validQueries.push(query); + this.queries.set(query.hash, { query, optimization }); + } const version = PostgresVersion.parse("17"); const pg = this.manager.getOrCreateConnection(conn); const ownStats = await Statistics.dumpStats(pg, version, "full"); @@ -99,23 +116,6 @@ export class QueryOptimizer extends EventEmitter { }); this.target = { connectable: conn, optimizer, statistics }; - const validQueries: RecentQuery[] = []; - for (const query of allRecentQueries) { - let optimization: LiveQueryOptimization; - const status = this.checkQueryUnsupported(query); - switch (status.type) { - case "ok": - optimization = { state: "waiting" }; - break; - case "not_supported": - optimization = this.onQueryUnsupported(); - break; - case "ignored": - continue; - } - validQueries.push(query); - this.queries.set(query.hash, { query, optimization }); - } this._allQueries = this.queries.size; await this.work(); return validQueries; @@ -171,6 +171,10 @@ export class QueryOptimizer extends EventEmitter { } } + getQueries(): OptimizedQuery[] { + return Array.from(this.queries.values()); + } + // private summarizeQueue() { // let waitingQueries = 0; // let optimizingQueries = 0; @@ -319,7 +323,10 @@ export class QueryOptimizer extends EventEmitter { } } - private onNoImprovements(recent: RecentQuery, result: Extract) { + private onNoImprovements( + recent: RecentQuery, + result: Extract, + ) { this.emit("noImprovements", recent, result); } @@ -334,12 +341,12 @@ export class QueryOptimizer extends EventEmitter { ); } - private onQueryUnsupported(): LiveQueryOptimization { + private onQueryUnsupported(reason: string): LiveQueryOptimization { // this.emit("queryUnsupported", recent); this._invalidQueries++; return { state: "not_supported", - reason: "Query is not supported", + reason, }; } @@ -379,7 +386,7 @@ export class QueryOptimizer extends EventEmitter { errorMessage: string, ): LiveQueryOptimization { const error = new Error(errorMessage); - this.emit("error", recent, error); + this.emit("error", error, recent); return { state: "error", error }; } diff --git a/src/remote/remote-controller.test.ts b/src/remote/remote-controller.test.ts index e4a6563..663559e 100644 --- a/src/remote/remote-controller.test.ts +++ b/src/remote/remote-controller.test.ts @@ -7,6 +7,7 @@ import { RemoteController } from "./remote-controller.ts"; import { ConnectionManager } from "../sync/connection-manager.ts"; import { RemoteSyncRequest } from "./remote.dto.ts"; import { assertSpyCalls, spy } from "@std/testing/mock"; +import { setTimeout } from "node:timers/promises"; Deno.test({ name: "controller syncs correctly", @@ -60,9 +61,6 @@ Deno.test({ try { const ws = new WebSocket(`ws://localhost:${server.addr.port}/postgres`); const messageFunction = spy(); - ws.addEventListener("open", (event) => { - console.log("OPENED", event); - }); ws.addEventListener("error", console.log); ws.addEventListener("message", messageFunction); @@ -79,7 +77,9 @@ Deno.test({ ); assertEquals(response?.status, 200); + await setTimeout(1000); + console.log("a", await response.json()); const sql = postgres( target.withDatabaseName(Remote.optimizingDbName).toString(), ); @@ -91,10 +91,8 @@ Deno.test({ assertEquals(indexesAfter.count, 1); assertEquals(tablesAfter[0], { tablename: "testing" }); const rows = await sql`select * from testing`; - // expect no rows to have been synced assertEquals(rows.length, 0); - // exactly one query must have been processed assertSpyCalls(messageFunction, 1); } finally { await Promise.all([sourceDb.stop(), targetDb.stop(), server.shutdown()]); diff --git a/src/remote/remote-controller.ts b/src/remote/remote-controller.ts index 7cb17f4..da84190 100644 --- a/src/remote/remote-controller.ts +++ b/src/remote/remote-controller.ts @@ -59,8 +59,10 @@ export class RemoteController { if (!this.syncResponse) { this.syncResponse = this.remote.syncFrom(db); } + const { schema } = await this.syncResponse; + const queries = this.remote.optimizer.getQueries(); - return Response.json(await this.syncResponse); + return Response.json({ schema, queries: { type: "ok", value: queries } }); } catch (error) { console.error(error); return Response.json({ @@ -84,14 +86,25 @@ export class RemoteController { return response; } - private eventNoImprovementsAvailable(query: RecentQuery, result: Extract) { + private eventNoImprovementsAvailable( + query: RecentQuery, + result: Extract, + ) { const indexesUsed = Array.from(result.existingIndexes); this.socket?.send( - JSON.stringify({ type: "noImprovements", query, cost: result.baseCost, indexesUsed }), + JSON.stringify({ + type: "noImprovements", + query, + cost: result.baseCost, + indexesUsed, + }), ); } - private eventImprovementsAvailable(query: RecentQuery, result: Extract) { + private eventImprovementsAvailable( + query: RecentQuery, + result: Extract, + ) { const indexesUsed = Array.from(result.existingIndexes); const recommendedIndexes = Array.from(result.newIndexes) .map((n) => result.triedIndexes.get(n)?.definition) @@ -108,7 +121,7 @@ export class RemoteController { ); } - private eventError(recentQuery: RecentQuery, error: Error) { + private eventError(error: Error, recentQuery: RecentQuery) { this.socket?.send( JSON.stringify({ type: "error", diff --git a/src/remote/remote.dto.ts b/src/remote/remote.dto.ts index 8d39ec5..fddcc37 100644 --- a/src/remote/remote.dto.ts +++ b/src/remote/remote.dto.ts @@ -1,7 +1,7 @@ import { z } from "zod"; import { Connectable } from "../sync/connectable.ts"; -import { RecentQuery } from "../sql/recent-query.ts"; import { FullSchema } from "../sync/schema_differ.ts"; +import { OptimizedQuery } from "./query-optimizer.ts"; export const RemoteSyncRequest = z.codec( z.string(), @@ -22,16 +22,20 @@ export const RemoteSyncFullSchemaResponse = z.discriminatedUnion("type", [ z.object({ type: z.literal("error"), error: z.string() }), ]); +export type RemoteSyncFullSchemaResponse = z.infer< + typeof RemoteSyncFullSchemaResponse +>; + export const RemoteSyncQueriesResponse = z.discriminatedUnion("type", [ z.object({ type: z.literal("ok"), - value: z.array(z.instanceof(RecentQuery)), + value: z.array(OptimizedQuery), }), z.object({ type: z.literal("error"), error: z.string() }), ]); export const RemoteSyncResponse = z.object({ - queries: RemoteSyncQueriesResponse, + // queries: RemoteSyncQueriesResponse, schema: RemoteSyncFullSchemaResponse, }); diff --git a/src/remote/remote.test.ts b/src/remote/remote.test.ts index 765a87a..4f4fa6a 100644 --- a/src/remote/remote.test.ts +++ b/src/remote/remote.test.ts @@ -73,9 +73,9 @@ Deno.test({ ); const result = await remote.syncFrom(source); - assertOk(result.queries); + const optimizedQueries = remote.optimizer.getQueries(); - const queries = result.queries.value.map((f) => f.query); + const queries = optimizedQueries.map((f) => f.query.query); assertArrayIncludes(queries, [ "create table testing(a int, b text)", "select * from testing where a = $1", @@ -195,10 +195,15 @@ Deno.test({ .withEnvironment({ POSTGRES_HOST_AUTH_METHOD: "trust", }) + .withCommand([ + "-c", + "shared_preload_libraries=pg_stat_statements,timescaledb", + ]) .withLogConsumer((a) => a.pipe(process.stdout)) .withCopyContentToContainer([ { content: ` + create extension if not exists pg_stat_statements; create table conditions( "time" timestamptz not null, device_id integer, @@ -217,6 +222,7 @@ Deno.test({ min(temperature) from conditions group by device_id, bucket; + select * from conditions where time < now(); `, target: "/docker-entrypoint-initdb.d/init.sql", }, @@ -237,6 +243,12 @@ Deno.test({ targetConn.withDatabaseName(PgIdentifier.fromString("optimizing_db")), ); await remote.syncFrom(sourceConn); + const queries = remote.optimizer.getQueries(); + const queryStrings = queries.map((q) => q.query.query); + + assertArrayIncludes(queryStrings, [ + "select * from conditions where time < $1", + ]); const indexesAfter = await t.exec( "select indexname from pg_indexes where schemaname = 'public'", ); @@ -246,8 +258,9 @@ Deno.test({ "Indexes were not copied over correctly from the source db", ); - assertEquals(indexesAfter[0], { indexname: "testing_1234" }); + assertEquals(indexesAfter[0], { indexname: "conditions_time_idx" }); } finally { + await manager.closeAll(); await Promise.all([source.stop(), target.stop()]); } }, diff --git a/src/remote/remote.ts b/src/remote/remote.ts index a05a936..fdd33a4 100644 --- a/src/remote/remote.ts +++ b/src/remote/remote.ts @@ -10,7 +10,7 @@ import { DumpCommand, RestoreCommand } from "../sync/schema-link.ts"; import { ConnectionManager } from "../sync/connection-manager.ts"; import { type RecentQuery } from "../sql/recent-query.ts"; import { type FullSchema, SchemaDiffer } from "../sync/schema_differ.ts"; -import { type RemoteSyncResponse } from "./remote.dto.ts"; +import { type RemoteSyncFullSchemaResponse } from "./remote.dto.ts"; import { QueryOptimizer } from "./query-optimizer.ts"; /** @@ -57,7 +57,7 @@ export class Remote { async syncFrom( source: Connectable, statsStrategy: StatisticsStrategy = { type: "pullFromSource" }, - ): Promise { + ): Promise<{ schema: RemoteSyncFullSchemaResponse }> { await this.resetDatabase(); const [_restoreResult, recentQueries, fullSchema, pulledStats] = await Promise @@ -96,17 +96,6 @@ export class Remote { ); return { - queries: recentQueries.status === "fulfilled" - ? { - type: "ok", - value: recentQueries.value, - } - : { - type: "error", - error: recentQueries.reason instanceof Error - ? recentQueries.reason.message - : "Unknown error", - }, schema: fullSchema.status === "fulfilled" ? { type: "ok", value: fullSchema.value } : { @@ -184,11 +173,11 @@ export class Remote { return { kind: "fromStatisticsExport", source: { kind: "inline" }, stats }; } - private getRecentQueries( + private async getRecentQueries( source: Connectable, ): Promise { const connector = this.manager.getConnectorFor(source); - return connector.getRecentQueries(); + return await connector.getRecentQueries(); } private getFullSchema(source: Connectable): Promise { diff --git a/src/sql/postgresjs.ts b/src/sql/postgresjs.ts index 082055d..ad6c7e1 100644 --- a/src/sql/postgresjs.ts +++ b/src/sql/postgresjs.ts @@ -85,5 +85,9 @@ export function wrapGenericPostgresInterface(pg: postgres.Sql): Postgres { yield* row as T[]; } }, + // @ts-expect-error | this will be added to the pg interface later + close() { + return pg.end(); + }, }; } diff --git a/src/sql/recent-query.ts b/src/sql/recent-query.ts index 76575db..bf365f0 100644 --- a/src/sql/recent-query.ts +++ b/src/sql/recent-query.ts @@ -79,7 +79,11 @@ export class RecentQuery { } static isSystemQuery(referencedTables: string[]): boolean { - return referencedTables.some((table) => table.startsWith("pg_")); + return referencedTables.some((table) => + table.startsWith("pg_") || + /* timescaledb jobs */ + table.startsWith("bgw_job_stat_") + ); } static isIntrospection(data: RawRecentQuery): boolean { diff --git a/src/sync/connection-manager.ts b/src/sync/connection-manager.ts index f547d10..e1bea94 100644 --- a/src/sync/connection-manager.ts +++ b/src/sync/connection-manager.ts @@ -50,4 +50,13 @@ export class ConnectionManager { : input; return new PostgresConnector(sql, this.segmentedQueryCache); } + + async closeAll(): Promise { + const closePromises = Array.from(this.connections.values()).map((conn) => + // @ts-expect-error | this will exist later + conn.close() + ); + this.connections.clear(); + await Promise.all(closePromises); + } } From 9b17a5598d79c1ec8cf3f9cbfb0075743e45a0e2 Mon Sep 17 00:00:00 2001 From: Xetera Date: Mon, 22 Dec 2025 22:16:19 +0300 Subject: [PATCH 26/29] feat: attach optimization to the query object --- .env | 2 +- devenv.nix | 1 - src/remote/optimization.ts | 26 +++++ src/remote/query-optimizer.ts | 190 ++++++++++++-------------------- src/remote/remote-controller.ts | 87 +++------------ src/remote/remote.dto.ts | 4 +- src/remote/remote.test.ts | 4 +- src/sql/recent-query.ts | 16 ++- 8 files changed, 127 insertions(+), 203 deletions(-) create mode 100644 src/remote/optimization.ts diff --git a/.env b/.env index 64d1c05..ce52c9d 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ # DEBUG=true -POSTGRES_URL=postgres://postgres:123@localhost:5432/umami_dev +POSTGRES_URL=postgres://xetera@localhost:5432/postgres LOG_PATH=/tmp/postgres_logs/postgres.log # STATISTICS_PATH=test/umami_test.json STATISTICS_PATH=statistics.json diff --git a/devenv.nix b/devenv.nix index 06705a4..10c5386 100644 --- a/devenv.nix +++ b/devenv.nix @@ -1,7 +1,6 @@ { pkgs, ... }: { - env.CI = "false"; dotenv.enable = true; packages = with pkgs; [ git diff --git a/src/remote/optimization.ts b/src/remote/optimization.ts new file mode 100644 index 0000000..054269d --- /dev/null +++ b/src/remote/optimization.ts @@ -0,0 +1,26 @@ +import z from "zod"; + +export const LiveQueryOptimization = z.discriminatedUnion("state", [ + z.object({ + state: z.literal("waiting"), + }), + z.object({ state: z.literal("optimizing") }), + z.object({ state: z.literal("not_supported"), reason: z.string() }), + z.object({ + state: z.literal("improvements_available"), + cost: z.number(), + optimizedCost: z.number(), + costReductionPercentage: z.number(), + indexRecommendations: z.array(z.string()), + indexesUsed: z.array(z.string()), + }), + z.object({ + state: z.literal("no_improvement_found"), + cost: z.number(), + indexesUsed: z.array(z.string()), + }), + z.object({ state: z.literal("timeout") }), + z.object({ state: z.literal("error"), error: z.instanceof(Error) }), +]); + +export type LiveQueryOptimization = z.infer; diff --git a/src/remote/query-optimizer.ts b/src/remote/query-optimizer.ts index a5bfdf5..b75f41d 100644 --- a/src/remote/query-optimizer.ts +++ b/src/remote/query-optimizer.ts @@ -1,5 +1,6 @@ import EventEmitter from "node:events"; -import { QueryHash, RecentQuery } from "../sql/recent-query.ts"; +import { OptimizedQuery, QueryHash, RecentQuery } from "../sql/recent-query.ts"; +import type { LiveQueryOptimization } from "./optimization.ts"; import { ConnectionManager } from "../sync/connection-manager.ts"; import { Sema } from "async-sema"; import { @@ -13,18 +14,16 @@ import { } from "@query-doctor/core"; import { Connectable } from "../sync/connectable.ts"; import { parse } from "@libpg-query/parser"; -import z from "zod"; const MINIMUM_COST_CHANGE_PERCENTAGE = 5; const QUERY_TIMEOUT_MS = 10000; type EventMap = { - error: [Error, RecentQuery]; - timeout: [RecentQuery, number]; - zeroCostPlan: [RecentQuery]; - queryUnsupported: [RecentQuery]; - noImprovements: [RecentQuery, Extract]; - improvementsAvailable: [RecentQuery, Extract]; + error: [Error, OptimizedQuery]; + timeout: [OptimizedQuery, number]; + zeroCostPlan: [OptimizedQuery]; + noImprovements: [OptimizedQuery]; + improvementsAvailable: [OptimizedQuery]; }; type Target = { @@ -80,9 +79,9 @@ export class QueryOptimizer extends EventEmitter { conn: Connectable, allRecentQueries: RecentQuery[], statsMode: StatisticsMode = QueryOptimizer.defaultStatistics, - ): Promise { + ): Promise { this.stop(); - const validQueries: RecentQuery[] = []; + const validQueries: OptimizedQuery[] = []; for (const query of allRecentQueries) { let optimization: LiveQueryOptimization; const status = this.checkQueryUnsupported(query); @@ -96,8 +95,10 @@ export class QueryOptimizer extends EventEmitter { case "ignored": continue; } - validQueries.push(query); - this.queries.set(query.hash, { query, optimization }); + const optimized = query.withOptimization(optimization); + + validQueries.push(optimized); + this.queries.set(query.hash, optimized); } const version = PostgresVersion.parse("17"); const pg = this.manager.getOrCreateConnection(conn); @@ -137,37 +138,37 @@ export class QueryOptimizer extends EventEmitter { } while (true) { - let recentQuery: RecentQuery | undefined; + let optimized: OptimizedQuery | undefined; const token = await this.semaphore.acquire(); try { for (const [hash, entry] of this.queries.entries()) { if (entry.optimization.state !== "waiting") { continue; } - this.queries.set(hash, { - query: entry.query, - optimization: { state: "optimizing" }, - }); - recentQuery = entry.query; + this.queries.set( + hash, + entry.withOptimization({ state: "optimizing" }), + ); + optimized = entry; break; } } finally { this.semaphore.release(token); } - if (!recentQuery) { + if (!optimized) { this._finish.resolve(0); break; } this._validQueriesProcessed++; const optimization = await this.optimizeQuery( - recentQuery, + optimized, this.target, ); - this.queries.set(recentQuery.hash, { - query: recentQuery, - optimization, - }); + this.queries.set( + optimized.hash, + optimized.withOptimization(optimization), + ); } } @@ -175,44 +176,6 @@ export class QueryOptimizer extends EventEmitter { return Array.from(this.queries.values()); } - // private summarizeQueue() { - // let waitingQueries = 0; - // let optimizingQueries = 0; - // let improvementsAvailableQueries = 0; - // let noImprovementFoundQueries = 0; - // let timeoutQueries = 0; - // let errorQueries = 0; - // let notSupportedQueries = 0; - - // for (const [_hash, query] of this.queries.entries()) { - // if (query.optimization.state === "waiting") { - // waitingQueries++; - // } else if (query.optimization.state === "optimizing") { - // optimizingQueries++; - // } else if (query.optimization.state === "improvements_available") { - // improvementsAvailableQueries++; - // } else if (query.optimization.state === "no_improvement_found") { - // noImprovementFoundQueries++; - // } else if (query.optimization.state === "timeout") { - // timeoutQueries++; - // } else if (query.optimization.state === "error") { - // errorQueries++; - // } else if (query.optimization.state === "not_supported") { - // notSupportedQueries++; - // } - // } - // console.log("============"); - // console.log(`waiting: ${waitingQueries}`); - // console.log(`optimizing: ${optimizingQueries}`); - // console.log(`timeout: ${timeoutQueries}`); - // console.log(`error: ${errorQueries}`); - // console.log( - // `improvements: ${improvementsAvailableQueries}`, - // ); - // console.log(`no improvements: ${noImprovementFoundQueries}`); - // console.log("============"); - // } - private checkQueryUnsupported( query: RecentQuery, ): { type: "ok" } | { type: "ignored" } | { @@ -236,7 +199,7 @@ export class QueryOptimizer extends EventEmitter { } private async optimizeQuery( - recent: RecentQuery, + recent: OptimizedQuery, target: Target, timeoutMs = QUERY_TIMEOUT_MS, ): Promise { @@ -285,7 +248,7 @@ export class QueryOptimizer extends EventEmitter { private onOptimizeReady( result: OptimizeResult, - recent: RecentQuery, + recent: OptimizedQuery, ): LiveQueryOptimization { switch (result.kind) { case "ok": { @@ -299,7 +262,7 @@ export class QueryOptimizer extends EventEmitter { Math.abs(percentageReduction), ); if (costReductionPercentage < MINIMUM_COST_CHANGE_PERCENTAGE) { - this.onNoImprovements(recent, result); + this.onNoImprovements(recent, result.baseCost, indexesUsed); return { state: "no_improvement_found", cost: result.baseCost, @@ -324,15 +287,23 @@ export class QueryOptimizer extends EventEmitter { } private onNoImprovements( - recent: RecentQuery, - result: Extract, + recent: OptimizedQuery, + cost: number, + indexesUsed: string[], ) { - this.emit("noImprovements", recent, result); + this.emit( + "noImprovements", + recent.withOptimization({ + state: "no_improvement_found", + cost, + indexesUsed, + }), + ); } private getPotentialIndexCandidates( statistics: Statistics, - recent: RecentQuery, + recent: OptimizedQuery, ) { const analyzer = new Analyzer(parse); return analyzer.deriveIndexes( @@ -342,7 +313,6 @@ export class QueryOptimizer extends EventEmitter { } private onQueryUnsupported(reason: string): LiveQueryOptimization { - // this.emit("queryUnsupported", recent); this._invalidQueries++; return { state: "not_supported", @@ -351,27 +321,37 @@ export class QueryOptimizer extends EventEmitter { } private onImprovementsAvailable( - recent: RecentQuery, + recent: OptimizedQuery, result: Extract, ) { - this.emit("improvementsAvailable", recent, result); - this.queries.set(recent.hash, { - query: recent, - optimization: { - state: "improvements_available", - cost: result.baseCost, - optimizedCost: result.finalCost, - costReductionPercentage: 0, - indexRecommendations: [], - indexesUsed: [], - // costReductionPercentage, - // indexRecommendations, - // indexesUsed, - }, - }); + const optimized = recent.withOptimization( + this.resultToImprovementsAvailable(result), + ); + this.emit("improvementsAvailable", optimized); + this.queries.set( + optimized.hash, + optimized, + ); } - private onZeroCostPlan(recent: RecentQuery): LiveQueryOptimization { + private resultToImprovementsAvailable( + result: Extract, + ): LiveQueryOptimization { + const indexesUsed = Array.from(result.existingIndexes); + const indexRecommendations = Array.from(result.newIndexes) + .map((n) => result.triedIndexes.get(n)?.definition) + .filter((n) => n !== undefined); + return { + state: "improvements_available", + cost: result.baseCost, + optimizedCost: result.finalCost, + costReductionPercentage: 0, + indexRecommendations, + indexesUsed, + }; + } + + private onZeroCostPlan(recent: OptimizedQuery): LiveQueryOptimization { this.emit("zeroCostPlan", recent); return { state: "error", @@ -382,7 +362,7 @@ export class QueryOptimizer extends EventEmitter { } private onError( - recent: RecentQuery, + recent: OptimizedQuery, errorMessage: string, ): LiveQueryOptimization { const error = new Error(errorMessage); @@ -391,7 +371,7 @@ export class QueryOptimizer extends EventEmitter { } private onTimeout( - recent: RecentQuery, + recent: OptimizedQuery, waitedMs: number, ): LiveQueryOptimization { this.emit("timeout", recent, waitedMs); @@ -440,35 +420,3 @@ export function costDifferencePercentage( ): PercentageDifference { return ((newVal - oldVal) / oldVal) * 100; } - -export const LiveQueryOptimization = z.discriminatedUnion("state", [ - z.object({ - state: z.literal("waiting"), - }), - z.object({ state: z.literal("optimizing") }), - z.object({ state: z.literal("not_supported"), reason: z.string() }), - z.object({ - state: z.literal("improvements_available"), - cost: z.number(), - optimizedCost: z.number(), - costReductionPercentage: z.number(), - indexRecommendations: z.array(z.string()), - indexesUsed: z.array(z.string()), - }), - z.object({ - state: z.literal("no_improvement_found"), - cost: z.number(), - indexesUsed: z.array(z.string()), - }), - z.object({ state: z.literal("timeout") }), - z.object({ state: z.literal("error"), error: z.instanceof(Error) }), -]); - -export type LiveQueryOptimization = z.infer; - -export const OptimizedQuery = z.object({ - query: z.instanceof(RecentQuery), - optimization: LiveQueryOptimization, -}); - -export type OptimizedQuery = z.infer; diff --git a/src/remote/remote-controller.ts b/src/remote/remote-controller.ts index da84190..81f727d 100644 --- a/src/remote/remote-controller.ts +++ b/src/remote/remote-controller.ts @@ -1,9 +1,8 @@ import { env } from "../env.ts"; -import { RecentQuery } from "../sql/recent-query.ts"; +import { OptimizedQuery } from "../sql/recent-query.ts"; import { QueryOptimizer } from "./query-optimizer.ts"; import { RemoteSyncRequest } from "./remote.dto.ts"; import { Remote } from "./remote.ts"; -import { OptimizeResult } from "@query-doctor/core"; export class RemoteController { /** @@ -34,18 +33,14 @@ export class RemoteController { } private hookUpWebsockets(optimizer: QueryOptimizer) { - optimizer.on( - "noImprovements", - this.eventNoImprovementsAvailable.bind(this), - ); - optimizer.on( - "improvementsAvailable", - this.eventImprovementsAvailable.bind(this), - ); - optimizer.on("error", this.eventError.bind(this)); - optimizer.on("timeout", this.eventTimeout.bind(this)); - optimizer.on("zeroCostPlan", this.eventZeroCostPlan.bind(this)); - optimizer.on("queryUnsupported", this.eventQueryUnsupported.bind(this)); + const onQueryProcessed = this.eventOnQueryProcessed.bind(this); + const onError = this.eventError.bind(this); + optimizer.on("noImprovements", onQueryProcessed); + optimizer.on("improvementsAvailable", onQueryProcessed); + optimizer.on("error", onError); + optimizer.on("timeout", onQueryProcessed); + optimizer.on("zeroCostPlan", onQueryProcessed); + optimizer.on("queryUnsupported", onQueryProcessed); } private async onFullSync(request: Request): Promise { @@ -86,70 +81,20 @@ export class RemoteController { return response; } - private eventNoImprovementsAvailable( - query: RecentQuery, - result: Extract, - ) { - const indexesUsed = Array.from(result.existingIndexes); - this.socket?.send( - JSON.stringify({ - type: "noImprovements", - query, - cost: result.baseCost, - indexesUsed, - }), - ); - } - - private eventImprovementsAvailable( - query: RecentQuery, - result: Extract, - ) { - const indexesUsed = Array.from(result.existingIndexes); - const recommendedIndexes = Array.from(result.newIndexes) - .map((n) => result.triedIndexes.get(n)?.definition) - .filter((n) => n !== undefined); - this.socket?.send( - JSON.stringify({ - type: "improvementsAvailable", - query, - cost: result.baseCost, - optimizedCost: result.finalCost, - indexesUsed, - recommendedIndexes, - }), - ); + private eventOnQueryProcessed(query: OptimizedQuery) { + this.socket?.send(JSON.stringify({ + type: "queryProcessed", + query, + })); } - private eventError(error: Error, recentQuery: RecentQuery) { + private eventError(error: Error, query: OptimizedQuery) { this.socket?.send( JSON.stringify({ type: "error", - query: recentQuery, + query, error: error.message, }), ); } - - private eventTimeout(recentQuery: RecentQuery, waitedMs: number) { - this.socket?.send( - JSON.stringify({ - type: "timeout", - query: recentQuery, - waitTimeMs: waitedMs, - }), - ); - } - - private eventZeroCostPlan(recentQuery: RecentQuery) { - this.socket?.send( - JSON.stringify({ type: "zeroCostPlan", query: recentQuery }), - ); - } - - private eventQueryUnsupported(query: RecentQuery) { - this.socket?.send( - JSON.stringify({ type: "queryUnsupported", query }), - ); - } } diff --git a/src/remote/remote.dto.ts b/src/remote/remote.dto.ts index fddcc37..305a259 100644 --- a/src/remote/remote.dto.ts +++ b/src/remote/remote.dto.ts @@ -1,7 +1,7 @@ import { z } from "zod"; import { Connectable } from "../sync/connectable.ts"; import { FullSchema } from "../sync/schema_differ.ts"; -import { OptimizedQuery } from "./query-optimizer.ts"; +import { OptimizedQuery } from "../sql/recent-query.ts"; export const RemoteSyncRequest = z.codec( z.string(), @@ -29,7 +29,7 @@ export type RemoteSyncFullSchemaResponse = z.infer< export const RemoteSyncQueriesResponse = z.discriminatedUnion("type", [ z.object({ type: z.literal("ok"), - value: z.array(OptimizedQuery), + value: z.array(z.custom()), }), z.object({ type: z.literal("error"), error: z.string() }), ]); diff --git a/src/remote/remote.test.ts b/src/remote/remote.test.ts index 4f4fa6a..7071de3 100644 --- a/src/remote/remote.test.ts +++ b/src/remote/remote.test.ts @@ -75,7 +75,7 @@ Deno.test({ const result = await remote.syncFrom(source); const optimizedQueries = remote.optimizer.getQueries(); - const queries = optimizedQueries.map((f) => f.query.query); + const queries = optimizedQueries.map((f) => f.query); assertArrayIncludes(queries, [ "create table testing(a int, b text)", "select * from testing where a = $1", @@ -244,7 +244,7 @@ Deno.test({ ); await remote.syncFrom(sourceConn); const queries = remote.optimizer.getQueries(); - const queryStrings = queries.map((q) => q.query.query); + const queryStrings = queries.map((q) => q.query); assertArrayIncludes(queryStrings, [ "select * from conditions where time < $1", diff --git a/src/sql/recent-query.ts b/src/sql/recent-query.ts index bf365f0..3b9afc9 100644 --- a/src/sql/recent-query.ts +++ b/src/sql/recent-query.ts @@ -8,6 +8,7 @@ import { } from "@query-doctor/core"; import { parse } from "@libpg-query/parser"; import z from "zod"; +import type { LiveQueryOptimization } from "../remote/optimization.ts"; /** * Constructed by syncing with {@link SegmentedQueryCache.sync} @@ -38,11 +39,6 @@ export class RecentQuery { ) { this.username = data.username; this.query = data.query; - // this.formattedQuery = format(data.query, { - // language: "postgresql", - // keywordCase: "lower", - // linesBetweenQueries: 2, - // }); this.formattedQuery = data.query; this.meanTime = data.meanTime; this.calls = data.calls; @@ -57,6 +53,12 @@ export class RecentQuery { : false; } + withOptimization( + optimization: LiveQueryOptimization, + ): OptimizedQuery { + return Object.assign(this, { optimization }); + } + static async analyze( data: RawRecentQuery, hash: QueryHash, @@ -107,5 +109,9 @@ export type RawRecentQuery = { topLevel: boolean; }; +export type OptimizedQuery = RecentQuery & { + optimization: LiveQueryOptimization; +}; + export const QueryHash = z.string().brand<"QueryHash">(); export type QueryHash = z.infer; From 410d50bfacec766eeb0aa62b18e67c6361d3528a Mon Sep 17 00:00:00 2001 From: Xetera Date: Tue, 23 Dec 2025 12:13:22 +0300 Subject: [PATCH 27/29] fix: remove unused queryUnsupported --- src/remote/remote-controller.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/remote/remote-controller.ts b/src/remote/remote-controller.ts index 81f727d..f62ca49 100644 --- a/src/remote/remote-controller.ts +++ b/src/remote/remote-controller.ts @@ -40,7 +40,6 @@ export class RemoteController { optimizer.on("error", onError); optimizer.on("timeout", onQueryProcessed); optimizer.on("zeroCostPlan", onQueryProcessed); - optimizer.on("queryUnsupported", onQueryProcessed); } private async onFullSync(request: Request): Promise { From eab6e0aca859b4ca5eabbc434a673218f48e9cf3 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Sirois Date: Sat, 20 Dec 2025 10:44:56 +0400 Subject: [PATCH 28/29] fix: reset optimizer on connecting to socket --- src/remote/remote-controller.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/remote/remote-controller.ts b/src/remote/remote-controller.ts index f62ca49..9f249fe 100644 --- a/src/remote/remote-controller.ts +++ b/src/remote/remote-controller.ts @@ -73,6 +73,10 @@ export class RemoteController { console.log({ socket }); this.socket = socket; + socket.addEventListener("open", () => { + this.syncResponse = undefined; + }); + socket.addEventListener("close", () => { this.socket = undefined; }); From 7911fe05e47a07099cb33708f61a286d3c2cce95 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Sirois Date: Wed, 24 Dec 2025 09:55:42 +0400 Subject: [PATCH 29/29] feat: add nudges to queries --- src/sql/recent-query.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sql/recent-query.ts b/src/sql/recent-query.ts index 3b9afc9..dfd520c 100644 --- a/src/sql/recent-query.ts +++ b/src/sql/recent-query.ts @@ -4,6 +4,7 @@ import type { SegmentedQueryCache } from "../sync/seen-cache.ts"; import { Analyzer, DiscoveredColumnReference, + Nudge, SQLCommenterTag, } from "@query-doctor/core"; import { parse } from "@libpg-query/parser"; @@ -34,6 +35,7 @@ export class RecentQuery { readonly tableReferences: string[], readonly columnReferences: DiscoveredColumnReference[], readonly tags: SQLCommenterTag[], + readonly nudges: Nudge[], readonly hash: QueryHash, readonly seenAt: number, ) { @@ -71,6 +73,7 @@ export class RecentQuery { analysis.referencedTables, analysis.indexesToCheck, analysis.tags, + analysis.nudges, hash, seenAt, );