From 074909409afc3bf2b3faa5d6e74b94d1341f1452 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 13 May 2026 16:42:23 +0200 Subject: [PATCH] fix: upgrade pg package - Update pg and pg-protocol to latest version. This fix an issue with password handling in connectionString containing utf-8 chars in it --- CLAUDE.md | 205 ++++++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 117 ++++++++------------------ package.json | 6 +- 3 files changed, 243 insertions(+), 85 deletions(-) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..85e460f8 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,205 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Overview + +postgres-meta is a RESTful API for managing PostgreSQL databases. It provides a normalized interface over the PostgreSQL system catalog, exposing database objects (tables, columns, functions, etc.) through a Fastify-based REST API. The library can be used both as a standalone server and as an npm package. + +## Development Commands + +### Development Server +```bash +npm run dev # Start dev server with Docker DB (auto-cleans on exit) +npm run dev:code # Start dev server without DB setup (if DB already running) +``` + +The dev server uses nodemon with ts-node/esm loader and pipes output through pino-pretty for readable logs. + +### Building +```bash +npm run build # Compile TypeScript + copy SQL files to dist/ +npm run clean # Remove dist/ and tsconfig.tsbuildinfo +npm run check # Type-check without emitting files +``` + +Build process: TypeScript compilation (tsc) + copying `src/lib/sql/*.sql` files to `dist/lib/sql/` via cpy-cli. + +### Testing +```bash +npm test # Full test: db:clean -> db:run -> test:run -> db:clean +npm run test:run # Run tests only (DB must be running) +npm run test:update # Update test snapshots (runs db:clean -> db:run first) +``` + +Tests use Vitest with snapshot testing. Test files are split between `test/lib/` (library tests) and `test/server/` (API tests). All tests import from `test/index.test.ts` which orchestrates test execution. + +**Important**: Tests require Docker. The test DB is managed via `test/db/docker-compose.yml` and runs on port 5432. Tests run sequentially (`maxConcurrency: 1`) and use `pool: 'forks'` to avoid memory issues. + +### Database Management +```bash +npm run db:run # Start test DB in Docker (detached, with healthcheck) +npm run db:clean # Stop and remove test DB containers +``` + +### Type Generation +```bash +npm run gen:types:typescript # Generate TypeScript types from DB schema +npm run gen:types:python # Generate Python types (Pydantic models) +npm run gen:types:go # Generate Go types +npm run gen:types:swift # Generate Swift types (beta) + +# With custom DB connection: +PG_META_DB_URL=postgresql://... npm run gen:types:typescript +``` + +Type generation is controlled by `PG_META_GENERATE_TYPES` env var and runs the server in special mode (exits after generating types to stdout). + +### Code Quality +```bash +npm run format # Format code with Prettier +``` + +## Architecture + +### Two-Layer Design + +1. **Library Layer** (`src/lib/`): Core PostgreSQL introspection logic + - `PostgresMeta.ts`: Main class that aggregates all metadata managers + - `PostgresMeta*.ts`: Individual managers for each database object type (columns, tables, functions, etc.) + - `sql/*.sql.ts`: SQL query templates for fetching metadata from system catalogs + - `Parser.ts`: SQL parsing, formatting, and deparsing (uses pgsql-parser) + - `db.ts`: Connection pooling and query execution with error handling + +2. **Server Layer** (`src/server/`): REST API built with Fastify + - `server.ts`: Entry point, handles normal server mode + type generation mode + - `app.ts`: Main Fastify app with routes, CORS, Swagger docs + - `admin-app.ts`: Admin server (runs on PG_META_PORT + 1) for metrics + - `routes/*.ts`: REST endpoints mapping to library methods + - `templates/*.ts`: Type generation templates for different languages + +### Object Manager Pattern + +Each PostgreSQL object type has a dedicated manager class following this pattern: +- `PostgresMetaTables`, `PostgresMetaColumns`, `PostgresMetaFunctions`, etc. +- Each manager has methods: `list()`, `retrieve()`, `create()`, `update()`, `remove()` +- Managers compose SQL from `src/lib/sql/*.sql.ts` templates +- All methods return `PostgresMetaResult` = `{ data: T | null, error: Error | null }` + +### SQL Query Organization + +SQL queries are defined in `src/lib/sql/*.sql.ts` as TypeScript template literals: +- Queries use `pg-format` for safe parameterization +- Complex queries join against `pg_catalog` and `information_schema` +- Build process copies SQL files to `dist/lib/sql/` for runtime access + +### Connection Pooling + +- Uses custom `@supabase/pg` fork (v0.0.3) instead of standard `pg` +- Connection pool initialized in `db.ts` via `init(config)` +- Pool configuration supports: + - `connectionTimeoutMillis`: Connection timeout (default 15s) + - `query_timeout`: Query timeout (default 55s) + - `maxResultSize`: Max result size in bytes (default 2GB, configurable via `PG_META_MAX_RESULT_SIZE_MB`) + - SSL configuration via `PG_META_DB_SSL_ROOT_CERT` +- Custom error handling for `RESULT_SIZE_EXCEEDED` errors +- Pool auto-reconnects on connection errors by calling `pool.end()` and reinitializing + +### Error Handling + +Database errors are formatted to mimic `psql` output: +- Includes severity, error code, message, line number, position marker +- Position calculation accounts for injected `SET statement_timeout` prefix +- Returns structured errors: `{ code, message, formattedError, position, detail, hint }` + +### Type Generation + +Type generation (`npm run gen:types:*`) works by: +1. Connecting to a database (test DB or custom via `PG_META_DB_URL`) +2. Fetching all schemas, tables, columns, relationships, functions, types +3. Passing data to language-specific templates in `src/server/templates/*.ts` +4. Templates output type definitions to stdout + +Environment variables: +- `PG_META_GENERATE_TYPES`: Language (typescript, python, go, swift) +- `PG_META_GENERATE_TYPES_INCLUDED_SCHEMAS`: Comma-separated schemas to include +- `PG_META_GENERATE_TYPES_DETECT_ONE_TO_ONE_RELATIONSHIPS`: Enable 1:1 relationship detection +- `PG_META_POSTGREST_VERSION`: PostgREST version for TypeScript template compatibility + +## Environment Variables + +Required for server operation: +```bash +PG_META_HOST=0.0.0.0 # Server host +PG_META_PORT=8080 # Server port (admin runs on +1) +PG_META_DB_HOST=localhost # PostgreSQL host +PG_META_DB_NAME=postgres # Database name +PG_META_DB_USER=postgres # Database user +PG_META_DB_PORT=5432 # PostgreSQL port +PG_META_DB_PASSWORD=postgres # Database password (or use CRYPTO_KEY for secrets) +PG_META_DB_SSL_MODE=disable # SSL mode (disable, require, verify-full, etc.) +``` + +Alternative connection: +```bash +PG_META_DB_URL=postgresql://... # Full connection string (overrides individual params) +``` + +Performance tuning: +```bash +PG_CONN_TIMEOUT_SECS=15 # Connection timeout (default: 15) +PG_QUERY_TIMEOUT_SECS=55 # Query timeout (default: 55) +PG_META_MAX_RESULT_SIZE_MB=20 # Max query result size in MB (default: 2048MB) +PG_META_MAX_BODY_LIMIT_MB=3 # Max request body size in MB (default: 3MB) +``` + +## Testing Notes + +### Snapshot Testing +Tests use Vitest inline snapshots (`toMatchInlineSnapshot`). When fixing bugs: +1. Add test case reproducing the bug +2. Fix the bug +3. Run `npm run test:update` (adds `-u` flag to vitest) +4. Review git diff of snapshots - `id` field changes are expected +5. Remove `-u` flag before committing (or use `npm run test` directly) + +### Test Structure +- `test/lib/*.ts`: Direct library tests using `pgMeta` instance +- `test/server/*.ts`: HTTP API tests using Fastify test utils +- `test/index.test.ts`: Main entry point that imports all test modules +- `test/db/`: Docker Compose config for test database with SSL enabled + +### Custom Database Connection +To test against a different database: +```bash +PG_META_DB_URL=postgresql://user:pass@host:port/dbname npm run dev:code +``` + +## Module System + +This project uses **ESM** (ES Modules): +- `"type": "module"` in package.json +- Import statements use `.js` extension (TypeScript convention for ESM) +- Node >=20 required +- ts-node runs with `--loader ts-node/esm` flag +- Uses import attributes for JSON: `import pkg from '#package.json' with { type: 'json' }` + +## Special Imports + +The `#package.json` import is defined in package.json `imports` field: +```json +"imports": { + "#package.json": "./package.json" +} +``` + +This allows importing package.json metadata in ESM without path resolution issues. + +## OpenAPI Documentation + +Generate OpenAPI spec: +```bash +npm run docs:export > openapi.json +``` + +Uses Fastify's `@fastify/swagger` plugin. Routes define schemas using `@sinclair/typebox` for automatic OpenAPI generation. diff --git a/package-lock.json b/package-lock.json index e8fe798a..220f2625 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,10 +19,10 @@ "crypto-js": "^4.0.0", "fastify": "^4.24.3", "fastify-metrics": "^10.0.0", - "pg": "npm:@supabase/pg@0.0.3", - "pg-connection-string": "^2.7.0", + "pg": "npm:@supabase/pg@8.21.1", + "pg-connection-string": "^2.12.0", "pg-format": "^1.0.4", - "pg-protocol": "npm:@supabase/pg-protocol@0.0.2", + "pg-protocol": "npm:@supabase/pg-protocol@1.13.2", "pgsql-parser": "^17.8.2", "pino": "^9.5.0", "postgres-array": "^3.0.1", @@ -840,7 +840,6 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=8.0.0" } @@ -862,7 +861,6 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-1.30.1.tgz", "integrity": "sha512-s5vvxXPVdjqS3kTLKMeBMvop9hbWkwzBpu+mUO2M7sZtlkyDJGwFe33wRKnbaYDo8ExRVBIIdwIGrqpxHuKttA==", "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=14" }, @@ -899,7 +897,6 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.57.2.tgz", "integrity": "sha512-BdBGhQBh8IjZ2oIIX6F2/Q3LKm/FDDKi6ccYKcBTeilh6SNdNKveDOLk73BkSJjQLJk6qe4Yh+hHw1UPhCDdrg==", "license": "Apache-2.0", - "peer": true, "dependencies": { "@opentelemetry/api-logs": "0.57.2", "@types/shimmer": "^1.2.0", @@ -1439,7 +1436,6 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.32.0.tgz", "integrity": "sha512-s0OpmpQFSfMrmedAn9Lhg4KWJELHCU6uU9dtIJ28N8UGhf9Y55im5X8fEzwhwDwiSqN+ZPSNrDJF7ivf/AuRPQ==", "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=14" } @@ -1877,8 +1873,7 @@ "version": "0.31.28", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.31.28.tgz", "integrity": "sha512-/s55Jujywdw/Jpan+vsy6JZs1z2ZTGxTmbZTPiuSL2wz9mfzA2gN1zzaqmvfi4pq+uOt7Du85fkiwv5ymW84aQ==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@tsconfig/node10": { "version": "1.0.11", @@ -1945,7 +1940,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.28.tgz", "integrity": "sha512-DHlH/fNL6Mho38jTy7/JT7sn2wnXI+wULR6PV4gy4VHLVvnrV/d3pHAMQHhc4gjdLmK2ZiPoMxzp6B3yRajLSQ==", "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~6.19.2" } @@ -2149,7 +2143,6 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz", "integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==", "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3298,7 +3291,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "@fastify/ajv-compiler": "^3.5.0", "@fastify/error": "^3.4.0", @@ -5180,22 +5172,22 @@ }, "node_modules/pg": { "name": "@supabase/pg", - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/@supabase/pg/-/pg-0.0.3.tgz", - "integrity": "sha512-WW4VdNYQocmcg7dZYk92vHY2nhhMhxoJhZ20m7PzuKe8p4vSdkv2tSM8HUCpZDiLxC6djfVeMk39ukcwEpFdzg==", + "version": "8.21.1", + "resolved": "https://registry.npmjs.org/@supabase/pg/-/pg-8.21.1.tgz", + "integrity": "sha512-FAdaGrfvEkGnMMxaHlhXX0gpuyDP9cp5y68B+ym78Q//6TRnbT3V/laajXxtfCy8BAFYe3oADHCHbEdVXkL2gA==", "license": "MIT", "dependencies": { - "pg-connection-string": "^2.7.0", - "pg-pool": "^3.8.0", - "pg-protocol": "npm:@supabase/pg-protocol@^0.0.2", - "pg-types": "^2.1.0", - "pgpass": "1.x" + "pg-connection-string": "^2.12.0", + "pg-pool": "^3.13.0", + "pg-protocol": "npm:@supabase/pg-protocol@^1.13.1", + "pg-types": "2.2.0", + "pgpass": "1.0.5" }, "engines": { - "node": ">= 8.0.0" + "node": ">= 16.0.0" }, "optionalDependencies": { - "pg-cloudflare": "^1.1.1" + "pg-cloudflare": "^1.3.0" }, "peerDependencies": { "pg-native": ">=3.0.1" @@ -5207,16 +5199,16 @@ } }, "node_modules/pg-cloudflare": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", - "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", + "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", "license": "MIT", "optional": true }, "node_modules/pg-connection-string": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.7.0.tgz", - "integrity": "sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA==", + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.12.0.tgz", + "integrity": "sha512-U7qg+bpswf3Cs5xLzRqbXbQl85ng0mfSV/J0nnA31MCLgvEaAo7CIhmeyrmJpOr7o+zm0rXK+hNnT5l9RHkCkQ==", "license": "MIT" }, "node_modules/pg-format": { @@ -5246,11 +5238,20 @@ "node": ">=4" } }, + "node_modules/pg-pool": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.13.0.tgz", + "integrity": "sha512-gB+R+Xud1gLFuRD/QgOIgGOBE2KCQPaPwkzBBGC9oG69pHTkhQeIuejVIk3/cnDyX39av2AxomQiyPT13WKHQA==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, "node_modules/pg-protocol": { "name": "@supabase/pg-protocol", - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/@supabase/pg-protocol/-/pg-protocol-0.0.2.tgz", - "integrity": "sha512-OLp5LeWRmfJy4vwV753hsCE90vzNSTSAwhsbinCIeoT455DHBufrkktVc4YvPXYEFj+EzpVKw/N0piX+AvBMBg==", + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@supabase/pg-protocol/-/pg-protocol-1.13.2.tgz", + "integrity": "sha512-VIA/Uh1xtKqQ/agrfZQPp1dJf448NM7ku7OjMGKmEwKOvBAjdUM4gKftSm08l9ZBJk1h3rpOJGaYWd6EgdGKTw==", "license": "MIT" }, "node_modules/pg-types": { @@ -5271,42 +5272,6 @@ "node": ">=10" } }, - "node_modules/pg/node_modules/pg": { - "version": "8.14.1", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.14.1.tgz", - "integrity": "sha512-0TdbqfjwIun9Fm/r89oB7RFQ0bLgduAhiIqIXOsyKoiC/L54DbuAAzIEN/9Op0f1Po9X7iCPXGoa/Ah+2aI8Xw==", - "license": "MIT", - "dependencies": { - "pg-connection-string": "^2.7.0", - "pg-pool": "^3.8.0", - "pg-protocol": "^1.8.0", - "pg-types": "^2.1.0", - "pgpass": "1.x" - }, - "engines": { - "node": ">= 8.0.0" - }, - "optionalDependencies": { - "pg-cloudflare": "^1.1.1" - }, - "peerDependencies": { - "pg-native": ">=3.0.1" - }, - "peerDependenciesMeta": { - "pg-native": { - "optional": true - } - } - }, - "node_modules/pg/node_modules/pg-pool": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.8.0.tgz", - "integrity": "sha512-VBw3jiVm6ZOdLBTIcXLNdSotb6Iy3uOCwDGFAksZCXmi10nyRvnP2v3jl4d+IsLYRyXf6o9hIm/ZtUzlByNUdw==", - "license": "MIT", - "peerDependencies": { - "pg": ">=8.0" - } - }, "node_modules/pg/node_modules/pg-types": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", @@ -5323,12 +5288,6 @@ "node": ">=4" } }, - "node_modules/pg/node_modules/pg/node_modules/pg-protocol": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.8.0.tgz", - "integrity": "sha512-jvuYlEkL03NRvOoyoRktBK7+qU5kOvlAwvmrH8sr3wbLrOdVWsRxQfz8mMy9sZFsqJ1hEWNfdWKI4SAmoL+j7g==", - "license": "MIT" - }, "node_modules/pg/node_modules/postgres-array": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", @@ -5339,9 +5298,9 @@ } }, "node_modules/pg/node_modules/postgres-bytea": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", - "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", "license": "MIT", "engines": { "node": ">=0.10.0" @@ -5624,7 +5583,6 @@ "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.3.tgz", "integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==", "license": "MIT", - "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -6767,7 +6725,6 @@ "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -6971,7 +6928,6 @@ "integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -7045,7 +7001,6 @@ "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", @@ -7159,7 +7114,6 @@ "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -7173,7 +7127,6 @@ "integrity": "sha512-BbcFDqNyBlfSpATmTtXOAOj71RNKDDvjBM/uPfnxxVGrG+FSH2RQIwgeEngTaTkuU/h0ScFvf+tRcKfYXzBybQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@vitest/expect": "3.0.9", "@vitest/mocker": "3.0.9", diff --git a/package.json b/package.json index 58b95839..ed9b8ae2 100644 --- a/package.json +++ b/package.json @@ -52,10 +52,10 @@ "crypto-js": "^4.0.0", "fastify": "^4.24.3", "fastify-metrics": "^10.0.0", - "pg": "npm:@supabase/pg@0.0.3", - "pg-connection-string": "^2.7.0", + "pg": "npm:@supabase/pg@8.21.1", + "pg-connection-string": "^2.12.0", "pg-format": "^1.0.4", - "pg-protocol": "npm:@supabase/pg-protocol@0.0.2", + "pg-protocol": "npm:@supabase/pg-protocol@1.13.2", "pgsql-parser": "^17.8.2", "pino": "^9.5.0", "postgres-array": "^3.0.1",