diff --git a/README.md b/README.md
index ba3ecda..7c42f7c 100644
--- a/README.md
+++ b/README.md
@@ -49,7 +49,7 @@ The data dictionary functionality is based on [pgdd](https://github.com/rustproo
These include COMMENTS and are helpful for understanding the structure
of your database, from a data modeling lens.
-| Function | Scenario |
+| Function | Scenario |
|---------|---------|
| `columns` | List all database column objects |
| `databases` | List all databases |
@@ -64,12 +64,13 @@ of your database, from a data modeling lens.
To get a full map of data objects, use `(read-data-dictionary db)` which
returns a map, with keywords mirroring the above functions.
+
### 🛠️ Operational Diagnostics
Diagnostic stats based on [ecto_psql_extras](https://github.com/pawurb/ecto_psql_extras/tree/main).
These are valuable for looking at your database through an operations or DBA lens.
-| Function | Scenario |
+| Function | Scenario |
|---------|---------|
| `all-locks` | Queries with active locks |
| `bloat` | Table and index "bloat" in your database ordered by most wasteful |
@@ -106,6 +107,7 @@ Use the `(diagnose (read-stats db))` and `(diagnose-warnings (read-stats db))` f
to evaluate the stats according to a set of heuristics.
+
## Usage
Check out the [examples](./examples/pgbench_tutorial.clj) if you're looking to create a fresh namespace.
@@ -138,6 +140,7 @@ Do a quick health check
; :version "PostgreSQL 16.1 (Debian 16.1-1.pgdg110+1) on x86_64-pc-linux-gnu..."}
```
+
Generate a data dictionary summarizing all major objects in your database.
```clojure
@@ -169,6 +172,7 @@ Generate a data dictionary summarizing all major objects in your database.
; :bytes_per_row 16384}
```
+
Create a full map of diagnostic stats.
```clojure
@@ -207,6 +211,7 @@ Create a full map of diagnostic stats.
; :application_name "psql"}
```
+
All of the stats and data dictionary keywords mirror the name of a public function in the
`postgres-extras-clj.core` namespace so you can invoke them selectively,
instead of getting them from the full map.
@@ -263,27 +268,20 @@ To create your own diagnostics:
; ... many more
```
-## Development
-
-Test runner with coverage
-
- clj -X:test
-
-Run NREPL and interactive terminal REPL in one
-
- clj -M:dev
-
-
-Build a jar. Output in `./target/com.github.perrygeo/postgres-extras-clj-*.jar`
-
- clj -T:build jar
+## Development
-Deploy to Clojars.
-Set `CLOJARS_USERNAME` and `CLOJARS_PASSWORD` env vars.
-Assumes that `clj -T:build jar` has already been run.
+```
+$ bb tasks
+The following tasks are available:
+
+test Run tests with coverage
+dev-db Start a develpment Postgres database
+nrepl Start nREPL server for development
+build Build JAR file
+deploy Deploy JAR after running tests
+```
- clj -T:build deploy
## License
diff --git a/bb.edn b/bb.edn
new file mode 100644
index 0000000..3667236
--- /dev/null
+++ b/bb.edn
@@ -0,0 +1,18 @@
+{:paths ["src"]
+ :tasks
+ {test {:doc "Run tests"
+ :task (shell "clojure -X:test")}
+
+ dev-db {:doc "Start a develpment Postgres database"
+ :task (shell {:dir "dev/infra"} "docker compose up")}
+
+ nrepl {:doc "Start nREPL server for development"
+ :task (shell "clj -M:dev")}
+
+ build {:doc "Build JAR file"
+ :task (shell "clj -T:build jar")}
+
+ deploy {:doc "Deploy JAR after running tests"
+ ; TODO Set `CLOJARS_USERNAME` and `CLOJARS_PASSWORD` env vars.
+ :depends [test build]
+ :task (shell "clj -T:build deploy")}}}
diff --git a/deps.edn b/deps.edn
index 027bdbe..97b0c60 100644
--- a/deps.edn
+++ b/deps.edn
@@ -1,39 +1,35 @@
{:paths ["src" "resources"]
:deps {com.layerware/hugsql {:mvn/version "0.5.3"}
- org.postgresql/postgresql {:mvn/version "42.7.3"}
+ org.postgresql/postgresql {:mvn/version "42.7.7"}
org.clojure/tools.logging {:mvn/version "1.3.0"}}
:aliases
- {;;
- ;; $ clj -X:test # Test runner with embedded postgres and coverage
- ;;
+ {;; $ clj -X:test # Test runner with embedded postgres and coverage
:test {:extra-deps {com.layerware/hugsql-adapter-next-jdbc {:mvn/version "0.5.3"}
com.fzakaria/slf4j-timbre {:mvn/version "0.4.1"}
io.zonky.test.postgres/embedded-postgres-binaries-linux-amd64 {:mvn/version "16.2.0"}
- io.zonky.test/embedded-postgres {:mvn/version "2.0.7"}
+ io.zonky.test/embedded-postgres {:mvn/version "2.1.0"}
lambdaisland/kaocha {:mvn/version "1.91.1392"}
lambdaisland/kaocha-cloverage {:mvn/version "1.1.89"}
seancorfield/next.jdbc {:mvn/version "1.2.659"}}
:exec-fn kaocha.runner/exec-fn
:extra-paths ["test"]
:exec-args {}}
- ;;
+
;; $ clj -M:dev # NREPL and interactive terminal REPL in one
- ;;
:dev {:extra-paths ["test" "examples"]
- :extra-deps {cider/cider-nrepl {:mvn/version "0.49.3"}
+ :extra-deps {cider/cider-nrepl {:mvn/version "0.57.0"}
com.fzakaria/slf4j-timbre {:mvn/version "0.4.1"}
org.scicloj/clay {:mvn/version "2-beta11"}
- io.zonky.test/embedded-postgres {:mvn/version "2.0.7"}
+ io.zonky.test/embedded-postgres {:mvn/version "2.1.0"}
io.zonky.test.postgres/embedded-postgres-binaries-linux-amd64 {:mvn/version "16.2.0"}
com.layerware/hugsql-adapter-next-jdbc {:mvn/version "0.5.3"}
seancorfield/next.jdbc {:mvn/version "1.2.659"}}
:main-opts ["-m" "nrepl.cmdline"
"--middleware" "[cider.nrepl/cider-middleware]"
"--interactive"]}
- ;;
+
;; $ clj -T:build jar # Build target/com.github.perrygeo/postgres-extras-clj-{version}.jar
;; $ clj -T:build deploy # Deploy to Clojars
- ;;
:build {:deps {io.github.clojure/tools.build {:git/tag "v0.10.3" :git/sha "15ead66"}
slipset/deps-deploy {:mvn/version "0.2.0"}}
:ns-default build}}}
diff --git a/dev/infra/docker-compose.yml b/dev/infra/docker-compose.yml
index c6dea24..62cc82f 100644
--- a/dev/infra/docker-compose.yml
+++ b/dev/infra/docker-compose.yml
@@ -7,7 +7,7 @@ services:
- POSTGRES_DB=postgres
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=password
- image: postgis/postgis:16-3.4
+ image: postgres:16.9-bullseye
ports:
- "127.0.0.1:5432:5432"
expose:
@@ -15,7 +15,10 @@ services:
restart: unless-stopped
volumes:
- ./init-db.sh:/docker-entrypoint-initdb.d/init-db.sh # where 'main' database is created
- - ./pgdata:/var/lib/postgresql/data # storage in local pgdata directory, .gitignored
+ - pgdata:/var/lib/postgresql/data # storage in docker volume
- ./postgresql.conf:/etc/postgresql/postgresql.conf # configuration
- ./pg_hba.conf:/etc/postgresql/pg_hba.conf # access rules
+volumes:
+ pgdata:
+ driver: local
diff --git a/dev/infra/postgresql.conf b/dev/infra/postgresql.conf
index 760deb0..e1ab55f 100644
--- a/dev/infra/postgresql.conf
+++ b/dev/infra/postgresql.conf
@@ -1,9 +1,9 @@
listen_addresses = '*'
port = 5432 # (change requires restart)
-max_connections = 400 # (change requires restart)
+max_connections = 40 # (change requires restart)
-shared_buffers = 2056MB # min 128kB
-work_mem = 40MB # min 64kB
+shared_buffers = 1024MB # min 128kB
+work_mem = 40MB # min 64kB
maintenance_work_mem = 640MB # min 1MB
dynamic_shared_memory_type = posix # the default is the first option
max_parallel_workers_per_gather = 6 # taken from max_parallel_workers
diff --git a/examples/pgbench_tutorial.clj b/examples/pgbench_tutorial.clj
index f3350ec..6cb2342 100644
--- a/examples/pgbench_tutorial.clj
+++ b/examples/pgbench_tutorial.clj
@@ -47,25 +47,21 @@
(show
(filter #(not (:system_object %)) f)))
-^:kindly/hide-code
(defn meta-as-header [x]
- (kind/md (str "### " (:doc (meta x)))))
+ (kind/md (str "### " x "\n" (:doc (meta x)))))
-;;
;; ## Setup
-;;
;; ### Dependencies
-^:kindly/hide-code
-(kind/md
- "```clojure
- {:deps {com.layerware/hugsql {:mvn/version \"0.5.3\"}
- com.layerware/hugsql-adapter-next-jdbc {:mvn/version \"0.5.3\"}
- org.scicloj/clay {:mvn/version \"2-beta11\"}
- org.postgresql/postgresql {:mvn/version \"42.7.3\"}
- seancorfield/next.jdbc {:mvn/version \"1.2.659\"}}}
- ```")
+; ```clojure
+; {:deps
+; {com.layerware/hugsql {:mvn/version "0.5.3"}
+; com.layerware/hugsql-adapter-next-jdbc {:mvn/version "0.5.3"}
+; org.scicloj/clay {:mvn/version "2-beta11"}
+; org.postgresql/postgresql {:mvn/version "42.7.3"}
+; seancorfield/next.jdbc {:mvn/version "1.2.659"}}}
+; ```
; ### Database benchmark
@@ -76,19 +72,15 @@
; To initialize
-^:kindly/hide-code
-(kind/md
- "```bash
- PGPASSWORD=password pgbench --host localhost --port 5432 --username postgres -s 10 -F 100 -i -I dtgvpf main
- ```")
+; ```bash
+; PGPASSWORD=password pgbench --host localhost --port 5432 --username postgres -s 10 -F 100 -i -I dtgvpf main
+; ```
; Then run the benchmarks
-^:kindly/hide-code
-(kind/md
- "```bash
- PGPASSWORD=password pgbench --host localhost --port 5432 --username postgres --client 32 --transactions 1000 --jobs 8 main
- ```")
+; ```bash
+; PGPASSWORD=password pgbench --host localhost --port 5432 --username postgres --client 32 --transactions 1000 --jobs 8 main
+; ```
;; ### Driver Setup
@@ -100,22 +92,18 @@
;; or from a JDBC URI.
-(def db2
+(def db-from-uri
(jdbc/get-datasource
"jdbc:postgresql://localhost:5432/main?user=postgres&password=password"))
;; Independently, we need to tell hugsql to expect next-jdbc.
+
(hugsql/set-adapter! (next-adapter/hugsql-adapter-next-jdbc))
;; Do a health check to ensure connectivity
-
-^:kindly/hide-code
-(comment
- (pgex/health-check db2))
-
(pgex/health-check db)
-
+(pgex/health-check db-from-uri)
; ## Settings at a glance
@@ -134,6 +122,7 @@
; Here's my configuration tuned for test/dev on my laptop.
(show (pgex/db-settings db))
+
; example row
^:kindly/hide-code (first (pgex/db-settings db))
@@ -149,37 +138,37 @@
^:kindly/hide-code (meta-as-header #'pgex/databases)
(show (pgex/databases db))
; example row
-^:kindly/hide-code (first (:databases dd))
+(first (:databases dd))
^:kindly/hide-code (meta-as-header #'pgex/schemas)
(show-public (pgex/schemas db))
; example row
-^:kindly/hide-code (first (:schemas dd))
+(first (:schemas dd))
^:kindly/hide-code (meta-as-header #'pgex/views)
(show-public (pgex/views db))
; example row
-^:kindly/hide-code (first (:views dd))
+(first (:views dd))
^:kindly/hide-code (meta-as-header #'pgex/indexes)
(show-public (pgex/indexes db))
; example row
-^:kindly/hide-code (first (:indexes dd))
+(first (:indexes dd))
^:kindly/hide-code (meta-as-header #'pgex/columns)
(show-public (pgex/columns db))
; example row
-^:kindly/hide-code (first (:columns dd))
+(first (:columns dd))
^:kindly/hide-code (meta-as-header #'pgex/tables)
(show-public (pgex/tables db))
; example row
-^:kindly/hide-code (first (:tables dd))
+(first (:tables dd))
^:kindly/hide-code (meta-as-header #'pgex/functions)
(show-public (pgex/functions db))
; example row
-^:kindly/hide-code (first (:functions dd))
+(first (:functions dd))
#_(filter #(= (:t_name %) "pgbench_accounts") (pgex/tables db))
@@ -190,127 +179,120 @@
(def stats (pgex/read-stats db {:limit 100}))
(keys stats)
-
^:kindly/hide-code (meta-as-header #'pgex/vacuum-stats)
(show (pgex/vacuum-stats db))
; example row
-^:kindly/hide-code (first (:vacuum-stats stats))
+(first (:vacuum-stats stats))
^:kindly/hide-code (meta-as-header #'pgex/index-usage)
(show (pgex/index-usage db))
; example row
-^:kindly/hide-code (first (:index-usage stats))
+(first (:index-usage stats))
^:kindly/hide-code (meta-as-header #'pgex/total-index-size)
(pgex/total-index-size db)
; example row
-^:kindly/hide-code (first (:total-index-size stats))
+(first (:total-index-size stats))
^:kindly/hide-code (meta-as-header #'pgex/cache-hit)
(show (pgex/cache-hit db))
; example row
-^:kindly/hide-code (first (:cache-hit stats))
+(first (:cache-hit stats))
^:kindly/hide-code (meta-as-header #'pgex/records-rank)
(show (pgex/records-rank db))
; example row
-^:kindly/hide-code (first (:records-rank stats))
+(first (:records-rank stats))
^:kindly/hide-code (meta-as-header #'pgex/index-cache-hit)
(show (pgex/index-cache-hit db))
; example row
-^:kindly/hide-code (first (:index-cache-hit stats))
+(first (:index-cache-hit stats))
^:kindly/hide-code (meta-as-header #'pgex/outliers)
(show (pgex/outliers db {:limit 10}))
; example row
-^:kindly/hide-code (first (:outliers stats))
-
+(first (:outliers stats))
^:kindly/hide-code (meta-as-header #'pgex/extensions)
(show (pgex/extensions db))
; example row
-^:kindly/hide-code (first (:extensions stats))
+(first (:extensions stats))
^:kindly/hide-code (meta-as-header #'pgex/total-table-size)
(show (pgex/total-table-size db))
; example row
-^:kindly/hide-code (first (:total-table-size stats))
+(first (:total-table-size stats))
^:kindly/hide-code (meta-as-header #'pgex/bloat)
(show (pgex/bloat db))
; example row
-^:kindly/hide-code (first (:bloat stats))
+(first (:bloat stats))
^:kindly/hide-code (meta-as-header #'pgex/calls)
(show (pgex/calls db {:limit 10}))
; example row
-^:kindly/hide-code (first (:calls stats))
+(first (:calls stats))
^:kindly/hide-code (meta-as-header #'pgex/table-size)
(show (pgex/table-size db))
; example row
-^:kindly/hide-code (first (:table-size stats))
+(first (:table-size stats))
^:kindly/hide-code (meta-as-header #'pgex/connections)
(show (pgex/connections db))
; example row
-^:kindly/hide-code (first (:connections stats))
+(first (:connections stats))
^:kindly/hide-code (meta-as-header #'pgex/table-cache-hit)
(show (pgex/table-cache-hit db))
; example row
-^:kindly/hide-code (first (:table-cache-hit stats))
+(first (:table-cache-hit stats))
^:kindly/hide-code (meta-as-header #'pgex/table-indexes-size)
(show (pgex/table-indexes-size db))
; example row
-^:kindly/hide-code (first (:table-indexes-size stats))
-
+(first (:table-indexes-size stats))
^:kindly/hide-code (meta-as-header #'pgex/seq-scans)
(show (pgex/seq-scans db))
; example row
-^:kindly/hide-code (first (:seq-scans stats))
+(first (:seq-scans stats))
^:kindly/hide-code (meta-as-header #'pgex/index-size)
(show (pgex/index-size db))
; example row
-^:kindly/hide-code (first (:index-size stats))
+(first (:index-size stats))
-^:kindly/hide-code
-(comment
- ;;;
- ;;; These show zero results with the pgbench example
- ;;; show them once we can reproduce the required conditions
- ;;;
- ^:kindly/hide-code (meta-as-header #'pgex/partition-children)
- (show (pgex/partition-children db))
+;; These show no results for the pgbench example
+;; but can be valuable in other scenarios.
- ^:kindly/hide-code (meta-as-header #'pgex/partition-parents)
- (show (pgex/partition-parents db))
+^:kindly/hide-code (meta-as-header #'pgex/partition-children)
+(show (pgex/partition-children db))
- ^:kindly/hide-code (meta-as-header #'pgex/duplicate-indexes)
- (show (pgex/duplicate-indexes db))
+^:kindly/hide-code (meta-as-header #'pgex/partition-parents)
+(show (pgex/partition-parents db))
- ^:kindly/hide-code (meta-as-header #'pgex/locks)
- (show (pgex/locks db))
+^:kindly/hide-code (meta-as-header #'pgex/duplicate-indexes)
+(show (pgex/duplicate-indexes db))
- ^:kindly/hide-code (meta-as-header #'pgex/null-indexes)
- (show (pgex/null-indexes db))
+^:kindly/hide-code (meta-as-header #'pgex/locks)
+(show (pgex/locks db))
- ^:kindly/hide-code (meta-as-header #'pgex/all-locks)
- (show (pgex/all-locks db))
+^:kindly/hide-code (meta-as-header #'pgex/null-indexes)
+(show (pgex/null-indexes db))
- ^:kindly/hide-code (meta-as-header #'pgex/blocking)
- (show (pgex/blocking db))
+^:kindly/hide-code (meta-as-header #'pgex/all-locks)
+(show (pgex/all-locks db))
- ^:kindly/hide-code (meta-as-header #'pgex/unused-indexes)
- (show (pgex/unused-indexes db {:min_scans 50}))
+^:kindly/hide-code (meta-as-header #'pgex/blocking)
+(show (pgex/blocking db))
- ^:kindly/hide-code (meta-as-header #'pgex/long-running-queries)
- (show (pgex/long-running-queries db)))
+^:kindly/hide-code (meta-as-header #'pgex/unused-indexes)
+(show (pgex/unused-indexes db {:min_scans 50}))
+^:kindly/hide-code (meta-as-header #'pgex/long-running-queries)
+(show (pgex/long-running-queries db))
;; ## The Kill Switch
@@ -319,9 +301,8 @@
;; vital if you need to take a heavily-used database
;; down for maintenance or emergency.
;;
-;;; ```clojure
-;;; (pgex/kill-all! db) ;; use with caution
-;;; ```
+(comment
+ (pgex/kill-all! db)) ;; use with caution!
;; ## Diagnostics