Use environment variables for configuration in developer tooling

Avoid the need for configuration files for temporary database connections.

.circleci/config.yml

@@ -66,8 +66,7 @@ jobs:
       - run:
           name: run spec tests
           command: |
-            POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://circleci@localhost" postgrest_test) \
-              stack test
+            test/create_test_db "postgres://circleci@localhost" postgrest_test stack test
 
   # Run memory usage tests based on stack and docker.
   stack-test-memory:
@@ -111,8 +110,7 @@ jobs:
           name: run memory usage tests
           command: |
             export PATH="~/.local/bin:$PATH"
-            test/create_test_db "postgres://circleci@localhost" postgrest_test
-            test/memory-tests.sh
+            test/create_test_db "postgres://circleci@localhost" postgrest_test test/memory-tests.sh
 
   # Publish a new release. This only runs when a release is tagged (see
   # workflow below).

nix/tests.nix

@@ -140,7 +140,7 @@ let
 
         ${withTmpDb postgresql} \
             ${cabal-install}/bin/cabal v2-run ${devCabalOptions} --verbose=0 -- \
-            postgrest --dump-schema "$rootdir"/test/io-tests/configs/simple.config \
+            postgrest --dump-schema \
             | ${yq}/bin/yq -y .
       '';
 in

test/Main.hs

@@ -57,7 +57,7 @@ main :: IO ()
 main = do
   getTime <- mkAutoUpdate defaultUpdateSettings { updateAction = getCurrentTime }
 
-  testDbConn <- getEnvVarWithDefault "POSTGREST_TEST_CONNECTION" "postgres://postgrest_test@localhost/postgrest_test"
+  testDbConn <- getEnvVarWithDefault "PGRST_DB_URI" "postgres://postgrest_test@localhost/postgrest_test"
 
   pool <- P.acquire (3, 10, toS testDbConn)
 

test/QueryCost.hs

@@ -24,7 +24,7 @@ import Test.Hspec
 
 main :: IO ()
 main = do
-  testDbConn <- getEnvVarWithDefault "POSTGREST_TEST_CONNECTION" "postgres://postgrest_test@localhost/postgrest_test"
+  testDbConn <- getEnvVarWithDefault "PGRST_DB_URI" "postgres://postgrest_test@localhost/postgrest_test"
   pool <- P.acquire (3, 10, toS testDbConn)
 
   hspec $ describe "QueryCost" $

test/create_test_db

@@ -68,4 +68,9 @@ PGDATABASE=$DB PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" --set=db=$
 EOF
 
 # Create a new connection string to use with the test runner
-echo 'postgres://'${TEST_USER_NAME}':'$TEST_USER_PASS'@'$HOST_PORT'/'$DB
+export PGRST_DB_URI="postgres://${TEST_USER_NAME}:$TEST_USER_PASS@$HOST_PORT/$DB"
+export PGRST_DB_ANON_ROLE="postgrest_test_anonymous"
+export PGRST_DB_SCHEMAS="test"
+
+shift 2
+"$@"

test/io-tests/configs/app-settings.config

@@ -1,6 +1,3 @@
-db-uri = "$(POSTGREST_TEST_CONNECTION)"
-db-schemas = "test"
-db-anon-role = "postgrest_test_anonymous"
 db-pool = 1
 db-pool-timeout = 1
 server-host = "127.0.0.1"

test/io-tests/configs/base64-secret-from-file.config

@@ -1,6 +1,3 @@
-db-uri = "$(POSTGREST_TEST_CONNECTION)"
-db-schemas = "test"
-db-anon-role = "postgrest_test_anonymous"
 db-pool = 1
 server-host = "127.0.0.1"
 server-port = 49421

test/io-tests/configs/dburi-from-file.config

@@ -1,6 +1,4 @@
 db-uri = "@/dev/stdin"
-db-schemas = "test"
-db-anon-role = "postgrest_test_anonymous"
 db-pool = 1
 server-host = "127.0.0.1"
 server-port = 49421

test/io-tests/configs/role-claim-key.config

@@ -1,6 +1,3 @@
-db-uri = "$(POSTGREST_TEST_CONNECTION)"
-db-schemas = "test"
-db-anon-role = "postgrest_test_anonymous"
 db-pool = 1
 server-host = "127.0.0.1"
 server-port = 49421

test/io-tests/configs/secret-from-file.config

@@ -1,6 +1,3 @@
-db-uri = "$(POSTGREST_TEST_CONNECTION)"
-db-schemas = "test"
-db-anon-role = "postgrest_test_anonymous"
 db-pool = 1
 server-host = "127.0.0.1"
 server-port = 49421

test/io-tests/configs/sigusr2-settings.config

@@ -1,6 +1,4 @@
-db-uri = "$(POSTGREST_TEST_CONNECTION)"
 db-schemas = "test"
-db-anon-role = "postgrest_test_anonymous"
 db-pool = 1
 server-host = "127.0.0.1"
 server-port = 49421

test/io-tests/configs/simple.config

@@ -1,6 +1,3 @@
-db-uri = "$(POSTGREST_TEST_CONNECTION)"
-db-schemas = "test"
-db-anon-role = "postgrest_test_anonymous"
 db-pool = 1
 server-host = "127.0.0.1"
 server-port = 49421

test/io-tests/configs/unix-socket.config

@@ -1,7 +1,3 @@
-db-uri = "$(POSTGREST_TEST_CONNECTION)"
-db-schemas = "test"
-db-anon-role = "postgrest_test_anonymous"
 db-pool = 1
-server-host = "127.0.0.1"
 server-unix-socket = "$(POSTGREST_TEST_SOCKET)"
 jwt-secret = "reallyreallyreallyreallyverysafe"

test/io-tests/test_io.py

@@ -55,19 +55,27 @@ class PostgrestProcess:
 @pytest.fixture
 def dburi():
     "Postgres database connection URI."
-    return os.getenv("POSTGREST_TEST_CONNECTION").encode("utf-8")
+    return os.getenv("PGRST_DB_URI").encode("utf-8")
+
+
+def mkenv(moreenv):
+    """
+    Create env from os.environ and moreenv, while
+    filtering None values to allow overriding "unset".
+    """
+    env = {**os.environ, **(moreenv or {})}
+    return {k: v for k, v in env.items() if v is not None}
 
 
 def dumpconfig(configpath=None, moreenv=None, stdin=None):
     "Dump the config as parsed by PostgREST."
 
-    env = {**os.environ, **(moreenv or {})}
     command = ["postgrest", "--dump-config"]
     if configpath:
         command += [configpath]
 
     process = subprocess.Popen(
-        command, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE
+        command, env=mkenv(moreenv), stdin=subprocess.PIPE, stdout=subprocess.PIPE
     )
     process.stdin.write(stdin or b"")
     result = process.communicate()[0]
@@ -81,7 +89,6 @@ def dumpconfig(configpath=None, moreenv=None, stdin=None):
 @contextlib.contextmanager
 def run(configpath, stdin=None, moreenv=None, socket=None):
     "Run PostgREST and yield an endpoint that is ready for connections."
-    env = {**os.environ, **(moreenv or {})}
 
     if socket:
         baseurl = "http+unix://" + urllib.parse.quote_plus(str(socket))
@@ -89,7 +96,7 @@ def run(configpath, stdin=None, moreenv=None, socket=None):
         baseurl = BASEURL
 
     command = ["postgrest", configpath]
-    process = subprocess.Popen(command, stdin=subprocess.PIPE, env=env)
+    process = subprocess.Popen(command, stdin=subprocess.PIPE, env=mkenv(moreenv))
 
     try:
         process.stdin.write(stdin or b"")
@@ -144,7 +151,10 @@ def test_expected_config(expectedconfig):
 
     """
     expected = expectedconfig.read_text()
-    assert dumpconfig(CONFIGSDIR / expectedconfig.name) == expected
+    config = CONFIGSDIR / expectedconfig.name
+
+    unset = {"PGRST_DB_URI": None, "PGRST_DB_ANON_ROLE": None, "PGRST_DB_SCHEMAS": None}
+    assert dumpconfig(config, moreenv=unset) == expected
 
 
 def test_expected_config_from_environment():
@@ -223,13 +233,17 @@ def test_read_secret_from_file(secretpath):
 
 def test_read_dburi_from_file_without_eol(dburi):
     "Reading the dburi from a file with a single line should work."
-    with run(CONFIGSDIR / "dburi-from-file.config", stdin=dburi):
+    config = CONFIGSDIR / "dburi-from-file.config"
+    unset = {"PGRST_DB_URI": None}
+    with run(config, moreenv=unset, stdin=dburi):
         pass
 
 
 def test_read_dburi_from_file_with_eol(dburi):
     "Reading the dburi from a file containing a newline should work."
-    with run(CONFIGSDIR / "dburi-from-file.config", stdin=dburi + b"\n"):
+    config = CONFIGSDIR / "dburi-from-file.config"
+    unset = {"PGRST_DB_URI": None}
+    with run(config, moreenv=unset, stdin=dburi + b"\n"):
         pass
 
 
@@ -350,8 +364,9 @@ def test_db_schema_reload(tmp_path):
     configfile.write_text(config)
 
     headers = {"Accept-Profile": "v1"}
+    unset = {"PGRST_DB_SCHEMAS": None}
 
-    with run(configfile) as postgrest:
+    with run(configfile, moreenv=unset) as postgrest:
         response = postgrest.session.get("/parents", headers=headers)
         assert response.status_code == 404
 

test/memory-tests.sh

@@ -1,12 +1,15 @@
 #! /usr/bin/env bash
 
 # This test script expects that a `postgrest` executable with profiling enabled
-# is on the PATH. With stack, for example, you can run `stack install --profile
-# postgrest`.
+# is on the PATH.
 
 set -eu
 
-export POSTGREST_TEST_CONNECTION=${POSTGREST_TEST_CONNECTION:-"postgres:///postgrest_test"}
+# PGRST_DB_URI, PGRST_DB_ANON_ROLE and PGRST_DB_SCHEMAS are expected to be set by with_tmp_db
+export PGRST_DB_POOL="1"
+export PGRST_SERVER_HOST="127.0.0.1"
+export PGRST_SERVER_PORT="49421"
+export PGRST_JWT_SECRET="reallyreallyreallyreallyverysafe"
 
 trap "kill 0" int term exit
 
@@ -18,7 +21,7 @@ ko(){ result 'not ok' "- $1"; failedTests=$(( $failedTests + 1 )); }
 
 pgrPort=49421
 
-pgrStart(){ postgrest test/memory-tests/config +RTS -p -h > /dev/null & pgrPID="$!"; }
+pgrStart(){ postgrest +RTS -p -h > /dev/null & pgrPID="$!"; }
 pgrStop(){ kill "$pgrPID" 2>/dev/null; }
 
 checkPgrStarted(){

test/memory-tests/config

@@ -1,8 +0,0 @@
-db-uri = "$(POSTGREST_TEST_CONNECTION)"
-db-schema = "test"
-db-anon-role = "postgrest_test_anonymous"
-db-pool = 1
-server-host = "127.0.0.1"
-server-port = 49421
-
-jwt-secret = "reallyreallyreallyreallyverysafe"

test/with_tmp_db

@@ -8,7 +8,7 @@ usage() {
 
 USAGE: $0 COMMAND
 
-Runs the given COMMAND with the POSTGREST_TEST_CONNECTION environment variable
+Runs the given COMMAND with the PGRST_DB_URI environment variable
 set to a temporary database that is ready for running the PostgREST test suite.
 
 You'll need to have the Postgres binaries 'initdb', 'pg_ctl' and 'psql' on your
@@ -59,7 +59,9 @@ export PGHOST="$tmpdir/socket"
 export PGUSER=postgrest_test_authenticator
 export PGDATABASE=postgres
 export DB_URI="postgresql://$PGDATABASE?host=$PGHOST&user=$PGUSER"
-export POSTGREST_TEST_CONNECTION="$DB_URI"
+export PGRST_DB_URI="$DB_URI"
+export PGRST_DB_SCHEMAS="test"
+export PGRST_DB_ANON_ROLE="postgrest_test_anonymous"
 
 log "Initializing database cluster..."
 # We try to make the database cluster as independent as possible from the host