Improve test db management (#747)

.gitignore

@@ -6,6 +6,6 @@ cabal.sandbox.config
 hscope.out
 codex.tags
 .anvil
-.stack-work
+.stack-work*
 tags
 site

CHANGELOG.md

@@ -6,6 +6,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ## Unreleased
 
 ### Added
+- Allow test database to be on another host - @dsimunic
 - New `Prefer` header value: `params=single-object` to pass all form values as a single json object to a stored procedure - @dsimunic
 - Ability to generate an OpenAPI spec - @mainx07, @hudayou, @ruslantalpa, @begriffs
 - Ability to generate an OpenAPI spec behind a proxy - @hudayou

circle.yml

@@ -6,8 +6,6 @@ dependencies:
     - curl -L https://github.com/commercialhaskell/stack/releases/download/v1.1.2/stack-1.1.2-linux-x86_64.tar.gz | tar zx -C /tmp
     - sudo mv /tmp/stack-1.1.2-linux-x86_64/stack /usr/bin
     - sudo apt-get update; sudo apt-get install --only-upgrade binutils
-    - createuser --superuser --no-password postgrest_test
-    - createdb -O postgrest_test -U ubuntu postgrest_test
   override:
     - stack setup
     - rm -fr $(stack path --dist-dir) $(stack path --local-install-root)
@@ -17,7 +15,7 @@ dependencies:
 
 test:
   override:
-    - stack test --test-arguments "--skip \"returns a valid openapi\""
+    - POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://ubuntu@localhost" postgrest_test) stack test --test-arguments "--skip \"returns a valid openapi\""
     - git ls-files | grep '\.l\?hs$' | xargs stack exec -- hlint -X QuasiQuotes -X NoPatternSynonyms "$@"
     - stack exec -- cabal update
     - stack exec --no-ghc-package-path -- cabal install --only-d --dry-run

test/Dockerfile.test

@@ -0,0 +1,13 @@
+FROM debian:jessie
+
+ENV PATH /root/.local/bin:$PATH
+
+RUN apt-get update \
+    && apt-get install -y wget libpq-dev pkg-config libpcre3 libpcre3-dev \
+       postgresql-client debconf locales \
+    && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+    && echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen \
+    && locale-gen \
+    && echo 'export LC_ALL=en_US.UTF-8' >> /etc/profile \
+    && wget -qO- https://get.haskellstack.org/ | sh
+

test/Main.hs

@@ -29,7 +29,7 @@ import Protolude
 
 main :: IO ()
 main = do
-  setupDb
+  testDbConn <- getEnvVarWithDefault "POSTGREST_TEST_CONNECTION" "postgres://postgrest_test@localhost/postgrest_test"
 
   pool <- P.acquire (3, 10, toS testDbConn)
   -- ask for the OS time at most once per second
@@ -39,29 +39,30 @@ main = do
 
   result <- P.use pool $ getDbStructure "test"
   refDbStructure <- newIORef $ either (panic.show) id result
-  let withApp = return $ postgrest testCfg refDbStructure pool getTime
-      ltdApp  = return $ postgrest testLtdRowsCfg refDbStructure pool getTime
-      unicodeApp = return $ postgrest testUnicodeCfg refDbStructure pool getTime
-      proxyApp = return $ postgrest testProxyCfg refDbStructure pool getTime
-      noJwtApp = return $ postgrest testCfgNoJWT refDbStructure pool getTime
+  let withApp = return $ postgrest (testCfg testDbConn) refDbStructure pool getTime
+      ltdApp  = return $ postgrest (testLtdRowsCfg testDbConn) refDbStructure pool getTime
+      unicodeApp = return $ postgrest (testUnicodeCfg testDbConn) refDbStructure pool getTime
+      proxyApp = return $ postgrest (testProxyCfg testDbConn) refDbStructure pool getTime
+      noJwtApp = return $ postgrest (testCfgNoJWT testDbConn) refDbStructure pool getTime
 
+  let reset = resetDb testDbConn
   hspec $ do
-    mapM_ (beforeAll_ resetDb . before withApp) specs
+    mapM_ (beforeAll_ reset . before withApp) specs
 
     -- this test runs with a different server flag
-    beforeAll_ resetDb . before ltdApp $
+    beforeAll_ reset . before ltdApp $
       describe "Feature.QueryLimitedSpec" Feature.QueryLimitedSpec.spec
 
     -- this test runs with a different schema
-    beforeAll_ resetDb . before unicodeApp $
+    beforeAll_ reset . before unicodeApp $
       describe "Feature.UnicodeSpec" Feature.UnicodeSpec.spec
 
     -- this test runs with a proxy
-    beforeAll_ resetDb . before proxyApp $
+    beforeAll_ reset . before proxyApp $
       describe "Feature.ProxySpec" Feature.ProxySpec.spec
 
     -- this test runs without a JWT secret
-    beforeAll_ resetDb . before noJwtApp $
+    beforeAll_ reset . before noJwtApp $
       describe "Feature.NoJwtSpec" Feature.NoJwtSpec.spec
 
  where

test/SpecHelper.hs

@@ -2,6 +2,9 @@ module SpecHelper where
 
 import Control.Monad (void)
 
+import qualified System.IO.Error as E
+import System.Environment (getEnv)
+
 import Codec.Binary.Base64.String (encode)
 import Data.CaseInsensitive (CI(..))
 import Data.List (lookup)
@@ -46,45 +49,37 @@ validateOpenApiResponse headers = do
        in
        D4.fetchFilesystemAndValidate schemaContext ((fromJust . decode) respBody) `shouldReturn` Right ()
 
-testDbConn :: Text
-testDbConn = "postgres://postgrest_test_authenticator@localhost:5432/postgrest_test"
+getEnvVarWithDefault :: Text -> Text -> IO Text
+getEnvVarWithDefault var def = do
+  varValue <- getEnv (toS var) `E.catchIOError` const (return $ toS def)
+  return $ toS varValue
 
-testCfg :: AppConfig
-testCfg =
+testCfg :: Text -> AppConfig
+testCfg testDbConn =
   AppConfig testDbConn "postgrest_test_anonymous" Nothing "test" "localhost" 3000 (Just "safe") 10 Nothing (Just "test.switch_role") True
 
-testCfgNoJWT :: AppConfig
-testCfgNoJWT =
+testCfgNoJWT :: Text -> AppConfig
+testCfgNoJWT testDbConn =
   AppConfig testDbConn "postgrest_test_anonymous" Nothing "test" "localhost" 3000 Nothing 10 Nothing Nothing True
 
-testUnicodeCfg :: AppConfig
-testUnicodeCfg =
+testUnicodeCfg :: Text -> AppConfig
+testUnicodeCfg testDbConn =
   AppConfig testDbConn "postgrest_test_anonymous" Nothing "تست" "localhost" 3000 (Just "safe") 10 Nothing Nothing True
 
-testLtdRowsCfg :: AppConfig
-testLtdRowsCfg =
+testLtdRowsCfg :: Text -> AppConfig
+testLtdRowsCfg testDbConn =
   AppConfig testDbConn "postgrest_test_anonymous" Nothing "test" "localhost" 3000 (Just "safe") 10 (Just 2) Nothing True
 
-testProxyCfg :: AppConfig
-testProxyCfg =
+testProxyCfg :: Text -> AppConfig
+testProxyCfg testDbConn =
   AppConfig testDbConn "postgrest_test_anonymous" (Just "https://postgrest.com/openapi.json") "test" "localhost" 3000 (Just "safe") 10 Nothing Nothing True
 
-setupDb :: IO ()
-setupDb = do
-  void $ readProcess "psql" ["-d", "postgres", "-a", "-f", "test/fixtures/database.sql"] []
-  void $ readProcess "psql" ["-d", "postgrest_test", "-a", "-c", "CREATE EXTENSION IF NOT EXISTS pgcrypto;"] []
-  loadFixture "roles"
-  loadFixture "schema"
-  loadFixture "jwt"
-  loadFixture "privileges"
-  resetDb
+resetDb :: Text -> IO ()
+resetDb dbConn = loadFixture dbConn "data"
 
-resetDb :: IO ()
-resetDb = loadFixture "data"
-
-loadFixture :: FilePath -> IO()
-loadFixture name =
-  void $ readProcess "psql" ["-U", "postgrest_test", "-d", "postgrest_test", "-a", "-f", "test/fixtures/" ++ name ++ ".sql"] []
+loadFixture :: Text -> FilePath -> IO()
+loadFixture dbConn name =
+  void $ readProcess "psql" [toS dbConn, "-a", "-f", "test/fixtures/" ++ name ++ ".sql"] []
 
 rangeHdrs :: ByteRange -> [Header]
 rangeHdrs r = [rangeUnit, (hRange, renderByteRange r)]

test/create_test_db

@@ -0,0 +1,93 @@
+#! /bin/bash
+if [ -z "$1" ]
+  then
+    echo "Please supply the connection uri for the user with create database privileges"
+    exit -1
+fi
+
+if [ -z "$2" ]
+  then
+    echo "Please supply the test database name"
+    exit -1
+fi
+if [[ $1 != postgres://* ]]
+then
+  echo "Please use a valid connection URI (https://www.postgresql.org/docs/current/static/libpq-connect.html#AEN45347)"
+  exit -1
+fi
+
+BASEPATH=$( cd $(dirname $0) ; pwd -P )
+#Remove database path from the connection uri--prevents setting up the new database name with PGDATABASE
+URI=$(echo $1 | cut -d'/' -f1-3)
+#Extract host and port--we need this to form the new connection string
+HOST_PORT=$(echo $URI | cut -d'/' -f3 | cut -d'@' -f2 )
+DB=$2
+# Specify the username of choice, or let the script create a random unique user by appending the database name
+TEST_USER_NAME=${3:-postgrest_test_$DB}
+# New password will get assigned only if the user does not already exist
+# Otherwise make sure to provide the correct password for the existing user
+TEST_USER_PASS=${4:-$(cat /dev/urandom | env LC_CTYPE=C tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1)a}
+
+PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" -Xq >/dev/null -c 'select rolcreatedb from pg_authid where rolname = current_user;' 2>/dev/null
+if [ $? -ne 0 ]; then
+  echo "ERROR: Please specify the user with 'Create DB' permissions, and ensure that the default database for the username exists."
+  exit 1
+fi
+
+# plpgsql does not like psql variables, easier to pull this part off with bash variables
+PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" -Xq >/dev/null <<EOF
+SELECT pg_terminate_backend(pg_stat_activity.pid)
+FROM pg_stat_activity
+WHERE pg_stat_activity.datname = '$DB'
+  AND pid <> pg_backend_pid();
+
+drop database if exists "$DB";
+
+create database "$DB" encoding = 'UTF8';
+
+DO \$\$
+  BEGIN
+    IF NOT EXISTS (SELECT * FROM pg_catalog.pg_roles WHERE rolname = '$TEST_USER_NAME')
+      THEN CREATE ROLE $TEST_USER_NAME WITH LOGIN NOINHERIT PASSWORD '$TEST_USER_PASS';
+    END IF;
+  END \$\$;
+DO \$\$
+  BEGIN
+    IF NOT EXISTS (SELECT * FROM pg_catalog.pg_roles WHERE rolname = 'postgrest_test_anonymous')
+      THEN CREATE ROLE postgrest_test_anonymous;
+    END IF;
+  END \$\$;
+DO \$\$
+  BEGIN
+    IF NOT EXISTS (SELECT * FROM pg_catalog.pg_roles WHERE rolname = 'postgrest_test_default_role')
+      THEN CREATE ROLE postgrest_test_default_role;
+    END IF;
+  END \$\$;
+DO \$\$
+  BEGIN
+    IF NOT EXISTS (SELECT * FROM pg_catalog.pg_roles WHERE rolname = 'postgrest_test_author')
+      THEN CREATE ROLE postgrest_test_author;
+    END IF;
+  END \$\$;
+
+DO \$\$
+  BEGIN
+  IF NOT EXISTS (select * from pg_roles where rolname = 'postgrest_test_anonymous' AND pg_has_role('$TEST_USER_NAME', oid, 'member'))
+    THEN GRANT postgrest_test_anonymous TO $TEST_USER_NAME;
+  END IF;
+  IF NOT EXISTS (select * from pg_roles where rolname = 'postgrest_test_author' AND pg_has_role('$TEST_USER_NAME', oid, 'member'))
+    THEN GRANT postgrest_test_author TO $TEST_USER_NAME;
+  END IF;
+  IF NOT EXISTS (select * from pg_roles where rolname = 'postgrest_test_default_role' AND pg_has_role('$TEST_USER_NAME', oid, 'member'))
+    THEN GRANT postgrest_test_default_role TO $TEST_USER_NAME;
+  END IF;
+END \$\$;
+EOF
+
+PGDATABASE=$DB PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" --set=db=$DB -Xqf $BASEPATH/fixtures/database.sql
+PGDATABASE=$DB PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" -Xqf $BASEPATH/fixtures/schema.sql
+PGDATABASE=$DB PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" -Xqf $BASEPATH/fixtures/jwt.sql
+PGDATABASE=$DB PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" --set=test_user_name="$TEST_USER_NAME" -Xqf $BASEPATH/fixtures/privileges.sql
+
+# Create a new connection string to use with the test runner
+echo 'postgres://'${TEST_USER_NAME}':'$TEST_USER_PASS'@'$HOST_PORT'/'$DB

test/destroy_test_db

@@ -0,0 +1,66 @@
+#! /bin/bash
+if [ -z "$1" ]
+  then
+    echo "Please supply the connection uri for the user with create database privileges"
+    exit -1
+fi
+
+if [ -z "$2" ]
+  then
+    echo "Please supply the test database name"
+    exit -1
+fi
+if [[ $1 != postgres://* ]]
+then
+  echo "Please use a valid connection URI (https://www.postgresql.org/docs/current/static/libpq-connect.html#AEN45347)"
+  exit -1
+fi
+
+BASEPATH=$( cd $(dirname $0) ; pwd -P )
+#Remove database path from the connection uri
+URI=$(echo $1 | cut -d'/' -f1-3)
+DB=$2
+
+PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" -Xq >/dev/null -c 'select rolcreatedb from pg_authid where rolname = current_user;' 2>/dev/null
+if [ $? -ne 0 ]; then
+  echo "ERROR: Please specify the user with 'Create DB' permissions, and ensure that the default database for the username exists."
+  exit 1
+fi
+
+# plpgsql does not like psql variables, easier to pull this part off with bash variables
+PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" -Xq >/dev/null <<EOF
+SELECT pg_terminate_backend(pg_stat_activity.pid)
+FROM pg_stat_activity
+WHERE pg_stat_activity.datname = '$DB'
+  AND pid <> pg_backend_pid();
+
+drop database if exists "$DB";
+
+-- Find all test users that were members of role 'postgrest_test_author'--that way we don't have to know the 
+-- test user name (in case it was auto-generated).
+DO \$\$
+DECLARE
+    mem text;
+BEGIN
+    FOR mem IN 
+      SELECT pg_get_userbyid(member)
+      FROM pg_roles r 
+      JOIN pg_auth_members m
+      ON m.roleid = r.oid 
+      WHERE rolname = 'postgrest_test_author' 
+    LOOP
+      EXECUTE 'drop role '|| mem || ';';
+    END LOOP;
+END \$\$;
+
+DO \$\$
+DECLARE
+    r text;
+BEGIN
+    FOR r IN 
+      VALUES('postgrest_test_author'),('postgrest_test_anonymous'),('postgrest_test_default_role')
+    LOOP    
+      EXECUTE 'drop role if exists '|| r || ';';
+    END LOOP;
+END \$\$;
+EOF

test/fixtures/database.sql

@@ -1,6 +1,3 @@
-DROP DATABASE IF EXISTS postgrest_test;
-DROP ROLE IF EXISTS postgrest_test;
-CREATE USER postgrest_test createdb createrole;
-CREATE DATABASE postgrest_test OWNER postgrest_test;
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
 
-ALTER DATABASE postgrest_test SET request.jwt.claim.id = '-1';
+ALTER DATABASE :db SET request.jwt.claim.id = '-1';

test/fixtures/privileges.sql

@@ -57,3 +57,8 @@ TO postgrest_test_anonymous;
 -- Privileges for non anonymous users
 GRANT USAGE ON SCHEMA test TO postgrest_test_author;
 GRANT ALL ON TABLE authors_only TO postgrest_test_author;
+
+GRANT USAGE ON SCHEMA postgrest,private,test to :test_user_name;
+GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA postgrest,private,test TO :test_user_name;
+GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA postgrest,private,test TO :test_user_name;
+

test/fixtures/roles.sql

@@ -1,7 +0,0 @@
-DROP ROLE IF EXISTS postgrest_test_authenticator, postgrest_test_anonymous, postgrest_test_default_role, postgrest_test_author;
-CREATE ROLE postgrest_test_authenticator WITH login noinherit;
-CREATE ROLE postgrest_test_anonymous;
-CREATE ROLE postgrest_test_default_role;
-CREATE ROLE postgrest_test_author;
-
-GRANT postgrest_test_anonymous, postgrest_test_default_role, postgrest_test_author TO postgrest_test_authenticator;