diff --git a/.gitignore b/.gitignore index 79b2122d4..d4653b751 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,6 @@ cabal.sandbox.config hscope.out codex.tags .anvil -.stack-work +.stack-work* tags site diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dadbddc1..42a663364 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased ### Added +- Allow test database to be on another host - @dsimunic - New `Prefer` header value: `params=single-object` to pass all form values as a single json object to a stored procedure - @dsimunic - Ability to generate an OpenAPI spec - @mainx07, @hudayou, @ruslantalpa, @begriffs - Ability to generate an OpenAPI spec behind a proxy - @hudayou diff --git a/circle.yml b/circle.yml index 8c20f9958..89da414f9 100644 --- a/circle.yml +++ b/circle.yml @@ -6,8 +6,6 @@ dependencies: - curl -L https://github.com/commercialhaskell/stack/releases/download/v1.1.2/stack-1.1.2-linux-x86_64.tar.gz | tar zx -C /tmp - sudo mv /tmp/stack-1.1.2-linux-x86_64/stack /usr/bin - sudo apt-get update; sudo apt-get install --only-upgrade binutils - - createuser --superuser --no-password postgrest_test - - createdb -O postgrest_test -U ubuntu postgrest_test override: - stack setup - rm -fr $(stack path --dist-dir) $(stack path --local-install-root) @@ -17,7 +15,7 @@ dependencies: test: override: - - stack test --test-arguments "--skip \"returns a valid openapi\"" + - POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://ubuntu@localhost" postgrest_test) stack test --test-arguments "--skip \"returns a valid openapi\"" - git ls-files | grep '\.l\?hs$' | xargs stack exec -- hlint -X QuasiQuotes -X NoPatternSynonyms "$@" - stack exec -- cabal update - stack exec --no-ghc-package-path -- cabal install --only-d --dry-run diff --git a/test/Dockerfile.test b/test/Dockerfile.test new file mode 100644 index 000000000..116ce2393 --- /dev/null +++ b/test/Dockerfile.test @@ -0,0 +1,13 @@ +FROM debian:jessie + +ENV PATH /root/.local/bin:$PATH + +RUN apt-get update \ + && apt-get install -y wget libpq-dev pkg-config libpcre3 libpcre3-dev \ + postgresql-client debconf locales \ + && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen \ + && locale-gen \ + && echo 'export LC_ALL=en_US.UTF-8' >> /etc/profile \ + && wget -qO- https://get.haskellstack.org/ | sh + diff --git a/test/Main.hs b/test/Main.hs index e7ca843cd..4be07c0cf 100644 --- a/test/Main.hs +++ b/test/Main.hs @@ -29,7 +29,7 @@ import Protolude main :: IO () main = do - setupDb + testDbConn <- getEnvVarWithDefault "POSTGREST_TEST_CONNECTION" "postgres://postgrest_test@localhost/postgrest_test" pool <- P.acquire (3, 10, toS testDbConn) -- ask for the OS time at most once per second @@ -39,29 +39,30 @@ main = do result <- P.use pool $ getDbStructure "test" refDbStructure <- newIORef $ either (panic.show) id result - let withApp = return $ postgrest testCfg refDbStructure pool getTime - ltdApp = return $ postgrest testLtdRowsCfg refDbStructure pool getTime - unicodeApp = return $ postgrest testUnicodeCfg refDbStructure pool getTime - proxyApp = return $ postgrest testProxyCfg refDbStructure pool getTime - noJwtApp = return $ postgrest testCfgNoJWT refDbStructure pool getTime + let withApp = return $ postgrest (testCfg testDbConn) refDbStructure pool getTime + ltdApp = return $ postgrest (testLtdRowsCfg testDbConn) refDbStructure pool getTime + unicodeApp = return $ postgrest (testUnicodeCfg testDbConn) refDbStructure pool getTime + proxyApp = return $ postgrest (testProxyCfg testDbConn) refDbStructure pool getTime + noJwtApp = return $ postgrest (testCfgNoJWT testDbConn) refDbStructure pool getTime + let reset = resetDb testDbConn hspec $ do - mapM_ (beforeAll_ resetDb . before withApp) specs + mapM_ (beforeAll_ reset . before withApp) specs -- this test runs with a different server flag - beforeAll_ resetDb . before ltdApp $ + beforeAll_ reset . before ltdApp $ describe "Feature.QueryLimitedSpec" Feature.QueryLimitedSpec.spec -- this test runs with a different schema - beforeAll_ resetDb . before unicodeApp $ + beforeAll_ reset . before unicodeApp $ describe "Feature.UnicodeSpec" Feature.UnicodeSpec.spec -- this test runs with a proxy - beforeAll_ resetDb . before proxyApp $ + beforeAll_ reset . before proxyApp $ describe "Feature.ProxySpec" Feature.ProxySpec.spec -- this test runs without a JWT secret - beforeAll_ resetDb . before noJwtApp $ + beforeAll_ reset . before noJwtApp $ describe "Feature.NoJwtSpec" Feature.NoJwtSpec.spec where diff --git a/test/SpecHelper.hs b/test/SpecHelper.hs index 8f61a3d16..247cb330c 100644 --- a/test/SpecHelper.hs +++ b/test/SpecHelper.hs @@ -2,6 +2,9 @@ module SpecHelper where import Control.Monad (void) +import qualified System.IO.Error as E +import System.Environment (getEnv) + import Codec.Binary.Base64.String (encode) import Data.CaseInsensitive (CI(..)) import Data.List (lookup) @@ -46,45 +49,37 @@ validateOpenApiResponse headers = do in D4.fetchFilesystemAndValidate schemaContext ((fromJust . decode) respBody) `shouldReturn` Right () -testDbConn :: Text -testDbConn = "postgres://postgrest_test_authenticator@localhost:5432/postgrest_test" +getEnvVarWithDefault :: Text -> Text -> IO Text +getEnvVarWithDefault var def = do + varValue <- getEnv (toS var) `E.catchIOError` const (return $ toS def) + return $ toS varValue -testCfg :: AppConfig -testCfg = +testCfg :: Text -> AppConfig +testCfg testDbConn = AppConfig testDbConn "postgrest_test_anonymous" Nothing "test" "localhost" 3000 (Just "safe") 10 Nothing (Just "test.switch_role") True -testCfgNoJWT :: AppConfig -testCfgNoJWT = +testCfgNoJWT :: Text -> AppConfig +testCfgNoJWT testDbConn = AppConfig testDbConn "postgrest_test_anonymous" Nothing "test" "localhost" 3000 Nothing 10 Nothing Nothing True -testUnicodeCfg :: AppConfig -testUnicodeCfg = +testUnicodeCfg :: Text -> AppConfig +testUnicodeCfg testDbConn = AppConfig testDbConn "postgrest_test_anonymous" Nothing "تست" "localhost" 3000 (Just "safe") 10 Nothing Nothing True -testLtdRowsCfg :: AppConfig -testLtdRowsCfg = +testLtdRowsCfg :: Text -> AppConfig +testLtdRowsCfg testDbConn = AppConfig testDbConn "postgrest_test_anonymous" Nothing "test" "localhost" 3000 (Just "safe") 10 (Just 2) Nothing True -testProxyCfg :: AppConfig -testProxyCfg = +testProxyCfg :: Text -> AppConfig +testProxyCfg testDbConn = AppConfig testDbConn "postgrest_test_anonymous" (Just "https://postgrest.com/openapi.json") "test" "localhost" 3000 (Just "safe") 10 Nothing Nothing True -setupDb :: IO () -setupDb = do - void $ readProcess "psql" ["-d", "postgres", "-a", "-f", "test/fixtures/database.sql"] [] - void $ readProcess "psql" ["-d", "postgrest_test", "-a", "-c", "CREATE EXTENSION IF NOT EXISTS pgcrypto;"] [] - loadFixture "roles" - loadFixture "schema" - loadFixture "jwt" - loadFixture "privileges" - resetDb +resetDb :: Text -> IO () +resetDb dbConn = loadFixture dbConn "data" -resetDb :: IO () -resetDb = loadFixture "data" - -loadFixture :: FilePath -> IO() -loadFixture name = - void $ readProcess "psql" ["-U", "postgrest_test", "-d", "postgrest_test", "-a", "-f", "test/fixtures/" ++ name ++ ".sql"] [] +loadFixture :: Text -> FilePath -> IO() +loadFixture dbConn name = + void $ readProcess "psql" [toS dbConn, "-a", "-f", "test/fixtures/" ++ name ++ ".sql"] [] rangeHdrs :: ByteRange -> [Header] rangeHdrs r = [rangeUnit, (hRange, renderByteRange r)] diff --git a/test/create_test_db b/test/create_test_db new file mode 100755 index 000000000..c69c702ef --- /dev/null +++ b/test/create_test_db @@ -0,0 +1,93 @@ +#! /bin/bash +if [ -z "$1" ] + then + echo "Please supply the connection uri for the user with create database privileges" + exit -1 +fi + +if [ -z "$2" ] + then + echo "Please supply the test database name" + exit -1 +fi +if [[ $1 != postgres://* ]] +then + echo "Please use a valid connection URI (https://www.postgresql.org/docs/current/static/libpq-connect.html#AEN45347)" + exit -1 +fi + +BASEPATH=$( cd $(dirname $0) ; pwd -P ) +#Remove database path from the connection uri--prevents setting up the new database name with PGDATABASE +URI=$(echo $1 | cut -d'/' -f1-3) +#Extract host and port--we need this to form the new connection string +HOST_PORT=$(echo $URI | cut -d'/' -f3 | cut -d'@' -f2 ) +DB=$2 +# Specify the username of choice, or let the script create a random unique user by appending the database name +TEST_USER_NAME=${3:-postgrest_test_$DB} +# New password will get assigned only if the user does not already exist +# Otherwise make sure to provide the correct password for the existing user +TEST_USER_PASS=${4:-$(cat /dev/urandom | env LC_CTYPE=C tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1)a} + +PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" -Xq >/dev/null -c 'select rolcreatedb from pg_authid where rolname = current_user;' 2>/dev/null +if [ $? -ne 0 ]; then + echo "ERROR: Please specify the user with 'Create DB' permissions, and ensure that the default database for the username exists." + exit 1 +fi + +# plpgsql does not like psql variables, easier to pull this part off with bash variables +PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" -Xq >/dev/null < pg_backend_pid(); + +drop database if exists "$DB"; + +create database "$DB" encoding = 'UTF8'; + +DO \$\$ + BEGIN + IF NOT EXISTS (SELECT * FROM pg_catalog.pg_roles WHERE rolname = '$TEST_USER_NAME') + THEN CREATE ROLE $TEST_USER_NAME WITH LOGIN NOINHERIT PASSWORD '$TEST_USER_PASS'; + END IF; + END \$\$; +DO \$\$ + BEGIN + IF NOT EXISTS (SELECT * FROM pg_catalog.pg_roles WHERE rolname = 'postgrest_test_anonymous') + THEN CREATE ROLE postgrest_test_anonymous; + END IF; + END \$\$; +DO \$\$ + BEGIN + IF NOT EXISTS (SELECT * FROM pg_catalog.pg_roles WHERE rolname = 'postgrest_test_default_role') + THEN CREATE ROLE postgrest_test_default_role; + END IF; + END \$\$; +DO \$\$ + BEGIN + IF NOT EXISTS (SELECT * FROM pg_catalog.pg_roles WHERE rolname = 'postgrest_test_author') + THEN CREATE ROLE postgrest_test_author; + END IF; + END \$\$; + +DO \$\$ + BEGIN + IF NOT EXISTS (select * from pg_roles where rolname = 'postgrest_test_anonymous' AND pg_has_role('$TEST_USER_NAME', oid, 'member')) + THEN GRANT postgrest_test_anonymous TO $TEST_USER_NAME; + END IF; + IF NOT EXISTS (select * from pg_roles where rolname = 'postgrest_test_author' AND pg_has_role('$TEST_USER_NAME', oid, 'member')) + THEN GRANT postgrest_test_author TO $TEST_USER_NAME; + END IF; + IF NOT EXISTS (select * from pg_roles where rolname = 'postgrest_test_default_role' AND pg_has_role('$TEST_USER_NAME', oid, 'member')) + THEN GRANT postgrest_test_default_role TO $TEST_USER_NAME; + END IF; +END \$\$; +EOF + +PGDATABASE=$DB PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" --set=db=$DB -Xqf $BASEPATH/fixtures/database.sql +PGDATABASE=$DB PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" -Xqf $BASEPATH/fixtures/schema.sql +PGDATABASE=$DB PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" -Xqf $BASEPATH/fixtures/jwt.sql +PGDATABASE=$DB PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" --set=test_user_name="$TEST_USER_NAME" -Xqf $BASEPATH/fixtures/privileges.sql + +# Create a new connection string to use with the test runner +echo 'postgres://'${TEST_USER_NAME}':'$TEST_USER_PASS'@'$HOST_PORT'/'$DB diff --git a/test/destroy_test_db b/test/destroy_test_db new file mode 100755 index 000000000..4f5ccbe96 --- /dev/null +++ b/test/destroy_test_db @@ -0,0 +1,66 @@ +#! /bin/bash +if [ -z "$1" ] + then + echo "Please supply the connection uri for the user with create database privileges" + exit -1 +fi + +if [ -z "$2" ] + then + echo "Please supply the test database name" + exit -1 +fi +if [[ $1 != postgres://* ]] +then + echo "Please use a valid connection URI (https://www.postgresql.org/docs/current/static/libpq-connect.html#AEN45347)" + exit -1 +fi + +BASEPATH=$( cd $(dirname $0) ; pwd -P ) +#Remove database path from the connection uri +URI=$(echo $1 | cut -d'/' -f1-3) +DB=$2 + +PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" -Xq >/dev/null -c 'select rolcreatedb from pg_authid where rolname = current_user;' 2>/dev/null +if [ $? -ne 0 ]; then + echo "ERROR: Please specify the user with 'Create DB' permissions, and ensure that the default database for the username exists." + exit 1 +fi + +# plpgsql does not like psql variables, easier to pull this part off with bash variables +PGOPTIONS='-c client_min_messages=WARNING' psql "$URI" -Xq >/dev/null < pg_backend_pid(); + +drop database if exists "$DB"; + +-- Find all test users that were members of role 'postgrest_test_author'--that way we don't have to know the +-- test user name (in case it was auto-generated). +DO \$\$ +DECLARE + mem text; +BEGIN + FOR mem IN + SELECT pg_get_userbyid(member) + FROM pg_roles r + JOIN pg_auth_members m + ON m.roleid = r.oid + WHERE rolname = 'postgrest_test_author' + LOOP + EXECUTE 'drop role '|| mem || ';'; + END LOOP; +END \$\$; + +DO \$\$ +DECLARE + r text; +BEGIN + FOR r IN + VALUES('postgrest_test_author'),('postgrest_test_anonymous'),('postgrest_test_default_role') + LOOP + EXECUTE 'drop role if exists '|| r || ';'; + END LOOP; +END \$\$; +EOF diff --git a/test/fixtures/database.sql b/test/fixtures/database.sql index a92e998e7..d89aaa6fe 100644 --- a/test/fixtures/database.sql +++ b/test/fixtures/database.sql @@ -1,6 +1,3 @@ -DROP DATABASE IF EXISTS postgrest_test; -DROP ROLE IF EXISTS postgrest_test; -CREATE USER postgrest_test createdb createrole; -CREATE DATABASE postgrest_test OWNER postgrest_test; +CREATE EXTENSION IF NOT EXISTS pgcrypto; -ALTER DATABASE postgrest_test SET request.jwt.claim.id = '-1'; +ALTER DATABASE :db SET request.jwt.claim.id = '-1'; diff --git a/test/fixtures/privileges.sql b/test/fixtures/privileges.sql index 54a5cd160..3621d2c64 100644 --- a/test/fixtures/privileges.sql +++ b/test/fixtures/privileges.sql @@ -57,3 +57,8 @@ TO postgrest_test_anonymous; -- Privileges for non anonymous users GRANT USAGE ON SCHEMA test TO postgrest_test_author; GRANT ALL ON TABLE authors_only TO postgrest_test_author; + +GRANT USAGE ON SCHEMA postgrest,private,test to :test_user_name; +GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA postgrest,private,test TO :test_user_name; +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA postgrest,private,test TO :test_user_name; + diff --git a/test/fixtures/roles.sql b/test/fixtures/roles.sql deleted file mode 100644 index a146a8fc8..000000000 --- a/test/fixtures/roles.sql +++ /dev/null @@ -1,7 +0,0 @@ -DROP ROLE IF EXISTS postgrest_test_authenticator, postgrest_test_anonymous, postgrest_test_default_role, postgrest_test_author; -CREATE ROLE postgrest_test_authenticator WITH login noinherit; -CREATE ROLE postgrest_test_anonymous; -CREATE ROLE postgrest_test_default_role; -CREATE ROLE postgrest_test_author; - -GRANT postgrest_test_anonymous, postgrest_test_default_role, postgrest_test_author TO postgrest_test_authenticator;