feat: Make db-uri optional

The default is now "postgresql://" which falls back to LIBPQ environment variables.

Resolves #1991, Ref #1823
This commit is contained in:
Wolfgang Walther
2022-01-22 15:13:19 +01:00
parent ec09b87940
commit 9ed30c0ba4
21 changed files with 108 additions and 107 deletions
+1
View File
@@ -12,6 +12,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
+ A `<host>:<admin_server_port>/live` endpoint is available for checking if postgrest is running on its port/socket. 200 OK = alive, 503 = dead.
+ A `<host>:<admin_server_port>/ready` endpoint is available for checking a correct internal state(the database connection plus the schema cache). 200 OK = ready, 503 = not ready.
- #1988, Add the current user to the request log on stdout - @DavidLindbom, @wolfgangwalther
- #1991, Add the ability to run without `db-uri` using libpq's PG environment variables to connect. @wolfgangwalther
### Fixed
+3
View File
@@ -45,6 +45,9 @@ let
inRootDir = true;
}
''
# previously required settings to make this work with older branches
export PGRST_DB_URI="postgresql://"
export PGRST_DB_CONFIG="false"
export PGRST_DB_POOL="1"
export PGRST_DB_TX_END="rollback-allow-override"
+1 -2
View File
@@ -35,7 +35,7 @@ let
}
''
# avoid starting multiple layers of withTmpDb
if test -v PGRST_DB_URI; then
if test -v PGHOST; then
exec "$_arg_command" "''${_arg_leftovers[@]}"
fi
@@ -53,7 +53,6 @@ let
export PGHOST="$tmpdir/socket"
export PGUSER
export PGDATABASE
export PGRST_DB_URI="postgresql:///$PGDATABASE?host=$PGHOST&user=$PGUSER"
export PGRST_DB_SCHEMAS
export PGRST_DB_ANON_ROLE
+1 -1
View File
@@ -226,7 +226,7 @@ parser optPath env dbSettings =
<*> (fromMaybe True <$> optBool "db-config")
<*> parseTxEnd "db-tx-end" snd
<*> parseTxEnd "db-tx-end" fst
<*> reqString "db-uri"
<*> (fromMaybe "postgresql://" <$> optString "db-uri")
<*> (fromMaybe True <$> optBool "db-use-legacy-gucs")
<*> pure optPath
<*> pure Nothing
-1
View File
@@ -1,5 +1,4 @@
db-anon-role = "required"
db-uri = "required"
db-schema = "provided_through_alias"
max-rows = 1000
-1
View File
@@ -1,4 +1,3 @@
db-uri = "required"
db-schemas = "required"
db-anon-role = "required"
-1
View File
@@ -1,4 +1,3 @@
db-uri = "required"
db-schemas = "required"
db-anon-role = "required"
+1 -2
View File
@@ -1,5 +1,4 @@
db-uri = "required"
db-schemas = "required"
db-anon-role = "required"
# Not the default, but only works with proper db-uri
# Not the default, but only works with PG* variables, which are not set
db-config = false
+1 -1
View File
@@ -11,7 +11,7 @@ db-root-spec = "open_alias"
db-schemas = "provided_through_alias"
db-config = true
db-tx-end = "commit"
db-uri = "required"
db-uri = "postgresql://"
db-use-legacy-gucs = true
jwt-aud = ""
jwt-role-claim-key = ".\"aliased\""
@@ -11,7 +11,7 @@ db-root-spec = ""
db-schemas = "required"
db-config = true
db-tx-end = "commit"
db-uri = "required"
db-uri = "postgresql://"
db-use-legacy-gucs = true
jwt-aud = ""
jwt-role-claim-key = ".\"role\""
@@ -11,7 +11,7 @@ db-root-spec = ""
db-schemas = "required"
db-config = true
db-tx-end = "commit"
db-uri = "required"
db-uri = "postgresql://"
db-use-legacy-gucs = true
jwt-aud = ""
jwt-role-claim-key = ".\"role\""
+1 -1
View File
@@ -11,7 +11,7 @@ db-root-spec = ""
db-schemas = "required"
db-config = false
db-tx-end = "commit"
db-uri = "required"
db-uri = "postgresql://"
db-use-legacy-gucs = true
jwt-aud = ""
jwt-role-claim-key = ".\"role\""
@@ -11,7 +11,7 @@ db-root-spec = "other_root"
db-schemas = "test,other_tenant1,other_tenant2"
db-config = true
db-tx-end = "rollback-allow-override"
db-uri = "<REPLACED_WITH_DB_URI>"
db-uri = "postgresql://"
db-use-legacy-gucs = false
jwt-aud = "https://otherexample.org"
jwt-role-claim-key = ".\"other\".\"role\""
@@ -11,7 +11,7 @@ db-root-spec = "root"
db-schemas = "test,tenant1,tenant2"
db-config = true
db-tx-end = "commit-allow-override"
db-uri = "<REPLACED_WITH_DB_URI>"
db-uri = "postgresql://"
db-use-legacy-gucs = false
jwt-aud = "https://example.org"
jwt-role-claim-key = ".\"a\".\"role\""
+1 -1
View File
@@ -11,7 +11,7 @@ db-root-spec = ""
db-schemas = "required"
db-config = true
db-tx-end = "commit"
db-uri = "required"
db-uri = "postgresql://"
db-use-legacy-gucs = true
jwt-aud = ""
jwt-role-claim-key = ".\"role\""
-1
View File
@@ -1,7 +1,6 @@
# tests how config options fall back with invalid types
db-anon-role = "required"
db-schemas = "required"
db-uri = "required"
# expects string
app.settings.test = false
-7
View File
@@ -27,18 +27,11 @@ cli:
- name: missing db-anon-role
expect: error
env:
PGRST_DB_URI: required
PGRST_DB_SCHEMAS: required
- name: missing db-schemas
expect: error
env:
PGRST_DB_ANON_ROLE: required
PGRST_DB_URI: required
- name: missing db-uri
expect: error
env:
PGRST_DB_ANON_ROLE: required
PGRST_DB_SCHEMAS: required
# failures: wrong config values
- name: invalid server-unix-socket-mode not octal
expect: error
+35 -17
View File
@@ -79,14 +79,19 @@ class PostgrestProcess:
@pytest.fixture
def dburi():
"Postgres database connection URI."
return os.getenv("PGRST_DB_URI").encode()
dbname = os.environ["PGDATABASE"]
host = os.environ["PGHOST"]
user = os.environ["PGUSER"]
return f"postgresql://?dbname={dbname}&host={host}&user={user}".encode()
@pytest.fixture
def defaultenv():
"Default environment for PostgREST."
return {
"PGRST_DB_URI": os.environ["PGRST_DB_URI"],
"PGDATABASE": os.environ["PGDATABASE"],
"PGHOST": os.environ["PGHOST"],
"PGUSER": os.environ["PGUSER"],
"PGRST_DB_SCHEMAS": "public",
"PGRST_DB_ANON_ROLE": os.environ["PGRST_DB_ANON_ROLE"],
"PGRST_DB_CONFIG": "false",
@@ -309,20 +314,14 @@ def test_expected_config_from_db_settings(defaultenv, role, expectedconfig):
config = CONFIGSDIR / "no-defaults.config"
db_uri = defaultenv["PGRST_DB_URI"].replace(
"user=postgrest_test_authenticator", f"user={role}"
)
env = {
**defaultenv,
"PGRST_DB_URI": db_uri,
"PGUSER": role,
"PGRST_DB_URI": "postgresql://",
"PGRST_DB_CONFIG": "true",
}
expected = (
(CONFIGSDIR / "expected" / expectedconfig)
.read_text()
.replace("<REPLACED_WITH_DB_URI>", env["PGRST_DB_URI"])
)
expected = (CONFIGSDIR / "expected" / expectedconfig).read_text()
assert dumpconfig(configpath=config, env=env) == expected
@@ -418,9 +417,26 @@ def test_read_secret_from_stdin_dbconfig(defaultenv):
assert response.status_code == 200
def test_connect_with_dburi(dburi, defaultenv):
"Connecting with db-uri instead of LIPQ* environment variables should work."
defaultenv_without_libpq = {
key: value
for key, value in defaultenv.items()
if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
}
env = {**defaultenv_without_libpq, "PGRST_DB_URI": dburi.decode()}
with run(env=env):
pass
def test_read_dburi_from_stdin_without_eol(dburi, defaultenv):
"Reading the dburi from stdin with a single line should work."
env = {**defaultenv, "PGRST_DB_URI": "@/dev/stdin"}
defaultenv_without_libpq = {
key: value
for key, value in defaultenv.items()
if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
}
env = {**defaultenv_without_libpq, "PGRST_DB_URI": "@/dev/stdin"}
with run(env=env, stdin=dburi):
pass
@@ -428,7 +444,12 @@ def test_read_dburi_from_stdin_without_eol(dburi, defaultenv):
def test_read_dburi_from_stdin_with_eol(dburi, defaultenv):
"Reading the dburi from stdin containing a newline should work."
env = {**defaultenv, "PGRST_DB_URI": "@/dev/stdin"}
defaultenv_without_libpq = {
key: value
for key, value in defaultenv.items()
if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
}
env = {**defaultenv_without_libpq, "PGRST_DB_URI": "@/dev/stdin"}
with run(env=env, stdin=dburi + b"\n"):
pass
@@ -802,12 +823,9 @@ def test_admin_ready_wo_channel(defaultenv):
def test_admin_ready_includes_schema_cache_state(defaultenv):
"Should get a failed response from the admin server ready endpoint when the schema cache is not loaded"
db_uri = defaultenv["PGRST_DB_URI"].replace(
"postgrest_test_authenticator", "limited_authenticator"
)
env = {
**defaultenv,
"PGRST_DB_URI": db_uri,
"PGUSER": "limited_authenticator",
"PGRST_DB_ANON_ROLE": "limited_authenticator",
}
+7 -11
View File
@@ -58,22 +58,19 @@ import qualified Feature.UpsertSpec
main :: IO ()
main = do
testDbConn <- getEnvVarWithDefault "PGRST_DB_URI" "postgres://postgrest_test@localhost/postgrest_test"
pool <- P.acquire (3, 10, toS testDbConn)
pool <- P.acquire (3, 10, toUtf8 $ configDbUri testCfg)
actualPgVersion <- either (panic.show) id <$> P.use pool queryPgVersion
baseDbStructure <-
loadDbStructure pool
(configDbSchemas $ testCfg testDbConn)
(configDbExtraSearchPath $ testCfg testDbConn)
(configDbSchemas testCfg)
(configDbExtraSearchPath testCfg)
actualPgVersion
let
-- For tests that run with the same refDbStructure
app cfg = do
let config = cfg testDbConn
app config = do
appState <- AppState.initWithPool pool config
AppState.putPgVersion appState actualPgVersion
AppState.putDbStructure appState (Just baseDbStructure)
@@ -82,8 +79,7 @@ main = do
return ((), postgrest LogCrit appState $ pure ())
-- For tests that run with a different DbStructure(depends on configSchemas)
appDbs cfg = do
let config = cfg testDbConn
appDbs config = do
customDbStructure <-
loadDbStructure pool
(configDbSchemas config)
@@ -120,8 +116,8 @@ main = do
let analyze :: IO ()
analyze = do
analyzeTable testDbConn "items"
analyzeTable testDbConn "child_entities"
analyzeTable "items"
analyzeTable "child_entities"
specs = uncurry describe <$> [
("Feature.AndOrParamsSpec" , Feature.AndOrParamsSpec.spec actualPgVersion)
+2 -6
View File
@@ -11,8 +11,7 @@ import qualified Hasql.Transaction as HT
import qualified Hasql.Transaction.Sessions as HT
import Text.Heredoc
import Protolude hiding (get, toS)
import Protolude.Conv (toS)
import Protolude hiding (get, toS)
import PostgREST.Query.QueryBuilder (requestToCallProcQuery)
import PostgREST.Request.Types
@@ -20,14 +19,11 @@ import PostgREST.Request.Types
import PostgREST.DbStructure.Identifiers
import PostgREST.DbStructure.Proc
import SpecHelper (getEnvVarWithDefault)
import Test.Hspec
main :: IO ()
main = do
testDbConn <- getEnvVarWithDefault "PGRST_DB_URI" "postgres://postgrest_test@localhost/postgrest_test"
pool <- P.acquire (3, 10, toS testDbConn)
pool <- P.acquire (3, 10, "postgresql://")
hspec $ describe "QueryCost" $
context "call proc query" $ do
+50 -50
View File
@@ -72,8 +72,8 @@ getEnvVarWithDefault :: Text -> Text -> IO Text
getEnvVarWithDefault var def = toS <$>
getEnv (toS var) `E.catchIOError` const (return $ toS def)
_baseCfg :: AppConfig
_baseCfg = let secret = Just $ encodeUtf8 "reallyreallyreallyreallyverysafe" in
baseCfg :: AppConfig
baseCfg = let secret = Just $ encodeUtf8 "reallyreallyreallyreallyverysafe" in
AppConfig {
configAppSettings = [ ("app.settings.app_host", "localhost") , ("app.settings.external_api_secret", "0123456789abcdef") ]
, configDbAnonRole = "postgrest_test_anonymous"
@@ -88,7 +88,7 @@ _baseCfg = let secret = Just $ encodeUtf8 "reallyreallyreallyreallyverysafe" in
, configDbRootSpec = Nothing
, configDbSchemas = fromList ["test"]
, configDbConfig = False
, configDbUri = mempty
, configDbUri = "postgresql://"
, configDbUseLegacyGucs = True
, configFilePath = Nothing
, configJWKS = parseSecret <$> secret
@@ -109,90 +109,90 @@ _baseCfg = let secret = Just $ encodeUtf8 "reallyreallyreallyreallyverysafe" in
, configAdminServerPort = Nothing
}
testCfg :: Text -> AppConfig
testCfg testDbConn = _baseCfg { configDbUri = testDbConn }
testCfg :: AppConfig
testCfg = baseCfg
testCfgDisallowRollback :: Text -> AppConfig
testCfgDisallowRollback testDbConn = (testCfg testDbConn) { configDbTxAllowOverride = False, configDbTxRollbackAll = False }
testCfgDisallowRollback :: AppConfig
testCfgDisallowRollback = baseCfg { configDbTxAllowOverride = False, configDbTxRollbackAll = False }
testCfgForceRollback :: Text -> AppConfig
testCfgForceRollback testDbConn = (testCfg testDbConn) { configDbTxAllowOverride = False, configDbTxRollbackAll = True }
testCfgForceRollback :: AppConfig
testCfgForceRollback = baseCfg { configDbTxAllowOverride = False, configDbTxRollbackAll = True }
testCfgNoJWT :: Text -> AppConfig
testCfgNoJWT testDbConn = (testCfg testDbConn) { configJwtSecret = Nothing, configJWKS = Nothing }
testCfgNoJWT :: AppConfig
testCfgNoJWT = baseCfg { configJwtSecret = Nothing, configJWKS = Nothing }
testUnicodeCfg :: Text -> AppConfig
testUnicodeCfg testDbConn = (testCfg testDbConn) { configDbSchemas = fromList ["تست"] }
testUnicodeCfg :: AppConfig
testUnicodeCfg = baseCfg { configDbSchemas = fromList ["تست"] }
testMaxRowsCfg :: Text -> AppConfig
testMaxRowsCfg testDbConn = (testCfg testDbConn) { configDbMaxRows = Just 2 }
testMaxRowsCfg :: AppConfig
testMaxRowsCfg = baseCfg { configDbMaxRows = Just 2 }
testDisabledOpenApiCfg :: Text -> AppConfig
testDisabledOpenApiCfg testDbConn = (testCfg testDbConn) { configOpenApiMode = OADisabled }
testDisabledOpenApiCfg :: AppConfig
testDisabledOpenApiCfg = baseCfg { configOpenApiMode = OADisabled }
testIgnorePrivOpenApiCfg :: Text -> AppConfig
testIgnorePrivOpenApiCfg testDbConn = (testCfg testDbConn) { configOpenApiMode = OAIgnorePriv, configDbSchemas = fromList ["test", "v1"] }
testIgnorePrivOpenApiCfg :: AppConfig
testIgnorePrivOpenApiCfg = baseCfg { configOpenApiMode = OAIgnorePriv, configDbSchemas = fromList ["test", "v1"] }
testProxyCfg :: Text -> AppConfig
testProxyCfg testDbConn = (testCfg testDbConn) { configOpenApiServerProxyUri = Just "https://postgrest.com/openapi.json" }
testProxyCfg :: AppConfig
testProxyCfg = baseCfg { configOpenApiServerProxyUri = Just "https://postgrest.com/openapi.json" }
testCfgBinaryJWT :: Text -> AppConfig
testCfgBinaryJWT testDbConn =
testCfgBinaryJWT :: AppConfig
testCfgBinaryJWT =
let secret = Just . B64.decodeLenient $ "cmVhbGx5cmVhbGx5cmVhbGx5cmVhbGx5dmVyeXNhZmU=" in
(testCfg testDbConn) {
baseCfg {
configJwtSecret = secret
, configJWKS = parseSecret <$> secret
}
testCfgAudienceJWT :: Text -> AppConfig
testCfgAudienceJWT testDbConn =
testCfgAudienceJWT :: AppConfig
testCfgAudienceJWT =
let secret = Just . B64.decodeLenient $ "cmVhbGx5cmVhbGx5cmVhbGx5cmVhbGx5dmVyeXNhZmU=" in
(testCfg testDbConn) {
baseCfg {
configJwtSecret = secret
, configJwtAudience = Just "youraudience"
, configJWKS = parseSecret <$> secret
}
testCfgAsymJWK :: Text -> AppConfig
testCfgAsymJWK testDbConn =
testCfgAsymJWK :: AppConfig
testCfgAsymJWK =
let secret = Just $ encodeUtf8 [str|{"alg":"RS256","e":"AQAB","key_ops":["verify"],"kty":"RSA","n":"0etQ2Tg187jb04MWfpuogYGV75IFrQQBxQaGH75eq_FpbkyoLcEpRUEWSbECP2eeFya2yZ9vIO5ScD-lPmovePk4Aa4SzZ8jdjhmAbNykleRPCxMg0481kz6PQhnHRUv3nF5WP479CnObJKqTVdEagVL66oxnX9VhZG9IZA7k0Th5PfKQwrKGyUeTGczpOjaPqbxlunP73j9AfnAt4XCS8epa-n3WGz1j-wfpr_ys57Aq-zBCfqP67UYzNpeI1AoXsJhD9xSDOzvJgFRvc3vm2wjAW4LEMwi48rCplamOpZToIHEPIaPzpveYQwDnB1HFTR1ove9bpKJsHmi-e2uzQ","use":"sig"}|]
in (testCfg testDbConn) {
in baseCfg {
configJwtSecret = secret
, configJWKS = parseSecret <$> secret
}
testCfgAsymJWKSet :: Text -> AppConfig
testCfgAsymJWKSet testDbConn =
testCfgAsymJWKSet :: AppConfig
testCfgAsymJWKSet =
let secret = Just $ encodeUtf8 [str|{"keys": [{"alg":"RS256","e":"AQAB","key_ops":["verify"],"kty":"RSA","n":"0etQ2Tg187jb04MWfpuogYGV75IFrQQBxQaGH75eq_FpbkyoLcEpRUEWSbECP2eeFya2yZ9vIO5ScD-lPmovePk4Aa4SzZ8jdjhmAbNykleRPCxMg0481kz6PQhnHRUv3nF5WP479CnObJKqTVdEagVL66oxnX9VhZG9IZA7k0Th5PfKQwrKGyUeTGczpOjaPqbxlunP73j9AfnAt4XCS8epa-n3WGz1j-wfpr_ys57Aq-zBCfqP67UYzNpeI1AoXsJhD9xSDOzvJgFRvc3vm2wjAW4LEMwi48rCplamOpZToIHEPIaPzpveYQwDnB1HFTR1ove9bpKJsHmi-e2uzQ","use":"sig"}]}|]
in (testCfg testDbConn) {
in baseCfg {
configJwtSecret = secret
, configJWKS = parseSecret <$> secret
}
testNonexistentSchemaCfg :: Text -> AppConfig
testNonexistentSchemaCfg testDbConn = (testCfg testDbConn) { configDbSchemas = fromList ["nonexistent"] }
testNonexistentSchemaCfg :: AppConfig
testNonexistentSchemaCfg = baseCfg { configDbSchemas = fromList ["nonexistent"] }
testCfgExtraSearchPath :: Text -> AppConfig
testCfgExtraSearchPath testDbConn = (testCfg testDbConn) { configDbExtraSearchPath = ["public", "extensions"] }
testCfgExtraSearchPath :: AppConfig
testCfgExtraSearchPath = baseCfg { configDbExtraSearchPath = ["public", "extensions"] }
testCfgRootSpec :: Text -> AppConfig
testCfgRootSpec testDbConn = (testCfg testDbConn) { configDbRootSpec = Just $ QualifiedIdentifier mempty "root"}
testCfgRootSpec :: AppConfig
testCfgRootSpec = baseCfg { configDbRootSpec = Just $ QualifiedIdentifier mempty "root"}
testCfgHtmlRawOutput :: Text -> AppConfig
testCfgHtmlRawOutput testDbConn = (testCfg testDbConn) { configRawMediaTypes = ["text/html"] }
testCfgHtmlRawOutput :: AppConfig
testCfgHtmlRawOutput = baseCfg { configRawMediaTypes = ["text/html"] }
testCfgResponseHeaders :: Text -> AppConfig
testCfgResponseHeaders testDbConn = (testCfg testDbConn) { configDbPreRequest = Just $ QualifiedIdentifier mempty "custom_headers" }
testCfgResponseHeaders :: AppConfig
testCfgResponseHeaders = baseCfg { configDbPreRequest = Just $ QualifiedIdentifier mempty "custom_headers" }
testMultipleSchemaCfg :: Text -> AppConfig
testMultipleSchemaCfg testDbConn = (testCfg testDbConn) { configDbSchemas = fromList ["v1", "v2"] }
testMultipleSchemaCfg :: AppConfig
testMultipleSchemaCfg = baseCfg { configDbSchemas = fromList ["v1", "v2"] }
testCfgLegacyGucs :: Text -> AppConfig
testCfgLegacyGucs testDbConn = (testCfg testDbConn) { configDbUseLegacyGucs = False }
testCfgLegacyGucs :: AppConfig
testCfgLegacyGucs = baseCfg { configDbUseLegacyGucs = False }
analyzeTable :: Text -> Text -> IO ()
analyzeTable dbConn tableName =
void $ readProcess "psql" ["--set", "ON_ERROR_STOP=1", toS dbConn, "-a", "-c", toS $ "ANALYZE test.\"" <> tableName <> "\""] []
analyzeTable :: Text -> IO ()
analyzeTable tableName =
void $ readProcess "psql" ["--set", "ON_ERROR_STOP=1", "-a", "-c", toS $ "ANALYZE test.\"" <> tableName <> "\""] []
rangeHdrs :: ByteRange -> [Header]
rangeHdrs r = [rangeUnit, (hRange, renderByteRange r)]