jimmy/postgrest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: Separated query and loading internal sleep configs

Browse Source 
To make schema cache loading wait tests robust it is necessary to provide three separate internal config variables:
* "internal-schema-cache-query-sleep" - introduces delay in schema queries execution
* "internal-schema-cache-load-sleep" - introduces delay between schema queries execution and processing their results
* "internal-schema-cache-relationship-load-sleep" - introduces delay in processing relationship query results

Thanks to these changes it is now possible to test various schema loading scenarios with the right granularity robustly (eg. make sure requests wait for schema loading but not for relationship loading).
This commit is contained in:
Michał Kłeczek
2025-10-17 13:42:35 -05:00
committed by Steve Chavez
parent 07681d1b5b
commit c08b87749b
7 changed files with 32 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/PostgREST/Config.hs
+6 -2
View File
@@ -115,7 +115,9 @@ data AppConfig = AppConfig
, configAdminServerPort :: Maybe Int
, configRoleSettings :: RoleSettings
, configRoleIsoLvl :: RoleIsolationLvl
, configInternalSCSleep :: Maybe Int32
, configInternalSCQuerySleep :: Maybe Int32
, configInternalSCLoadSleep :: Maybe Int32
, configInternalSCRelLoadSleep :: Maybe Int32
}
data LogLevel = LogCrit | LogError | LogWarn | LogInfo | LogDebug
@@ -298,7 +300,9 @@ parser optPath env dbSettings roleSettings roleIsolationLvl =
<*> parseAdminServerPort "admin-server-port"
<*> pure roleSettings
<*> pure roleIsolationLvl
<*> optInt "internal-schema-cache-sleep"
<*> optInt "internal-schema-cache-query-sleep"
<*> optInt "internal-schema-cache-load-sleep"
<*> optInt "internal-schema-cache-relationship-load-sleep"
where
parseAppSettings :: C.Key -> C.Parser C.Config [(Text, Text)]
parseAppSettings key = addFromEnv . fmap (fmap coerceText) <$> C.subassocs key C.value
src/PostgREST/SchemaCache.hs
+7 -5
View File
@@ -25,8 +25,6 @@ module PostgREST.SchemaCache
, decodeFuncs
) where
import Control.Monad.Extra (whenJust)
import Data.Aeson ((.=))
import qualified Data.Aeson as JSON
import qualified Data.HashMap.Strict as HM
@@ -69,6 +67,7 @@ import PostgREST.SchemaCache.Table (Column (..), ColumnMap,
import qualified PostgREST.MediaType as MediaType
import Protolude
import System.IO.Unsafe (unsafePerformIO)
data SchemaCache = SchemaCache
{ dbTables :: TablesMap
@@ -152,14 +151,16 @@ querySchemaCache conf@AppConfig{..} = do
tzones <- SQL.statement mempty $ timezones prepared
_ <-
let sleepCall = SQL.Statement "select pg_sleep($1 / 1000.0)" (param HE.int4) HD.noResult prepared in
whenJust configInternalSCSleep (`SQL.statement` sleepCall) -- only used for testing
for_ configInternalSCQuerySleep (`SQL.statement` sleepCall) -- only used for testing
let tabsWViewsPks = addViewPrimaryKeys tabs keyDeps
rels = addInverseRels $ addM2MRels tabsWViewsPks $ addViewM2OAndO2ORels keyDeps m2oRels
return $ removeInternal schemas $ SchemaCache {
-- Add delay in loading schema cache when internal-schema-cache-load-sleep config is set
return $ delayEval configInternalSCLoadSleep $ removeInternal schemas $ SchemaCache {
dbTables = tabsWViewsPks
, dbRelationships = getOverrideRelationshipsMap rels cRels
-- Add delay in loading relationships when internal-schema-cache-relationship-load-sleep config is set
, dbRelationships = delayEval configInternalSCRelLoadSleep $ getOverrideRelationshipsMap rels cRels
, dbRoutines = funcs
, dbRepresentations = reps
, dbMediaHandlers = HM.union mHdlers initialMediaHandlers -- the custom handlers will override the initial ones
@@ -168,6 +169,7 @@ querySchemaCache conf@AppConfig{..} = do
where
schemas = toList configDbSchemas
prepared = configDbPreparedStatements
delayEval confDelay result = maybe result (unsafePerformIO . (($> result) . (threadDelay . (1000 *) . fromIntegral))) confDelay
-- | overrides detected relationships with the computed relationships and gets the RelationshipsMap
getOverrideRelationshipsMap :: [Relationship] -> [Relationship] -> RelationshipsMap
test/io/conftest.py
+1 -1
View File
@@ -61,7 +61,7 @@ def slow_schema_cache_env(defaultenv):
"Slow schema cache load environment PostgREST."
return {
**defaultenv,
"PGRST_INTERNAL_SCHEMA_CACHE_SLEEP": "1000", # this does a pg_sleep internally, it will cause the schema cache query to be slow
"PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP": "1000", # this does a pg_sleep internally, it will cause the schema cache query to be slow
# the slow schema cache query will keep using one pool connection until it finishes
# to prevent requests waiting for PGRST_DB_POOL_ACQUISITION_TIMEOUT we'll increase the pool size (must be >= 2)
"PGRST_DB_POOL": "2",
test/io/test_big_schema.py
+8 -11
View File
@@ -2,7 +2,6 @@
import pytest
from util import parse_server_timings_header
from postgrest import run
@@ -14,7 +13,7 @@ def test_requests_with_resource_embedding_wait_for_schema_cache_reload(defaulten
"PGRST_DB_SCHEMAS": "apflora",
"PGRST_DB_POOL": "2",
"PGRST_DB_ANON_ROLE": "postgrest_test_anonymous",
"PGRST_SERVER_TIMING_ENABLED": "true",
"PGRST_INTERNAL_SCHEMA_CACHE_RELATIONSHIP_LOAD_SLEEP": "5100",
}
with run(env=env, wait_max_seconds=30) as postgrest:
@@ -27,10 +26,7 @@ def test_requests_with_resource_embedding_wait_for_schema_cache_reload(defaulten
response = postgrest.session.get("/tpopmassn?select=*,tpop(*)")
assert response.status_code == 200
plan_dur = parse_server_timings_header(response.headers["Server-Timing"])[
"plan"
]
assert plan_dur > 10000.0
assert response.elapsed.total_seconds() > 5
def test_requests_without_resource_embedding_wait_for_schema_cache_reload(defaultenv):
@@ -41,7 +37,8 @@ def test_requests_without_resource_embedding_wait_for_schema_cache_reload(defaul
"PGRST_DB_SCHEMAS": "apflora",
"PGRST_DB_POOL": "2",
"PGRST_DB_ANON_ROLE": "postgrest_test_anonymous",
"PGRST_SERVER_TIMING_ENABLED": "true",
"PGRST_INTERNAL_SCHEMA_CACHE_LOAD_SLEEP": "1100",
"PGRST_INTERNAL_SCHEMA_CACHE_RELATIONSHIP_LOAD_SLEEP": "5000",
}
with run(env=env, wait_max_seconds=30) as postgrest:
@@ -54,10 +51,10 @@ def test_requests_without_resource_embedding_wait_for_schema_cache_reload(defaul
response = postgrest.session.get("/tpopmassn")
assert response.status_code == 200
plan_dur = parse_server_timings_header(response.headers["Server-Timing"])[
"plan"
]
assert plan_dur < 10000.0
assert (
response.elapsed.total_seconds() > 1
and response.elapsed.total_seconds() < 5
)
# TODO: This test fails now because of https://github.com/PostgREST/postgrest/pull/2122
test/io/test_cli.py
+2 -2
View File
@@ -312,13 +312,13 @@ def test_cli_ready_flag_fail_when_schema_cache_not_loaded(defaultenv, metapostgr
**defaultenv,
"PGUSER": role,
"PGRST_DB_ANON_ROLE": role,
"PGRST_INTERNAL_SCHEMA_CACHE_SLEEP": "500",
"PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP": "500",
}
port = freeport()
with run(env=env, port=port) as postgrest:
# The schema cache query takes at least 500ms, due to PGRST_INTERNAL_SCHEMA_CACHE_SLEEP above.
# The schema cache query takes at least 500ms, due to PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP above.
# Make it impossible to load the schema cache, by setting statement timeout to 400ms.
set_statement_timeout(metapostgrest, role, 400)
test/io/test_io.py
+5 -5
View File
@@ -825,11 +825,11 @@ def test_admin_ready_includes_schema_cache_state(defaultenv, metapostgrest):
**defaultenv,
"PGUSER": role,
"PGRST_DB_ANON_ROLE": role,
"PGRST_INTERNAL_SCHEMA_CACHE_SLEEP": "500",
"PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP": "500",
}
with run(env=env) as postgrest:
# The schema cache query takes at least 500ms, due to PGRST_INTERNAL_SCHEMA_CACHE_SLEEP above.
# The schema cache query takes at least 500ms, due to PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP above.
# Make it impossible to load the schema cache, by setting statement timeout to 400ms.
set_statement_timeout(metapostgrest, role, 400)
@@ -855,11 +855,11 @@ def test_metrics_include_schema_cache_fails(defaultenv, metapostgrest):
env = {
**defaultenv,
"PGUSER": role,
"PGRST_INTERNAL_SCHEMA_CACHE_SLEEP": "50",
"PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP": "50",
}
with run(env=env) as postgrest:
# The schema cache query takes at least 20ms, due to PGRST_INTERNAL_SCHEMA_CACHE_SLEEP above.
# The schema cache query takes at least 20ms, due to PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP above.
# Make it impossible to load the schema cache, by setting statement timeout to 100ms.
set_statement_timeout(metapostgrest, role, 20)
@@ -1320,7 +1320,7 @@ def test_schema_cache_concurrent_notifications(slow_schema_cache_env):
"schema cache should be up-to-date whenever a notification is sent while another reload is in progress, see https://github.com/PostgREST/postgrest/issues/2791"
internal_sleep = (
int(slow_schema_cache_env["PGRST_INTERNAL_SCHEMA_CACHE_SLEEP"]) / 1000
int(slow_schema_cache_env["PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP"]) / 1000
)
with run(env=slow_schema_cache_env, wait_for_readiness=False) as postgrest:
test/spec/SpecHelper.hs
+3 -1
View File
@@ -157,7 +157,9 @@ baseCfg = let secret = encodeUtf8 "reallyreallyreallyreallyverysafe" in
, configAdminServerPort = Nothing
, configRoleSettings = mempty
, configRoleIsoLvl = mempty
, configInternalSCSleep = Nothing
, configInternalSCQuerySleep = Nothing
, configInternalSCLoadSleep = Nothing
, configInternalSCRelLoadSleep = Nothing
, configServerTimingEnabled = True
}