jimmy/postgrest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: consistently analyze tables exactly once

Browse Source 
Disabling the autovacuum daemon should also help reproducibility in
theory, although I don't know of any cases where we hit a problem with
that.

VACUUM changes the order of rows that PostgreSQL returns for some table
without explicit ordering, thus doing the latter to make it consistently
reproducible.

After ANALYZE estimates are 100% exact for the moment, so some requests
which returned 206 Partial Response now return 200 instead. The fact
that PostgREST returns 206 on an unfiltered endpoint can probably be
considered a bug.
This commit is contained in:
Wolfgang Walther
2026-05-17 20:03:15 +00:00
parent abe3e511bf
commit c7260bddab
10 changed files with 33 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nix/tools/withTools.nix
+2
View File
@@ -15,6 +15,7 @@ let
let let
commandName = "postgrest-with-${name}"; commandName = "postgrest-with-${name}";
postgresqlConf = writeText "postgresql.conf" " postgresqlConf = writeText "postgresql.conf" "
autovacuum = false
listen_addresses = '' listen_addresses = ''
log_statement = all log_statement = all
"; ";
@@ -143,6 +144,7 @@ let
load_start=$SECONDS load_start=$SECONDS
>&2 printf "${commandName}: Loading fixtures under the postgres role..." >&2 printf "${commandName}: Loading fixtures under the postgres role..."
psql -U postgres -v PGUSER="$PGUSER" -v ON_ERROR_STOP=1 -f "$_arg_fixtures" >> "$setuplog" psql -U postgres -v PGUSER="$PGUSER" -v ON_ERROR_STOP=1 -f "$_arg_fixtures" >> "$setuplog"
psql -U postgres -v ON_ERROR_STOP=1 -c "VACUUM ANALYZE;" >> "$setuplog"
load_end=$((SECONDS - load_start)) load_end=$((SECONDS - load_start))
>&2 printf " done in %ss. Running command...\n" "$load_end" >&2 printf " done in %ss. Running command...\n" "$load_end"
fi fi
src/PostgREST/Config/PgVersion.hs
-4
View File
@@ -4,7 +4,6 @@ module PostgREST.Config.PgVersion
( PgVersion(..) ( PgVersion(..)
, minimumPgVersion , minimumPgVersion
, pgVersion150 , pgVersion150
, pgVersion170
, pgVersion180 , pgVersion180
) where ) where
@@ -33,8 +32,5 @@ pgVersion140 = PgVersion 140000 "14.0" "14.0"
pgVersion150 :: PgVersion pgVersion150 :: PgVersion
pgVersion150 = PgVersion 150000 "15.0" "15.0" pgVersion150 = PgVersion 150000 "15.0" "15.0"
pgVersion170 :: PgVersion
pgVersion170 = PgVersion 170000 "17.0" "17.0"
pgVersion180 :: PgVersion pgVersion180 :: PgVersion
pgVersion180 = PgVersion 180000 "18.0" "18.0" pgVersion180 = PgVersion 180000 "18.0" "18.0"
test/io/test_io.py
+2 -2
View File
@@ -938,12 +938,12 @@ def test_log_query(level, defaultenv):
response = postgrest.session.get( response = postgrest.session.get(
"/projects", headers={"Prefer": "count=estimated"} "/projects", headers={"Prefer": "count=estimated"}
) )
assert response.status_code == 206 assert response.status_code == 200
response = postgrest.session.get( response = postgrest.session.get(
"/projects", headers={"Prefer": "count=planned"} "/projects", headers={"Prefer": "count=planned"}
) )
assert response.status_code == 206 assert response.status_code == 200
response = postgrest.session.get("/infinite_recursion") response = postgrest.session.get("/infinite_recursion")
assert response.status_code == 500 assert response.status_code == 500
test/spec/Feature/Query/AggregateFunctionsSpec.hs
+4 -4
View File
@@ -154,24 +154,24 @@ allowed =
[json|[{"total_budget": 9501.06}]|] [json|[{"total_budget": 9501.06}]|]
{ matchHeaders = [matchContentTypeJson] } { matchHeaders = [matchContentTypeJson] }
it "supports aggregates from a spread relationships grouped by spreaded fields from other relationships" $ do it "supports aggregates from a spread relationships grouped by spreaded fields from other relationships" $ do
get "/processes?select=...process_costs(cost.sum()),...process_categories(name)" `shouldRespondWith` get "/processes?select=...process_costs(cost.sum()),...process_categories(name)&order=process_categories(name)" `shouldRespondWith`
[json|[ [json|[
{"sum": 400.00, "name": "Batch"}, {"sum": 400.00, "name": "Batch"},
{"sum": 350.00, "name": "Mass"}]|] {"sum": 350.00, "name": "Mass"}]|]
{ matchHeaders = [matchContentTypeJson] } { matchHeaders = [matchContentTypeJson] }
get "/processes?select=...process_costs(cost_sum:cost.sum()),...process_categories(category:name)" `shouldRespondWith` get "/processes?select=...process_costs(cost_sum:cost.sum()),...process_categories(category:name)&order=process_categories(category)" `shouldRespondWith`
[json|[ [json|[
{"cost_sum": 400.00, "category": "Batch"}, {"cost_sum": 400.00, "category": "Batch"},
{"cost_sum": 350.00, "category": "Mass"}]|] {"cost_sum": 350.00, "category": "Mass"}]|]
{ matchHeaders = [matchContentTypeJson] } { matchHeaders = [matchContentTypeJson] }
it "supports aggregates on spreaded fields from nested relationships" $ do it "supports aggregates on spreaded fields from nested relationships" $ do
get "/process_supervisor?select=...processes(factory_id,...process_costs(cost.sum()))" `shouldRespondWith` get "/process_supervisor?select=...processes(factory_id,...process_costs(cost.sum()))&order=processes(factory_id).desc" `shouldRespondWith`
[json|[ [json|[
{"factory_id": 3, "sum": 110.00}, {"factory_id": 3, "sum": 110.00},
{"factory_id": 2, "sum": 500.00}, {"factory_id": 2, "sum": 500.00},
{"factory_id": 1, "sum": 350.00}]|] {"factory_id": 1, "sum": 350.00}]|]
{ matchHeaders = [matchContentTypeJson] } { matchHeaders = [matchContentTypeJson] }
get "/process_supervisor?select=...processes(factory_id,...process_costs(cost_sum:cost.sum()))" `shouldRespondWith` get "/process_supervisor?select=...processes(factory_id,...process_costs(cost_sum:cost.sum()))&order=processes(factory_id).desc" `shouldRespondWith`
[json|[ [json|[
{"factory_id": 3, "cost_sum": 110.00}, {"factory_id": 3, "cost_sum": 110.00},
{"factory_id": 2, "cost_sum": 500.00}, {"factory_id": 2, "cost_sum": 500.00},
test/spec/Feature/Query/EmbedDisambiguationSpec.hs
+1 -1
View File
@@ -236,7 +236,7 @@ spec =
[json|[{"createdAt":"2015-12-08T04:22:57.472738","article":{"id": 1},"user":{"name": "Angela Martin"}}]|] [json|[{"createdAt":"2015-12-08T04:22:57.472738","article":{"id": 1},"user":{"name": "Angela Martin"}}]|]
it "can specify a view!fk" $ it "can specify a view!fk" $
get "/message?select=id,body,sender:person_detail!message_sender_fkey(name,sent),recipient:person_detail!message_recipient_fkey(name,received)&id=lt.4" `shouldRespondWith` get "/message?select=id,body,sender:person_detail!message_sender_fkey(name,sent),recipient:person_detail!message_recipient_fkey(name,received)&id=lt.4&order=id" `shouldRespondWith`
[json| [json|
[{"id":1,"body":"Hello Jane","sender":{"name":"John","sent":2},"recipient":{"name":"Jane","received":2}}, [{"id":1,"body":"Hello Jane","sender":{"name":"John","sent":2},"recipient":{"name":"Jane","received":2}},
{"id":2,"body":"Hi John","sender":{"name":"Jane","sent":1},"recipient":{"name":"John","received":1}}, {"id":2,"body":"Hi John","sender":{"name":"Jane","sent":1},"recipient":{"name":"John","received":1}},
test/spec/Feature/Query/PlanSpec.hs
+8 -9
View File
@@ -15,12 +15,11 @@ import Test.Hspec hiding (pendingWith)
import Test.Hspec.Wai import Test.Hspec.Wai
import Test.Hspec.Wai.JSON import Test.Hspec.Wai.JSON
import PostgREST.Config.PgVersion (PgVersion, pgVersion170) import Protolude hiding (get)
import Protolude hiding (get)
import SpecHelper import SpecHelper
spec :: PgVersion -> SpecWith ((), Application) spec :: SpecWith ((), Application)
spec actualPgVersion = do spec = do
describe "read table/view plan" $ do describe "read table/view plan" $ do
it "outputs the total cost for a single filter on a table" $ do it "outputs the total cost for a single filter on a table" $ do
r <- request methodGet "/projects?id=in.(1,2,3)" r <- request methodGet "/projects?id=in.(1,2,3)"
@@ -34,7 +33,7 @@ spec actualPgVersion = do
resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8") resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
resHeaders `shouldSatisfy` notZeroContentLength resHeaders `shouldSatisfy` notZeroContentLength
resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" } resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
totalCost `shouldBe` (if actualPgVersion >= pgVersion170 then 11.32 else 15.63) totalCost `shouldBe` 1.11
it "outputs the total cost for a single filter on a view" $ do it "outputs the total cost for a single filter on a view" $ do
r <- request methodGet "/projects_view?id=gt.2" r <- request methodGet "/projects_view?id=gt.2"
@@ -47,7 +46,7 @@ spec actualPgVersion = do
liftIO $ do liftIO $ do
resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8") resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" } resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
totalCost `shouldBe` 24.28 totalCost `shouldBe` 1.1
it "outputs blocks info when using the buffers option" $ do it "outputs blocks info when using the buffers option" $ do
r <- request methodGet "/projects" (acceptHdrs "application/vnd.pgrst.plan+json; options=buffers") "" r <- request methodGet "/projects" (acceptHdrs "application/vnd.pgrst.plan+json; options=buffers") ""
@@ -143,7 +142,7 @@ spec actualPgVersion = do
resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8") resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
resHeaders `shouldSatisfy` notZeroContentLength resHeaders `shouldSatisfy` notZeroContentLength
resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" } resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
totalCost `shouldBe` 8.23 totalCost `shouldBe` 1.13
it "outputs the total cost for a delete" $ do it "outputs the total cost for a delete" $ do
r <- request methodDelete "/projects?id=in.(1,2,3)" r <- request methodDelete "/projects?id=in.(1,2,3)"
@@ -157,7 +156,7 @@ spec actualPgVersion = do
resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8") resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
resHeaders `shouldSatisfy` notZeroContentLength resHeaders `shouldSatisfy` notZeroContentLength
resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" } resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
totalCost `shouldBe` (if actualPgVersion >= pgVersion170 then 11.37 else 15.68) totalCost `shouldBe` 1.16
it "outputs the total cost for a single upsert" $ do it "outputs the total cost for a single upsert" $ do
r <- request methodPut "/tiobe_pls?name=eq.Go" r <- request methodPut "/tiobe_pls?name=eq.Go"
@@ -490,7 +489,7 @@ spec actualPgVersion = do
liftIO $ do liftIO $ do
resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=analyze; charset=utf-8") resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=analyze; charset=utf-8")
totalCost `shouldSatisfy` (> 49.0) totalCost `shouldSatisfy` (> 2.0)
aggregateQty `shouldSatisfy` (> 1) aggregateQty `shouldSatisfy` (> 1)
context "functions with count=exact" $ do context "functions with count=exact" $ do
test/spec/Feature/Query/QuerySpec.hs
+7 -7
View File
@@ -692,13 +692,13 @@ spec = do
{ matchHeaders = [matchContentTypeJson] } { matchHeaders = [matchContentTypeJson] }
it "requesting data using many<->many relation defined by composite keys" $ it "requesting data using many<->many relation defined by composite keys" $
get "/users_tasks?user_id=eq.1&task_id=eq.1&select=user_id,files(filename,content)" `shouldRespondWith` get "/users_tasks?user_id=eq.1&task_id=eq.1&select=user_id,files(filename,content)&files.order=filename" `shouldRespondWith`
[json|[{"user_id":1,"files":[{"filename":"autoexec.bat","content":"@ECHO OFF"},{"filename":"command.com","content":"#include <unix.h>"},{"filename":"README.md","content":"# make $$$!"}]}]|] [json|[{"user_id":1,"files":[{"filename":"README.md","content":"# make $$$!"},{"filename":"autoexec.bat","content":"@ECHO OFF"},{"filename":"command.com","content":"#include <unix.h>"}]}]|]
{ matchHeaders = [matchContentTypeJson] } { matchHeaders = [matchContentTypeJson] }
it "requesting data using many<->many (composite keys) relation using hint" $ it "requesting data using many<->many (composite keys) relation using hint" $
get "/users_tasks?user_id=eq.1&task_id=eq.1&select=user_id,files!touched_files(filename,content)" `shouldRespondWith` get "/users_tasks?user_id=eq.1&task_id=eq.1&select=user_id,files!touched_files(filename,content)&files.order=filename" `shouldRespondWith`
[json|[{"user_id":1,"files":[{"filename":"autoexec.bat","content":"@ECHO OFF"},{"filename":"command.com","content":"#include <unix.h>"},{"filename":"README.md","content":"# make $$$!"}]}]|] [json|[{"user_id":1,"files":[{"filename":"README.md","content":"# make $$$!"},{"filename":"autoexec.bat","content":"@ECHO OFF"},{"filename":"command.com","content":"#include <unix.h>"}]}]|]
{ matchHeaders = [matchContentTypeJson] } { matchHeaders = [matchContentTypeJson] }
it "requesting children with composite key" $ it "requesting children with composite key" $
@@ -1607,11 +1607,11 @@ spec = do
] |] ] |]
{ matchHeaders = [matchContentTypeJson] } { matchHeaders = [matchContentTypeJson] }
it "formats through join" $ it "formats through join" $
get "/datarep_next_two_todos?select=id,name,first_item:datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)" `shouldRespondWith` get "/datarep_next_two_todos?select=id,name,first_item:datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)&order=id" `shouldRespondWith`
[json| [{"id":1,"name":"school related","first_item":{"label_color":"#000100","due_at":"2018-01-03T00:00:00Z"}},{"id":2,"name":"do these first","first_item":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |] [json| [{"id":1,"name":"school related","first_item":{"label_color":"#000100","due_at":"2018-01-03T00:00:00Z"}},{"id":2,"name":"do these first","first_item":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |]
{ matchHeaders = [matchContentTypeJson] } { matchHeaders = [matchContentTypeJson] }
it "formats through join with star select" $ it "formats through join with star select" $
get "/datarep_next_two_todos?select=id,name,second_item:datarep_todos!datarep_next_two_todos_second_item_id_fkey(*)" `shouldRespondWith` get "/datarep_next_two_todos?select=id,name,second_item:datarep_todos!datarep_next_two_todos_second_item_id_fkey(*)&order=id" `shouldRespondWith`
[json| [ [json| [
{"id":1,"name":"school related","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}}, {"id":1,"name":"school related","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}},
{"id":2,"name":"do these first","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}} {"id":2,"name":"do these first","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}}
@@ -1645,7 +1645,7 @@ spec = do
] |] ] |]
{ matchHeaders = [matchContentTypeJson] } { matchHeaders = [matchContentTypeJson] }
it "uses text parser on value for filter across relations" $ it "uses text parser on value for filter across relations" $
get "/datarep_next_two_todos?select=id,name,datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)&datarep_todos.label_color=neq.000100" `shouldRespondWith` get "/datarep_next_two_todos?select=id,name,datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)&datarep_todos.label_color=neq.000100&order=id" `shouldRespondWith`
[json| [{"id":1,"name":"school related","datarep_todos":null},{"id":2,"name":"do these first","datarep_todos":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |] [json| [{"id":1,"name":"school related","datarep_todos":null},{"id":2,"name":"do these first","datarep_todos":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |]
{ matchHeaders = [matchContentTypeJson] } { matchHeaders = [matchContentTypeJson] }
-- This is not supported by data reps (would be hard to make it work with high performance). So the test just -- This is not supported by data reps (would be hard to make it work with high performance). So the test just
test/spec/Feature/Query/RelatedQueriesSpec.hs
+6 -6
View File
@@ -323,7 +323,7 @@ spec = describe "related queries" $ do
]|] ]|]
{ matchStatus = 206 { matchStatus = 206
, matchHeaders = [ matchContentTypeJson , matchHeaders = [ matchContentTypeJson
, "Content-Range" <:> "0-3/1200" ] , "Content-Range" <:> "0-3/5" ]
} }
request methodGet "/projects?select=name,clients()&clients=is.null" request methodGet "/projects?select=name,clients()&clients=is.null"
[("Prefer", "count=planned")] "" [("Prefer", "count=planned")] ""
@@ -340,9 +340,9 @@ spec = describe "related queries" $ do
{"id":1,"name":"Walmart"}, {"id":1,"name":"Walmart"},
{"id":2,"name":"Target"} {"id":2,"name":"Target"}
]|] ]|]
{ matchStatus = 206 { matchStatus = 200
, matchHeaders = [ matchContentTypeJson , matchHeaders = [ matchContentTypeJson
, "Content-Range" <:> "0-1/952" ] , "Content-Range" <:> "0-1/2" ]
} }
it "works with count=estimated" $ do it "works with count=estimated" $ do
@@ -357,7 +357,7 @@ spec = describe "related queries" $ do
]|] ]|]
{ matchStatus = 206 { matchStatus = 206
, matchHeaders = [ matchContentTypeJson , matchHeaders = [ matchContentTypeJson
, "Content-Range" <:> "0-3/1200" ] , "Content-Range" <:> "0-3/5" ]
} }
request methodGet "/projects?select=name,clients()&clients=is.null" request methodGet "/projects?select=name,clients()&clients=is.null"
[("Prefer", "count=estimated")] "" [("Prefer", "count=estimated")] ""
@@ -374,7 +374,7 @@ spec = describe "related queries" $ do
{"id":1,"name":"Walmart"}, {"id":1,"name":"Walmart"},
{"id":2,"name":"Target"} {"id":2,"name":"Target"}
]|] ]|]
{ matchStatus = 206 { matchStatus = 200
, matchHeaders = [ matchContentTypeJson , matchHeaders = [ matchContentTypeJson
, "Content-Range" <:> "0-1/952" ] , "Content-Range" <:> "0-1/2" ]
} }
test/spec/Main.hs
+1 -1
View File
@@ -243,7 +243,7 @@ main = do
-- this test runs with db-plan-enabled = true -- this test runs with db-plan-enabled = true
parallel $ before planEnabledApp $ parallel $ before planEnabledApp $
describe "Feature.Query.PlanSpec.spec" $ Feature.Query.PlanSpec.spec actualPgVersion describe "Feature.Query.PlanSpec.spec" Feature.Query.PlanSpec.spec
-- this test runs with server-trace-header set -- this test runs with server-trace-header set
parallel $ before obsApp $ parallel $ before obsApp $
test/spec/fixtures/data.sql
Vendored
+2
View File
@@ -983,3 +983,5 @@ VALUES (1, '2025-01-01 10:00','2025-01-01 11:00', 'vacation'),
(2, '2024-11-01 09:00','2024-11-01 10:00', 'vacation'), (2, '2024-11-01 09:00','2024-11-01 10:00', 'vacation'),
(3, '2024-12-02 13:00','2024-12-02 14:00', 'vacation'), (3, '2024-12-02 13:00','2024-12-02 14:00', 'vacation'),
(1, '2023-01-02 20:00','2023-01-01 21:00', 'work'); (1, '2023-01-02 20:00','2023-01-01 21:00', 'work');
INSERT INTO bets (id) SELECT generate_series(1,1000);