Reduce memory usage of uniform json array restriction

src/PostgREST/ApiRequest.hs

@@ -1,3 +1,4 @@
+{-# LANGUAGE LambdaCase #-}
 {-|
 Module      : PostgREST.ApiRequest
 Description : PostgREST functions to translate HTTP request to a domain type called ApiRequest.
@@ -278,21 +279,21 @@ payloadAttributes raw json =
   -- Test that Array contains only Objects having the same keys
   case json of
     JSON.Array arr ->
-      let objs :: V.Vector JSON.Object
+      case arr V.!? 0 of
-          objs = foldr -- filter non-objects, map to raw objects
+        Just (JSON.Object o) ->
-                   (\val result -> case val of
+          let canonicalKeys = S.fromList $ M.keys o
-                      JSON.Object o -> V.cons o result
+              areKeysUniform = all (\case
-                      _ -> result)
+                JSON.Object x -> S.fromList (M.keys x) == canonicalKeys
-                   V.empty arr
+                _ -> False) arr in
-          keysPerObj = V.map (S.fromList . M.keys) objs
+          if areKeysUniform
-          canonicalKeys = fromMaybe S.empty $ keysPerObj V.!? 0
+            then Just $ PayloadJSON raw (PJArray $ V.length arr) canonicalKeys
-          areKeysUniform = all (==canonicalKeys) keysPerObj
-          arrLength = V.length arr in
-      if (V.length objs == arrLength) && areKeysUniform
-        then Just $ PayloadJSON raw (PJArray arrLength) canonicalKeys
             else Nothing
+        Just _ -> Nothing
+        Nothing -> Just emptyPJArray
 
     JSON.Object o -> Just $ PayloadJSON raw PJObject (S.fromList $ M.keys o)
 
     -- truncate everything else to an empty array.
-    _ -> Just $ PayloadJSON (JSON.encode emptyArray) (PJArray 0) S.empty
+    _ -> Just emptyPJArray
+  where
+    emptyPJArray = PayloadJSON (JSON.encode emptyArray) (PJArray 0) S.empty

test/Feature/InsertSpec.hs

@@ -49,6 +49,12 @@ spec = do
           , matchHeaders = [matchContentTypeJson]
           }
 
+    context "non uniform json array" $ do
+      it "rejects json array that isn't exclusivily composed of objects" $
+        post "/articles" [json| [{"id": 100, "body": "xxxxx"}, 123, "xxxx", {"id": 111, "body": "xxxx"}] |] `shouldRespondWith` 400
+      it "rejects json array that has objects with different keys" $
+        post "/articles" [json| [{"id": 100, "body": "xxxxx"}, {"id": 111, "body": "xxxx", "owner": "me"}] |] `shouldRespondWith` 400
+
     context "requesting full representation" $ do
       it "includes related data after insert" $
         request methodPost "/projects?select=id,name,clients{id,name}"

test/fixtures/privileges.sql

@@ -62,6 +62,7 @@ GRANT ALL ON TABLE
     , being_part
     , part
     , leak
+    , perf_articles
 TO postgrest_test_anonymous;
 
 GRANT INSERT ON TABLE insertonly TO postgrest_test_anonymous;

test/fixtures/schema.sql

@@ -1351,4 +1351,9 @@ create table test.leak(
   blob bytea
 );
 
-CREATE FUNCTION test.leak(blob bytea) RETURNS void AS $$ BEGIN END; $$ LANGUAGE plpgsql;
+create function test.leak(blob bytea) returns void as $$ begin end; $$ language plpgsql;
+
+create table test.perf_articles(
+  id integer not null,
+  body text not null
+);

test/memory-tests.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 currentTest=1
 failedTests=0
 result(){ echo "$1 $currentTest $2"; currentTest=$(( $currentTest + 1 )); }
@@ -9,10 +9,7 @@ pgrPort=49421
 
 pgrStopAll(){ pkill -f "$(stack path --local-install-root)/bin/postgrest"; }
 
-pgrStart(){
+pgrStart(){ stack exec -- postgrest test/memory-tests/config +RTS -p -h >/dev/null & pgrPID="$!"; }
-  stack build --profile
-  stack exec -- postgrest test/memory-tests/config +RTS -p -h >/dev/null & pgrPID="$!";
-}
 pgrStop(){ kill "$pgrPID" 2>/dev/null; }
 
 setUp(){ pgrStopAll; }
@@ -29,7 +26,7 @@ rootStatus(){
   curl -s -o /dev/null -I -w '%{http_code}' "http://localhost:$pgrPort/"
 }
 
-memoryTest(){
+jsonKeyTest(){
   pgrStart
   checkPgrStarted
   factor=$(( 3*$(numfmt --from=si $1)/4 )) # 3/4 on $1 is need to maintain the specified size because of base64
@@ -47,9 +44,9 @@ memoryTest(){
     MAX_BYTES=$(numfmt --from=si $4)
     if test $BYTES -le $MAX_BYTES
     then
-      ok "$2 $3: with a $1 payload size the memory usage($BYTES_FMT bytes) is less than $4"
+      ok "$2 $3: with a json key of $1 the memory usage($BYTES_FMT bytes) is less than $4"
     else
-      ko "$2 $3: with a $1 payload size the memory usage($BYTES_FMT bytes) is more than $4"
+      ko "$2 $3: with a json key of $1 the memory usage($BYTES_FMT bytes) is more than $4"
     fi
   else
     pgrStop
@@ -57,21 +54,63 @@ memoryTest(){
   fi
 }
 
+postJsonArrayTest(){
+  pgrStart
+  checkPgrStarted
+  arr=()
+  arr+=('[')
+  for i in $(seq 1 $(expr $1 - 1))
+  do
+    arr+=("{\"id\": $i, \"body\": \"xxxxxxx\"},")
+  done
+  arr+=("{\"id\": $1, \"body\": \"xxxxxxx\"}")
+  arr+=(']')
+  payload=$(echo ${arr[*]})
+  httpStatus=$(echo $payload | curl -s -H "Content-Type: application/json" -d @- -w '%{http_code}' http://localhost:$pgrPort$2 | tr -d '"')
+  if test "$httpStatus" -ge 200 && test "$httpStatus" -lt 210
+  then
+    pgrStop
+    while [ ! -s postgrest.prof ]
+    do
+      sleep 1
+    done
+    BYTES_FMT=$(cat postgrest.prof | grep -o -P '(?<=alloc =).*(?=bytes)' | tr -d ' ')
+    BYTES=$(echo $BYTES_FMT | tr -d ',')
+    MAX_BYTES=$(numfmt --from=si $3)
+    PAYLOAD_SIZE=$(echo $payload | wc -c | numfmt --to=si)
+    if test $BYTES -le $MAX_BYTES
+    then
+      ok "POST $2: with a json payload of $PAYLOAD_SIZE that has $1 array values the memory usage($BYTES_FMT bytes) is less than $3"
+    else
+      ko "POST $2: with a json payload of $PAYLOAD_SIZE that has $1 array values the memory usage($BYTES_FMT bytes) is more than $3"
+    fi
+  else
+    pgrStop
+    ko "POST $2: request failed with http $httpStatus"
+  fi
+}
+
+stack build --profile
+
 setUp
 
 echo "Running memory usage tests.."
 
-memoryTest "1M" "POST" "/rpc/leak" "15M"
+jsonKeyTest "1M" "POST" "/rpc/leak" "15M"
-memoryTest "1M" "POST" "/leak" "15M"
+jsonKeyTest "1M" "POST" "/leak" "15M"
-memoryTest "1M" "PATCH" "/leak?id=eq.1" "15M"
+jsonKeyTest "1M" "PATCH" "/leak?id=eq.1" "15M"
 
-memoryTest "10M" "POST" "/rpc/leak" "105M"
+jsonKeyTest "10M" "POST" "/rpc/leak" "105M"
-memoryTest "10M" "POST" "/leak" "105M"
+jsonKeyTest "10M" "POST" "/leak" "105M"
-memoryTest "10M" "PATCH" "/leak?id=eq.1" "105M"
+jsonKeyTest "10M" "PATCH" "/leak?id=eq.1" "105M"
 
-memoryTest "100M" "POST" "/rpc/leak" "895M"
+jsonKeyTest "100M" "POST" "/rpc/leak" "895M"
-memoryTest "100M" "POST" "/leak" "895M"
+jsonKeyTest "100M" "POST" "/leak" "895M"
-memoryTest "100M" "PATCH" "/leak?id=eq.1" "895M"
+jsonKeyTest "100M" "PATCH" "/leak?id=eq.1" "895M"
+
+postJsonArrayTest "1000" "/perf_articles" "20M"
+postJsonArrayTest "10000" "/perf_articles" "120M"
+postJsonArrayTest "100000" "/perf_articles" "1.15G"
 
 cleanUp