Add memory usage tests

.circleci/config.yml

@@ -138,6 +138,50 @@ jobs:
           name: run tests
           command: POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://circleci@localhost" postgrest_test) stack test
 
+  build-prof-test:
+    docker:
+      - image: circleci/buildpack-deps:trusty
+        environment:
+          - PGHOST=localhost
+          - TERM=xterm
+      - image: circleci/postgres:9.6.2
+        environment:
+          - POSTGRES_USER=circleci
+          - POSTGRES_DB=circleci
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+          - v1-stack-prof-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }}
+      - run:
+          name: install stack & dependencies
+          command: |
+            curl -L https://github.com/commercialhaskell/stack/releases/download/v1.1.2/stack-1.1.2-linux-x86_64.tar.gz | tar zx -C /tmp
+            sudo mv /tmp/stack-1.1.2-linux-x86_64/stack /usr/bin
+            sudo apt-get update
+            sudo apt-get install -y libgmp-dev
+            sudo apt-get install -y postgresql-client
+            stack setup
+      - run:
+          name: build with profiling enabled
+          command: |
+            stack build --profile -j1
+      - run:
+          name: run memory usage tests
+          command: |
+            test/create_test_db "postgres://circleci@localhost" postgrest_test
+            psql "postgres:///postgrest_test" -f test/fixtures/database.sql
+            psql "postgres:///postgrest_test" -f test/fixtures/roles.sql
+            psql "postgres:///postgrest_test" -f test/fixtures/schema.sql
+            psql "postgres:///postgrest_test" -f test/fixtures/jwt.sql
+            psql "postgres:///postgrest_test" -f test/fixtures/privileges.sql
+            test/memory-tests.sh
+      - save_cache:
+          paths:
+            - "~/.stack"
+            - ".stack-work"
+          key: v1-stack-prof-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }}
+
   centos6:
     <<: *build-distro-bin
 
@@ -188,10 +232,15 @@ workflows:
           filters:
             tags:
               only: /v[0-9]+(\.[0-9]+)*/
+      - build-prof-test:
+          filters:
+            tags:
+              only: /v[0-9]+(\.[0-9]+)*/
       - centos6:
           requires:
             - build-test
             - build-test-9.6
+            - build-prof-test
           filters:
             tags:
               only: /v[0-9]+(\.[0-9]+)*/
@@ -201,6 +250,7 @@ workflows:
           requires:
             - build-test
             - build-test-9.6
+            - build-prof-test
           filters:
             tags:
               only: /v[0-9]+(\.[0-9]+)*/
@@ -210,6 +260,7 @@ workflows:
           requires:
             - build-test
             - build-test-9.6
+            - build-prof-test
           filters:
             tags:
               only: /v[0-9]+(\.[0-9]+)*/
@@ -219,6 +270,7 @@ workflows:
           requires:
             - build-test
             - build-test-9.6
+            - build-prof-test
           filters:
             tags:
               only: /v[0-9]+(\.[0-9]+)*/

test/fixtures/privileges.sql

@@ -61,6 +61,7 @@ GRANT ALL ON TABLE
     , descendant
     , being_part
     , part
+    , leak
 TO postgrest_test_anonymous;
 
 GRANT INSERT ON TABLE insertonly TO postgrest_test_anonymous;
@@ -69,6 +70,7 @@ GRANT USAGE ON SEQUENCE
       auto_incrementing_pk_id_seq
     , items_id_seq
     , callcounter_count
+    , leak_id_seq
 TO postgrest_test_anonymous;
 
 -- Privileges for non anonymous users

test/fixtures/schema.sql

@@ -1345,3 +1345,10 @@ $$ language sql;
 create or replace function test.overloaded(a text, b text, c text) returns text as $$
   select a || b || c
 $$ language sql;
+
+create table test.leak(
+  id serial primary key,
+  blob bytea
+);
+
+CREATE FUNCTION test.leak(blob bytea) RETURNS void AS $$ BEGIN END; $$ LANGUAGE plpgsql;

test/memory-tests.sh

@@ -0,0 +1,78 @@
+#!/bin/sh
+currentTest=1
+failedTests=0
+result(){ echo "$1 $currentTest $2"; currentTest=$(( $currentTest + 1 )); }
+ok(){ result 'ok' "- $1"; }
+ko(){ result 'not ok' "- $1"; failedTests=$(( $failedTests + 1 )); }
+
+pgrPort=49421
+
+pgrStopAll(){ pkill -f "$(stack path --local-install-root)/bin/postgrest"; }
+
+pgrStart(){
+  stack build --profile
+  stack exec -- postgrest test/memory-tests/config +RTS -p -h >/dev/null & pgrPID="$!";
+}
+pgrStop(){ kill "$pgrPID" 2>/dev/null; }
+
+setUp(){ pgrStopAll; }
+cleanUp(){ pgrStopAll; }
+
+checkPgrStarted(){
+  while pgrStarted && test $(rootStatus) -ne 200
+  do
+    sleep 1
+  done
+}
+pgrStarted(){ kill -0 "$pgrPID" 2>/dev/null; }
+rootStatus(){
+  curl -s -o /dev/null -I -w '%{http_code}' "http://localhost:$pgrPort/"
+}
+
+memoryTest(){
+  pgrStart
+  checkPgrStarted
+  factor=$(( 3*$(numfmt --from=si $1)/4 )) # 3/4 on $1 is need to maintain the specified size because of base64
+  payload="{\"blob\" : \"$(dd if=/dev/zero bs=$factor count=1 status=none | base64)\"}"
+  httpStatus=$(echo $payload | curl -s -H "Content-Type: application/json" --request $2 -d @- -w '%{http_code}' http://localhost:$pgrPort$3 | tr -d '"')
+  if test "$httpStatus" -ge 200 && test "$httpStatus" -lt 210
+  then
+    pgrStop
+    while [ ! -s postgrest.prof ]
+    do
+      sleep 1
+    done
+    BYTES_FMT=$(cat postgrest.prof | grep -o -P '(?<=alloc =).*(?=bytes)' | tr -d ' ')
+    BYTES=$(echo $BYTES_FMT | tr -d ',')
+    MAX_BYTES=$(numfmt --from=si $4)
+    if test $BYTES -le $MAX_BYTES
+    then
+      ok "$2 $3: with a $1 payload size the memory usage($BYTES_FMT bytes) is less than $4"
+    else
+      ko "$2 $3: with a $1 payload size the memory usage($BYTES_FMT bytes) is more than $4"
+    fi
+  else
+    pgrStop
+    ko "$2 $3: request failed with http $httpStatus"
+  fi
+}
+
+setUp
+
+echo "Running memory usage tests.."
+
+memoryTest "1M" "POST" "/rpc/leak" "15M"
+memoryTest "1M" "POST" "/leak" "15M"
+memoryTest "1M" "PATCH" "/leak?id=eq.1" "15M"
+
+memoryTest "10M" "POST" "/rpc/leak" "105M"
+memoryTest "10M" "POST" "/leak" "105M"
+memoryTest "10M" "PATCH" "/leak?id=eq.1" "105M"
+
+memoryTest "100M" "POST" "/rpc/leak" "895M"
+memoryTest "100M" "POST" "/leak" "895M"
+memoryTest "100M" "PATCH" "/leak?id=eq.1" "895M"
+
+cleanUp
+
+exit $failedTests

test/memory-tests/config

@@ -0,0 +1,8 @@
+db-uri = "postgres:///postgrest_test"
+db-schema = "test"
+db-anon-role = "postgrest_test_anonymous"
+db-pool = 1
+server-host = "*4"
+server-port = 49421
+
+jwt-secret = "reallyreallyreallyreallyverysafe"