Nixify io and memory tests (#1538)

* Include tests for io and memory in the Nix environment.
* include spec tests in nix-shell by default
* install the io and memory tests in CI
This commit is contained in:
Remo Rechkemmer
2020-05-28 12:45:14 -05:00
committed by GitHub
parent 784ebe57d7
commit 69b09e312a
18 changed files with 211 additions and 108 deletions
+21 -4
View File
@@ -74,6 +74,8 @@ jobs:
- run:
name: run io tests
command: |
export PATH="~/.local/bin:$PATH"
stack install --fast -j1
test/io-tests.sh
# Run memory usage tests based on stack and docker.
@@ -111,12 +113,13 @@ jobs:
- ".stack-work"
key: v1-stack-prof-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }}
- run:
name: build with profiling enabled
name: install with profiling enabled
command: |
stack build --profile -j1
stack install --profile -j1
- run:
name: run memory usage tests
command: |
export PATH="~/.local/bin:$PATH"
test/create_test_db "postgres://circleci@localhost" postgrest_test
test/memory-tests.sh
@@ -190,12 +193,20 @@ jobs:
nix-env -iA cachix -f https://cachix.org/api/v1/install
cachix use postgrest
- run:
name: Install testing scripts for all supported PostgreSQL versions
command: nix-env -f default.nix -iA tests
name: Install testing scripts
command: nix-env -f default.nix -iA tests tests.ioTests tests.memoryTests
# Our utility scripts use cabal to run the tests, which will pick up all
# the libraries that are built with Nix. So this should only ever build
# the PostgREST itself and be reasonably quick. We accelerate that part
# further by caching `~/.cabal` and `dist-newstyle`.
#
# CircleCI looks for caches by prefix. When saving to the the cache
# below, we append the current time to the key ('-{{ epoch }}'), so a new
# cache will always be uploaded. If the `postgrest.cabal` and
# `nix/nixpkgs-version.nix` files are unchanged, the latest matching
# cache will be used. If there is no cache for those files, CircleCI will
# try the next, more general key and use the latest cache that matches
# it.
- restore_cache:
keys:
- nix-test-{{ checksum "postgrest.cabal" }}-{{ checksum "nix/nixpkgs-version.nix" }}
@@ -219,6 +230,12 @@ jobs:
- run:
name: Run the spec tests against PostgreSQL 12
command: postgrest-test-spec-postgresql-12
- run:
name: Run io tests
command: postgrest-test-io
- run:
name: Run memory tests
command: postgrest-test-memory
- save_cache:
paths:
- "~/.cabal"
+2
View File
@@ -15,3 +15,5 @@ site
*.swp
result*
dist-newstyle
postgrest.hp
postgrest.prof
+22 -21
View File
@@ -55,8 +55,8 @@ let
patches =
pkgs.callPackage nix/patches { };
# Base dynamic derivation for the PostgREST package.
drv =
# Dynamic derivation for PostgREST
postgrest =
pkgs.haskell.packages."${compiler}".callCabal2nix name src { };
# Function that derives a fully static Haskell package based on
@@ -64,9 +64,14 @@ let
staticHaskellPackage =
import nix/static-haskell-package.nix { inherit nixpkgs compiler patches allOverlays; };
# Static derivation for the PostgREST executable.
drvStatic =
staticHaskellPackage name src;
profiledHaskellPackages =
pkgs.haskell.packages."${compiler}".extend (self: super:
{
mkDerivation =
args:
super.mkDerivation (args // { enableLibraryProfiling = true; });
}
);
lib =
pkgs.haskell.lib;
@@ -78,27 +83,26 @@ rec {
# libraries and documentation. We disable running the test suite on Nix
# builds, as they require a database to be set up.
postgrestPackage =
lib.dontCheck (lib.enableCabalFlag drv "FailOnWarn");
# Derivation for just the PostgREST binary, where we strip all dynamic
# libraries and documentation, leaving only the executable. Note that the
# executable is static with regards to Haskell libraries, but not system
# libraries like glibc and libpq.
postgrest =
lib.justStaticExecutables postgrestPackage;
lib.dontCheck (lib.enableCabalFlag postgrest "FailOnWarn");
# Static executable.
postgrestStatic =
lib.justStaticExecutables (lib.dontCheck drvStatic);
lib.justStaticExecutables (lib.dontCheck (staticHaskellPackage name src));
# Profiled dynamic executable.
postgrestProfiled =
lib.enableExecutableProfiling (
lib.dontHaddock (
lib.dontCheck (profiledHaskellPackages.callCabal2nix name src { })
)
);
# Docker images and loading script.
docker =
pkgs.callPackage nix/docker { postgrest = postgrestStatic; };
# Environment in which PostgREST can be built with cabal, useful e.g. for
# defining a shell for `nix-shell`.
env =
drv.env;
postgrest.env;
# Utility for updating the pinned version of Nixpkgs.
nixpkgsUpgrade =
@@ -106,10 +110,7 @@ rec {
# Scripts for running tests.
tests =
pkgs.callPackage nix/tests.nix {
inherit postgresqlVersions;
postgrestBuildEnv = env;
};
pkgs.callPackage nix/tests.nix { inherit postgrest postgrestStatic postgrestProfiled postgresqlVersions; };
# Development tools, including linting and styling scripts.
devtools =
+38 -18
View File
@@ -17,16 +17,18 @@ artifacts from the `/nix/store`, you can run `nix-collect-garbage`.
To build PostgREST from your local checkout of the repository, run:
```bash
nix-build --attr postgrest
nix-build --attr postgrestPackage
```
This will create a `result` directory that contains the PostgREST binary at
`result/bin/postgrest`. The `--attr` parameter (or short: `-A`) tells Nix to
build the `postgrest` attribute from the Nix expression it finds in our
build the `postgrestPackage` attribute from the Nix expression it finds in our
`default.nix` (see below for details). Nix will take care of getting the right
GHC version and all the build dependencies.
## Binary cache
We recommend that you use the PostgREST binary cache on
[cachix](https://cachix.org/):
@@ -66,17 +68,37 @@ The PostgREST utilities available in `nix-shell` all have names that begin with
```
# Note: The utilities listed here might not be up to date.
[nix-shell]$ postgrest-<tab>
postgrest-lint postgrest-test-spec-postgresql-10
postgrest-style postgrest-test-spec-postgresql-11
postgrest-style-check postgrest-test-spec-postgresql-12
postgrest-test-all postgrest-test-spec-postgresql-9.4
postgrest-test-spec postgrest-test-spec-postgresql-9.5
postgrest-test-spec-all postgrest-test-spec-postgresql-9.6
postgrest-lint postgrest-test-spec-postgresql-11
postgrest-style postgrest-test-spec-postgresql-12
postgrest-style-check postgrest-test-spec-postgresql-9.4
postgrest-test-spec postgrest-test-spec-postgresql-9.5
postgrest-test-spec-all postgrest-test-spec-postgresql-9.6
postgrest-test-spec-postgresql-10
[nix-shell]$
```
Some additional modules like `ioTests`, `memoryTests`, `docker` and `release`
have large dependencies that would need to be built before the shell becomes
available, which could take an especially long time if the cachix binary cache
is not used. You can activate those by passing a flag to `nix-shell`, which
will make the respective utilites available:
```
$ nix-shell --arg ioTests true
[nix-shell]$ postgrest-<tab>
postgrest-lint postgrest-test-spec-postgresql-10
postgrest-style postgrest-test-spec-postgresql-11
postgrest-style-check postgrest-test-spec-postgresql-12
postgrest-test-io postgrest-test-spec-postgresql-9.4
postgrest-test-spec postgrest-test-spec-postgresql-9.5
postgrest-test-spec-all postgrest-test-spec-postgresql-9.6
```
Note that `postgrest-tests-io` is now also available.
To run one-off commands, you can also use `nix-shell --run <command>`, which
will lauch the Nix shell, run that one command and exit. Note that the tab
completion will not work with `nix-shell --run`, as Nix has yet to evaluate
@@ -94,19 +116,17 @@ $ nix-shell --run "postgrest-foo --bar"
A third option is to install utilities that you use very often locally:
```
$ nix-env -f default.nix -iA style
$ nix-env -f default.nix -iA devtools
# `postgrest-style` can now be run directly:
$ postgrest-style
```
Note that this does not yet work for all utilities (e.g. `postgrest-test-spec` currently
needs to be run within the Nix shell environment).
If you use `nix-shell` very often, you might like to use https://github.com/xzfc/cached-nix-shell,
which skips evaluating all our Nix expressions if nothing changed, reducing startup time for the
shell considerably.
If you use `nix-shell` very often, you might like to use
https://github.com/xzfc/cached-nix-shell, which skips evaluating all our Nix
expressions if nothing changed, reducing startup time for the shell
considerably.
## Testing
@@ -116,14 +136,14 @@ temporary test databases:
```bash
# Run the tests against the most recent version of PostgreSQL:
$ nix-shell --run postgrest-test-spec
$ nix-shell --arg tests true --run postgrest-test-spec
# Run the tests against all supported versions of PostgreSQL:
$ nix-shell --run postgrest-test-spec-all
$ nix-shell --arg tests true --run postgrest-test-spec-all
# Run the tests against a specific version of PostgreSQL (use tab-completion in
# nix-shell to see all available versions):
$ nix-shell --run postgrest-test-spec-postgresql-9.5.21
$ nix-shell --arg tests true --run postgrest-test-spec-postgresql-9.5
```
+49 -6
View File
@@ -2,11 +2,16 @@
{ buildEnv
, cabal-install
, curl
, git
, haskell
, lib
, postgrestBuildEnv
, ncat
, postgresql
, postgresqlVersions
, postgrest
, postgrestStatic
, postgrestProfiled
, runtimeShell
, writeShellScript
, writeShellScriptBin
@@ -33,7 +38,7 @@ let
''
set -euo pipefail
export PATH="$(cat ${postgrestBuildEnv})"/bin:"$PATH"
export PATH="$(cat ${postgrest.env})"/bin:"$PATH"
cat << EOF
@@ -51,9 +56,6 @@ let
EOF
'';
defaultTestSpec =
testSpec "postgrest-test-spec" postgresql;
# Create a `testSpec` for each PostgreSQL version that we want to test
# against.
testSpecVersions =
@@ -73,6 +75,36 @@ let
${lib.concatStringsSep "\n" testRunners}
'';
testIO =
name: postgresql:
writeShellScriptBin
name
''
set -euo pipefail
rootdir="$(${git}/bin/git rev-parse --show-toplevel)"
cd "$rootdir"
export PATH="${postgrestStatic}/bin:${curl}/bin:${ncat}/bin:$PATH"
${withTmpDb postgresql} "$rootdir"/test/io-tests.sh
'';
testMemory =
name: postgresql:
writeShellScriptBin
name
''
set -euo pipefail
rootdir="$(${git}/bin/git rev-parse --show-toplevel)"
cd "$rootdir"
export PATH="${postgrestProfiled}/bin:${curl}/bin:$PATH"
${withTmpDb postgresql} "$rootdir/test/memory-tests.sh"
'';
in
# Create an environment that contains all the utility scripts for running tests
# that we defined above.
@@ -82,7 +114,18 @@ buildEnv {
paths =
[
defaultTestSpec
(testSpec "postgrest-test-spec" postgresql)
testSpecAllVersions
] ++ testSpecVersions;
}
# The IO an memory tests have large dependencies (a static and a profiled
# build of PostgREST respectively) and are run less often than the spec
# tests, so we don't include them in the default test environment. We make
# them available through separate attributes:
// {
ioTests =
(testIO "postgrest-test-io" postgresql);
memoryTests =
(testMemory "postgrest-test-memory" postgresql);
}
+31 -18
View File
@@ -1,26 +1,39 @@
with (import ./default.nix);
pkgs.lib.overrideDerivation env (
# The additional modules below have large dependencies and are therefore
# disabled by default. You can activate them by passing arguments to nix-shell,
# e.g.:
#
# nix-shell --arg release true
#
# This will provide you with a shell where the `postgrest-release-*` scripts
# are available.
#
# We highly recommend that use the PostgREST binary cache by installing cachix
# (https://app.cachix.org/) and running `cachix use postgrest`.
{ ioTests ? false, memoryTests ? false, docker ? false, release ? false }:
let
postgrest =
import ./default.nix;
pkgs =
postgrest.pkgs;
lib =
pkgs.lib;
in
lib.overrideDerivation postgrest.env (
base: {
buildInputs =
base.buildInputs ++ [
pkgs.cabal-install
pkgs.stack
pkgs.cabal2nix
pkgs.postgresql
nixpkgsUpgrade
tests
devtools
# We don't include the `postgrest-docker-load` here, as that would
# cause the shell to depend on building the Docker images and in turn
# on the static executable. Use `nix-shell default.nix -A dockerLoad`
# to get a shell with that script on the PATH.
];
shellHook =
''
# Set our pinned version of Nixpkgs in the NIX_PATH so that
# `stack --nix` also uses that version.
NIX_PATH="nixpkgs=${nixpkgs}"
'';
postgrest.nixpkgsUpgrade
postgrest.devtools
postgrest.tests
]
++ lib.optional ioTests postgrest.tests.ioTests
++ lib.optional memoryTests postgrest.tests.memoryTests
++ lib.optional docker postgrest.docker
++ lib.optional release postgrest.release;
}
)
+1
View File
@@ -50,6 +50,7 @@ main = do
cost <- exec pool [str| [{"id": 1}, {"id": 4}] |] $
requestToCallProcQuery (QualifiedIdentifier "test" "get_projects_below") [PgArg "id" "int" True] False (Just MultipleObjects) []
liftIO $ do
-- lower bound needed for now to make sure that cost is not Nothing
cost `shouldSatisfy` (> Just 2000)
cost `shouldSatisfy` (< Just 2100)
+27 -23
View File
@@ -1,7 +1,10 @@
#!/bin/sh
#!/usr/bin/env bash
# Run unit tests for Input/Ouput of PostgREST seen as a black box
# with test output in Test Anything Protocol format.
#
# These tests expect that `postgrest` is on the PATH, as well as `curl` and
# `ncat` (from the nmap package in some distribution).
#
# References:
# [1] Test Anything Protocol
# https://testanything.org/
@@ -12,9 +15,15 @@
# [3] List of TCP and UDP port numbers
# https://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers
#
set -eu
export POSTGREST_TEST_CONNECTION=${POSTGREST_TEST_CONNECTION:-"postgres:///postgrest_test"}
cd "$(dirname "$0")"
cd io-tests
trap "kill 0" int term exit
# Port for Test PostgREST Server (must match config)
pgrPort=49421 # in range 4915265535: for private or temporary use
@@ -30,11 +39,11 @@ ko(){ result 'not ok' "- $1"; failedTests=$(( $failedTests + 1 )); }
comment(){ echo "# $1"; }
# Utilities to start/stop test PostgREST server running in the background
pgrStart(){ stack exec -- postgrest "$1" >/dev/null & pgrPID="$!"; }
pgrStartRead(){ stack exec -- postgrest "$1" >/dev/null < "$2" & pgrPID="$!"; }
pgrStart(){ postgrest $1 >/dev/null 2>/dev/null & pgrPID="$!"; }
pgrStartRead(){ postgrest $1 <$2 >/dev/null & pgrPID="$!"; }
pgrStartStdin(){ postgrest $1 >/dev/null <<< "$2" & pgrPID="$!"; }
pgrStarted(){ kill -0 "$pgrPID" 2>/dev/null; }
pgrStop(){ kill "$pgrPID" 2>/dev/null; }
pgrStopAll(){ pkill -f "$(stack path --local-install-root)/bin/postgrest"; }
pgrStop(){ kill "$pgrPID" 2>/dev/null; pgrPID=""; }
# Utilities to send HTTP requests to the PostgREST server
rootStatus(){
@@ -47,10 +56,6 @@ authorsStatus(){
"http://localhost:$pgrPort/authors_only"
}
# Start and End of Unit Tests
setUp(){ pgrStopAll; }
cleanUp(){ pgrStopAll; }
# Unit Test Templates
readSecretFromFile(){
case "$1" in
@@ -82,9 +87,9 @@ readSecretFromFile(){
pgrStop
}
readDbUriFromFile(){
readDbUriFromStdin(){
pgrConfig="dburi-from-file.config"
pgrStartRead "./configs/$pgrConfig" "./dburis/$1"
pgrStartStdin "./configs/$pgrConfig" "$1"
while pgrStarted && test "$( rootStatus )" -ne 200
do
# wait for the server to start
@@ -93,9 +98,9 @@ readDbUriFromFile(){
done
if pgrStarted
then
ok "connection with $2 dburi read from a file"
ok "connection with $2 dburi read from stdin / a file"
else
ko "connection with $2 dburi read from a file"
ko "connection with $2 dburi read from stdin / a file"
fi
pgrStop
}
@@ -109,7 +114,7 @@ reqWithRoleClaimKey(){
sleep 0.1 \
|| sleep 1 # fallback: subsecond sleep is not standard and may fail
done
authorsJwt=$(psql -qtAX postgrest_test -c "select jwt.sign('$2', 'reallyreallyreallyreallyverysafe');")
authorsJwt=$(psql -qtAX "$POSTGREST_TEST_CONNECTION" -c "select jwt.sign('$2', 'reallyreallyreallyreallyverysafe');")
httpStatus="$( authorsStatus "$authorsJwt" )"
if test "$httpStatus" -eq $3
then
@@ -132,10 +137,10 @@ invalidRoleClaimKey(){
if pgrStarted
then
ko "invalid jspath \"$1\": accepted"
pgrStop
else
ok "invalid jspath \"$1\": rejected"
fi
pgrStop
}
# ensure iat claim is successful in the presence of pgrst time cache, see https://github.com/PostgREST/postgrest/issues/1139
@@ -148,7 +153,7 @@ ensureIatClaimWorks(){
|| sleep 1 # fallback: subsecond sleep is not standard and may fail
done
for i in {1..10}; do \
iatJwt=$(psql -qtAX postgrest_test -c "select jwt.sign(row_to_json(r), 'reallyreallyreallyreallyverysafe') from ( select 'postgrest_test_author' as role, extract(epoch from now()) as iat) r")
iatJwt=$(psql -qtAX "$POSTGREST_TEST_CONNECTION" -c "select jwt.sign(row_to_json(r), 'reallyreallyreallyreallyverysafe') from ( select 'postgrest_test_author' as role, extract(epoch from now()) as iat) r")
httpStatus="$( authorsStatus $iatJwt )"
if test "$httpStatus" -ne 200
then
@@ -209,9 +214,7 @@ socketConnection(){
test -n "$(command -v curl)" || bailOut 'curl is not available'
# PRE: postgres must be running
psql -l 1>/dev/null 2>/dev/null || bailOut 'postgres is not running'
setUp
psql -l "$POSTGREST_TEST_CONNECTION" 1>/dev/null 2>/dev/null || bailOut 'postgres is not running'
echo "Running IO tests.."
@@ -231,8 +234,10 @@ readSecretFromFile ascii.b64 'Base64 (ASCII)'
readSecretFromFile utf8.b64 'Base64 (UTF-8)'
readSecretFromFile binary.b64 'Base64 (binary)'
readDbUriFromFile uri.noeol "(no EOL)"
readDbUriFromFile uri.txt "(EOL)"
eol=$'\x0a'
readDbUriFromStdin "$POSTGREST_TEST_CONNECTION" "(no EOL)"
readDbUriFromStdin "$POSTGREST_TEST_CONNECTION$eol" "(EOL)"
reqWithRoleClaimKey '.postgrest.a_role' '{"postgrest":{"a_role":"postgrest_test_author"}}' 200
reqWithRoleClaimKey '.customObject.manyRoles[1]' '{"customObject":{"manyRoles": ["other", "postgrest_test_author"]}}' 200
@@ -250,7 +255,6 @@ invalidRoleClaimKey 1234
ensureIatClaimWorks
ensureAppSettings
cleanUp
trap - int term exit
exit $failedTests
+1 -1
View File
@@ -1,4 +1,4 @@
db-uri = "postgres:///postgrest_test"
db-uri = "$(POSTGREST_TEST_CONNECTION)"
db-schema = "test"
db-anon-role = "postgrest_test_anonymous"
db-pool = 1
@@ -1,4 +1,4 @@
db-uri = "postgres:///postgrest_test"
db-uri = "$(POSTGREST_TEST_CONNECTION)"
db-schema = "test"
db-anon-role = "postgrest_test_anonymous"
db-pool = 1
+1 -1
View File
@@ -1,4 +1,4 @@
db-uri = "postgres:///postgrest_test"
db-uri = "$(POSTGREST_TEST_CONNECTION)"
db-schema = "test"
db-anon-role = "postgrest_test_anonymous"
db-pool = 1
@@ -1,4 +1,4 @@
db-uri = "postgres:///postgrest_test"
db-uri = "$(POSTGREST_TEST_CONNECTION)"
db-schema = "test"
db-anon-role = "postgrest_test_anonymous"
db-pool = 1
+1 -1
View File
@@ -1,4 +1,4 @@
db-uri = "postgres:///postgrest_test"
db-uri = "$(POSTGREST_TEST_CONNECTION)"
db-schema = "test"
db-anon-role = "postgrest_test_anonymous"
db-pool = 1
+1 -1
View File
@@ -1,4 +1,4 @@
db-uri = "postgres:///postgrest_test"
db-uri = "$(POSTGREST_TEST_CONNECTION)"
db-schema = "test"
db-anon-role = "postgrest_test_anonymous"
db-pool = 1
-1
View File
@@ -1 +0,0 @@
postgres:///postgrest_test
-1
View File
@@ -1 +0,0 @@
postgres:///postgrest_test
+13 -9
View File
@@ -1,4 +1,15 @@
#! /usr/bin/env bash
# This test script expects that a `postgrest` executable with profiling enabled
# is on the PATH. With stack, for example, you can run `stack install --profile
# postgrest`.
set -eu
export POSTGREST_TEST_CONNECTION=${POSTGREST_TEST_CONNECTION:-"postgres:///postgrest_test"}
trap "kill 0" int term exit
currentTest=1
failedTests=0
result(){ echo "$1 $currentTest $2"; currentTest=$(( $currentTest + 1 )); }
@@ -7,14 +18,9 @@ ko(){ result 'not ok' "- $1"; failedTests=$(( $failedTests + 1 )); }
pgrPort=49421
pgrStopAll(){ pkill -f "$(stack path --profile --local-install-root)/bin/postgrest"; }
pgrStart(){ stack exec --profile -- postgrest test/memory-tests/config +RTS -p -h >/dev/null & pgrPID="$!"; }
pgrStart(){ postgrest test/memory-tests/config +RTS -p -h > /dev/null & pgrPID="$!"; }
pgrStop(){ kill "$pgrPID" 2>/dev/null; }
setUp(){ pgrStopAll; }
cleanUp(){ pgrStopAll; }
checkPgrStarted(){
while pgrStarted && test $(rootStatus) -ne 200
do
@@ -90,8 +96,6 @@ postJsonArrayTest(){
fi
}
setUp
echo "Running memory usage tests.."
jsonKeyTest "1M" "POST" "/rpc/leak?columns=blob" "13M"
@@ -110,6 +114,6 @@ postJsonArrayTest "1000" "/perf_articles?columns=id,body" "11M"
postJsonArrayTest "10000" "/perf_articles?columns=id,body" "11M"
postJsonArrayTest "100000" "/perf_articles?columns=id,body" "21M"
cleanUp
trap - int term exit
exit $failedTests
+1 -1
View File
@@ -1,4 +1,4 @@
db-uri = "postgres:///postgrest_test"
db-uri = "$(POSTGREST_TEST_CONNECTION)"
db-schema = "test"
db-anon-role = "postgrest_test_anonymous"
db-pool = 1