From c7f0d423238a70c23a82ed136fc18e1843c007ca Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 30 Dec 2020 13:22:25 +0100 Subject: [PATCH] Add postgrest-coverage to show and upload hpc reports to codecov --- .circleci/config.yml | 7 ++ .codecov.yml | 17 +++++ .gitignore | 3 + default.nix | 10 ++- nix/devtools.nix | 2 + nix/tests.nix | 92 ++++++++++++++++++++++++++- postgrest.cabal | 71 ++++++++++++--------- stack.yaml | 4 -- test/Feature/InsertSpec.hs | 7 -- test/Feature/QueryLimitedSpec.hs | 3 +- test/Feature/RollbackSpec.hs | 3 +- test/Feature/RpcPreRequestGucsSpec.hs | 9 +-- test/Feature/UnicodeSpec.hs | 3 +- test/Feature/UpdateSpec.hs | 7 +- test/QueryCost.hs | 1 - test/coverage.overlay | 0 test/io-tests/test_io.py | 9 ++- 17 files changed, 179 insertions(+), 69 deletions(-) create mode 100644 .codecov.yml create mode 100644 test/coverage.overlay diff --git a/.circleci/config.yml b/.circleci/config.yml index e6c8f2422..69832948f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -234,6 +234,13 @@ jobs: name: Run memory tests command: postgrest-test-memory when: always + - run: + name: Run coverage + command: postgrest-coverage + - run: + name: Upload coverage to codecov + command: bash <(curl -s https://codecov.io/bash) -f coverage/codecov.json + workflows: version: 2 diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 000000000..1780dc0cd --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,17 @@ +codecov: + branch: master + +comment: false + +coverage: + status: + project: + default: + target: auto + threshold: 0% + only_pulls: false + patch: + default: + target: auto + threshold: 0% + only_pulls: true diff --git a/.gitignore b/.gitignore index fd5e22ee2..d00cdb788 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ dist-newstyle postgrest.hp postgrest.prof __pycache__ +*.tix +coverage +.hpc diff --git a/default.nix b/default.nix index 683a7d71e..6ef1ca859 100644 --- a/default.nix +++ b/default.nix @@ -66,7 +66,7 @@ let # Options passed to cabal in dev tools and tests devCabalOptions = - "-f FailOnWarn --test-show-detail=direct"; + "-f dev --test-show-detail=direct"; profiledHaskellPackages = pkgs.haskell.packages."${compiler}".extend (self: super: @@ -87,7 +87,7 @@ rec { # libraries and documentation. We disable running the test suite on Nix # builds, as they require a database to be set up. postgrestPackage = - lib.dontCheck (lib.enableCabalFlag postgrest "FailOnWarn"); + lib.dontCheck postgrest; # Static executable. postgrestStatic = @@ -114,7 +114,11 @@ rec { # Scripts for running tests. tests = - pkgs.callPackage nix/tests.nix { inherit postgrest postgrestStatic postgrestProfiled postgresqlVersions devCabalOptions; }; + pkgs.callPackage nix/tests.nix { + inherit postgrest postgrestProfiled postgresqlVersions devCabalOptions; + ghc = pkgs.haskell.compiler."${compiler}"; + hpc-codecov = pkgs.haskell.packages."${compiler}".hpc-codecov; + }; # Linting and styling scripts. style = diff --git a/nix/devtools.nix b/nix/devtools.nix index 5aeaf7de0..55aa773c3 100644 --- a/nix/devtools.nix +++ b/nix/devtools.nix @@ -72,6 +72,8 @@ let } '' ${cabal-install}/bin/cabal v2-clean + # clean old coverage data, too + rm -rf .hpc coverage ''; check = diff --git a/nix/tests.nix b/nix/tests.nix index 2b2555f69..93a2395b8 100644 --- a/nix/tests.nix +++ b/nix/tests.nix @@ -5,15 +5,15 @@ , checkedShellScript , curl , devCabalOptions -, diffutils +, ghc +, gnugrep , haskell +, hpc-codecov , lib , postgresql , postgresqlVersions , postgrest , postgrestProfiled -, postgrestStatic -, procps , python3 , runtimeShell , yq @@ -164,6 +164,90 @@ let postgrest --dump-schema \ | ${yq}/bin/yq -y . ''; + + coverage = + name: postgresql: + checkedShellScript + { + inherit name; + docs = "Run spec and io tests while collecting hpc coverage data."; + inRootDir = true; + } + '' + env="$(cat ${postgrest.env})" + export PATH="$env/bin:$PATH" + + # clean up previous coverage reports + mkdir -p coverage + rm -rf coverage/* + + # temporary directory to collect data in + tmpdir="$(mktemp -d)" + + # we keep the tmpdir when an error occurs for debugging and only remove it on success + trap 'echo Temporary directory kept at: $tmpdir' ERR SIGINT SIGTERM + + # build once before running all the tests + ${cabal-install}/bin/cabal v2-build ${devCabalOptions} --enable-tests all + + # collect all tests + HPCTIXFILE="$tmpdir"/io.tix \ + ${withTmpDb postgresql} ${cabal-install}/bin/cabal v2-exec ${devCabalOptions} \ + ${ioTestPython}/bin/pytest -- -v test/io-tests + + HPCTIXFILE="$tmpdir"/spec.tix \ + ${withTmpDb postgresql} ${cabal-install}/bin/cabal v2-test ${devCabalOptions} + + # collect all the tix files + ${ghc}/bin/hpc sum --union --exclude=Paths_postgrest --output="$tmpdir"/tests.tix "$tmpdir"/io.tix "$tmpdir"/spec.tix + + # prepare the overlay + ${ghc}/bin/hpc overlay --output="$tmpdir"/overlay.tix test/coverage.overlay + ${ghc}/bin/hpc sum --union --output="$tmpdir"/tests-overlay.tix "$tmpdir"/tests.tix "$tmpdir"/overlay.tix + + # check nothing in the overlay is actually tested + ${ghc}/bin/hpc map --function=inv --output="$tmpdir"/inverted.tix "$tmpdir"/tests.tix + ${ghc}/bin/hpc combine --function=sub \ + --output="$tmpdir"/check.tix "$tmpdir"/overlay.tix "$tmpdir"/inverted.tix + # returns zero exit code if any count="" lines are found, i.e. + # something is covered by both the overlay and the tests + if ${ghc}/bin/hpc report --xml "$tmpdir"/check.tix | ${gnugrep}/bin/grep -qP 'count="[^0]' + then + ${ghc}/bin/hpc markup --highlight-covered --destdir=coverage/overlay "$tmpdir"/overlay.tix || true + ${ghc}/bin/hpc markup --highlight-covered --destdir=coverage/check "$tmpdir"/check.tix || true + echo "ERROR: Something is covered by both the tests and the overlay:" + echo "file://$(pwd)/coverage/check/hpc_index.html" + exit 1 + else + # copy the result .tix file to the coverage/ dir to make it available to postgrest-coverage-draft-overlay, too + cp "$tmpdir"/tests-overlay.tix coverage/postgrest.tix + # prepare codecov json report + ${hpc-codecov}/bin/hpc-codecov --mix=.hpc --out=coverage/codecov.json coverage/postgrest.tix + + # create html and stdout reports + # TODO: The markup command fails when run outside nix-shell (i.e. in CI!) + # Need to fix it properly in the future instead of adding the || true + ${ghc}/bin/hpc markup --destdir=coverage coverage/postgrest.tix || true + echo "file://$(pwd)/coverage/hpc_index.html" + ${ghc}/bin/hpc report coverage/postgrest.tix "$@" + fi + + rm -rf "$tmpdir" + ''; + + coverageDraftOverlay = + name: + checkedShellScript + { + inherit name; + docs = "Create a draft overlay from current coverage report."; + inRootDir = true; + } + '' + ${ghc}/bin/hpc draft --output=test/coverage.overlay coverage/postgrest.tix + sed -i 's|^module \(.*\):|module \1/|g' test/coverage.overlay + ''; + in # Create an environment that contains all the utility scripts for running tests # that we defined above. @@ -179,6 +263,8 @@ buildEnv testSpecAllVersions.bin (testIO "postgrest-test-io" postgresql).bin (dumpSchema "postgrest-dump-schema" postgresql).bin + (coverage "postgrest-coverage" postgresql).bin + (coverageDraftOverlay "postgrest-coverage-draft-overlay").bin ] ++ testSpecVersions; } # The memory tests have large dependencies (a profiled build of PostgREST) diff --git a/postgrest.cabal b/postgrest.cabal index a843a2327..560b260bf 100644 --- a/postgrest.cabal +++ b/postgrest.cabal @@ -19,12 +19,17 @@ source-repository head type: git location: git://github.com/PostgREST/postgrest.git -flag FailOnWarn +flag dev default: False manual: True - description: No warnings allowed + description: Development flags library + default-language: Haskell2010 + default-extensions: OverloadedStrings + QuasiQuotes + NoImplicitPrelude + hs-source-dirs: src exposed-modules: PostgREST.ApiRequest PostgREST.App PostgREST.Auth @@ -43,7 +48,6 @@ library PostgREST.Private.Common PostgREST.Private.ProxyUri PostgREST.Private.QueryFragment - hs-source-dirs: src build-depends: base >= 4.9 && < 4.15 , HTTP >= 4000.3.7 && < 4000.4 , Ranged-sets >= 0.3 && < 0.5 @@ -88,15 +92,12 @@ library , wai-extra >= 3.0.19 && < 3.2 , wai-logger >= 2.3.2 , wai-middleware-static >= 0.8.1 && < 0.10 - default-language: Haskell2010 - default-extensions: OverloadedStrings - QuasiQuotes - NoImplicitPrelude - if flag(FailOnWarn) - ghc-options: -O2 -Werror -Wall -fwarn-identities + if flag(dev) + ghc-options: -O0 -Werror -Wall -fwarn-identities -fno-spec-constr -optP-Wno-nonportable-include-path + -fhpc -hpcdir .hpc else - ghc-options: -O2 -Wall -fwarn-identities + ghc-options: -O2 -Werror -Wall -fwarn-identities -fno-spec-constr -optP-Wno-nonportable-include-path -- -fno-spec-constr may help keep compile time memory use in check, -- see https://gitlab.haskell.org/ghc/ghc/issues/16017#note_219304 @@ -105,8 +106,12 @@ library -- see https://github.com/commercialhaskell/stack/issues/3918 executable postgrest - main-is: Main.hs + default-language: Haskell2010 + default-extensions: OverloadedStrings + QuasiQuotes + NoImplicitPrelude hs-source-dirs: main + main-is: Main.hs build-depends: base >= 4.9 && < 4.15 , aeson >= 1.4.7 && < 1.6 , auto-update >= 0.1.4 && < 0.2 @@ -126,17 +131,14 @@ executable postgrest , time >= 1.6 && < 1.11 , wai >= 3.2.1 && < 3.3 , warp >= 3.2.12 && < 3.4 - default-language: Haskell2010 - default-extensions: OverloadedStrings - QuasiQuotes - NoImplicitPrelude - if flag(FailOnWarn) + if flag(dev) ghc-options: -threaded -rtsopts "-with-rtsopts=-N -I2" - -O2 -Werror -Wall -fwarn-identities + -O0 -Werror -Wall -fwarn-identities -fno-spec-constr -optP-Wno-nonportable-include-path + -fhpc -hpcdir .hpc else ghc-options: -threaded -rtsopts "-with-rtsopts=-N -I2" - -O2 -Wall -fwarn-identities + -O2 -Werror -Wall -fwarn-identities -fno-spec-constr -optP-Wno-nonportable-include-path if !os(windows) @@ -145,6 +147,11 @@ executable postgrest test-suite spec type: exitcode-stdio-1.0 + default-language: Haskell2010 + default-extensions: OverloadedStrings + QuasiQuotes + NoImplicitPrelude + hs-source-dirs: test main-is: Main.hs other-modules: Feature.AndOrParamsSpec Feature.AsymmetricJwtSpec @@ -179,7 +186,6 @@ test-suite spec Feature.UpsertSpec SpecHelper TestTypes - hs-source-dirs: test build-depends: base >= 4.9 && < 4.15 , aeson >= 1.4.7 && < 1.6 , aeson-qq >= 0.8.1 && < 0.9 @@ -211,20 +217,21 @@ test-suite spec , transformers-base >= 0.4.4 && < 0.5 , wai >= 3.2.1 && < 3.3 , wai-extra >= 3.0.19 && < 3.2 - default-language: Haskell2010 - default-extensions: OverloadedStrings - QuasiQuotes - NoImplicitPrelude ghc-options: -threaded -rtsopts -with-rtsopts=-N + -O0 -Werror -Wall -fwarn-identities + -fno-spec-constr -optP-Wno-nonportable-include-path + -fno-warn-missing-signatures -Test-Suite spec-querycost - Type: exitcode-stdio-1.0 - Default-Language: Haskell2010 - default-extensions: OverloadedStrings, QuasiQuotes, NoImplicitPrelude - Hs-Source-Dirs: test - Main-Is: QueryCost.hs - Other-Modules: SpecHelper - Build-Depends: base >= 4.9 && < 4.15 +test-suite spec-querycost + type: exitcode-stdio-1.0 + default-language: Haskell2010 + default-extensions: OverloadedStrings + QuasiQuotes + NoImplicitPrelude + hs-source-dirs: test + main-is: QueryCost.hs + other-modules: SpecHelper + build-depends: base >= 4.9 && < 4.15 , aeson >= 1.4.7 && < 1.6 , aeson-qq >= 0.8.1 && < 0.9 , async >= 2.1.1 && < 2.3 @@ -256,3 +263,5 @@ Test-Suite spec-querycost , transformers-base >= 0.4.4 && < 0.5 , wai >= 3.2.1 && < 3.3 , wai-extra >= 3.0.19 && < 3.2 + ghc-options: -O0 -Werror -Wall -fwarn-identities + -fno-spec-constr -optP-Wno-nonportable-include-path diff --git a/stack.yaml b/stack.yaml index 17c56ef9b..23fb22b07 100644 --- a/stack.yaml +++ b/stack.yaml @@ -1,9 +1,5 @@ resolver: lts-16.26 # 2020-12-13, GHC 8.8.4 -flags: - postgrest: - FailOnWarn: true - nix: packages: - pcre diff --git a/test/Feature/InsertSpec.hs b/test/Feature/InsertSpec.hs index 9fb4d3420..998bfca84 100644 --- a/test/Feature/InsertSpec.hs +++ b/test/Feature/InsertSpec.hs @@ -1,14 +1,10 @@ module Feature.InsertSpec where -import qualified Data.Aeson as JSON - import Data.List (lookup) -import Data.Maybe (fromJust) import Network.Wai (Application) import Network.Wai.Test (SResponse (simpleBody, simpleHeaders, simpleStatus)) import Test.Hspec hiding (pendingWith) import Test.Hspec.Wai.Matcher (bodyEquals) -import TestTypes (CompoundPK (..), IncPK (..)) import Network.HTTP.Types import Test.Hspec.Wai @@ -441,9 +437,6 @@ spec actualPgVersion = do describe "Row level permission" $ it "set user_id when inserting rows" $ do - post "/postgrest/users" [json| { "id":"jdoe", "pass": "1234", "role": "postgrest_test_author" } |] - post "/postgrest/users" [json| { "id":"jroe", "pass": "1234", "role": "postgrest_test_author" } |] - request methodPost "/authors_only" [ authHeaderJWT "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoicG9zdGdyZXN0X3Rlc3RfYXV0aG9yIiwiaWQiOiJqZG9lIn0.B-lReuGNDwAlU1GOC476MlO0vAt9JNoHIlxg2vwMaO0", ("Prefer", "return=representation") ] [json| { "secret": "nyancat" } |] diff --git a/test/Feature/QueryLimitedSpec.hs b/test/Feature/QueryLimitedSpec.hs index b6c16b1ac..97d37d012 100644 --- a/test/Feature/QueryLimitedSpec.hs +++ b/test/Feature/QueryLimitedSpec.hs @@ -1,7 +1,6 @@ module Feature.QueryLimitedSpec where -import Network.Wai (Application) -import Network.Wai.Test (SResponse (simpleHeaders, simpleStatus)) +import Network.Wai (Application) import Network.HTTP.Types import Test.Hspec diff --git a/test/Feature/RollbackSpec.hs b/test/Feature/RollbackSpec.hs index db18f8b98..24bdbcd07 100644 --- a/test/Feature/RollbackSpec.hs +++ b/test/Feature/RollbackSpec.hs @@ -7,8 +7,7 @@ import Test.Hspec import Test.Hspec.Wai import Test.Hspec.Wai.JSON -import Protolude hiding (get) -import SpecHelper +import Protolude hiding (get) -- two helpers functions to make sure that each test can setup and cleanup properly diff --git a/test/Feature/RpcPreRequestGucsSpec.hs b/test/Feature/RpcPreRequestGucsSpec.hs index 8b1fe6a07..c615b0c23 100644 --- a/test/Feature/RpcPreRequestGucsSpec.hs +++ b/test/Feature/RpcPreRequestGucsSpec.hs @@ -1,18 +1,13 @@ module Feature.RpcPreRequestGucsSpec where -import qualified Data.ByteString.Lazy as BL (empty) - -import Network.Wai (Application) -import Network.Wai.Test (SResponse (simpleBody, simpleHeaders, simpleStatus)) +import Network.Wai (Application) import Network.HTTP.Types import Test.Hspec hiding (pendingWith) import Test.Hspec.Wai import Test.Hspec.Wai.JSON -import Text.Heredoc -import Protolude hiding (get, put) -import SpecHelper +import Protolude hiding (get, put) spec :: SpecWith ((), Application) spec = diff --git a/test/Feature/UnicodeSpec.hs b/test/Feature/UnicodeSpec.hs index 5241a6271..532c4ce33 100644 --- a/test/Feature/UnicodeSpec.hs +++ b/test/Feature/UnicodeSpec.hs @@ -7,8 +7,7 @@ import Test.Hspec import Test.Hspec.Wai import Test.Hspec.Wai.JSON -import Protolude hiding (get) -import SpecHelper +import Protolude hiding (get) spec :: SpecWith ((), Application) spec = diff --git a/test/Feature/UpdateSpec.hs b/test/Feature/UpdateSpec.hs index 11dc721b4..e511be2fa 100644 --- a/test/Feature/UpdateSpec.hs +++ b/test/Feature/UpdateSpec.hs @@ -1,14 +1,11 @@ module Feature.UpdateSpec where -import Data.List (lookup) -import Network.Wai (Application) -import Network.Wai.Test (SResponse (simpleBody, simpleHeaders, simpleStatus)) -import Test.Hspec hiding (pendingWith) +import Network.Wai (Application) +import Test.Hspec hiding (pendingWith) import Network.HTTP.Types import Test.Hspec.Wai import Test.Hspec.Wai.JSON -import Text.Heredoc import Protolude hiding (get) import SpecHelper diff --git a/test/QueryCost.hs b/test/QueryCost.hs index 246464156..eb118111c 100644 --- a/test/QueryCost.hs +++ b/test/QueryCost.hs @@ -5,7 +5,6 @@ import qualified Data.Aeson.Lens as L import qualified Hasql.Decoders as HD import qualified Hasql.DynamicStatements.Snippet as H import qualified Hasql.DynamicStatements.Statement as H -import qualified Hasql.Encoders as HE import qualified Hasql.Pool as P import qualified Hasql.Statement as H import qualified Hasql.Transaction as HT diff --git a/test/coverage.overlay b/test/coverage.overlay new file mode 100644 index 000000000..e69de29bb diff --git a/test/io-tests/test_io.py b/test/io-tests/test_io.py index 5bb543354..6c7517f4a 100644 --- a/test/io-tests/test_io.py +++ b/test/io-tests/test_io.py @@ -75,13 +75,16 @@ def defaultenv(): def dumpconfig(configpath=None, env=None, stdin=None): "Dump the config as parsed by PostgREST." + env = env or {} + command = [POSTGREST_BIN, "--dump-config"] + env["HPCTIXFILE"] = os.getenv("HPCTIXFILE", "") if configpath: command.append(configpath) process = subprocess.Popen( - command, env=env or {}, stdin=subprocess.PIPE, stdout=subprocess.PIPE + command, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE ) process.stdin.write(stdin or b"") @@ -96,6 +99,7 @@ def dumpconfig(configpath=None, env=None, stdin=None): @contextlib.contextmanager def run(configpath=None, stdin=None, env=None, port=None): "Run PostgREST and yield an endpoint that is ready for connections." + env = env or {} with tempfile.TemporaryDirectory() as tmpdir: if port: @@ -108,11 +112,12 @@ def run(configpath=None, stdin=None, env=None, port=None): baseurl = "http+unix://" + urllib.parse.quote_plus(str(socketfile)) command = [POSTGREST_BIN] + env["HPCTIXFILE"] = os.getenv("HPCTIXFILE", "") if configpath: command.append(configpath) - process = subprocess.Popen(command, stdin=subprocess.PIPE, env=env or {}) + process = subprocess.Popen(command, stdin=subprocess.PIPE, env=env) try: process.stdin.write(stdin or b"")