Add improved query for allSynonyms

This query works with views with subselects, and it's a lot faster,
on a complex schema the previous query was taking around 8 mins,
this query takes less than half a second(403.072 ms).

Also reorder view embedding tests

CHANGELOG.md

@@ -12,6 +12,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Fixed
 
 - #1113, Fix UPSERT failing when having a camel case PK column - @steve-chavez
+- #945, Fix slow start-up time on big schemas - @steve-chavez
 
 ### Changed
 

src/PostgREST/DbStructure.hs

@@ -40,7 +40,7 @@ getDbStructure :: Schema -> PgVersion -> H.Session DbStructure
 getDbStructure schema pgVer = do
   tabs      <- H.query () allTables
   cols      <- H.query schema $ allColumns tabs
-  syns      <- H.query () $ allSynonyms cols
+  syns      <- H.query schema $ allSynonyms cols
   childRels <- H.query () $ allChildRelations tabs cols
   keys      <- H.query () $ allPrimaryKeys tabs
   procs     <- H.query schema allProcs
@@ -689,79 +689,67 @@ pkFromRow :: [Table] -> (Schema, Text, Text) -> Maybe PrimaryKey
 pkFromRow tabs (s, t, n) = PrimaryKey <$> table <*> pure n
   where table = find (\tbl -> tableSchema tbl == s && tableName tbl == t) tabs
 
-allSynonyms :: [Column] -> H.Query () [Synonym]
+allSynonyms :: [Column] -> H.Query Schema [Synonym]
 allSynonyms cols =
-  H.statement sql HE.unit (decodeSynonyms cols) True
- where
-  -- query explanation at https://gist.github.com/ruslantalpa/2eab8c930a65e8043d8f
-  sql = [q|
-    with view_columns as (
-        select
-            c.oid as view_oid,
-            a.attname::information_schema.sql_identifier as column_name
-        from pg_attribute a
-        join pg_class c on a.attrelid = c.oid
-        join pg_namespace nc on c.relnamespace = nc.oid
-        where
-            not pg_is_other_temp_schema(nc.oid)
-            and a.attnum > 0
-            and not a.attisdropped
-            and (c.relkind = 'v'::"char")
-            and nc.nspname not in ('information_schema', 'pg_catalog')
+  H.statement sql (HE.value HE.text) (decodeSynonyms cols) True
+  -- query explanation at https://gist.github.com/steve-chavez/7ee0e6590cddafb532e5f00c46275569
+  where sql = [q|
+    with
+    views as (
+      select
+        n.nspname   as view_schema,
+        c.relname   as view_name,
+        r.ev_action as view_definition
+      from pg_class c
+      join pg_namespace n on n.oid = c.relnamespace
+      join pg_rewrite r on r.ev_class = c.oid
+      where (c.relkind = 'v'::char) and n.nspname = $1
     ),
-    view_column_usage as (
-        select distinct
-            v.oid as view_oid,
-            nv.nspname::information_schema.sql_identifier as view_schema,
-            v.relname::information_schema.sql_identifier as view_name,
-            nt.nspname::information_schema.sql_identifier as table_schema,
-            t.relname::information_schema.sql_identifier as table_name,
-            a.attname::information_schema.sql_identifier as column_name,
-            pg_get_viewdef(v.oid)::information_schema.character_data as view_definition
-        from pg_namespace nv
-        join pg_class v on nv.oid = v.relnamespace
-        join pg_depend dv on v.oid = dv.refobjid
-        join pg_depend dt on dv.objid = dt.objid
-        join pg_class t on dt.refobjid = t.oid
-        join pg_namespace nt on t.relnamespace = nt.oid
-        join pg_attribute a on t.oid = a.attrelid and dt.refobjsubid = a.attnum
-
-        where
-            nv.nspname not in ('information_schema', 'pg_catalog')
-            and v.relkind = 'v'::"char"
-            and dv.refclassid = 'pg_class'::regclass::oid
-            and dv.classid = 'pg_rewrite'::regclass::oid
-            and dv.deptype = 'i'::"char"
-            and dv.refobjid <> dt.refobjid
-            and dt.classid = 'pg_rewrite'::regclass::oid
-            and dt.refclassid = 'pg_class'::regclass::oid
-            and (t.relkind = any (array['r'::"char", 'v'::"char", 'f'::"char"]))
+    removed_subselects as(
+      select
+        view_schema, view_name,
+        regexp_replace(view_definition, ':subselect {.*?:constraintDeps <>} :location', '', 'g') as x
+      from views
     ),
-    candidates as (
-        select
-            vcu.*,
-            (
-                select case when match is not null then coalesce(match[8], match[7], match[4]) end
-                from regexp_matches(
-                    CONCAT('SELECT ', SPLIT_PART(vcu.view_definition, 'SELECT', 2)),
-                    CONCAT('SELECT.*?((',vcu.table_name,')|(\w+))\.(', vcu.column_name, ')(\s+AS\s+("([^"]+)"|([^, \n\t]+)))?.*?FROM.*?(',vcu.table_schema,'\.|)(\2|',vcu.table_name,'\s+(as\s)?\3)'),
-                    'nsi'
-                ) match
-            ) as view_column_name
-        from view_column_usage as vcu
+    target_lists as(
+      select
+        view_schema, view_name,
+        regexp_split_to_array(x, 'targetList') as x
+      from removed_subselects
+    ),
+    last_target_list_wo_tail as(
+      select
+        view_schema, view_name,
+        (regexp_split_to_array(x[array_upper(x, 1)], ':onConflict'))[1] as x
+      from target_lists
+    ),
+    target_entries as(
+      select
+        view_schema, view_name,
+        unnest((regexp_split_to_array(x, 'TARGETENTRY'))[2:]) as entry
+      from last_target_list_wo_tail
+    ),
+    results as(
+      select
+        view_schema, view_name,
+        substring(entry from ':resname (.*?) :') as view_colum_name,
+        substring(entry from ':resorigtbl (.*?) :') as resorigtbl,
+        substring(entry from ':resorigcol (.*?) :') as resorigcol
+      from target_entries
     )
     select
-        c.table_schema,
-        c.table_name,
-        c.column_name as table_column_name,
-        c.view_schema,
-        c.view_name,
-        c.view_column_name
-    from view_columns as vc, candidates as c
-    where
-        vc.view_oid = c.view_oid
-        and vc.column_name = c.view_column_name
-    order by c.view_schema, c.view_name, c.table_name, c.view_column_name
+      sch.nspname as table_schema,
+      tbl.relname as table_name,
+      col.attname as table_column_name,
+      res.view_schema,
+      res.view_name,
+      res.view_colum_name
+    from results res
+    join pg_class tbl on tbl.oid::text = res.resorigtbl
+    join pg_attribute col on col.attrelid = tbl.oid and col.attnum::text = res.resorigcol
+    join pg_namespace sch on sch.oid = tbl.relnamespace
+    where resorigtbl <> '0'
+    order by view_schema, view_name, view_colum_name;
     |]
 
 synonymFromRow :: [Column] -> (Text,Text,Text,Text,Text,Text) -> Maybe Synonym

test/Feature/QuerySpec.hs

@@ -223,13 +223,12 @@ spec = do
         , matchHeaders = []
         }
 
-
     it "requesting parents and children" $
       get "/projects?id=eq.1&select=id, name, clients(*), tasks(id, name)" `shouldRespondWith`
         [json|[{"id":1,"name":"Windows 7","clients":{"id":1,"name":"Microsoft"},"tasks":[{"id":1,"name":"Design w7"},{"id":2,"name":"Code w7"}]}]|]
         { matchHeaders = [matchContentTypeJson] }
 
-    it "requesting parent without specifying primary key" $ do
+    it "requesting parent without specifying primary key" $
       get "/projects?select=name,client(name)" `shouldRespondWith`
         [json|[
           {"name":"Windows 7","client":{"name": "Microsoft"}},
@@ -239,9 +238,6 @@ spec = do
           {"name":"Orphan","client":null}
         ]|]
         { matchHeaders = [matchContentTypeJson] }
-      get "/articleStars?select=createdAt,article(owner),user(name)&limit=1" `shouldRespondWith`
-        [json|[{"createdAt":"2015-12-08T04:22:57.472738","article":{"owner": "postgrest_test_authenticator"},"user":{"name": "Angela Martin"}}]|]
-        { matchHeaders = [matchContentTypeJson] }
 
     it "requesting parent and renaming primary key" $
       get "/projects?select=name,client(clientId:id,name)" `shouldRespondWith`
@@ -313,40 +309,11 @@ spec = do
         [json|[{"id":1,"tasks":[{"id":1},{"id":2},{"id":3},{"id":4}]},{"id":2,"tasks":[{"id":5},{"id":6},{"id":7}]},{"id":3,"tasks":[{"id":1},{"id":5}]}]|]
         { matchHeaders = [matchContentTypeJson] }
 
-    it "requesting parents and children on views" $
-      get "/projects_view?id=eq.1&select=id, name, clients(*), tasks(id, name)" `shouldRespondWith`
-        [json|[{"id":1,"name":"Windows 7","clients":{"id":1,"name":"Microsoft"},"tasks":[{"id":1,"name":"Design w7"},{"id":2,"name":"Code w7"}]}]|]
-        { matchHeaders = [matchContentTypeJson] }
-
-    it "requesting parents and children on views with renamed keys" $
-      get "/projects_view_alt?t_id=eq.1&select=t_id, name, clients(*), tasks(id, name)" `shouldRespondWith`
-        [json|[{"t_id":1,"name":"Windows 7","clients":{"id":1,"name":"Microsoft"},"tasks":[{"id":1,"name":"Design w7"},{"id":2,"name":"Code w7"}]}]|]
-        { matchHeaders = [matchContentTypeJson] }
-
-    it "detects parent relations when having many views of a private table" $ do
-      get "/books?select=title,author(name)&id=eq.5" `shouldRespondWith`
-        [json|[ { "title": "Farenheit 451", "author": { "name": "Ray Bradbury" } } ]|]
-        { matchHeaders = [matchContentTypeJson] }
-      get "/forties_books?select=title,author(name)&limit=1" `shouldRespondWith`
-        [json|[ { "title": "1984", "author": { "name": "George Orwell" } } ]|]
-        { matchHeaders = [matchContentTypeJson] }
-      get "/fifties_books?select=title,author(name)&limit=1" `shouldRespondWith`
-        [json|[ { "title": "The Catcher in the Rye", "author": { "name": "J.D. Salinger" } } ]|]
-        { matchHeaders = [matchContentTypeJson] }
-      get "/sixties_books?select=title,author(name)&limit=1" `shouldRespondWith`
-        [json|[ { "title": "To Kill a Mockingbird", "author": { "name": "Harper Lee" } } ]|]
-        { matchHeaders = [matchContentTypeJson] }
-
     it "requesting children with composite key" $
       get "/users_tasks?user_id=eq.2&task_id=eq.6&select=*, comments(content)" `shouldRespondWith`
         [json|[{"user_id":2,"task_id":6,"comments":[{"content":"Needs to be delivered ASAP"}]}]|]
         { matchHeaders = [matchContentTypeJson] }
 
-    it "detect relations in views from exposed schema that are based on tables in private schema and have columns renames" $
-      get "/articles?id=eq.1&select=id,articleStars(users(*))" `shouldRespondWith`
-        [json|[{"id":1,"articleStars":[{"users":{"id":1,"name":"Angela Martin"}},{"users":{"id":2,"name":"Michael Scott"}},{"users":{"id":3,"name":"Dwight Schrute"}}]}]|]
-        { matchHeaders = [matchContentTypeJson] }
-
     it "can embed by FK column name" $
       get "/projects?id=in.(1,3)&select=id,name,client_id(id,name)" `shouldRespondWith`
         [json|[{"id":1,"name":"Windows 7","client_id":{"id":1,"name":"Microsoft"}},{"id":3,"name":"IOS","client_id":{"id":2,"name":"Apple"}}]|]
@@ -362,8 +329,53 @@ spec = do
         [json|[{"id":1,"name":"Windows 7","client_id":1,"client":{"id":1,"name":"Microsoft"}},{"id":3,"name":"IOS","client_id":2,"client":{"id":2,"name":"Apple"}}]|]
         { matchHeaders = [matchContentTypeJson] }
 
-    it "can detect fk relations through views to tables in the public schema" $
-      get "/consumers_view?select=*,orders_view(*)" `shouldRespondWith` 200
+    describe "view embedding" $ do
+      it "can detect fk relations through views to tables in the public schema" $
+        get "/consumers_view?select=*,orders_view(*)" `shouldRespondWith` 200
+
+      it "can request parent without specifying primary key" $
+        get "/articleStars?select=createdAt,article(owner),user(name)&limit=1" `shouldRespondWith`
+          [json|[{"createdAt":"2015-12-08T04:22:57.472738","article":{"owner": "postgrest_test_authenticator"},"user":{"name": "Angela Martin"}}]|]
+          { matchHeaders = [matchContentTypeJson] }
+
+      it "can detect relations in views from exposed schema that are based on tables in private schema and have columns renames" $
+        get "/articles?id=eq.1&select=id,articleStars(users(*))" `shouldRespondWith`
+          [json|[{"id":1,"articleStars":[{"users":{"id":1,"name":"Angela Martin"}},{"users":{"id":2,"name":"Michael Scott"}},{"users":{"id":3,"name":"Dwight Schrute"}}]}]|]
+          { matchHeaders = [matchContentTypeJson] }
+
+      it "works when requesting parents and children on views" $
+        get "/projects_view?id=eq.1&select=id, name, clients(*), tasks(id, name)" `shouldRespondWith`
+          [json|[{"id":1,"name":"Windows 7","clients":{"id":1,"name":"Microsoft"},"tasks":[{"id":1,"name":"Design w7"},{"id":2,"name":"Code w7"}]}]|]
+          { matchHeaders = [matchContentTypeJson] }
+
+      it "works when requesting parents and children on views with renamed keys" $
+        get "/projects_view_alt?t_id=eq.1&select=t_id, name, clients(*), tasks(id, name)" `shouldRespondWith`
+          [json|[{"t_id":1,"name":"Windows 7","clients":{"id":1,"name":"Microsoft"},"tasks":[{"id":1,"name":"Design w7"},{"id":2,"name":"Code w7"}]}]|]
+          { matchHeaders = [matchContentTypeJson] }
+
+      it "detects parent relations when having many views of a private table" $ do
+        get "/books?select=title,author(name)&id=eq.5" `shouldRespondWith`
+          [json|[ { "title": "Farenheit 451", "author": { "name": "Ray Bradbury" } } ]|]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/forties_books?select=title,author(name)&limit=1" `shouldRespondWith`
+          [json|[ { "title": "1984", "author": { "name": "George Orwell" } } ]|]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/fifties_books?select=title,author(name)&limit=1" `shouldRespondWith`
+          [json|[ { "title": "The Catcher in the Rye", "author": { "name": "J.D. Salinger" } } ]|]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/sixties_books?select=title,author(name)&limit=1" `shouldRespondWith`
+          [json|[ { "title": "To Kill a Mockingbird", "author": { "name": "Harper Lee" } } ]|]
+          { matchHeaders = [matchContentTypeJson] }
+
+      it "works with views that have subselects" $ do
+        get "/authors_books_number?select=*,books(title)&id=eq.1" `shouldRespondWith`
+          [json|[ {"id":1, "name":"George Orwell","num_in_forties":1,"num_in_fifties":0,"num_in_sixties":0,"num_in_all_decades":1,
+                   "books":[{"title":"1984"}]} ]|]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/authors_have_book_in_decade?select=*,books(title)&id=eq.3" `shouldRespondWith`
+          [json|[ {"id":3,"name":"Antoine de Saint-Exupéry","has_book_in_forties":true,"has_book_in_fifties":false,"has_book_in_sixties":false,
+                   "books":[{"title":"The Little Prince"}]} ]|]
+          { matchHeaders = [matchContentTypeJson] }
 
     describe "path fixed" $ do
       it "works when requesting children 2 levels" $
@@ -371,13 +383,7 @@ spec = do
           [json|[{"id":1,"projects":[{"id":1,"tasks":[{"id":1},{"id":2}]},{"id":2,"tasks":[{"id":3},{"id":4}]}]}]|]
           { matchHeaders = [matchContentTypeJson] }
 
-      it "works with parent relation" $ do
-        get "/message?select=id,body,sender:person_detail.sender(name,sent),recipient:person_detail.recipient(name,received)&id=lt.4" `shouldRespondWith`
-          [json|
-            [{"id":1,"body":"Hello Jane","sender":{"name":"John","sent":2},"recipient":{"name":"Jane","received":2}},
-             {"id":2,"body":"Hi John","sender":{"name":"Jane","sent":1},"recipient":{"name":"John","received":1}},
-             {"id":3,"body":"How are you doing?","sender":{"name":"John","sent":2},"recipient":{"name":"Jane","received":2}}] |]
-          { matchHeaders = [matchContentTypeJson] }
+      it "works with parent relation" $
         get "/message?select=id,body,sender:person.sender(name),recipient:person.recipient(name)&id=lt.4" `shouldRespondWith`
           [json|
             [{"id":1,"body":"Hello Jane","sender":{"name":"John"},"recipient":{"name":"Jane"}},
@@ -385,6 +391,14 @@ spec = do
              {"id":3,"body":"How are you doing?","sender":{"name":"John"},"recipient":{"name":"Jane"}}] |]
           { matchHeaders = [matchContentTypeJson] }
 
+      it "works with a parent view relation" $
+        get "/message?select=id,body,sender:person_detail.sender(name,sent),recipient:person_detail.recipient(name,received)&id=lt.4" `shouldRespondWith`
+          [json|
+            [{"id":1,"body":"Hello Jane","sender":{"name":"John","sent":2},"recipient":{"name":"Jane","received":2}},
+             {"id":2,"body":"Hi John","sender":{"name":"Jane","sent":1},"recipient":{"name":"John","received":1}},
+             {"id":3,"body":"How are you doing?","sender":{"name":"John","sent":2},"recipient":{"name":"Jane","received":2}}] |]
+          { matchHeaders = [matchContentTypeJson] }
+
       it "works with many<->many relation" $
         get "/tasks?select=id,users:users.users_tasks(id)" `shouldRespondWith`
           [json|[{"id":1,"users":[{"id":1},{"id":3}]},{"id":2,"users":[{"id":1}]},{"id":3,"users":[{"id":1}]},{"id":4,"users":[{"id":1}]},{"id":5,"users":[{"id":2},{"id":3}]},{"id":6,"users":[{"id":2}]},{"id":7,"users":[{"id":2}]},{"id":8,"users":[]}]|]

test/fixtures/privileges.sql

@@ -82,6 +82,8 @@ GRANT ALL ON TABLE
     , "UnitTest"
     , json_arr
     , jsonb_test
+    , authors_books_number
+    , authors_have_book_in_decade
 TO postgrest_test_anonymous;
 
 GRANT INSERT ON TABLE insertonly TO postgrest_test_anonymous;

test/fixtures/schema.sql

@@ -1120,17 +1120,17 @@ create view images_base64 as (
   select name, replace(encode(img, 'base64'), E'\n', '') as img from images
 );
 
-create function test.ret_enum(val text) returns test.enum_menagerie_type as $$ 
+create function test.ret_enum(val text) returns test.enum_menagerie_type as $$
   select val::test.enum_menagerie_type;
 $$ language sql;
 
 create domain one_nine as integer check (value >= 1 and value <= 9);
 
-create function test.ret_array() returns integer[] as $$ 
+create function test.ret_array() returns integer[] as $$
   select '{1,2,3}'::integer[];
 $$ language sql;
 
-create function test.ret_domain(val integer) returns test.one_nine as $$ 
+create function test.ret_domain(val integer) returns test.one_nine as $$
   select val::test.one_nine;
 $$ language sql;
 
@@ -1144,20 +1144,20 @@ $$ language sql;
 
 create function test.ret_scalars() returns table(
   a text, b test.enum_menagerie_type, c test.one_nine, d int4range
-) as $$ 
-  select row('scalars'::text, enum_first(null::test.enum_menagerie_type), 
+) as $$
+  select row('scalars'::text, enum_first(null::test.enum_menagerie_type),
               1::test.one_nine, int4range(10, 20));
 $$ language sql;
 
 create type test.point_2d as (x integer, y integer);
 
-create function test.ret_point_2d() returns test.point_2d as $$ 
+create function test.ret_point_2d() returns test.point_2d as $$
   select row(10, 5)::test.point_2d;
 $$ language sql;
 
 create type private.point_3d as (x integer, y integer, z integer);
 
-create function test.ret_point_3d() returns private.point_3d as $$ 
+create function test.ret_point_3d() returns private.point_3d as $$
   select row(7, -3, 4)::private.point_3d;
 $$ language sql;
 
@@ -1171,17 +1171,17 @@ create function test.ret_rows_with_base64_bin() returns setof test.images_base64
   select i.name, i.img from test.images_base64 i;
 $$ language sql;
 
-create function test.single_article(id integer) returns test.articles as $$ 
+create function test.single_article(id integer) returns test.articles as $$
   select a.* from test.articles a where a.id = $1;
 $$ language sql;
 
-create function test.get_guc_value(name text) returns text as $$ 
+create function test.get_guc_value(name text) returns text as $$
   select nullif(current_setting(name), '')::text;
 $$ language sql;
 
 create table w_or_wo_comma_names ( name text );
 
-create table items_with_different_col_types ( 
+create table items_with_different_col_types (
   int_data integer,
   text_data text,
   bool_data bool,
@@ -1194,20 +1194,20 @@ create table items_with_different_col_types (
 
 -- Tables used for testing complex boolean logic with and/or query params
 
-create table entities ( 
+create table entities (
   id integer primary key,
   name text,
   arr integer[],
   text_search_vector tsvector
 );
 
-create table child_entities ( 
+create table child_entities (
   id integer primary key,
   name text,
   parent_id integer references entities(id)
 );
 
-create table grandchild_entities ( 
+create table grandchild_entities (
   id integer primary key,
   name text,
   parent_id integer references child_entities(id),
@@ -1466,3 +1466,59 @@ create table jsonb_test(
   id integer primary key,
   data jsonb
 );
+
+create view test.authors_books_number as
+select
+  id,
+  name,
+  (
+    select
+      count(*)
+    from forties_books where author_id = authors.id
+  ) as num_in_forties,
+  (
+    select
+      count(*)
+    from fifties_books where author_id = authors.id
+  ) as num_in_fifties,
+  (
+    select
+      count(*)
+    from sixties_books where author_id = authors.id
+  ) as num_in_sixties,
+  (
+    select
+      count(*)
+    from (
+      select id
+      from forties_books where author_id = authors.id
+      union
+      select id
+      from fifties_books where author_id = authors.id
+      union
+      select id
+      from sixties_books where author_id = authors.id
+    ) _
+  ) as num_in_all_decades
+from private.authors;
+
+create view test.authors_have_book_in_decade as
+select
+  id,
+  name,
+  CASE
+    WHEN (x.id IN (SELECT author_id FROM test.forties_books))
+    THEN true
+    ELSE false
+  END AS has_book_in_forties,
+  CASE
+    WHEN (x.id IN (SELECT author_id FROM test.fifties_books))
+    THEN true
+    ELSE false
+  END AS has_book_in_fifties,
+  CASE
+    WHEN (x.id IN (SELECT author_id FROM test.sixties_books))
+    THEN true
+    ELSE false
+  END AS has_book_in_sixties
+from private.authors x;