Skip to content

Commit 58ba01f

Browse files
jberrymanrakeshkky0x777
authored
Improve performance of replace_metadata on tracking tables. Closes hasura#3802 (hasura#4182)
* Improve performance of replace_metadata on tracking tables. Closes hasura#3802 On the 1000 table track case this went from >20min to 8 seconds, the bottleneck being all the former calls (for each table) to buildSchemaCache. * refactor replace metadata Only save metadata in hdb_catalog schema and build schema cache (strictly) * remove `withBuildSchemaCache` function Co-authored-by: rakeshkky <12475069+rakeshkky@users.noreply.github.com> Co-authored-by: Vamshi Surabhi <0x777@users.noreply.github.com>
1 parent e33f3a8 commit 58ba01f

File tree

6 files changed

+75
-89
lines changed

6 files changed

+75
-89
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ Read more about check constraints on [Postgres Docs](https://www.postgresql.org/
3030
(close #3969) (#4145)
3131

3232
### Bug fixes and improvements
33+
- server: improve performance of replace_metadata tracking many tables (fix #3802)
3334
- server: option to reload remote schemas in 'reload_metadata' API (fix #3792, #4117)
3435
- server: fix various space leaks to avoid excessive memory consumption
3536
- server: fix postgres query error when computed fields included in mutation response (fix #4035)

‎server/src-lib/Hasura/GraphQL/Schema/Merge.hs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -93,12 +93,12 @@ checkSchemaConflicts gCtx remoteCtx = do
9393

9494
checkConflictingNode
9595
:: (MonadError QErr m)
96-
=> GCtx
96+
=> TypeMap
97+
-- ^ See 'GCtx'.
9798
-> G.Name
9899
-> m ()
99-
checkConflictingNode gCtx node = do
100-
let typeMap = _gTypes gCtx
101-
hQR = _otiFields <$>
100+
checkConflictingNode typeMap node = do
101+
let hQR = _otiFields <$>
102102
(getObjTyM =<< Map.lookup hQRName typeMap)
103103
hMR = _otiFields <$>
104104
(getObjTyM =<< Map.lookup hMRName typeMap)

‎server/src-lib/Hasura/RQL/DDL/Metadata.hs

Lines changed: 57 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
{-# LANGUAGE RecordWildCards #-}
12
module Hasura.RQL.DDL.Metadata
23
( runReplaceMetadata
34
, runExportMetadata
@@ -26,10 +27,10 @@ import Hasura.RQL.DDL.ComputedField (dropComputedFieldFromCatalo
2627
import Hasura.RQL.DDL.EventTrigger (delEventTriggerFromCatalog, subTableP2)
2728
import Hasura.RQL.DDL.Metadata.Types
2829
import Hasura.RQL.DDL.Permission.Internal (dropPermFromCatalog)
29-
import Hasura.RQL.DDL.RemoteSchema (addRemoteSchemaP2, fetchRemoteSchemas,
30+
import Hasura.RQL.DDL.RemoteSchema (addRemoteSchemaToCatalog, fetchRemoteSchemas,
3031
removeRemoteSchemaFromCatalog)
32+
import Hasura.RQL.DDL.Schema.Catalog (saveTableToCatalog)
3133
import Hasura.RQL.Types
32-
import Hasura.Server.Version (HasVersion)
3334
import Hasura.SQL.Types
3435

3536
import qualified Database.PG.Query as Q
@@ -41,8 +42,9 @@ import qualified Hasura.RQL.DDL.QueryCollection as Collection
4142
import qualified Hasura.RQL.DDL.Relationship as Relationship
4243
import qualified Hasura.RQL.DDL.Schema as Schema
4344

44-
clearMetadata :: Q.TxE QErr ()
45-
clearMetadata = Q.catchE defaultTxErrorHandler $ do
45+
-- | Purge all user-defined metadata; metadata with is_system_defined = false
46+
clearUserMetadata :: MonadTx m => m ()
47+
clearUserMetadata = liftTx $ Q.catchE defaultTxErrorHandler $ do
4648
Q.unitQ "DELETE FROM hdb_catalog.hdb_function WHERE is_system_defined <> 'true'" () False
4749
Q.unitQ "DELETE FROM hdb_catalog.hdb_permission WHERE is_system_defined <> 'true'" () False
4850
Q.unitQ "DELETE FROM hdb_catalog.hdb_relationship WHERE is_system_defined <> 'true'" () False
@@ -60,7 +62,7 @@ runClearMetadata
6062
:: (MonadTx m, CacheRWM m)
6163
=> ClearMetadata -> m EncJSON
6264
runClearMetadata _ = do
63-
liftTx clearMetadata
65+
clearUserMetadata
6466
buildSchemaCacheStrict
6567
return successMsg
6668

@@ -124,69 +126,53 @@ applyQP1 (ReplaceMetadata _ tables functionsMeta schemas collections
124126
getDups l =
125127
l L.\\ HS.toList (HS.fromList l)
126128

127-
applyQP2
128-
:: ( HasVersion
129-
, MonadIO m
130-
, MonadTx m
131-
, CacheRWM m
132-
, HasSystemDefined m
133-
, HasHttpManager m
134-
)
135-
=> ReplaceMetadata
136-
-> m EncJSON
137-
applyQP2 (ReplaceMetadata _ tables functionsMeta
138-
schemas collections allowlist customTypes actions) = do
139-
140-
liftTx clearMetadata
129+
applyQP2 :: (CacheRWM m, MonadTx m, HasSystemDefined m) => ReplaceMetadata -> m EncJSON
130+
applyQP2 replaceMetadata = do
131+
clearUserMetadata
132+
saveMetadata replaceMetadata
141133
buildSchemaCacheStrict
134+
pure successMsg
135+
136+
saveMetadata :: (MonadTx m, HasSystemDefined m) => ReplaceMetadata -> m ()
137+
saveMetadata (ReplaceMetadata _ tables functionsMeta
138+
schemas collections allowlist customTypes actions) = do
142139

143140
withPathK "tables" $ do
144-
-- tables and views
145-
indexedForM_ tables $ \tableMeta -> do
146-
let tableName = tableMeta ^. tmTable
147-
isEnum = tableMeta ^. tmIsEnum
148-
config = tableMeta ^. tmConfiguration
149-
void $ Schema.trackExistingTableOrViewP2 tableName isEnum config
150-
151-
indexedForM_ tables $ \table -> do
141+
indexedForM_ tables $ \TableMeta{..} -> do
142+
-- Save table
143+
saveTableToCatalog _tmTable _tmIsEnum _tmConfiguration
144+
152145
-- Relationships
153146
withPathK "object_relationships" $
154-
indexedForM_ (table ^. tmObjectRelationships) $ \objRel ->
155-
Relationship.insertRelationshipToCatalog (table ^. tmTable) ObjRel objRel
147+
indexedForM_ _tmObjectRelationships $ \objRel ->
148+
Relationship.insertRelationshipToCatalog _tmTable ObjRel objRel
156149
withPathK "array_relationships" $
157-
indexedForM_ (table ^. tmArrayRelationships) $ \arrRel ->
158-
Relationship.insertRelationshipToCatalog (table ^. tmTable) ArrRel arrRel
150+
indexedForM_ _tmArrayRelationships $ \arrRel ->
151+
Relationship.insertRelationshipToCatalog _tmTable ArrRel arrRel
152+
159153
-- Computed Fields
160154
withPathK "computed_fields" $
161-
indexedForM_ (table ^. tmComputedFields) $
155+
indexedForM_ _tmComputedFields $
162156
\(ComputedFieldMeta name definition comment) ->
163157
ComputedField.addComputedFieldToCatalog $
164-
ComputedField.AddComputedField (table ^. tmTable) name definition comment
165-
166-
-- Permissions
167-
indexedForM_ tables $ \table -> do
168-
let tableName = table ^. tmTable
169-
tabInfo <- modifyErrAndSet500 ("apply " <> ) $ askTableCoreInfo tableName
170-
withPathK "insert_permissions" $ processPerms tabInfo $
171-
table ^. tmInsertPermissions
172-
withPathK "select_permissions" $ processPerms tabInfo $
173-
table ^. tmSelectPermissions
174-
withPathK "update_permissions" $ processPerms tabInfo $
175-
table ^. tmUpdatePermissions
176-
withPathK "delete_permissions" $ processPerms tabInfo $
177-
table ^. tmDeletePermissions
178-
179-
indexedForM_ tables $ \table ->
158+
ComputedField.AddComputedField _tmTable name definition comment
159+
160+
-- Permissions
161+
withPathK "insert_permissions" $ processPerms _tmTable _tmInsertPermissions
162+
withPathK "select_permissions" $ processPerms _tmTable _tmSelectPermissions
163+
withPathK "update_permissions" $ processPerms _tmTable _tmUpdatePermissions
164+
withPathK "delete_permissions" $ processPerms _tmTable _tmDeletePermissions
165+
166+
-- Event triggers
180167
withPathK "event_triggers" $
181-
indexedForM_ (table ^. tmEventTriggers) $ \etc ->
182-
subTableP2 (table ^. tmTable) False etc
168+
indexedForM_ _tmEventTriggers $ \etc -> subTableP2 _tmTable False etc
183169

184170
-- sql functions
185171
withPathK "functions" $ case functionsMeta of
186-
FMVersion1 qualifiedFunctions -> indexedForM_ qualifiedFunctions $
187-
\qf -> void $ Schema.trackFunctionP2 qf Schema.emptyFunctionConfig
188-
FMVersion2 functionsV2 -> indexedForM_ functionsV2 $
189-
\(Schema.TrackFunctionV2 function config) -> void $ Schema.trackFunctionP2 function config
172+
FMVersion1 qualifiedFunctions -> indexedForM_ qualifiedFunctions $
173+
\qf -> Schema.saveFunctionToCatalog qf Schema.emptyFunctionConfig
174+
FMVersion2 functionsV2 -> indexedForM_ functionsV2 $
175+
\(Schema.TrackFunctionV2 function config) -> Schema.saveFunctionToCatalog function config
190176

191177
-- query collections
192178
systemDefined <- askSystemDefined
@@ -200,32 +186,30 @@ applyQP2 (ReplaceMetadata _ tables functionsMeta
200186

201187
-- remote schemas
202188
withPathK "remote_schemas" $
203-
indexedMapM_ (void . addRemoteSchemaP2) schemas
189+
indexedMapM_ (liftTx . addRemoteSchemaToCatalog) schemas
204190

205-
CustomTypes.persistCustomTypes customTypes
206-
207-
for_ actions $ \action -> do
208-
let createAction =
209-
CreateAction (_amName action) (_amDefinition action) (_amComment action)
210-
Action.persistCreateAction createAction
211-
for_ (_amPermissions action) $ \permission -> do
212-
let createActionPermission = CreateActionPermission (_amName action)
213-
(_apmRole permission) Nothing (_apmComment permission)
214-
Action.persistCreateActionPermission createActionPermission
215-
216-
buildSchemaCacheStrict
217-
return successMsg
191+
-- custom types
192+
withPathK "custom_types" $
193+
CustomTypes.persistCustomTypes customTypes
218194

195+
-- actions
196+
withPathK "actions" $
197+
indexedForM_ actions $ \action -> do
198+
let createAction =
199+
CreateAction (_amName action) (_amDefinition action) (_amComment action)
200+
Action.persistCreateAction createAction
201+
withPathK "permissions" $
202+
indexedForM_ (_amPermissions action) $ \permission -> do
203+
let createActionPermission = CreateActionPermission (_amName action)
204+
(_apmRole permission) Nothing (_apmComment permission)
205+
Action.persistCreateActionPermission createActionPermission
219206
where
220-
processPerms tabInfo perms = indexedForM_ perms $ Permission.addPermP2 (_tciName tabInfo)
207+
processPerms tableName perms = indexedForM_ perms $ Permission.addPermP2 tableName
221208

222209
runReplaceMetadata
223-
:: ( HasVersion
224-
, MonadIO m
225-
, MonadTx m
210+
:: ( MonadTx m
226211
, CacheRWM m
227212
, HasSystemDefined m
228-
, HasHttpManager m
229213
)
230214
=> ReplaceMetadata -> m EncJSON
231215
runReplaceMetadata q = do

‎server/src-lib/Hasura/RQL/DDL/RemoteSchema.hs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ module Hasura.RQL.DDL.RemoteSchema
66
, fetchRemoteSchemas
77
, addRemoteSchemaP1
88
, addRemoteSchemaP2Setup
9-
, addRemoteSchemaP2
9+
, addRemoteSchemaToCatalog
1010
) where
1111

1212
import Hasura.EncJSON

‎server/src-lib/Hasura/RQL/DDL/Schema/Function.hs

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -157,9 +157,12 @@ mkFunctionInfo qf systemDefined config rawFuncInfo =
157157
let argsText = T.intercalate "," $ map getFuncArgNameTxt args
158158
in "the function arguments " <> argsText <> " are not in compliance with GraphQL spec"
159159

160-
saveFunctionToCatalog :: QualifiedFunction -> FunctionConfig -> SystemDefined -> Q.TxE QErr ()
161-
saveFunctionToCatalog (QualifiedObject sn fn) config systemDefined =
162-
Q.unitQE defaultTxErrorHandler [Q.sql|
160+
saveFunctionToCatalog
161+
:: (MonadTx m, HasSystemDefined m)
162+
=> QualifiedFunction -> FunctionConfig -> m ()
163+
saveFunctionToCatalog (QualifiedObject sn fn) config = do
164+
systemDefined <- askSystemDefined
165+
liftTx $ Q.unitQE defaultTxErrorHandler [Q.sql|
163166
INSERT INTO "hdb_catalog"."hdb_function"
164167
(function_schema, function_name, configuration, is_system_defined)
165168
VALUES ($1, $2, $3, $4)
@@ -205,8 +208,7 @@ trackFunctionP1 qf = do
205208
trackFunctionP2 :: (MonadTx m, CacheRWM m, HasSystemDefined m)
206209
=> QualifiedFunction -> FunctionConfig -> m EncJSON
207210
trackFunctionP2 qf config = do
208-
systemDefined <- askSystemDefined
209-
liftTx $ saveFunctionToCatalog qf config systemDefined
211+
saveFunctionToCatalog qf config
210212
buildSchemaCacheFor $ MOFunction qf
211213
return successMsg
212214

‎server/src-lib/Hasura/RQL/DDL/Schema/Table.hs

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
module Hasura.RQL.DDL.Schema.Table
55
( TrackTable(..)
66
, runTrackTableQ
7-
, trackExistingTableOrViewP2
87

98
, TrackTableV2(..)
109
, runTrackTableV2Q
@@ -38,6 +37,7 @@ import Hasura.SQL.Types
3837

3938
import qualified Database.PG.Query as Q
4039
import qualified Hasura.GraphQL.Schema as GS
40+
import qualified Hasura.GraphQL.Context as GC
4141
import qualified Hasura.Incremental as Inc
4242
import qualified Language.GraphQL.Draft.Syntax as G
4343

@@ -103,9 +103,8 @@ trackExistingTableOrViewP2
103103
:: (MonadTx m, CacheRWM m, HasSystemDefined m)
104104
=> QualifiedTable -> Bool -> TableConfig -> m EncJSON
105105
trackExistingTableOrViewP2 tableName isEnum config = do
106-
sc <- askSchemaCache
107-
let defGCtx = scDefaultRemoteGCtx sc
108-
GS.checkConflictingNode defGCtx $ GS.qualObjectToName tableName
106+
typeMap <- GC._gTypes . scDefaultRemoteGCtx <$> askSchemaCache
107+
GS.checkConflictingNode typeMap $ GS.qualObjectToName tableName
109108
saveTableToCatalog tableName isEnum config
110109
buildSchemaCacheFor (MOTable tableName)
111110
return successMsg
@@ -212,9 +211,9 @@ processTableChanges ti tableDiff = do
212211

213212
withNewTabName newTN = do
214213
let tnGQL = GS.qualObjectToName newTN
215-
defGCtx = scDefaultRemoteGCtx sc
214+
typeMap = GC._gTypes $ scDefaultRemoteGCtx sc
216215
-- check for GraphQL schema conflicts on new name
217-
GS.checkConflictingNode defGCtx tnGQL
216+
GS.checkConflictingNode typeMap tnGQL
218217
procAlteredCols sc tn
219218
-- update new table in catalog
220219
renameTableInCatalog newTN tn

0 commit comments

Comments
 (0)