From 6646d0488e95a6f964464198b478f08c2a0af5c9 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 31 Oct 2024 10:53:04 -0400 Subject: [PATCH 1/6] chore: add PGLocks query to analyze what locks are held in pg --- coderd/database/db.go | 103 ++++++++++++++++++++++ coderd/database/dbauthz/dbauthz.go | 3 + coderd/database/dbmem/dbmem.go | 3 + coderd/database/dbmetrics/querymetrics.go | 7 ++ 4 files changed, 116 insertions(+) diff --git a/coderd/database/db.go b/coderd/database/db.go index 75a28066ce905..37456708266e9 100644 --- a/coderd/database/db.go +++ b/coderd/database/db.go @@ -12,6 +12,9 @@ import ( "context" "database/sql" "errors" + "fmt" + "sort" + "strings" "time" "github.com/jmoiron/sqlx" @@ -28,6 +31,7 @@ type Store interface { wrapper Ping(ctx context.Context) (time.Duration, error) + PGLocks(ctx context.Context) (PGLocks, error) InTx(func(Store) error, *TxOptions) error } @@ -134,6 +138,98 @@ func (q *sqlQuerier) Ping(ctx context.Context) (time.Duration, error) { return time.Since(start), err } +type PGLocks []PGLock + +func (l PGLocks) String() string { + // Try to group things together by relation name. + sort.Slice(l, func(i, j int) bool { + return safeString(l[i].RelationName) < safeString(l[j].RelationName) + }) + + var out strings.Builder + for i, lock := range l { + if i != 0 { + out.WriteString("\n") + } + out.WriteString(lock.String()) + } + return out.String() +} + +// PGLock docs see: https://www.postgresql.org/docs/current/view-pg-locks.html#VIEW-PG-LOCKS +type PGLock struct { + // LockType see: https://www.postgresql.org/docs/current/monitoring-stats.html#WAIT-EVENT-LOCK-TABLE + LockType *string `db:"locktype"` + Database *string `db:"database"` // oid + Relation *string `db:"relation"` // oid + RelationName *string `db:"relation_name"` + Page *int `db:"page"` + Tuple *int `db:"tuple"` + VirtualXID *string `db:"virtualxid"` + TransactionID *string `db:"transactionid"` // xid + ClassID *string `db:"classid"` // oid + ObjID *string `db:"objid"` // oid + ObjSubID *int `db:"objsubid"` + VirtualTransaction *string `db:"virtualtransaction"` + PID int `db:"pid"` + Mode *string `db:"mode"` + Granted bool `db:"granted"` + FastPath *bool `db:"fastpath"` + WaitStart *time.Time `db:"waitstart"` +} + +func (l PGLock) String() string { + granted := "granted" + if !l.Granted { + granted = "waiting" + } + var details string + switch safeString(l.LockType) { + case "relation": + details = "" + case "page": + details = fmt.Sprintf("page=%d", *l.Page) + case "tuple": + details = fmt.Sprintf("page=%d tuple=%d", *l.Page, *l.Tuple) + case "virtualxid": + details = "waiting to acquire virtual tx id lock" + default: + details = "???" + } + return fmt.Sprintf("%d-%5s [%s] %s/%s/%s: %s", + l.PID, + safeString(l.TransactionID), + granted, + safeString(l.RelationName), + safeString(l.LockType), + safeString(l.Mode), + details, + ) +} + +// PGLocks returns a list of all locks in the database currently in use. +func (q *sqlQuerier) PGLocks(ctx context.Context) (PGLocks, error) { + rows, err := q.sdb.QueryContext(ctx, ` + SELECT + relation::regclass AS relation_name, + * + FROM pg_locks; + `) + if err != nil { + return nil, err + } + + defer rows.Close() + + var locks []PGLock + err = sqlx.StructScan(rows, &locks) + if err != nil { + return nil, err + } + + return locks, err +} + func DefaultTXOptions() *TxOptions { return &TxOptions{ Isolation: sql.LevelDefault, @@ -218,3 +314,10 @@ func (q *sqlQuerier) runTx(function func(Store) error, txOpts *sql.TxOptions) er } return nil } + +func safeString(s *string) string { + if s == nil { + return "" + } + return *s +} diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index ae6b307b3e7d3..b81a93559e70c 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -602,6 +602,9 @@ func prepareSQLFilter(ctx context.Context, authorizer rbac.Authorizer, action po func (q *querier) Ping(ctx context.Context) (time.Duration, error) { return q.db.Ping(ctx) } +func (q *querier) PGLocks(ctx context.Context) (database.PGLocks, error) { + return q.db.PGLocks(ctx) +} // InTx runs the given function in a transaction. func (q *querier) InTx(function func(querier database.Store) error, txOpts *database.TxOptions) error { diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 4f54598744dd0..ebe051771a107 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -338,6 +338,9 @@ func newUniqueConstraintError(uc database.UniqueConstraint) *pq.Error { func (*FakeQuerier) Ping(_ context.Context) (time.Duration, error) { return 0, nil } +func (*FakeQuerier) PGLocks(ctx context.Context) (database.PGLocks, error) { + return []database.PGLock{}, nil +} func (tx *fakeTx) AcquireLock(_ context.Context, id int64) error { if _, ok := tx.FakeQuerier.locks[id]; ok { diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 7e74aab3b9de0..e1cfec5bac9ca 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -66,6 +66,13 @@ func (m queryMetricsStore) Ping(ctx context.Context) (time.Duration, error) { return duration, err } +func (m queryMetricsStore) PGLocks(ctx context.Context) (database.PGLocks, error) { + start := time.Now() + locks, err := m.s.PGLocks(ctx) + m.queryLatencies.WithLabelValues("PGLocks").Observe(time.Since(start).Seconds()) + return locks, err +} + func (m queryMetricsStore) InTx(f func(database.Store) error, options *database.TxOptions) error { return m.dbMetrics.InTx(f, options) } From 2f8cf7a0d0bc097f8c1d9c441b2908fc20a6a7de Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 31 Oct 2024 11:04:43 -0400 Subject: [PATCH 2/6] move to pglocks.go file --- coderd/database/db.go | 95 ----------------------------- coderd/database/pglocks.go | 119 +++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 95 deletions(-) create mode 100644 coderd/database/pglocks.go diff --git a/coderd/database/db.go b/coderd/database/db.go index 37456708266e9..0f923a861efb4 100644 --- a/coderd/database/db.go +++ b/coderd/database/db.go @@ -12,9 +12,6 @@ import ( "context" "database/sql" "errors" - "fmt" - "sort" - "strings" "time" "github.com/jmoiron/sqlx" @@ -138,98 +135,6 @@ func (q *sqlQuerier) Ping(ctx context.Context) (time.Duration, error) { return time.Since(start), err } -type PGLocks []PGLock - -func (l PGLocks) String() string { - // Try to group things together by relation name. - sort.Slice(l, func(i, j int) bool { - return safeString(l[i].RelationName) < safeString(l[j].RelationName) - }) - - var out strings.Builder - for i, lock := range l { - if i != 0 { - out.WriteString("\n") - } - out.WriteString(lock.String()) - } - return out.String() -} - -// PGLock docs see: https://www.postgresql.org/docs/current/view-pg-locks.html#VIEW-PG-LOCKS -type PGLock struct { - // LockType see: https://www.postgresql.org/docs/current/monitoring-stats.html#WAIT-EVENT-LOCK-TABLE - LockType *string `db:"locktype"` - Database *string `db:"database"` // oid - Relation *string `db:"relation"` // oid - RelationName *string `db:"relation_name"` - Page *int `db:"page"` - Tuple *int `db:"tuple"` - VirtualXID *string `db:"virtualxid"` - TransactionID *string `db:"transactionid"` // xid - ClassID *string `db:"classid"` // oid - ObjID *string `db:"objid"` // oid - ObjSubID *int `db:"objsubid"` - VirtualTransaction *string `db:"virtualtransaction"` - PID int `db:"pid"` - Mode *string `db:"mode"` - Granted bool `db:"granted"` - FastPath *bool `db:"fastpath"` - WaitStart *time.Time `db:"waitstart"` -} - -func (l PGLock) String() string { - granted := "granted" - if !l.Granted { - granted = "waiting" - } - var details string - switch safeString(l.LockType) { - case "relation": - details = "" - case "page": - details = fmt.Sprintf("page=%d", *l.Page) - case "tuple": - details = fmt.Sprintf("page=%d tuple=%d", *l.Page, *l.Tuple) - case "virtualxid": - details = "waiting to acquire virtual tx id lock" - default: - details = "???" - } - return fmt.Sprintf("%d-%5s [%s] %s/%s/%s: %s", - l.PID, - safeString(l.TransactionID), - granted, - safeString(l.RelationName), - safeString(l.LockType), - safeString(l.Mode), - details, - ) -} - -// PGLocks returns a list of all locks in the database currently in use. -func (q *sqlQuerier) PGLocks(ctx context.Context) (PGLocks, error) { - rows, err := q.sdb.QueryContext(ctx, ` - SELECT - relation::regclass AS relation_name, - * - FROM pg_locks; - `) - if err != nil { - return nil, err - } - - defer rows.Close() - - var locks []PGLock - err = sqlx.StructScan(rows, &locks) - if err != nil { - return nil, err - } - - return locks, err -} - func DefaultTXOptions() *TxOptions { return &TxOptions{ Isolation: sql.LevelDefault, diff --git a/coderd/database/pglocks.go b/coderd/database/pglocks.go new file mode 100644 index 0000000000000..667715dd6c588 --- /dev/null +++ b/coderd/database/pglocks.go @@ -0,0 +1,119 @@ +package database + +import ( + "context" + "fmt" + "reflect" + "sort" + "strings" + "time" + + "github.com/jmoiron/sqlx" + + "github.com/coder/coder/v2/coderd/util/slice" +) + +// PGLock docs see: https://www.postgresql.org/docs/current/view-pg-locks.html#VIEW-PG-LOCKS +type PGLock struct { + // LockType see: https://www.postgresql.org/docs/current/monitoring-stats.html#WAIT-EVENT-LOCK-TABLE + LockType *string `db:"locktype"` + Database *string `db:"database"` // oid + Relation *string `db:"relation"` // oid + RelationName *string `db:"relation_name"` + Page *int `db:"page"` + Tuple *int `db:"tuple"` + VirtualXID *string `db:"virtualxid"` + TransactionID *string `db:"transactionid"` // xid + ClassID *string `db:"classid"` // oid + ObjID *string `db:"objid"` // oid + ObjSubID *int `db:"objsubid"` + VirtualTransaction *string `db:"virtualtransaction"` + PID int `db:"pid"` + Mode *string `db:"mode"` + Granted bool `db:"granted"` + FastPath *bool `db:"fastpath"` + WaitStart *time.Time `db:"waitstart"` +} + +func (l PGLock) Equal(b PGLock) bool { + // Lazy, but hope this works + return reflect.DeepEqual(l, b) +} + +func (l PGLock) String() string { + granted := "granted" + if !l.Granted { + granted = "waiting" + } + var details string + switch safeString(l.LockType) { + case "relation": + details = "" + case "page": + details = fmt.Sprintf("page=%d", *l.Page) + case "tuple": + details = fmt.Sprintf("page=%d tuple=%d", *l.Page, *l.Tuple) + case "virtualxid": + details = "waiting to acquire virtual tx id lock" + default: + details = "???" + } + return fmt.Sprintf("%d-%5s [%s] %s/%s/%s: %s", + l.PID, + safeString(l.TransactionID), + granted, + safeString(l.RelationName), + safeString(l.LockType), + safeString(l.Mode), + details, + ) +} + +// PGLocks returns a list of all locks in the database currently in use. +func (q *sqlQuerier) PGLocks(ctx context.Context) (PGLocks, error) { + rows, err := q.sdb.QueryContext(ctx, ` + SELECT + relation::regclass AS relation_name, + * + FROM pg_locks; + `) + if err != nil { + return nil, err + } + + defer rows.Close() + + var locks []PGLock + err = sqlx.StructScan(rows, &locks) + if err != nil { + return nil, err + } + + return locks, err +} + +type PGLocks []PGLock + +func (l PGLocks) String() string { + // Try to group things together by relation name. + sort.Slice(l, func(i, j int) bool { + return safeString(l[i].RelationName) < safeString(l[j].RelationName) + }) + + var out strings.Builder + for i, lock := range l { + if i != 0 { + out.WriteString("\n") + } + out.WriteString(lock.String()) + } + return out.String() +} + +// Difference returns the difference between two sets of locks. +// This is helpful to determine what changed between the two sets. +func (l PGLocks) Difference(to PGLocks) (new PGLocks, removed PGLocks) { + return slice.SymmetricDifferenceFunc(l, to, func(a, b PGLock) bool { + return a.Equal(b) + }) +} From 6e0c67a4693327a267ededa828dff555add3a9d8 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 31 Oct 2024 11:08:41 -0400 Subject: [PATCH 3/6] fmt --- coderd/database/dbmem/dbmem.go | 1 + 1 file changed, 1 insertion(+) diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index ebe051771a107..f8e1ccd5834de 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -338,6 +338,7 @@ func newUniqueConstraintError(uc database.UniqueConstraint) *pq.Error { func (*FakeQuerier) Ping(_ context.Context) (time.Duration, error) { return 0, nil } + func (*FakeQuerier) PGLocks(ctx context.Context) (database.PGLocks, error) { return []database.PGLock{}, nil } From ed52eb5f9e7245f838e18d78a42767ef200bdb22 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 31 Oct 2024 11:30:58 -0400 Subject: [PATCH 4/6] make gen --- coderd/database/dbauthz/dbauthz.go | 1 + coderd/database/dbmock/dbmock.go | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index b81a93559e70c..9bf98aade03c4 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -602,6 +602,7 @@ func prepareSQLFilter(ctx context.Context, authorizer rbac.Authorizer, action po func (q *querier) Ping(ctx context.Context) (time.Duration, error) { return q.db.Ping(ctx) } + func (q *querier) PGLocks(ctx context.Context) (database.PGLocks, error) { return q.db.PGLocks(ctx) } diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index ffc9ab79f777e..27b398a062051 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -4299,6 +4299,21 @@ func (mr *MockStoreMockRecorder) OrganizationMembers(arg0, arg1 any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OrganizationMembers", reflect.TypeOf((*MockStore)(nil).OrganizationMembers), arg0, arg1) } +// PGLocks mocks base method. +func (m *MockStore) PGLocks(arg0 context.Context) (database.PGLocks, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PGLocks", arg0) + ret0, _ := ret[0].(database.PGLocks) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PGLocks indicates an expected call of PGLocks. +func (mr *MockStoreMockRecorder) PGLocks(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PGLocks", reflect.TypeOf((*MockStore)(nil).PGLocks), arg0) +} + // Ping mocks base method. func (m *MockStore) Ping(arg0 context.Context) (time.Duration, error) { m.ctrl.T.Helper() From 0ecdab5ff47b1d36bd27acfea6f1996588b4a7a4 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 31 Oct 2024 11:56:08 -0400 Subject: [PATCH 5/6] linting --- coderd/database/dbauthz/dbauthz_test.go | 5 ++++- coderd/database/dbmem/dbmem.go | 2 +- coderd/database/pglocks.go | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 439cf1bdaec19..ae50309e96d66 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -152,7 +152,10 @@ func TestDBAuthzRecursive(t *testing.T) { for i := 2; i < method.Type.NumIn(); i++ { ins = append(ins, reflect.New(method.Type.In(i)).Elem()) } - if method.Name == "InTx" || method.Name == "Ping" || method.Name == "Wrappers" { + if method.Name == "InTx" || + method.Name == "Ping" || + method.Name == "Wrappers" || + method.Name == "PGLocks" { continue } // Log the name of the last method, so if there is a panic, it is diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index f8e1ccd5834de..e38c3e107013f 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -339,7 +339,7 @@ func (*FakeQuerier) Ping(_ context.Context) (time.Duration, error) { return 0, nil } -func (*FakeQuerier) PGLocks(ctx context.Context) (database.PGLocks, error) { +func (*FakeQuerier) PGLocks(_ context.Context) (database.PGLocks, error) { return []database.PGLock{}, nil } diff --git a/coderd/database/pglocks.go b/coderd/database/pglocks.go index 667715dd6c588..85e1644b3825c 100644 --- a/coderd/database/pglocks.go +++ b/coderd/database/pglocks.go @@ -103,9 +103,9 @@ func (l PGLocks) String() string { var out strings.Builder for i, lock := range l { if i != 0 { - out.WriteString("\n") + _, _ = out.WriteString("\n") } - out.WriteString(lock.String()) + _, _ = out.WriteString(lock.String()) } return out.String() } From d615075fc9d29f37460b6d7393b10980654585cc Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Thu, 31 Oct 2024 12:08:41 -0400 Subject: [PATCH 6/6] ignore PGLocks --- coderd/database/dbauthz/setup_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/coderd/database/dbauthz/setup_test.go b/coderd/database/dbauthz/setup_test.go index df9d551101a25..52e8dd42fea9c 100644 --- a/coderd/database/dbauthz/setup_test.go +++ b/coderd/database/dbauthz/setup_test.go @@ -34,6 +34,7 @@ var errMatchAny = xerrors.New("match any error") var skipMethods = map[string]string{ "InTx": "Not relevant", "Ping": "Not relevant", + "PGLocks": "Not relevant", "Wrappers": "Not relevant", "AcquireLock": "Not relevant", "TryAcquireLock": "Not relevant",