From 3e0645cdeb77cf91e46f62af69a6d91e27b8a089 Mon Sep 17 00:00:00 2001 From: Jakub Domeracki Date: Wed, 16 Jul 2025 14:15:48 +0200 Subject: [PATCH 01/10] chore: publish CLI binaries and detached signatures to releases.coder.com (#18901) Cherry pick (https://github.com/coder/coder/commit/e4d3453e2b55edfc5a9650083f4bffc765423b1c) Starting with version 2.24.X , Coder CLI binaries & corresponding detached signatures will get published to the GCS bucket releases.coder.com. --- .github/workflows/release.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 71567cd540502..1d5ebb5aa23b8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -634,6 +634,29 @@ jobs: - name: ls build run: ls -lh build + - name: Publish Coder CLI binaries and detached signatures to GCS + if: ${{ !inputs.dry_run && github.ref == 'refs/heads/main' && github.repository_owner == 'coder'}} + run: | + set -euxo pipefail + + version="$(./scripts/version.sh)" + + binaries=( + "coder-darwin-amd64" + "coder-darwin-arm64" + "coder-linux-amd64" + "coder-linux-arm64" + "coder-linux-armv7" + "coder-windows-amd64.exe" + "coder-windows-arm64.exe" + ) + + for binary in "${binaries[@]}"; do + detached_signature="${binary}.asc" + gcloud storage cp "./site/out/bin/${binary}" "gs://releases.coder.com/coder-cli/${version}/${binary}" + gcloud storage cp "./site/out/bin/${detached_signature}" "gs://releases.coder.com/coder-cli/${version}/${detached_signature}" + done + - name: Publish release run: | set -euo pipefail From d6b2ca1c3b984880d31c580a235cd8840231f933 Mon Sep 17 00:00:00 2001 From: "gcp-cherry-pick-bot[bot]" <98988430+gcp-cherry-pick-bot[bot]@users.noreply.github.com> Date: Tue, 5 Aug 2025 12:17:06 +0500 Subject: [PATCH 02/10] chore: add openai icon (cherry-pick #19118) (#19175) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: ケイラ Co-authored-by: 35C4n0r <70096901+35C4n0r@users.noreply.github.com> --- site/src/theme/externalImages.ts | 1 + site/src/theme/icons.json | 1 + site/static/icon/openai.svg | 2 ++ 3 files changed, 4 insertions(+) create mode 100644 site/static/icon/openai.svg diff --git a/site/src/theme/externalImages.ts b/site/src/theme/externalImages.ts index f736e91e7b745..15713559036d0 100644 --- a/site/src/theme/externalImages.ts +++ b/site/src/theme/externalImages.ts @@ -156,6 +156,7 @@ export const defaultParametersForBuiltinIcons = new Map([ ["/icon/kasmvnc.svg", "whiteWithColor"], ["/icon/kiro.svg", "whiteWithColor"], ["/icon/memory.svg", "monochrome"], + ["/icon/openai.svg", "monochrome"], ["/icon/rust.svg", "monochrome"], ["/icon/terminal.svg", "monochrome"], ["/icon/widgets.svg", "monochrome"], diff --git a/site/src/theme/icons.json b/site/src/theme/icons.json index d687b1f5991fc..6a8fd0c7a6222 100644 --- a/site/src/theme/icons.json +++ b/site/src/theme/icons.json @@ -83,6 +83,7 @@ "nomad.svg", "novnc.svg", "okta.svg", + "openai.svg", "personalize.svg", "php.svg", "phpstorm.svg", diff --git a/site/static/icon/openai.svg b/site/static/icon/openai.svg new file mode 100644 index 0000000000000..3b4eff961f37e --- /dev/null +++ b/site/static/icon/openai.svg @@ -0,0 +1,2 @@ + +OpenAI icon \ No newline at end of file From bc502b52f5a2d7a371d038bfaebe11bcfae978a9 Mon Sep 17 00:00:00 2001 From: Ethan <39577870+ethanndickson@users.noreply.github.com> Date: Wed, 6 Aug 2025 13:11:45 +1000 Subject: [PATCH 03/10] chore: cherry-pick coder desktop + corporate vpn fixes for 2.24 (#19177) This backports the coder/coder fixes for https://github.com/coder/coder-desktop-windows/issues/147 and https://github.com/coder/coder-desktop-macos/issues/201 to v2.24, and fixes a bug where Coder Desktop logs were duplicated on Windows. 1. https://github.com/coder/coder/pull/19124 (`c64b3c0d8327edbfad4cc7bc373bdb873cd7e26b`) - Required for CI to pass. 2. https://github.com/coder/coder/pull/19125 (`9f2dc72d9a3d9feecf28bc30cd1234394a0e5c37`) - Required for CI to pass. 3. https://github.com/coder/coder/pull/19143 (`914daac2dcf700817d4ae248a6e8014f64654fe7`) - Required for CI to pass. 4. https://github.com/coder/coder/pull/19023 (`618121cba39833db7c1ca4653d7ce7745d5aff57`) - First implementation of fix for https://github.com/coder/coder-desktop-windows/issues/147 5. https://github.com/coder/coder/pull/19069 (`ba0b124a1daaddde590f5839afebe803330981be`) - Partially reverts previous commit, actual fix for https://github.com/coder/coder-desktop-windows/issues/147 6. https://github.com/coder/coder/pull/19052 (`19171e27c9624c435f97d4ae5e3b61c649bc7063`) - Avoid duplicating logs on Coder Desktop Windows 7. https://github.com/coder/coder/pull/19080 (`18b037d0086cd6eca11e6180b36a302badaa5a5f`) - Updates coder/tailscale reference, lets Coder Desktop macOS use the slim binary to run Coder Connect. I've tested my latest build of Coder Desktop against a Coder server running this branch, just as a sanity check. --------- Co-authored-by: Cian Johnston Co-authored-by: Dean Sheather --- .github/workflows/ci.yaml | 7 +++- .github/workflows/release.yaml | 4 +- cli/vpndaemon_darwin.go | 73 ++++++++++++++++++++++++++++++++++ cli/vpndaemon_other.go | 2 +- cli/vpndaemon_windows.go | 5 +-- go.mod | 2 +- go.sum | 4 +- tailnet/conn.go | 14 ++++++- vpn/tunnel.go | 6 --- 9 files changed, 98 insertions(+), 19 deletions(-) create mode 100644 cli/vpndaemon_darwin.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7a0bdbd435f98..8e442e4a539ca 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -428,6 +428,11 @@ jobs: - name: Disable Spotlight Indexing if: runner.os == 'macOS' run: | + enabled=$(sudo mdutil -a -s | grep "Indexing enabled" | wc -l) + if [ $enabled -eq 0 ]; then + echo "Spotlight indexing is already disabled" + exit 0 + fi sudo mdutil -a -i off sudo mdutil -X / sudo launchctl bootout system /System/Library/LaunchDaemons/com.apple.metadata.mds.plist @@ -1082,7 +1087,7 @@ jobs: - name: Switch XCode Version uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0 with: - xcode-version: "16.0.0" + xcode-version: "16.1.0" - name: Setup Go uses: ./.github/actions/setup-go diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1d5ebb5aa23b8..d032cd3e8731c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -60,7 +60,7 @@ jobs: - name: Switch XCode Version uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0 with: - xcode-version: "16.0.0" + xcode-version: "16.1.0" - name: Setup Go uses: ./.github/actions/setup-go @@ -655,7 +655,7 @@ jobs: detached_signature="${binary}.asc" gcloud storage cp "./site/out/bin/${binary}" "gs://releases.coder.com/coder-cli/${version}/${binary}" gcloud storage cp "./site/out/bin/${detached_signature}" "gs://releases.coder.com/coder-cli/${version}/${detached_signature}" - done + done - name: Publish release run: | diff --git a/cli/vpndaemon_darwin.go b/cli/vpndaemon_darwin.go new file mode 100644 index 0000000000000..a1b836dd6b0c3 --- /dev/null +++ b/cli/vpndaemon_darwin.go @@ -0,0 +1,73 @@ +//go:build darwin + +package cli + +import ( + "golang.org/x/xerrors" + + "cdr.dev/slog" + "github.com/coder/coder/v2/vpn" + "github.com/coder/serpent" +) + +func (r *RootCmd) vpnDaemonRun() *serpent.Command { + var ( + rpcReadFD int64 + rpcWriteFD int64 + ) + + cmd := &serpent.Command{ + Use: "run", + Short: "Run the VPN daemon on macOS.", + Middleware: serpent.Chain( + serpent.RequireNArgs(0), + ), + Options: serpent.OptionSet{ + { + Flag: "rpc-read-fd", + Env: "CODER_VPN_DAEMON_RPC_READ_FD", + Description: "The file descriptor for the pipe to read from the RPC connection.", + Value: serpent.Int64Of(&rpcReadFD), + Required: true, + }, + { + Flag: "rpc-write-fd", + Env: "CODER_VPN_DAEMON_RPC_WRITE_FD", + Description: "The file descriptor for the pipe to write to the RPC connection.", + Value: serpent.Int64Of(&rpcWriteFD), + Required: true, + }, + }, + Handler: func(inv *serpent.Invocation) error { + ctx := inv.Context() + + if rpcReadFD < 0 || rpcWriteFD < 0 { + return xerrors.Errorf("rpc-read-fd (%v) and rpc-write-fd (%v) must be positive", rpcReadFD, rpcWriteFD) + } + if rpcReadFD == rpcWriteFD { + return xerrors.Errorf("rpc-read-fd (%v) and rpc-write-fd (%v) must be different", rpcReadFD, rpcWriteFD) + } + + pipe, err := vpn.NewBidirectionalPipe(uintptr(rpcReadFD), uintptr(rpcWriteFD)) + if err != nil { + return xerrors.Errorf("create bidirectional RPC pipe: %w", err) + } + defer pipe.Close() + + tunnel, err := vpn.NewTunnel(ctx, slog.Make().Leveled(slog.LevelDebug), pipe, + vpn.NewClient(), + vpn.UseOSNetworkingStack(), + vpn.UseAsLogger(), + ) + if err != nil { + return xerrors.Errorf("create new tunnel for client: %w", err) + } + defer tunnel.Close() + + <-ctx.Done() + return nil + }, + } + + return cmd +} diff --git a/cli/vpndaemon_other.go b/cli/vpndaemon_other.go index 2e3e39b1b99ba..1526efb011889 100644 --- a/cli/vpndaemon_other.go +++ b/cli/vpndaemon_other.go @@ -1,4 +1,4 @@ -//go:build !windows +//go:build !windows && !darwin package cli diff --git a/cli/vpndaemon_windows.go b/cli/vpndaemon_windows.go index cf74558ffa1ab..6c2d147da25ff 100644 --- a/cli/vpndaemon_windows.go +++ b/cli/vpndaemon_windows.go @@ -63,10 +63,7 @@ func (r *RootCmd) vpnDaemonRun() *serpent.Command { defer pipe.Close() logger.Info(ctx, "starting tunnel") - tunnel, err := vpn.NewTunnel(ctx, logger, pipe, vpn.NewClient(), - vpn.UseOSNetworkingStack(), - vpn.UseCustomLogSinks(sinks...), - ) + tunnel, err := vpn.NewTunnel(ctx, logger, pipe, vpn.NewClient(), vpn.UseOSNetworkingStack()) if err != nil { return xerrors.Errorf("create new tunnel for client: %w", err) } diff --git a/go.mod b/go.mod index ad9555590006d..b4404754589bb 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ replace github.com/tcnksm/go-httpstat => github.com/coder/go-httpstat v0.0.0-202 // There are a few minor changes we make to Tailscale that we're slowly upstreaming. Compare here: // https://github.com/tailscale/tailscale/compare/main...coder:tailscale:main -replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20250611020837-f14d20d23d8c +replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20250729141742-067f1e5d9716 // This is replaced to include // 1. a fix for a data race: c.f. https://github.com/tailscale/wireguard-go/pull/25 diff --git a/go.sum b/go.sum index 45070b45fd564..8496867edf78f 100644 --- a/go.sum +++ b/go.sum @@ -926,8 +926,8 @@ github.com/coder/serpent v0.10.0 h1:ofVk9FJXSek+SmL3yVE3GoArP83M+1tX+H7S4t8BSuM= github.com/coder/serpent v0.10.0/go.mod h1:cZFW6/fP+kE9nd/oRkEHJpG6sXCtQ+AX7WMMEHv0Y3Q= github.com/coder/ssh v0.0.0-20231128192721-70855dedb788 h1:YoUSJ19E8AtuUFVYBpXuOD6a/zVP3rcxezNsoDseTUw= github.com/coder/ssh v0.0.0-20231128192721-70855dedb788/go.mod h1:aGQbuCLyhRLMzZF067xc84Lh7JDs1FKwCmF1Crl9dxQ= -github.com/coder/tailscale v1.1.1-0.20250611020837-f14d20d23d8c h1:d/qBIi3Ez7KkopRgNtfdvTMqvqBg47d36qVfkd3C5EQ= -github.com/coder/tailscale v1.1.1-0.20250611020837-f14d20d23d8c/go.mod h1:l7ml5uu7lFh5hY28lGYM4b/oFSmuPHYX6uk4RAu23Lc= +github.com/coder/tailscale v1.1.1-0.20250729141742-067f1e5d9716 h1:hi7o0sA+RPBq8Rvvz+hNrC/OTL2897OKREMIRIuQeTs= +github.com/coder/tailscale v1.1.1-0.20250729141742-067f1e5d9716/go.mod h1:l7ml5uu7lFh5hY28lGYM4b/oFSmuPHYX6uk4RAu23Lc= github.com/coder/terraform-config-inspect v0.0.0-20250107175719-6d06d90c630e h1:JNLPDi2P73laR1oAclY6jWzAbucf70ASAvf5mh2cME0= github.com/coder/terraform-config-inspect v0.0.0-20250107175719-6d06d90c630e/go.mod h1:Gz/z9Hbn+4KSp8A2FBtNszfLSdT2Tn/uAKGuVqqWmDI= github.com/coder/terraform-provider-coder/v2 v2.7.1-0.20250623193313-e890833351e2 h1:vtGzECz5CyzuxMODexWdIRxhYLqyTcHafuJpH60PYhM= diff --git a/tailnet/conn.go b/tailnet/conn.go index c3ebd246c539f..709d5b2958453 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -65,7 +65,9 @@ const EnvMagicsockDebugLogging = "CODER_MAGICSOCK_DEBUG_LOGGING" func init() { // Globally disable network namespacing. All networking happens in - // userspace. + // userspace unless the connection is configured to use a TUN. + // NOTE: this exists in init() so it affects all connections (incl. DERP) + // made by tailscale packages by default. netns.SetEnabled(false) // Tailscale, by default, "trims" the set of peers down to ones that we are // "actively" communicating with in an effort to save memory. Since @@ -100,6 +102,7 @@ type Options struct { BlockEndpoints bool Logger slog.Logger ListenPort uint16 + // CaptureHook is a callback that captures Disco packets and packets sent // into the tailnet tunnel. CaptureHook capture.Callback @@ -154,7 +157,14 @@ func NewConn(options *Options) (conn *Conn, err error) { return nil, xerrors.New("At least one IP range must be provided") } - netns.SetEnabled(options.TUNDev != nil) + useNetNS := options.TUNDev != nil + options.Logger.Debug(context.Background(), "network isolation configuration", slog.F("use_netns", useNetNS)) + netns.SetEnabled(useNetNS) + // The Coder soft isolation mode is a workaround to allow Coder Connect to + // connect to Coder servers behind corporate VPNs, and relaxes some of the + // loop protections that come with Tailscale. + // See the comment above the netns function for more details. + netns.SetCoderSoftIsolation(useNetNS) var telemetryStore *TelemetryStore if options.TelemetrySink != nil { diff --git a/vpn/tunnel.go b/vpn/tunnel.go index e4624ac1822b0..30ee56c2396fa 100644 --- a/vpn/tunnel.go +++ b/vpn/tunnel.go @@ -192,12 +192,6 @@ func UseAsLogger() TunnelOption { } } -func UseCustomLogSinks(sinks ...slog.Sink) TunnelOption { - return func(t *Tunnel) { - t.clientLogger = t.clientLogger.AppendSinks(sinks...) - } -} - func WithClock(clock quartz.Clock) TunnelOption { return func(t *Tunnel) { t.clock = clock From 9df4992076bdb10dfd2a09f8b6ed2fc2c9373f9d Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 7 Aug 2025 13:00:59 +0200 Subject: [PATCH 04/10] fix: pin Nix version to 2.28.4 to avoid JSON type error (#19223) Pin Nix version to 2.28.4 in dogfood workflow Pins the Nix version in the dogfood workflow to 2.28.4 to avoid a JSON type error that occurs with Nix 2.29 and above. Change-Id: Ie024d5070dbe5901952fc52463c6602363ef8886 Signed-off-by: Thomas Kosiewski [tk@coder.com](mailto:tk@coder.com) Signed-off-by: Thomas Kosiewski --- .github/workflows/dogfood.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dogfood.yaml b/.github/workflows/dogfood.yaml index b5b2447bd44bf..952e31b7c98ec 100644 --- a/.github/workflows/dogfood.yaml +++ b/.github/workflows/dogfood.yaml @@ -35,7 +35,11 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup Nix - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31 + uses: nixbuild/nix-quick-install-action@63ca48f939ee3b8d835f4126562537df0fee5b91 # v32 + with: + # Pinning to 2.28 here, as Nix gets a "error: [json.exception.type_error.302] type must be array, but is string" + # on version 2.29 and above. + nix_version: "2.28.4" - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3 with: From 7f6cefdb56e8283f6b524f015d4562edb83b4391 Mon Sep 17 00:00:00 2001 From: Spike Curtis Date: Thu, 7 Aug 2025 16:11:49 +0400 Subject: [PATCH 05/10] fix: upgrade to 1.24.6 to fix race in lib/pq queries (#19214) (#19219) THIS IS A SECURITY FIX - cherry-picks #19214 upgrade to go 1.24.6 to avoid https://github.com/golang/go/issues/74831 (CVE-2025-47907) Also points to a new version of our lib/pq fork that worked around the Go issue, which should restore better performance. --- .github/actions/setup-go/action.yaml | 2 +- dogfood/coder/Dockerfile | 2 +- go.mod | 4 ++-- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup-go/action.yaml b/.github/actions/setup-go/action.yaml index a8a88621dda18..097a1b6cfd119 100644 --- a/.github/actions/setup-go/action.yaml +++ b/.github/actions/setup-go/action.yaml @@ -4,7 +4,7 @@ description: | inputs: version: description: "The Go version to use." - default: "1.24.4" + default: "1.24.6" use-preinstalled-go: description: "Whether to use preinstalled Go." default: "false" diff --git a/dogfood/coder/Dockerfile b/dogfood/coder/Dockerfile index dbafcd7add427..95559758bbd63 100644 --- a/dogfood/coder/Dockerfile +++ b/dogfood/coder/Dockerfile @@ -11,7 +11,7 @@ RUN cargo install jj-cli typos-cli watchexec-cli FROM ubuntu:jammy@sha256:0e5e4a57c2499249aafc3b40fcd541e9a456aab7296681a3994d631587203f97 AS go # Install Go manually, so that we can control the version -ARG GO_VERSION=1.24.4 +ARG GO_VERSION=1.24.6 # Boring Go is needed to build FIPS-compliant binaries. RUN apt-get update && \ diff --git a/go.mod b/go.mod index b4404754589bb..f5e41808e8781 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/coder/coder/v2 -go 1.24.4 +go 1.24.6 // Required until a v3 of chroma is created to lazily initialize all XML files. // None of our dependencies seem to use the registries anyways, so this @@ -58,7 +58,7 @@ replace github.com/imulab/go-scim/pkg/v2 => github.com/coder/go-scim/pkg/v2 v2.0 // Adds support for a new Listener from a driver.Connector // This lets us use rotating authentication tokens for passwords in connection strings // which we use in the awsiamrds package. -replace github.com/lib/pq => github.com/coder/pq v1.10.5-0.20240813183442-0c420cb5a048 +replace github.com/lib/pq => github.com/coder/pq v1.10.5-0.20250807075151-6ad9b0a25151 // Removes an init() function that causes terminal sequences to be printed to the web terminal when // used in conjunction with agent-exec. See https://github.com/coder/coder/pull/15817 diff --git a/go.sum b/go.sum index 8496867edf78f..a282a9ebc3727 100644 --- a/go.sum +++ b/go.sum @@ -912,8 +912,8 @@ github.com/coder/go-scim/pkg/v2 v2.0.0-20230221055123-1d63c1222136 h1:0RgB61LcNs github.com/coder/go-scim/pkg/v2 v2.0.0-20230221055123-1d63c1222136/go.mod h1:VkD1P761nykiq75dz+4iFqIQIZka189tx1BQLOp0Skc= github.com/coder/guts v1.5.0 h1:a94apf7xMf5jDdg1bIHzncbRiTn3+BvBZgrFSDbUnyI= github.com/coder/guts v1.5.0/go.mod h1:0Sbv5Kp83u1Nl7MIQiV2zmacJ3o02I341bkWkjWXSUQ= -github.com/coder/pq v1.10.5-0.20240813183442-0c420cb5a048 h1:3jzYUlGH7ZELIH4XggXhnTnP05FCYiAFeQpoN+gNR5I= -github.com/coder/pq v1.10.5-0.20240813183442-0c420cb5a048/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/coder/pq v1.10.5-0.20250807075151-6ad9b0a25151 h1:YAxwg3lraGNRwoQ18H7R7n+wsCqNve7Brdvj0F1rDnU= +github.com/coder/pq v1.10.5-0.20250807075151-6ad9b0a25151/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc= github.com/coder/preview v1.0.1 h1:f6q+RjNelwnkyXfGbmVlb4dcUOQ0z4mPsb2kuQpFHuU= From 5ff749617b40cb87a2bf5c312e449a22d654863b Mon Sep 17 00:00:00 2001 From: Ethan <39577870+ethanndickson@users.noreply.github.com> Date: Thu, 7 Aug 2025 22:21:15 +1000 Subject: [PATCH 06/10] chore(coderd/database): optimize AuditLogs queries (cherry-pick #18600) (#19193) A customer who recently upgraded their deployment to 2.24 is seeing their audit log queries take longer than a minute to load (resulting in a gateway timeout). As such, Support's requested we backport this fix to 2.24 (in the next patch), as it does not require a database migration. ### Original PR Description (https://github.com/coder/coder/pull/18600): Closes #17689 This PR optimizes the audit logs query performance by extracting the count operation into a separate query and replacing the OR-based workspace_builds with conditional joins. ## Query changes * Extracted count query to separate one * Replaced single `workspace_builds` join with OR conditions with separate conditional joins * Added conditional joins * `wb_build` for workspace_build audit logs (which is a direct lookup) * `wb_workspace` for workspace create audit logs (via workspace) Optimized AuditLogsOffset query: https://explain.dalibo.com/plan/4g1hbedg4a564bg8 New CountAuditLogs query: https://explain.dalibo.com/plan/ga2fbcecb9efbce3 Co-authored-by: Kacper Sawicki --- coderd/audit.go | 24 +- coderd/database/dbauthz/dbauthz.go | 20 ++ coderd/database/dbauthz/dbauthz_test.go | 10 + coderd/database/dbauthz/setup_test.go | 15 +- coderd/database/dbmem/dbmem.go | 85 ++++- coderd/database/dbmetrics/querymetrics.go | 14 + coderd/database/dbmock/dbmock.go | 30 ++ coderd/database/modelqueries.go | 50 ++- coderd/database/modelqueries_internal_test.go | 41 +++ coderd/database/querier.go | 1 + coderd/database/querier_test.go | 58 ++- coderd/database/queries.sql.go | 332 ++++++++++++------ coderd/database/queries/auditlogs.sql | 295 ++++++++++------ coderd/searchquery/search.go | 25 +- coderd/searchquery/search_test.go | 7 +- 15 files changed, 763 insertions(+), 244 deletions(-) diff --git a/coderd/audit.go b/coderd/audit.go index 63b6e49ebb05a..786707768c05e 100644 --- a/coderd/audit.go +++ b/coderd/audit.go @@ -46,7 +46,7 @@ func (api *API) auditLogs(rw http.ResponseWriter, r *http.Request) { } queryStr := r.URL.Query().Get("q") - filter, errs := searchquery.AuditLogs(ctx, api.Database, queryStr) + filter, countFilter, errs := searchquery.AuditLogs(ctx, api.Database, queryStr) if len(errs) > 0 { httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ Message: "Invalid audit search query.", @@ -62,9 +62,12 @@ func (api *API) auditLogs(rw http.ResponseWriter, r *http.Request) { if filter.Username == "me" { filter.UserID = apiKey.UserID filter.Username = "" + countFilter.UserID = apiKey.UserID + countFilter.Username = "" } - dblogs, err := api.Database.GetAuditLogsOffset(ctx, filter) + // Use the same filters to count the number of audit logs + count, err := api.Database.CountAuditLogs(ctx, countFilter) if dbauthz.IsNotAuthorizedError(err) { httpapi.Forbidden(rw) return @@ -73,9 +76,8 @@ func (api *API) auditLogs(rw http.ResponseWriter, r *http.Request) { httpapi.InternalServerError(rw, err) return } - // GetAuditLogsOffset does not return ErrNoRows because it uses a window function to get the count. - // So we need to check if the dblogs is empty and return an empty array if so. - if len(dblogs) == 0 { + // If count is 0, then we don't need to query audit logs + if count == 0 { httpapi.Write(ctx, rw, http.StatusOK, codersdk.AuditLogResponse{ AuditLogs: []codersdk.AuditLog{}, Count: 0, @@ -83,9 +85,19 @@ func (api *API) auditLogs(rw http.ResponseWriter, r *http.Request) { return } + dblogs, err := api.Database.GetAuditLogsOffset(ctx, filter) + if dbauthz.IsNotAuthorizedError(err) { + httpapi.Forbidden(rw) + return + } + if err != nil { + httpapi.InternalServerError(rw, err) + return + } + httpapi.Write(ctx, rw, http.StatusOK, codersdk.AuditLogResponse{ AuditLogs: api.convertAuditLogs(ctx, dblogs), - Count: dblogs[0].Count, + Count: count, }) } diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index d63e049abf8ee..a2c3b1d5705da 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -1301,6 +1301,22 @@ func (q *querier) CleanTailnetTunnels(ctx context.Context) error { return q.db.CleanTailnetTunnels(ctx) } +func (q *querier) CountAuditLogs(ctx context.Context, arg database.CountAuditLogsParams) (int64, error) { + // Shortcut if the user is an owner. The SQL filter is noticeable, + // and this is an easy win for owners. Which is the common case. + err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceAuditLog) + if err == nil { + return q.db.CountAuditLogs(ctx, arg) + } + + prep, err := prepareSQLFilter(ctx, q.auth, policy.ActionRead, rbac.ResourceAuditLog.Type) + if err != nil { + return 0, xerrors.Errorf("(dev error) prepare sql filter: %w", err) + } + + return q.db.CountAuthorizedAuditLogs(ctx, arg, prep) +} + func (q *querier) CountInProgressPrebuilds(ctx context.Context) ([]database.CountInProgressPrebuildsRow, error) { if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceWorkspace.All()); err != nil { return nil, err @@ -5256,3 +5272,7 @@ func (q *querier) GetAuthorizedUsers(ctx context.Context, arg database.GetUsersP func (q *querier) GetAuthorizedAuditLogsOffset(ctx context.Context, arg database.GetAuditLogsOffsetParams, _ rbac.PreparedAuthorized) ([]database.GetAuditLogsOffsetRow, error) { return q.GetAuditLogsOffset(ctx, arg) } + +func (q *querier) CountAuthorizedAuditLogs(ctx context.Context, arg database.CountAuditLogsParams, _ rbac.PreparedAuthorized) (int64, error) { + return q.CountAuditLogs(ctx, arg) +} diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 6d1c8c3df601c..e83b2bd4710c5 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -327,6 +327,16 @@ func (s *MethodTestSuite) TestAuditLogs() { LimitOpt: 10, }, emptyPreparedAuthorized{}).Asserts(rbac.ResourceAuditLog, policy.ActionRead) })) + s.Run("CountAuditLogs", s.Subtest(func(db database.Store, check *expects) { + _ = dbgen.AuditLog(s.T(), db, database.AuditLog{}) + _ = dbgen.AuditLog(s.T(), db, database.AuditLog{}) + check.Args(database.CountAuditLogsParams{}).Asserts(rbac.ResourceAuditLog, policy.ActionRead).WithNotAuthorized("nil") + })) + s.Run("CountAuthorizedAuditLogs", s.Subtest(func(db database.Store, check *expects) { + _ = dbgen.AuditLog(s.T(), db, database.AuditLog{}) + _ = dbgen.AuditLog(s.T(), db, database.AuditLog{}) + check.Args(database.CountAuditLogsParams{}, emptyPreparedAuthorized{}).Asserts(rbac.ResourceAuditLog, policy.ActionRead) + })) } func (s *MethodTestSuite) TestFile() { diff --git a/coderd/database/dbauthz/setup_test.go b/coderd/database/dbauthz/setup_test.go index 29ca421d6f11e..555a17fb2070f 100644 --- a/coderd/database/dbauthz/setup_test.go +++ b/coderd/database/dbauthz/setup_test.go @@ -271,7 +271,7 @@ func (s *MethodTestSuite) NotAuthorizedErrorTest(ctx context.Context, az *coderd // This is unfortunate, but if we are using `Filter` the error returned will be nil. So filter out // any case where the error is nil and the response is an empty slice. - if err != nil || !hasEmptySliceResponse(resp) { + if err != nil || !hasEmptyResponse(resp) { // Expect the default error if testCase.notAuthorizedExpect == "" { s.ErrorContainsf(err, "unauthorized", "error string should have a good message") @@ -296,8 +296,8 @@ func (s *MethodTestSuite) NotAuthorizedErrorTest(ctx context.Context, az *coderd resp, err := callMethod(ctx) // This is unfortunate, but if we are using `Filter` the error returned will be nil. So filter out - // any case where the error is nil and the response is an empty slice. - if err != nil || !hasEmptySliceResponse(resp) { + // any case where the error is nil and the response is an empty slice or int64(0). + if err != nil || !hasEmptyResponse(resp) { if testCase.cancelledCtxExpect == "" { s.Errorf(err, "method should an error with cancellation") s.ErrorIsf(err, context.Canceled, "error should match context.Canceled") @@ -308,13 +308,20 @@ func (s *MethodTestSuite) NotAuthorizedErrorTest(ctx context.Context, az *coderd }) } -func hasEmptySliceResponse(values []reflect.Value) bool { +func hasEmptyResponse(values []reflect.Value) bool { for _, r := range values { if r.Kind() == reflect.Slice || r.Kind() == reflect.Array { if r.Len() == 0 { return true } } + + // Special case for int64, as it's the return type for count query. + if r.Kind() == reflect.Int64 { + if r.Int() == 0 { + return true + } + } } return false } diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index cd1067e61dbb5..42d61244d9098 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -1779,6 +1779,10 @@ func (*FakeQuerier) CleanTailnetTunnels(context.Context) error { return ErrUnimplemented } +func (q *FakeQuerier) CountAuditLogs(ctx context.Context, arg database.CountAuditLogsParams) (int64, error) { + return q.CountAuthorizedAuditLogs(ctx, arg, nil) +} + func (q *FakeQuerier) CountInProgressPrebuilds(ctx context.Context) ([]database.CountInProgressPrebuildsRow, error) { return nil, ErrUnimplemented } @@ -13930,7 +13934,6 @@ func (q *FakeQuerier) GetAuthorizedAuditLogsOffset(ctx context.Context, arg data UserQuietHoursSchedule: sql.NullString{String: user.QuietHoursSchedule, Valid: userValid}, UserStatus: database.NullUserStatus{UserStatus: user.Status, Valid: userValid}, UserRoles: user.RBACRoles, - Count: 0, }) if len(logs) >= int(arg.LimitOpt) { @@ -13938,10 +13941,82 @@ func (q *FakeQuerier) GetAuthorizedAuditLogsOffset(ctx context.Context, arg data } } - count := int64(len(logs)) - for i := range logs { - logs[i].Count = count + return logs, nil +} + +func (q *FakeQuerier) CountAuthorizedAuditLogs(ctx context.Context, arg database.CountAuditLogsParams, prepared rbac.PreparedAuthorized) (int64, error) { + if err := validateDatabaseType(arg); err != nil { + return 0, err } - return logs, nil + // Call this to match the same function calls as the SQL implementation. + // It functionally does nothing for filtering. + if prepared != nil { + _, err := prepared.CompileToSQL(ctx, regosql.ConvertConfig{ + VariableConverter: regosql.AuditLogConverter(), + }) + if err != nil { + return 0, err + } + } + + q.mutex.RLock() + defer q.mutex.RUnlock() + + var count int64 + + // q.auditLogs are already sorted by time DESC, so no need to sort after the fact. + for _, alog := range q.auditLogs { + if arg.RequestID != uuid.Nil && arg.RequestID != alog.RequestID { + continue + } + if arg.OrganizationID != uuid.Nil && arg.OrganizationID != alog.OrganizationID { + continue + } + if arg.Action != "" && string(alog.Action) != arg.Action { + continue + } + if arg.ResourceType != "" && !strings.Contains(string(alog.ResourceType), arg.ResourceType) { + continue + } + if arg.ResourceID != uuid.Nil && alog.ResourceID != arg.ResourceID { + continue + } + if arg.Username != "" { + user, err := q.getUserByIDNoLock(alog.UserID) + if err == nil && !strings.EqualFold(arg.Username, user.Username) { + continue + } + } + if arg.Email != "" { + user, err := q.getUserByIDNoLock(alog.UserID) + if err == nil && !strings.EqualFold(arg.Email, user.Email) { + continue + } + } + if !arg.DateFrom.IsZero() { + if alog.Time.Before(arg.DateFrom) { + continue + } + } + if !arg.DateTo.IsZero() { + if alog.Time.After(arg.DateTo) { + continue + } + } + if arg.BuildReason != "" { + workspaceBuild, err := q.getWorkspaceBuildByIDNoLock(context.Background(), alog.ResourceID) + if err == nil && !strings.EqualFold(arg.BuildReason, string(workspaceBuild.Reason)) { + continue + } + } + // If the filter exists, ensure the object is authorized. + if prepared != nil && prepared.Authorize(ctx, alog.RBACObject()) != nil { + continue + } + + count++ + } + + return count, nil } diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 0d68d0c15e1be..ca2b0c2ce7fa5 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -186,6 +186,13 @@ func (m queryMetricsStore) CleanTailnetTunnels(ctx context.Context) error { return r0 } +func (m queryMetricsStore) CountAuditLogs(ctx context.Context, arg database.CountAuditLogsParams) (int64, error) { + start := time.Now() + r0, r1 := m.s.CountAuditLogs(ctx, arg) + m.queryLatencies.WithLabelValues("CountAuditLogs").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m queryMetricsStore) CountInProgressPrebuilds(ctx context.Context) ([]database.CountInProgressPrebuildsRow, error) { start := time.Now() r0, r1 := m.s.CountInProgressPrebuilds(ctx) @@ -3321,3 +3328,10 @@ func (m queryMetricsStore) GetAuthorizedAuditLogsOffset(ctx context.Context, arg m.queryLatencies.WithLabelValues("GetAuthorizedAuditLogsOffset").Observe(time.Since(start).Seconds()) return r0, r1 } + +func (m queryMetricsStore) CountAuthorizedAuditLogs(ctx context.Context, arg database.CountAuditLogsParams, prepared rbac.PreparedAuthorized) (int64, error) { + start := time.Now() + r0, r1 := m.s.CountAuthorizedAuditLogs(ctx, arg, prepared) + m.queryLatencies.WithLabelValues("CountAuthorizedAuditLogs").Observe(time.Since(start).Seconds()) + return r0, r1 +} diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 03222782a5d68..9d7d6c74cb0ce 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -247,6 +247,36 @@ func (mr *MockStoreMockRecorder) CleanTailnetTunnels(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanTailnetTunnels", reflect.TypeOf((*MockStore)(nil).CleanTailnetTunnels), ctx) } +// CountAuditLogs mocks base method. +func (m *MockStore) CountAuditLogs(ctx context.Context, arg database.CountAuditLogsParams) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountAuditLogs", ctx, arg) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountAuditLogs indicates an expected call of CountAuditLogs. +func (mr *MockStoreMockRecorder) CountAuditLogs(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountAuditLogs", reflect.TypeOf((*MockStore)(nil).CountAuditLogs), ctx, arg) +} + +// CountAuthorizedAuditLogs mocks base method. +func (m *MockStore) CountAuthorizedAuditLogs(ctx context.Context, arg database.CountAuditLogsParams, prepared rbac.PreparedAuthorized) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountAuthorizedAuditLogs", ctx, arg, prepared) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountAuthorizedAuditLogs indicates an expected call of CountAuthorizedAuditLogs. +func (mr *MockStoreMockRecorder) CountAuthorizedAuditLogs(ctx, arg, prepared any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountAuthorizedAuditLogs", reflect.TypeOf((*MockStore)(nil).CountAuthorizedAuditLogs), ctx, arg, prepared) +} + // CountInProgressPrebuilds mocks base method. func (m *MockStore) CountInProgressPrebuilds(ctx context.Context) ([]database.CountInProgressPrebuildsRow, error) { m.ctrl.T.Helper() diff --git a/coderd/database/modelqueries.go b/coderd/database/modelqueries.go index eaf73af07f6d5..785ccf86afd27 100644 --- a/coderd/database/modelqueries.go +++ b/coderd/database/modelqueries.go @@ -478,6 +478,7 @@ func (q *sqlQuerier) GetAuthorizedUsers(ctx context.Context, arg GetUsersParams, type auditLogQuerier interface { GetAuthorizedAuditLogsOffset(ctx context.Context, arg GetAuditLogsOffsetParams, prepared rbac.PreparedAuthorized) ([]GetAuditLogsOffsetRow, error) + CountAuthorizedAuditLogs(ctx context.Context, arg CountAuditLogsParams, prepared rbac.PreparedAuthorized) (int64, error) } func (q *sqlQuerier) GetAuthorizedAuditLogsOffset(ctx context.Context, arg GetAuditLogsOffsetParams, prepared rbac.PreparedAuthorized) ([]GetAuditLogsOffsetRow, error) { @@ -548,7 +549,6 @@ func (q *sqlQuerier) GetAuthorizedAuditLogsOffset(ctx context.Context, arg GetAu &i.OrganizationName, &i.OrganizationDisplayName, &i.OrganizationIcon, - &i.Count, ); err != nil { return nil, err } @@ -563,6 +563,54 @@ func (q *sqlQuerier) GetAuthorizedAuditLogsOffset(ctx context.Context, arg GetAu return items, nil } +func (q *sqlQuerier) CountAuthorizedAuditLogs(ctx context.Context, arg CountAuditLogsParams, prepared rbac.PreparedAuthorized) (int64, error) { + authorizedFilter, err := prepared.CompileToSQL(ctx, regosql.ConvertConfig{ + VariableConverter: regosql.AuditLogConverter(), + }) + if err != nil { + return 0, xerrors.Errorf("compile authorized filter: %w", err) + } + + filtered, err := insertAuthorizedFilter(countAuditLogs, fmt.Sprintf(" AND %s", authorizedFilter)) + if err != nil { + return 0, xerrors.Errorf("insert authorized filter: %w", err) + } + + query := fmt.Sprintf("-- name: CountAuthorizedAuditLogs :one\n%s", filtered) + + rows, err := q.db.QueryContext(ctx, query, + arg.ResourceType, + arg.ResourceID, + arg.OrganizationID, + arg.ResourceTarget, + arg.Action, + arg.UserID, + arg.Username, + arg.Email, + arg.DateFrom, + arg.DateTo, + arg.BuildReason, + arg.RequestID, + ) + if err != nil { + return 0, err + } + defer rows.Close() + var count int64 + for rows.Next() { + if err := rows.Scan(&count); err != nil { + return 0, err + } + } + if err := rows.Close(); err != nil { + return 0, err + } + if err := rows.Err(); err != nil { + return 0, err + } + return count, nil +} + func insertAuthorizedFilter(query string, replaceWith string) (string, error) { if !strings.Contains(query, authorizedQueryPlaceholder) { return "", xerrors.Errorf("query does not contain authorized replace string, this is not an authorized query") diff --git a/coderd/database/modelqueries_internal_test.go b/coderd/database/modelqueries_internal_test.go index 992eb269ddc14..4f675a1b60785 100644 --- a/coderd/database/modelqueries_internal_test.go +++ b/coderd/database/modelqueries_internal_test.go @@ -1,9 +1,12 @@ package database import ( + "regexp" + "strings" "testing" "time" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" "github.com/coder/coder/v2/testutil" @@ -54,3 +57,41 @@ func TestWorkspaceTableConvert(t *testing.T) { "'workspace.WorkspaceTable()' is not missing at least 1 field when converting to 'WorkspaceTable'. "+ "To resolve this, go to the 'func (w Workspace) WorkspaceTable()' and ensure all fields are converted.") } + +// TestAuditLogsQueryConsistency ensures that GetAuditLogsOffset and CountAuditLogs +// have identical WHERE clauses to prevent filtering inconsistencies. +// This test is a guard rail to prevent developer oversight mistakes. +func TestAuditLogsQueryConsistency(t *testing.T) { + t.Parallel() + + getWhereClause := extractWhereClause(getAuditLogsOffset) + require.NotEmpty(t, getWhereClause, "failed to extract WHERE clause from GetAuditLogsOffset") + + countWhereClause := extractWhereClause(countAuditLogs) + require.NotEmpty(t, countWhereClause, "failed to extract WHERE clause from CountAuditLogs") + + // Compare the WHERE clauses + if diff := cmp.Diff(getWhereClause, countWhereClause); diff != "" { + t.Errorf("GetAuditLogsOffset and CountAuditLogs WHERE clauses must be identical to ensure consistent filtering.\nDiff:\n%s", diff) + } +} + +// extractWhereClause extracts the WHERE clause from a SQL query string +func extractWhereClause(query string) string { + // Find WHERE and get everything after it + wherePattern := regexp.MustCompile(`(?is)WHERE\s+(.*)`) + whereMatches := wherePattern.FindStringSubmatch(query) + if len(whereMatches) < 2 { + return "" + } + + whereClause := whereMatches[1] + + // Remove ORDER BY, LIMIT, OFFSET clauses from the end + whereClause = regexp.MustCompile(`(?is)\s+(ORDER BY|LIMIT|OFFSET).*$`).ReplaceAllString(whereClause, "") + + // Remove SQL comments + whereClause = regexp.MustCompile(`(?m)--.*$`).ReplaceAllString(whereClause, "") + + return strings.TrimSpace(whereClause) +} diff --git a/coderd/database/querier.go b/coderd/database/querier.go index b1c13d31ceb6d..4d5052b42aadc 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -64,6 +64,7 @@ type sqlcQuerier interface { CleanTailnetCoordinators(ctx context.Context) error CleanTailnetLostPeers(ctx context.Context) error CleanTailnetTunnels(ctx context.Context) error + CountAuditLogs(ctx context.Context, arg CountAuditLogsParams) (int64, error) // CountInProgressPrebuilds returns the number of in-progress prebuilds, grouped by preset ID and transition. // Prebuild considered in-progress if it's in the "starting", "stopping", or "deleting" state. CountInProgressPrebuilds(ctx context.Context) ([]CountInProgressPrebuildsRow, error) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 5ae43138bd634..f80f68115ad2c 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -1567,6 +1567,26 @@ func TestAuditLogDefaultLimit(t *testing.T) { require.Len(t, rows, 100) } +func TestAuditLogCount(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + sqlDB := testSQLDB(t) + err := migrations.Up(sqlDB) + require.NoError(t, err) + db := database.New(sqlDB) + + ctx := testutil.Context(t, testutil.WaitLong) + + dbgen.AuditLog(t, db, database.AuditLog{}) + + count, err := db.CountAuditLogs(ctx, database.CountAuditLogsParams{}) + require.NoError(t, err) + require.Equal(t, int64(1), count) +} + func TestWorkspaceQuotas(t *testing.T) { t.Parallel() orgMemberIDs := func(o database.OrganizationMember) uuid.UUID { @@ -1947,9 +1967,13 @@ func TestAuthorizedAuditLogs(t *testing.T) { }) // When: The user queries for audit logs + count, err := db.CountAuditLogs(memberCtx, database.CountAuditLogsParams{}) + require.NoError(t, err) logs, err := db.GetAuditLogsOffset(memberCtx, database.GetAuditLogsOffsetParams{}) require.NoError(t, err) - // Then: No logs returned + + // Then: No logs returned and count is 0 + require.Equal(t, int64(0), count, "count should be 0") require.Len(t, logs, 0, "no logs should be returned") }) @@ -1965,10 +1989,14 @@ func TestAuthorizedAuditLogs(t *testing.T) { }) // When: the auditor queries for audit logs + count, err := db.CountAuditLogs(siteAuditorCtx, database.CountAuditLogsParams{}) + require.NoError(t, err) logs, err := db.GetAuditLogsOffset(siteAuditorCtx, database.GetAuditLogsOffsetParams{}) require.NoError(t, err) - // Then: All logs are returned - require.ElementsMatch(t, auditOnlyIDs(allLogs), auditOnlyIDs(logs)) + + // Then: All logs are returned and count matches + require.Equal(t, int64(len(allLogs)), count, "count should match total number of logs") + require.ElementsMatch(t, auditOnlyIDs(allLogs), auditOnlyIDs(logs), "all logs should be returned") }) t.Run("SingleOrgAuditor", func(t *testing.T) { @@ -1984,10 +2012,14 @@ func TestAuthorizedAuditLogs(t *testing.T) { }) // When: The auditor queries for audit logs + count, err := db.CountAuditLogs(orgAuditCtx, database.CountAuditLogsParams{}) + require.NoError(t, err) logs, err := db.GetAuditLogsOffset(orgAuditCtx, database.GetAuditLogsOffsetParams{}) require.NoError(t, err) - // Then: Only the logs for the organization are returned - require.ElementsMatch(t, orgAuditLogs[orgID], auditOnlyIDs(logs)) + + // Then: Only the logs for the organization are returned and count matches + require.Equal(t, int64(len(orgAuditLogs[orgID])), count, "count should match organization logs") + require.ElementsMatch(t, orgAuditLogs[orgID], auditOnlyIDs(logs), "only organization logs should be returned") }) t.Run("TwoOrgAuditors", func(t *testing.T) { @@ -2004,10 +2036,16 @@ func TestAuthorizedAuditLogs(t *testing.T) { }) // When: The user queries for audit logs + count, err := db.CountAuditLogs(multiOrgAuditCtx, database.CountAuditLogsParams{}) + require.NoError(t, err) logs, err := db.GetAuditLogsOffset(multiOrgAuditCtx, database.GetAuditLogsOffsetParams{}) require.NoError(t, err) - // Then: All logs for both organizations are returned - require.ElementsMatch(t, append(orgAuditLogs[first], orgAuditLogs[second]...), auditOnlyIDs(logs)) + + // Then: All logs for both organizations are returned and count matches + expectedLogs := append([]uuid.UUID{}, orgAuditLogs[first]...) + expectedLogs = append(expectedLogs, orgAuditLogs[second]...) + require.Equal(t, int64(len(expectedLogs)), count, "count should match sum of both organizations") + require.ElementsMatch(t, expectedLogs, auditOnlyIDs(logs), "logs from both organizations should be returned") }) t.Run("ErroneousOrg", func(t *testing.T) { @@ -2022,9 +2060,13 @@ func TestAuthorizedAuditLogs(t *testing.T) { }) // When: The user queries for audit logs + count, err := db.CountAuditLogs(userCtx, database.CountAuditLogsParams{}) + require.NoError(t, err) logs, err := db.GetAuditLogsOffset(userCtx, database.GetAuditLogsOffsetParams{}) require.NoError(t, err) - // Then: No logs are returned + + // Then: No logs are returned and count is 0 + require.Equal(t, int64(0), count, "count should be 0") require.Len(t, logs, 0, "no logs should be returned") }) } diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 396a044a3484b..a24de735bf539 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -441,140 +441,241 @@ func (q *sqlQuerier) UpdateAPIKeyByID(ctx context.Context, arg UpdateAPIKeyByIDP return err } -const getAuditLogsOffset = `-- name: GetAuditLogsOffset :many -SELECT - audit_logs.id, audit_logs.time, audit_logs.user_id, audit_logs.organization_id, audit_logs.ip, audit_logs.user_agent, audit_logs.resource_type, audit_logs.resource_id, audit_logs.resource_target, audit_logs.action, audit_logs.diff, audit_logs.status_code, audit_logs.additional_fields, audit_logs.request_id, audit_logs.resource_icon, - -- sqlc.embed(users) would be nice but it does not seem to play well with - -- left joins. - users.username AS user_username, - users.name AS user_name, - users.email AS user_email, - users.created_at AS user_created_at, - users.updated_at AS user_updated_at, - users.last_seen_at AS user_last_seen_at, - users.status AS user_status, - users.login_type AS user_login_type, - users.rbac_roles AS user_roles, - users.avatar_url AS user_avatar_url, - users.deleted AS user_deleted, - users.quiet_hours_schedule AS user_quiet_hours_schedule, - COALESCE(organizations.name, '') AS organization_name, - COALESCE(organizations.display_name, '') AS organization_display_name, - COALESCE(organizations.icon, '') AS organization_icon, - COUNT(audit_logs.*) OVER () AS count -FROM - audit_logs - LEFT JOIN users ON audit_logs.user_id = users.id - LEFT JOIN - -- First join on workspaces to get the initial workspace create - -- to workspace build 1 id. This is because the first create is - -- is a different audit log than subsequent starts. - workspaces ON - audit_logs.resource_type = 'workspace' AND - audit_logs.resource_id = workspaces.id - LEFT JOIN - workspace_builds ON - -- Get the reason from the build if the resource type - -- is a workspace_build - ( - audit_logs.resource_type = 'workspace_build' - AND audit_logs.resource_id = workspace_builds.id - ) - OR - -- Get the reason from the build #1 if this is the first - -- workspace create. - ( - audit_logs.resource_type = 'workspace' AND - audit_logs.action = 'create' AND - workspaces.id = workspace_builds.workspace_id AND - workspace_builds.build_number = 1 - ) - LEFT JOIN organizations ON audit_logs.organization_id = organizations.id +const countAuditLogs = `-- name: CountAuditLogs :one +SELECT COUNT(*) +FROM audit_logs + LEFT JOIN users ON audit_logs.user_id = users.id + LEFT JOIN organizations ON audit_logs.organization_id = organizations.id + -- First join on workspaces to get the initial workspace create + -- to workspace build 1 id. This is because the first create is + -- is a different audit log than subsequent starts. + LEFT JOIN workspaces ON audit_logs.resource_type = 'workspace' + AND audit_logs.resource_id = workspaces.id + -- Get the reason from the build if the resource type + -- is a workspace_build + LEFT JOIN workspace_builds wb_build ON audit_logs.resource_type = 'workspace_build' + AND audit_logs.resource_id = wb_build.id + -- Get the reason from the build #1 if this is the first + -- workspace create. + LEFT JOIN workspace_builds wb_workspace ON audit_logs.resource_type = 'workspace' + AND audit_logs.action = 'create' + AND workspaces.id = wb_workspace.workspace_id + AND wb_workspace.build_number = 1 WHERE - -- Filter resource_type + -- Filter resource_type CASE - WHEN $1 :: text != '' THEN - resource_type = $1 :: resource_type + WHEN $1::text != '' THEN resource_type = $1::resource_type ELSE true END -- Filter resource_id AND CASE - WHEN $2 :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - resource_id = $2 + WHEN $2::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN resource_id = $2 ELSE true END - -- Filter organization_id - AND CASE - WHEN $3 :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - audit_logs.organization_id = $3 + -- Filter organization_id + AND CASE + WHEN $3::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.organization_id = $3 ELSE true END -- Filter by resource_target AND CASE - WHEN $4 :: text != '' THEN - resource_target = $4 + WHEN $4::text != '' THEN resource_target = $4 ELSE true END -- Filter action AND CASE - WHEN $5 :: text != '' THEN - action = $5 :: audit_action + WHEN $5::text != '' THEN action = $5::audit_action ELSE true END -- Filter by user_id AND CASE - WHEN $6 :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - user_id = $6 + WHEN $6::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN user_id = $6 ELSE true END -- Filter by username AND CASE - WHEN $7 :: text != '' THEN - user_id = (SELECT id FROM users WHERE lower(username) = lower($7) AND deleted = false) + WHEN $7::text != '' THEN user_id = ( + SELECT id + FROM users + WHERE lower(username) = lower($7) + AND deleted = false + ) ELSE true END -- Filter by user_email AND CASE - WHEN $8 :: text != '' THEN - users.email = $8 + WHEN $8::text != '' THEN users.email = $8 ELSE true END -- Filter by date_from AND CASE - WHEN $9 :: timestamp with time zone != '0001-01-01 00:00:00Z' THEN - "time" >= $9 + WHEN $9::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" >= $9 ELSE true END -- Filter by date_to AND CASE - WHEN $10 :: timestamp with time zone != '0001-01-01 00:00:00Z' THEN - "time" <= $10 + WHEN $10::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" <= $10 + ELSE true + END + -- Filter by build_reason + AND CASE + WHEN $11::text != '' THEN COALESCE(wb_build.reason::text, wb_workspace.reason::text) = $11 ELSE true END - -- Filter by build_reason - AND CASE - WHEN $11::text != '' THEN - workspace_builds.reason::text = $11 - ELSE true - END -- Filter request_id AND CASE - WHEN $12 :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - audit_logs.request_id = $12 + WHEN $12::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.request_id = $12 ELSE true END + -- Authorize Filter clause will be injected below in CountAuthorizedAuditLogs + -- @authorize_filter +` + +type CountAuditLogsParams struct { + ResourceType string `db:"resource_type" json:"resource_type"` + ResourceID uuid.UUID `db:"resource_id" json:"resource_id"` + OrganizationID uuid.UUID `db:"organization_id" json:"organization_id"` + ResourceTarget string `db:"resource_target" json:"resource_target"` + Action string `db:"action" json:"action"` + UserID uuid.UUID `db:"user_id" json:"user_id"` + Username string `db:"username" json:"username"` + Email string `db:"email" json:"email"` + DateFrom time.Time `db:"date_from" json:"date_from"` + DateTo time.Time `db:"date_to" json:"date_to"` + BuildReason string `db:"build_reason" json:"build_reason"` + RequestID uuid.UUID `db:"request_id" json:"request_id"` +} + +func (q *sqlQuerier) CountAuditLogs(ctx context.Context, arg CountAuditLogsParams) (int64, error) { + row := q.db.QueryRowContext(ctx, countAuditLogs, + arg.ResourceType, + arg.ResourceID, + arg.OrganizationID, + arg.ResourceTarget, + arg.Action, + arg.UserID, + arg.Username, + arg.Email, + arg.DateFrom, + arg.DateTo, + arg.BuildReason, + arg.RequestID, + ) + var count int64 + err := row.Scan(&count) + return count, err +} +const getAuditLogsOffset = `-- name: GetAuditLogsOffset :many +SELECT audit_logs.id, audit_logs.time, audit_logs.user_id, audit_logs.organization_id, audit_logs.ip, audit_logs.user_agent, audit_logs.resource_type, audit_logs.resource_id, audit_logs.resource_target, audit_logs.action, audit_logs.diff, audit_logs.status_code, audit_logs.additional_fields, audit_logs.request_id, audit_logs.resource_icon, + -- sqlc.embed(users) would be nice but it does not seem to play well with + -- left joins. + users.username AS user_username, + users.name AS user_name, + users.email AS user_email, + users.created_at AS user_created_at, + users.updated_at AS user_updated_at, + users.last_seen_at AS user_last_seen_at, + users.status AS user_status, + users.login_type AS user_login_type, + users.rbac_roles AS user_roles, + users.avatar_url AS user_avatar_url, + users.deleted AS user_deleted, + users.quiet_hours_schedule AS user_quiet_hours_schedule, + COALESCE(organizations.name, '') AS organization_name, + COALESCE(organizations.display_name, '') AS organization_display_name, + COALESCE(organizations.icon, '') AS organization_icon +FROM audit_logs + LEFT JOIN users ON audit_logs.user_id = users.id + LEFT JOIN organizations ON audit_logs.organization_id = organizations.id + -- First join on workspaces to get the initial workspace create + -- to workspace build 1 id. This is because the first create is + -- is a different audit log than subsequent starts. + LEFT JOIN workspaces ON audit_logs.resource_type = 'workspace' + AND audit_logs.resource_id = workspaces.id + -- Get the reason from the build if the resource type + -- is a workspace_build + LEFT JOIN workspace_builds wb_build ON audit_logs.resource_type = 'workspace_build' + AND audit_logs.resource_id = wb_build.id + -- Get the reason from the build #1 if this is the first + -- workspace create. + LEFT JOIN workspace_builds wb_workspace ON audit_logs.resource_type = 'workspace' + AND audit_logs.action = 'create' + AND workspaces.id = wb_workspace.workspace_id + AND wb_workspace.build_number = 1 +WHERE + -- Filter resource_type + CASE + WHEN $1::text != '' THEN resource_type = $1::resource_type + ELSE true + END + -- Filter resource_id + AND CASE + WHEN $2::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN resource_id = $2 + ELSE true + END + -- Filter organization_id + AND CASE + WHEN $3::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.organization_id = $3 + ELSE true + END + -- Filter by resource_target + AND CASE + WHEN $4::text != '' THEN resource_target = $4 + ELSE true + END + -- Filter action + AND CASE + WHEN $5::text != '' THEN action = $5::audit_action + ELSE true + END + -- Filter by user_id + AND CASE + WHEN $6::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN user_id = $6 + ELSE true + END + -- Filter by username + AND CASE + WHEN $7::text != '' THEN user_id = ( + SELECT id + FROM users + WHERE lower(username) = lower($7) + AND deleted = false + ) + ELSE true + END + -- Filter by user_email + AND CASE + WHEN $8::text != '' THEN users.email = $8 + ELSE true + END + -- Filter by date_from + AND CASE + WHEN $9::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" >= $9 + ELSE true + END + -- Filter by date_to + AND CASE + WHEN $10::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" <= $10 + ELSE true + END + -- Filter by build_reason + AND CASE + WHEN $11::text != '' THEN COALESCE(wb_build.reason::text, wb_workspace.reason::text) = $11 + ELSE true + END + -- Filter request_id + AND CASE + WHEN $12::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.request_id = $12 + ELSE true + END -- Authorize Filter clause will be injected below in GetAuthorizedAuditLogsOffset -- @authorize_filter -ORDER BY - "time" DESC -LIMIT - -- a limit of 0 means "no limit". The audit log table is unbounded +ORDER BY "time" DESC +LIMIT -- a limit of 0 means "no limit". The audit log table is unbounded -- in size, and is expected to be quite large. Implement a default -- limit of 100 to prevent accidental excessively large queries. - COALESCE(NULLIF($14 :: int, 0), 100) -OFFSET - $13 + COALESCE(NULLIF($14::int, 0), 100) OFFSET $13 ` type GetAuditLogsOffsetParams struct { @@ -611,7 +712,6 @@ type GetAuditLogsOffsetRow struct { OrganizationName string `db:"organization_name" json:"organization_name"` OrganizationDisplayName string `db:"organization_display_name" json:"organization_display_name"` OrganizationIcon string `db:"organization_icon" json:"organization_icon"` - Count int64 `db:"count" json:"count"` } // GetAuditLogsBefore retrieves `row_limit` number of audit logs before the provided @@ -671,7 +771,6 @@ func (q *sqlQuerier) GetAuditLogsOffset(ctx context.Context, arg GetAuditLogsOff &i.OrganizationName, &i.OrganizationDisplayName, &i.OrganizationIcon, - &i.Count, ); err != nil { return nil, err } @@ -687,26 +786,41 @@ func (q *sqlQuerier) GetAuditLogsOffset(ctx context.Context, arg GetAuditLogsOff } const insertAuditLog = `-- name: InsertAuditLog :one -INSERT INTO - audit_logs ( - id, - "time", - user_id, - organization_id, - ip, - user_agent, - resource_type, - resource_id, - resource_target, - action, - diff, - status_code, - additional_fields, - request_id, - resource_icon - ) -VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING id, time, user_id, organization_id, ip, user_agent, resource_type, resource_id, resource_target, action, diff, status_code, additional_fields, request_id, resource_icon +INSERT INTO audit_logs ( + id, + "time", + user_id, + organization_id, + ip, + user_agent, + resource_type, + resource_id, + resource_target, + action, + diff, + status_code, + additional_fields, + request_id, + resource_icon + ) +VALUES ( + $1, + $2, + $3, + $4, + $5, + $6, + $7, + $8, + $9, + $10, + $11, + $12, + $13, + $14, + $15 + ) +RETURNING id, time, user_id, organization_id, ip, user_agent, resource_type, resource_id, resource_target, action, diff, status_code, additional_fields, request_id, resource_icon ` type InsertAuditLogParams struct { diff --git a/coderd/database/queries/auditlogs.sql b/coderd/database/queries/auditlogs.sql index 9016908a75feb..6269f21cd27e4 100644 --- a/coderd/database/queries/auditlogs.sql +++ b/coderd/database/queries/auditlogs.sql @@ -1,158 +1,239 @@ -- GetAuditLogsBefore retrieves `row_limit` number of audit logs before the provided -- ID. -- name: GetAuditLogsOffset :many -SELECT - sqlc.embed(audit_logs), - -- sqlc.embed(users) would be nice but it does not seem to play well with - -- left joins. - users.username AS user_username, - users.name AS user_name, - users.email AS user_email, - users.created_at AS user_created_at, - users.updated_at AS user_updated_at, - users.last_seen_at AS user_last_seen_at, - users.status AS user_status, - users.login_type AS user_login_type, - users.rbac_roles AS user_roles, - users.avatar_url AS user_avatar_url, - users.deleted AS user_deleted, - users.quiet_hours_schedule AS user_quiet_hours_schedule, - COALESCE(organizations.name, '') AS organization_name, - COALESCE(organizations.display_name, '') AS organization_display_name, - COALESCE(organizations.icon, '') AS organization_icon, - COUNT(audit_logs.*) OVER () AS count -FROM - audit_logs - LEFT JOIN users ON audit_logs.user_id = users.id - LEFT JOIN - -- First join on workspaces to get the initial workspace create - -- to workspace build 1 id. This is because the first create is - -- is a different audit log than subsequent starts. - workspaces ON - audit_logs.resource_type = 'workspace' AND - audit_logs.resource_id = workspaces.id - LEFT JOIN - workspace_builds ON - -- Get the reason from the build if the resource type - -- is a workspace_build - ( - audit_logs.resource_type = 'workspace_build' - AND audit_logs.resource_id = workspace_builds.id - ) - OR - -- Get the reason from the build #1 if this is the first - -- workspace create. - ( - audit_logs.resource_type = 'workspace' AND - audit_logs.action = 'create' AND - workspaces.id = workspace_builds.workspace_id AND - workspace_builds.build_number = 1 - ) - LEFT JOIN organizations ON audit_logs.organization_id = organizations.id +SELECT sqlc.embed(audit_logs), + -- sqlc.embed(users) would be nice but it does not seem to play well with + -- left joins. + users.username AS user_username, + users.name AS user_name, + users.email AS user_email, + users.created_at AS user_created_at, + users.updated_at AS user_updated_at, + users.last_seen_at AS user_last_seen_at, + users.status AS user_status, + users.login_type AS user_login_type, + users.rbac_roles AS user_roles, + users.avatar_url AS user_avatar_url, + users.deleted AS user_deleted, + users.quiet_hours_schedule AS user_quiet_hours_schedule, + COALESCE(organizations.name, '') AS organization_name, + COALESCE(organizations.display_name, '') AS organization_display_name, + COALESCE(organizations.icon, '') AS organization_icon +FROM audit_logs + LEFT JOIN users ON audit_logs.user_id = users.id + LEFT JOIN organizations ON audit_logs.organization_id = organizations.id + -- First join on workspaces to get the initial workspace create + -- to workspace build 1 id. This is because the first create is + -- is a different audit log than subsequent starts. + LEFT JOIN workspaces ON audit_logs.resource_type = 'workspace' + AND audit_logs.resource_id = workspaces.id + -- Get the reason from the build if the resource type + -- is a workspace_build + LEFT JOIN workspace_builds wb_build ON audit_logs.resource_type = 'workspace_build' + AND audit_logs.resource_id = wb_build.id + -- Get the reason from the build #1 if this is the first + -- workspace create. + LEFT JOIN workspace_builds wb_workspace ON audit_logs.resource_type = 'workspace' + AND audit_logs.action = 'create' + AND workspaces.id = wb_workspace.workspace_id + AND wb_workspace.build_number = 1 WHERE - -- Filter resource_type + -- Filter resource_type CASE - WHEN @resource_type :: text != '' THEN - resource_type = @resource_type :: resource_type + WHEN @resource_type::text != '' THEN resource_type = @resource_type::resource_type ELSE true END -- Filter resource_id AND CASE - WHEN @resource_id :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - resource_id = @resource_id + WHEN @resource_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN resource_id = @resource_id ELSE true END - -- Filter organization_id - AND CASE - WHEN @organization_id :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - audit_logs.organization_id = @organization_id + -- Filter organization_id + AND CASE + WHEN @organization_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.organization_id = @organization_id ELSE true END -- Filter by resource_target AND CASE - WHEN @resource_target :: text != '' THEN - resource_target = @resource_target + WHEN @resource_target::text != '' THEN resource_target = @resource_target ELSE true END -- Filter action AND CASE - WHEN @action :: text != '' THEN - action = @action :: audit_action + WHEN @action::text != '' THEN action = @action::audit_action ELSE true END -- Filter by user_id AND CASE - WHEN @user_id :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - user_id = @user_id + WHEN @user_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN user_id = @user_id ELSE true END -- Filter by username AND CASE - WHEN @username :: text != '' THEN - user_id = (SELECT id FROM users WHERE lower(username) = lower(@username) AND deleted = false) + WHEN @username::text != '' THEN user_id = ( + SELECT id + FROM users + WHERE lower(username) = lower(@username) + AND deleted = false + ) ELSE true END -- Filter by user_email AND CASE - WHEN @email :: text != '' THEN - users.email = @email + WHEN @email::text != '' THEN users.email = @email ELSE true END -- Filter by date_from AND CASE - WHEN @date_from :: timestamp with time zone != '0001-01-01 00:00:00Z' THEN - "time" >= @date_from + WHEN @date_from::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" >= @date_from ELSE true END -- Filter by date_to AND CASE - WHEN @date_to :: timestamp with time zone != '0001-01-01 00:00:00Z' THEN - "time" <= @date_to + WHEN @date_to::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" <= @date_to + ELSE true + END + -- Filter by build_reason + AND CASE + WHEN @build_reason::text != '' THEN COALESCE(wb_build.reason::text, wb_workspace.reason::text) = @build_reason ELSE true END - -- Filter by build_reason - AND CASE - WHEN @build_reason::text != '' THEN - workspace_builds.reason::text = @build_reason - ELSE true - END -- Filter request_id AND CASE - WHEN @request_id :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - audit_logs.request_id = @request_id + WHEN @request_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.request_id = @request_id ELSE true END - -- Authorize Filter clause will be injected below in GetAuthorizedAuditLogsOffset -- @authorize_filter -ORDER BY - "time" DESC -LIMIT - -- a limit of 0 means "no limit". The audit log table is unbounded +ORDER BY "time" DESC +LIMIT -- a limit of 0 means "no limit". The audit log table is unbounded -- in size, and is expected to be quite large. Implement a default -- limit of 100 to prevent accidental excessively large queries. - COALESCE(NULLIF(@limit_opt :: int, 0), 100) -OFFSET - @offset_opt; + COALESCE(NULLIF(@limit_opt::int, 0), 100) OFFSET @offset_opt; -- name: InsertAuditLog :one -INSERT INTO - audit_logs ( - id, - "time", - user_id, - organization_id, - ip, - user_agent, - resource_type, - resource_id, - resource_target, - action, - diff, - status_code, - additional_fields, - request_id, - resource_icon - ) -VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING *; +INSERT INTO audit_logs ( + id, + "time", + user_id, + organization_id, + ip, + user_agent, + resource_type, + resource_id, + resource_target, + action, + diff, + status_code, + additional_fields, + request_id, + resource_icon + ) +VALUES ( + $1, + $2, + $3, + $4, + $5, + $6, + $7, + $8, + $9, + $10, + $11, + $12, + $13, + $14, + $15 + ) +RETURNING *; + +-- name: CountAuditLogs :one +SELECT COUNT(*) +FROM audit_logs + LEFT JOIN users ON audit_logs.user_id = users.id + LEFT JOIN organizations ON audit_logs.organization_id = organizations.id + -- First join on workspaces to get the initial workspace create + -- to workspace build 1 id. This is because the first create is + -- is a different audit log than subsequent starts. + LEFT JOIN workspaces ON audit_logs.resource_type = 'workspace' + AND audit_logs.resource_id = workspaces.id + -- Get the reason from the build if the resource type + -- is a workspace_build + LEFT JOIN workspace_builds wb_build ON audit_logs.resource_type = 'workspace_build' + AND audit_logs.resource_id = wb_build.id + -- Get the reason from the build #1 if this is the first + -- workspace create. + LEFT JOIN workspace_builds wb_workspace ON audit_logs.resource_type = 'workspace' + AND audit_logs.action = 'create' + AND workspaces.id = wb_workspace.workspace_id + AND wb_workspace.build_number = 1 +WHERE + -- Filter resource_type + CASE + WHEN @resource_type::text != '' THEN resource_type = @resource_type::resource_type + ELSE true + END + -- Filter resource_id + AND CASE + WHEN @resource_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN resource_id = @resource_id + ELSE true + END + -- Filter organization_id + AND CASE + WHEN @organization_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.organization_id = @organization_id + ELSE true + END + -- Filter by resource_target + AND CASE + WHEN @resource_target::text != '' THEN resource_target = @resource_target + ELSE true + END + -- Filter action + AND CASE + WHEN @action::text != '' THEN action = @action::audit_action + ELSE true + END + -- Filter by user_id + AND CASE + WHEN @user_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN user_id = @user_id + ELSE true + END + -- Filter by username + AND CASE + WHEN @username::text != '' THEN user_id = ( + SELECT id + FROM users + WHERE lower(username) = lower(@username) + AND deleted = false + ) + ELSE true + END + -- Filter by user_email + AND CASE + WHEN @email::text != '' THEN users.email = @email + ELSE true + END + -- Filter by date_from + AND CASE + WHEN @date_from::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" >= @date_from + ELSE true + END + -- Filter by date_to + AND CASE + WHEN @date_to::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" <= @date_to + ELSE true + END + -- Filter by build_reason + AND CASE + WHEN @build_reason::text != '' THEN COALESCE(wb_build.reason::text, wb_workspace.reason::text) = @build_reason + ELSE true + END + -- Filter request_id + AND CASE + WHEN @request_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.request_id = @request_id + ELSE true + END + -- Authorize Filter clause will be injected below in CountAuthorizedAuditLogs + -- @authorize_filter +; diff --git a/coderd/searchquery/search.go b/coderd/searchquery/search.go index 721e593d4dd8d..634d4b6632ed3 100644 --- a/coderd/searchquery/search.go +++ b/coderd/searchquery/search.go @@ -33,7 +33,9 @@ import ( // - resource_type: string (enum) // - action: string (enum) // - build_reason: string (enum) -func AuditLogs(ctx context.Context, db database.Store, query string) (database.GetAuditLogsOffsetParams, []codersdk.ValidationError) { +func AuditLogs(ctx context.Context, db database.Store, query string) (database.GetAuditLogsOffsetParams, + database.CountAuditLogsParams, []codersdk.ValidationError, +) { // Always lowercase for all searches. query = strings.ToLower(query) values, errors := searchTerms(query, func(term string, values url.Values) error { @@ -41,7 +43,8 @@ func AuditLogs(ctx context.Context, db database.Store, query string) (database.G return nil }) if len(errors) > 0 { - return database.GetAuditLogsOffsetParams{}, errors + // nolint:exhaustruct // We don't need to initialize these structs because we return an error. + return database.GetAuditLogsOffsetParams{}, database.CountAuditLogsParams{}, errors } const dateLayout = "2006-01-02" @@ -63,8 +66,24 @@ func AuditLogs(ctx context.Context, db database.Store, query string) (database.G filter.DateTo = filter.DateTo.Add(23*time.Hour + 59*time.Minute + 59*time.Second) } + // Prepare the count filter, which uses the same parameters as the GetAuditLogsOffsetParams. + // nolint:exhaustruct // UserID is not obtained from the query parameters. + countFilter := database.CountAuditLogsParams{ + RequestID: filter.RequestID, + ResourceID: filter.ResourceID, + ResourceTarget: filter.ResourceTarget, + Username: filter.Username, + Email: filter.Email, + DateFrom: filter.DateFrom, + DateTo: filter.DateTo, + OrganizationID: filter.OrganizationID, + ResourceType: filter.ResourceType, + Action: filter.Action, + BuildReason: filter.BuildReason, + } + parser.ErrorExcessParams(values) - return filter, parser.Errors + return filter, countFilter, parser.Errors } func Users(query string) (database.GetUsersParams, []codersdk.ValidationError) { diff --git a/coderd/searchquery/search_test.go b/coderd/searchquery/search_test.go index dd879839e552f..2b7f4f402e008 100644 --- a/coderd/searchquery/search_test.go +++ b/coderd/searchquery/search_test.go @@ -343,6 +343,7 @@ func TestSearchAudit(t *testing.T) { Name string Query string Expected database.GetAuditLogsOffsetParams + ExpectedCountParams database.CountAuditLogsParams ExpectedErrorContains string }{ { @@ -372,6 +373,9 @@ func TestSearchAudit(t *testing.T) { Expected: database.GetAuditLogsOffsetParams{ ResourceTarget: "foo", }, + ExpectedCountParams: database.CountAuditLogsParams{ + ResourceTarget: "foo", + }, }, { Name: "RequestID", @@ -386,7 +390,7 @@ func TestSearchAudit(t *testing.T) { // Do not use a real database, this is only used for an // organization lookup. db := dbmem.New() - values, errs := searchquery.AuditLogs(context.Background(), db, c.Query) + values, countValues, errs := searchquery.AuditLogs(context.Background(), db, c.Query) if c.ExpectedErrorContains != "" { require.True(t, len(errs) > 0, "expect some errors") var s strings.Builder @@ -397,6 +401,7 @@ func TestSearchAudit(t *testing.T) { } else { require.Len(t, errs, 0, "expected no error") require.Equal(t, c.Expected, values, "expected values") + require.Equal(t, c.ExpectedCountParams, countValues, "expected count values") } }) } From c219a9a748728e2b62d8009651efa97e953cce92 Mon Sep 17 00:00:00 2001 From: Ethan <39577870+ethanndickson@users.noreply.github.com> Date: Thu, 7 Aug 2025 22:32:09 +1000 Subject: [PATCH 07/10] fix: use system context for querying workspaces when deleting users (#19211) (#19227) Backport of #19211 to our ESR, v2.24, since this is a kind of bad (albeit rare) bug and very easy to fix. ### Original PR Closes #19209. In `templates.go`, we do this to make sure we count ALL workspaces for a template before we try and delete that template: https://github.com/coder/coder/blob/dc598856e3be0926573dbbe2ec680e95a139093a/coderd/templates.go#L81-L99 However, we weren't doing the same when attempting to delete users, leading to the linked issue. We can solve the issue the same way as we do for templates. --- coderd/users.go | 5 ++++- coderd/users_test.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/coderd/users.go b/coderd/users.go index e2f6fd79c7d75..d43efcc661cb7 100644 --- a/coderd/users.go +++ b/coderd/users.go @@ -542,7 +542,10 @@ func (api *API) deleteUser(rw http.ResponseWriter, r *http.Request) { return } - workspaces, err := api.Database.GetWorkspaces(ctx, database.GetWorkspacesParams{ + // This query is ONLY done to get the workspace count, so we use a system + // context to return ALL workspaces. Not just workspaces the user can view. + // nolint:gocritic + workspaces, err := api.Database.GetWorkspaces(dbauthz.AsSystemRestricted(ctx), database.GetWorkspacesParams{ OwnerID: user.ID, }) if err != nil { diff --git a/coderd/users_test.go b/coderd/users_test.go index bd0f138b6a339..7b12c57ee1b98 100644 --- a/coderd/users_test.go +++ b/coderd/users_test.go @@ -378,6 +378,43 @@ func TestDeleteUser(t *testing.T) { require.ErrorAs(t, err, &apiErr, "should be a coderd error") require.Equal(t, http.StatusForbidden, apiErr.StatusCode(), "should be forbidden") }) + t.Run("CountCheckIncludesAllWorkspaces", func(t *testing.T) { + t.Parallel() + client, _ := coderdtest.NewWithProvisionerCloser(t, nil) + firstUser := coderdtest.CreateFirstUser(t, client) + + // Create a target user who will own a workspace + targetUserClient, targetUser := coderdtest.CreateAnotherUser(t, client, firstUser.OrganizationID) + + // Create a User Admin who should not have permission to see the target user's workspace + userAdminClient, userAdmin := coderdtest.CreateAnotherUser(t, client, firstUser.OrganizationID) + + // Grant User Admin role to the userAdmin + userAdmin, err := client.UpdateUserRoles(context.Background(), userAdmin.ID.String(), codersdk.UpdateRoles{ + Roles: []string{rbac.RoleUserAdmin().String()}, + }) + require.NoError(t, err) + + // Create a template and workspace owned by the target user + version := coderdtest.CreateTemplateVersion(t, client, firstUser.OrganizationID, nil) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) + template := coderdtest.CreateTemplate(t, client, firstUser.OrganizationID, version.ID) + _ = coderdtest.CreateWorkspace(t, targetUserClient, template.ID) + + workspaces, err := userAdminClient.Workspaces(context.Background(), codersdk.WorkspaceFilter{ + Owner: targetUser.Username, + }) + require.NoError(t, err) + require.Len(t, workspaces.Workspaces, 0) + + // Attempt to delete the target user - this should fail because the + // user has a workspace not visible to the deleting user. + err = userAdminClient.DeleteUser(context.Background(), targetUser.ID) + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusExpectationFailed, apiErr.StatusCode()) + require.Contains(t, apiErr.Message, "has workspaces") + }) } func TestNotifyUserStatusChanged(t *testing.T) { From 1e284fe835a894cf1ada451ac0452e1b31dd3ce2 Mon Sep 17 00:00:00 2001 From: Jakub Domeracki Date: Thu, 7 Aug 2025 17:42:01 +0200 Subject: [PATCH 08/10] chore: fix CLI binary publishing for releases.coder.com (#19231) --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d032cd3e8731c..1b3678fe3f09d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -635,7 +635,7 @@ jobs: run: ls -lh build - name: Publish Coder CLI binaries and detached signatures to GCS - if: ${{ !inputs.dry_run && github.ref == 'refs/heads/main' && github.repository_owner == 'coder'}} + if: ${{ !inputs.dry_run }} run: | set -euxo pipefail From 1be409cf2e38274454585df2f40d76e5a5ea0b55 Mon Sep 17 00:00:00 2001 From: Jakub Domeracki Date: Thu, 7 Aug 2025 18:05:50 +0200 Subject: [PATCH 09/10] chore: revert CLI binary publishing for releases.coder.com (#19233) --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1b3678fe3f09d..d032cd3e8731c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -635,7 +635,7 @@ jobs: run: ls -lh build - name: Publish Coder CLI binaries and detached signatures to GCS - if: ${{ !inputs.dry_run }} + if: ${{ !inputs.dry_run && github.ref == 'refs/heads/main' && github.repository_owner == 'coder'}} run: | set -euxo pipefail From b1e8f0a7fcfebf7b17a234980fdb264b5b94cb86 Mon Sep 17 00:00:00 2001 From: Ethan <39577870+ethanndickson@users.noreply.github.com> Date: Tue, 12 Aug 2025 22:30:58 +1000 Subject: [PATCH 10/10] ci: fix gcp service accounts (#19312) (#19314) Backport of #19312 --- .github/workflows/ci.yaml | 16 ++++++++-------- .github/workflows/dogfood.yaml | 4 ++-- .github/workflows/pr-deploy.yaml | 2 +- .github/workflows/release.yaml | 8 ++++---- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8e442e4a539ca..9c023c0ecf52f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -256,8 +256,8 @@ jobs: pushd /tmp/proto curl -L -o protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v23.4/protoc-23.4-linux-x86_64.zip unzip protoc.zip - cp -r ./bin/* /usr/local/bin - cp -r ./include /usr/local/bin/include + sudo cp -r ./bin/* /usr/local/bin + sudo cp -r ./include /usr/local/bin/include popd - name: make gen @@ -988,8 +988,8 @@ jobs: pushd /tmp/proto curl -L -o protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v23.4/protoc-23.4-linux-x86_64.zip unzip protoc.zip - cp -r ./bin/* /usr/local/bin - cp -r ./include /usr/local/bin/include + sudo cp -r ./bin/* /usr/local/bin + sudo cp -r ./include /usr/local/bin/include popd - name: Setup Go @@ -1225,8 +1225,8 @@ jobs: id: gcloud_auth uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10 with: - workload_identity_provider: ${{ secrets.GCP_CODE_SIGNING_WORKLOAD_ID_PROVIDER }} - service_account: ${{ secrets.GCP_CODE_SIGNING_SERVICE_ACCOUNT }} + workload_identity_provider: ${{ vars.GCP_CODE_SIGNING_WORKLOAD_ID_PROVIDER }} + service_account: ${{ vars.GCP_CODE_SIGNING_SERVICE_ACCOUNT }} token_format: "access_token" - name: Setup GCloud SDK @@ -1526,8 +1526,8 @@ jobs: - name: Authenticate to Google Cloud uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10 with: - workload_identity_provider: projects/573722524737/locations/global/workloadIdentityPools/github/providers/github - service_account: coder-ci@coder-dogfood.iam.gserviceaccount.com + workload_identity_provider: ${{ vars.GCP_WORKLOAD_ID_PROVIDER }} + service_account: ${{ vars.GCP_SERVICE_ACCOUNT }} - name: Set up Google Cloud SDK uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4 diff --git a/.github/workflows/dogfood.yaml b/.github/workflows/dogfood.yaml index 952e31b7c98ec..9c67bee45fa95 100644 --- a/.github/workflows/dogfood.yaml +++ b/.github/workflows/dogfood.yaml @@ -131,8 +131,8 @@ jobs: - name: Authenticate to Google Cloud uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10 with: - workload_identity_provider: projects/573722524737/locations/global/workloadIdentityPools/github/providers/github - service_account: coder-ci@coder-dogfood.iam.gserviceaccount.com + workload_identity_provider: ${{ vars.GCP_WORKLOAD_ID_PROVIDER }} + service_account: ${{ vars.GCP_SERVICE_ACCOUNT }} - name: Terraform init and validate run: | diff --git a/.github/workflows/pr-deploy.yaml b/.github/workflows/pr-deploy.yaml index fe64c47aebc6e..e9ff759331a3f 100644 --- a/.github/workflows/pr-deploy.yaml +++ b/.github/workflows/pr-deploy.yaml @@ -420,7 +420,7 @@ jobs: curl -fsSL "$URL" -o "${DEST}" chmod +x "${DEST}" "${DEST}" version - mv "${DEST}" /usr/local/bin/coder + sudo mv "${DEST}" /usr/local/bin/coder - name: Create first user if: needs.get_info.outputs.NEW == 'true' || github.event.inputs.deploy == 'true' diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d032cd3e8731c..ddd78a8306228 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -288,8 +288,8 @@ jobs: id: gcloud_auth uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10 with: - workload_identity_provider: ${{ secrets.GCP_CODE_SIGNING_WORKLOAD_ID_PROVIDER }} - service_account: ${{ secrets.GCP_CODE_SIGNING_SERVICE_ACCOUNT }} + workload_identity_provider: ${{ vars.GCP_CODE_SIGNING_WORKLOAD_ID_PROVIDER }} + service_account: ${{ vars.GCP_CODE_SIGNING_SERVICE_ACCOUNT }} token_format: "access_token" - name: Setup GCloud SDK @@ -698,8 +698,8 @@ jobs: - name: Authenticate to Google Cloud uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10 with: - workload_identity_provider: ${{ secrets.GCP_WORKLOAD_ID_PROVIDER }} - service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} + workload_identity_provider: ${{ vars.GCP_WORKLOAD_ID_PROVIDER }} + service_account: ${{ vars.GCP_SERVICE_ACCOUNT }} - name: Setup GCloud SDK uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # 2.1.4