Skip to content

feat(support): fetch data concurrently #12385

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 5, 2024
Merged

feat(support): fetch data concurrently #12385

Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
512f404
concurrent fetch
johnstcn Feb 29, 2024
b0cd05a
apply suggestions from pr
johnstcn Mar 5, 2024
a1e1dc4
do not early exit on error
johnstcn Mar 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
204 changes: 135 additions & 69 deletions support/support.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ import (
"io"
"net/http"
"strings"
"sync"

"golang.org/x/sync/errgroup"
"golang.org/x/xerrors"

"github.com/google/uuid"
Expand Down Expand Up @@ -61,34 +63,47 @@ type Deps struct {
}

func DeploymentInfo(ctx context.Context, client *codersdk.Client, log slog.Logger) Deployment {
// Note: each goroutine assigns to a different struct field, hence no mutex.
var d Deployment

bi, err := client.BuildInfo(ctx)
if err != nil {
log.Error(ctx, "fetch build info", slog.Error(err))
} else {
eg, ctx := errgroup.WithContext(ctx)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just realized we only log errors, so we might be interested in partial results?

If so, might make sense to just use eg := &errgroup.Group{}. This way the other Go routines won't be interrupted when one runs into an error. (Sorry I didn't take this into account in my previous review.)

eg.Go(func() error {
bi, err := client.BuildInfo(ctx)
if err != nil {
return xerrors.Errorf("fetch build info: %w", err)
}
d.BuildInfo = &bi
}
return nil
})

dc, err := client.DeploymentConfig(ctx)
if err != nil {
log.Error(ctx, "fetch deployment config", slog.Error(err))
} else {
eg.Go(func() error {
dc, err := client.DeploymentConfig(ctx)
if err != nil {
return xerrors.Errorf("fetch deployment config: %w", err)
}
d.Config = dc
}
return nil
})

hr, err := client.DebugHealth(ctx)
if err != nil {
log.Error(ctx, "fetch health report", slog.Error(err))
} else {
eg.Go(func() error {
hr, err := client.DebugHealth(ctx)
if err != nil {
return xerrors.Errorf("fetch health report: %w", err)
}
d.HealthReport = &hr
}
return nil
})

exp, err := client.Experiments(ctx)
if err != nil {
log.Error(ctx, "fetch experiments", slog.Error(err))
} else {
eg.Go(func() error {
exp, err := client.Experiments(ctx)
if err != nil {
return xerrors.Errorf("fetch experiments: %w", err)
}
d.Experiments = exp
return nil
})

if err := eg.Wait(); err != nil {
log.Error(ctx, "fetch deployment information", slog.Error(err))
}

return d
Expand All @@ -97,41 +112,50 @@ func DeploymentInfo(ctx context.Context, client *codersdk.Client, log slog.Logge
func NetworkInfo(ctx context.Context, client *codersdk.Client, log slog.Logger, agentID uuid.UUID) Network {
var n Network

coordResp, err := client.Request(ctx, http.MethodGet, "/api/v2/debug/coordinator", nil)
if err != nil {
log.Error(ctx, "fetch coordinator debug page", slog.Error(err))
} else {
eg, ctx := errgroup.WithContext(ctx)
eg.Go(func() error {
coordResp, err := client.Request(ctx, http.MethodGet, "/api/v2/debug/coordinator", nil)
if err != nil {
return xerrors.Errorf("fetch coordinator debug page: %w", err)
}
defer coordResp.Body.Close()
bs, err := io.ReadAll(coordResp.Body)
if err != nil {
log.Error(ctx, "read coordinator debug page", slog.Error(err))
} else {
n.CoordinatorDebug = string(bs)
return xerrors.Errorf("read coordinator debug page: %w", err)
}
}
n.CoordinatorDebug = string(bs)
return nil
})

tailResp, err := client.Request(ctx, http.MethodGet, "/api/v2/debug/tailnet", nil)
if err != nil {
log.Error(ctx, "fetch tailnet debug page", slog.Error(err))
} else {
eg.Go(func() error {
tailResp, err := client.Request(ctx, http.MethodGet, "/api/v2/debug/tailnet", nil)
if err != nil {
return xerrors.Errorf("fetch tailnet debug page: %w", err)
}
defer tailResp.Body.Close()
bs, err := io.ReadAll(tailResp.Body)
if err != nil {
log.Error(ctx, "read tailnet debug page", slog.Error(err))
} else {
n.TailnetDebug = string(bs)
return xerrors.Errorf("read tailnet debug page: %w", err)
}
n.TailnetDebug = string(bs)
return nil
})

eg.Go(func() error {
if agentID == uuid.Nil {
log.Warn(ctx, "agent id required for agent connection info")
return nil
}
}

if agentID != uuid.Nil {
connInfo, err := client.WorkspaceAgentConnectionInfo(ctx, agentID)
if err != nil {
log.Error(ctx, "fetch agent conn info", slog.Error(err), slog.F("agent_id", agentID.String()))
} else {
n.NetcheckLocal = &connInfo
return xerrors.Errorf("fetch agent conn info: %w", err)
}
} else {
log.Warn(ctx, "agent id required for agent connection info")
n.NetcheckLocal = &connInfo
return nil
})

if err := eg.Wait(); err != nil {
log.Error(ctx, "fetch network information", slog.Error(err))
}

return n
Expand All @@ -149,43 +173,59 @@ func WorkspaceInfo(ctx context.Context, client *codersdk.Client, log slog.Logger
log.Error(ctx, "no agent id specified")
}

// dependency, cannot fetch concurrently
ws, err := client.Workspace(ctx, workspaceID)
if err != nil {
log.Error(ctx, "fetch workspace", slog.Error(err), slog.F("workspace_id", workspaceID))
return w
}
w.Workspace = ws

agt, err := client.WorkspaceAgent(ctx, agentID)
if err != nil {
log.Error(ctx, "fetch workspace agent", slog.Error(err), slog.F("agent_id", agentID))
}
eg, ctx := errgroup.WithContext(ctx)

w.Workspace = ws
w.Agent = agt
eg.Go(func() error {
agt, err := client.WorkspaceAgent(ctx, agentID)
if err != nil {
return xerrors.Errorf("fetch workspace agent: %w", err)
}
w.Agent = agt
return nil
})

buildLogCh, closer, err := client.WorkspaceBuildLogsAfter(ctx, ws.LatestBuild.ID, 0)
if err != nil {
log.Error(ctx, "fetch provisioner job logs", slog.Error(err), slog.F("job_id", ws.LatestBuild.Job.ID.String()))
} else {
eg.Go(func() error {
buildLogCh, closer, err := client.WorkspaceBuildLogsAfter(ctx, ws.LatestBuild.ID, 0)
if err != nil {
return xerrors.Errorf("fetch provisioner job logs: %w", err)
}
defer closer.Close()
var logs []codersdk.ProvisionerJobLog
for log := range buildLogCh {
w.BuildLogs = append(w.BuildLogs, log)
logs = append(w.BuildLogs, log)
}
w.BuildLogs = logs
return nil
})

eg.Go(func() error {
if len(w.Workspace.LatestBuild.Resources) == 0 {
log.Warn(ctx, "workspace build has no resources")
return nil
}
agentLogCh, closer, err := client.WorkspaceAgentLogsAfter(ctx, agentID, 0, false)
if err != nil {
return xerrors.Errorf("fetch agent startup logs: %w", err)
}
}

if len(w.Workspace.LatestBuild.Resources) == 0 {
log.Warn(ctx, "workspace build has no resources")
return w
}

agentLogCh, closer, err := client.WorkspaceAgentLogsAfter(ctx, agentID, 0, false)
if err != nil {
log.Error(ctx, "fetch agent startup logs", slog.Error(err), slog.F("agent_id", agentID.String()))
} else {
defer closer.Close()
var logs []codersdk.WorkspaceAgentLog
for logChunk := range agentLogCh {
w.AgentStartupLogs = append(w.AgentStartupLogs, logChunk...)
logs = append(w.AgentStartupLogs, logChunk...)
}
w.AgentStartupLogs = logs
return nil
})

if err := eg.Wait(); err != nil {
log.Error(ctx, "fetch workspace information", slog.Error(err))
}

return w
Expand Down Expand Up @@ -225,9 +265,35 @@ func Run(ctx context.Context, d *Deps) (*Bundle, error) {
}
}

b.Deployment = DeploymentInfo(ctx, d.Client, d.Log)
b.Workspace = WorkspaceInfo(ctx, d.Client, d.Log, d.WorkspaceID, d.AgentID)
b.Network = NetworkInfo(ctx, d.Client, d.Log, d.AgentID)
var (
wg sync.WaitGroup
m sync.Mutex
)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Switch for errgroup too?

wg.Add(1)
go func() {
defer wg.Done()
di := DeploymentInfo(ctx, d.Client, d.Log)
m.Lock()
b.Deployment = di
m.Unlock()
}()
wg.Add(1)
go func() {
defer wg.Done()
wi := WorkspaceInfo(ctx, d.Client, d.Log, d.WorkspaceID, d.AgentID)
m.Lock()
b.Workspace = wi
m.Unlock()
}()
wg.Add(1)
go func() {
defer wg.Done()
ni := NetworkInfo(ctx, d.Client, d.Log, d.AgentID)
m.Lock()
b.Network = ni
m.Unlock()
}()
wg.Wait()

return &b, nil
}