From 17e99001f2635bae37f234aafb8a522c4790c70b Mon Sep 17 00:00:00 2001 From: kylecarbs Date: Sun, 12 Jun 2022 23:09:57 +0000 Subject: [PATCH 1/7] feat: Add anonymized telemetry to report product usage This adds a background service to report telemetry to a Coder server for usage data. There will be realtime event data sent in the future, but for now usage will report on a CRON. --- cli/server.go | 46 ++ cli/server_test.go | 37 ++ coderd/coderd.go | 2 + coderd/database/databasefake/databasefake.go | 117 +++- coderd/database/dump.sql | 8 + .../migrations/000023_site_config.down.sql | 1 + .../migrations/000023_site_config.up.sql | 4 + coderd/database/models.go | 5 + coderd/database/querier.go | 13 +- coderd/database/queries.sql.go | 476 +++++++++++--- coderd/database/queries/parameterschemas.sql | 3 + coderd/database/queries/provisionerjobs.sql | 3 + coderd/database/queries/siteconfig.sql | 5 + coderd/database/queries/templates.sql | 3 + coderd/database/queries/templateversions.sql | 3 + coderd/database/queries/workspaceagents.sql | 3 + coderd/database/queries/workspaceapps.sql | 3 + coderd/database/queries/workspacebuilds.sql | 3 + .../database/queries/workspaceresources.sql | 3 + coderd/database/queries/workspaces.sql | 2 +- coderd/telemetry/telemetry.go | 585 ++++++++++++++++++ coderd/telemetry/telemetry_test.go | 141 +++++ coderd/workspaces.go | 4 +- go.mod | 7 + go.sum | 9 + 25 files changed, 1412 insertions(+), 74 deletions(-) create mode 100644 coderd/database/migrations/000023_site_config.down.sql create mode 100644 coderd/database/migrations/000023_site_config.up.sql create mode 100644 coderd/database/queries/siteconfig.sql create mode 100644 coderd/telemetry/telemetry.go create mode 100644 coderd/telemetry/telemetry_test.go diff --git a/cli/server.go b/cli/server.go index b69c6c5c83489..19e3a2b7aed83 100644 --- a/cli/server.go +++ b/cli/server.go @@ -25,6 +25,7 @@ import ( "github.com/briandowns/spinner" "github.com/coreos/go-systemd/daemon" "github.com/google/go-github/v43/github" + "github.com/google/uuid" "github.com/pion/turn/v2" "github.com/pion/webrtc/v3" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -49,6 +50,7 @@ import ( "github.com/coder/coder/coderd/database/databasefake" "github.com/coder/coder/coderd/devtunnel" "github.com/coder/coder/coderd/gitsshkey" + "github.com/coder/coder/coderd/telemetry" "github.com/coder/coder/coderd/tracing" "github.com/coder/coder/coderd/turnconn" "github.com/coder/coder/codersdk" @@ -80,6 +82,8 @@ func server() *cobra.Command { oauth2GithubClientSecret string oauth2GithubAllowedOrganizations []string oauth2GithubAllowSignups bool + telemetryEnabled bool + telemetryURL string tlsCertFile string tlsClientCAFile string tlsClientAuth string @@ -301,6 +305,45 @@ func server() *cobra.Command { } } + deploymentID, err := options.Database.GetDeploymentID(cmd.Context()) + if errors.Is(err, sql.ErrNoRows) { + err = nil + } + if err != nil { + return xerrors.Errorf("get deployment id: %w", err) + } + if deploymentID == "" { + deploymentID = uuid.NewString() + err = options.Database.InsertDeploymentID(cmd.Context(), deploymentID) + if err != nil { + return xerrors.Errorf("set deployment id: %w", err) + } + } + + // Parse the raw telemetry URL! + telemetryURL, err := url.Parse(telemetryURL) + if err != nil { + return xerrors.Errorf("parse telemetry url: %w", err) + } + // Disable telemetry if in dev-mode. If the telemetry flag + // is manually specified, override this behavior! + if buildModeDev && !cmd.Flags().Changed("telemetry") { + telemetryEnabled = false + } + reporter, err := telemetry.New(telemetry.Options{ + DeploymentID: deploymentID, + Database: options.Database, + Logger: logger.Named("telemetry"), + URL: telemetryURL, + DevMode: dev, + Disabled: !telemetryEnabled, + }) + if err != nil { + return xerrors.Errorf("create telemetry reporter: %w", err) + } + defer reporter.Close() + options.Telemetry = reporter + coderAPI := coderd.New(options) client := codersdk.New(localURL) if tlsEnable { @@ -537,6 +580,9 @@ func server() *cobra.Command { "Specifies organizations the user must be a member of to authenticate with GitHub.") cliflag.BoolVarP(root.Flags(), &oauth2GithubAllowSignups, "oauth2-github-allow-signups", "", "CODER_OAUTH2_GITHUB_ALLOW_SIGNUPS", false, "Specifies whether new users can sign up with GitHub.") + cliflag.BoolVarP(root.Flags(), &telemetryEnabled, "telemetry", "", "CODER_TELEMETRY", true, "Specifies whether telemetry is enabled or not. Coder collects anonymized usage data to help improve our product!") + cliflag.StringVarP(root.Flags(), &telemetryURL, "telemetry-url", "", "CODER_TELEMETRY_URL", "https://telemetry.coder.com", "Specifies a URL to send telemetry to.") + _ = root.Flags().MarkHidden("telemetry-url") cliflag.BoolVarP(root.Flags(), &tlsEnable, "tls-enable", "", "CODER_TLS_ENABLE", false, "Specifies if TLS will be enabled") cliflag.StringVarP(root.Flags(), &tlsCertFile, "tls-cert-file", "", "CODER_TLS_CERT_FILE", "", "Specifies the path to the certificate for TLS. It requires a PEM-encoded file. "+ diff --git a/cli/server_test.go b/cli/server_test.go index 7af01138e8c2c..160bf978851c7 100644 --- a/cli/server_test.go +++ b/cli/server_test.go @@ -8,11 +8,13 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "encoding/json" "encoding/pem" "fmt" "math/big" "net" "net/http" + "net/http/httptest" "net/url" "os" "runtime" @@ -20,6 +22,7 @@ import ( "testing" "time" + "github.com/go-chi/chi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/goleak" @@ -27,6 +30,7 @@ import ( "github.com/coder/coder/cli/clitest" "github.com/coder/coder/coderd/coderdtest" "github.com/coder/coder/coderd/database/postgres" + "github.com/coder/coder/coderd/telemetry" "github.com/coder/coder/codersdk" ) @@ -322,6 +326,39 @@ func TestServer(t *testing.T) { require.ErrorIs(t, <-errC, context.Canceled) require.Error(t, goleak.Find()) }) + t.Run("Telemetry", func(t *testing.T) { + t.Parallel() + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + deployment := make(chan struct{}, 64) + snapshot := make(chan *telemetry.Snapshot, 64) + r := chi.NewRouter() + r.Post("/deployment", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + deployment <- struct{}{} + }) + r.Post("/snapshot", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + ss := &telemetry.Snapshot{} + err := json.NewDecoder(r.Body).Decode(ss) + require.NoError(t, err) + snapshot <- ss + }) + server := httptest.NewServer(r) + t.Cleanup(server.Close) + + root, _ := clitest.New(t, "server", "--dev", "--tunnel=false", "--address", ":0", "--telemetry", "true", "--telemetry-url", server.URL) + var buf strings.Builder + errC := make(chan error) + root.SetOutput(&buf) + go func() { + errC <- root.ExecuteContext(ctx) + }() + + <-deployment + <-snapshot + }) } func generateTLSCertificate(t testing.TB) (certPath, keyPath string) { diff --git a/coderd/coderd.go b/coderd/coderd.go index 2cd94d1eac646..63edd27fd0a9b 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -25,6 +25,7 @@ import ( "github.com/coder/coder/coderd/httpapi" "github.com/coder/coder/coderd/httpmw" "github.com/coder/coder/coderd/rbac" + "github.com/coder/coder/coderd/telemetry" "github.com/coder/coder/coderd/tracing" "github.com/coder/coder/coderd/turnconn" "github.com/coder/coder/coderd/wsconncache" @@ -52,6 +53,7 @@ type Options struct { ICEServers []webrtc.ICEServer SecureAuthCookie bool SSHKeygenAlgorithm gitsshkey.Algorithm + Telemetry *telemetry.Reporter TURNServer *turnconn.Server TracerProvider *sdktrace.TracerProvider } diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index fcc214a5745b7..6c30169bfb4ee 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -66,6 +66,8 @@ type fakeQuerier struct { workspaceBuilds []database.WorkspaceBuild workspaceApps []database.WorkspaceApp workspaces []database.Workspace + + deploymentID string } // InTx doesn't rollback data properly for in-memory yet. @@ -315,7 +317,7 @@ func (q *fakeQuerier) GetAuthorizationUserRoles(_ context.Context, userID uuid.U }, nil } -func (q *fakeQuerier) GetWorkspacesWithFilter(_ context.Context, arg database.GetWorkspacesWithFilterParams) ([]database.Workspace, error) { +func (q *fakeQuerier) GetWorkspaces(_ context.Context, arg database.GetWorkspacesParams) ([]database.Workspace, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -415,6 +417,19 @@ func (q *fakeQuerier) GetWorkspaceAppsByAgentID(_ context.Context, id uuid.UUID) return apps, nil } +func (q *fakeQuerier) GetWorkspaceAppsCreatedAfter(_ context.Context, after time.Time) ([]database.WorkspaceApp, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + apps := make([]database.WorkspaceApp, 0) + for _, app := range q.workspaceApps { + if app.CreatedAt.After(after) { + apps = append(apps, app) + } + } + return apps, nil +} + func (q *fakeQuerier) GetWorkspaceAppsByAgentIDs(_ context.Context, ids []uuid.UUID) ([]database.WorkspaceApp, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -641,6 +656,19 @@ func (q *fakeQuerier) GetWorkspaceBuildByWorkspaceIDAndBuildNumber(_ context.Con return database.WorkspaceBuild{}, sql.ErrNoRows } +func (q *fakeQuerier) GetWorkspaceBuildsCreatedAfter(_ context.Context, after time.Time) ([]database.WorkspaceBuild, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + workspaceBuilds := make([]database.WorkspaceBuild, 0) + for _, workspaceBuild := range q.workspaceBuilds { + if workspaceBuild.CreatedAt.After(after) { + workspaceBuilds = append(workspaceBuilds, workspaceBuild) + } + } + return workspaceBuilds, nil +} + func (q *fakeQuerier) GetWorkspacesByOrganizationIDs(_ context.Context, req database.GetWorkspacesByOrganizationIDsParams) ([]database.Workspace, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -845,6 +873,19 @@ func (q *fakeQuerier) GetTemplateVersionsByTemplateID(_ context.Context, arg dat return version, nil } +func (q *fakeQuerier) GetTemplateVersionsCreatedAfter(_ context.Context, after time.Time) ([]database.TemplateVersion, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + versions := make([]database.TemplateVersion, 0) + for _, version := range q.templateVersions { + if version.CreatedAt.After(after) { + versions = append(versions, version) + } + } + return versions, nil +} + func (q *fakeQuerier) GetTemplateVersionByTemplateIDAndName(_ context.Context, arg database.GetTemplateVersionByTemplateIDAndNameParams) (database.TemplateVersion, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -904,6 +945,19 @@ func (q *fakeQuerier) GetParameterSchemasByJobID(_ context.Context, jobID uuid.U return parameters, nil } +func (q *fakeQuerier) GetParameterSchemasCreatedAfter(_ context.Context, after time.Time) ([]database.ParameterSchema, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + parameters := make([]database.ParameterSchema, 0) + for _, parameterSchema := range q.parameterSchemas { + if parameterSchema.CreatedAt.After(after) { + parameters = append(parameters, parameterSchema) + } + } + return parameters, nil +} + func (q *fakeQuerier) GetParameterValueByScopeAndName(_ context.Context, arg database.GetParameterValueByScopeAndNameParams) (database.ParameterValue, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -923,6 +977,13 @@ func (q *fakeQuerier) GetParameterValueByScopeAndName(_ context.Context, arg dat return database.ParameterValue{}, sql.ErrNoRows } +func (q *fakeQuerier) GetTemplates(_ context.Context) ([]database.Template, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + return q.templates, nil +} + func (q *fakeQuerier) GetTemplatesByOrganization(_ context.Context, arg database.GetTemplatesByOrganizationParams) ([]database.Template, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -1109,6 +1170,19 @@ func (q *fakeQuerier) GetWorkspaceAgentsByResourceIDs(_ context.Context, resourc return workspaceAgents, nil } +func (q *fakeQuerier) GetWorkspaceAgentsCreatedAfter(_ context.Context, after time.Time) ([]database.WorkspaceAgent, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + workspaceAgents := make([]database.WorkspaceAgent, 0) + for _, agent := range q.provisionerJobAgents { + if agent.CreatedAt.After(after) { + workspaceAgents = append(workspaceAgents, agent) + } + } + return workspaceAgents, nil +} + func (q *fakeQuerier) GetWorkspaceAppByAgentIDAndName(_ context.Context, arg database.GetWorkspaceAppByAgentIDAndNameParams) (database.WorkspaceApp, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -1180,6 +1254,19 @@ func (q *fakeQuerier) GetWorkspaceResourcesByJobID(_ context.Context, jobID uuid return resources, nil } +func (q *fakeQuerier) GetWorkspaceResourcesCreatedAfter(_ context.Context, after time.Time) ([]database.WorkspaceResource, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + resources := make([]database.WorkspaceResource, 0) + for _, resource := range q.provisionerJobResources { + if resource.CreatedAt.After(after) { + resources = append(resources, resource) + } + } + return resources, nil +} + func (q *fakeQuerier) GetProvisionerJobsByIDs(_ context.Context, ids []uuid.UUID) ([]database.ProvisionerJob, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -1200,6 +1287,19 @@ func (q *fakeQuerier) GetProvisionerJobsByIDs(_ context.Context, ids []uuid.UUID return jobs, nil } +func (q *fakeQuerier) GetProvisionerJobsCreatedAfter(_ context.Context, after time.Time) ([]database.ProvisionerJob, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + jobs := make([]database.ProvisionerJob, 0) + for _, job := range q.provisionerJobs { + if job.CreatedAt.After(after) { + jobs = append(jobs, job) + } + } + return jobs, nil +} + func (q *fakeQuerier) GetProvisionerLogsByIDBetween(_ context.Context, arg database.GetProvisionerLogsByIDBetweenParams) ([]database.ProvisionerJobLog, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -1980,3 +2080,18 @@ func (q *fakeQuerier) InsertAuditLog(_ context.Context, arg database.InsertAudit return alog, nil } + +func (q *fakeQuerier) InsertDeploymentID(_ context.Context, id string) error { + q.mutex.Lock() + defer q.mutex.Unlock() + + q.deploymentID = id + return nil +} + +func (q *fakeQuerier) GetDeploymentID(_ context.Context) (string, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + return q.deploymentID, nil +} diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 95ef9ff0df3d6..6dbd041c5192c 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -226,6 +226,11 @@ CREATE TABLE provisioner_jobs ( worker_id uuid ); +CREATE TABLE site_config ( + key character varying(256) NOT NULL, + value character varying(8192) NOT NULL +); + CREATE TABLE template_versions ( id uuid NOT NULL, template_id uuid, @@ -378,6 +383,9 @@ ALTER TABLE ONLY provisioner_job_logs ALTER TABLE ONLY provisioner_jobs ADD CONSTRAINT provisioner_jobs_pkey PRIMARY KEY (id); +ALTER TABLE ONLY site_config + ADD CONSTRAINT site_config_key_key UNIQUE (key); + ALTER TABLE ONLY template_versions ADD CONSTRAINT template_versions_pkey PRIMARY KEY (id); diff --git a/coderd/database/migrations/000023_site_config.down.sql b/coderd/database/migrations/000023_site_config.down.sql new file mode 100644 index 0000000000000..b5af43f645d46 --- /dev/null +++ b/coderd/database/migrations/000023_site_config.down.sql @@ -0,0 +1 @@ +DROP TABLE site_config; diff --git a/coderd/database/migrations/000023_site_config.up.sql b/coderd/database/migrations/000023_site_config.up.sql new file mode 100644 index 0000000000000..16e0b617dd494 --- /dev/null +++ b/coderd/database/migrations/000023_site_config.up.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS site_config ( + key varchar(256) NOT NULL UNIQUE, + value varchar(8192) NOT NULL +); diff --git a/coderd/database/models.go b/coderd/database/models.go index 2454049da0ac0..f77d40c6631fb 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -426,6 +426,11 @@ type ProvisionerJobLog struct { Output string `db:"output" json:"output"` } +type SiteConfig struct { + Key string `db:"key" json:"key"` + Value string `db:"value" json:"value"` +} + type Template struct { ID uuid.UUID `db:"id" json:"id"` CreatedAt time.Time `db:"created_at" json:"created_at"` diff --git a/coderd/database/querier.go b/coderd/database/querier.go index c3f57d3a9f795..a18fb8e4a4f85 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -6,6 +6,7 @@ package database import ( "context" + "time" "github.com/google/uuid" ) @@ -28,6 +29,7 @@ type querier interface { // This function returns roles for authorization purposes. Implied member roles // are included. GetAuthorizationUserRoles(ctx context.Context, userID uuid.UUID) (GetAuthorizationUserRolesRow, error) + GetDeploymentID(ctx context.Context) (string, error) GetFileByHash(ctx context.Context, hash string) (File, error) GetGitSSHKey(ctx context.Context, userID uuid.UUID) (GitSSHKey, error) GetLatestWorkspaceBuildByWorkspaceID(ctx context.Context, workspaceID uuid.UUID) (WorkspaceBuild, error) @@ -40,12 +42,14 @@ type querier interface { GetOrganizations(ctx context.Context) ([]Organization, error) GetOrganizationsByUserID(ctx context.Context, userID uuid.UUID) ([]Organization, error) GetParameterSchemasByJobID(ctx context.Context, jobID uuid.UUID) ([]ParameterSchema, error) + GetParameterSchemasCreatedAfter(ctx context.Context, createdAt time.Time) ([]ParameterSchema, error) GetParameterValueByScopeAndName(ctx context.Context, arg GetParameterValueByScopeAndNameParams) (ParameterValue, error) GetParameterValuesByScope(ctx context.Context, arg GetParameterValuesByScopeParams) ([]ParameterValue, error) GetProvisionerDaemonByID(ctx context.Context, id uuid.UUID) (ProvisionerDaemon, error) GetProvisionerDaemons(ctx context.Context) ([]ProvisionerDaemon, error) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (ProvisionerJob, error) GetProvisionerJobsByIDs(ctx context.Context, ids []uuid.UUID) ([]ProvisionerJob, error) + GetProvisionerJobsCreatedAfter(ctx context.Context, createdAt time.Time) ([]ProvisionerJob, error) GetProvisionerLogsByIDBetween(ctx context.Context, arg GetProvisionerLogsByIDBetweenParams) ([]ProvisionerJobLog, error) GetTemplateByID(ctx context.Context, id uuid.UUID) (Template, error) GetTemplateByOrganizationAndName(ctx context.Context, arg GetTemplateByOrganizationAndNameParams) (Template, error) @@ -53,6 +57,8 @@ type querier interface { GetTemplateVersionByJobID(ctx context.Context, jobID uuid.UUID) (TemplateVersion, error) GetTemplateVersionByTemplateIDAndName(ctx context.Context, arg GetTemplateVersionByTemplateIDAndNameParams) (TemplateVersion, error) GetTemplateVersionsByTemplateID(ctx context.Context, arg GetTemplateVersionsByTemplateIDParams) ([]TemplateVersion, error) + GetTemplateVersionsCreatedAfter(ctx context.Context, createdAt time.Time) ([]TemplateVersion, error) + GetTemplates(ctx context.Context) ([]Template, error) GetTemplatesByIDs(ctx context.Context, ids []uuid.UUID) ([]Template, error) GetTemplatesByOrganization(ctx context.Context, arg GetTemplatesByOrganizationParams) ([]Template, error) GetUserByEmailOrUsername(ctx context.Context, arg GetUserByEmailOrUsernameParams) (User, error) @@ -64,25 +70,30 @@ type querier interface { GetWorkspaceAgentByID(ctx context.Context, id uuid.UUID) (WorkspaceAgent, error) GetWorkspaceAgentByInstanceID(ctx context.Context, authInstanceID string) (WorkspaceAgent, error) GetWorkspaceAgentsByResourceIDs(ctx context.Context, ids []uuid.UUID) ([]WorkspaceAgent, error) + GetWorkspaceAgentsCreatedAfter(ctx context.Context, createdAt time.Time) ([]WorkspaceAgent, error) GetWorkspaceAppByAgentIDAndName(ctx context.Context, arg GetWorkspaceAppByAgentIDAndNameParams) (WorkspaceApp, error) GetWorkspaceAppsByAgentID(ctx context.Context, agentID uuid.UUID) ([]WorkspaceApp, error) GetWorkspaceAppsByAgentIDs(ctx context.Context, ids []uuid.UUID) ([]WorkspaceApp, error) + GetWorkspaceAppsCreatedAfter(ctx context.Context, createdAt time.Time) ([]WorkspaceApp, error) GetWorkspaceBuildByID(ctx context.Context, id uuid.UUID) (WorkspaceBuild, error) GetWorkspaceBuildByJobID(ctx context.Context, jobID uuid.UUID) (WorkspaceBuild, error) GetWorkspaceBuildByWorkspaceID(ctx context.Context, arg GetWorkspaceBuildByWorkspaceIDParams) ([]WorkspaceBuild, error) GetWorkspaceBuildByWorkspaceIDAndBuildNumber(ctx context.Context, arg GetWorkspaceBuildByWorkspaceIDAndBuildNumberParams) (WorkspaceBuild, error) GetWorkspaceBuildByWorkspaceIDAndName(ctx context.Context, arg GetWorkspaceBuildByWorkspaceIDAndNameParams) (WorkspaceBuild, error) + GetWorkspaceBuildsCreatedAfter(ctx context.Context, createdAt time.Time) ([]WorkspaceBuild, error) GetWorkspaceByID(ctx context.Context, id uuid.UUID) (Workspace, error) GetWorkspaceByOwnerIDAndName(ctx context.Context, arg GetWorkspaceByOwnerIDAndNameParams) (Workspace, error) GetWorkspaceOwnerCountsByTemplateIDs(ctx context.Context, ids []uuid.UUID) ([]GetWorkspaceOwnerCountsByTemplateIDsRow, error) GetWorkspaceResourceByID(ctx context.Context, id uuid.UUID) (WorkspaceResource, error) GetWorkspaceResourcesByJobID(ctx context.Context, jobID uuid.UUID) ([]WorkspaceResource, error) + GetWorkspaceResourcesCreatedAfter(ctx context.Context, createdAt time.Time) ([]WorkspaceResource, error) + GetWorkspaces(ctx context.Context, arg GetWorkspacesParams) ([]Workspace, error) GetWorkspacesAutostart(ctx context.Context) ([]Workspace, error) GetWorkspacesByOrganizationIDs(ctx context.Context, arg GetWorkspacesByOrganizationIDsParams) ([]Workspace, error) GetWorkspacesByTemplateID(ctx context.Context, arg GetWorkspacesByTemplateIDParams) ([]Workspace, error) - GetWorkspacesWithFilter(ctx context.Context, arg GetWorkspacesWithFilterParams) ([]Workspace, error) InsertAPIKey(ctx context.Context, arg InsertAPIKeyParams) (APIKey, error) InsertAuditLog(ctx context.Context, arg InsertAuditLogParams) (AuditLog, error) + InsertDeploymentID(ctx context.Context, value string) error InsertFile(ctx context.Context, arg InsertFileParams) (File, error) InsertGitSSHKey(ctx context.Context, arg InsertGitSSHKeyParams) (GitSSHKey, error) InsertOrganization(ctx context.Context, arg InsertOrganizationParams) (Organization, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index edb2fc774589c..e5612a1eb4b3c 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -845,6 +845,50 @@ func (q *sqlQuerier) GetParameterSchemasByJobID(ctx context.Context, jobID uuid. return items, nil } +const getParameterSchemasCreatedAfter = `-- name: GetParameterSchemasCreatedAfter :many +SELECT id, created_at, job_id, name, description, default_source_scheme, default_source_value, allow_override_source, default_destination_scheme, allow_override_destination, default_refresh, redisplay_value, validation_error, validation_condition, validation_type_system, validation_value_type FROM parameter_schemas WHERE created_at > $1 +` + +func (q *sqlQuerier) GetParameterSchemasCreatedAfter(ctx context.Context, createdAt time.Time) ([]ParameterSchema, error) { + rows, err := q.db.QueryContext(ctx, getParameterSchemasCreatedAfter, createdAt) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ParameterSchema + for rows.Next() { + var i ParameterSchema + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.JobID, + &i.Name, + &i.Description, + &i.DefaultSourceScheme, + &i.DefaultSourceValue, + &i.AllowOverrideSource, + &i.DefaultDestinationScheme, + &i.AllowOverrideDestination, + &i.DefaultRefresh, + &i.RedisplayValue, + &i.ValidationError, + &i.ValidationCondition, + &i.ValidationTypeSystem, + &i.ValidationValueType, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const insertParameterSchema = `-- name: InsertParameterSchema :one INSERT INTO parameter_schemas ( @@ -1470,6 +1514,49 @@ func (q *sqlQuerier) GetProvisionerJobsByIDs(ctx context.Context, ids []uuid.UUI return items, nil } +const getProvisionerJobsCreatedAfter = `-- name: GetProvisionerJobsCreatedAfter :many +SELECT id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, storage_source, type, input, worker_id FROM provisioner_jobs WHERE created_at > $1 +` + +func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, createdAt time.Time) ([]ProvisionerJob, error) { + rows, err := q.db.QueryContext(ctx, getProvisionerJobsCreatedAfter, createdAt) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ProvisionerJob + for rows.Next() { + var i ProvisionerJob + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.StartedAt, + &i.CanceledAt, + &i.CompletedAt, + &i.Error, + &i.OrganizationID, + &i.InitiatorID, + &i.Provisioner, + &i.StorageMethod, + &i.StorageSource, + &i.Type, + &i.Input, + &i.WorkerID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const insertProvisionerJob = `-- name: InsertProvisionerJob :one INSERT INTO provisioner_jobs ( @@ -1601,6 +1688,26 @@ func (q *sqlQuerier) UpdateProvisionerJobWithCompleteByID(ctx context.Context, a return err } +const getDeploymentID = `-- name: GetDeploymentID :one +SELECT value FROM site_config WHERE key = 'deployment_id' +` + +func (q *sqlQuerier) GetDeploymentID(ctx context.Context) (string, error) { + row := q.db.QueryRowContext(ctx, getDeploymentID) + var value string + err := row.Scan(&value) + return value, err +} + +const insertDeploymentID = `-- name: InsertDeploymentID :exec +INSERT INTO site_config (key, value) VALUES ('deployment_id', $1) +` + +func (q *sqlQuerier) InsertDeploymentID(ctx context.Context, value string) error { + _, err := q.db.ExecContext(ctx, insertDeploymentID, value) + return err +} + const getTemplateByID = `-- name: GetTemplateByID :one SELECT id, created_at, updated_at, organization_id, deleted, name, provisioner, active_version_id, description, max_ttl, min_autostart_interval, created_by @@ -1671,6 +1778,46 @@ func (q *sqlQuerier) GetTemplateByOrganizationAndName(ctx context.Context, arg G return i, err } +const getTemplates = `-- name: GetTemplates :many +SELECT id, created_at, updated_at, organization_id, deleted, name, provisioner, active_version_id, description, max_ttl, min_autostart_interval, created_by FROM templates +` + +func (q *sqlQuerier) GetTemplates(ctx context.Context) ([]Template, error) { + rows, err := q.db.QueryContext(ctx, getTemplates) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Template + for rows.Next() { + var i Template + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.OrganizationID, + &i.Deleted, + &i.Name, + &i.Provisioner, + &i.ActiveVersionID, + &i.Description, + &i.MaxTtl, + &i.MinAutostartInterval, + &i.CreatedBy, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getTemplatesByIDs = `-- name: GetTemplatesByIDs :many SELECT id, created_at, updated_at, organization_id, deleted, name, provisioner, active_version_id, description, max_ttl, min_autostart_interval, created_by @@ -2063,6 +2210,42 @@ func (q *sqlQuerier) GetTemplateVersionsByTemplateID(ctx context.Context, arg Ge return items, nil } +const getTemplateVersionsCreatedAfter = `-- name: GetTemplateVersionsCreatedAfter :many +SELECT id, template_id, organization_id, created_at, updated_at, name, readme, job_id FROM template_versions WHERE created_at > $1 +` + +func (q *sqlQuerier) GetTemplateVersionsCreatedAfter(ctx context.Context, createdAt time.Time) ([]TemplateVersion, error) { + rows, err := q.db.QueryContext(ctx, getTemplateVersionsCreatedAfter, createdAt) + if err != nil { + return nil, err + } + defer rows.Close() + var items []TemplateVersion + for rows.Next() { + var i TemplateVersion + if err := rows.Scan( + &i.ID, + &i.TemplateID, + &i.OrganizationID, + &i.CreatedAt, + &i.UpdatedAt, + &i.Name, + &i.Readme, + &i.JobID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const insertTemplateVersion = `-- name: InsertTemplateVersion :one INSERT INTO template_versions ( @@ -2735,6 +2918,51 @@ func (q *sqlQuerier) GetWorkspaceAgentsByResourceIDs(ctx context.Context, ids [] return items, nil } +const getWorkspaceAgentsCreatedAfter = `-- name: GetWorkspaceAgentsCreatedAfter :many +SELECT id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory FROM workspace_agents WHERE created_at > $1 +` + +func (q *sqlQuerier) GetWorkspaceAgentsCreatedAfter(ctx context.Context, createdAt time.Time) ([]WorkspaceAgent, error) { + rows, err := q.db.QueryContext(ctx, getWorkspaceAgentsCreatedAfter, createdAt) + if err != nil { + return nil, err + } + defer rows.Close() + var items []WorkspaceAgent + for rows.Next() { + var i WorkspaceAgent + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.Name, + &i.FirstConnectedAt, + &i.LastConnectedAt, + &i.DisconnectedAt, + &i.ResourceID, + &i.AuthToken, + &i.AuthInstanceID, + &i.Architecture, + &i.EnvironmentVariables, + &i.OperatingSystem, + &i.StartupScript, + &i.InstanceMetadata, + &i.ResourceMetadata, + &i.Directory, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const insertWorkspaceAgent = `-- name: InsertWorkspaceAgent :one INSERT INTO workspace_agents ( @@ -2939,6 +3167,42 @@ func (q *sqlQuerier) GetWorkspaceAppsByAgentIDs(ctx context.Context, ids []uuid. return items, nil } +const getWorkspaceAppsCreatedAfter = `-- name: GetWorkspaceAppsCreatedAfter :many +SELECT id, created_at, agent_id, name, icon, command, url, relative_path FROM workspace_apps WHERE created_at > $1 +` + +func (q *sqlQuerier) GetWorkspaceAppsCreatedAfter(ctx context.Context, createdAt time.Time) ([]WorkspaceApp, error) { + rows, err := q.db.QueryContext(ctx, getWorkspaceAppsCreatedAfter, createdAt) + if err != nil { + return nil, err + } + defer rows.Close() + var items []WorkspaceApp + for rows.Next() { + var i WorkspaceApp + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.AgentID, + &i.Name, + &i.Icon, + &i.Command, + &i.Url, + &i.RelativePath, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const insertWorkspaceApp = `-- name: InsertWorkspaceApp :one INSERT INTO workspace_apps ( @@ -3290,6 +3554,46 @@ func (q *sqlQuerier) GetWorkspaceBuildByWorkspaceIDAndName(ctx context.Context, return i, err } +const getWorkspaceBuildsCreatedAfter = `-- name: GetWorkspaceBuildsCreatedAfter :many +SELECT id, created_at, updated_at, workspace_id, template_version_id, name, build_number, transition, initiator_id, provisioner_state, job_id, deadline FROM workspace_builds WHERE created_at > $1 +` + +func (q *sqlQuerier) GetWorkspaceBuildsCreatedAfter(ctx context.Context, createdAt time.Time) ([]WorkspaceBuild, error) { + rows, err := q.db.QueryContext(ctx, getWorkspaceBuildsCreatedAfter, createdAt) + if err != nil { + return nil, err + } + defer rows.Close() + var items []WorkspaceBuild + for rows.Next() { + var i WorkspaceBuild + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.WorkspaceID, + &i.TemplateVersionID, + &i.Name, + &i.BuildNumber, + &i.Transition, + &i.InitiatorID, + &i.ProvisionerState, + &i.JobID, + &i.Deadline, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const insertWorkspaceBuild = `-- name: InsertWorkspaceBuild :one INSERT INTO workspace_builds ( @@ -3448,6 +3752,40 @@ func (q *sqlQuerier) GetWorkspaceResourcesByJobID(ctx context.Context, jobID uui return items, nil } +const getWorkspaceResourcesCreatedAfter = `-- name: GetWorkspaceResourcesCreatedAfter :many +SELECT id, created_at, job_id, transition, type, name FROM workspace_resources WHERE created_at > $1 +` + +func (q *sqlQuerier) GetWorkspaceResourcesCreatedAfter(ctx context.Context, createdAt time.Time) ([]WorkspaceResource, error) { + rows, err := q.db.QueryContext(ctx, getWorkspaceResourcesCreatedAfter, createdAt) + if err != nil { + return nil, err + } + defer rows.Close() + var items []WorkspaceResource + for rows.Next() { + var i WorkspaceResource + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.JobID, + &i.Transition, + &i.Type, + &i.Name, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const insertWorkspaceResource = `-- name: InsertWorkspaceResource :one INSERT INTO workspace_resources (id, created_at, job_id, transition, type, name) @@ -3590,23 +3928,48 @@ func (q *sqlQuerier) GetWorkspaceOwnerCountsByTemplateIDs(ctx context.Context, i return items, nil } -const getWorkspacesAutostart = `-- name: GetWorkspacesAutostart :many +const getWorkspaces = `-- name: GetWorkspaces :many SELECT - id, created_at, updated_at, owner_id, organization_id, template_id, deleted, name, autostart_schedule, ttl + id, created_at, updated_at, owner_id, organization_id, template_id, deleted, name, autostart_schedule, ttl FROM - workspaces + workspaces WHERE - deleted = false -AND -( - (autostart_schedule IS NOT NULL AND autostart_schedule <> '') - OR - (ttl IS NOT NULL AND ttl > 0) -) + -- Optionally include deleted workspaces + deleted = $1 + -- Filter by organization_id + AND CASE + WHEN $2 :: uuid != '00000000-00000000-00000000-00000000' THEN + organization_id = $2 + ELSE true + END + -- Filter by owner_id + AND CASE + WHEN $3 :: uuid != '00000000-00000000-00000000-00000000' THEN + owner_id = $3 + ELSE true + END + -- Filter by name, matching on substring + AND CASE + WHEN $4 :: text != '' THEN + LOWER(name) LIKE '%' || LOWER($4) || '%' + ELSE true + END ` -func (q *sqlQuerier) GetWorkspacesAutostart(ctx context.Context) ([]Workspace, error) { - rows, err := q.db.QueryContext(ctx, getWorkspacesAutostart) +type GetWorkspacesParams struct { + Deleted bool `db:"deleted" json:"deleted"` + OrganizationID uuid.UUID `db:"organization_id" json:"organization_id"` + OwnerID uuid.UUID `db:"owner_id" json:"owner_id"` + Name string `db:"name" json:"name"` +} + +func (q *sqlQuerier) GetWorkspaces(ctx context.Context, arg GetWorkspacesParams) ([]Workspace, error) { + rows, err := q.db.QueryContext(ctx, getWorkspaces, + arg.Deleted, + arg.OrganizationID, + arg.OwnerID, + arg.Name, + ) if err != nil { return nil, err } @@ -3639,17 +4002,23 @@ func (q *sqlQuerier) GetWorkspacesAutostart(ctx context.Context) ([]Workspace, e return items, nil } -const getWorkspacesByOrganizationIDs = `-- name: GetWorkspacesByOrganizationIDs :many -SELECT id, created_at, updated_at, owner_id, organization_id, template_id, deleted, name, autostart_schedule, ttl FROM workspaces WHERE organization_id = ANY($1 :: uuid [ ]) AND deleted = $2 +const getWorkspacesAutostart = `-- name: GetWorkspacesAutostart :many +SELECT + id, created_at, updated_at, owner_id, organization_id, template_id, deleted, name, autostart_schedule, ttl +FROM + workspaces +WHERE + deleted = false +AND +( + (autostart_schedule IS NOT NULL AND autostart_schedule <> '') + OR + (ttl IS NOT NULL AND ttl > 0) +) ` -type GetWorkspacesByOrganizationIDsParams struct { - Ids []uuid.UUID `db:"ids" json:"ids"` - Deleted bool `db:"deleted" json:"deleted"` -} - -func (q *sqlQuerier) GetWorkspacesByOrganizationIDs(ctx context.Context, arg GetWorkspacesByOrganizationIDsParams) ([]Workspace, error) { - rows, err := q.db.QueryContext(ctx, getWorkspacesByOrganizationIDs, pq.Array(arg.Ids), arg.Deleted) +func (q *sqlQuerier) GetWorkspacesAutostart(ctx context.Context) ([]Workspace, error) { + rows, err := q.db.QueryContext(ctx, getWorkspacesAutostart) if err != nil { return nil, err } @@ -3682,23 +4051,17 @@ func (q *sqlQuerier) GetWorkspacesByOrganizationIDs(ctx context.Context, arg Get return items, nil } -const getWorkspacesByTemplateID = `-- name: GetWorkspacesByTemplateID :many -SELECT - id, created_at, updated_at, owner_id, organization_id, template_id, deleted, name, autostart_schedule, ttl -FROM - workspaces -WHERE - template_id = $1 - AND deleted = $2 +const getWorkspacesByOrganizationIDs = `-- name: GetWorkspacesByOrganizationIDs :many +SELECT id, created_at, updated_at, owner_id, organization_id, template_id, deleted, name, autostart_schedule, ttl FROM workspaces WHERE organization_id = ANY($1 :: uuid [ ]) AND deleted = $2 ` -type GetWorkspacesByTemplateIDParams struct { - TemplateID uuid.UUID `db:"template_id" json:"template_id"` - Deleted bool `db:"deleted" json:"deleted"` +type GetWorkspacesByOrganizationIDsParams struct { + Ids []uuid.UUID `db:"ids" json:"ids"` + Deleted bool `db:"deleted" json:"deleted"` } -func (q *sqlQuerier) GetWorkspacesByTemplateID(ctx context.Context, arg GetWorkspacesByTemplateIDParams) ([]Workspace, error) { - rows, err := q.db.QueryContext(ctx, getWorkspacesByTemplateID, arg.TemplateID, arg.Deleted) +func (q *sqlQuerier) GetWorkspacesByOrganizationIDs(ctx context.Context, arg GetWorkspacesByOrganizationIDsParams) ([]Workspace, error) { + rows, err := q.db.QueryContext(ctx, getWorkspacesByOrganizationIDs, pq.Array(arg.Ids), arg.Deleted) if err != nil { return nil, err } @@ -3731,48 +4094,23 @@ func (q *sqlQuerier) GetWorkspacesByTemplateID(ctx context.Context, arg GetWorks return items, nil } -const getWorkspacesWithFilter = `-- name: GetWorkspacesWithFilter :many +const getWorkspacesByTemplateID = `-- name: GetWorkspacesByTemplateID :many SELECT - id, created_at, updated_at, owner_id, organization_id, template_id, deleted, name, autostart_schedule, ttl + id, created_at, updated_at, owner_id, organization_id, template_id, deleted, name, autostart_schedule, ttl FROM - workspaces + workspaces WHERE - -- Optionally include deleted workspaces - deleted = $1 - -- Filter by organization_id - AND CASE - WHEN $2 :: uuid != '00000000-00000000-00000000-00000000' THEN - organization_id = $2 - ELSE true - END - -- Filter by owner_id - AND CASE - WHEN $3 :: uuid != '00000000-00000000-00000000-00000000' THEN - owner_id = $3 - ELSE true - END - -- Filter by name, matching on substring - AND CASE - WHEN $4 :: text != '' THEN - LOWER(name) LIKE '%' || LOWER($4) || '%' - ELSE true - END + template_id = $1 + AND deleted = $2 ` -type GetWorkspacesWithFilterParams struct { - Deleted bool `db:"deleted" json:"deleted"` - OrganizationID uuid.UUID `db:"organization_id" json:"organization_id"` - OwnerID uuid.UUID `db:"owner_id" json:"owner_id"` - Name string `db:"name" json:"name"` +type GetWorkspacesByTemplateIDParams struct { + TemplateID uuid.UUID `db:"template_id" json:"template_id"` + Deleted bool `db:"deleted" json:"deleted"` } -func (q *sqlQuerier) GetWorkspacesWithFilter(ctx context.Context, arg GetWorkspacesWithFilterParams) ([]Workspace, error) { - rows, err := q.db.QueryContext(ctx, getWorkspacesWithFilter, - arg.Deleted, - arg.OrganizationID, - arg.OwnerID, - arg.Name, - ) +func (q *sqlQuerier) GetWorkspacesByTemplateID(ctx context.Context, arg GetWorkspacesByTemplateIDParams) ([]Workspace, error) { + rows, err := q.db.QueryContext(ctx, getWorkspacesByTemplateID, arg.TemplateID, arg.Deleted) if err != nil { return nil, err } diff --git a/coderd/database/queries/parameterschemas.sql b/coderd/database/queries/parameterschemas.sql index e813138a8a89f..194a97930b6ca 100644 --- a/coderd/database/queries/parameterschemas.sql +++ b/coderd/database/queries/parameterschemas.sql @@ -6,6 +6,9 @@ FROM WHERE job_id = $1; +-- name: GetParameterSchemasCreatedAfter :many +SELECT * FROM parameter_schemas WHERE created_at > $1; + -- name: InsertParameterSchema :one INSERT INTO parameter_schemas ( diff --git a/coderd/database/queries/provisionerjobs.sql b/coderd/database/queries/provisionerjobs.sql index 116299804c6b1..da4fa6d1824d0 100644 --- a/coderd/database/queries/provisionerjobs.sql +++ b/coderd/database/queries/provisionerjobs.sql @@ -46,6 +46,9 @@ FROM WHERE id = ANY(@ids :: uuid [ ]); +-- name: GetProvisionerJobsCreatedAfter :many +SELECT * FROM provisioner_jobs WHERE created_at > $1; + -- name: InsertProvisionerJob :one INSERT INTO provisioner_jobs ( diff --git a/coderd/database/queries/siteconfig.sql b/coderd/database/queries/siteconfig.sql new file mode 100644 index 0000000000000..269aed61ea0c4 --- /dev/null +++ b/coderd/database/queries/siteconfig.sql @@ -0,0 +1,5 @@ +-- name: InsertDeploymentID :exec +INSERT INTO site_config (key, value) VALUES ('deployment_id', $1); + +-- name: GetDeploymentID :one +SELECT value FROM site_config WHERE key = 'deployment_id'; diff --git a/coderd/database/queries/templates.sql b/coderd/database/queries/templates.sql index c3b3753083351..55257e0f544bf 100644 --- a/coderd/database/queries/templates.sql +++ b/coderd/database/queries/templates.sql @@ -37,6 +37,9 @@ WHERE organization_id = $1 AND deleted = $2; +-- name: GetTemplates :many +SELECT * FROM templates; + -- name: InsertTemplate :one INSERT INTO templates ( diff --git a/coderd/database/queries/templateversions.sql b/coderd/database/queries/templateversions.sql index 148c8856deeb0..231b3512553df 100644 --- a/coderd/database/queries/templateversions.sql +++ b/coderd/database/queries/templateversions.sql @@ -40,6 +40,9 @@ FROM WHERE job_id = $1; +-- name: GetTemplateVersionsCreatedAfter :many +SELECT * FROM template_versions WHERE created_at > $1; + -- name: GetTemplateVersionByTemplateIDAndName :one SELECT * diff --git a/coderd/database/queries/workspaceagents.sql b/coderd/database/queries/workspaceagents.sql index de15fe451687b..3d6269a700d62 100644 --- a/coderd/database/queries/workspaceagents.sql +++ b/coderd/database/queries/workspaceagents.sql @@ -34,6 +34,9 @@ FROM WHERE resource_id = ANY(@ids :: uuid [ ]); +-- name: GetWorkspaceAgentsCreatedAfter :many +SELECT * FROM workspace_agents WHERE created_at > $1; + -- name: InsertWorkspaceAgent :one INSERT INTO workspace_agents ( diff --git a/coderd/database/queries/workspaceapps.sql b/coderd/database/queries/workspaceapps.sql index 61898d1caf56c..e4be54176d962 100644 --- a/coderd/database/queries/workspaceapps.sql +++ b/coderd/database/queries/workspaceapps.sql @@ -7,6 +7,9 @@ SELECT * FROM workspace_apps WHERE agent_id = ANY(@ids :: uuid [ ]); -- name: GetWorkspaceAppByAgentIDAndName :one SELECT * FROM workspace_apps WHERE agent_id = $1 AND name = $2; +-- name: GetWorkspaceAppsCreatedAfter :many +SELECT * FROM workspace_apps WHERE created_at > $1; + -- name: InsertWorkspaceApp :one INSERT INTO workspace_apps ( diff --git a/coderd/database/queries/workspacebuilds.sql b/coderd/database/queries/workspacebuilds.sql index 4bace271cb125..df2ce44c94f1e 100644 --- a/coderd/database/queries/workspacebuilds.sql +++ b/coderd/database/queries/workspacebuilds.sql @@ -18,6 +18,9 @@ WHERE LIMIT 1; +-- name: GetWorkspaceBuildsCreatedAfter :many +SELECT * FROM workspace_builds WHERE created_at > $1; + -- name: GetWorkspaceBuildByWorkspaceIDAndName :one SELECT * diff --git a/coderd/database/queries/workspaceresources.sql b/coderd/database/queries/workspaceresources.sql index c120cf41a8d57..3cd93d6e6e6a6 100644 --- a/coderd/database/queries/workspaceresources.sql +++ b/coderd/database/queries/workspaceresources.sql @@ -14,6 +14,9 @@ FROM WHERE job_id = $1; +-- name: GetWorkspaceResourcesCreatedAfter :many +SELECT * FROM workspace_resources WHERE created_at > $1; + -- name: InsertWorkspaceResource :one INSERT INTO workspace_resources (id, created_at, job_id, transition, type, name) diff --git a/coderd/database/queries/workspaces.sql b/coderd/database/queries/workspaces.sql index 1b9f6a88f6256..3355ac6c6b445 100644 --- a/coderd/database/queries/workspaces.sql +++ b/coderd/database/queries/workspaces.sql @@ -8,7 +8,7 @@ WHERE LIMIT 1; --- name: GetWorkspacesWithFilter :many +-- name: GetWorkspaces :many SELECT * FROM diff --git a/coderd/telemetry/telemetry.go b/coderd/telemetry/telemetry.go new file mode 100644 index 0000000000000..ce6c21c571c5a --- /dev/null +++ b/coderd/telemetry/telemetry.go @@ -0,0 +1,585 @@ +package telemetry + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "runtime" + "strings" + "sync" + "time" + + "github.com/elastic/go-sysinfo" + "github.com/google/uuid" + "golang.org/x/sync/errgroup" + "golang.org/x/xerrors" + + "cdr.dev/slog" + + "github.com/coder/coder/buildinfo" + "github.com/coder/coder/coderd/database" +) + +type Options struct { + Database database.Store + Logger slog.Logger + URL *url.URL + + DeploymentID string + DevMode bool + // Disabled determines whether telemetry will be collected + // and sent. This allows callers to still execute the API + // without having to check whether it's enabled. + Disabled bool + SnapshotFrequency time.Duration +} + +// New constructs a reporter for telemetry data. +// Duplicate data will be sent, it's on the server-side to index by UUID. +// Data is anonymized prior to being sent! +func New(options Options) (*Reporter, error) { + if options.SnapshotFrequency == 0 { + // Report six times a day by default! + options.SnapshotFrequency = 4 * time.Hour + } + snapshotURL, err := options.URL.Parse("/snapshot") + if err != nil { + return nil, xerrors.Errorf("parse snapshot url: %w", err) + } + deploymentURL, err := options.URL.Parse("/deployment") + if err != nil { + return nil, xerrors.Errorf("parse deployment url: %w", err) + } + + ctx, cancelFunc := context.WithCancel(context.Background()) + reporter := &Reporter{ + ctx: ctx, + closed: make(chan struct{}), + closeFunc: cancelFunc, + options: options, + deploymentURL: deploymentURL, + snapshotURL: snapshotURL, + startedAt: database.Now(), + } + if !options.Disabled { + go reporter.runSnapshotter() + } + return reporter, nil +} + +// Reporter sends data to the telemetry server. +type Reporter struct { + ctx context.Context + closed chan struct{} + closeMutex sync.Mutex + closeFunc context.CancelFunc + + options Options + deploymentURL, + snapshotURL *url.URL + startedAt time.Time + shutdownAt *time.Time +} + +// Snapshot reports a snapshot to the telemetry server. +// The contents of the snapshot can be a partial representation of +// the database. For example, if a new user is added, a snapshot +// can contain just that user entry. +func (r *Reporter) Snapshot(ctx context.Context, snapshot *Snapshot) { + if r.options.Disabled { + return + } + data, err := json.Marshal(snapshot) + if err != nil { + r.options.Logger.Error(ctx, "marshal snapshot: %w", slog.Error(err)) + return + } + req, err := http.NewRequestWithContext(ctx, "POST", r.snapshotURL.String(), bytes.NewReader(data)) + if err != nil { + r.options.Logger.Error(ctx, "create request", slog.Error(err)) + return + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + // If the request fails it's not necessarily an error. + // In an airgapped environment, it's fine if this fails! + r.options.Logger.Debug(ctx, "submit", slog.Error(err)) + return + } + if resp.StatusCode != http.StatusAccepted { + r.options.Logger.Debug(ctx, "bad response from telemetry server", slog.F("status", resp.StatusCode)) + return + } + r.options.Logger.Debug(ctx, "submitted snapshot") +} + +func (r *Reporter) Close() { + r.closeMutex.Lock() + defer r.closeMutex.Unlock() + if r.isClosed() { + return + } + close(r.closed) + now := database.Now() + r.shutdownAt = &now + // Report a final collection of telemetry prior to close! + // This could indicate final actions a user has taken, and + // the time the deployment was shutdown. + r.report() + r.closeFunc() +} + +func (r *Reporter) isClosed() bool { + select { + case <-r.closed: + return true + default: + return false + } +} + +func (r *Reporter) runSnapshotter() { + first := true + ticker := time.NewTicker(r.options.SnapshotFrequency) + defer ticker.Stop() + for { + if !first { + select { + case <-r.closed: + return + case <-ticker.C: + } + // Skip the ticker on the first run to report instantly! + first = false + } + r.closeMutex.Lock() + r.report() + r.closeMutex.Unlock() + } +} + +func (r *Reporter) report() { + // Submit deployment information before creating a snapshot! + // This is separated from the snapshot API call to reduce + // duplicate data from being inserted. Snapshot may be called + // numerous times simaltanously if there is lots of activity! + err := r.deployment() + if err != nil { + r.options.Logger.Debug(r.ctx, "update deployment", slog.Error(err)) + return + } + snapshot, err := r.createSnapshot() + if errors.Is(err, context.Canceled) { + return + } + if err != nil { + r.options.Logger.Error(r.ctx, "create snapshot", slog.Error(err)) + return + } + r.Snapshot(r.ctx, snapshot) +} + +// deployment collects host information and reports it to the telemetry server. +func (r *Reporter) deployment() error { + if r.options.Disabled { + return nil + } + sysInfoHost, err := sysinfo.Host() + if err != nil { + return xerrors.Errorf("get host info: %w", err) + } + mem, err := sysInfoHost.Memory() + if err != nil { + return xerrors.Errorf("get memory info: %w", err) + } + sysInfo := sysInfoHost.Info() + + containerized := false + if sysInfo.Containerized != nil { + containerized = *sysInfo.Containerized + } + data, err := json.Marshal(&Deployment{ + ID: r.options.DeploymentID, + Architecture: sysInfo.Architecture, + Containerized: containerized, + DevMode: r.options.DevMode, + OSType: sysInfo.OS.Type, + OSFamily: sysInfo.OS.Family, + OSPlatform: sysInfo.OS.Platform, + OSName: sysInfo.OS.Name, + OSVersion: sysInfo.OS.Version, + CPUCores: runtime.NumCPU(), + MemoryTotal: mem.Total, + MachineID: sysInfo.UniqueID, + Version: buildinfo.Version(), + StartedAt: r.startedAt, + ShutdownAt: r.shutdownAt, + }) + if err != nil { + return xerrors.Errorf("marshal deployment: %w", err) + } + req, err := http.NewRequestWithContext(r.ctx, "POST", r.deploymentURL.String(), bytes.NewReader(data)) + if err != nil { + return xerrors.Errorf("create deployment request: %w", err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return xerrors.Errorf("perform request: %w", err) + } + if resp.StatusCode != http.StatusAccepted { + return xerrors.Errorf("update deployment: %w", err) + } + r.options.Logger.Debug(r.ctx, "submitted deployment info") + return nil +} + +// createSnapshot collects a full snapshot from the database. +func (r *Reporter) createSnapshot() (*Snapshot, error) { + var ( + ctx = r.ctx + createdAfter = database.Now().AddDate(0, 0, -1) + eg errgroup.Group + snapshot = &Snapshot{ + DeploymentID: r.options.DeploymentID, + } + ) + + eg.Go(func() error { + schemas, err := r.options.Database.GetParameterSchemasCreatedAfter(ctx, createdAfter) + if err != nil { + return xerrors.Errorf("get parameter schemas: %w", err) + } + snapshot.ParameterSchemas = make([]ParameterSchema, 0, len(schemas)) + for _, schema := range schemas { + snapshot.ParameterSchemas = append(snapshot.ParameterSchemas, ParameterSchema{ + ID: schema.ID, + JobID: schema.JobID, + Name: schema.Name, + ValidationCondition: schema.ValidationCondition, + }) + } + return nil + }) + eg.Go(func() error { + jobs, err := r.options.Database.GetProvisionerJobsCreatedAfter(ctx, createdAfter) + if err != nil { + return xerrors.Errorf("get provisioner jobs: %w", err) + } + snapshot.ProvisionerJobs = make([]ProvisionerJob, 0, len(jobs)) + for _, job := range jobs { + snapJob := ProvisionerJob{ + ID: job.ID, + OrganizationID: job.OrganizationID, + InitiatorID: job.InitiatorID, + CreatedAt: job.CreatedAt, + UpdatedAt: job.UpdatedAt, + Error: job.Error.String, + Type: job.Type, + } + if job.StartedAt.Valid { + snapJob.StartedAt = &job.StartedAt.Time + } + if job.CanceledAt.Valid { + snapJob.CanceledAt = &job.CanceledAt.Time + } + if job.CompletedAt.Valid { + snapJob.CompletedAt = &job.CompletedAt.Time + } + snapshot.ProvisionerJobs = append(snapshot.ProvisionerJobs, snapJob) + } + return nil + }) + eg.Go(func() error { + templates, err := r.options.Database.GetTemplates(ctx) + if err != nil { + return xerrors.Errorf("get templates: %w", err) + } + snapshot.Templates = make([]Template, 0, len(templates)) + for _, dbTemplate := range templates { + snapshot.Templates = append(snapshot.Templates, Template{ + ID: dbTemplate.ID, + CreatedBy: dbTemplate.CreatedBy.UUID, + CreatedAt: dbTemplate.CreatedAt, + UpdatedAt: dbTemplate.UpdatedAt, + OrganizationID: dbTemplate.OrganizationID, + Deleted: dbTemplate.Deleted, + ActiveVersionID: dbTemplate.ActiveVersionID, + Name: dbTemplate.Name, + Description: dbTemplate.Description != "", + }) + } + return nil + }) + eg.Go(func() error { + templateVersions, err := r.options.Database.GetTemplateVersionsCreatedAfter(ctx, createdAfter) + if err != nil { + return xerrors.Errorf("get template versions: %w", err) + } + snapshot.TemplateVersions = make([]TemplateVersion, 0, len(templateVersions)) + for _, version := range templateVersions { + snapVersion := TemplateVersion{ + ID: version.ID, + CreatedAt: version.CreatedAt, + OrganizationID: version.OrganizationID, + JobID: version.JobID, + } + if version.TemplateID.Valid { + snapVersion.TemplateID = &version.TemplateID.UUID + } + snapshot.TemplateVersions = append(snapshot.TemplateVersions, snapVersion) + } + return nil + }) + eg.Go(func() error { + users, err := r.options.Database.GetUsers(ctx, database.GetUsersParams{}) + if err != nil { + return xerrors.Errorf("get users: %w", err) + } + snapshot.Users = make([]User, 0, len(users)) + for _, dbUser := range users { + emailHashed := "" + atSymbol := strings.LastIndex(dbUser.Email, "@") + if atSymbol >= 0 { + hash := sha256.Sum256([]byte(dbUser.Email[:atSymbol])) + emailHashed = fmt.Sprintf("%x%s", hash[:], dbUser.Email[atSymbol:]) + } + + snapshot.Users = append(snapshot.Users, User{ + ID: dbUser.ID, + EmailHashed: emailHashed, + RBACRoles: dbUser.RBACRoles, + CreatedAt: dbUser.CreatedAt, + }) + } + return nil + }) + eg.Go(func() error { + workspaces, err := r.options.Database.GetWorkspaces(ctx, database.GetWorkspacesParams{}) + if err != nil { + return xerrors.Errorf("get workspaces: %w", err) + } + snapshot.Workspaces = make([]Workspace, 0, len(workspaces)) + for _, dbWorkspace := range workspaces { + snapshot.Workspaces = append(snapshot.Workspaces, Workspace{ + OrganizationID: dbWorkspace.OrganizationID, + OwnerID: dbWorkspace.OwnerID, + TemplateID: dbWorkspace.TemplateID, + CreatedAt: dbWorkspace.CreatedAt, + Deleted: dbWorkspace.Deleted, + }) + } + return nil + }) + eg.Go(func() error { + workspaceApps, err := r.options.Database.GetWorkspaceAppsCreatedAfter(ctx, createdAfter) + if err != nil { + return xerrors.Errorf("get workspace apps: %w", err) + } + snapshot.WorkspaceApps = make([]WorkspaceApp, 0, len(workspaceApps)) + for _, app := range workspaceApps { + snapshot.WorkspaceApps = append(snapshot.WorkspaceApps, WorkspaceApp{ + ID: app.ID, + CreatedAt: app.CreatedAt, + AgentID: app.AgentID, + Icon: app.Icon != "", + RelativePath: app.RelativePath, + }) + } + return nil + }) + eg.Go(func() error { + workspaceAgents, err := r.options.Database.GetWorkspaceAgentsCreatedAfter(ctx, createdAfter) + if err != nil { + return xerrors.Errorf("get workspace agents: %w", err) + } + snapshot.WorkspaceAgents = make([]WorkspaceAgent, 0, len(workspaceAgents)) + for _, agent := range workspaceAgents { + snapshot.WorkspaceAgents = append(snapshot.WorkspaceAgents, WorkspaceAgent{ + ID: agent.ID, + CreatedAt: agent.CreatedAt, + ResourceID: agent.ResourceID, + InstanceAuth: agent.AuthInstanceID.Valid, + Architecture: agent.Architecture, + OperatingSystem: agent.OperatingSystem, + EnvironmentVariables: agent.EnvironmentVariables.Valid, + StartupScript: agent.StartupScript.Valid, + Directory: agent.Directory != "", + }) + } + return nil + }) + eg.Go(func() error { + workspaceBuilds, err := r.options.Database.GetWorkspaceBuildsCreatedAfter(ctx, createdAfter) + if err != nil { + return xerrors.Errorf("get workspace builds: %w", err) + } + snapshot.WorkspaceBuilds = make([]WorkspaceBuild, 0, len(workspaceBuilds)) + for _, build := range workspaceBuilds { + snapshot.WorkspaceBuilds = append(snapshot.WorkspaceBuilds, WorkspaceBuild{ + ID: build.ID, + CreatedAt: build.CreatedAt, + WorkspaceID: build.WorkspaceID, + JobID: build.JobID, + TemplateVersionID: build.TemplateVersionID, + BuildNumber: uint32(build.BuildNumber), + }) + } + return nil + }) + eg.Go(func() error { + workspaceResources, err := r.options.Database.GetWorkspaceResourcesCreatedAfter(ctx, createdAfter) + if err != nil { + return xerrors.Errorf("get workspace resources: %w", err) + } + snapshot.WorkspaceResources = make([]WorkspaceResource, 0, len(workspaceResources)) + for _, resource := range workspaceResources { + snapshot.WorkspaceResources = append(snapshot.WorkspaceResources, WorkspaceResource{ + ID: resource.ID, + JobID: resource.JobID, + Transition: resource.Transition, + Type: resource.Type, + }) + } + return nil + }) + + err := eg.Wait() + if err != nil { + return nil, err + } + return snapshot, nil +} + +// Snapshot represents a point-in-time anonymized database dump. +// Data is aggregated by latest on the server-side, so partial data +// can be sent without issue. +type Snapshot struct { + DeploymentID string `json:"deployment_id"` + + ParameterSchemas []ParameterSchema `json:"parameter_schemas"` + ProvisionerJobs []ProvisionerJob `json:"provisioner_jobs"` + Templates []Template `json:"templates"` + TemplateVersions []TemplateVersion `json:"template_versions"` + Users []User `json:"users"` + Workspaces []Workspace `json:"workspaces"` + WorkspaceApps []WorkspaceApp `json:"workspace_apps"` + WorkspaceAgents []WorkspaceAgent `json:"workspace_agents"` + WorkspaceBuilds []WorkspaceBuild `json:"workspace_build"` + WorkspaceResources []WorkspaceResource `json:"workspace_resources"` +} + +// Deployment contains information about the host running Coder. +type Deployment struct { + ID string `json:"id" validate:"required"` + Architecture string `json:"architecture"` + Containerized bool `json:"containerized"` + DevMode bool `json:"dev_mode" validate:"required"` + OSType string `json:"os_type"` + OSFamily string `json:"os_family"` + OSPlatform string `json:"os_platform"` + OSName string `json:"os_name"` + OSVersion string `json:"os_version"` + CPUCores int `json:"cpu_cores"` + MemoryTotal uint64 `json:"memory_total"` + MachineID string `json:"machine_id"` + Version string `json:"version" validate:"required"` + StartedAt time.Time `json:"started_at"` + ShutdownAt *time.Time `json:"shutdown_at"` +} + +type User struct { + ID uuid.UUID `json:"uuid"` + CreatedAt time.Time `json:"created_at"` + EmailHashed string `json:"email_hashed"` + RBACRoles []string `json:"rbac_roles"` + Status database.UserStatus `json:"status"` +} + +type WorkspaceResource struct { + ID uuid.UUID `json:"id"` + JobID uuid.UUID `json:"job_id"` + Transition database.WorkspaceTransition `json:"transition"` + Type string `json:"type"` +} + +type WorkspaceAgent struct { + ID uuid.UUID `json:"id"` + CreatedAt time.Time `json:"created_at"` + ResourceID uuid.UUID `json:"resource_id"` + InstanceAuth bool `json:"instance_auth"` + Architecture string `json:"architecture"` + OperatingSystem string `json:"operating_system"` + EnvironmentVariables bool `json:"environment_variables"` + StartupScript bool `json:"startup_script"` + Directory bool `json:"directory"` +} + +type WorkspaceApp struct { + ID uuid.UUID `json:"id"` + CreatedAt time.Time `json:"created_at"` + AgentID uuid.UUID `json:"agent_id"` + Icon bool `json:"icon"` + RelativePath bool `json:"relative_path"` +} + +type WorkspaceBuild struct { + ID uuid.UUID `json:"id"` + CreatedAt time.Time `json:"created_at"` + WorkspaceID uuid.UUID `json:"workspace_id"` + TemplateVersionID uuid.UUID `json:"template_version_id"` + JobID uuid.UUID `json:"job_id"` + BuildNumber uint32 `json:"build_number"` +} + +type Workspace struct { + OrganizationID uuid.UUID `json:"organization_id"` + OwnerID uuid.UUID `json:"owner_id"` + TemplateID uuid.UUID `json:"template_id"` + CreatedAt time.Time `json:"created_at"` + Deleted bool `json:"deleted"` +} + +type Template struct { + ID uuid.UUID `json:"id"` + CreatedBy uuid.UUID `json:"created_by"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + OrganizationID uuid.UUID `json:"organization_id"` + Deleted bool `json:"deleted"` + ActiveVersionID uuid.UUID `json:"active_version_id"` + Name string `json:"name"` + Description bool `json:"description"` +} + +type TemplateVersion struct { + ID uuid.UUID `json:"id"` + CreatedAt time.Time `json:"created_at"` + TemplateID *uuid.UUID `json:"template_id,omitempty"` + OrganizationID uuid.UUID `json:"organization_id"` + JobID uuid.UUID `json:"job_id"` +} + +type ProvisionerJob struct { + ID uuid.UUID `json:"id"` + OrganizationID uuid.UUID `json:"organization_id"` + InitiatorID uuid.UUID `json:"initiator_id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + StartedAt *time.Time `json:"started_at,omitempty"` + CanceledAt *time.Time `json:"canceled_at,omitempty"` + CompletedAt *time.Time `json:"completed_at,omitempty"` + Error string `json:"error"` + Type database.ProvisionerJobType `json:"type"` +} + +type ParameterSchema struct { + ID uuid.UUID `json:"parameter_schema"` + JobID uuid.UUID `json:"job_id"` + Name string `json:"name"` + ValidationCondition string `json:"validation_condition"` +} diff --git a/coderd/telemetry/telemetry_test.go b/coderd/telemetry/telemetry_test.go new file mode 100644 index 0000000000000..5b4b9282ee96e --- /dev/null +++ b/coderd/telemetry/telemetry_test.go @@ -0,0 +1,141 @@ +package telemetry_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/go-chi/chi" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" + + "cdr.dev/slog" + "cdr.dev/slog/sloggers/slogtest" + "github.com/coder/coder/coderd/database" + "github.com/coder/coder/coderd/database/databasefake" + "github.com/coder/coder/coderd/telemetry" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} + +func TestTelemetry(t *testing.T) { + t.Parallel() + t.Run("Snapshot", func(t *testing.T) { + t.Parallel() + db := databasefake.New() + ctx := context.Background() + _, err := db.InsertParameterSchema(ctx, database.InsertParameterSchemaParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + }) + require.NoError(t, err) + _, err = db.InsertProvisionerJob(ctx, database.InsertProvisionerJobParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + }) + require.NoError(t, err) + _, err = db.InsertTemplate(ctx, database.InsertTemplateParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + }) + require.NoError(t, err) + _, err = db.InsertTemplateVersion(ctx, database.InsertTemplateVersionParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + }) + require.NoError(t, err) + _, err = db.InsertUser(ctx, database.InsertUserParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + }) + require.NoError(t, err) + _, err = db.InsertWorkspace(ctx, database.InsertWorkspaceParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + }) + require.NoError(t, err) + _, err = db.InsertWorkspaceApp(ctx, database.InsertWorkspaceAppParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + }) + require.NoError(t, err) + _, err = db.InsertWorkspaceAgent(ctx, database.InsertWorkspaceAgentParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + }) + require.NoError(t, err) + _, err = db.InsertWorkspaceBuild(ctx, database.InsertWorkspaceBuildParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + }) + require.NoError(t, err) + _, err = db.InsertWorkspaceResource(ctx, database.InsertWorkspaceResourceParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + }) + require.NoError(t, err) + snapshot := collectSnapshot(t, db) + require.Len(t, snapshot.ParameterSchemas, 1) + require.Len(t, snapshot.ProvisionerJobs, 1) + require.Len(t, snapshot.Templates, 1) + require.Len(t, snapshot.TemplateVersions, 1) + require.Len(t, snapshot.Users, 1) + require.Len(t, snapshot.Workspaces, 1) + require.Len(t, snapshot.WorkspaceApps, 1) + require.Len(t, snapshot.WorkspaceAgents, 1) + require.Len(t, snapshot.WorkspaceBuilds, 1) + require.Len(t, snapshot.WorkspaceResources, 1) + }) + t.Run("HashedEmail", func(t *testing.T) { + t.Parallel() + db := databasefake.New() + _, err := db.InsertUser(context.Background(), database.InsertUserParams{ + ID: uuid.New(), + Email: "kyle@coder.com", + CreatedAt: database.Now(), + }) + require.NoError(t, err) + snapshot := collectSnapshot(t, db) + require.Len(t, snapshot.Users, 1) + require.Equal(t, snapshot.Users[0].EmailHashed, "bb44bf07cf9a2db0554bba63a03d822c927deae77df101874496df5a6a3e896d@coder.com") + }) +} + +func collectSnapshot(t *testing.T, db database.Store) *telemetry.Snapshot { + t.Helper() + deployment := make(chan struct{}, 64) + snapshot := make(chan *telemetry.Snapshot, 64) + r := chi.NewRouter() + r.Post("/deployment", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + deployment <- struct{}{} + }) + r.Post("/snapshot", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + ss := &telemetry.Snapshot{} + err := json.NewDecoder(r.Body).Decode(ss) + require.NoError(t, err) + snapshot <- ss + }) + server := httptest.NewServer(r) + t.Cleanup(server.Close) + serverURL, err := url.Parse(server.URL) + require.NoError(t, err) + reporter, err := telemetry.New(telemetry.Options{ + Database: db, + Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), + URL: serverURL, + DeploymentID: uuid.NewString(), + DevMode: false, + }) + require.NoError(t, err) + t.Cleanup(reporter.Close) + <-deployment + return <-snapshot +} diff --git a/coderd/workspaces.go b/coderd/workspaces.go index b9e09f50513b4..73233f3ae3068 100644 --- a/coderd/workspaces.go +++ b/coderd/workspaces.go @@ -108,7 +108,7 @@ func (api *API) workspaces(rw http.ResponseWriter, r *http.Request) { ownerFilter := r.URL.Query().Get("owner") nameFilter := r.URL.Query().Get("name") - filter := database.GetWorkspacesWithFilterParams{Deleted: false} + filter := database.GetWorkspacesParams{Deleted: false} if orgFilter != "" { orgID, err := uuid.Parse(orgFilter) if err == nil { @@ -137,7 +137,7 @@ func (api *API) workspaces(rw http.ResponseWriter, r *http.Request) { filter.Name = nameFilter } - workspaces, err := api.Database.GetWorkspacesWithFilter(r.Context(), filter) + workspaces, err := api.Database.GetWorkspaces(r.Context(), filter) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ Message: "Internal error fetching workspaces.", diff --git a/go.mod b/go.mod index 38d06b60a52f2..4061434f43802 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/coder/retry v1.3.0 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf github.com/creack/pty v1.1.18 + github.com/elastic/go-sysinfo v1.8.0 github.com/fatih/color v1.13.0 github.com/fatih/structs v1.1.0 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa @@ -128,6 +129,12 @@ require ( storj.io/drpc v0.0.30 ) +require ( + github.com/elastic/go-windows v1.0.0 // indirect + github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect + howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect +) + require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect diff --git a/go.sum b/go.sum index e7077ce680395..823544f738206 100644 --- a/go.sum +++ b/go.sum @@ -517,6 +517,10 @@ github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:Htrtb github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elastic/go-sysinfo v1.8.0 h1:hwmVlZLfTVTP+L0hSS2BD/G8GNPmcl4JEMoOktSw/wc= +github.com/elastic/go-sysinfo v1.8.0/go.mod h1:JfllUnzoQV/JRYymbH3dO1yggI3mV2oTKSXsDHM+uIM= +github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= +github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -993,6 +997,7 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jedib0t/go-pretty/v6 v6.3.1 h1:aOXiD9oqiuLH8btPQW6SfgtQN5zwhyfzZls8a6sPJ/I= github.com/jedib0t/go-pretty/v6 v6.3.1/go.mod h1:FMkOpgGD3EZ91cW8g/96RfxoV7bdeJyzXPYgz1L1ln0= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -1005,6 +1010,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= @@ -2449,6 +2456,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.1/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= +howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= k8s.io/api v0.16.13/go.mod h1:QWu8UWSTiuQZMMeYjwLs6ILu5O74qKSJ0c+4vrchDxs= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= From 9ed5ecd88ac11fa6a28c9d401e3320eae44aee6d Mon Sep 17 00:00:00 2001 From: kylecarbs Date: Sun, 12 Jun 2022 23:40:09 +0000 Subject: [PATCH 2/7] Fix flake and requested changes --- cli/server.go | 10 ++++----- cli/server_test.go | 2 +- coderd/database/databasefake/databasefake.go | 2 +- coderd/database/dump.sql | 6 +++--- .../migrations/000023_site_config.up.sql | 2 +- coderd/database/queries.sql.go | 4 ++-- coderd/database/queries/siteconfig.sql | 4 ++-- coderd/telemetry/telemetry.go | 21 +++++++++++++------ 8 files changed, 30 insertions(+), 21 deletions(-) diff --git a/cli/server.go b/cli/server.go index 19e3a2b7aed83..8e54c7cf2b087 100644 --- a/cli/server.go +++ b/cli/server.go @@ -82,7 +82,7 @@ func server() *cobra.Command { oauth2GithubClientSecret string oauth2GithubAllowedOrganizations []string oauth2GithubAllowSignups bool - telemetryEnabled bool + telemetryEnable bool telemetryURL string tlsCertFile string tlsClientCAFile string @@ -327,8 +327,8 @@ func server() *cobra.Command { } // Disable telemetry if in dev-mode. If the telemetry flag // is manually specified, override this behavior! - if buildModeDev && !cmd.Flags().Changed("telemetry") { - telemetryEnabled = false + if buildModeDev && !cmd.Flags().Changed("telemetry-enable") { + telemetryEnable = false } reporter, err := telemetry.New(telemetry.Options{ DeploymentID: deploymentID, @@ -336,7 +336,7 @@ func server() *cobra.Command { Logger: logger.Named("telemetry"), URL: telemetryURL, DevMode: dev, - Disabled: !telemetryEnabled, + Disabled: !telemetryEnable, }) if err != nil { return xerrors.Errorf("create telemetry reporter: %w", err) @@ -580,7 +580,7 @@ func server() *cobra.Command { "Specifies organizations the user must be a member of to authenticate with GitHub.") cliflag.BoolVarP(root.Flags(), &oauth2GithubAllowSignups, "oauth2-github-allow-signups", "", "CODER_OAUTH2_GITHUB_ALLOW_SIGNUPS", false, "Specifies whether new users can sign up with GitHub.") - cliflag.BoolVarP(root.Flags(), &telemetryEnabled, "telemetry", "", "CODER_TELEMETRY", true, "Specifies whether telemetry is enabled or not. Coder collects anonymized usage data to help improve our product!") + cliflag.BoolVarP(root.Flags(), &telemetryEnable, "telemetry-enable", "", "CODER_TELEMETRY_ENABLE", true, "Specifies whether telemetry is enabled or not. Coder collects anonymized usage data to help improve our product!") cliflag.StringVarP(root.Flags(), &telemetryURL, "telemetry-url", "", "CODER_TELEMETRY_URL", "https://telemetry.coder.com", "Specifies a URL to send telemetry to.") _ = root.Flags().MarkHidden("telemetry-url") cliflag.BoolVarP(root.Flags(), &tlsEnable, "tls-enable", "", "CODER_TLS_ENABLE", false, "Specifies if TLS will be enabled") diff --git a/cli/server_test.go b/cli/server_test.go index 160bf978851c7..ace35c92a9349 100644 --- a/cli/server_test.go +++ b/cli/server_test.go @@ -348,7 +348,7 @@ func TestServer(t *testing.T) { server := httptest.NewServer(r) t.Cleanup(server.Close) - root, _ := clitest.New(t, "server", "--dev", "--tunnel=false", "--address", ":0", "--telemetry", "true", "--telemetry-url", server.URL) + root, _ := clitest.New(t, "server", "--dev", "--tunnel=false", "--address", ":0", "--telemetry-enable", "true", "--telemetry-url", server.URL) var buf strings.Builder errC := make(chan error) root.SetOutput(&buf) diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index 6c30169bfb4ee..fc1e2ce3618a2 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -981,7 +981,7 @@ func (q *fakeQuerier) GetTemplates(_ context.Context) ([]database.Template, erro q.mutex.RLock() defer q.mutex.RUnlock() - return q.templates, nil + return q.templates[:], nil } func (q *fakeQuerier) GetTemplatesByOrganization(_ context.Context, arg database.GetTemplatesByOrganizationParams) ([]database.Template, error) { diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 6dbd041c5192c..f82aafdfbf9c9 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -226,7 +226,7 @@ CREATE TABLE provisioner_jobs ( worker_id uuid ); -CREATE TABLE site_config ( +CREATE TABLE site_configs ( key character varying(256) NOT NULL, value character varying(8192) NOT NULL ); @@ -383,8 +383,8 @@ ALTER TABLE ONLY provisioner_job_logs ALTER TABLE ONLY provisioner_jobs ADD CONSTRAINT provisioner_jobs_pkey PRIMARY KEY (id); -ALTER TABLE ONLY site_config - ADD CONSTRAINT site_config_key_key UNIQUE (key); +ALTER TABLE ONLY site_configs + ADD CONSTRAINT site_configs_key_key UNIQUE (key); ALTER TABLE ONLY template_versions ADD CONSTRAINT template_versions_pkey PRIMARY KEY (id); diff --git a/coderd/database/migrations/000023_site_config.up.sql b/coderd/database/migrations/000023_site_config.up.sql index 16e0b617dd494..70c64e511c2d8 100644 --- a/coderd/database/migrations/000023_site_config.up.sql +++ b/coderd/database/migrations/000023_site_config.up.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS site_config ( +CREATE TABLE IF NOT EXISTS site_configs ( key varchar(256) NOT NULL UNIQUE, value varchar(8192) NOT NULL ); diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index e5612a1eb4b3c..e58e624592496 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -1689,7 +1689,7 @@ func (q *sqlQuerier) UpdateProvisionerJobWithCompleteByID(ctx context.Context, a } const getDeploymentID = `-- name: GetDeploymentID :one -SELECT value FROM site_config WHERE key = 'deployment_id' +SELECT value FROM site_configs WHERE key = 'deployment_id' ` func (q *sqlQuerier) GetDeploymentID(ctx context.Context) (string, error) { @@ -1700,7 +1700,7 @@ func (q *sqlQuerier) GetDeploymentID(ctx context.Context) (string, error) { } const insertDeploymentID = `-- name: InsertDeploymentID :exec -INSERT INTO site_config (key, value) VALUES ('deployment_id', $1) +INSERT INTO site_configs (key, value) VALUES ('deployment_id', $1) ` func (q *sqlQuerier) InsertDeploymentID(ctx context.Context, value string) error { diff --git a/coderd/database/queries/siteconfig.sql b/coderd/database/queries/siteconfig.sql index 269aed61ea0c4..9d3936e23886d 100644 --- a/coderd/database/queries/siteconfig.sql +++ b/coderd/database/queries/siteconfig.sql @@ -1,5 +1,5 @@ -- name: InsertDeploymentID :exec -INSERT INTO site_config (key, value) VALUES ('deployment_id', $1); +INSERT INTO site_configs (key, value) VALUES ('deployment_id', $1); -- name: GetDeploymentID :one -SELECT value FROM site_config WHERE key = 'deployment_id'; +SELECT value FROM site_configs WHERE key = 'deployment_id'; diff --git a/coderd/telemetry/telemetry.go b/coderd/telemetry/telemetry.go index ce6c21c571c5a..dfd719e6e0352 100644 --- a/coderd/telemetry/telemetry.go +++ b/coderd/telemetry/telemetry.go @@ -28,7 +28,8 @@ import ( type Options struct { Database database.Store Logger slog.Logger - URL *url.URL + // URL is an endpoint to direct telemetry towards! + URL *url.URL DeploymentID string DevMode bool @@ -155,9 +156,13 @@ func (r *Reporter) runSnapshotter() { case <-ticker.C: } // Skip the ticker on the first run to report instantly! - first = false } + first = false r.closeMutex.Lock() + if r.isClosed() { + r.closeMutex.Unlock() + return + } r.report() r.closeMutex.Unlock() } @@ -345,6 +350,8 @@ func (r *Reporter) createSnapshot() (*Snapshot, error) { emailHashed := "" atSymbol := strings.LastIndex(dbUser.Email, "@") if atSymbol >= 0 { + // We hash the beginning of the user to allow for indexing users + // by email between deployments. hash := sha256.Sum256([]byte(dbUser.Email[:atSymbol])) emailHashed = fmt.Sprintf("%x%s", hash[:], dbUser.Email[atSymbol:]) } @@ -366,6 +373,7 @@ func (r *Reporter) createSnapshot() (*Snapshot, error) { snapshot.Workspaces = make([]Workspace, 0, len(workspaces)) for _, dbWorkspace := range workspaces { snapshot.Workspaces = append(snapshot.Workspaces, Workspace{ + ID: dbWorkspace.ID, OrganizationID: dbWorkspace.OrganizationID, OwnerID: dbWorkspace.OwnerID, TemplateID: dbWorkspace.TemplateID, @@ -475,10 +483,10 @@ type Snapshot struct { // Deployment contains information about the host running Coder. type Deployment struct { - ID string `json:"id" validate:"required"` + ID string `json:"id"` Architecture string `json:"architecture"` Containerized bool `json:"containerized"` - DevMode bool `json:"dev_mode" validate:"required"` + DevMode bool `json:"dev_mode"` OSType string `json:"os_type"` OSFamily string `json:"os_family"` OSPlatform string `json:"os_platform"` @@ -487,7 +495,7 @@ type Deployment struct { CPUCores int `json:"cpu_cores"` MemoryTotal uint64 `json:"memory_total"` MachineID string `json:"machine_id"` - Version string `json:"version" validate:"required"` + Version string `json:"version"` StartedAt time.Time `json:"started_at"` ShutdownAt *time.Time `json:"shutdown_at"` } @@ -537,6 +545,7 @@ type WorkspaceBuild struct { } type Workspace struct { + ID uuid.UUID `json:"id"` OrganizationID uuid.UUID `json:"organization_id"` OwnerID uuid.UUID `json:"owner_id"` TemplateID uuid.UUID `json:"template_id"` @@ -578,7 +587,7 @@ type ProvisionerJob struct { } type ParameterSchema struct { - ID uuid.UUID `json:"parameter_schema"` + ID uuid.UUID `json:"id"` JobID uuid.UUID `json:"job_id"` Name string `json:"name"` ValidationCondition string `json:"validation_condition"` From 52932857b908babe285e5a91990ae425028f4022 Mon Sep 17 00:00:00 2001 From: kylecarbs Date: Mon, 13 Jun 2022 23:44:04 +0000 Subject: [PATCH 3/7] Add reporting options for setup --- cli/server.go | 4 + coderd/telemetry/telemetry.go | 279 +++++++++++++++++++++------------- 2 files changed, 176 insertions(+), 107 deletions(-) diff --git a/cli/server.go b/cli/server.go index 8e54c7cf2b087..d7e038f005afb 100644 --- a/cli/server.go +++ b/cli/server.go @@ -337,6 +337,10 @@ func server() *cobra.Command { URL: telemetryURL, DevMode: dev, Disabled: !telemetryEnable, + GitHubOAuth: oauth2GithubClientID != "", + Prometheus: promEnabled, + STUN: len(stunServers) != 0, + Tunnel: tunnel, }) if err != nil { return xerrors.Errorf("create telemetry reporter: %w", err) diff --git a/coderd/telemetry/telemetry.go b/coderd/telemetry/telemetry.go index dfd719e6e0352..d08e724660205 100644 --- a/coderd/telemetry/telemetry.go +++ b/coderd/telemetry/telemetry.go @@ -21,7 +21,6 @@ import ( "cdr.dev/slog" - "github.com/coder/coder/buildinfo" "github.com/coder/coder/coderd/database" ) @@ -37,13 +36,22 @@ type Options struct { // and sent. This allows callers to still execute the API // without having to check whether it's enabled. Disabled bool + GitHubOAuth bool + Prometheus bool + STUN bool SnapshotFrequency time.Duration + Tunnel bool } // New constructs a reporter for telemetry data. // Duplicate data will be sent, it's on the server-side to index by UUID. // Data is anonymized prior to being sent! func New(options Options) (*Reporter, error) { + if options.Disabled { + return &Reporter{ + options: options, + }, nil + } if options.SnapshotFrequency == 0 { // Report six times a day by default! options.SnapshotFrequency = 4 * time.Hour @@ -87,39 +95,47 @@ type Reporter struct { shutdownAt *time.Time } -// Snapshot reports a snapshot to the telemetry server. -// The contents of the snapshot can be a partial representation of -// the database. For example, if a new user is added, a snapshot -// can contain just that user entry. -func (r *Reporter) Snapshot(ctx context.Context, snapshot *Snapshot) { +// Report sends a snapshot to the telemetry server. +// The contents of the snapshot can be a partial representation of the +// database. For example, if a new user is added, a snapshot can +// contain just that user entry. +func (r *Reporter) Report(snapshot *Snapshot) { if r.options.Disabled { return } - data, err := json.Marshal(snapshot) - if err != nil { - r.options.Logger.Error(ctx, "marshal snapshot: %w", slog.Error(err)) - return - } - req, err := http.NewRequestWithContext(ctx, "POST", r.snapshotURL.String(), bytes.NewReader(data)) - if err != nil { - r.options.Logger.Error(ctx, "create request", slog.Error(err)) - return - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - // If the request fails it's not necessarily an error. - // In an airgapped environment, it's fine if this fails! - r.options.Logger.Debug(ctx, "submit", slog.Error(err)) - return - } - if resp.StatusCode != http.StatusAccepted { - r.options.Logger.Debug(ctx, "bad response from telemetry server", slog.F("status", resp.StatusCode)) - return - } - r.options.Logger.Debug(ctx, "submitted snapshot") + snapshot.DeploymentID = r.options.DeploymentID + + // Runs in a goroutine so it's non-blocking to callers! + go func() { + data, err := json.Marshal(snapshot) + if err != nil { + r.options.Logger.Error(r.ctx, "marshal snapshot: %w", slog.Error(err)) + return + } + req, err := http.NewRequestWithContext(r.ctx, "POST", r.snapshotURL.String(), bytes.NewReader(data)) + if err != nil { + r.options.Logger.Error(r.ctx, "create request", slog.Error(err)) + return + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + // If the request fails it's not necessarily an error. + // In an airgapped environment, it's fine if this fails! + r.options.Logger.Debug(r.ctx, "submit", slog.Error(err)) + return + } + if resp.StatusCode != http.StatusAccepted { + r.options.Logger.Debug(r.ctx, "bad response from telemetry server", slog.F("status", resp.StatusCode)) + return + } + r.options.Logger.Debug(r.ctx, "submitted snapshot") + }() } func (r *Reporter) Close() { + if r.options.Disabled { + return + } r.closeMutex.Lock() defer r.closeMutex.Unlock() if r.isClosed() { @@ -131,7 +147,7 @@ func (r *Reporter) Close() { // Report a final collection of telemetry prior to close! // This could indicate final actions a user has taken, and // the time the deployment was shutdown. - r.report() + r.reportWithDeployment() r.closeFunc() } @@ -163,12 +179,12 @@ func (r *Reporter) runSnapshotter() { r.closeMutex.Unlock() return } - r.report() + r.reportWithDeployment() r.closeMutex.Unlock() } } -func (r *Reporter) report() { +func (r *Reporter) reportWithDeployment() { // Submit deployment information before creating a snapshot! // This is separated from the snapshot API call to reduce // duplicate data from being inserted. Snapshot may be called @@ -186,14 +202,11 @@ func (r *Reporter) report() { r.options.Logger.Error(r.ctx, "create snapshot", slog.Error(err)) return } - r.Snapshot(r.ctx, snapshot) + r.Report(snapshot) } // deployment collects host information and reports it to the telemetry server. func (r *Reporter) deployment() error { - if r.options.Disabled { - return nil - } sysInfoHost, err := sysinfo.Host() if err != nil { return xerrors.Errorf("get host info: %w", err) @@ -213,6 +226,10 @@ func (r *Reporter) deployment() error { Architecture: sysInfo.Architecture, Containerized: containerized, DevMode: r.options.DevMode, + GitHubOAuth: r.options.GitHubOAuth, + Prometheus: r.options.Prometheus, + STUN: r.options.STUN, + Tunnel: r.options.Tunnel, OSType: sysInfo.OS.Type, OSFamily: sysInfo.OS.Family, OSPlatform: sysInfo.OS.Platform, @@ -221,7 +238,6 @@ func (r *Reporter) deployment() error { CPUCores: runtime.NumCPU(), MemoryTotal: mem.Total, MachineID: sysInfo.UniqueID, - Version: buildinfo.Version(), StartedAt: r.startedAt, ShutdownAt: r.shutdownAt, }) @@ -277,25 +293,7 @@ func (r *Reporter) createSnapshot() (*Snapshot, error) { } snapshot.ProvisionerJobs = make([]ProvisionerJob, 0, len(jobs)) for _, job := range jobs { - snapJob := ProvisionerJob{ - ID: job.ID, - OrganizationID: job.OrganizationID, - InitiatorID: job.InitiatorID, - CreatedAt: job.CreatedAt, - UpdatedAt: job.UpdatedAt, - Error: job.Error.String, - Type: job.Type, - } - if job.StartedAt.Valid { - snapJob.StartedAt = &job.StartedAt.Time - } - if job.CanceledAt.Valid { - snapJob.CanceledAt = &job.CanceledAt.Time - } - if job.CompletedAt.Valid { - snapJob.CompletedAt = &job.CompletedAt.Time - } - snapshot.ProvisionerJobs = append(snapshot.ProvisionerJobs, snapJob) + snapshot.ProvisionerJobs = append(snapshot.ProvisionerJobs, ConvertProvisionerJob(job)) } return nil }) @@ -345,6 +343,15 @@ func (r *Reporter) createSnapshot() (*Snapshot, error) { if err != nil { return xerrors.Errorf("get users: %w", err) } + var firstUser database.User + for _, dbUser := range users { + if firstUser.CreatedAt.IsZero() { + firstUser = dbUser + } + if dbUser.CreatedAt.After(firstUser.CreatedAt) { + firstUser = dbUser + } + } snapshot.Users = make([]User, 0, len(users)) for _, dbUser := range users { emailHashed := "" @@ -355,13 +362,16 @@ func (r *Reporter) createSnapshot() (*Snapshot, error) { hash := sha256.Sum256([]byte(dbUser.Email[:atSymbol])) emailHashed = fmt.Sprintf("%x%s", hash[:], dbUser.Email[atSymbol:]) } - - snapshot.Users = append(snapshot.Users, User{ + user := User{ ID: dbUser.ID, EmailHashed: emailHashed, RBACRoles: dbUser.RBACRoles, CreatedAt: dbUser.CreatedAt, - }) + } + if firstUser.ID == dbUser.ID { + user.Email = dbUser.Email + } + snapshot.Users = append(snapshot.Users, user) } return nil }) @@ -372,14 +382,7 @@ func (r *Reporter) createSnapshot() (*Snapshot, error) { } snapshot.Workspaces = make([]Workspace, 0, len(workspaces)) for _, dbWorkspace := range workspaces { - snapshot.Workspaces = append(snapshot.Workspaces, Workspace{ - ID: dbWorkspace.ID, - OrganizationID: dbWorkspace.OrganizationID, - OwnerID: dbWorkspace.OwnerID, - TemplateID: dbWorkspace.TemplateID, - CreatedAt: dbWorkspace.CreatedAt, - Deleted: dbWorkspace.Deleted, - }) + snapshot.Workspaces = append(snapshot.Workspaces, ConvertWorkspace(dbWorkspace)) } return nil }) @@ -390,13 +393,7 @@ func (r *Reporter) createSnapshot() (*Snapshot, error) { } snapshot.WorkspaceApps = make([]WorkspaceApp, 0, len(workspaceApps)) for _, app := range workspaceApps { - snapshot.WorkspaceApps = append(snapshot.WorkspaceApps, WorkspaceApp{ - ID: app.ID, - CreatedAt: app.CreatedAt, - AgentID: app.AgentID, - Icon: app.Icon != "", - RelativePath: app.RelativePath, - }) + snapshot.WorkspaceApps = append(snapshot.WorkspaceApps, ConvertWorkspaceApp(app)) } return nil }) @@ -407,17 +404,7 @@ func (r *Reporter) createSnapshot() (*Snapshot, error) { } snapshot.WorkspaceAgents = make([]WorkspaceAgent, 0, len(workspaceAgents)) for _, agent := range workspaceAgents { - snapshot.WorkspaceAgents = append(snapshot.WorkspaceAgents, WorkspaceAgent{ - ID: agent.ID, - CreatedAt: agent.CreatedAt, - ResourceID: agent.ResourceID, - InstanceAuth: agent.AuthInstanceID.Valid, - Architecture: agent.Architecture, - OperatingSystem: agent.OperatingSystem, - EnvironmentVariables: agent.EnvironmentVariables.Valid, - StartupScript: agent.StartupScript.Valid, - Directory: agent.Directory != "", - }) + snapshot.WorkspaceAgents = append(snapshot.WorkspaceAgents, ConvertWorkspaceAgent(agent)) } return nil }) @@ -428,14 +415,7 @@ func (r *Reporter) createSnapshot() (*Snapshot, error) { } snapshot.WorkspaceBuilds = make([]WorkspaceBuild, 0, len(workspaceBuilds)) for _, build := range workspaceBuilds { - snapshot.WorkspaceBuilds = append(snapshot.WorkspaceBuilds, WorkspaceBuild{ - ID: build.ID, - CreatedAt: build.CreatedAt, - WorkspaceID: build.WorkspaceID, - JobID: build.JobID, - TemplateVersionID: build.TemplateVersionID, - BuildNumber: uint32(build.BuildNumber), - }) + snapshot.WorkspaceBuilds = append(snapshot.WorkspaceBuilds, ConvertWorkspaceBuild(build)) } return nil }) @@ -446,12 +426,7 @@ func (r *Reporter) createSnapshot() (*Snapshot, error) { } snapshot.WorkspaceResources = make([]WorkspaceResource, 0, len(workspaceResources)) for _, resource := range workspaceResources { - snapshot.WorkspaceResources = append(snapshot.WorkspaceResources, WorkspaceResource{ - ID: resource.ID, - JobID: resource.JobID, - Transition: resource.Transition, - Type: resource.Type, - }) + snapshot.WorkspaceResources = append(snapshot.WorkspaceResources, ConvertWorkspaceResource(resource)) } return nil }) @@ -463,6 +438,90 @@ func (r *Reporter) createSnapshot() (*Snapshot, error) { return snapshot, nil } +// ConvertWorkspace anonymizes a workspace. +func ConvertWorkspace(workspace database.Workspace) Workspace { + return Workspace{ + ID: workspace.ID, + OrganizationID: workspace.OrganizationID, + OwnerID: workspace.OwnerID, + TemplateID: workspace.TemplateID, + CreatedAt: workspace.CreatedAt, + Deleted: workspace.Deleted, + AutostartSchedule: workspace.AutostartSchedule.String, + } +} + +// ConvertWorkspaceBuild anonymizes a workspace build. +func ConvertWorkspaceBuild(build database.WorkspaceBuild) WorkspaceBuild { + return WorkspaceBuild{ + ID: build.ID, + CreatedAt: build.CreatedAt, + WorkspaceID: build.WorkspaceID, + JobID: build.JobID, + TemplateVersionID: build.TemplateVersionID, + BuildNumber: uint32(build.BuildNumber), + } +} + +// ConvertProvisionerJob anonymizes a provisioner job. +func ConvertProvisionerJob(job database.ProvisionerJob) ProvisionerJob { + snapJob := ProvisionerJob{ + ID: job.ID, + OrganizationID: job.OrganizationID, + InitiatorID: job.InitiatorID, + CreatedAt: job.CreatedAt, + UpdatedAt: job.UpdatedAt, + Error: job.Error.String, + Type: job.Type, + } + if job.StartedAt.Valid { + snapJob.StartedAt = &job.StartedAt.Time + } + if job.CanceledAt.Valid { + snapJob.CanceledAt = &job.CanceledAt.Time + } + if job.CompletedAt.Valid { + snapJob.CompletedAt = &job.CompletedAt.Time + } + return snapJob +} + +// ConvertWorkspaceAgent anonymizes a workspace agent. +func ConvertWorkspaceAgent(agent database.WorkspaceAgent) WorkspaceAgent { + return WorkspaceAgent{ + ID: agent.ID, + CreatedAt: agent.CreatedAt, + ResourceID: agent.ResourceID, + InstanceAuth: agent.AuthInstanceID.Valid, + Architecture: agent.Architecture, + OperatingSystem: agent.OperatingSystem, + EnvironmentVariables: agent.EnvironmentVariables.Valid, + StartupScript: agent.StartupScript.Valid, + Directory: agent.Directory != "", + } +} + +// ConvertWorkspaceApp anonymizes a workspace app. +func ConvertWorkspaceApp(app database.WorkspaceApp) WorkspaceApp { + return WorkspaceApp{ + ID: app.ID, + CreatedAt: app.CreatedAt, + AgentID: app.AgentID, + Icon: app.Icon != "", + RelativePath: app.RelativePath, + } +} + +// ConvertWorkspaceResource anonymizes a workspace resource. +func ConvertWorkspaceResource(resource database.WorkspaceResource) WorkspaceResource { + return WorkspaceResource{ + ID: resource.ID, + JobID: resource.JobID, + Transition: resource.Transition, + Type: resource.Type, + } +} + // Snapshot represents a point-in-time anonymized database dump. // Data is aggregated by latest on the server-side, so partial data // can be sent without issue. @@ -487,6 +546,10 @@ type Deployment struct { Architecture string `json:"architecture"` Containerized bool `json:"containerized"` DevMode bool `json:"dev_mode"` + Tunnel bool `json:"tunnel"` + GitHubOAuth bool `json:"github_oauth"` + Prometheus bool `json:"prometheus"` + STUN bool `json:"stun"` OSType string `json:"os_type"` OSFamily string `json:"os_family"` OSPlatform string `json:"os_platform"` @@ -495,14 +558,15 @@ type Deployment struct { CPUCores int `json:"cpu_cores"` MemoryTotal uint64 `json:"memory_total"` MachineID string `json:"machine_id"` - Version string `json:"version"` StartedAt time.Time `json:"started_at"` ShutdownAt *time.Time `json:"shutdown_at"` } type User struct { - ID uuid.UUID `json:"uuid"` - CreatedAt time.Time `json:"created_at"` + ID uuid.UUID `json:"uuid"` + CreatedAt time.Time `json:"created_at"` + // Email is only filled in for the first/admin user! + Email string `json:"email"` EmailHashed string `json:"email_hashed"` RBACRoles []string `json:"rbac_roles"` Status database.UserStatus `json:"status"` @@ -545,12 +609,13 @@ type WorkspaceBuild struct { } type Workspace struct { - ID uuid.UUID `json:"id"` - OrganizationID uuid.UUID `json:"organization_id"` - OwnerID uuid.UUID `json:"owner_id"` - TemplateID uuid.UUID `json:"template_id"` - CreatedAt time.Time `json:"created_at"` - Deleted bool `json:"deleted"` + ID uuid.UUID `json:"id"` + OrganizationID uuid.UUID `json:"organization_id"` + OwnerID uuid.UUID `json:"owner_id"` + TemplateID uuid.UUID `json:"template_id"` + CreatedAt time.Time `json:"created_at"` + Deleted bool `json:"deleted"` + AutostartSchedule string `json:"autostart_schedule"` } type Template struct { From 09fdadef87e7bce66fdc4522db8561e70e7de6cd Mon Sep 17 00:00:00 2001 From: kylecarbs Date: Thu, 16 Jun 2022 15:57:51 +0000 Subject: [PATCH 4/7] Add reporting for workspaces --- cli/server.go | 40 ++++----- cli/server_test.go | 4 +- coderd/coderd.go | 2 +- coderd/coderdtest/coderdtest.go | 2 + coderd/telemetry/telemetry.go | 155 ++++++++++++++++++-------------- coderd/templates.go | 9 +- coderd/users.go | 13 +++ coderd/workspaces.go | 6 ++ 8 files changed, 137 insertions(+), 94 deletions(-) diff --git a/cli/server.go b/cli/server.go index b01d53799774a..4e0803078a0b3 100644 --- a/cli/server.go +++ b/cli/server.go @@ -258,6 +258,7 @@ func server() *cobra.Command { SSHKeygenAlgorithm: sshKeygenAlgorithm, TURNServer: turnServer, TracerProvider: tracerProvider, + Telemetry: telemetry.NewNoop(), } if oauth2GithubClientSecret != "" { @@ -310,28 +311,27 @@ func server() *cobra.Command { if err != nil { return xerrors.Errorf("parse telemetry url: %w", err) } - // Disable telemetry if in dev-mode. If the telemetry flag - // is manually specified, override this behavior! - if !cmd.Flags().Changed("telemetry-enable") { + // Disable telemetry if the in-memory database is used unless explicitly defined! + if inMemoryDatabase && !cmd.Flags().Changed("telemetry") { telemetryEnable = false } - reporter, err := telemetry.New(telemetry.Options{ - BuiltinPostgres: builtinPostgres, - DeploymentID: deploymentID, - Database: options.Database, - Logger: logger.Named("telemetry"), - URL: telemetryURL, - Disabled: !telemetryEnable, - GitHubOAuth: oauth2GithubClientID != "", - Prometheus: promEnabled, - STUN: len(stunServers) != 0, - Tunnel: tunnel, - }) - if err != nil { - return xerrors.Errorf("create telemetry reporter: %w", err) + if telemetryEnable { + options.Telemetry, err = telemetry.New(telemetry.Options{ + BuiltinPostgres: builtinPostgres, + DeploymentID: deploymentID, + Database: options.Database, + Logger: logger.Named("telemetry"), + URL: telemetryURL, + GitHubOAuth: oauth2GithubClientID != "", + Prometheus: promEnabled, + STUN: len(stunServers) != 0, + Tunnel: tunnel, + }) + if err != nil { + return xerrors.Errorf("create telemetry reporter: %w", err) + } + defer options.Telemetry.Close() } - defer reporter.Close() - options.Telemetry = reporter coderAPI := coderd.New(options) client := codersdk.New(localURL) @@ -533,7 +533,7 @@ func server() *cobra.Command { "Specifies organizations the user must be a member of to authenticate with GitHub.") cliflag.BoolVarP(root.Flags(), &oauth2GithubAllowSignups, "oauth2-github-allow-signups", "", "CODER_OAUTH2_GITHUB_ALLOW_SIGNUPS", false, "Specifies whether new users can sign up with GitHub.") - cliflag.BoolVarP(root.Flags(), &telemetryEnable, "telemetry-enable", "", "CODER_TELEMETRY_ENABLE", true, "Specifies whether telemetry is enabled or not. Coder collects anonymized usage data to help improve our product!") + cliflag.BoolVarP(root.Flags(), &telemetryEnable, "telemetry", "", "CODER_TELEMETRY", true, "Specifies whether telemetry is enabled or not. Coder collects anonymized usage data to help improve our product!") cliflag.StringVarP(root.Flags(), &telemetryURL, "telemetry-url", "", "CODER_TELEMETRY_URL", "https://telemetry.coder.com", "Specifies a URL to send telemetry to.") _ = root.Flags().MarkHidden("telemetry-url") cliflag.BoolVarP(root.Flags(), &tlsEnable, "tls-enable", "", "CODER_TLS_ENABLE", false, "Specifies if TLS will be enabled") diff --git a/cli/server_test.go b/cli/server_test.go index 0aefc3307c569..1532793caa321 100644 --- a/cli/server_test.go +++ b/cli/server_test.go @@ -259,10 +259,8 @@ func TestServer(t *testing.T) { server := httptest.NewServer(r) t.Cleanup(server.Close) - root, _ := clitest.New(t, "server", "--dev", "--tunnel=false", "--address", ":0", "--telemetry-enable", "true", "--telemetry-url", server.URL) - var buf strings.Builder + root, _ := clitest.New(t, "server", "--in-memory", "--address", ":0", "--telemetry", "--telemetry-url", server.URL) errC := make(chan error) - root.SetOutput(&buf) go func() { errC <- root.ExecuteContext(ctx) }() diff --git a/coderd/coderd.go b/coderd/coderd.go index e005b13e49c38..c4822145b5a60 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -55,7 +55,7 @@ type Options struct { ICEServers []webrtc.ICEServer SecureAuthCookie bool SSHKeygenAlgorithm gitsshkey.Algorithm - Telemetry *telemetry.Reporter + Telemetry telemetry.Reporter TURNServer *turnconn.Server TracerProvider *sdktrace.TracerProvider } diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index ea068927004df..bf72369c60858 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -26,6 +26,7 @@ import ( "time" "github.com/coder/coder/coderd/rbac" + "github.com/coder/coder/coderd/telemetry" "github.com/coder/coder/coderd/util/ptr" "cloud.google.com/go/compute/metadata" @@ -164,6 +165,7 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, *coderd.API) TURNServer: turnServer, APIRateLimit: options.APIRateLimit, Authorizer: options.Authorizer, + Telemetry: telemetry.NewNoop(), }) srv.Config.Handler = coderAPI.Handler if options.IncludeProvisionerD { diff --git a/coderd/telemetry/telemetry.go b/coderd/telemetry/telemetry.go index 1633f616885ff..96ec3d81fb0c3 100644 --- a/coderd/telemetry/telemetry.go +++ b/coderd/telemetry/telemetry.go @@ -37,12 +37,8 @@ type Options struct { // URL is an endpoint to direct telemetry towards! URL *url.URL - BuiltinPostgres bool - DeploymentID string - // Disabled determines whether telemetry will be collected - // and sent. This allows callers to still execute the API - // without having to check whether it's enabled. - Disabled bool + BuiltinPostgres bool + DeploymentID string GitHubOAuth bool Prometheus bool STUN bool @@ -53,12 +49,7 @@ type Options struct { // New constructs a reporter for telemetry data. // Duplicate data will be sent, it's on the server-side to index by UUID. // Data is anonymized prior to being sent! -func New(options Options) (*Reporter, error) { - if options.Disabled { - return &Reporter{ - options: options, - }, nil - } +func New(options Options) (Reporter, error) { if options.SnapshotFrequency == 0 { // Report once every 30mins by default! options.SnapshotFrequency = 30 * time.Minute @@ -73,7 +64,7 @@ func New(options Options) (*Reporter, error) { } ctx, cancelFunc := context.WithCancel(context.Background()) - reporter := &Reporter{ + reporter := &remoteReporter{ ctx: ctx, closed: make(chan struct{}), closeFunc: cancelFunc, @@ -82,14 +73,26 @@ func New(options Options) (*Reporter, error) { snapshotURL: snapshotURL, startedAt: database.Now(), } - if !options.Disabled { - go reporter.runSnapshotter() - } + go reporter.runSnapshotter() return reporter, nil } +// NewNoop creates a new telemetry reporter that entirely discards all requests. +func NewNoop() Reporter { + return &noopReporter{} +} + // Reporter sends data to the telemetry server. -type Reporter struct { +type Reporter interface { + // Report sends a snapshot to the telemetry server. + // The contents of the snapshot can be a partial representation of the + // database. For example, if a new user is added, a snapshot can + // contain just that user entry. + Report(snapshot *Snapshot) + Close() +} + +type remoteReporter struct { ctx context.Context closed chan struct{} closeMutex sync.Mutex @@ -102,14 +105,7 @@ type Reporter struct { shutdownAt *time.Time } -// Report sends a snapshot to the telemetry server. -// The contents of the snapshot can be a partial representation of the -// database. For example, if a new user is added, a snapshot can -// contain just that user entry. -func (r *Reporter) Report(snapshot *Snapshot) { - if r.options.Disabled { - return - } +func (r *remoteReporter) Report(snapshot *Snapshot) { snapshot.DeploymentID = r.options.DeploymentID // Runs in a goroutine so it's non-blocking to callers! @@ -140,10 +136,7 @@ func (r *Reporter) Report(snapshot *Snapshot) { }() } -func (r *Reporter) Close() { - if r.options.Disabled { - return - } +func (r *remoteReporter) Close() { r.closeMutex.Lock() defer r.closeMutex.Unlock() if r.isClosed() { @@ -159,7 +152,7 @@ func (r *Reporter) Close() { r.closeFunc() } -func (r *Reporter) isClosed() bool { +func (r *remoteReporter) isClosed() bool { select { case <-r.closed: return true @@ -168,7 +161,7 @@ func (r *Reporter) isClosed() bool { } } -func (r *Reporter) runSnapshotter() { +func (r *remoteReporter) runSnapshotter() { first := true ticker := time.NewTicker(r.options.SnapshotFrequency) defer ticker.Stop() @@ -192,7 +185,7 @@ func (r *Reporter) runSnapshotter() { } } -func (r *Reporter) reportWithDeployment() { +func (r *remoteReporter) reportWithDeployment() { // Submit deployment information before creating a snapshot! // This is separated from the snapshot API call to reduce // duplicate data from being inserted. Snapshot may be called @@ -214,7 +207,7 @@ func (r *Reporter) reportWithDeployment() { } // deployment collects host information and reports it to the telemetry server. -func (r *Reporter) deployment() error { +func (r *remoteReporter) deployment() error { sysInfoHost, err := sysinfo.Host() if err != nil { return xerrors.Errorf("get host info: %w", err) @@ -269,7 +262,7 @@ func (r *Reporter) deployment() error { } // createSnapshot collects a full snapshot from the database. -func (r *Reporter) createSnapshot() (*Snapshot, error) { +func (r *remoteReporter) createSnapshot() (*Snapshot, error) { var ( ctx = r.ctx // For resources that grow in size very quickly (like workspace builds), @@ -315,17 +308,7 @@ func (r *Reporter) createSnapshot() (*Snapshot, error) { } snapshot.Templates = make([]Template, 0, len(templates)) for _, dbTemplate := range templates { - snapshot.Templates = append(snapshot.Templates, Template{ - ID: dbTemplate.ID, - CreatedBy: dbTemplate.CreatedBy, - CreatedAt: dbTemplate.CreatedAt, - UpdatedAt: dbTemplate.UpdatedAt, - OrganizationID: dbTemplate.OrganizationID, - Deleted: dbTemplate.Deleted, - ActiveVersionID: dbTemplate.ActiveVersionID, - Name: dbTemplate.Name, - Description: dbTemplate.Description != "", - }) + snapshot.Templates = append(snapshot.Templates, ConvertTemplate(dbTemplate)) } return nil }) @@ -336,16 +319,7 @@ func (r *Reporter) createSnapshot() (*Snapshot, error) { } snapshot.TemplateVersions = make([]TemplateVersion, 0, len(templateVersions)) for _, version := range templateVersions { - snapVersion := TemplateVersion{ - ID: version.ID, - CreatedAt: version.CreatedAt, - OrganizationID: version.OrganizationID, - JobID: version.JobID, - } - if version.TemplateID.Valid { - snapVersion.TemplateID = &version.TemplateID.UUID - } - snapshot.TemplateVersions = append(snapshot.TemplateVersions, snapVersion) + snapshot.TemplateVersions = append(snapshot.TemplateVersions, ConvertTemplateVersion(version)) } return nil }) @@ -356,6 +330,9 @@ func (r *Reporter) createSnapshot() (*Snapshot, error) { } var firstUser database.User for _, dbUser := range users { + if dbUser.Status != database.UserStatusActive { + continue + } if firstUser.CreatedAt.IsZero() { firstUser = dbUser } @@ -365,20 +342,8 @@ func (r *Reporter) createSnapshot() (*Snapshot, error) { } snapshot.Users = make([]User, 0, len(users)) for _, dbUser := range users { - emailHashed := "" - atSymbol := strings.LastIndex(dbUser.Email, "@") - if atSymbol >= 0 { - // We hash the beginning of the user to allow for indexing users - // by email between deployments. - hash := sha256.Sum256([]byte(dbUser.Email[:atSymbol])) - emailHashed = fmt.Sprintf("%x%s", hash[:], dbUser.Email[atSymbol:]) - } - user := User{ - ID: dbUser.ID, - EmailHashed: emailHashed, - RBACRoles: dbUser.RBACRoles, - CreatedAt: dbUser.CreatedAt, - } + user := ConvertUser(dbUser) + // If it's the first user, we'll send the email! if firstUser.ID == dbUser.ID { user.Email = dbUser.Email } @@ -533,6 +498,53 @@ func ConvertWorkspaceResource(resource database.WorkspaceResource) WorkspaceReso } } +// ConvertUser anonymizes a user. +func ConvertUser(dbUser database.User) User { + emailHashed := "" + atSymbol := strings.LastIndex(dbUser.Email, "@") + if atSymbol >= 0 { + // We hash the beginning of the user to allow for indexing users + // by email between deployments. + hash := sha256.Sum256([]byte(dbUser.Email[:atSymbol])) + emailHashed = fmt.Sprintf("%x%s", hash[:], dbUser.Email[atSymbol:]) + } + return User{ + ID: dbUser.ID, + EmailHashed: emailHashed, + RBACRoles: dbUser.RBACRoles, + CreatedAt: dbUser.CreatedAt, + } +} + +// ConvertTemplate anonymizes a template. +func ConvertTemplate(dbTemplate database.Template) Template { + return Template{ + ID: dbTemplate.ID, + CreatedBy: dbTemplate.CreatedBy, + CreatedAt: dbTemplate.CreatedAt, + UpdatedAt: dbTemplate.UpdatedAt, + OrganizationID: dbTemplate.OrganizationID, + Deleted: dbTemplate.Deleted, + ActiveVersionID: dbTemplate.ActiveVersionID, + Name: dbTemplate.Name, + Description: dbTemplate.Description != "", + } +} + +// ConvertTemplateVersion anonymizes a template version. +func ConvertTemplateVersion(version database.TemplateVersion) TemplateVersion { + snapVersion := TemplateVersion{ + ID: version.ID, + CreatedAt: version.CreatedAt, + OrganizationID: version.OrganizationID, + JobID: version.JobID, + } + if version.TemplateID.Valid { + snapVersion.TemplateID = &version.TemplateID.UUID + } + return snapVersion +} + // Snapshot represents a point-in-time anonymized database dump. // Data is aggregated by latest on the server-side, so partial data // can be sent without issue. @@ -668,3 +680,8 @@ type ParameterSchema struct { Name string `json:"name"` ValidationCondition string `json:"validation_condition"` } + +type noopReporter struct{} + +func (n *noopReporter) Report(_ *Snapshot) {} +func (n *noopReporter) Close() {} diff --git a/coderd/templates.go b/coderd/templates.go index aab99adca1858..60ab03ef938d3 100644 --- a/coderd/templates.go +++ b/coderd/templates.go @@ -16,6 +16,7 @@ import ( "github.com/coder/coder/coderd/httpapi" "github.com/coder/coder/coderd/httpmw" "github.com/coder/coder/coderd/rbac" + "github.com/coder/coder/coderd/telemetry" "github.com/coder/coder/coderd/util/ptr" "github.com/coder/coder/codersdk" ) @@ -180,10 +181,11 @@ func (api *API) postTemplateByOrganization(rw http.ResponseWriter, r *http.Reque minAutostartInterval = time.Duration(*createTemplate.MinAutostartIntervalMillis) * time.Millisecond } + var dbTemplate database.Template var template codersdk.Template err = api.Database.InTx(func(db database.Store) error { now := database.Now() - dbTemplate, err := db.InsertTemplate(r.Context(), database.InsertTemplateParams{ + dbTemplate, err = db.InsertTemplate(r.Context(), database.InsertTemplateParams{ ID: uuid.New(), CreatedAt: now, UpdatedAt: now, @@ -244,6 +246,11 @@ func (api *API) postTemplateByOrganization(rw http.ResponseWriter, r *http.Reque return } + api.Telemetry.Report(&telemetry.Snapshot{ + Templates: []telemetry.Template{telemetry.ConvertTemplate(dbTemplate)}, + TemplateVersions: []telemetry.TemplateVersion{telemetry.ConvertTemplateVersion(templateVersion)}, + }) + httpapi.Write(rw, http.StatusCreated, template) } diff --git a/coderd/users.go b/coderd/users.go index 781cc213d1797..fcdf04f0b53c9 100644 --- a/coderd/users.go +++ b/coderd/users.go @@ -20,6 +20,7 @@ import ( "github.com/coder/coder/coderd/httpapi" "github.com/coder/coder/coderd/httpmw" "github.com/coder/coder/coderd/rbac" + "github.com/coder/coder/coderd/telemetry" "github.com/coder/coder/coderd/userpassword" "github.com/coder/coder/codersdk" "github.com/coder/coder/cryptorand" @@ -86,6 +87,13 @@ func (api *API) postFirstUser(rw http.ResponseWriter, r *http.Request) { return } + telemetryUser := telemetry.ConvertUser(user) + // Send the initial users email address! + telemetryUser.Email = user.Email + api.Telemetry.Report(&telemetry.Snapshot{ + Users: []telemetry.User{telemetryUser}, + }) + // TODO: @emyrk this currently happens outside the database tx used to create // the user. Maybe I add this ability to grant roles in the createUser api // and add some rbac bypass when calling api functions this way?? @@ -252,6 +260,11 @@ func (api *API) postUser(rw http.ResponseWriter, r *http.Request) { return } + // Report when users are added! + api.Telemetry.Report(&telemetry.Snapshot{ + Users: []telemetry.User{telemetry.ConvertUser(user)}, + }) + httpapi.Write(rw, http.StatusCreated, convertUser(user, []uuid.UUID{createUser.OrganizationID})) } diff --git a/coderd/workspaces.go b/coderd/workspaces.go index 51cce47d688ae..188a61242c03f 100644 --- a/coderd/workspaces.go +++ b/coderd/workspaces.go @@ -27,6 +27,7 @@ import ( "github.com/coder/coder/coderd/httpapi" "github.com/coder/coder/coderd/httpmw" "github.com/coder/coder/coderd/rbac" + "github.com/coder/coder/coderd/telemetry" "github.com/coder/coder/coderd/util/ptr" "github.com/coder/coder/codersdk" ) @@ -457,6 +458,11 @@ func (api *API) postWorkspacesByOrganization(rw http.ResponseWriter, r *http.Req return } + api.Telemetry.Report(&telemetry.Snapshot{ + Workspaces: []telemetry.Workspace{telemetry.ConvertWorkspace(workspace)}, + WorkspaceBuilds: []telemetry.WorkspaceBuild{telemetry.ConvertWorkspaceBuild(workspaceBuild)}, + }) + httpapi.Write(rw, http.StatusCreated, convertWorkspace(workspace, workspaceBuild, templateVersionJob, template, findUser(apiKey.UserID, users), findUser(workspaceBuild.InitiatorID, users))) } From 9b690bd646662a7339fe8e4ca00cc659c02c5c3c Mon Sep 17 00:00:00 2001 From: kylecarbs Date: Thu, 16 Jun 2022 16:01:32 +0000 Subject: [PATCH 5/7] Add resources as they are reported --- coderd/provisionerdaemons.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/coderd/provisionerdaemons.go b/coderd/provisionerdaemons.go index dd0c3ea3be906..c9788ea685fcf 100644 --- a/coderd/provisionerdaemons.go +++ b/coderd/provisionerdaemons.go @@ -26,6 +26,7 @@ import ( "github.com/coder/coder/coderd/httpapi" "github.com/coder/coder/coderd/parameter" "github.com/coder/coder/coderd/rbac" + "github.com/coder/coder/coderd/telemetry" "github.com/coder/coder/provisionerd/proto" "github.com/coder/coder/provisionersdk" sdkproto "github.com/coder/coder/provisionersdk/proto" @@ -80,6 +81,7 @@ func (api *API) ListenProvisionerDaemon(ctx context.Context) (client proto.DRPCP Database: api.Database, Pubsub: api.Pubsub, Provisioners: daemon.Provisioners, + Telemetry: api.Telemetry, Logger: api.Logger.Named(fmt.Sprintf("provisionerd-%s", daemon.Name)), }) if err != nil { @@ -127,6 +129,7 @@ type provisionerdServer struct { Provisioners []database.ProvisionerType Database database.Store Pubsub database.Pubsub + Telemetry telemetry.Reporter } // AcquireJob queries the database to lock a job. @@ -543,6 +546,10 @@ func (server *provisionerdServer) CompleteJob(ctx context.Context, completed *pr return nil, xerrors.Errorf("you don't have permission to update this job") } + telemetrySnapshot := &telemetry.Snapshot{} + // Items are added to this snapshot as they complete! + defer server.Telemetry.Report(telemetrySnapshot) + switch jobType := completed.Type.(type) { case *proto.CompletedJob_TemplateImport_: for transition, resources := range map[database.WorkspaceTransition][]*sdkproto.Resource{ @@ -556,7 +563,7 @@ func (server *provisionerdServer) CompleteJob(ctx context.Context, completed *pr slog.F("resource_type", resource.Type), slog.F("transition", transition)) - err = insertWorkspaceResource(ctx, server.Database, jobID, transition, resource) + err = insertWorkspaceResource(ctx, server.Database, jobID, transition, resource, telemetrySnapshot) if err != nil { return nil, xerrors.Errorf("insert resource: %w", err) } @@ -625,7 +632,7 @@ func (server *provisionerdServer) CompleteJob(ctx context.Context, completed *pr } // This could be a bulk insert to improve performance. for _, protoResource := range jobType.WorkspaceBuild.Resources { - err = insertWorkspaceResource(ctx, db, job.ID, workspaceBuild.Transition, protoResource) + err = insertWorkspaceResource(ctx, db, job.ID, workspaceBuild.Transition, protoResource, telemetrySnapshot) if err != nil { return xerrors.Errorf("insert provisioner job: %w", err) } @@ -656,7 +663,7 @@ func (server *provisionerdServer) CompleteJob(ctx context.Context, completed *pr slog.F("resource_name", resource.Name), slog.F("resource_type", resource.Type)) - err = insertWorkspaceResource(ctx, server.Database, jobID, database.WorkspaceTransitionStart, resource) + err = insertWorkspaceResource(ctx, server.Database, jobID, database.WorkspaceTransitionStart, resource, telemetrySnapshot) if err != nil { return nil, xerrors.Errorf("insert resource: %w", err) } @@ -686,7 +693,7 @@ func (server *provisionerdServer) CompleteJob(ctx context.Context, completed *pr return &proto.Empty{}, nil } -func insertWorkspaceResource(ctx context.Context, db database.Store, jobID uuid.UUID, transition database.WorkspaceTransition, protoResource *sdkproto.Resource) error { +func insertWorkspaceResource(ctx context.Context, db database.Store, jobID uuid.UUID, transition database.WorkspaceTransition, protoResource *sdkproto.Resource, snapshot *telemetry.Snapshot) error { resource, err := db.InsertWorkspaceResource(ctx, database.InsertWorkspaceResourceParams{ ID: uuid.New(), CreatedAt: database.Now(), @@ -698,6 +705,8 @@ func insertWorkspaceResource(ctx context.Context, db database.Store, jobID uuid. if err != nil { return xerrors.Errorf("insert provisioner job resource %q: %w", protoResource.Name, err) } + snapshot.WorkspaceResources = append(snapshot.WorkspaceResources, telemetry.ConvertWorkspaceResource(resource)) + for _, agent := range protoResource.Agents { var instanceID sql.NullString if agent.GetInstanceId() != "" { @@ -745,9 +754,10 @@ func insertWorkspaceResource(ctx context.Context, db database.Store, jobID uuid. if err != nil { return xerrors.Errorf("insert agent: %w", err) } + snapshot.WorkspaceAgents = append(snapshot.WorkspaceAgents, telemetry.ConvertWorkspaceAgent(dbAgent)) for _, app := range agent.Apps { - _, err := db.InsertWorkspaceApp(ctx, database.InsertWorkspaceAppParams{ + dbApp, err := db.InsertWorkspaceApp(ctx, database.InsertWorkspaceAppParams{ ID: uuid.New(), CreatedAt: database.Now(), AgentID: dbAgent.ID, @@ -766,6 +776,7 @@ func insertWorkspaceResource(ctx context.Context, db database.Store, jobID uuid. if err != nil { return xerrors.Errorf("insert app: %w", err) } + snapshot.WorkspaceApps = append(snapshot.WorkspaceApps, telemetry.ConvertWorkspaceApp(dbApp)) } } return nil From b74d12742a9bf2a89ad73dc647dbb6f38668e25d Mon Sep 17 00:00:00 2001 From: kylecarbs Date: Thu, 16 Jun 2022 16:50:44 +0000 Subject: [PATCH 6/7] Track API key usage --- coderd/database/databasefake/databasefake.go | 13 +++++++ coderd/database/querier.go | 1 + coderd/database/queries.sql.go | 41 ++++++++++++++++++++ coderd/database/queries/apikeys.sql | 3 ++ coderd/telemetry/telemetry.go | 31 +++++++++++++++ coderd/telemetry/telemetry_test.go | 7 +++- 6 files changed, 95 insertions(+), 1 deletion(-) diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index e9c78809c6938..e41c70b874ed8 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -130,6 +130,19 @@ func (q *fakeQuerier) GetAPIKeyByID(_ context.Context, id string) (database.APIK return database.APIKey{}, sql.ErrNoRows } +func (q *fakeQuerier) GetAPIKeysLastUsedAfter(_ context.Context, after time.Time) ([]database.APIKey, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + apiKeys := make([]database.APIKey, 0) + for _, key := range q.apiKeys { + if key.LastUsed.After(after) { + apiKeys = append(apiKeys, key) + } + } + return apiKeys, nil +} + func (q *fakeQuerier) DeleteAPIKeyByID(_ context.Context, id string) error { q.mutex.Lock() defer q.mutex.Unlock() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index cc35b27bba56e..f3108dc8b00ec 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -23,6 +23,7 @@ type querier interface { DeleteGitSSHKey(ctx context.Context, userID uuid.UUID) error DeleteParameterValueByID(ctx context.Context, id uuid.UUID) error GetAPIKeyByID(ctx context.Context, id string) (APIKey, error) + GetAPIKeysLastUsedAfter(ctx context.Context, lastUsed time.Time) ([]APIKey, error) // GetAuditLogsBefore retrieves `limit` number of audit logs before the provided // ID. GetAuditLogsBefore(ctx context.Context, arg GetAuditLogsBeforeParams) ([]AuditLog, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 411e15864dcd0..b91a784a2fc84 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -60,6 +60,47 @@ func (q *sqlQuerier) GetAPIKeyByID(ctx context.Context, id string) (APIKey, erro return i, err } +const getAPIKeysLastUsedAfter = `-- name: GetAPIKeysLastUsedAfter :many +SELECT id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, oauth_access_token, oauth_refresh_token, oauth_id_token, oauth_expiry, lifetime_seconds FROM api_keys WHERE last_used > $1 +` + +func (q *sqlQuerier) GetAPIKeysLastUsedAfter(ctx context.Context, lastUsed time.Time) ([]APIKey, error) { + rows, err := q.db.QueryContext(ctx, getAPIKeysLastUsedAfter, lastUsed) + if err != nil { + return nil, err + } + defer rows.Close() + var items []APIKey + for rows.Next() { + var i APIKey + if err := rows.Scan( + &i.ID, + &i.HashedSecret, + &i.UserID, + &i.LastUsed, + &i.ExpiresAt, + &i.CreatedAt, + &i.UpdatedAt, + &i.LoginType, + &i.OAuthAccessToken, + &i.OAuthRefreshToken, + &i.OAuthIDToken, + &i.OAuthExpiry, + &i.LifetimeSeconds, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const insertAPIKey = `-- name: InsertAPIKey :one INSERT INTO api_keys ( diff --git a/coderd/database/queries/apikeys.sql b/coderd/database/queries/apikeys.sql index 9e7cc8e252b7d..c24f779e1d08a 100644 --- a/coderd/database/queries/apikeys.sql +++ b/coderd/database/queries/apikeys.sql @@ -8,6 +8,9 @@ WHERE LIMIT 1; +-- name: GetAPIKeysLastUsedAfter :many +SELECT * FROM api_keys WHERE last_used > $1; + -- name: InsertAPIKey :one INSERT INTO api_keys ( diff --git a/coderd/telemetry/telemetry.go b/coderd/telemetry/telemetry.go index 96ec3d81fb0c3..d14dd85beb09e 100644 --- a/coderd/telemetry/telemetry.go +++ b/coderd/telemetry/telemetry.go @@ -274,6 +274,17 @@ func (r *remoteReporter) createSnapshot() (*Snapshot, error) { } ) + eg.Go(func() error { + apiKeys, err := r.options.Database.GetAPIKeysLastUsedAfter(ctx, createdAfter) + if err != nil { + return xerrors.Errorf("get api keys last used: %w", err) + } + snapshot.APIKeys = make([]APIKey, 0, len(apiKeys)) + for _, apiKey := range apiKeys { + snapshot.APIKeys = append(snapshot.APIKeys, ConvertAPIKey(apiKey)) + } + return nil + }) eg.Go(func() error { schemas, err := r.options.Database.GetParameterSchemasCreatedAfter(ctx, createdAfter) if err != nil { @@ -414,6 +425,17 @@ func (r *remoteReporter) createSnapshot() (*Snapshot, error) { return snapshot, nil } +// ConvertAPIKey anonymizes an API key. +func ConvertAPIKey(apiKey database.APIKey) APIKey { + return APIKey{ + ID: apiKey.ID, + UserID: apiKey.UserID, + CreatedAt: apiKey.CreatedAt, + LastUsed: apiKey.LastUsed, + LoginType: apiKey.LoginType, + } +} + // ConvertWorkspace anonymizes a workspace. func ConvertWorkspace(workspace database.Workspace) Workspace { return Workspace{ @@ -551,6 +573,7 @@ func ConvertTemplateVersion(version database.TemplateVersion) TemplateVersion { type Snapshot struct { DeploymentID string `json:"deployment_id"` + APIKeys []APIKey `json:"api_keys"` ParameterSchemas []ParameterSchema `json:"parameter_schemas"` ProvisionerJobs []ProvisionerJob `json:"provisioner_jobs"` Templates []Template `json:"templates"` @@ -585,6 +608,14 @@ type Deployment struct { ShutdownAt *time.Time `json:"shutdown_at"` } +type APIKey struct { + ID string `json:"id"` + UserID uuid.UUID `json:"user_id"` + CreatedAt time.Time `json:"created_at"` + LastUsed time.Time `json:"last_used"` + LoginType database.LoginType `json:"login_type"` +} + type User struct { ID uuid.UUID `json:"uuid"` CreatedAt time.Time `json:"created_at"` diff --git a/coderd/telemetry/telemetry_test.go b/coderd/telemetry/telemetry_test.go index a9e43a3227082..4e78b19ec8c54 100644 --- a/coderd/telemetry/telemetry_test.go +++ b/coderd/telemetry/telemetry_test.go @@ -31,7 +31,12 @@ func TestTelemetry(t *testing.T) { t.Parallel() db := databasefake.New() ctx := context.Background() - _, err := db.InsertParameterSchema(ctx, database.InsertParameterSchemaParams{ + _, err := db.InsertAPIKey(ctx, database.InsertAPIKeyParams{ + ID: uuid.NewString(), + LastUsed: database.Now(), + }) + require.NoError(t, err) + _, err = db.InsertParameterSchema(ctx, database.InsertParameterSchemaParams{ ID: uuid.New(), CreatedAt: database.Now(), }) From 35aac55308635f95fa798b617f6a45ec75b32f32 Mon Sep 17 00:00:00 2001 From: kylecarbs Date: Fri, 17 Jun 2022 04:20:54 +0000 Subject: [PATCH 7/7] Ensure telemetry is tracked prior to exit --- cli/server.go | 10 ++---- cli/server_test.go | 2 +- coderd/provisionerdaemons.go | 27 ++++++++++------ coderd/telemetry/telemetry.go | 58 ++++++++++++++++++----------------- 4 files changed, 51 insertions(+), 46 deletions(-) diff --git a/cli/server.go b/cli/server.go index 5c6bf63793de7..2696dd4616117 100644 --- a/cli/server.go +++ b/cli/server.go @@ -83,7 +83,6 @@ func server() *cobra.Command { oauth2GithubClientSecret string oauth2GithubAllowedOrganizations []string oauth2GithubAllowSignups bool - telemetryEnable bool telemetryURL string tlsCertFile string tlsClientCAFile string @@ -312,11 +311,7 @@ func server() *cobra.Command { if err != nil { return xerrors.Errorf("parse telemetry url: %w", err) } - // Disable telemetry if the in-memory database is used unless explicitly defined! - if inMemoryDatabase && !cmd.Flags().Changed("telemetry") { - telemetryEnable = false - } - if telemetryEnable { + if !inMemoryDatabase || cmd.Flags().Changed("telemetry-url") { options.Telemetry, err = telemetry.New(telemetry.Options{ BuiltinPostgres: builtinPostgres, DeploymentID: deploymentID, @@ -487,6 +482,8 @@ func server() *cobra.Command { <-devTunnelErrChan } + // Ensures a last report can be sent before exit! + options.Telemetry.Close() cmd.Println("Waiting for WebSocket connections to close...") shutdownConns() coderAPI.Close() @@ -534,7 +531,6 @@ func server() *cobra.Command { "Specifies organizations the user must be a member of to authenticate with GitHub.") cliflag.BoolVarP(root.Flags(), &oauth2GithubAllowSignups, "oauth2-github-allow-signups", "", "CODER_OAUTH2_GITHUB_ALLOW_SIGNUPS", false, "Specifies whether new users can sign up with GitHub.") - cliflag.BoolVarP(root.Flags(), &telemetryEnable, "telemetry", "", "CODER_TELEMETRY", true, "Specifies whether telemetry is enabled or not. Coder collects anonymized usage data to help improve our product!") cliflag.StringVarP(root.Flags(), &telemetryURL, "telemetry-url", "", "CODER_TELEMETRY_URL", "https://telemetry.coder.com", "Specifies a URL to send telemetry to.") _ = root.Flags().MarkHidden("telemetry-url") cliflag.BoolVarP(root.Flags(), &tlsEnable, "tls-enable", "", "CODER_TLS_ENABLE", false, "Specifies if TLS will be enabled") diff --git a/cli/server_test.go b/cli/server_test.go index 1532793caa321..a7be6b7d021e8 100644 --- a/cli/server_test.go +++ b/cli/server_test.go @@ -259,7 +259,7 @@ func TestServer(t *testing.T) { server := httptest.NewServer(r) t.Cleanup(server.Close) - root, _ := clitest.New(t, "server", "--in-memory", "--address", ":0", "--telemetry", "--telemetry-url", server.URL) + root, _ := clitest.New(t, "server", "--in-memory", "--address", ":0", "--telemetry-url", server.URL) errC := make(chan error) go func() { errC <- root.ExecuteContext(ctx) diff --git a/coderd/provisionerdaemons.go b/coderd/provisionerdaemons.go index c9788ea685fcf..272a1d0f604c7 100644 --- a/coderd/provisionerdaemons.go +++ b/coderd/provisionerdaemons.go @@ -493,21 +493,28 @@ func (server *provisionerdServer) FailJob(ctx context.Context, failJob *proto.Fa if job.CompletedAt.Valid { return nil, xerrors.Errorf("job already completed") } + job.CompletedAt = sql.NullTime{ + Time: database.Now(), + Valid: true, + } + job.Error = sql.NullString{ + String: failJob.Error, + Valid: failJob.Error != "", + } + err = server.Database.UpdateProvisionerJobWithCompleteByID(ctx, database.UpdateProvisionerJobWithCompleteByIDParams{ - ID: jobID, - CompletedAt: sql.NullTime{ - Time: database.Now(), - Valid: true, - }, - UpdatedAt: database.Now(), - Error: sql.NullString{ - String: failJob.Error, - Valid: failJob.Error != "", - }, + ID: jobID, + CompletedAt: job.CompletedAt, + UpdatedAt: database.Now(), + Error: job.Error, }) if err != nil { return nil, xerrors.Errorf("update provisioner job: %w", err) } + server.Telemetry.Report(&telemetry.Snapshot{ + ProvisionerJobs: []telemetry.ProvisionerJob{telemetry.ConvertProvisionerJob(job)}, + }) + switch jobType := failJob.Type.(type) { case *proto.FailedJob_WorkspaceBuild_: if jobType.WorkspaceBuild.State == nil { diff --git a/coderd/telemetry/telemetry.go b/coderd/telemetry/telemetry.go index 4de72da9b0199..0506eac387732 100644 --- a/coderd/telemetry/telemetry.go +++ b/coderd/telemetry/telemetry.go @@ -106,34 +106,34 @@ type remoteReporter struct { } func (r *remoteReporter) Report(snapshot *Snapshot) { - snapshot.DeploymentID = r.options.DeploymentID + go r.reportSync(snapshot) +} - // Runs in a goroutine so it's non-blocking to callers! - go func() { - data, err := json.Marshal(snapshot) - if err != nil { - r.options.Logger.Error(r.ctx, "marshal snapshot: %w", slog.Error(err)) - return - } - req, err := http.NewRequestWithContext(r.ctx, "POST", r.snapshotURL.String(), bytes.NewReader(data)) - if err != nil { - r.options.Logger.Error(r.ctx, "create request", slog.Error(err)) - return - } - req.Header.Set(VersionHeader, buildinfo.Version()) - resp, err := http.DefaultClient.Do(req) - if err != nil { - // If the request fails it's not necessarily an error. - // In an airgapped environment, it's fine if this fails! - r.options.Logger.Debug(r.ctx, "submit", slog.Error(err)) - return - } - if resp.StatusCode != http.StatusAccepted { - r.options.Logger.Debug(r.ctx, "bad response from telemetry server", slog.F("status", resp.StatusCode)) - return - } - r.options.Logger.Debug(r.ctx, "submitted snapshot") - }() +func (r *remoteReporter) reportSync(snapshot *Snapshot) { + snapshot.DeploymentID = r.options.DeploymentID + data, err := json.Marshal(snapshot) + if err != nil { + r.options.Logger.Error(r.ctx, "marshal snapshot: %w", slog.Error(err)) + return + } + req, err := http.NewRequestWithContext(r.ctx, "POST", r.snapshotURL.String(), bytes.NewReader(data)) + if err != nil { + r.options.Logger.Error(r.ctx, "create request", slog.Error(err)) + return + } + req.Header.Set(VersionHeader, buildinfo.Version()) + resp, err := http.DefaultClient.Do(req) + if err != nil { + // If the request fails it's not necessarily an error. + // In an airgapped environment, it's fine if this fails! + r.options.Logger.Debug(r.ctx, "submit", slog.Error(err)) + return + } + if resp.StatusCode != http.StatusAccepted { + r.options.Logger.Debug(r.ctx, "bad response from telemetry server", slog.F("status", resp.StatusCode)) + return + } + r.options.Logger.Debug(r.ctx, "submitted snapshot") } func (r *remoteReporter) Close() { @@ -203,7 +203,7 @@ func (r *remoteReporter) reportWithDeployment() { r.options.Logger.Error(r.ctx, "create snapshot", slog.Error(err)) return } - r.Report(snapshot) + r.reportSync(snapshot) } // deployment collects host information and reports it to the telemetry server. @@ -446,6 +446,7 @@ func ConvertWorkspace(workspace database.Workspace) Workspace { TemplateID: workspace.TemplateID, CreatedAt: workspace.CreatedAt, Deleted: workspace.Deleted, + Name: workspace.Name, AutostartSchedule: workspace.AutostartSchedule.String, } } @@ -670,6 +671,7 @@ type Workspace struct { TemplateID uuid.UUID `json:"template_id"` CreatedAt time.Time `json:"created_at"` Deleted bool `json:"deleted"` + Name string `json:"name"` AutostartSchedule string `json:"autostart_schedule"` }