Skip to content

Commit 3002a27

Browse files
kylecarbskylecarbs
authored
ci: Replace DataDog CI with custom upload script (#169)
* ci: Replace DataDog CI with custom upload script This will reduce CI time by ~6 minutes across all of our runners. It's a bit janky, but I believe worth the slight maintainance burden. * Fix test race when job would complete too early * Fix job cancelation override * Fix race where provisioner job is inserted before project version
1 parent 4cd0261 commit 3002a27

File tree

9 files changed

+248
-74
lines changed

9 files changed

+248
-74
lines changed

‎.github/workflows/coder.yaml

Lines changed: 12 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -159,45 +159,27 @@ jobs:
159159
-covermode=atomic -coverprofile="gotests.coverage"
160160
-timeout=3m -count=5 -race -short -parallel=2
161161

162+
- name: Upload DataDog Trace
163+
if: (success() || failure()) && github.actor != 'dependabot[bot]'
164+
env:
165+
DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
166+
DD_DATABASE: fake
167+
GIT_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
168+
run: go run scripts/datadog-cireport/main.go gotests.xml
169+
162170
- name: Test with PostgreSQL Database
163171
if: runner.os == 'Linux'
164172
run: DB=true gotestsum --junitfile="gotests.xml" --packages="./..." --
165173
-covermode=atomic -coverprofile="gotests.coverage" -timeout=3m
166174
-count=1 -race -parallel=2
167175

168-
- name: Setup Node for DataDog CLI
169-
uses: actions/setup-node@v2
170-
if: always() && github.actor != 'dependabot[bot]'
171-
with:
172-
node-version: "14"
173-
174-
- name: Cache DataDog CLI
175-
if: always() && github.actor != 'dependabot[bot]'
176-
uses: actions/cache@v2
177-
with:
178-
path: |
179-
~/.npm
180-
%LocalAppData%\npm-cache
181-
key: datadogci-
182-
restore-keys: datadogci-
183-
184176
- name: Upload DataDog Trace
185-
if: always() && github.actor != 'dependabot[bot]'
186-
# See: https://docs.datadoghq.com/continuous_integration/setup_tests/junit_upload/#collecting-environment-configuration-metadata
177+
if: (success() || failure()) && github.actor != 'dependabot[bot]'
187178
env:
188179
DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
189-
DD_GIT_REPOSITORY_URL: ${{ github.repositoryUrl }}
190-
DD_GIT_BRANCH: ${{ github.head_ref }}
191-
DD_GIT_COMMIT_SHA: ${{ github.sha }}
192-
DD_GIT_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
193-
DD_GIT_COMMIT_AUTHOR_NAME: ${{ github.event.head_commit.author.name }}
194-
DD_GIT_COMMIT_AUTHOR_EMAIL: ${{ github.event.head_commit.author.email }}
195-
DD_GIT_COMMIT_COMMITTER_NAME: ${{ github.event.head_commit.committer.name }}
196-
DD_GIT_COMMIT_COMMITTER_EMAIL: ${{ github.event.head_commit.committer.email }}
197-
DD_TAGS: ${{ format('os.platform:{0},os.architecture:{1}', runner.os, runner.arch) }}
198-
run: |
199-
npm install -g @datadog/datadog-ci
200-
datadog-ci junit upload --service coder gotests.xml
180+
DD_DATABASE: postgresql
181+
GIT_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
182+
run: go run scripts/datadog-cireport/main.go gotests.xml
201183

202184
- uses: codecov/codecov-action@v2
203185
if: github.actor != 'dependabot[bot]'

‎codecov.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,4 @@ ignore:
3131
- peerbroker/proto
3232
- provisionerd/proto
3333
- provisionersdk/proto
34+
- scripts/datadog-cireport

‎coderd/projectversion.go

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -125,16 +125,29 @@ func (api *api) postProjectVersionByOrganization(rw http.ResponseWriter, r *http
125125
var provisionerJob database.ProvisionerJob
126126
var projectVersion database.ProjectVersion
127127
err = api.Database.InTx(func(db database.Store) error {
128-
projectVersionID := uuid.New()
128+
provisionerJobID := uuid.New()
129+
projectVersion, err = api.Database.InsertProjectVersion(r.Context(), database.InsertProjectVersionParams{
130+
ID: uuid.New(),
131+
ProjectID: project.ID,
132+
CreatedAt: database.Now(),
133+
UpdatedAt: database.Now(),
134+
Name: namesgenerator.GetRandomName(1),
135+
StorageMethod: createProjectVersion.StorageMethod,
136+
StorageSource: createProjectVersion.StorageSource,
137+
ImportJobID: provisionerJobID,
138+
})
139+
if err != nil {
140+
return xerrors.Errorf("insert project version: %s", err)
141+
}
142+
129143
input, err := json.Marshal(projectImportJob{
130-
ProjectVersionID: projectVersionID,
144+
ProjectVersionID: projectVersion.ID,
131145
})
132146
if err != nil {
133147
return xerrors.Errorf("marshal import job: %w", err)
134148
}
135-
136149
provisionerJob, err = db.InsertProvisionerJob(r.Context(), database.InsertProvisionerJobParams{
137-
ID: uuid.New(),
150+
ID: provisionerJobID,
138151
CreatedAt: database.Now(),
139152
UpdatedAt: database.Now(),
140153
InitiatorID: apiKey.UserID,
@@ -146,20 +159,6 @@ func (api *api) postProjectVersionByOrganization(rw http.ResponseWriter, r *http
146159
if err != nil {
147160
return xerrors.Errorf("insert provisioner job: %w", err)
148161
}
149-
150-
projectVersion, err = api.Database.InsertProjectVersion(r.Context(), database.InsertProjectVersionParams{
151-
ID: projectVersionID,
152-
ProjectID: project.ID,
153-
CreatedAt: database.Now(),
154-
UpdatedAt: database.Now(),
155-
Name: namesgenerator.GetRandomName(1),
156-
StorageMethod: createProjectVersion.StorageMethod,
157-
StorageSource: createProjectVersion.StorageSource,
158-
ImportJobID: provisionerJob.ID,
159-
})
160-
if err != nil {
161-
return xerrors.Errorf("insert project version: %s", err)
162-
}
163162
return nil
164163
})
165164
if err != nil {

‎coderd/provisionerdaemons.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,13 @@ func (server *provisionerdServer) CancelJob(ctx context.Context, cancelJob *prot
377377
if err != nil {
378378
return nil, xerrors.Errorf("parse job id: %w", err)
379379
}
380+
job, err := server.Database.GetProvisionerJobByID(ctx, jobID)
381+
if err != nil {
382+
return nil, xerrors.Errorf("get provisioner job: %w", err)
383+
}
384+
if job.CompletedAt.Valid {
385+
return nil, xerrors.Errorf("job already completed")
386+
}
380387
err = server.Database.UpdateProvisionerJobWithCompleteByID(ctx, database.UpdateProvisionerJobWithCompleteByIDParams{
381388
ID: jobID,
382389
CompletedAt: sql.NullTime{

‎coderd/workspacehistory.go

Lines changed: 19 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -126,17 +126,32 @@ func (api *api) postWorkspaceHistoryByUser(rw http.ResponseWriter, r *http.Reque
126126
// This must happen in a transaction to ensure history can be inserted, and
127127
// the prior history can update it's "after" column to point at the new.
128128
err = api.Database.InTx(func(db database.Store) error {
129-
// Generate the ID before-hand so the provisioner job is aware of it!
130-
workspaceHistoryID := uuid.New()
129+
provisionerJobID := uuid.New()
130+
workspaceHistory, err = db.InsertWorkspaceHistory(r.Context(), database.InsertWorkspaceHistoryParams{
131+
ID: uuid.New(),
132+
CreatedAt: database.Now(),
133+
UpdatedAt: database.Now(),
134+
WorkspaceID: workspace.ID,
135+
ProjectVersionID: projectVersion.ID,
136+
BeforeID: priorHistoryID,
137+
Name: namesgenerator.GetRandomName(1),
138+
Initiator: user.ID,
139+
Transition: createBuild.Transition,
140+
ProvisionJobID: provisionerJobID,
141+
})
142+
if err != nil {
143+
return xerrors.Errorf("insert workspace history: %w", err)
144+
}
145+
131146
input, err := json.Marshal(workspaceProvisionJob{
132-
WorkspaceHistoryID: workspaceHistoryID,
147+
WorkspaceHistoryID: workspaceHistory.ID,
133148
})
134149
if err != nil {
135150
return xerrors.Errorf("marshal provision job: %w", err)
136151
}
137152

138153
provisionerJob, err = db.InsertProvisionerJob(r.Context(), database.InsertProvisionerJobParams{
139-
ID: uuid.New(),
154+
ID: provisionerJobID,
140155
CreatedAt: database.Now(),
141156
UpdatedAt: database.Now(),
142157
InitiatorID: user.ID,
@@ -149,22 +164,6 @@ func (api *api) postWorkspaceHistoryByUser(rw http.ResponseWriter, r *http.Reque
149164
return xerrors.Errorf("insert provisioner job: %w", err)
150165
}
151166

152-
workspaceHistory, err = db.InsertWorkspaceHistory(r.Context(), database.InsertWorkspaceHistoryParams{
153-
ID: workspaceHistoryID,
154-
CreatedAt: database.Now(),
155-
UpdatedAt: database.Now(),
156-
WorkspaceID: workspace.ID,
157-
ProjectVersionID: projectVersion.ID,
158-
BeforeID: priorHistoryID,
159-
Name: namesgenerator.GetRandomName(1),
160-
Initiator: user.ID,
161-
Transition: createBuild.Transition,
162-
ProvisionJobID: provisionerJob.ID,
163-
})
164-
if err != nil {
165-
return xerrors.Errorf("insert workspace history: %w", err)
166-
}
167-
168167
if priorHistoryID.Valid {
169168
// Update the prior history entries "after" column.
170169
err = db.UpdateWorkspaceHistoryByID(r.Context(), database.UpdateWorkspaceHistoryByIDParams{

‎coderd/workspacehistory_test.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,12 @@ func TestPostWorkspaceHistoryByUser(t *testing.T) {
5959
t.Parallel()
6060
client := coderdtest.New(t)
6161
user := coderdtest.CreateInitialUser(t, client)
62-
coderdtest.NewProvisionerDaemon(t, client)
62+
closeDaemon := coderdtest.NewProvisionerDaemon(t, client)
6363
project := coderdtest.CreateProject(t, client, user.Organization)
6464
version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, nil)
6565
coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name)
66+
// Close here so workspace history doesn't process!
67+
closeDaemon.Close()
6668
workspace := coderdtest.CreateWorkspace(t, client, "me", project.ID)
6769
_, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{
6870
ProjectVersionID: version.ID,

‎codersdk/workspaces_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,12 +220,13 @@ func TestFollowWorkspaceHistoryLogsAfter(t *testing.T) {
220220
})
221221
coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name)
222222
workspace := coderdtest.CreateWorkspace(t, client, "", project.ID)
223+
after := database.Now()
223224
history, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{
224225
ProjectVersionID: version.ID,
225226
Transition: database.WorkspaceTransitionCreate,
226227
})
227228
require.NoError(t, err)
228-
logs, err := client.FollowWorkspaceHistoryLogsAfter(context.Background(), "", workspace.Name, history.Name, time.Time{})
229+
logs, err := client.FollowWorkspaceHistoryLogsAfter(context.Background(), "", workspace.Name, history.Name, after)
229230
require.NoError(t, err)
230231
_, ok := <-logs
231232
require.True(t, ok)

‎rules.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,18 +10,17 @@ func xerrors(m dsl.Matcher) {
1010
m.Import("errors")
1111
m.Import("fmt")
1212
m.Import("golang.org/x/xerrors")
13-
msg := "Use xerrors to provide additional stacktrace information!"
1413

1514
m.Match("fmt.Errorf($*args)").
1615
Suggest("xerrors.New($args)").
17-
Report(msg)
16+
Report("Use xerrors to provide additional stacktrace information!")
1817

1918
m.Match("fmt.Errorf($*args)").
2019
Suggest("xerrors.Errorf($args)").
21-
Report(msg)
20+
Report("Use xerrors to provide additional stacktrace information!")
2221

2322
m.Match("errors.New($msg)").
2423
Where(m["msg"].Type.Is("string")).
2524
Suggest("xerrors.New($msg)").
26-
Report(msg)
25+
Report("Use xerrors to provide additional stacktrace information!")
2726
}

0 commit comments

Comments
 (0)