From 4f4f2d7b4186bc8e21c1b9c7b01dffacacfb1ea7 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 6 Feb 2022 00:27:37 +0000 Subject: [PATCH 1/4] ci: Replace DataDog CI with custom upload script This will reduce CI time by ~6 minutes across all of our runners. It's a bit janky, but I believe worth the slight maintainance burden. --- .github/workflows/coder.yaml | 40 ++----- codecov.yml | 1 + rules.go | 7 +- scripts/datadog-cireport/main.go | 178 +++++++++++++++++++++++++++++++ 4 files changed, 192 insertions(+), 34 deletions(-) create mode 100644 scripts/datadog-cireport/main.go diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml index 22c7583290147..05baf5b62c28e 100644 --- a/.github/workflows/coder.yaml +++ b/.github/workflows/coder.yaml @@ -159,45 +159,25 @@ jobs: -covermode=atomic -coverprofile="gotests.coverage" -timeout=3m -count=5 -race -short -parallel=2 + - name: Upload DataDog Trace + if: (success() || failure()) && github.actor != 'dependabot[bot]' + env: + DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_DATABASE: fake + run: go run scripts/datadog-cireport/main.go gotests.xml + - name: Test with PostgreSQL Database if: runner.os == 'Linux' run: DB=true gotestsum --junitfile="gotests.xml" --packages="./..." -- -covermode=atomic -coverprofile="gotests.coverage" -timeout=3m -count=1 -race -parallel=2 - - name: Setup Node for DataDog CLI - uses: actions/setup-node@v2 - if: always() && github.actor != 'dependabot[bot]' - with: - node-version: "14" - - - name: Cache DataDog CLI - if: always() && github.actor != 'dependabot[bot]' - uses: actions/cache@v2 - with: - path: | - ~/.npm - %LocalAppData%\npm-cache - key: datadogci- - restore-keys: datadogci- - - name: Upload DataDog Trace - if: always() && github.actor != 'dependabot[bot]' - # See: https://docs.datadoghq.com/continuous_integration/setup_tests/junit_upload/#collecting-environment-configuration-metadata + if: (success() || failure()) && github.actor != 'dependabot[bot]' env: DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_GIT_REPOSITORY_URL: ${{ github.repositoryUrl }} - DD_GIT_BRANCH: ${{ github.head_ref }} - DD_GIT_COMMIT_SHA: ${{ github.sha }} - DD_GIT_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - DD_GIT_COMMIT_AUTHOR_NAME: ${{ github.event.head_commit.author.name }} - DD_GIT_COMMIT_AUTHOR_EMAIL: ${{ github.event.head_commit.author.email }} - DD_GIT_COMMIT_COMMITTER_NAME: ${{ github.event.head_commit.committer.name }} - DD_GIT_COMMIT_COMMITTER_EMAIL: ${{ github.event.head_commit.committer.email }} - DD_TAGS: ${{ format('os.platform:{0},os.architecture:{1}', runner.os, runner.arch) }} - run: | - npm install -g @datadog/datadog-ci - datadog-ci junit upload --service coder gotests.xml + DD_DATABASE: postgresql + run: go run scripts/datadog-cireport/main.go gotests.xml - uses: codecov/codecov-action@v2 if: github.actor != 'dependabot[bot]' diff --git a/codecov.yml b/codecov.yml index 1ccc943684949..7666963e4f0cf 100644 --- a/codecov.yml +++ b/codecov.yml @@ -31,3 +31,4 @@ ignore: - peerbroker/proto - provisionerd/proto - provisionersdk/proto + - scripts/datadog-cireport diff --git a/rules.go b/rules.go index 7a95c89e016ed..a81f59395e676 100644 --- a/rules.go +++ b/rules.go @@ -10,18 +10,17 @@ func xerrors(m dsl.Matcher) { m.Import("errors") m.Import("fmt") m.Import("golang.org/x/xerrors") - msg := "Use xerrors to provide additional stacktrace information!" m.Match("fmt.Errorf($*args)"). Suggest("xerrors.New($args)"). - Report(msg) + Report("Use xerrors to provide additional stacktrace information!") m.Match("fmt.Errorf($*args)"). Suggest("xerrors.Errorf($args)"). - Report(msg) + Report("Use xerrors to provide additional stacktrace information!") m.Match("errors.New($msg)"). Where(m["msg"].Type.Is("string")). Suggest("xerrors.New($msg)"). - Report(msg) + Report("Use xerrors to provide additional stacktrace information!") } diff --git a/scripts/datadog-cireport/main.go b/scripts/datadog-cireport/main.go new file mode 100644 index 0000000000000..ddfba312e8b1d --- /dev/null +++ b/scripts/datadog-cireport/main.go @@ -0,0 +1,178 @@ +package main + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "log" + "mime/multipart" + "net/http" + "net/textproto" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" +) + +// The DataDog "cireport" API is not publicly documented, +// but implementation is available in their open-source CLI +// built for CI: https://github.com/DataDog/datadog-ci +// +// It's built using node, and took ~3 minutes to install and +// run on our Windows runner, and ~1 minute on all others. +// +// This script models that code as much as possible. +func main() { + apiKey := os.Getenv("DATADOG_API_KEY") + if apiKey == "" { + log.Fatal("DATADOG_API_KEY must be set!") + } + if len(os.Args) <= 1 { + log.Fatal("You must supply a filename to upload!") + } + + // Code (almost) verbatim translated from: + // https://github.com/DataDog/datadog-ci/blob/78d0da28e1c1af44333deabf1c9486e2ad66b8af/src/helpers/ci.ts#L194-L229 + var ( + githubServerURL = os.Getenv("GITHUB_SERVER_URL") + githubRepository = os.Getenv("GITHUB_REPOSITORY") + githubSHA = os.Getenv("GITHUB_SHA") + githubRunID = os.Getenv("GITHUB_RUN_ID") + pipelineURL = fmt.Sprintf("%s/%s/actions/runs/%s", githubServerURL, githubRepository, githubRunID) + jobURL = fmt.Sprintf("%s/%s/commit/%s/checks", githubServerURL, githubRepository, githubSHA) + ) + if os.Getenv("GITHUB_RUN_ATTEMPT") != "" { + pipelineURL += fmt.Sprintf("/attempts/%s", os.Getenv("GITHUB_RUN_ATTEMPT")) + } + + commitMessage, err := exec.Command("git", "show", "-s", "--format=%s").CombinedOutput() + if err != nil { + log.Fatalf("Get commit message: %s", err) + } + commitData, err := exec.Command("git", "show", "-s", "--format=%an,%ae,%ad,%cn,%ce,%cd").CombinedOutput() + if err != nil { + log.Fatalf("Get commit data: %s", err) + } + commitParts := strings.Split(string(commitData), ",") + + tags := map[string]string{ + "service": "coder", + "_dd.cireport_version": "2", + + "database": os.Getenv("DD_DATABASE"), + + // Additional tags found in DataDog docs. See: + // https://docs.datadoghq.com/continuous_integration/setup_tests/junit_upload/#collecting-environment-configuration-metadata + "os.platform": runtime.GOOS, + "os.architecture": runtime.GOARCH, + + "ci.job.url": jobURL, + "ci.pipeline.id": githubRunID, + "ci.pipeline.name": os.Getenv("GITHUB_WORKFLOW"), + "ci.pipeline.number": os.Getenv("GITHUB_RUN_NUMBER"), + "ci.pipeline.url": pipelineURL, + "ci.provider.name": "github", + "ci.workspace_path": os.Getenv("GITHUB_WORKSPACE"), + + "git.branch": os.Getenv("GITHUB_HEAD_REF"), + "git.commit.sha": githubSHA, + "git.repository_url": fmt.Sprintf("%s/%s.git", githubServerURL, githubRepository), + + "git.commit.message": strings.TrimSpace(string(commitMessage)), + "git.commit.author.name": commitParts[0], + "git.commit.author.email": commitParts[1], + "git.commit.author.date": commitParts[2], + "git.commit.committer.name": commitParts[3], + "git.commit.committer.email": commitParts[4], + "git.commit.committer.date": commitParts[5], + } + + xmlFilePath := filepath.Clean(os.Args[1]) + xmlFileData, err := os.ReadFile(xmlFilePath) + if err != nil { + log.Fatalf("Read %q: %s", xmlFilePath, err) + } + // https://github.com/DataDog/datadog-ci/blob/78d0da28e1c1af44333deabf1c9486e2ad66b8af/src/commands/junit/api.ts#L53 + var xmlCompressedBuffer bytes.Buffer + xmlGzipWriter := gzip.NewWriter(&xmlCompressedBuffer) + _, err = xmlGzipWriter.Write(xmlFileData) + if err != nil { + log.Fatalf("Write xml: %s", err) + } + err = xmlGzipWriter.Close() + if err != nil { + log.Fatalf("Close xml gzip writer: %s", err) + } + + // Represents FormData. See: + // https://github.com/DataDog/datadog-ci/blob/78d0da28e1c1af44333deabf1c9486e2ad66b8af/src/commands/junit/api.ts#L27 + var multipartBuffer bytes.Buffer + multipartWriter := multipart.NewWriter(&multipartBuffer) + + // Adds the event data. See: + // https://github.com/DataDog/datadog-ci/blob/78d0da28e1c1af44333deabf1c9486e2ad66b8af/src/commands/junit/api.ts#L42 + eventMimeHeader := make(textproto.MIMEHeader) + eventMimeHeader.Set("Content-Disposition", `form-data; name="event"; filename="event.json"`) + eventMimeHeader.Set("Content-Type", "application/json") + eventMultipartWriter, err := multipartWriter.CreatePart(eventMimeHeader) + if err != nil { + log.Fatalf("Create event multipart: %s", err) + } + eventJSON, err := json.Marshal(tags) + if err != nil { + log.Fatalf("Marshal tags: %s", err) + } + _, err = eventMultipartWriter.Write(eventJSON) + if err != nil { + log.Fatalf("Write event JSON: %s", err) + } + + // This seems really strange, but better to follow the implementation. See: + // https://github.com/DataDog/datadog-ci/blob/78d0da28e1c1af44333deabf1c9486e2ad66b8af/src/commands/junit/api.ts#L44-L55 + xmlFilename := fmt.Sprintf("%s-coder-%s-%s-%s", filepath.Base(xmlFilePath), githubSHA, pipelineURL, jobURL) + xmlFilename = regexp.MustCompile("[^a-z0-9]").ReplaceAllString(xmlFilename, "_") + + xmlMimeHeader := make(textproto.MIMEHeader) + xmlMimeHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="junit_xml_report_file"; filename="%s.xml.gz"`, xmlFilename)) + xmlMimeHeader.Set("Content-Type", "application/octet-stream") + inputWriter, err := multipartWriter.CreatePart(xmlMimeHeader) + if err != nil { + log.Fatalf("Create xml.gz multipart: %s", err) + } + _, err = inputWriter.Write(xmlCompressedBuffer.Bytes()) + if err != nil { + log.Fatalf("Write xml.gz: %s", err) + } + err = multipartWriter.Close() + if err != nil { + log.Fatalf("Close: %s", err) + } + + ctx := context.Background() + req, err := http.NewRequestWithContext(ctx, "POST", "https://cireport-intake.datadoghq.com/api/v2/cireport", &multipartBuffer) + if err != nil { + log.Fatalf("Create request: %s", err) + } + req.Header.Set("Content-Type", multipartWriter.FormDataContentType()) + req.Header.Set("DD-API-KEY", apiKey) + + res, err := http.DefaultClient.Do(req) + if err != nil { + log.Fatalf("Do request: %s", err) + } + defer res.Body.Close() + var msg json.RawMessage + err = json.NewDecoder(res.Body).Decode(&msg) + if err != nil { + log.Fatalf("Decode response: %s", err) + } + msg, err = json.MarshalIndent(msg, "", "\t") + if err != nil { + log.Fatalf("Pretty print: %s", err) + } + _, _ = fmt.Printf("Status code: %d\nResponse: %s\n", res.StatusCode, msg) +} From 2dda551ebff7525ff2cdf5feb08695270fd71a56 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 6 Feb 2022 01:40:46 +0000 Subject: [PATCH 2/4] Fix test race when job would complete too early --- coderd/workspacehistory_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/coderd/workspacehistory_test.go b/coderd/workspacehistory_test.go index b7ef8855264fb..a1e1d71351baa 100644 --- a/coderd/workspacehistory_test.go +++ b/coderd/workspacehistory_test.go @@ -59,10 +59,12 @@ func TestPostWorkspaceHistoryByUser(t *testing.T) { t.Parallel() client := coderdtest.New(t) user := coderdtest.CreateInitialUser(t, client) - coderdtest.NewProvisionerDaemon(t, client) + closeDaemon := coderdtest.NewProvisionerDaemon(t, client) project := coderdtest.CreateProject(t, client, user.Organization) version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, nil) coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name) + // Close here so workspace history doesn't process! + closeDaemon.Close() workspace := coderdtest.CreateWorkspace(t, client, "me", project.ID) _, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{ ProjectVersionID: version.ID, From 49799a0ededdd4c85e15df51752615ba1baadf7d Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 6 Feb 2022 02:30:37 +0000 Subject: [PATCH 3/4] Fix job cancelation override --- .github/workflows/coder.yaml | 2 ++ coderd/provisionerdaemons.go | 7 +++++++ scripts/datadog-cireport/main.go | 14 ++++++++++---- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml index 05baf5b62c28e..86dda496008c5 100644 --- a/.github/workflows/coder.yaml +++ b/.github/workflows/coder.yaml @@ -164,6 +164,7 @@ jobs: env: DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} DD_DATABASE: fake + GIT_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: go run scripts/datadog-cireport/main.go gotests.xml - name: Test with PostgreSQL Database @@ -177,6 +178,7 @@ jobs: env: DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} DD_DATABASE: postgresql + GIT_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: go run scripts/datadog-cireport/main.go gotests.xml - uses: codecov/codecov-action@v2 diff --git a/coderd/provisionerdaemons.go b/coderd/provisionerdaemons.go index f6d616383ab1a..87611a8ba6734 100644 --- a/coderd/provisionerdaemons.go +++ b/coderd/provisionerdaemons.go @@ -377,6 +377,13 @@ func (server *provisionerdServer) CancelJob(ctx context.Context, cancelJob *prot if err != nil { return nil, xerrors.Errorf("parse job id: %w", err) } + job, err := server.Database.GetProvisionerJobByID(ctx, jobID) + if err != nil { + return nil, xerrors.Errorf("get provisioner job: %w", err) + } + if job.CompletedAt.Valid { + return nil, xerrors.Errorf("job already completed") + } err = server.Database.UpdateProvisionerJobWithCompleteByID(ctx, database.UpdateProvisionerJobWithCompleteByIDParams{ ID: jobID, CompletedAt: sql.NullTime{ diff --git a/scripts/datadog-cireport/main.go b/scripts/datadog-cireport/main.go index ddfba312e8b1d..5dbf420ec537e 100644 --- a/scripts/datadog-cireport/main.go +++ b/scripts/datadog-cireport/main.go @@ -49,7 +49,7 @@ func main() { pipelineURL += fmt.Sprintf("/attempts/%s", os.Getenv("GITHUB_RUN_ATTEMPT")) } - commitMessage, err := exec.Command("git", "show", "-s", "--format=%s").CombinedOutput() + commitMessage, err := exec.Command("git", "log", "-1", "--pretty=format:%s").CombinedOutput() if err != nil { log.Fatalf("Get commit message: %s", err) } @@ -63,7 +63,7 @@ func main() { "service": "coder", "_dd.cireport_version": "2", - "database": os.Getenv("DD_DATABASE"), + "test.traits": fmt.Sprintf(`{"database":["%s"]}`, os.Getenv("DD_DATABASE")), // Additional tags found in DataDog docs. See: // https://docs.datadoghq.com/continuous_integration/setup_tests/junit_upload/#collecting-environment-configuration-metadata @@ -82,7 +82,7 @@ func main() { "git.commit.sha": githubSHA, "git.repository_url": fmt.Sprintf("%s/%s.git", githubServerURL, githubRepository), - "git.commit.message": strings.TrimSpace(string(commitMessage)), + "git.commit.message": string(commitMessage), "git.commit.author.name": commitParts[0], "git.commit.author.email": commitParts[1], "git.commit.author.date": commitParts[2], @@ -174,5 +174,11 @@ func main() { if err != nil { log.Fatalf("Pretty print: %s", err) } - _, _ = fmt.Printf("Status code: %d\nResponse: %s\n", res.StatusCode, msg) + _, _ = fmt.Println(string(msg)) + msg, err = json.MarshalIndent(tags, "", "\t") + if err != nil { + log.Fatalf("Marshal tags: %s", err) + } + _, _ = fmt.Println(string(msg)) + _, _ = fmt.Printf("Status: %d\n", res.StatusCode) } From 763367ec412211b0d32699fa96f003dcbffb4335 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 6 Feb 2022 03:18:10 +0000 Subject: [PATCH 4/4] Fix race where provisioner job is inserted before project version --- coderd/projectversion.go | 35 ++++++++++++++++----------------- coderd/workspacehistory.go | 39 ++++++++++++++++++------------------- codersdk/workspaces_test.go | 3 ++- 3 files changed, 38 insertions(+), 39 deletions(-) diff --git a/coderd/projectversion.go b/coderd/projectversion.go index 1f045e1c09101..b27b4959af0b0 100644 --- a/coderd/projectversion.go +++ b/coderd/projectversion.go @@ -125,16 +125,29 @@ func (api *api) postProjectVersionByOrganization(rw http.ResponseWriter, r *http var provisionerJob database.ProvisionerJob var projectVersion database.ProjectVersion err = api.Database.InTx(func(db database.Store) error { - projectVersionID := uuid.New() + provisionerJobID := uuid.New() + projectVersion, err = api.Database.InsertProjectVersion(r.Context(), database.InsertProjectVersionParams{ + ID: uuid.New(), + ProjectID: project.ID, + CreatedAt: database.Now(), + UpdatedAt: database.Now(), + Name: namesgenerator.GetRandomName(1), + StorageMethod: createProjectVersion.StorageMethod, + StorageSource: createProjectVersion.StorageSource, + ImportJobID: provisionerJobID, + }) + if err != nil { + return xerrors.Errorf("insert project version: %s", err) + } + input, err := json.Marshal(projectImportJob{ - ProjectVersionID: projectVersionID, + ProjectVersionID: projectVersion.ID, }) if err != nil { return xerrors.Errorf("marshal import job: %w", err) } - provisionerJob, err = db.InsertProvisionerJob(r.Context(), database.InsertProvisionerJobParams{ - ID: uuid.New(), + ID: provisionerJobID, CreatedAt: database.Now(), UpdatedAt: database.Now(), InitiatorID: apiKey.UserID, @@ -146,20 +159,6 @@ func (api *api) postProjectVersionByOrganization(rw http.ResponseWriter, r *http if err != nil { return xerrors.Errorf("insert provisioner job: %w", err) } - - projectVersion, err = api.Database.InsertProjectVersion(r.Context(), database.InsertProjectVersionParams{ - ID: projectVersionID, - ProjectID: project.ID, - CreatedAt: database.Now(), - UpdatedAt: database.Now(), - Name: namesgenerator.GetRandomName(1), - StorageMethod: createProjectVersion.StorageMethod, - StorageSource: createProjectVersion.StorageSource, - ImportJobID: provisionerJob.ID, - }) - if err != nil { - return xerrors.Errorf("insert project version: %s", err) - } return nil }) if err != nil { diff --git a/coderd/workspacehistory.go b/coderd/workspacehistory.go index dae2a4d157436..5d7cfac8ef0fe 100644 --- a/coderd/workspacehistory.go +++ b/coderd/workspacehistory.go @@ -126,17 +126,32 @@ func (api *api) postWorkspaceHistoryByUser(rw http.ResponseWriter, r *http.Reque // This must happen in a transaction to ensure history can be inserted, and // the prior history can update it's "after" column to point at the new. err = api.Database.InTx(func(db database.Store) error { - // Generate the ID before-hand so the provisioner job is aware of it! - workspaceHistoryID := uuid.New() + provisionerJobID := uuid.New() + workspaceHistory, err = db.InsertWorkspaceHistory(r.Context(), database.InsertWorkspaceHistoryParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + UpdatedAt: database.Now(), + WorkspaceID: workspace.ID, + ProjectVersionID: projectVersion.ID, + BeforeID: priorHistoryID, + Name: namesgenerator.GetRandomName(1), + Initiator: user.ID, + Transition: createBuild.Transition, + ProvisionJobID: provisionerJobID, + }) + if err != nil { + return xerrors.Errorf("insert workspace history: %w", err) + } + input, err := json.Marshal(workspaceProvisionJob{ - WorkspaceHistoryID: workspaceHistoryID, + WorkspaceHistoryID: workspaceHistory.ID, }) if err != nil { return xerrors.Errorf("marshal provision job: %w", err) } provisionerJob, err = db.InsertProvisionerJob(r.Context(), database.InsertProvisionerJobParams{ - ID: uuid.New(), + ID: provisionerJobID, CreatedAt: database.Now(), UpdatedAt: database.Now(), InitiatorID: user.ID, @@ -149,22 +164,6 @@ func (api *api) postWorkspaceHistoryByUser(rw http.ResponseWriter, r *http.Reque return xerrors.Errorf("insert provisioner job: %w", err) } - workspaceHistory, err = db.InsertWorkspaceHistory(r.Context(), database.InsertWorkspaceHistoryParams{ - ID: workspaceHistoryID, - CreatedAt: database.Now(), - UpdatedAt: database.Now(), - WorkspaceID: workspace.ID, - ProjectVersionID: projectVersion.ID, - BeforeID: priorHistoryID, - Name: namesgenerator.GetRandomName(1), - Initiator: user.ID, - Transition: createBuild.Transition, - ProvisionJobID: provisionerJob.ID, - }) - if err != nil { - return xerrors.Errorf("insert workspace history: %w", err) - } - if priorHistoryID.Valid { // Update the prior history entries "after" column. err = db.UpdateWorkspaceHistoryByID(r.Context(), database.UpdateWorkspaceHistoryByIDParams{ diff --git a/codersdk/workspaces_test.go b/codersdk/workspaces_test.go index 5ca82e4fbf4f4..e5d170c170311 100644 --- a/codersdk/workspaces_test.go +++ b/codersdk/workspaces_test.go @@ -220,12 +220,13 @@ func TestFollowWorkspaceHistoryLogsAfter(t *testing.T) { }) coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name) workspace := coderdtest.CreateWorkspace(t, client, "", project.ID) + after := database.Now() history, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{ ProjectVersionID: version.ID, Transition: database.WorkspaceTransitionCreate, }) require.NoError(t, err) - logs, err := client.FollowWorkspaceHistoryLogsAfter(context.Background(), "", workspace.Name, history.Name, time.Time{}) + logs, err := client.FollowWorkspaceHistoryLogsAfter(context.Background(), "", workspace.Name, history.Name, after) require.NoError(t, err) _, ok := <-logs require.True(t, ok)