coder · ammario · Nov 17, 2022 · Oct 25, 2022 · Nov 7, 2022 · Nov 7, 2022
diff --git a/cli/loadtest_test.go b/cli/loadtest_test.go
@@ -26,6 +26,7 @@ import (
 )
 
 func TestLoadTest(t *testing.T) {
+	t.Skipf("This test is flakey. See https://github.com/coder/coder/issues/4942")
 	t.Parallel()
 
 	t.Run("PlaceboFromStdin", func(t *testing.T) {

diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go
@@ -300,17 +300,18 @@ func (q *fakeQuerier) GetTemplateAverageBuildTime(ctx context.Context, arg datab
 		}
 	}
 
-	tryMedian := func(fs []float64) float64 {
+	tryPercentile := func(fs []float64, p float64) float64 {
 		if len(fs) == 0 {
 			return -1
 		}
 		sort.Float64s(fs)
-		return fs[len(fs)/2]
+		return fs[int(float64(len(fs))*p/100)]
 	}
+
 	var row database.GetTemplateAverageBuildTimeRow
-	row.DeleteMedian = tryMedian(deleteTimes)
-	row.StopMedian = tryMedian(stopTimes)
-	row.StartMedian = tryMedian(startTimes)
+	row.Delete50, row.Delete95 = tryPercentile(deleteTimes, 50), tryPercentile(deleteTimes, 95)
+	row.Stop50, row.Stop95 = tryPercentile(stopTimes, 50), tryPercentile(stopTimes, 95)
+	row.Start50, row.Start95 = tryPercentile(startTimes, 50), tryPercentile(startTimes, 95)
 	return row, nil
 }
 

diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go
diff --git a/coderd/database/queries/templates.sql b/coderd/database/queries/templates.sql
@@ -142,8 +142,11 @@ ORDER BY
 SELECT
 	-- Postgres offers no clear way to DRY this short of a function or other
 	-- complexities.
-	coalesce((PERCENTILE_DISC(0.5) WITHIN GROUP(ORDER BY exec_time_sec) FILTER (WHERE transition = 'start')), -1)::FLOAT AS start_median,
-	coalesce((PERCENTILE_DISC(0.5) WITHIN GROUP(ORDER BY exec_time_sec) FILTER (WHERE transition = 'stop')), -1)::FLOAT AS stop_median,
-	coalesce((PERCENTILE_DISC(0.5) WITHIN GROUP(ORDER BY exec_time_sec) FILTER (WHERE transition = 'delete')), -1)::FLOAT AS delete_median
+	coalesce((PERCENTILE_DISC(0.5) WITHIN GROUP(ORDER BY exec_time_sec) FILTER (WHERE transition = 'start')), -1)::FLOAT AS start_50,
+	coalesce((PERCENTILE_DISC(0.5) WITHIN GROUP(ORDER BY exec_time_sec) FILTER (WHERE transition = 'stop')), -1)::FLOAT AS stop_50,
+	coalesce((PERCENTILE_DISC(0.5) WITHIN GROUP(ORDER BY exec_time_sec) FILTER (WHERE transition = 'delete')), -1)::FLOAT AS delete_50,
+	coalesce((PERCENTILE_DISC(0.95) WITHIN GROUP(ORDER BY exec_time_sec) FILTER (WHERE transition = 'start')), -1)::FLOAT AS start_95,
+	coalesce((PERCENTILE_DISC(0.95) WITHIN GROUP(ORDER BY exec_time_sec) FILTER (WHERE transition = 'stop')), -1)::FLOAT AS stop_95,
+	coalesce((PERCENTILE_DISC(0.95) WITHIN GROUP(ORDER BY exec_time_sec) FILTER (WHERE transition = 'delete')), -1)::FLOAT AS delete_95
 FROM build_times
 ;
diff --git a/coderd/metricscache/metricscache.go b/coderd/metricscache/metricscache.go
@@ -242,7 +242,11 @@ func (c *Cache) TemplateUniqueUsers(id uuid.UUID) (int, bool) {
 }
 
 func (c *Cache) TemplateBuildTimeStats(id uuid.UUID) codersdk.TemplateBuildTimeStats {
-	var unknown codersdk.TemplateBuildTimeStats
+	unknown := codersdk.TemplateBuildTimeStats{
+		codersdk.WorkspaceTransitionStart:  {},
+		codersdk.WorkspaceTransitionStop:   {},
+		codersdk.WorkspaceTransitionDelete: {},
+	}
 
 	m := c.templateAverageBuildTime.Load()
 	if m == nil {
@@ -256,7 +260,7 @@ func (c *Cache) TemplateBuildTimeStats(id uuid.UUID) codersdk.TemplateBuildTimeS
 		return unknown
 	}
 
-	convertMedian := func(m float64) *int64 {
+	convertMillis := func(m float64) *int64 {
 		if m <= 0 {
 			return nil
 		}
@@ -265,8 +269,17 @@ func (c *Cache) TemplateBuildTimeStats(id uuid.UUID) codersdk.TemplateBuildTimeS
 	}
 
 	return codersdk.TemplateBuildTimeStats{
-		StartMillis:  convertMedian(resp.StartMedian),
-		StopMillis:   convertMedian(resp.StopMedian),
-		DeleteMillis: convertMedian(resp.DeleteMedian),
+		codersdk.WorkspaceTransitionStart: {
+			P50: convertMillis(resp.Start50),
+			P95: convertMillis(resp.Start95),
+		},
+		codersdk.WorkspaceTransitionStop: {
+			P50: convertMillis(resp.Stop50),
+			P95: convertMillis(resp.Stop95),
+		},
+		codersdk.WorkspaceTransitionDelete: {
+			P50: convertMillis(resp.Delete50),
+			P95: convertMillis(resp.Delete95),
+		},
 	}
 }
diff --git a/coderd/metricscache/metricscache_test.go b/coderd/metricscache/metricscache_test.go
@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"cdr.dev/slog/sloggers/slogtest"
@@ -204,6 +203,12 @@ func clockTime(t time.Time, hour, minute, sec int) time.Time {
 	return time.Date(t.Year(), t.Month(), t.Day(), hour, minute, sec, t.Nanosecond(), t.Location())
 }
 
+func requireBuildTimeStatsEmpty(t *testing.T, stats codersdk.TemplateBuildTimeStats) {
+	require.Empty(t, stats[codersdk.WorkspaceTransitionStart])
+	require.Empty(t, stats[codersdk.WorkspaceTransitionStop])
+	require.Empty(t, stats[codersdk.WorkspaceTransitionDelete])
+}
+
 func TestCache_BuildTime(t *testing.T) {
 	t.Parallel()
 
@@ -296,7 +301,7 @@ func TestCache_BuildTime(t *testing.T) {
 			require.NoError(t, err)
 
 			gotStats := cache.TemplateBuildTimeStats(template.ID)
-			require.Empty(t, gotStats, "should not have loaded yet")
+			requireBuildTimeStatsEmpty(t, gotStats)
 
 			for _, row := range tt.args.rows {
 				_, err := db.InsertProvisionerJob(ctx, database.InsertProvisionerJobParams{
@@ -328,36 +333,32 @@ func TestCache_BuildTime(t *testing.T) {
 			}
 
 			if tt.want.loads {
+				wantTransition := codersdk.WorkspaceTransition(tt.args.transition)
 				require.Eventuallyf(t, func() bool {
 					stats := cache.TemplateBuildTimeStats(template.ID)
-					return stats != codersdk.TemplateBuildTimeStats{}
+					return stats[wantTransition] != codersdk.TransitionStats{}
 				}, testutil.WaitLong, testutil.IntervalMedium,
 					"BuildTime never populated",
 				)
 
 				gotStats = cache.TemplateBuildTimeStats(template.ID)
-
-				if tt.args.transition == database.WorkspaceTransitionDelete {
-					require.Nil(t, gotStats.StopMillis)
-					require.Nil(t, gotStats.StartMillis)
-					require.Equal(t, tt.want.buildTimeMs, *gotStats.DeleteMillis)
-				}
-				if tt.args.transition == database.WorkspaceTransitionStart {
-					require.Nil(t, gotStats.StopMillis)
-					require.Nil(t, gotStats.DeleteMillis)
-					require.Equal(t, tt.want.buildTimeMs, *gotStats.StartMillis)
-				}
-				if tt.args.transition == database.WorkspaceTransitionStop {
-					require.Nil(t, gotStats.StartMillis)
-					require.Nil(t, gotStats.DeleteMillis)
-					require.Equal(t, tt.want.buildTimeMs, *gotStats.StopMillis)
+				for transition, stats := range gotStats {
+					if transition == wantTransition {
+						require.Equal(t, tt.want.buildTimeMs, *stats.P50)
+					} else {
+						require.Empty(
+							t, stats, "%v", transition,
+						)
+					}
 				}
 			} else {
+				var stats codersdk.TemplateBuildTimeStats
 				require.Never(t, func() bool {
-					stats := cache.TemplateBuildTimeStats(template.ID)
-					return !assert.Empty(t, stats)
+					stats = cache.TemplateBuildTimeStats(template.ID)
+					requireBuildTimeStatsEmpty(t, stats)
+					return t.Failed()
 				}, testutil.WaitShort/2, testutil.IntervalMedium,
-					"BuildTimeStats populated",
+					"BuildTimeStats populated", stats,
 				)
 			}
 		})

diff --git a/coderd/templates_test.go b/coderd/templates_test.go
@@ -536,7 +536,7 @@ func TestTemplateMetrics(t *testing.T) {
 	})
 	template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
 	require.Equal(t, -1, template.ActiveUserCount)
-	require.Empty(t, template.BuildTimeStats)
+	require.Empty(t, template.BuildTimeStats[codersdk.WorkspaceTransitionStart])
 
 	coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
 	workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
@@ -607,7 +607,7 @@ func TestTemplateMetrics(t *testing.T) {
 	require.Eventuallyf(t, func() bool {
 		template, err = client.Template(ctx, template.ID)
 		require.NoError(t, err)
-		startMs := template.BuildTimeStats.StartMillis
+		startMs := template.BuildTimeStats[codersdk.WorkspaceTransitionStart].P50
 		return startMs != nil && *startMs > 1
 	},
 		testutil.WaitShort, testutil.IntervalFast,

diff --git a/codersdk/templates.go b/codersdk/templates.go
@@ -33,12 +33,12 @@ type Template struct {
 	CreatedByName    string                 `json:"created_by_name"`
 }
 
-type TemplateBuildTimeStats struct {
-	StartMillis  *int64 `json:"start_ms"`
-	StopMillis   *int64 `json:"stop_ms"`
-	DeleteMillis *int64 `json:"delete_ms"`
+type TransitionStats struct {
+	P50 *int64
+	P95 *int64
 }
 
+type TemplateBuildTimeStats map[WorkspaceTransition]TransitionStats
 type UpdateActiveTemplateVersion struct {
 	ID uuid.UUID `json:"id" validate:"required"`
 }

diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts
@@ -651,11 +651,10 @@ export interface TemplateACL {
 }
 
 // From codersdk/templates.go
-export interface TemplateBuildTimeStats {
-  readonly start_ms?: number
-  readonly stop_ms?: number
-  readonly delete_ms?: number
-}
+export type TemplateBuildTimeStats = Record<
+  WorkspaceTransition,
+  TransitionStats
+>
 
 // From codersdk/templates.go
 export interface TemplateDAUsResponse {
@@ -697,6 +696,12 @@ export interface TraceConfig {
   readonly capture_logs: DeploymentConfigField<boolean>
 }
 
+// From codersdk/templates.go
+export interface TransitionStats {
+  readonly P50?: number
+  readonly P95?: number
+}
+
 // From codersdk/templates.go
 export interface UpdateActiveTemplateVersion {
   readonly id: string

@@ -42,7 +42,7 @@ export const TemplateStats: FC<TemplateStatsProps> = ({
       />
       <StatsItem
         label={Language.buildTimeLabel}
-        value={formatTemplateBuildTime(template.build_time_stats.start_ms)}
+        value={formatTemplateBuildTime(template.build_time_stats.start.P50)}
       />
       <StatsItem
         label={Language.activeVersionLabel}

@@ -19,7 +19,7 @@ import { WorkspaceStats } from "../WorkspaceStats/WorkspaceStats"
 import { AlertBanner } from "../AlertBanner/AlertBanner"
 import { useTranslation } from "react-i18next"
 import {
-  EstimateTransitionTime,
+  ActiveTransition,
   WorkspaceBuildProgress,
 } from "components/WorkspaceBuildProgress/WorkspaceBuildProgress"
 import { AgentRow } from "components/Resources/AgentRow"
@@ -115,13 +115,9 @@ export const Workspace: FC<React.PropsWithChildren<WorkspaceProps>> = ({
     />
   )
 
-  let buildTimeEstimate: number | undefined = undefined
-  let isTransitioning: boolean | undefined = undefined
+  let transitionStats: TypesGen.TransitionStats | undefined = undefined
   if (template !== undefined) {
-    ;[buildTimeEstimate, isTransitioning] = EstimateTransitionTime(
-      template,
-      workspace,
-    )
+    transitionStats = ActiveTransition(template, workspace)
   }
 
   return (
@@ -195,10 +191,10 @@ export const Workspace: FC<React.PropsWithChildren<WorkspaceProps>> = ({
           handleUpdate={handleUpdate}
         />
 
-        {isTransitioning !== undefined && isTransitioning && (
+        {transitionStats !== undefined && (
           <WorkspaceBuildProgress
             workspace={workspace}
-            buildEstimate={buildTimeEstimate}
+            transitionStats={transitionStats}
           />
         )}
 

@@ -21,7 +21,10 @@ const Template: Story<WorkspaceBuildProgressProps> = (args) => (
 
 export const Starting = Template.bind({})
 Starting.args = {
-  buildEstimate: 10000,
+  transitionStats: {
+    P50: 10000,
+    P95: 10010,
+  },
   workspace: {
     ...MockStartingWorkspace,
     latest_build: {
@@ -39,11 +42,17 @@ Starting.args = {
 export const StartingUnknown = Template.bind({})
 StartingUnknown.args = {
   ...Starting.args,
-  buildEstimate: undefined,
+  transitionStats: undefined,
 }
 
 export const StartingPassedEstimate = Template.bind({})
 StartingPassedEstimate.args = {
   ...Starting.args,
-  buildEstimate: 1000,
+  transitionStats: { P50: 1000, P95: 1000 },
+}
+
+export const StartingHighVariaton = Template.bind({})
+StartingHighVariaton.args = {
+  ...Starting.args,
+  transitionStats: { P50: 10000, P95: 20000 },
 }