reshuffling

coder · bcpeinhardt · Jul 31, 2025 · Jul 15, 2025 · Jul 15, 2025 · Jul 15, 2025
commit 8115e4aa31d75dd65a578717ec93b1a05ca326d7
diff --git a/coderd/database/migrations/000353_add_provisioner_logs_overflowed.up.sql b/coderd/database/migrations/000353_add_provisioner_logs_overflowed.up.sql
@@ -3,4 +3,4 @@
   ALTER TABLE provisioner_jobs ADD COLUMN logs_overflowed boolean NOT NULL DEFAULT false;
 
   COMMENT ON COLUMN provisioner_jobs.logs_length IS 'Total length of provisioner logs';
-  COMMENT ON COLUMN provisioner_jobs.logs_overflowed IS 'Whether the provisioner logs overflowed in length';
+  COMMENT ON COLUMN provisioner_jobs.logs_overflowed IS 'Whether the provisioner logs overflowed in length';
diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go
@@ -907,43 +907,65 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest)
 		insertParams := database.InsertProvisionerJobLogsParams{
 			JobID: parsedID,
 		}
+
+		newLogSize := 0
+		overflowedErrorMsg := "Provisioner logs exceeded the max size of 1MB. Will not continue to write provisioner logs for workspace build."
+		lenErrMsg := len(overflowedErrorMsg)
+
+		var (
+			createdAt time.Time
+			level     database.LogLevel
+			stage     string
+			source    database.LogSource
+			output    string
+		)
+
 		for _, log := range request.Logs {
-			logLevel, err := convertLogLevel(log.Level)
+			// Build our log params
+			level, err = convertLogLevel(log.Level)
 			if err != nil {
 				return nil, xerrors.Errorf("convert log level: %w", err)
 			}
-			logSource, err := convertLogSource(log.Source)
+			source, err = convertLogSource(log.Source)
 			if err != nil {
 				return nil, xerrors.Errorf("convert log source: %w", err)
 			}
-			insertParams.CreatedAt = append(insertParams.CreatedAt, time.UnixMilli(log.CreatedAt))
-			insertParams.Level = append(insertParams.Level, logLevel)
-			insertParams.Stage = append(insertParams.Stage, log.Stage)
-			insertParams.Source = append(insertParams.Source, logSource)
-			insertParams.Output = append(insertParams.Output, log.Output)
+			createdAt = time.UnixMilli(log.CreatedAt)
+			stage = log.Stage
+			output = log.Output
+
+			// Check if we would overflow the job logs (not leaving enough room for the error message)
+			willOverflow := int64(job.LogsLength)+int64(newLogSize)+int64(lenErrMsg)+int64(len(output)) > 1048576
+			if willOverflow {
+				s.Logger.Debug(ctx, "provisioner job logs overflowed 1MB size limit in database.", slog.F("job_id", parsedID))
+				err = s.Database.UpdateProvisionerJobLogsOverflowed(ctx, database.UpdateProvisionerJobLogsOverflowedParams{
+					ID:             parsedID,
+					LogsOverflowed: true,
+				})
+				if err != nil {
+					s.Logger.Error(ctx, "failed to set logs overflowed flag", slog.F("job_id", parsedID), slog.Error(err))
+				}
+
+				level = database.LogLevelWarn
+				output = overflowedErrorMsg
+			}
+
+			newLogSize += len(output)
+
+			insertParams.CreatedAt = append(insertParams.CreatedAt, createdAt)
+			insertParams.Level = append(insertParams.Level, level)
+			insertParams.Stage = append(insertParams.Stage, stage)
+			insertParams.Source = append(insertParams.Source, source)
+			insertParams.Output = append(insertParams.Output, output)
 			s.Logger.Debug(ctx, "job log",
 				slog.F("job_id", parsedID),
-				slog.F("stage", log.Stage),
-				slog.F("output", log.Output))
-		}
+				slog.F("stage", stage),
+				slog.F("output", output))
 
-		newLogSize := 0
-		for _, log := range request.Logs {
-			newLogSize += len(log.Output)
-		}
-
-		willOverflow := int64(job.LogsLength)+int64(newLogSize) > 1048576
-		if willOverflow {
-			err = s.Database.UpdateProvisionerJobLogsOverflowed(ctx, database.UpdateProvisionerJobLogsOverflowedParams{
-				ID:             parsedID,
-				LogsOverflowed: true,
-			})
-			if err != nil {
-				s.Logger.Error(ctx, "failed to set logs overflowed flag", slog.F("job_id", parsedID), slog.Error(err))
+			// Don't write any more logs because there's no room.
+			if willOverflow {
+				break
 			}
-			return &proto.UpdateJobResponse{
-				Canceled: job.CanceledAt.Valid,
-			}, nil
 		}
 
 		err = s.Database.UpdateProvisionerJobLogsLength(ctx, database.UpdateProvisionerJobLogsLengthParams{

@@ -64,7 +64,8 @@ export const WorkspaceBuildLogs: FC<WorkspaceBuildLogsProps> = ({
 				created_at: new Date().toISOString(),
 				log_level: "error",
 				log_source: "provisioner",
-				output: "Provisioner logs exceeded the max size of 1MB!",
+				output:
+					"Provisioner logs exceeded the max size of 1MB. Will not continue to write provisioner logs for workspace build.",
 				stage: "overflow",
 			});
 		}

@@ -222,8 +222,8 @@ export const WorkspaceBuildPageView: FC<WorkspaceBuildPageViewProps> = ({
 								borderBottom: `1px solid ${theme.palette.divider}`,
 							}}
 						>
-							Provisioner logs exceeded the maximum size of 1MB and were
-							truncated.
+							Provisioner logs exceeded the max size of 1MB. Will not continue
+							to write provisioner logs for workspace build.
 						</Alert>
 					)}