Improve bulk log performance

coder · kylecarbs · Mar 23, 2023 · Feb 23, 2023 · Feb 23, 2023 · Feb 23, 2023
commit adb06eafffb94f65f70d263ae673d10e9a614b96
diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go
@@ -274,7 +274,6 @@ func (api *API) patchWorkspaceAgentStartupLogs(rw http.ResponseWriter, r *http.R
 		CreatedAfter: lowestID - 1,
 	})
 	if err != nil {
-
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
 			Message: "Failed to marshal startup logs notify message",
 			Detail:  err.Error(),
@@ -374,32 +373,28 @@ func (api *API) workspaceAgentStartupLogs(rw http.ResponseWriter, r *http.Reques
 	ctx, wsNetConn := websocketNetConn(ctx, conn, websocket.MessageText)
 	defer wsNetConn.Close() // Also closes conn.
 
-	logIdsDone := make(map[int64]bool)
-
 	// The Go stdlib JSON encoder appends a newline character after message write.
 	encoder := json.NewEncoder(wsNetConn)
-	for _, provisionerJobLog := range logs {
-		logIdsDone[provisionerJobLog.ID] = true
-		err = encoder.Encode(convertWorkspaceAgentStartupLog(provisionerJobLog))
-		if err != nil {
-			return
-		}
+	err = encoder.Encode(convertWorkspaceAgentStartupLogs(logs))
+	if err != nil {
+		return
 	}
 	if agent.LifecycleState == database.WorkspaceAgentLifecycleStateReady {
 		// The startup script has finished running, so we can close the connection.
 		return
 	}
 
 	var (
-		bufferedLogs  = make(chan *database.WorkspaceAgentStartupLog, 128)
+		// It's not impossible that
+		bufferedLogs  = make(chan []database.WorkspaceAgentStartupLog, 128)
 		endOfLogs     atomic.Bool
 		lastSentLogID atomic.Int64
 	)
 
-	sendLog := func(log *database.WorkspaceAgentStartupLog) {
+	sendLogs := func(logs []database.WorkspaceAgentStartupLog) {
 		select {
-		case bufferedLogs <- log:
-			lastSentLogID.Store(log.ID)
+		case bufferedLogs <- logs:
+			lastSentLogID.Store(logs[len(logs)-1].ID)
 		default:
 			logger.Warn(ctx, "workspace agent startup log overflowing channel")
 		}
@@ -427,13 +422,7 @@ func (api *API) workspaceAgentStartupLogs(rw http.ResponseWriter, r *http.Reques
 					logger.Warn(ctx, "failed to get workspace agent startup logs after", slog.Error(err))
 					return
 				}
-				for _, log := range logs {
-					if endOfLogs.Load() {
-						return
-					}
-					log := log
-					sendLog(&log)
-				}
+				sendLogs(logs)
 			}
 
 			if jlMsg.EndOfLogs {
@@ -446,10 +435,7 @@ func (api *API) workspaceAgentStartupLogs(rw http.ResponseWriter, r *http.Reques
 					logger.Warn(ctx, "get workspace agent startup logs after", slog.Error(err))
 					return
 				}
-				for _, log := range logs {
-					log := log
-					sendLog(&log)
-				}
+				sendLogs(logs)
 				bufferedLogs <- nil
 			}
 		},
@@ -468,19 +454,15 @@ func (api *API) workspaceAgentStartupLogs(rw http.ResponseWriter, r *http.Reques
 		case <-ctx.Done():
 			logger.Debug(context.Background(), "job logs context canceled")
 			return
-		case log, ok := <-bufferedLogs:
+		case logs, ok := <-bufferedLogs:
 			// A nil log is sent when complete!
-			if !ok || log == nil {
+			if !ok || logs == nil {
 				logger.Debug(context.Background(), "reached the end of published logs")
 				return
 			}
-			if logIdsDone[log.ID] {
-				logger.Debug(ctx, "subscribe duplicated log")
-			} else {
-				err = encoder.Encode(convertWorkspaceAgentStartupLog(*log))
-				if err != nil {
-					return
-				}
+			err = encoder.Encode(convertWorkspaceAgentStartupLogs(logs))
+			if err != nil {
+				return
 			}
 		}
 	}

diff --git a/site/src/components/Resources/AgentRow.tsx b/site/src/components/Resources/AgentRow.tsx
@@ -17,6 +17,8 @@ import { VSCodeDesktopButton } from "components/VSCodeDesktopButton/VSCodeDeskto
 import { useMachine } from "@xstate/react"
 import { workspaceAgentLogsMachine } from "xServices/workspaceAgentLogs/workspaceAgentLogsXService"
 import { Line, Logs } from "components/Logs/Logs"
+import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"
+import { darcula } from "react-syntax-highlighter/dist/cjs/styles/prism"
 
 export interface AgentRowProps {
   agent: WorkspaceAgent
@@ -45,6 +47,8 @@ export const AgentRow: FC<AgentRowProps> = ({
     context: { agentID: agent.id },
   })
 
+  console.log("drac", darcula)
+
   return (
     <Stack direction="column" key={agent.id} spacing={0}>
       <Stack
@@ -149,8 +153,18 @@ export const AgentRow: FC<AgentRowProps> = ({
       </Stack>
 
       <div>
+        Startup Script
+        <SyntaxHighlighter
+          style={darcula}
+          language="bash"
+          useInlineStyles={false}
+          codeTagProps={{ style: {} }}
+        >
+          {String(agent.startup_script)}
+        </SyntaxHighlighter>
         {logsMachine.context.startupLogs && (
           <Logs
+            className={styles.agentStartupLogs}
             lines={logsMachine.context.startupLogs.map(
               (log): Line => ({
                 level: "info",
@@ -195,4 +209,10 @@ const useStyles = makeStyles((theme) => ({
     color: theme.palette.text.secondary,
     marginTop: theme.spacing(0.5),
   },
+
+  agentStartupLogs: {
+    maxHeight: 200,
+    display: "flex",
+    flexDirection: "column-reverse",
+  },
 }))
diff --git a/site/src/xServices/workspaceAgentLogs/workspaceAgentLogsXService.ts b/site/src/xServices/workspaceAgentLogs/workspaceAgentLogsXService.ts
@@ -8,8 +8,8 @@ export const workspaceAgentLogsMachine = createMachine(
     id: "workspaceAgentLogsMachine",
     schema: {
       events: {} as {
-        type: "ADD_STARTUP_LOG"
-        log: TypesGen.WorkspaceAgentStartupLog
+        type: "ADD_STARTUP_LOGS"
+        logs: TypesGen.WorkspaceAgentStartupLog[]
       },
       context: {} as {
         agentID: string
@@ -45,8 +45,8 @@ export const workspaceAgentLogsMachine = createMachine(
       },
     },
     on: {
-      ADD_STARTUP_LOG: {
-        actions: "addStartupLog",
+      ADD_STARTUP_LOGS: {
+        actions: "addStartupLogs",
       },
     },
   },
@@ -65,7 +65,7 @@ export const workspaceAgentLogsMachine = createMachine(
           )
           socket.binaryType = "blob"
           socket.addEventListener("message", (event) => {
-            callback({ type: "ADD_STARTUP_LOG", log: JSON.parse(event.data) })
+            callback({ type: "ADD_STARTUP_LOGS", logs: JSON.parse(event.data) })
           })
           socket.addEventListener("error", () => {
             reject(new Error("socket errored"))
@@ -80,10 +80,10 @@ export const workspaceAgentLogsMachine = createMachine(
       assignStartupLogs: assign({
         startupLogs: (_, { data }) => data,
       }),
-      addStartupLog: assign({
+      addStartupLogs: assign({
         startupLogs: (context, event) => {
           const previousLogs = context.startupLogs ?? []
-          return [...previousLogs, event.log]
+          return [...previousLogs, ...event.logs]
         },
       }),
     },