Fix crash when out of memory while opening a terminal window: Handle out-of-memory more gracefully.

npt-1707 · npt-1707 · commit d301c25ed035 · 2025-04-29T15:06:02.000+08:00
diff --git a/liteidex/src/3rdparty/libvterm/src/screen.c b/liteidex/src/3rdparty/libvterm/src/screen.c
@@ -95,8 +95,7 @@ static ScreenCell *realloc_buffer(VTermScreen *screen, ScreenCell *buffer, int n
     }
   }
 
-  if(buffer)
-    vterm_allocator_free(screen->vt, buffer);
+  vterm_allocator_free(screen->vt, buffer);
 
   return new_buffer;
 }
@@ -520,8 +519,7 @@ static int resize(int new_rows, int new_cols, VTermPos *delta, void *user)
   screen->rows = new_rows;
   screen->cols = new_cols;
 
-  if(screen->sb_buffer)
-    vterm_allocator_free(screen->vt, screen->sb_buffer);
+  vterm_allocator_free(screen->vt, screen->sb_buffer);
 
   screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * new_cols);
 
@@ -625,10 +623,13 @@ static VTermStateCallbacks state_cbs = {
 static VTermScreen *screen_new(VTerm *vt)
 {
   VTermState *state = vterm_obtain_state(vt);
-  if(!state)
+  if (state == NULL)
     return NULL;
 
   VTermScreen *screen = vterm_allocator_malloc(vt, sizeof(VTermScreen));
+  if (screen == NULL)
+    return NULL;
+
   int rows, cols;
 
   vterm_get_size(vt, &rows, &cols);
@@ -652,6 +653,12 @@ static VTermScreen *screen_new(VTerm *vt)
 
   screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * cols);
 
+  if (screen->buffer == NULL || screen->sb_buffer == NULL)
+  {
+    vterm_screen_free(screen);
+    return NULL;
+  }
+
   vterm_state_set_callbacks(screen->state, &state_cbs, screen);
 
   return screen;
@@ -660,11 +667,8 @@ static VTermScreen *screen_new(VTerm *vt)
 INTERNAL void vterm_screen_free(VTermScreen *screen)
 {
   vterm_allocator_free(screen->vt, screen->buffers[0]);
-  if(screen->buffers[1])
-    vterm_allocator_free(screen->vt, screen->buffers[1]);
-
+  vterm_allocator_free(screen->vt, screen->buffers[1]);
   vterm_allocator_free(screen->vt, screen->sb_buffer);
-
   vterm_allocator_free(screen->vt, screen);
 }
 
diff --git a/liteidex/src/3rdparty/libvterm/src/state.c b/liteidex/src/3rdparty/libvterm/src/state.c
@@ -52,6 +52,8 @@ static VTermState *vterm_state_new(VTerm *vt)
 {
   VTermState *state = vterm_allocator_malloc(vt, sizeof(VTermState));
 
+  if (state == NULL)
+    return NULL;
   state->vt = vt;
 
   state->rows = vt->rows;
@@ -1736,6 +1738,8 @@ VTermState *vterm_obtain_state(VTerm *vt)
     return vt->state;
 
   VTermState *state = vterm_state_new(vt);
+  if (state == NULL)
+    return NULL;
   vt->state = state;
 
   state->combine_chars_size = 16;
diff --git a/liteidex/src/3rdparty/libvterm/src/vterm.c b/liteidex/src/3rdparty/libvterm/src/vterm.c
@@ -37,6 +37,8 @@ VTerm *vterm_new_with_allocator(int rows, int cols, VTermAllocatorFunctions *fun
   /* Need to bootstrap using the allocator function directly */
   VTerm *vt = (*funcs->malloc)(sizeof(VTerm), allocdata);
 
+  if (vt == NULL)
+    return NULL;
   vt->allocator = funcs;
   vt->allocdata = allocdata;
 
@@ -51,13 +53,24 @@ VTerm *vterm_new_with_allocator(int rows, int cols, VTermAllocatorFunctions *fun
   vt->parser.strbuffer_len = 64;
   vt->parser.strbuffer_cur = 0;
   vt->parser.strbuffer = vterm_allocator_malloc(vt, vt->parser.strbuffer_len);
+  if (vt->parser.strbuffer == NULL)
+  {
+    vterm_allocator_free(vt, vt);
+    return NULL;
+  }
 
   vt->outfunc = NULL;
   vt->outdata = NULL;
 
   vt->outbuffer_len = 64;
   vt->outbuffer_cur = 0;
   vt->outbuffer = vterm_allocator_malloc(vt, vt->outbuffer_len);
+  if (vt->outbuffer == NULL)
+  {
+    vterm_allocator_free(vt, vt->parser.strbuffer);
+    vterm_allocator_free(vt, vt);
+    return NULL;
+  }
 
   vt->tmpbuffer_len = 64;
   vt->tmpbuffer = vterm_allocator_malloc(vt, vt->tmpbuffer_len);
@@ -87,7 +100,8 @@ INTERNAL void *vterm_allocator_malloc(VTerm *vt, size_t size)
 
 INTERNAL void vterm_allocator_free(VTerm *vt, void *ptr)
 {
-  (*vt->allocator->free)(ptr, vt->allocdata);
+  if (ptr)
+    (*vt->allocator->free)(ptr, vt->allocdata);
 }
 
 void vterm_get_size(const VTerm *vt, int *rowsp, int *colsp)
