validate uncompressed dump size on both client and server

jgoux · jgoux · commit 5b03c29a1c27 · 2024-08-19T09:30:34.000+02:00
diff --git a/apps/proxy/package.json b/apps/proxy/package.json
@@ -10,7 +10,7 @@
     "type-check": "tsc --noEmit"
   },
   "dependencies": {
-    "@electric-sql/pglite": "0.2.0",
+    "@electric-sql/pglite": "0.2.2",
     "@supabase/supabase-js": "^2.45.1",
     "find-up": "^7.0.0",
     "pg-gateway": "0.3.0-alpha.6",
diff --git a/apps/web/app/api/databases/[id]/upload/route.ts b/apps/web/app/api/databases/[id]/upload/route.ts
@@ -6,6 +6,7 @@ import { Readable } from 'stream'
 import { createClient } from '~/utils/supabase/server'
 import { createScramSha256Data } from 'pg-gateway'
 import { generateDatabasePassword } from '~/utils/generate-database-password'
+import { getUncompressedSizeInMB } from '~/utils/get-uncompressed-size-in-mb'
 
 const wildcardDomain = process.env.NEXT_PUBLIC_WILDCARD_DOMAIN ?? 'db.example.com'
 const s3Client = new S3Client({ forcePathStyle: true })
@@ -62,8 +63,7 @@ export async function POST(
     )
   }
 
-  const dumpSizeInMB = dump.size / (1024 * 1024)
-  if (dumpSizeInMB > 100) {
+  if ((await getUncompressedSizeInMB(dump)) > 100) {
     return NextResponse.json(
       {
         success: false,
diff --git a/apps/web/data/deployed-databases/deployed-database-create-mutation.ts b/apps/web/data/deployed-databases/deployed-database-create-mutation.ts
@@ -2,6 +2,7 @@ import { useMutation, UseMutationOptions, useQueryClient } from '@tanstack/react
 import { getDeployedDatabasesQueryKey } from './deployed-databases-query'
 import { useApp } from '~/components/app-provider'
 import type { DatabaseUploadResponse } from '~/app/api/databases/[id]/upload/route'
+import { getUncompressedSizeInMB } from '~/utils/get-uncompressed-size-in-mb'
 
 export type DeployedDatabaseCreateVariables = {
   databaseId: string
@@ -33,14 +34,12 @@ export const useDeployedDatabaseCreateMutation = ({
 
       const db = await dbManager.getDbInstance(variables.databaseId)
 
-      const dump = await db.dumpDataDir()
-      const dumpSizeInMB = dump.size / (1024 * 1024)
-      if (dumpSizeInMB > 100) {
+      const dump = await db.dumpDataDir('gzip')
+      if ((await getUncompressedSizeInMB(dump)) > 100) {
         throw new Error("You can't deploy a database that is bigger than 100MB")
       }
-
       const formData = new FormData()
-      formData.append('dump', dump, 'dump.tar')
+      formData.append('dump', dump, 'dump.tar.gz')
       formData.append('name', variables.name ?? 'My database')
       formData.append('created_at', variables.createdAt.toISOString())
 
@@ -66,3 +65,29 @@ export const useDeployedDatabaseCreateMutation = ({
     ...options,
   })
 }
+
+export async function zip(file: Blob | File): Promise<Uint8Array> {
+  const cs = new CompressionStream('gzip')
+  const writer = cs.writable.getWriter()
+  const reader = cs.readable.getReader()
+
+  writer.write(file)
+  writer.close()
+
+  const chunks: Uint8Array[] = []
+
+  while (true) {
+    const { value, done } = await reader.read()
+    if (done) break
+    if (value) chunks.push(value)
+  }
+
+  const compressed = new Uint8Array(chunks.reduce((acc, chunk) => acc + chunk.length, 0))
+  let offset = 0
+  chunks.forEach((chunk) => {
+    compressed.set(chunk, offset)
+    offset += chunk.length
+  })
+
+  return compressed
+}
diff --git a/apps/web/package.json b/apps/web/package.json
@@ -14,7 +14,7 @@
     "@aws-sdk/client-s3": "^3.620.1",
     "@aws-sdk/lib-storage": "^3.620.1",
     "@dagrejs/dagre": "^1.1.2",
-    "@electric-sql/pglite": "0.2.0",
+    "@electric-sql/pglite": "0.2.2",
     "@gregnr/postgres-meta": "^0.82.0-dev.2",
     "@monaco-editor/react": "^4.6.0",
     "@postgres-new/supabase": "*",
diff --git a/apps/web/utils/get-uncompressed-size-in-mb.ts b/apps/web/utils/get-uncompressed-size-in-mb.ts
@@ -0,0 +1,9 @@
+/**
+ * Get the uncompressed size of a gzipped file in MB
+ */
+export async function getUncompressedSizeInMB(gzippedData: Blob | File): Promise<number> {
+  // The last 4 bytes of the file contain the uncompressed size
+  const arrayBuffer = await gzippedData.slice(-4).arrayBuffer()
+  const view = new DataView(arrayBuffer)
+  return view.getUint32(0, true) / (1024 * 1024)
+}
diff --git a/package-lock.json b/package-lock.json

Original file line number	Diff line number	Diff line change
`@@ -6,6 +6,7 @@ import { Readable } from 'stream'`
`6`	`6`	`import { createClient } from '~/utils/supabase/server'`
`7`	`7`	`import { createScramSha256Data } from 'pg-gateway'`
`8`	`8`	`import { generateDatabasePassword } from '~/utils/generate-database-password'`
	`9`	`+import { getUncompressedSizeInMB } from '~/utils/get-uncompressed-size-in-mb'`
`9`	`10`
`10`	`11`	`const wildcardDomain = process.env.NEXT_PUBLIC_WILDCARD_DOMAIN ?? 'db.example.com'`
`11`	`12`	`const s3Client = new S3Client({ forcePathStyle: true })`
`@@ -62,8 +63,7 @@ export async function POST(`
`62`	`63`	`)`
`63`	`64`	`}`
`64`	`65`
`65`		`- const dumpSizeInMB = dump.size / (1024 * 1024)`
`66`		`- if (dumpSizeInMB > 100) {`
	`66`	`+ if ((await getUncompressedSizeInMB(dump)) > 100) {`
`67`	`67`	`return NextResponse.json(`
`68`	`68`	`{`
`69`	`69`	`success: false,`