Base upload fix

coder · BrunoQuaresma · Feb 16, 2023 · Feb 15, 2023 · Feb 15, 2023 · Feb 15, 2023
commit 9a7c981ab36663fb9bab251fdf72145dd8b8045a
diff --git a/site/js-untar.d.ts b/site/js-untar.d.ts
@@ -1,14 +1,19 @@
 declare module "js-untar" {
-  interface File {
+  export interface File {
     name: string
+    mode: string
     blob: Blob
+    gid: number
+    uid: number
+    mtime: number
+    type: "0" | "1" | "2" | "3" | "4" | "5" //https://en.wikipedia.org/wiki/Tar_(computing) on Type flag field
   }
 
   const Untar: (buffer: ArrayBuffer) => {
     then: (
-      resolve?: () => Promise<void>,
+      resolve?: (files: File[]) => void,
       reject?: () => Promise<void>,
-      progress: (file: File) => Promise<void>,
+      progress?: (file: File) => Promise<void>,
     ) => Promise<void>
   }
 

diff --git a/site/package.json b/site/package.json
@@ -39,6 +39,7 @@
     "@tanstack/react-query": "4.22.4",
     "@testing-library/react-hooks": "8.0.1",
     "@types/color-convert": "2.0.0",
+    "@types/file-saver": "^2.0.5",
     "@types/react-color": "3.0.6",
     "@vitejs/plugin-react": "2.1.0",
     "@xstate/inspect": "0.6.5",
@@ -54,6 +55,7 @@
     "dayjs": "1.11.4",
     "emoji-mart": "5.4.0",
     "eventsourcemock": "2.0.0",
+    "file-saver": "^2.0.5",
     "formik": "2.2.9",
     "front-matter": "4.0.2",
     "history": "5.3.0",

diff --git a/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/TemplateVersionEditorPage.tsx b/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/TemplateVersionEditorPage.tsx
@@ -16,14 +16,21 @@ type Params = {
 export const TemplateVersionEditorPage: FC = () => {
   const { version: versionName, template: templateName } = useParams() as Params
   const orgId = useOrganizationId()
-  const { isSuccess, data } = useTemplateVersionData(
-    orgId,
-    templateName,
-    versionName,
-  )
   const [editorState, sendEvent] = useMachine(templateVersionEditorMachine, {
     context: { orgId },
   })
+  const { isSuccess, data } = useTemplateVersionData(
+    {
+      orgId,
+      templateName,
+      versionName,
+    },
+    {
+      onSuccess(data) {
+        sendEvent({ type: "INITIALIZE", untarFiles: data.untarFiles })
+      },
+    },
+  )
 
   return (
     <>
@@ -34,7 +41,7 @@ export const TemplateVersionEditorPage: FC = () => {
       {isSuccess && (
         <TemplateVersionEditor
           template={data.template}
-          templateVersion={editorState.context.version || data.currentVersion}
+          templateVersion={editorState.context.version || data.version}
           defaultFileTree={data.fileTree}
           onPreview={(fileTree) => {
             sendEvent({

diff --git a/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/data.ts b/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/data.ts
@@ -1,32 +1,49 @@
-import { useQuery } from "@tanstack/react-query"
-import { getTemplateByName, getTemplateVersionByName } from "api/api"
+import { useQuery, UseQueryOptions } from "@tanstack/react-query"
+import { getFile, getTemplateByName, getTemplateVersionByName } from "api/api"
 import { createTemplateVersionFileTree } from "util/templateVersion"
+import untar, { File as UntarFile } from "js-untar"
 
 const getTemplateVersionData = async (
   orgId: string,
   templateName: string,
   versionName: string,
 ) => {
-  const [template, currentVersion] = await Promise.all([
+  const [template, version] = await Promise.all([
     getTemplateByName(orgId, templateName),
     getTemplateVersionByName(orgId, templateName, versionName),
   ])
-  const fileTree = await createTemplateVersionFileTree(currentVersion)
+  const tarFile = await getFile(version.job.file_id)
+  let untarFiles: UntarFile[] = []
+  await untar(tarFile).then((files) => {
+    untarFiles = files
+  })
+  const fileTree = await createTemplateVersionFileTree(untarFiles)
 
   return {
     template,
-    currentVersion,
+    version,
     fileTree,
+    untarFiles,
   }
 }
 
+type GetTemplateVersionResponse = Awaited<
+  ReturnType<typeof getTemplateVersionData>
+>
+
+type UseTemplateVersionDataParams = {
+  orgId: string
+  templateName: string
+  versionName: string
+}
+
 export const useTemplateVersionData = (
-  orgId: string,
-  templateName: string,
-  versionName: string,
+  { templateName, versionName, orgId }: UseTemplateVersionDataParams,
+  options?: UseQueryOptions<GetTemplateVersionResponse>,
 ) => {
   return useQuery({
     queryKey: ["templateVersion", templateName, versionName],
     queryFn: () => getTemplateVersionData(orgId, templateName, versionName),
+    ...options,
   })
 }
diff --git a/site/src/util/templateVersion.ts b/site/src/util/templateVersion.ts
@@ -1,6 +1,6 @@
 import * as API from "api/api"
 import { TemplateVersion } from "api/typesGenerated"
-import untar from "js-untar"
+import untar, { File as UntarFile } from "js-untar"
 import { FileTree, setFile } from "./filetree"
 
 /**
@@ -41,21 +41,22 @@ export const getTemplateVersionFiles = async (
 
 const allowedExtensions = ["tf", "md", "Dockerfile"]
 
+export const isAllowedFile = (name: string) => {
+  return allowedExtensions.some((ext) => name.endsWith(ext))
+}
+
 export const createTemplateVersionFileTree = async (
-  version: TemplateVersion,
+  untarFiles: UntarFile[],
 ): Promise<FileTree> => {
   let fileTree: FileTree = {}
-  const tarFile = await API.getFile(version.job.file_id)
   const blobs: Record<string, Blob> = {}
 
-  await untar(tarFile).then(undefined, undefined, async (file) => {
-    if (allowedExtensions.some((ext) => file.name.endsWith(ext))) {
-      blobs[file.name] = file.blob
+  for (const untarFile of untarFiles) {
+    if (isAllowedFile(untarFile.name)) {
+      blobs[untarFile.name] = untarFile.blob
     }
-  })
+  }
 
-  // We don't want to get the blob text during untar to not block the main thread.
-  // Also, by doing it here, we can make all the loading in parallel.
   await Promise.all(
     Object.entries(blobs).map(async ([fullPath, blob]) => {
       const content = await blob.text()

diff --git a/site/src/xServices/templateVersionEditor/templateVersionEditorXService.ts b/site/src/xServices/templateVersionEditor/templateVersionEditorXService.ts
@@ -8,7 +8,10 @@ import {
 import { assign, createMachine } from "xstate"
 import * as API from "api/api"
 import Tar from "tar-js"
+import { File as UntarFile } from "js-untar"
 import { FileTree, traverse } from "util/filetree"
+import { isAllowedFile } from "util/templateVersion"
+import { saveAs } from "file-saver"
 
 export interface CreateVersionData {
   file: File
@@ -22,6 +25,7 @@ export interface TemplateVersionEditorMachineContext {
   version?: TemplateVersion
   resources?: WorkspaceResource[]
   buildLogs?: ProvisionerJobLog[]
+  untarFiles?: UntarFile[]
 }
 
 export const templateVersionEditorMachine = createMachine(
@@ -31,6 +35,7 @@ export const templateVersionEditorMachine = createMachine(
     schema: {
       context: {} as TemplateVersionEditorMachineContext,
       events: {} as
+        | { type: "INITIALIZE"; untarFiles: UntarFile[] }
         | {
             type: "CREATE_VERSION"
             fileTree: FileTree
@@ -61,8 +66,16 @@ export const templateVersionEditorMachine = createMachine(
       },
     },
     tsTypes: {} as import("./templateVersionEditorXService.typegen").Typegen0,
-    initial: "idle",
+    initial: "initializing",
     states: {
+      initializing: {
+        on: {
+          INITIALIZE: {
+            actions: ["assignUntarFiles"],
+            target: "idle",
+          },
+        },
+      },
       idle: {
         on: {
           CREATE_VERSION: {
@@ -201,20 +214,42 @@ export const templateVersionEditorMachine = createMachine(
           }
         },
       }),
+      assignUntarFiles: assign({
+        untarFiles: (_, { untarFiles }) => untarFiles,
+      }),
     },
     services: {
-      uploadTar: (ctx) => {
-        if (!ctx.fileTree) {
-          throw new Error("files must be set")
+      uploadTar: async ({ fileTree, untarFiles }) => {
+        if (!fileTree) {
+          throw new Error("file tree must to be set")
+        }
+        if (!untarFiles) {
+          throw new Error("untar files must to be set")
         }
         const tar = new Tar()
-        let out: Uint8Array = new Uint8Array()
-        traverse(ctx.fileTree, (content, _filename, fullPath) => {
+        // Add previous non editable files
+        for (const untarFile of untarFiles) {
+          if (!isAllowedFile(untarFile.name) && untarFile.type === "0") {
+            // TODO: Maybe there is a way to use blob directly to append in tar
+            // or a way where converting to text is necessary. I tried to find
+            // it but I didn't have success tho.
+            const content = await untarFile.blob.text()
+            tar.append(untarFile.name, content, {
+              mode: Number(untarFile.mode),
+              gid: untarFile.gid,
+              uid: untarFile.uid,
+              mtime: untarFile.mtime,
+            })
+          }
+        }
+        // Add the editable files
+        traverse(fileTree, (content, _filename, fullPath) => {
           if (typeof content === "string") {
-            out = tar.append(fullPath, content)
+            tar.append(fullPath, content)
           }
         })
-        return API.uploadTemplateFile(new File([out], "template.tar"))
+        saveAs(new Blob([tar.out]), "template.tar")
+        return API.uploadTemplateFile(new File([tar.out], "template.tar"))
       },
       createBuild: (ctx) => {
         if (!ctx.uploadResponse) {

diff --git a/site/yarn.lock b/site/yarn.lock
@@ -3144,6 +3144,11 @@
   resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.51.tgz#cfd70924a25a3fd32b218e5e420e6897e1ac4f40"
   integrity sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ==
 
+"@types/file-saver@^2.0.5":
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/@types/file-saver/-/file-saver-2.0.5.tgz#9ee342a5d1314bb0928375424a2f162f97c310c7"
+  integrity sha512-zv9kNf3keYegP5oThGLaPk8E081DFDuwfqjtiTzm6PoxChdJ1raSuADf2YGCVIyrSynLrgc8JWv296s7Q7pQSQ==
+
 "@types/glob@*":
   version "8.0.0"
   resolved "https://registry.yarnpkg.com/@types/glob/-/glob-8.0.0.tgz#321607e9cbaec54f687a0792b2d1d370739455d2"
@@ -7278,6 +7283,11 @@ file-loader@^6.2.0:
     loader-utils "^2.0.0"
     schema-utils "^3.0.0"
 
+file-saver@^2.0.5:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/file-saver/-/file-saver-2.0.5.tgz#d61cfe2ce059f414d899e9dd6d4107ee25670c38"
+  integrity sha512-P9bmyZ3h/PRG+Nzga+rbdI4OEpNDzAVyy74uVO9ATgzLK6VtAsYybF/+TOCvrc0MO793d6+42lLyZTw7/ArVzA==
+
 file-system-cache@^1.0.5:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/file-system-cache/-/file-system-cache-1.1.0.tgz#984de17b976b75a77a27e08d6828137c1aa80fa1"