diff --git a/site/jest.setup.ts b/site/jest.setup.ts index f5f19c9f35aaa..d6eb950443481 100644 --- a/site/jest.setup.ts +++ b/site/jest.setup.ts @@ -3,6 +3,14 @@ import { cleanup } from "@testing-library/react" import crypto from "crypto" import { server } from "./src/testHelpers/server" import "jest-location-mock" +import { TextEncoder, TextDecoder } from "util" +import { Blob } from "buffer" + +global.TextEncoder = TextEncoder +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Polyfill for jsdom +global.TextDecoder = TextDecoder as any +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Polyfill for jsdom +global.Blob = Blob as any // Polyfill the getRandomValues that is used on utils/random.ts Object.defineProperty(global.self, "crypto", { diff --git a/site/js-untar.d.ts b/site/js-untar.d.ts index e28cb0a65c058..6309b4cc0974f 100644 --- a/site/js-untar.d.ts +++ b/site/js-untar.d.ts @@ -1,14 +1,21 @@ declare module "js-untar" { - interface File { + export interface File { name: string + mode: string blob: Blob + gid: number + uid: number + mtime: number + gname: string + uname: string + type: "0" | "1" | "2" | "3" | "4" | "5" //https://en.wikipedia.org/wiki/Tar_(computing) on Type flag field } const Untar: (buffer: ArrayBuffer) => { then: ( - resolve?: () => Promise, + resolve?: (files: File[]) => void, reject?: () => Promise, - progress: (file: File) => Promise, + progress?: (file: File) => Promise, ) => Promise } diff --git a/site/package.json b/site/package.json index e63df76dff585..660400ec669c6 100644 --- a/site/package.json +++ b/site/package.json @@ -76,7 +76,6 @@ "remark-gfm": "3.0.1", "rollup-plugin-visualizer": "5.9.0", "sourcemapped-stacktrace": "1.1.11", - "tar-js": "^0.3.0", "ts-prune": "0.10.3", "tzdata": "1.0.30", "ua-parser-js": "1.0.33", diff --git a/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/TemplateVersionEditorPage.tsx b/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/TemplateVersionEditorPage.tsx index 5b45f1767bd65..e6e55568cca83 100644 --- a/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/TemplateVersionEditorPage.tsx +++ b/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/TemplateVersionEditorPage.tsx @@ -16,14 +16,21 @@ type Params = { export const TemplateVersionEditorPage: FC = () => { const { version: versionName, template: templateName } = useParams() as Params const orgId = useOrganizationId() - const { isSuccess, data } = useTemplateVersionData( - orgId, - templateName, - versionName, - ) const [editorState, sendEvent] = useMachine(templateVersionEditorMachine, { context: { orgId }, }) + const { isSuccess, data } = useTemplateVersionData( + { + orgId, + templateName, + versionName, + }, + { + onSuccess(data) { + sendEvent({ type: "INITIALIZE", untarFiles: data.untarFiles }) + }, + }, + ) return ( <> @@ -34,7 +41,7 @@ export const TemplateVersionEditorPage: FC = () => { {isSuccess && ( { sendEvent({ diff --git a/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/data.ts b/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/data.ts index 6270e746d0b2c..f46b523107107 100644 --- a/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/data.ts +++ b/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/data.ts @@ -1,32 +1,49 @@ -import { useQuery } from "@tanstack/react-query" -import { getTemplateByName, getTemplateVersionByName } from "api/api" +import { useQuery, UseQueryOptions } from "@tanstack/react-query" +import { getFile, getTemplateByName, getTemplateVersionByName } from "api/api" import { createTemplateVersionFileTree } from "util/templateVersion" +import untar, { File as UntarFile } from "js-untar" const getTemplateVersionData = async ( orgId: string, templateName: string, versionName: string, ) => { - const [template, currentVersion] = await Promise.all([ + const [template, version] = await Promise.all([ getTemplateByName(orgId, templateName), getTemplateVersionByName(orgId, templateName, versionName), ]) - const fileTree = await createTemplateVersionFileTree(currentVersion) + const tarFile = await getFile(version.job.file_id) + let untarFiles: UntarFile[] = [] + await untar(tarFile).then((files) => { + untarFiles = files + }) + const fileTree = await createTemplateVersionFileTree(untarFiles) return { template, - currentVersion, + version, fileTree, + untarFiles, } } +type GetTemplateVersionResponse = Awaited< + ReturnType +> + +type UseTemplateVersionDataParams = { + orgId: string + templateName: string + versionName: string +} + export const useTemplateVersionData = ( - orgId: string, - templateName: string, - versionName: string, + { templateName, versionName, orgId }: UseTemplateVersionDataParams, + options?: UseQueryOptions, ) => { return useQuery({ queryKey: ["templateVersion", templateName, versionName], queryFn: () => getTemplateVersionData(orgId, templateName, versionName), + ...options, }) } diff --git a/site/src/util/tar.test.ts b/site/src/util/tar.test.ts new file mode 100644 index 0000000000000..d2e81af06f2ce --- /dev/null +++ b/site/src/util/tar.test.ts @@ -0,0 +1,52 @@ +import { TarReader, TarWriter, ITarFileInfo, TarFileType } from "./tar" + +const mtime = 1666666666666 + +test("tar", async () => { + // Write + const writer = new TarWriter() + writer.addFile("a.txt", "hello", { mtime }) + writer.addFile("b.txt", new Blob(["world"]), { mtime }) + writer.addFile("c.txt", "", { mtime }) + writer.addFolder("etc", { mtime }) + writer.addFile("etc/d.txt", "", { mtime }) + const blob = await writer.write() + + // Read + const reader = new TarReader() + const fileInfos = await reader.readFile(blob) + verifyFile(fileInfos[0], reader.getTextFile(fileInfos[0].name) as string, { + name: "a.txt", + content: "hello", + }) + verifyFile(fileInfos[1], reader.getTextFile(fileInfos[1].name) as string, { + name: "b.txt", + content: "world", + }) + verifyFile(fileInfos[2], reader.getTextFile(fileInfos[2].name) as string, { + name: "c.txt", + content: "", + }) + verifyFolder(fileInfos[3], { + name: "etc", + }) + verifyFile(fileInfos[4], reader.getTextFile(fileInfos[4].name) as string, { + name: "etc/d.txt", + content: "", + }) +}) + +function verifyFile( + info: ITarFileInfo, + content: string, + expected: { name: string; content: string }, +) { + expect(info.name).toEqual(expected.name) + expect(info.size).toEqual(expected.content.length) + expect(content).toEqual(expected.content) +} + +function verifyFolder(info: ITarFileInfo, expected: { name: string }) { + expect(info.name).toEqual(expected.name) + expect(info.type).toEqual(TarFileType.Dir) +} diff --git a/site/src/util/tar.ts b/site/src/util/tar.ts new file mode 100644 index 0000000000000..47543d1dc2470 --- /dev/null +++ b/site/src/util/tar.ts @@ -0,0 +1,321 @@ +// Based on https://github.com/gera2ld/tarjs +// and https://github.com/ankitrohatgi/tarballjs/blob/master/tarball.js +export enum TarFileType { + File = "0", + Dir = "5", +} +const encoder = new TextEncoder() +const utf8Encode = (input: string) => encoder.encode(input) +const decoder = new TextDecoder() +const utf8Decode = (input: Uint8Array) => decoder.decode(input) + +export interface ITarFileInfo { + name: string + type: TarFileType + size: number + headerOffset: number +} + +export interface ITarWriteItem { + name: string + type: TarFileType + data: ArrayBuffer | Promise | null + size: number + opts?: Partial +} + +export interface ITarWriteOptions { + uid: number + gid: number + mode: number + mtime: number + user: string + group: string +} + +export class TarReader { + private fileInfo: ITarFileInfo[] = [] + private _buffer: ArrayBuffer | null = null + + constructor() { + this.reset() + } + + get buffer() { + if (!this._buffer) { + throw new Error("Buffer is not set") + } + + return this._buffer + } + + reset() { + this.fileInfo = [] + this._buffer = null + } + + async readFile(file: ArrayBuffer | Uint8Array | Blob) { + this.reset() + this._buffer = await getArrayBuffer(file) + this.readFileInfo() + return this.fileInfo + } + + private readFileInfo() { + this.fileInfo = [] + let offset = 0 + let fileSize = 0 + let fileName = "" + let fileType: TarFileType + while (offset < this.buffer.byteLength - 512) { + fileName = this.readFileName(offset) + if (!fileName) { + break + } + fileType = this.readFileType(offset) + fileSize = this.readFileSize(offset) + + this.fileInfo.push({ + name: fileName, + type: fileType, + size: fileSize, + headerOffset: offset, + }) + + offset += 512 + 512 * Math.floor((fileSize + 511) / 512) + } + } + + private readString(offset: number, maxSize: number) { + let size = 0 + let view = new Uint8Array(this.buffer, offset, maxSize) + while (size < maxSize && view[size]) { + size += 1 + } + view = new Uint8Array(this.buffer, offset, size) + return utf8Decode(view) + } + + private readFileName(offset: number) { + return this.readString(offset, 100) + } + + private readFileType(offset: number) { + const typeView = new Uint8Array(this.buffer, offset + 156, 1) + const typeStr = String.fromCharCode(typeView[0]) + if (typeStr === "0") { + return TarFileType.File + } else if (typeStr === "5") { + return TarFileType.Dir + } else { + throw new Error("No supported file type") + } + } + + private readFileSize(offset: number) { + // offset = 124, length = 12 + const view = new Uint8Array(this.buffer, offset + 124, 12) + const sizeStr = utf8Decode(view) + return parseInt(sizeStr, 8) + } + + private readFileBlob(offset: number, size: number, mimetype: string) { + const view = new Uint8Array(this.buffer, offset, size) + return new Blob([view], { type: mimetype }) + } + + private readTextFile(offset: number, size: number) { + const view = new Uint8Array(this.buffer, offset, size) + return utf8Decode(view) + } + + getTextFile(filename: string) { + const item = this.fileInfo.find((info) => info.name === filename) + if (item) { + return this.readTextFile(item.headerOffset + 512, item.size) + } + } + + getFileBlob(filename: string, mimetype = "") { + const item = this.fileInfo.find((info) => info.name === filename) + if (item) { + return this.readFileBlob(item.headerOffset + 512, item.size, mimetype) + } + } +} + +export class TarWriter { + private fileData: ITarWriteItem[] = [] + private _buffer: ArrayBuffer | null = null + + get buffer() { + if (!this._buffer) { + throw new Error("Buffer is not set") + } + return this._buffer + } + + addFile( + name: string, + file: string | ArrayBuffer | Uint8Array | Blob, + opts?: Partial, + ) { + const data = getArrayBuffer(file) + const size = (data as ArrayBuffer).byteLength ?? (file as Blob).size + const item: ITarWriteItem = { + name, + type: TarFileType.File, + data, + size, + opts, + } + this.fileData.push(item) + } + + addFolder(name: string, opts?: Partial) { + this.fileData.push({ + name, + type: TarFileType.Dir, + data: null, + size: 0, + opts, + }) + } + + private createBuffer() { + const dataSize = this.fileData.reduce( + (prev, item) => prev + 512 + 512 * Math.floor((item.size + 511) / 512), + 0, + ) + const bufSize = 10240 * Math.floor((dataSize + 10240 - 1) / 10240) + this._buffer = new ArrayBuffer(bufSize) + } + + async write() { + this.createBuffer() + const view = new Uint8Array(this.buffer) + let offset = 0 + for (const item of this.fileData) { + // write header + this.writeFileName(item.name, offset) + this.writeFileType(item.type, offset) + this.writeFileSize(item.size, offset) + this.fillHeader(offset, item.opts as Partial, item.type) + this.writeChecksum(offset) + + // write data + const data = new Uint8Array((await item.data) as ArrayBuffer) + view.set(data, offset + 512) + offset += 512 + 512 * Math.floor((item.size + 511) / 512) + } + return new Blob([this.buffer], { type: "application/x-tar" }) + } + + private writeString(str: string, offset: number, size: number) { + const strView = utf8Encode(str) + const view = new Uint8Array(this.buffer, offset, size) + for (let i = 0; i < size; i += 1) { + view[i] = i < strView.length ? strView[i] : 0 + } + } + + private writeFileName(name: string, offset: number) { + // offset: 0 + this.writeString(name, offset, 100) + } + + private writeFileType(type: TarFileType, offset: number) { + // offset: 156 + const typeView = new Uint8Array(this.buffer, offset + 156, 1) + typeView[0] = type.charCodeAt(0) + } + + private writeFileSize(size: number, offset: number) { + // offset: 124 + const sizeStr = size.toString(8).padStart(11, "0") + this.writeString(sizeStr, offset + 124, 12) + } + + private writeFileMode(mode: number, offset: number) { + // offset: 100 + this.writeString(mode.toString(8).padStart(7, "0"), offset + 100, 8) + } + + private writeFileUid(uid: number, offset: number) { + // offset: 108 + this.writeString(uid.toString(8).padStart(7, "0"), offset + 108, 8) + } + + private writeFileGid(gid: number, offset: number) { + // offset: 116 + this.writeString(gid.toString(8).padStart(7, "0"), offset + 116, 8) + } + + private writeFileMtime(mtime: number, offset: number) { + // offset: 136 + this.writeString(mtime.toString(8).padStart(11, "0"), offset + 136, 12) + } + + private writeFileUser(user: string, offset: number) { + // offset: 265 + this.writeString(user, offset + 265, 32) + } + + private writeFileGroup(group: string, offset: number) { + // offset: 297 + this.writeString(group, offset + 297, 32) + } + + private writeChecksum(offset: number) { + // offset: 148 + this.writeString(" ", offset + 148, 8) // first fill with spaces + + // add up header bytes + const header = new Uint8Array(this.buffer, offset, 512) + let chksum = 0 + for (let i = 0; i < 512; i += 1) { + chksum += header[i] + } + this.writeString(chksum.toString(8), offset + 148, 8) + } + + private fillHeader( + offset: number, + opts: Partial, + fileType: TarFileType, + ) { + const { uid, gid, mode, mtime, user, group } = { + uid: 1000, + gid: 1000, + mode: fileType === TarFileType.File ? 0o664 : 0o775, + mtime: ~~(Date.now() / 1000), + user: "tarballjs", + group: "tarballjs", + ...opts, + } + + this.writeFileMode(mode, offset) + this.writeFileUid(uid, offset) + this.writeFileGid(gid, offset) + this.writeFileMtime(mtime, offset) + + this.writeString("ustar", offset + 257, 6) // magic string + this.writeString("00", offset + 263, 2) // magic version + + this.writeFileUser(user, offset) + this.writeFileGroup(group, offset) + } +} + +function getArrayBuffer(file: string | ArrayBuffer | Uint8Array | Blob) { + if (typeof file === "string") { + return utf8Encode(file).buffer + } + if (file instanceof ArrayBuffer) { + return file + } + if (ArrayBuffer.isView(file)) { + return new Uint8Array(file).buffer + } + return file.arrayBuffer() +} diff --git a/site/src/util/templateVersion.ts b/site/src/util/templateVersion.ts index fbce12a9c7f6a..0ff4414b87d30 100644 --- a/site/src/util/templateVersion.ts +++ b/site/src/util/templateVersion.ts @@ -1,6 +1,6 @@ import * as API from "api/api" import { TemplateVersion } from "api/typesGenerated" -import untar from "js-untar" +import untar, { File as UntarFile } from "js-untar" import { FileTree, setFile } from "./filetree" /** @@ -41,21 +41,22 @@ export const getTemplateVersionFiles = async ( const allowedExtensions = ["tf", "md", "Dockerfile"] +export const isAllowedFile = (name: string) => { + return allowedExtensions.some((ext) => name.endsWith(ext)) +} + export const createTemplateVersionFileTree = async ( - version: TemplateVersion, + untarFiles: UntarFile[], ): Promise => { let fileTree: FileTree = {} - const tarFile = await API.getFile(version.job.file_id) const blobs: Record = {} - await untar(tarFile).then(undefined, undefined, async (file) => { - if (allowedExtensions.some((ext) => file.name.endsWith(ext))) { - blobs[file.name] = file.blob + for (const untarFile of untarFiles) { + if (isAllowedFile(untarFile.name)) { + blobs[untarFile.name] = untarFile.blob } - }) + } - // We don't want to get the blob text during untar to not block the main thread. - // Also, by doing it here, we can make all the loading in parallel. await Promise.all( Object.entries(blobs).map(async ([fullPath, blob]) => { const content = await blob.text() diff --git a/site/src/xServices/templateVersionEditor/templateVersionEditorXService.ts b/site/src/xServices/templateVersionEditor/templateVersionEditorXService.ts index 15bc033f0b594..34c9555f079c7 100644 --- a/site/src/xServices/templateVersionEditor/templateVersionEditorXService.ts +++ b/site/src/xServices/templateVersionEditor/templateVersionEditorXService.ts @@ -7,8 +7,10 @@ import { } from "api/typesGenerated" import { assign, createMachine } from "xstate" import * as API from "api/api" -import Tar from "tar-js" +import { File as UntarFile } from "js-untar" import { FileTree, traverse } from "util/filetree" +import { isAllowedFile } from "util/templateVersion" +import { TarWriter } from "util/tar" export interface CreateVersionData { file: File @@ -22,6 +24,7 @@ export interface TemplateVersionEditorMachineContext { version?: TemplateVersion resources?: WorkspaceResource[] buildLogs?: ProvisionerJobLog[] + untarFiles?: UntarFile[] } export const templateVersionEditorMachine = createMachine( @@ -31,6 +34,7 @@ export const templateVersionEditorMachine = createMachine( schema: { context: {} as TemplateVersionEditorMachineContext, events: {} as + | { type: "INITIALIZE"; untarFiles: UntarFile[] } | { type: "CREATE_VERSION" fileTree: FileTree @@ -61,8 +65,16 @@ export const templateVersionEditorMachine = createMachine( }, }, tsTypes: {} as import("./templateVersionEditorXService.typegen").Typegen0, - initial: "idle", + initial: "initializing", states: { + initializing: { + on: { + INITIALIZE: { + actions: ["assignUntarFiles"], + target: "idle", + }, + }, + }, idle: { on: { CREATE_VERSION: { @@ -201,20 +213,51 @@ export const templateVersionEditorMachine = createMachine( } }, }), + assignUntarFiles: assign({ + untarFiles: (_, { untarFiles }) => untarFiles, + }), }, services: { - uploadTar: (ctx) => { - if (!ctx.fileTree) { - throw new Error("files must be set") + uploadTar: async ({ fileTree, untarFiles }) => { + if (!fileTree) { + throw new Error("file tree must to be set") + } + if (!untarFiles) { + throw new Error("untar files must to be set") + } + const tar = new TarWriter() + + // Add previous non editable files + for (const untarFile of untarFiles) { + if (!isAllowedFile(untarFile.name)) { + if (untarFile.type === "5") { + tar.addFolder(untarFile.name, { + mode: parseInt(untarFile.mode, 8) & 0xfff, // https://github.com/beatgammit/tar-js/blob/master/lib/tar.js#L42 + mtime: untarFile.mtime, + user: untarFile.uname, + group: untarFile.gname, + }) + } else { + const buffer = await untarFile.blob.arrayBuffer() + tar.addFile(untarFile.name, new Uint8Array(buffer), { + mode: parseInt(untarFile.mode, 8) & 0xfff, // https://github.com/beatgammit/tar-js/blob/master/lib/tar.js#L42 + mtime: untarFile.mtime, + user: untarFile.uname, + group: untarFile.gname, + }) + } + } } - const tar = new Tar() - let out: Uint8Array = new Uint8Array() - traverse(ctx.fileTree, (content, _filename, fullPath) => { + // Add the editable files + traverse(fileTree, (content, _filename, fullPath) => { if (typeof content === "string") { - out = tar.append(fullPath, content) + tar.addFile(fullPath, content) + } else { + tar.addFolder(fullPath) } }) - return API.uploadTemplateFile(new File([out], "template.tar")) + const blob = await tar.write() + return API.uploadTemplateFile(new File([blob], "template.tar")) }, createBuild: (ctx) => { if (!ctx.uploadResponse) { diff --git a/site/yarn.lock b/site/yarn.lock index 2ea632bc7bd3b..591414a696afe 100644 --- a/site/yarn.lock +++ b/site/yarn.lock @@ -13678,11 +13678,6 @@ tapable@^2.1.1, tapable@^2.2.0: resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== -tar-js@^0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/tar-js/-/tar-js-0.3.0.tgz#6949aabfb0ba18bb1562ae51a439fd0f30183a17" - integrity sha512-9uqP2hJUZNKRkwPDe5nXxXdzo6w+BFBPq9x/tyi5/U/DneuSesO/HMb0y5TeWpfcv49YDJTs7SrrZeeu8ZHWDA== - tar-stream@^2.0.1: version "2.2.0" resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287"