diff --git a/site/js-untar.d.ts b/site/js-untar.d.ts deleted file mode 100644 index 6309b4cc0974f..0000000000000 --- a/site/js-untar.d.ts +++ /dev/null @@ -1,23 +0,0 @@ -declare module "js-untar" { - export interface File { - name: string - mode: string - blob: Blob - gid: number - uid: number - mtime: number - gname: string - uname: string - type: "0" | "1" | "2" | "3" | "4" | "5" //https://en.wikipedia.org/wiki/Tar_(computing) on Type flag field - } - - const Untar: (buffer: ArrayBuffer) => { - then: ( - resolve?: (files: File[]) => void, - reject?: () => Promise, - progress?: (file: File) => Promise, - ) => Promise - } - - export default Untar -} diff --git a/site/package.json b/site/package.json index 660400ec669c6..5fc3f102d1ea9 100644 --- a/site/package.json +++ b/site/package.json @@ -59,7 +59,6 @@ "history": "5.3.0", "i18next": "21.9.1", "jest-location-mock": "1.0.9", - "js-untar": "2.0.0", "just-debounce-it": "3.1.1", "lodash": "4.17.21", "playwright": "^1.29.2", @@ -103,7 +102,6 @@ "@types/react-helmet": "6.1.5", "@types/react-syntax-highlighter": "15.5.5", "@types/semver": "7.3.12", - "@types/tar-js": "^0.3.2", "@types/ua-parser-js": "0.7.36", "@types/uuid": "8.3.4", "@typescript-eslint/eslint-plugin": "5.50.0", diff --git a/site/site.go b/site/site.go index 4a0e73956388d..194c60b46635a 100644 --- a/site/site.go +++ b/site/site.go @@ -326,8 +326,6 @@ func cspHeaders(next http.Handler) http.Handler { // Report all violations back to the server to log CSPDirectiveReportURI: {"/api/v2/csp/reports"}, CSPFrameAncestors: {"'none'"}, - // worker for loading the .tar files on FE using js-untar - CSPDirectiveWorkerSrc: {"'self' blob:"}, // Only scripts can manipulate the dom. This prevents someone from // naming themselves something like ''. diff --git a/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/TemplateVersionEditorPage.tsx b/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/TemplateVersionEditorPage.tsx index e6e55568cca83..c6702a3ebf2b2 100644 --- a/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/TemplateVersionEditorPage.tsx +++ b/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/TemplateVersionEditorPage.tsx @@ -27,7 +27,7 @@ export const TemplateVersionEditorPage: FC = () => { }, { onSuccess(data) { - sendEvent({ type: "INITIALIZE", untarFiles: data.untarFiles }) + sendEvent({ type: "INITIALIZE", tarReader: data.tarReader }) }, }, ) diff --git a/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/data.ts b/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/data.ts index f46b523107107..1f9bfae22b8fb 100644 --- a/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/data.ts +++ b/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/data.ts @@ -1,7 +1,7 @@ import { useQuery, UseQueryOptions } from "@tanstack/react-query" import { getFile, getTemplateByName, getTemplateVersionByName } from "api/api" +import { TarReader } from "util/tar" import { createTemplateVersionFileTree } from "util/templateVersion" -import untar, { File as UntarFile } from "js-untar" const getTemplateVersionData = async ( orgId: string, @@ -13,17 +13,15 @@ const getTemplateVersionData = async ( getTemplateVersionByName(orgId, templateName, versionName), ]) const tarFile = await getFile(version.job.file_id) - let untarFiles: UntarFile[] = [] - await untar(tarFile).then((files) => { - untarFiles = files - }) - const fileTree = await createTemplateVersionFileTree(untarFiles) + const tarReader = new TarReader() + await tarReader.readFile(tarFile) + const fileTree = await createTemplateVersionFileTree(tarReader) return { template, version, fileTree, - untarFiles, + tarReader, } } diff --git a/site/src/util/tar.test.ts b/site/src/util/tar.test.ts index d2e81af06f2ce..f0eee8b7c2bd3 100644 --- a/site/src/util/tar.test.ts +++ b/site/src/util/tar.test.ts @@ -9,7 +9,12 @@ test("tar", async () => { writer.addFile("b.txt", new Blob(["world"]), { mtime }) writer.addFile("c.txt", "", { mtime }) writer.addFolder("etc", { mtime }) - writer.addFile("etc/d.txt", "", { mtime }) + writer.addFile("etc/d.txt", "Some text content", { + mtime, + user: "coder", + group: "codergroup", + mode: parseInt("777", 8), + }) const blob = await writer.write() // Read @@ -32,8 +37,11 @@ test("tar", async () => { }) verifyFile(fileInfos[4], reader.getTextFile(fileInfos[4].name) as string, { name: "etc/d.txt", - content: "", + content: "Some text content", }) + expect(fileInfos[4].group).toEqual("codergroup") + expect(fileInfos[4].user).toEqual("coder") + expect(fileInfos[4].mode).toEqual(parseInt("777", 8)) }) function verifyFile( diff --git a/site/src/util/tar.ts b/site/src/util/tar.ts index 47543d1dc2470..960b37ef83e59 100644 --- a/site/src/util/tar.ts +++ b/site/src/util/tar.ts @@ -13,6 +13,10 @@ export interface ITarFileInfo { name: string type: TarFileType size: number + mode: number + mtime: number + user: string + group: string headerOffset: number } @@ -34,7 +38,7 @@ export interface ITarWriteOptions { } export class TarReader { - private fileInfo: ITarFileInfo[] = [] + public fileInfo: ITarFileInfo[] = [] private _buffer: ArrayBuffer | null = null constructor() { @@ -64,22 +68,28 @@ export class TarReader { private readFileInfo() { this.fileInfo = [] let offset = 0 - let fileSize = 0 - let fileName = "" - let fileType: TarFileType + while (offset < this.buffer.byteLength - 512) { - fileName = this.readFileName(offset) + const fileName = this.readFileName(offset) if (!fileName) { break } - fileType = this.readFileType(offset) - fileSize = this.readFileSize(offset) + const fileType = this.readFileType(offset) + const fileSize = this.readFileSize(offset) + const fileMode = this.readFileMode(offset) + const fileMtime = this.readFileMtime(offset) + const fileUser = this.readFileUser(offset) + const fileGroup = this.readFileGroup(offset) this.fileInfo.push({ name: fileName, type: fileType, size: fileSize, headerOffset: offset, + mode: fileMode, + mtime: fileMtime, + user: fileUser, + group: fileGroup, }) offset += 512 + 512 * Math.floor((fileSize + 511) / 512) @@ -100,6 +110,24 @@ export class TarReader { return this.readString(offset, 100) } + private readFileMode(offset: number) { + const mode = this.readString(offset + 100, 8) + return parseInt(mode, 8) + } + + private readFileMtime(offset: number) { + const mtime = this.readString(offset + 136, 12) + return parseInt(mtime, 8) + } + + private readFileUser(offset: number) { + return this.readString(offset + 265, 32) + } + + private readFileGroup(offset: number) { + return this.readString(offset + 297, 32) + } + private readFileType(offset: number) { const typeView = new Uint8Array(this.buffer, offset + 156, 1) const typeStr = String.fromCharCode(typeView[0]) diff --git a/site/src/util/templateVersion.ts b/site/src/util/templateVersion.ts index 0ff4414b87d30..9122036149440 100644 --- a/site/src/util/templateVersion.ts +++ b/site/src/util/templateVersion.ts @@ -1,7 +1,7 @@ import * as API from "api/api" import { TemplateVersion } from "api/typesGenerated" -import untar, { File as UntarFile } from "js-untar" import { FileTree, setFile } from "./filetree" +import { TarReader } from "./tar" /** * Content by filename @@ -10,32 +10,16 @@ export type TemplateVersionFiles = Record export const getTemplateVersionFiles = async ( version: TemplateVersion, - allowedExtensions: string[], - allowedFiles: string[], ): Promise => { const files: TemplateVersionFiles = {} const tarFile = await API.getFile(version.job.file_id) - const blobs: Record = {} - - await untar(tarFile).then(undefined, undefined, async (file) => { - const paths = file.name.split("/") - const filename = paths[paths.length - 1] - const [_, extension] = filename.split(".") - - if ( - allowedExtensions.includes(extension) || - allowedFiles.includes(filename) - ) { - blobs[filename] = file.blob + const tarReader = new TarReader() + await tarReader.readFile(tarFile) + for (const file of tarReader.fileInfo) { + if (isAllowedFile(file.name)) { + files[file.name] = tarReader.getTextFile(file.name) as string } - }) - - await Promise.all( - Object.entries(blobs).map(async ([filename, blob]) => { - files[filename] = await blob.text() - }), - ) - + } return files } @@ -46,23 +30,17 @@ export const isAllowedFile = (name: string) => { } export const createTemplateVersionFileTree = async ( - untarFiles: UntarFile[], + tarReader: TarReader, ): Promise => { let fileTree: FileTree = {} - const blobs: Record = {} - - for (const untarFile of untarFiles) { - if (isAllowedFile(untarFile.name)) { - blobs[untarFile.name] = untarFile.blob + for (const file of tarReader.fileInfo) { + if (isAllowedFile(file.name)) { + fileTree = setFile( + file.name, + tarReader.getTextFile(file.name) as string, + fileTree, + ) } } - - await Promise.all( - Object.entries(blobs).map(async ([fullPath, blob]) => { - const content = await blob.text() - fileTree = setFile(fullPath, content, fileTree) - }), - ) - return fileTree } diff --git a/site/src/xServices/templateVersion/templateVersionXService.ts b/site/src/xServices/templateVersion/templateVersionXService.ts index bd7d371894723..ee29de5da24b7 100644 --- a/site/src/xServices/templateVersion/templateVersionXService.ts +++ b/site/src/xServices/templateVersion/templateVersionXService.ts @@ -162,23 +162,9 @@ export const templateVersionMachine = createMachine( } const loadFilesPromises: ReturnType[] = [] - const allowedExtensions = ["tf", "md"] - const allowedFiles = ["Dockerfile"] - loadFilesPromises.push( - getTemplateVersionFiles( - currentVersion, - allowedExtensions, - allowedFiles, - ), - ) + loadFilesPromises.push(getTemplateVersionFiles(currentVersion)) if (previousVersion) { - loadFilesPromises.push( - getTemplateVersionFiles( - previousVersion, - allowedExtensions, - allowedFiles, - ), - ) + loadFilesPromises.push(getTemplateVersionFiles(previousVersion)) } const [currentFiles, previousFiles] = await Promise.all( loadFilesPromises, diff --git a/site/src/xServices/templateVersionEditor/templateVersionEditorXService.ts b/site/src/xServices/templateVersionEditor/templateVersionEditorXService.ts index 34c9555f079c7..88ded1c42c01b 100644 --- a/site/src/xServices/templateVersionEditor/templateVersionEditorXService.ts +++ b/site/src/xServices/templateVersionEditor/templateVersionEditorXService.ts @@ -7,10 +7,9 @@ import { } from "api/typesGenerated" import { assign, createMachine } from "xstate" import * as API from "api/api" -import { File as UntarFile } from "js-untar" import { FileTree, traverse } from "util/filetree" import { isAllowedFile } from "util/templateVersion" -import { TarWriter } from "util/tar" +import { TarReader, TarWriter } from "util/tar" export interface CreateVersionData { file: File @@ -24,7 +23,7 @@ export interface TemplateVersionEditorMachineContext { version?: TemplateVersion resources?: WorkspaceResource[] buildLogs?: ProvisionerJobLog[] - untarFiles?: UntarFile[] + tarReader?: TarReader } export const templateVersionEditorMachine = createMachine( @@ -34,7 +33,7 @@ export const templateVersionEditorMachine = createMachine( schema: { context: {} as TemplateVersionEditorMachineContext, events: {} as - | { type: "INITIALIZE"; untarFiles: UntarFile[] } + | { type: "INITIALIZE"; tarReader: TarReader } | { type: "CREATE_VERSION" fileTree: FileTree @@ -70,7 +69,7 @@ export const templateVersionEditorMachine = createMachine( initializing: { on: { INITIALIZE: { - actions: ["assignUntarFiles"], + actions: ["assignTarReader"], target: "idle", }, }, @@ -213,38 +212,41 @@ export const templateVersionEditorMachine = createMachine( } }, }), - assignUntarFiles: assign({ - untarFiles: (_, { untarFiles }) => untarFiles, + assignTarReader: assign({ + tarReader: (_, { tarReader }) => tarReader, }), }, services: { - uploadTar: async ({ fileTree, untarFiles }) => { + uploadTar: async ({ fileTree, tarReader }) => { if (!fileTree) { throw new Error("file tree must to be set") } - if (!untarFiles) { - throw new Error("untar files must to be set") + if (!tarReader) { + throw new Error("tar reader must to be set") } const tar = new TarWriter() // Add previous non editable files - for (const untarFile of untarFiles) { - if (!isAllowedFile(untarFile.name)) { - if (untarFile.type === "5") { - tar.addFolder(untarFile.name, { - mode: parseInt(untarFile.mode, 8) & 0xfff, // https://github.com/beatgammit/tar-js/blob/master/lib/tar.js#L42 - mtime: untarFile.mtime, - user: untarFile.uname, - group: untarFile.gname, + for (const file of tarReader.fileInfo) { + if (!isAllowedFile(file.name)) { + if (file.type === "5") { + tar.addFolder(file.name, { + mode: file.mode, // https://github.com/beatgammit/tar-js/blob/master/lib/tar.js#L42 + mtime: file.mtime, + user: file.user, + group: file.group, }) } else { - const buffer = await untarFile.blob.arrayBuffer() - tar.addFile(untarFile.name, new Uint8Array(buffer), { - mode: parseInt(untarFile.mode, 8) & 0xfff, // https://github.com/beatgammit/tar-js/blob/master/lib/tar.js#L42 - mtime: untarFile.mtime, - user: untarFile.uname, - group: untarFile.gname, - }) + tar.addFile( + file.name, + tarReader.getTextFile(file.name) as string, + { + mode: file.mode, // https://github.com/beatgammit/tar-js/blob/master/lib/tar.js#L42 + mtime: file.mtime, + user: file.user, + group: file.group, + }, + ) } } } diff --git a/site/yarn.lock b/site/yarn.lock index 591414a696afe..af2e49b69af65 100644 --- a/site/yarn.lock +++ b/site/yarn.lock @@ -3455,11 +3455,6 @@ resolved "https://registry.yarnpkg.com/@types/tapable/-/tapable-1.0.8.tgz#b94a4391c85666c7b73299fd3ad79d4faa435310" integrity sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ== -"@types/tar-js@^0.3.2": - version "0.3.2" - resolved "https://registry.yarnpkg.com/@types/tar-js/-/tar-js-0.3.2.tgz#23d3c7c8ce22bec6ed8355c2169dd3fd6ebe2583" - integrity sha512-0ySOBNP+/Ey67EZ0QaCOgkt6zSAeayTwsoln02ztlyB5lF4NQD0sl5C7E5eKS+QUb7xgTEPdPo9LUjYOHUJVqQ== - "@types/testing-library__jest-dom@^5.9.1": version "5.14.5" resolved "https://registry.yarnpkg.com/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.5.tgz#d113709c90b3c75fdb127ec338dad7d5f86c974f" @@ -9492,11 +9487,6 @@ js-string-escape@^1.0.1: resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== -js-untar@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/js-untar/-/js-untar-2.0.0.tgz#b452d28dedd3b0be92c2ac9a7d70f612a93c7453" - integrity sha512-7CsDLrYQMbLxDt2zl9uKaPZSdmJMvGGQ7wo9hoB3J+z/VcO2w63bXFgHVnjF1+S9wD3zAu8FBVj7EYWjTQ3Z7g== - js-yaml@^3.10.0, js-yaml@^3.13.1: version "3.14.1" resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.1.tgz#dae812fdb3825fa306609a8717383c50c36a0537"