From 11fd5529bab87b59ce41fdf8df7bcaab005a683d Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 26 Jan 2024 15:35:08 +0100 Subject: [PATCH 1/6] feat: support template bundles as zip archives --- coderd/apidoc/docs.go | 2 +- coderd/apidoc/swagger.json | 2 +- coderd/files.go | 10 ++++++++-- docs/api/files.md | 10 +++++----- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index 098ea767e4ffe..95883b0225dd8 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -920,7 +920,7 @@ const docTemplate = `{ { "type": "string", "default": "application/x-tar", - "description": "Content-Type must be ` + "`" + `application/x-tar` + "`" + `", + "description": "Content-Type must be ` + "`" + `application/x-tar` + "`" + ` or ` + "`" + `application/zip` + "`" + `", "name": "Content-Type", "in": "header", "required": true diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 24bc5e29cc05c..fa5a3f61e925a 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -788,7 +788,7 @@ { "type": "string", "default": "application/x-tar", - "description": "Content-Type must be `application/x-tar`", + "description": "Content-Type must be `application/x-tar` or `application/zip`", "name": "Content-Type", "in": "header", "required": true diff --git a/coderd/files.go b/coderd/files.go index a04ba1eacedc3..a2084ce3c554a 100644 --- a/coderd/files.go +++ b/coderd/files.go @@ -21,6 +21,7 @@ import ( const ( tarMimeType = "application/x-tar" + zipMimeType = "application/zip" ) // @Summary Upload file @@ -30,7 +31,7 @@ const ( // @Produce json // @Accept application/x-tar // @Tags Files -// @Param Content-Type header string true "Content-Type must be `application/x-tar`" default(application/x-tar) +// @Param Content-Type header string true "Content-Type must be `application/x-tar` or `application/zip`" default(application/x-tar) // @Param file formData file true "File to be uploaded" // @Success 201 {object} codersdk.UploadResponse // @Router /files [post] @@ -41,7 +42,7 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) { contentType := r.Header.Get("Content-Type") switch contentType { - case tarMimeType: + case tarMimeType, zipMimeType: default: httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ Message: fmt.Sprintf("Unsupported content type header %q.", contentType), @@ -78,6 +79,11 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) { return } + if contentType == zipMimeType { + // FIXME Repack to .tar format. + contentType = tarMimeType + } + id := uuid.New() file, err = api.Database.InsertFile(ctx, database.InsertFileParams{ ID: id, diff --git a/docs/api/files.md b/docs/api/files.md index 81d93479aeb36..936be96cc575e 100644 --- a/docs/api/files.md +++ b/docs/api/files.md @@ -22,11 +22,11 @@ file: string ### Parameters -| Name | In | Type | Required | Description | -| -------------- | ------ | ------ | -------- | ---------------------------------------- | -| `Content-Type` | header | string | true | Content-Type must be `application/x-tar` | -| `body` | body | object | true | | -| `» file` | body | binary | true | File to be uploaded | +| Name | In | Type | Required | Description | +| -------------- | ------ | ------ | -------- | ------------------------------------------------------------- | +| `Content-Type` | header | string | true | Content-Type must be `application/x-tar` or `application/zip` | +| `body` | body | object | true | | +| `» file` | body | binary | true | File to be uploaded | ### Example responses From bc9a5551f8a79147a6f662c50b72557c17bc7287 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 26 Jan 2024 17:27:50 +0100 Subject: [PATCH 2/6] WIP --- coderd/files.go | 29 ++++++++++++++++++++++------ coderd/fileszip.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 coderd/fileszip.go diff --git a/coderd/files.go b/coderd/files.go index a2084ce3c554a..29396fffb0e20 100644 --- a/coderd/files.go +++ b/coderd/files.go @@ -1,6 +1,8 @@ package coderd import ( + "archive/zip" + "bytes" "crypto/sha256" "database/sql" "encoding/hex" @@ -40,7 +42,6 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) { apiKey := httpmw.APIKey(r) contentType := r.Header.Get("Content-Type") - switch contentType { case tarMimeType, zipMimeType: default: @@ -59,6 +60,27 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) { }) return } + + if contentType == zipMimeType { + zipReader, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ + Message: "Incomplete .zip archive file.", + Detail: err.Error(), + }) + return + } + + data, err = createTarFromZip(zipReader) + if err != nil { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error processing .zip archive.", + Detail: err.Error(), + }) + return + } + } + hashBytes := sha256.Sum256(data) hash := hex.EncodeToString(hashBytes[:]) file, err := api.Database.GetFileByHashAndCreator(ctx, database.GetFileByHashAndCreatorParams{ @@ -79,11 +101,6 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) { return } - if contentType == zipMimeType { - // FIXME Repack to .tar format. - contentType = tarMimeType - } - id := uuid.New() file, err = api.Database.InsertFile(ctx, database.InsertFileParams{ ID: id, diff --git a/coderd/fileszip.go b/coderd/fileszip.go new file mode 100644 index 0000000000000..1e316e8ad0339 --- /dev/null +++ b/coderd/fileszip.go @@ -0,0 +1,48 @@ +package coderd + +import ( + "archive/tar" + "archive/zip" + "bytes" + "io" +) + +func createTarFromZip(zipReader *zip.Reader) ([]byte, error) { + var tarBuffer bytes.Buffer + + tarWriter := tar.NewWriter(&tarBuffer) + defer tarWriter.Close() + + for _, zipFile := range zipReader.File { + err := processFileInZipArchive(zipFile, tarWriter) + if err != nil { + return nil, err + } + } + return tarBuffer.Bytes(), nil +} + +func processFileInZipArchive(zipFile *zip.File, tarWriter *tar.Writer) error { + zipFileReader, err := zipFile.Open() + if err != nil { + return err + } + defer zipFileReader.Close() + + tarHeader, err := tar.FileInfoHeader(zipFile.FileInfo(), "") + if err != nil { + return err + } + tarHeader.Name = zipFile.Name + + err = tarWriter.WriteHeader(tarHeader) + if err != nil { + return err + } + + _, err = io.Copy(tarWriter, zipFileReader) + if err != nil { + return err + } + return nil +} From af8a0d55c08abbfb08066f429d2013bc0064124e Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 26 Jan 2024 17:44:15 +0100 Subject: [PATCH 3/6] Buffer --- coderd/fileszip.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/coderd/fileszip.go b/coderd/fileszip.go index 1e316e8ad0339..5b8c0b5488105 100644 --- a/coderd/fileszip.go +++ b/coderd/fileszip.go @@ -4,9 +4,12 @@ import ( "archive/tar" "archive/zip" "bytes" + "errors" "io" ) +const zipCopyBufferSize = 4096 + func createTarFromZip(zipReader *zip.Reader) ([]byte, error) { var tarBuffer bytes.Buffer @@ -40,9 +43,14 @@ func processFileInZipArchive(zipFile *zip.File, tarWriter *tar.Writer) error { return err } - _, err = io.Copy(tarWriter, zipFileReader) - if err != nil { - return err + for { + _, err := io.CopyN(tarWriter, zipFileReader, zipCopyBufferSize) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return err + } } return nil } From b20a483a247330d48d7d1329c2e8ffad1c4c248f Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 30 Jan 2024 13:24:38 +0100 Subject: [PATCH 4/6] Zip Tar --- coderd/files.go | 2 +- coderd/files_test.go | 28 +++++++++++++++++++++++++++- coderd/fileszip.go | 6 +++++- codersdk/files.go | 1 + 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/coderd/files.go b/coderd/files.go index 29396fffb0e20..ad4d52a5a265b 100644 --- a/coderd/files.go +++ b/coderd/files.go @@ -71,7 +71,7 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) { return } - data, err = createTarFromZip(zipReader) + data, err = CreateTarFromZip(zipReader) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error processing .zip archive.", diff --git a/coderd/files_test.go b/coderd/files_test.go index 1a3f407a6e1f6..956ffe10b67d6 100644 --- a/coderd/files_test.go +++ b/coderd/files_test.go @@ -9,8 +9,10 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/require" + "github.com/coder/coder/v2/coderd" "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/provisioner/echo" "github.com/coder/coder/v2/testutil" ) @@ -72,7 +74,7 @@ func TestDownload(t *testing.T) { require.Equal(t, http.StatusNotFound, apiErr.StatusCode()) }) - t.Run("Insert", func(t *testing.T) { + t.Run("InsertTar_DownloadTar", func(t *testing.T) { t.Parallel() client := coderdtest.New(t, nil) _ = coderdtest.CreateFirstUser(t, client) @@ -87,4 +89,28 @@ func TestDownload(t *testing.T) { require.Len(t, data, 1024) require.Equal(t, codersdk.ContentTypeTar, contentType) }) + + t.Run("InsertZip_DownloadTar", func(t *testing.T) { + t.Parallel() + client := coderdtest.New(t, nil) + _ = coderdtest.CreateFirstUser(t, client) + + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + defer cancel() + + tarball, err := echo.Tar(&echo.Responses{ + Parse: echo.ParseComplete, + ProvisionApply: echo.ApplyComplete, + }) + require.NoError(t, err) + zipContent, err := coderd.CreateZipFromTar(tarball) + require.NoError(t, err) + + resp, err := client.Upload(ctx, codersdk.ContentTypeZip, bytes.NewReader(zipContent)) + require.NoError(t, err) + data, contentType, err := client.Download(ctx, resp.ID) + require.NoError(t, err) + require.Equal(t, codersdk.ContentTypeTar, contentType) + require.Len(t, data, 1024) // FIXME compare output + }) } diff --git a/coderd/fileszip.go b/coderd/fileszip.go index 5b8c0b5488105..aae7e5a3da45b 100644 --- a/coderd/fileszip.go +++ b/coderd/fileszip.go @@ -10,7 +10,7 @@ import ( const zipCopyBufferSize = 4096 -func createTarFromZip(zipReader *zip.Reader) ([]byte, error) { +func CreateTarFromZip(zipReader *zip.Reader) ([]byte, error) { var tarBuffer bytes.Buffer tarWriter := tar.NewWriter(&tarBuffer) @@ -54,3 +54,7 @@ func processFileInZipArchive(zipFile *zip.File, tarWriter *tar.Writer) error { } return nil } + +func CreateZipFromTar(tarball []byte) ([]byte, error) { + panic("not implemented yet") +} diff --git a/codersdk/files.go b/codersdk/files.go index 3525e9d785d6e..1cea1b83aefb8 100644 --- a/codersdk/files.go +++ b/codersdk/files.go @@ -12,6 +12,7 @@ import ( const ( ContentTypeTar = "application/x-tar" + ContentTypeZip = "application/zip" ) // UploadResponse contains the hash to reference the uploaded file. From fd868964c2dba54580fb7dd472ba96f005f24a05 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 30 Jan 2024 16:30:19 +0100 Subject: [PATCH 5/6] WIP --- coderd/files.go | 5 ++- coderd/files_test.go | 13 +++--- coderd/fileszip.go | 87 ++++++++++++++++++++++++++++----------- provisioner/echo/serve.go | 2 +- 4 files changed, 77 insertions(+), 30 deletions(-) diff --git a/coderd/files.go b/coderd/files.go index ad4d52a5a265b..2c36c088ba14b 100644 --- a/coderd/files.go +++ b/coderd/files.go @@ -24,6 +24,8 @@ import ( const ( tarMimeType = "application/x-tar" zipMimeType = "application/zip" + + httpFileMaxBytes = 10 * (10 << 20) ) // @Summary Upload file @@ -51,7 +53,7 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) { return } - r.Body = http.MaxBytesReader(rw, r.Body, 10*(10<<20)) + r.Body = http.MaxBytesReader(rw, r.Body, httpFileMaxBytes) data, err := io.ReadAll(r.Body) if err != nil { httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ @@ -79,6 +81,7 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) { }) return } + contentType = tarMimeType } hashBytes := sha256.Sum256(data) diff --git a/coderd/files_test.go b/coderd/files_test.go index 956ffe10b67d6..0d7704acdf256 100644 --- a/coderd/files_test.go +++ b/coderd/files_test.go @@ -1,6 +1,7 @@ package coderd_test import ( + "archive/tar" "bytes" "context" "net/http" @@ -95,22 +96,24 @@ func TestDownload(t *testing.T) { client := coderdtest.New(t, nil) _ = coderdtest.CreateFirstUser(t, client) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) - defer cancel() - tarball, err := echo.Tar(&echo.Responses{ Parse: echo.ParseComplete, ProvisionApply: echo.ApplyComplete, }) require.NoError(t, err) - zipContent, err := coderd.CreateZipFromTar(tarball) + + tarReader := tar.NewReader(bytes.NewReader(tarball)) + require.NoError(t, err) + zipContent, err := coderd.CreateZipFromTar(tarReader) require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + defer cancel() resp, err := client.Upload(ctx, codersdk.ContentTypeZip, bytes.NewReader(zipContent)) require.NoError(t, err) data, contentType, err := client.Download(ctx, resp.ID) require.NoError(t, err) require.Equal(t, codersdk.ContentTypeTar, contentType) - require.Len(t, data, 1024) // FIXME compare output + require.Equal(t, tarball, data) }) } diff --git a/coderd/fileszip.go b/coderd/fileszip.go index aae7e5a3da45b..b1ec529a1f016 100644 --- a/coderd/fileszip.go +++ b/coderd/fileszip.go @@ -6,55 +6,96 @@ import ( "bytes" "errors" "io" + "log" ) -const zipCopyBufferSize = 4096 - func CreateTarFromZip(zipReader *zip.Reader) ([]byte, error) { var tarBuffer bytes.Buffer + err := writeTarArchive(&tarBuffer, zipReader) + if err != nil { + return nil, err + } + return tarBuffer.Bytes(), nil +} - tarWriter := tar.NewWriter(&tarBuffer) +func writeTarArchive(tarBuffer *bytes.Buffer, zipReader *zip.Reader) error { + tarWriter := tar.NewWriter(tarBuffer) defer tarWriter.Close() - for _, zipFile := range zipReader.File { - err := processFileInZipArchive(zipFile, tarWriter) + for _, file := range zipReader.File { + err := processFileInZipArchive(file, tarWriter) if err != nil { - return nil, err + return err } } - return tarBuffer.Bytes(), nil + return tarWriter.Flush() } -func processFileInZipArchive(zipFile *zip.File, tarWriter *tar.Writer) error { - zipFileReader, err := zipFile.Open() +func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error { + fileReader, err := file.Open() if err != nil { return err } - defer zipFileReader.Close() + defer fileReader.Close() - tarHeader, err := tar.FileInfoHeader(zipFile.FileInfo(), "") + err = tarWriter.WriteHeader(&tar.Header{ + Name: file.Name, + Size: file.FileInfo().Size(), + Mode: 0o644, + }) if err != nil { return err } - tarHeader.Name = zipFile.Name - err = tarWriter.WriteHeader(tarHeader) + n, err := io.CopyN(tarWriter, fileReader, httpFileMaxBytes) + log.Println(file.Name, n, err) + if errors.Is(err, io.EOF) { + err = nil + } + return err +} + +func CreateZipFromTar(tarReader *tar.Reader) ([]byte, error) { + var zipBuffer bytes.Buffer + err := writeZipArchive(&zipBuffer, tarReader) if err != nil { - return err + return nil, err } + return zipBuffer.Bytes(), nil +} + +func writeZipArchive(zipBuffer *bytes.Buffer, tarReader *tar.Reader) error { + zipWriter := zip.NewWriter(zipBuffer) + defer zipWriter.Close() for { - _, err := io.CopyN(tarWriter, zipFileReader, zipCopyBufferSize) + tarHeader, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { - if errors.Is(err, io.EOF) { - break - } return err } - } - return nil -} -func CreateZipFromTar(tarball []byte) ([]byte, error) { - panic("not implemented yet") + zipHeader, err := zip.FileInfoHeader(tarHeader.FileInfo()) + if err != nil { + return err + } + zipHeader.Name = tarHeader.Name + + zipEntry, err := zipWriter.CreateHeader(zipHeader) + if err != nil { + return err + } + + _, err = io.CopyN(zipEntry, tarReader, httpFileMaxBytes) + if errors.Is(err, io.EOF) { + err = nil + } + if err != nil { + return err + } + } + return zipWriter.Flush() } diff --git a/provisioner/echo/serve.go b/provisioner/echo/serve.go index 3196afdaa58a7..871c46c4e34fe 100644 --- a/provisioner/echo/serve.go +++ b/provisioner/echo/serve.go @@ -306,7 +306,7 @@ func TarWithOptions(ctx context.Context, logger slog.Logger, responses *Response } } } - err := writer.Flush() + err := writer.Close() if err != nil { return nil, err } From 8f631ef409e40f66a3b55fb98795fa65079138af Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 31 Jan 2024 11:21:17 +0100 Subject: [PATCH 6/6] Implement backend --- coderd/files.go | 35 ++++++++++++++++++++++++++++++---- coderd/files_test.go | 40 ++++++++++++++++++++++++++++++++++++++- coderd/fileszip.go | 14 +++++++------- codersdk/files.go | 9 ++++++++- provisioner/echo/serve.go | 1 + 5 files changed, 86 insertions(+), 13 deletions(-) diff --git a/coderd/files.go b/coderd/files.go index 2c36c088ba14b..d5379c4d8b036 100644 --- a/coderd/files.go +++ b/coderd/files.go @@ -1,6 +1,7 @@ package coderd import ( + "archive/tar" "archive/zip" "bytes" "crypto/sha256" @@ -11,6 +12,7 @@ import ( "io" "net/http" + "cdr.dev/slog" "github.com/go-chi/chi/v5" "github.com/google/uuid" @@ -134,7 +136,10 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) { // @Success 200 // @Router /files/{fileID} [get] func (api *API) fileByID(rw http.ResponseWriter, r *http.Request) { - ctx := r.Context() + var ( + ctx = r.Context() + format = r.URL.Query().Get("format") + ) fileID := chi.URLParam(r, "fileID") if fileID == "" { @@ -165,7 +170,29 @@ func (api *API) fileByID(rw http.ResponseWriter, r *http.Request) { return } - rw.Header().Set("Content-Type", file.Mimetype) - rw.WriteHeader(http.StatusOK) - _, _ = rw.Write(file.Data) + switch format { + case codersdk.FormatZip: + if file.Mimetype != codersdk.ContentTypeTar { + httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ + Message: "Only .tar files can be converted to .zip format", + Detail: err.Error(), + }) + return + } + + rw.Header().Set("Content-Type", codersdk.ContentTypeZip) + rw.WriteHeader(http.StatusOK) + err = WriteZipArchive(rw, tar.NewReader(bytes.NewReader(file.Data))) + if err != nil { + api.Logger.Error(ctx, "invalid .zip archive", slog.F("file_id", fileID), slog.F("mimetype", file.Mimetype), slog.Error(err)) + } + case "": // no format? no conversion + rw.Header().Set("Content-Type", file.Mimetype) + _, _ = rw.Write(file.Data) + default: + httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ + Message: "Unsupported conversion format.", + Detail: err.Error(), + }) + } } diff --git a/coderd/files_test.go b/coderd/files_test.go index 0d7704acdf256..ff0eb60585742 100644 --- a/coderd/files_test.go +++ b/coderd/files_test.go @@ -80,13 +80,17 @@ func TestDownload(t *testing.T) { client := coderdtest.New(t, nil) _ = coderdtest.CreateFirstUser(t, client) + // given ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() + // when resp, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(make([]byte, 1024))) require.NoError(t, err) data, contentType, err := client.Download(ctx, resp.ID) require.NoError(t, err) + + // then require.Len(t, data, 1024) require.Equal(t, codersdk.ContentTypeTar, contentType) }) @@ -96,6 +100,7 @@ func TestDownload(t *testing.T) { client := coderdtest.New(t, nil) _ = coderdtest.CreateFirstUser(t, client) + // given tarball, err := echo.Tar(&echo.Responses{ Parse: echo.ParseComplete, ProvisionApply: echo.ApplyComplete, @@ -103,17 +108,50 @@ func TestDownload(t *testing.T) { require.NoError(t, err) tarReader := tar.NewReader(bytes.NewReader(tarball)) - require.NoError(t, err) zipContent, err := coderd.CreateZipFromTar(tarReader) require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() + + // when resp, err := client.Upload(ctx, codersdk.ContentTypeZip, bytes.NewReader(zipContent)) require.NoError(t, err) data, contentType, err := client.Download(ctx, resp.ID) require.NoError(t, err) + + // then require.Equal(t, codersdk.ContentTypeTar, contentType) require.Equal(t, tarball, data) }) + + t.Run("InsertTar_DownloadZip", func(t *testing.T) { + t.Parallel() + client := coderdtest.New(t, nil) + _ = coderdtest.CreateFirstUser(t, client) + + // given + tarball, err := echo.Tar(&echo.Responses{ + Parse: echo.ParseComplete, + ProvisionApply: echo.ApplyComplete, + }) + require.NoError(t, err) + + tarReader := tar.NewReader(bytes.NewReader(tarball)) + expectedZip, err := coderd.CreateZipFromTar(tarReader) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + defer cancel() + + // when + resp, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(tarball)) + require.NoError(t, err) + data, contentType, err := client.DownloadWithFormat(ctx, resp.ID, codersdk.FormatZip) + require.NoError(t, err) + + // then + require.Equal(t, codersdk.ContentTypeZip, contentType) + require.Equal(t, expectedZip, data) + }) } diff --git a/coderd/fileszip.go b/coderd/fileszip.go index b1ec529a1f016..b001958912fa1 100644 --- a/coderd/fileszip.go +++ b/coderd/fileszip.go @@ -18,8 +18,8 @@ func CreateTarFromZip(zipReader *zip.Reader) ([]byte, error) { return tarBuffer.Bytes(), nil } -func writeTarArchive(tarBuffer *bytes.Buffer, zipReader *zip.Reader) error { - tarWriter := tar.NewWriter(tarBuffer) +func writeTarArchive(w io.Writer, zipReader *zip.Reader) error { + tarWriter := tar.NewWriter(w) defer tarWriter.Close() for _, file := range zipReader.File { @@ -28,7 +28,7 @@ func writeTarArchive(tarBuffer *bytes.Buffer, zipReader *zip.Reader) error { return err } } - return tarWriter.Flush() + return nil } func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error { @@ -57,15 +57,15 @@ func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error { func CreateZipFromTar(tarReader *tar.Reader) ([]byte, error) { var zipBuffer bytes.Buffer - err := writeZipArchive(&zipBuffer, tarReader) + err := WriteZipArchive(&zipBuffer, tarReader) if err != nil { return nil, err } return zipBuffer.Bytes(), nil } -func writeZipArchive(zipBuffer *bytes.Buffer, tarReader *tar.Reader) error { - zipWriter := zip.NewWriter(zipBuffer) +func WriteZipArchive(w io.Writer, tarReader *tar.Reader) error { + zipWriter := zip.NewWriter(w) defer zipWriter.Close() for { @@ -97,5 +97,5 @@ func writeZipArchive(zipBuffer *bytes.Buffer, tarReader *tar.Reader) error { return err } } - return zipWriter.Flush() + return nil // don't need to flush as we call `writer.Close()` } diff --git a/codersdk/files.go b/codersdk/files.go index 1cea1b83aefb8..a14f2ca73d386 100644 --- a/codersdk/files.go +++ b/codersdk/files.go @@ -13,6 +13,8 @@ import ( const ( ContentTypeTar = "application/x-tar" ContentTypeZip = "application/zip" + + FormatZip = "zip" ) // UploadResponse contains the hash to reference the uploaded file. @@ -39,7 +41,12 @@ func (c *Client) Upload(ctx context.Context, contentType string, rd io.Reader) ( // Download fetches a file by uploaded hash. func (c *Client) Download(ctx context.Context, id uuid.UUID) ([]byte, string, error) { - res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/files/%s", id.String()), nil) + return c.DownloadWithFormat(ctx, id, "") +} + +// Download fetches a file by uploaded hash, but it forces format conversion. +func (c *Client) DownloadWithFormat(ctx context.Context, id uuid.UUID, format string) ([]byte, string, error) { + res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/files/%s?format=%s", id.String(), format), nil) if err != nil { return nil, "", err } diff --git a/provisioner/echo/serve.go b/provisioner/echo/serve.go index 871c46c4e34fe..53ec286b3c358 100644 --- a/provisioner/echo/serve.go +++ b/provisioner/echo/serve.go @@ -306,6 +306,7 @@ func TarWithOptions(ctx context.Context, logger slog.Logger, responses *Response } } } + // `writer.Close()` function flushes the writer buffer, and adds extra padding to create a legal tarball. err := writer.Close() if err != nil { return nil, err