Skip to content

feat: support template bundles as zip archives #11839

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jan 31, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion coderd/apidoc/docs.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion coderd/apidoc/swagger.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

69 changes: 61 additions & 8 deletions coderd/files.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package coderd

import (
"archive/tar"
"archive/zip"
"bytes"
"crypto/sha256"
"database/sql"
"encoding/hex"
Expand All @@ -9,6 +12,7 @@ import (
"io"
"net/http"

"cdr.dev/slog"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"

Expand All @@ -21,6 +25,9 @@ import (

const (
tarMimeType = "application/x-tar"
zipMimeType = "application/zip"

httpFileMaxBytes = 10 * (10 << 20)
)

// @Summary Upload file
Expand All @@ -30,7 +37,7 @@ const (
// @Produce json
// @Accept application/x-tar
// @Tags Files
// @Param Content-Type header string true "Content-Type must be `application/x-tar`" default(application/x-tar)
// @Param Content-Type header string true "Content-Type must be `application/x-tar` or `application/zip`" default(application/x-tar)
// @Param file formData file true "File to be uploaded"
// @Success 201 {object} codersdk.UploadResponse
// @Router /files [post]
Expand All @@ -39,17 +46,16 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) {
apiKey := httpmw.APIKey(r)

contentType := r.Header.Get("Content-Type")

switch contentType {
case tarMimeType:
case tarMimeType, zipMimeType:
default:
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
Message: fmt.Sprintf("Unsupported content type header %q.", contentType),
})
return
}

r.Body = http.MaxBytesReader(rw, r.Body, 10*(10<<20))
r.Body = http.MaxBytesReader(rw, r.Body, httpFileMaxBytes)
data, err := io.ReadAll(r.Body)
if err != nil {
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
Expand All @@ -58,6 +64,28 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) {
})
return
}

if contentType == zipMimeType {
zipReader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
Message: "Incomplete .zip archive file.",
Detail: err.Error(),
})
return
}

data, err = CreateTarFromZip(zipReader)
if err != nil {
httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
Message: "Internal error processing .zip archive.",
Detail: err.Error(),
})
return
}
contentType = tarMimeType
}

hashBytes := sha256.Sum256(data)
hash := hex.EncodeToString(hashBytes[:])
file, err := api.Database.GetFileByHashAndCreator(ctx, database.GetFileByHashAndCreatorParams{
Expand Down Expand Up @@ -108,7 +136,10 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) {
// @Success 200
// @Router /files/{fileID} [get]
func (api *API) fileByID(rw http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var (
ctx = r.Context()
format = r.URL.Query().Get("format")
)

fileID := chi.URLParam(r, "fileID")
if fileID == "" {
Expand Down Expand Up @@ -139,7 +170,29 @@ func (api *API) fileByID(rw http.ResponseWriter, r *http.Request) {
return
}

rw.Header().Set("Content-Type", file.Mimetype)
rw.WriteHeader(http.StatusOK)
_, _ = rw.Write(file.Data)
switch format {
case codersdk.FormatZip:
if file.Mimetype != codersdk.ContentTypeTar {
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
Message: "Only .tar files can be converted to .zip format",
Detail: err.Error(),
})
return
}

rw.Header().Set("Content-Type", codersdk.ContentTypeZip)
rw.WriteHeader(http.StatusOK)
err = WriteZipArchive(rw, tar.NewReader(bytes.NewReader(file.Data)))
if err != nil {
api.Logger.Error(ctx, "invalid .zip archive", slog.F("file_id", fileID), slog.F("mimetype", file.Mimetype), slog.Error(err))
}
case "": // no format? no conversion
rw.Header().Set("Content-Type", file.Mimetype)
_, _ = rw.Write(file.Data)
Comment on lines +189 to +191
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe no harm to allow explicitly setting ContentTypeTar?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought about it, but it would be another place to adjust if we decide to natively support other formats.

default:
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
Message: "Unsupported conversion format.",
Detail: err.Error(),
})
}
}
69 changes: 68 additions & 1 deletion coderd/files_test.go
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍 nice tests

Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package coderd_test

import (
"archive/tar"
"bytes"
"context"
"net/http"
Expand All @@ -9,8 +10,10 @@ import (
"github.com/google/uuid"
"github.com/stretchr/testify/require"

"github.com/coder/coder/v2/coderd"
"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/provisioner/echo"
"github.com/coder/coder/v2/testutil"
)

Expand Down Expand Up @@ -72,19 +75,83 @@ func TestDownload(t *testing.T) {
require.Equal(t, http.StatusNotFound, apiErr.StatusCode())
})

t.Run("Insert", func(t *testing.T) {
t.Run("InsertTar_DownloadTar", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, nil)
_ = coderdtest.CreateFirstUser(t, client)

// given
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()

// when
resp, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(make([]byte, 1024)))
require.NoError(t, err)
data, contentType, err := client.Download(ctx, resp.ID)
require.NoError(t, err)

// then
require.Len(t, data, 1024)
require.Equal(t, codersdk.ContentTypeTar, contentType)
})

t.Run("InsertZip_DownloadTar", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, nil)
_ = coderdtest.CreateFirstUser(t, client)

// given
tarball, err := echo.Tar(&echo.Responses{
Parse: echo.ParseComplete,
ProvisionApply: echo.ApplyComplete,
})
require.NoError(t, err)

tarReader := tar.NewReader(bytes.NewReader(tarball))
zipContent, err := coderd.CreateZipFromTar(tarReader)
require.NoError(t, err)

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()

// when
resp, err := client.Upload(ctx, codersdk.ContentTypeZip, bytes.NewReader(zipContent))
require.NoError(t, err)
data, contentType, err := client.Download(ctx, resp.ID)
require.NoError(t, err)

// then
require.Equal(t, codersdk.ContentTypeTar, contentType)
require.Equal(t, tarball, data)
})

t.Run("InsertTar_DownloadZip", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, nil)
_ = coderdtest.CreateFirstUser(t, client)

// given
tarball, err := echo.Tar(&echo.Responses{
Parse: echo.ParseComplete,
ProvisionApply: echo.ApplyComplete,
})
require.NoError(t, err)

tarReader := tar.NewReader(bytes.NewReader(tarball))
expectedZip, err := coderd.CreateZipFromTar(tarReader)
require.NoError(t, err)

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()

// when
resp, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(tarball))
require.NoError(t, err)
data, contentType, err := client.DownloadWithFormat(ctx, resp.ID, codersdk.FormatZip)
require.NoError(t, err)

// then
require.Equal(t, codersdk.ContentTypeZip, contentType)
require.Equal(t, expectedZip, data)
})
}
101 changes: 101 additions & 0 deletions coderd/fileszip.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package coderd

import (
"archive/tar"
"archive/zip"
"bytes"
"errors"
"io"
"log"
)

func CreateTarFromZip(zipReader *zip.Reader) ([]byte, error) {
var tarBuffer bytes.Buffer
err := writeTarArchive(&tarBuffer, zipReader)
if err != nil {
return nil, err
}
return tarBuffer.Bytes(), nil
}

func writeTarArchive(w io.Writer, zipReader *zip.Reader) error {
tarWriter := tar.NewWriter(w)
defer tarWriter.Close()

for _, file := range zipReader.File {
err := processFileInZipArchive(file, tarWriter)
if err != nil {
return err
}
}
return nil
}

func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error {
fileReader, err := file.Open()
if err != nil {
return err
}
defer fileReader.Close()

err = tarWriter.WriteHeader(&tar.Header{
Name: file.Name,
Size: file.FileInfo().Size(),
Mode: 0o644,
})
if err != nil {
return err
}

n, err := io.CopyN(tarWriter, fileReader, httpFileMaxBytes)
log.Println(file.Name, n, err)
if errors.Is(err, io.EOF) {
err = nil
}
return err
}

func CreateZipFromTar(tarReader *tar.Reader) ([]byte, error) {
var zipBuffer bytes.Buffer
err := WriteZipArchive(&zipBuffer, tarReader)
if err != nil {
return nil, err
}
return zipBuffer.Bytes(), nil
}

func WriteZipArchive(w io.Writer, tarReader *tar.Reader) error {
zipWriter := zip.NewWriter(w)
defer zipWriter.Close()

for {
tarHeader, err := tarReader.Next()
if errors.Is(err, io.EOF) {
break
}

if err != nil {
return err
}

zipHeader, err := zip.FileInfoHeader(tarHeader.FileInfo())
if err != nil {
return err
}
zipHeader.Name = tarHeader.Name

zipEntry, err := zipWriter.CreateHeader(zipHeader)
if err != nil {
return err
}

_, err = io.CopyN(zipEntry, tarReader, httpFileMaxBytes)
if errors.Is(err, io.EOF) {
err = nil
}
if err != nil {
return err
}
}
return nil // don't need to flush as we call `writer.Close()`
}
10 changes: 9 additions & 1 deletion codersdk/files.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ import (

const (
ContentTypeTar = "application/x-tar"
ContentTypeZip = "application/zip"

FormatZip = "zip"
)

// UploadResponse contains the hash to reference the uploaded file.
Expand All @@ -38,7 +41,12 @@ func (c *Client) Upload(ctx context.Context, contentType string, rd io.Reader) (

// Download fetches a file by uploaded hash.
func (c *Client) Download(ctx context.Context, id uuid.UUID) ([]byte, string, error) {
res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/files/%s", id.String()), nil)
return c.DownloadWithFormat(ctx, id, "")
}

// Download fetches a file by uploaded hash, but it forces format conversion.
func (c *Client) DownloadWithFormat(ctx context.Context, id uuid.UUID, format string) ([]byte, string, error) {
res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/files/%s?format=%s", id.String(), format), nil)
if err != nil {
return nil, "", err
}
Expand Down
10 changes: 5 additions & 5 deletions docs/api/files.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading