From d8d2a399b6395edffc37acd2ed09f7b188d44aff Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Sun, 12 Jun 2022 15:11:48 -0500 Subject: [PATCH 1/4] feat: gzip static http server assets Agent binary downloads go from 30MB to 13MB! --- coderd/coderd.go | 4 +++- go.mod | 1 + go.sum | 1 + provisionersdk/agent.go | 4 ++-- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index 2cd94d1eac646..3bf7f5e5a930f 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/NYTimes/gziphandler" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/pion/webrtc/v3" @@ -350,7 +351,8 @@ func New(options *Options) *API { r.Get("/state", api.workspaceBuildState) }) }) - r.NotFound(api.siteHandler.ServeHTTP) + + r.NotFound(gziphandler.GzipHandler(http.HandlerFunc(api.siteHandler.ServeHTTP)).ServeHTTP) return api } diff --git a/go.mod b/go.mod index 38d06b60a52f2..d18e8684bcf40 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( cdr.dev/slog v1.4.2-0.20220525200111-18dce5c2cd5f cloud.google.com/go/compute v1.6.1 github.com/AlecAivazis/survey/v2 v2.3.4 + github.com/NYTimes/gziphandler v1.1.1 github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 github.com/awalterschulze/gographviz v2.0.3+incompatible github.com/bgentry/speakeasy v0.1.0 diff --git a/go.sum b/go.sum index e7077ce680395..70199209fc861 100644 --- a/go.sum +++ b/go.sum @@ -128,6 +128,7 @@ github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfy github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= diff --git a/provisionersdk/agent.go b/provisionersdk/agent.go index 59d06556e8799..fb60629b59e03 100644 --- a/provisionersdk/agent.go +++ b/provisionersdk/agent.go @@ -21,7 +21,7 @@ set -eux pipefail BINARY_LOCATION=$(mktemp -d -t tmp.coderXXXXXX)/coder BINARY_URL=${ACCESS_URL}bin/coder-linux-${ARCH} if which curl >/dev/null 2>&1; then - curl -fsSL "${BINARY_URL}" -o "${BINARY_LOCATION}" + curl -fsSL --compressed "${BINARY_URL}" -o "${BINARY_LOCATION}" elif which wget >/dev/null 2>&1; then wget -q "${BINARY_URL}" -O "${BINARY_LOCATION}" elif which busybox >/dev/null 2>&1; then @@ -38,7 +38,7 @@ exec $BINARY_LOCATION agent` darwinScript = `#!/usr/bin/env sh set -eux pipefail BINARY_LOCATION=$(mktemp -d -t tmp.coderXXXXXX)/coder -curl -fsSL "${ACCESS_URL}bin/coder-darwin-${ARCH}" -o "${BINARY_LOCATION}" +curl -fsSL --compressed "${ACCESS_URL}bin/coder-darwin-${ARCH}" -o "${BINARY_LOCATION}" chmod +x $BINARY_LOCATION export CODER_AGENT_AUTH="${AUTH_TYPE}" export CODER_AGENT_URL="${ACCESS_URL}" From 1bf95d2b7b948e729c30fa0b0b8ff77760a1f166 Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Mon, 13 Jun 2022 10:45:49 -0500 Subject: [PATCH 2/4] use chi/middleware --- coderd/coderd.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index 3bf7f5e5a930f..e83a99663954d 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -9,7 +9,6 @@ import ( "sync" "time" - "github.com/NYTimes/gziphandler" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/pion/webrtc/v3" @@ -352,7 +351,7 @@ func New(options *Options) *API { }) }) - r.NotFound(gziphandler.GzipHandler(http.HandlerFunc(api.siteHandler.ServeHTTP)).ServeHTTP) + r.NotFound(middleware.Compress(5, "gzip", "deflate")(http.HandlerFunc(api.siteHandler.ServeHTTP)).ServeHTTP) return api } From dc75df772f796f85175171e42c279944b60e42c7 Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Mon, 13 Jun 2022 12:42:27 -0500 Subject: [PATCH 3/4] fixup! use chi/middleware --- coderd/coderd.go | 16 +++++++++++++--- go.mod | 4 ++-- go.sum | 7 ++++--- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index e83a99663954d..ad4d0a99221f2 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -4,19 +4,20 @@ import ( "context" "crypto/x509" "fmt" + "io" "net/http" "net/url" "sync" "time" + "github.com/andybalholm/brotli" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/pion/webrtc/v3" + sdktrace "go.opentelemetry.io/otel/sdk/trace" "golang.org/x/xerrors" "google.golang.org/api/idtoken" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - "cdr.dev/slog" "github.com/coder/coder/buildinfo" "github.com/coder/coder/coderd/awsidentity" @@ -351,7 +352,16 @@ func New(options *Options) *API { }) }) - r.NotFound(middleware.Compress(5, "gzip", "deflate")(http.HandlerFunc(api.siteHandler.ServeHTTP)).ServeHTTP) + cmp := middleware.NewCompressor(5, + "text/*", + "application/*", + "image/*", + ) + cmp.SetEncoder("br", func(w io.Writer, level int) io.Writer { + return brotli.NewWriterLevel(w, level) + }) + + r.NotFound(cmp.Handler(http.HandlerFunc(api.siteHandler.ServeHTTP)).ServeHTTP) return api } diff --git a/go.mod b/go.mod index d18e8684bcf40..b5a0f39fb6b97 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( cdr.dev/slog v1.4.2-0.20220525200111-18dce5c2cd5f cloud.google.com/go/compute v1.6.1 github.com/AlecAivazis/survey/v2 v2.3.4 - github.com/NYTimes/gziphandler v1.1.1 + github.com/andybalholm/brotli v1.0.4 github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 github.com/awalterschulze/gographviz v2.0.3+incompatible github.com/bgentry/speakeasy v0.1.0 @@ -181,7 +181,7 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.15.0 // indirect + github.com/klauspost/compress v1.15.6 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect diff --git a/go.sum b/go.sum index 70199209fc861..436fb3395242c 100644 --- a/go.sum +++ b/go.sum @@ -128,7 +128,6 @@ github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfy github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= @@ -166,6 +165,8 @@ github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:C github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/crlf v0.0.0-20171020200849-670099aa064f/go.mod h1:k8feO4+kXDxro6ErPXBRTJ/ro2mf0SsFG8s7doP9kJE= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= @@ -1051,8 +1052,8 @@ github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U= -github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY= +github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= From 24ffc35dc788e433738533fcc5f5d15df1906e94 Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Mon, 13 Jun 2022 12:51:20 -0500 Subject: [PATCH 4/4] fixup! use chi/middleware --- coderd/coderd.go | 32 ++++++++++++++++++++++---------- go.mod | 2 +- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index ad4d0a99221f2..b4d7db305ea1d 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -13,6 +13,7 @@ import ( "github.com/andybalholm/brotli" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" + "github.com/klauspost/compress/zstd" "github.com/pion/webrtc/v3" sdktrace "go.opentelemetry.io/otel/sdk/trace" "golang.org/x/xerrors" @@ -352,16 +353,7 @@ func New(options *Options) *API { }) }) - cmp := middleware.NewCompressor(5, - "text/*", - "application/*", - "image/*", - ) - cmp.SetEncoder("br", func(w io.Writer, level int) io.Writer { - return brotli.NewWriterLevel(w, level) - }) - - r.NotFound(cmp.Handler(http.HandlerFunc(api.siteHandler.ServeHTTP)).ServeHTTP) + r.NotFound(compressHandler(http.HandlerFunc(api.siteHandler.ServeHTTP)).ServeHTTP) return api } @@ -392,3 +384,23 @@ func debugLogRequest(log slog.Logger) func(http.Handler) http.Handler { }) } } + +func compressHandler(h http.Handler) http.Handler { + cmp := middleware.NewCompressor(5, + "text/*", + "application/*", + "image/*", + ) + cmp.SetEncoder("br", func(w io.Writer, level int) io.Writer { + return brotli.NewWriterLevel(w, level) + }) + cmp.SetEncoder("zstd", func(w io.Writer, level int) io.Writer { + zw, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(level))) + if err != nil { + panic("invalid zstd compressor: " + err.Error()) + } + return zw + }) + + return cmp.Handler(h) +} diff --git a/go.mod b/go.mod index b5a0f39fb6b97..4d335e96de29c 100644 --- a/go.mod +++ b/go.mod @@ -181,7 +181,7 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.15.6 // indirect + github.com/klauspost/compress v1.15.6 github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect