From ac450a6099e16c00494df65a50591bc6ff35cf7d Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 7 Nov 2024 19:11:01 +0000
Subject: [PATCH 1/5] fix: avoid BLOB_UNKNOWN by not enabling
ForceBuildMetadata
Ref: #385
Ref: coder/kaniko#34
---
envbuilder.go | 4 ++--
go.mod | 2 +-
go.sum | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/envbuilder.go b/envbuilder.go
index e6f9c8d4..f6cfc844 100644
--- a/envbuilder.go
+++ b/envbuilder.go
@@ -541,7 +541,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro
NoPush: !opts.PushImage || len(destinations) == 0,
CacheRunLayers: true,
CacheCopyLayers: true,
- ForceBuildMetadata: opts.PushImage, // Force layers with no changes to be cached, required for cache probing.
+ ForceBuildMetadata: false, // Force layers with no changes to be cached, required for cache probing.
CompressedCaching: true,
Compression: config.ZStd,
// Maps to "default" level, ~100-300 MB/sec according to
@@ -1269,7 +1269,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error)
NoPush: true,
CacheRunLayers: true,
CacheCopyLayers: true,
- ForceBuildMetadata: true, // Force layers with no changes to be cached, required for cache probing.
+ ForceBuildMetadata: false, // Force layers with no changes to be cached, required for cache probing.
CompressedCaching: true,
Compression: config.ZStd,
// Maps to "default" level, ~100-300 MB/sec according to
diff --git a/go.mod b/go.mod
index e066b206..9b070ff8 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.22.4
// There are a few options we need added to Kaniko!
// See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main
-replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241028054616-350cbb820e05
+replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241107190527-8aeb946dfc4d
// Required to import codersdk due to gvisor dependency.
replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374
diff --git a/go.sum b/go.sum
index 5872942d..19317cba 100644
--- a/go.sum
+++ b/go.sum
@@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0=
github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo=
-github.com/coder/kaniko v0.0.0-20241028054616-350cbb820e05 h1:KZc6vG/WnSWG8RtUevGrCdZbF7XJaaZ32ocig6sZLQk=
-github.com/coder/kaniko v0.0.0-20241028054616-350cbb820e05/go.mod h1:3rM/KOQ4LgF8mE+O1P6pLDa/E57mzxIxNdUOMKi1qpg=
+github.com/coder/kaniko v0.0.0-20241107190527-8aeb946dfc4d h1:qqoyQMtIHWYRw8RbDPT+pdOQER9UKm36tsJtiy9MVc8=
+github.com/coder/kaniko v0.0.0-20241107190527-8aeb946dfc4d/go.mod h1:3rM/KOQ4LgF8mE+O1P6pLDa/E57mzxIxNdUOMKi1qpg=
github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs=
github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc=
github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ=
From 9df56fca08f5e8031bdddf92a101bb7627a9e4af Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 7 Nov 2024 19:12:40 +0000
Subject: [PATCH 2/5] dbg
---
envbuilder.go | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/envbuilder.go b/envbuilder.go
index f6cfc844..dda35d9c 100644
--- a/envbuilder.go
+++ b/envbuilder.go
@@ -9,6 +9,7 @@ import (
"fmt"
"io"
"io/fs"
+ stdlog "log"
"maps"
"net"
"net/http"
@@ -43,6 +44,7 @@ import (
dockerconfig "github.com/docker/cli/cli/config"
"github.com/docker/cli/cli/config/configfile"
"github.com/fatih/color"
+ "github.com/google/go-containerregistry/pkg/logs"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/kballard/go-shellquote"
@@ -583,6 +585,19 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro
endStage("🏗️ Built image!")
if opts.PushImage {
endStage = startStage("🏗️ Pushing image...")
+ kOpts.PushRetry = 3
+ // kOpts.PushIgnoreImmutableTagErrors = true
+ logs.Debug = stdlog.New(os.Stderr, "", 0)
+ logs.Warn = stdlog.New(os.Stderr, "", 0)
+ logs.Progress = stdlog.New(os.Stderr, "", 0)
+ layers, _ := image.Layers()
+ for _, layer := range layers {
+ mediaType, _ := layer.MediaType()
+ diffID, _ := layer.DiffID()
+ digest, _ := layer.Digest()
+ size, _ := layer.Size()
+ opts.Logger(log.LevelDebug, "Layer: %s %s %s %d", mediaType, diffID, digest, size)
+ }
if err := executor.DoPush(image, kOpts); err == nil {
endStage("🏗️ Pushed image!")
} else if !opts.ExitOnPushFailure {
From 4b71b2416e07ad6e61e877193236f729f4cc55ce Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Wed, 20 Nov 2024 13:10:05 +0000
Subject: [PATCH 3/5] update fork and add test
---
Makefile | 6 ++-
envbuilder.go | 29 ++++++--------
go.mod | 2 +-
go.sum | 4 +-
integration/integration_test.go | 41 ++++++++++++++++++--
integration/testdata/blob-unknown/Dockerfile | 3 ++
6 files changed, 61 insertions(+), 24 deletions(-)
create mode 100644 integration/testdata/blob-unknown/Dockerfile
diff --git a/Makefile b/Makefile
index ce079015..14ed5182 100644
--- a/Makefile
+++ b/Makefile
@@ -67,7 +67,7 @@ test-registry-container: .registry-cache
# Pulls images referenced in integration tests and pushes them to the local cache.
.PHONY: test-images-push
-test-images-push: .registry-cache/docker/registry/v2/repositories/envbuilder-test-alpine .registry-cache/docker/registry/v2/repositories/envbuilder-test-ubuntu .registry-cache/docker/registry/v2/repositories/envbuilder-test-codercom-code-server
+test-images-push: .registry-cache/docker/registry/v2/repositories/envbuilder-test-alpine .registry-cache/docker/registry/v2/repositories/envbuilder-test-ubuntu .registry-cache/docker/registry/v2/repositories/envbuilder-test-codercom-code-server .registry-cache/docker/registry/v2/repositories/envbuilder-test-blob-unknown
.PHONY: test-images-pull
test-images-pull:
@@ -77,6 +77,7 @@ test-images-pull:
docker tag ubuntu:latest localhost:5000/envbuilder-test-ubuntu:latest
docker pull codercom/code-server:latest
docker tag codercom/code-server:latest localhost:5000/envbuilder-test-codercom-code-server:latest
+ docker build -t localhost:5000/envbuilder-test-blob-unknown:latest -f integration/testdata/blob-unknown/Dockerfile integration/testdata/blob-unknown
.registry-cache:
mkdir -p .registry-cache && chmod -R ag+w .registry-cache
@@ -89,3 +90,6 @@ test-images-pull:
.registry-cache/docker/registry/v2/repositories/envbuilder-test-codercom-code-server:
docker push localhost:5000/envbuilder-test-codercom-code-server:latest
+
+.registry-cache/docker/registry/v2/repositories/envbuilder-test-blob-unknown:
+ docker push localhost:5000/envbuilder-test-blob-unknown:latest
diff --git a/envbuilder.go b/envbuilder.go
index dda35d9c..2d7efec3 100644
--- a/envbuilder.go
+++ b/envbuilder.go
@@ -9,7 +9,6 @@ import (
"fmt"
"io"
"io/fs"
- stdlog "log"
"maps"
"net"
"net/http"
@@ -44,7 +43,6 @@ import (
dockerconfig "github.com/docker/cli/cli/config"
"github.com/docker/cli/cli/config/configfile"
"github.com/fatih/color"
- "github.com/google/go-containerregistry/pkg/logs"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/kballard/go-shellquote"
@@ -543,7 +541,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro
NoPush: !opts.PushImage || len(destinations) == 0,
CacheRunLayers: true,
CacheCopyLayers: true,
- ForceBuildMetadata: false, // Force layers with no changes to be cached, required for cache probing.
+ ForceBuildMetadata: opts.PushImage, // Force layers with no changes to be cached, required for cache probing.
CompressedCaching: true,
Compression: config.ZStd,
// Maps to "default" level, ~100-300 MB/sec according to
@@ -585,19 +583,16 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro
endStage("🏗️ Built image!")
if opts.PushImage {
endStage = startStage("🏗️ Pushing image...")
- kOpts.PushRetry = 3
- // kOpts.PushIgnoreImmutableTagErrors = true
- logs.Debug = stdlog.New(os.Stderr, "", 0)
- logs.Warn = stdlog.New(os.Stderr, "", 0)
- logs.Progress = stdlog.New(os.Stderr, "", 0)
- layers, _ := image.Layers()
- for _, layer := range layers {
- mediaType, _ := layer.MediaType()
- diffID, _ := layer.DiffID()
- digest, _ := layer.Digest()
- size, _ := layer.Size()
- opts.Logger(log.LevelDebug, "Layer: %s %s %s %d", mediaType, diffID, digest, size)
- }
+
+ // To debug registry issues, enable logging:
+ //
+ // import (
+ // stdlog "log"
+ // reglogs "github.com/google/go-containerregistry/pkg/logs"
+ // )
+ // reglogs.Debug = stdlog.New(os.Stderr, "", 0)
+ // reglogs.Warn = stdlog.New(os.Stderr, "", 0)
+ // reglogs.Progress = stdlog.New(os.Stderr, "", 0)
if err := executor.DoPush(image, kOpts); err == nil {
endStage("🏗️ Pushed image!")
} else if !opts.ExitOnPushFailure {
@@ -1284,7 +1279,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error)
NoPush: true,
CacheRunLayers: true,
CacheCopyLayers: true,
- ForceBuildMetadata: false, // Force layers with no changes to be cached, required for cache probing.
+ ForceBuildMetadata: true, // Force layers with no changes to be cached, required for cache probing.
CompressedCaching: true,
Compression: config.ZStd,
// Maps to "default" level, ~100-300 MB/sec according to
diff --git a/go.mod b/go.mod
index 9b070ff8..a9427751 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.22.4
// There are a few options we need added to Kaniko!
// See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main
-replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241107190527-8aeb946dfc4d
+replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241120115946-327ae3f1c802
// Required to import codersdk due to gvisor dependency.
replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374
diff --git a/go.sum b/go.sum
index 19317cba..c0ab432f 100644
--- a/go.sum
+++ b/go.sum
@@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0=
github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo=
-github.com/coder/kaniko v0.0.0-20241107190527-8aeb946dfc4d h1:qqoyQMtIHWYRw8RbDPT+pdOQER9UKm36tsJtiy9MVc8=
-github.com/coder/kaniko v0.0.0-20241107190527-8aeb946dfc4d/go.mod h1:3rM/KOQ4LgF8mE+O1P6pLDa/E57mzxIxNdUOMKi1qpg=
+github.com/coder/kaniko v0.0.0-20241120115946-327ae3f1c802 h1:6vcTuyI9pLlmEkSOw0cT+T74DuqYWMcBJFGj+32C7qk=
+github.com/coder/kaniko v0.0.0-20241120115946-327ae3f1c802/go.mod h1:3rM/KOQ4LgF8mE+O1P6pLDa/E57mzxIxNdUOMKi1qpg=
github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs=
github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc=
github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ=
diff --git a/integration/integration_test.go b/integration/integration_test.go
index aaa25d16..b7e75438 100644
--- a/integration/integration_test.go
+++ b/integration/integration_test.go
@@ -57,9 +57,10 @@ import (
)
const (
- testContainerLabel = "envbox-integration-test"
- testImageAlpine = "localhost:5000/envbuilder-test-alpine:latest"
- testImageUbuntu = "localhost:5000/envbuilder-test-ubuntu:latest"
+ testContainerLabel = "envbox-integration-test"
+ testImageAlpine = "localhost:5000/envbuilder-test-alpine:latest"
+ testImageUbuntu = "localhost:5000/envbuilder-test-ubuntu:latest"
+ testImageBlobUnknown = "localhost:5000/envbuilder-test-blob-unknown:latest"
// nolint:gosec // Throw-away key for testing. DO NOT REUSE.
testSSHKey = `-----BEGIN OPENSSH PRIVATE KEY-----
@@ -2354,6 +2355,38 @@ USER devalot
}
require.Fail(t, "expected pid 1 to be running as devalot")
})
+
+ t.Run("PushDuplicateLayersNoBlobUnknown", func(t *testing.T) {
+ t.Parallel()
+
+ srv := gittest.CreateGitServer(t, gittest.Options{
+ Files: map[string]string{
+ ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s
+USER root
+RUN echo "hi i r empty"
+RUN echo "who u"
+`, testImageBlobUnknown),
+ ".devcontainer/devcontainer.json": `{
+ "name": "Test",
+ "build": {
+ "dockerfile": "Dockerfile"
+ },
+ }`,
+ },
+ })
+
+ // NOTE(mafredri): The in-memory registry doesn't catch this error so we
+ // have to use registry:2.
+ ref, err := name.ParseReference(fmt.Sprintf("localhost:5000/test-blob-unknown-%s", uuid.NewString()))
+ require.NoError(t, err)
+ opts := []string{
+ envbuilderEnv("GIT_URL", srv.URL),
+ envbuilderEnv("CACHE_REPO", ref.String()),
+ envbuilderEnv("VERBOSE", "1"),
+ }
+
+ _ = pushImage(t, ref, nil, opts...)
+ })
}
func TestChownHomedir(t *testing.T) {
@@ -2532,6 +2565,8 @@ func getCachedImage(ctx context.Context, t *testing.T, cli *client.Client, env .
}
func startContainerFromRef(ctx context.Context, t *testing.T, cli *client.Client, ref name.Reference) container.CreateResponse {
+ t.Helper()
+
// Ensure that we can pull the image.
rc, err := cli.ImagePull(ctx, ref.String(), image.PullOptions{})
require.NoError(t, err)
diff --git a/integration/testdata/blob-unknown/Dockerfile b/integration/testdata/blob-unknown/Dockerfile
new file mode 100644
index 00000000..614d04a7
--- /dev/null
+++ b/integration/testdata/blob-unknown/Dockerfile
@@ -0,0 +1,3 @@
+FROM alpine:latest
+
+WORKDIR /home
From 9347b30c556de8bf8b956b2e8846edb2429b22c8 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Wed, 20 Nov 2024 13:14:05 +0000
Subject: [PATCH 4/5] Commentary
---
integration/testdata/blob-unknown/Dockerfile | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/integration/testdata/blob-unknown/Dockerfile b/integration/testdata/blob-unknown/Dockerfile
index 614d04a7..fffcc574 100644
--- a/integration/testdata/blob-unknown/Dockerfile
+++ b/integration/testdata/blob-unknown/Dockerfile
@@ -1,3 +1,7 @@
FROM alpine:latest
+# This will produce an empty layer via Docker. It will allow us to test for a
+# conflicting empty layer produced by Kaniko. This is to check against the
+# BLOB_UNKNOWN error when trying to upload the built image to a registry and
+# Kaniko having overwritten this blob with its own.
WORKDIR /home
From d72eaff987d73cf7e001f790e6e423eb1cbeb506 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Wed, 20 Nov 2024 13:22:36 +0000
Subject: [PATCH 5/5] fork you very much
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index a9427751..00987aa7 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.22.4
// There are a few options we need added to Kaniko!
// See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main
-replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241120115946-327ae3f1c802
+replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241120132148-131d6094d781
// Required to import codersdk due to gvisor dependency.
replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374
diff --git a/go.sum b/go.sum
index c0ab432f..1bb43776 100644
--- a/go.sum
+++ b/go.sum
@@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0=
github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo=
-github.com/coder/kaniko v0.0.0-20241120115946-327ae3f1c802 h1:6vcTuyI9pLlmEkSOw0cT+T74DuqYWMcBJFGj+32C7qk=
-github.com/coder/kaniko v0.0.0-20241120115946-327ae3f1c802/go.mod h1:3rM/KOQ4LgF8mE+O1P6pLDa/E57mzxIxNdUOMKi1qpg=
+github.com/coder/kaniko v0.0.0-20241120132148-131d6094d781 h1:/4SMdrjLQL1BseLSnMd9nYQSI+E63CXcyFGC7ZHHj8I=
+github.com/coder/kaniko v0.0.0-20241120132148-131d6094d781/go.mod h1:3rM/KOQ4LgF8mE+O1P6pLDa/E57mzxIxNdUOMKi1qpg=
github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs=
github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc=
github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ=