From cdb2e1bf254d1c89c903a0f2fd62fc34c87d3538 Mon Sep 17 00:00:00 2001 From: John Lago <750845+Lagoja@users.noreply.github.com> Date: Thu, 3 Oct 2024 11:41:53 -0700 Subject: [PATCH 1/3] Flake version to 0.13.3 --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 7eac76384f1..155106ff783 100644 --- a/flake.nix +++ b/flake.nix @@ -11,7 +11,7 @@ let pkgs = nixpkgs.legacyPackages.${system}; - lastTag = "0.13.2"; + lastTag = "0.13.3"; revision = if (self ? shortRev) then "${self.shortRev}" From 3df27b91f47baefb523eb443db81c7bf8ba620cc Mon Sep 17 00:00:00 2001 From: John Lago <750845+Lagoja@users.noreply.github.com> Date: Tue, 29 Oct 2024 10:40:30 -0700 Subject: [PATCH 2/3] Merge 0.13.6 into latest --- .github/workflows/debug.yaml | 2 +- .../cli_reference/devbox_completion_zsh.md | 11 +- docs/app/docs/configuration.md | 10 +- docs/app/docs/guides/creating_plugins.md | 2 +- docs/app/docs/guides/secrets.md | 4 +- docs/app/docs/quickstart.mdx | 2 +- docs/app/docusaurus.config.js | 2 +- flake.lock | 6 +- flake.nix | 2 +- go.mod | 2 +- go.sum | 4 +- internal/build/build.go | 33 +++ internal/devbox/devbox.go | 7 +- internal/devconfig/configfile/env.go | 13 +- internal/shellgen/flake_plan.go | 86 ++++++-- internal/shellgen/generate.go | 38 ++-- internal/shellgen/generate_test.go | 6 +- internal/shellgen/tmpl/glibc-patch.nix.tmpl | 27 +-- internal/telemetry/segment.go | 19 +- pkg/autodetect/autodetect.go | 11 +- pkg/autodetect/detector/go.go | 51 +++++ pkg/autodetect/detector/go_test.go | 103 ++++++++++ pkg/autodetect/detector/php.go | 111 ++++++++++ pkg/autodetect/detector/php_test.go | 193 ++++++++++++++++++ vendor-hash | 2 +- 25 files changed, 651 insertions(+), 96 deletions(-) create mode 100644 pkg/autodetect/detector/go.go create mode 100644 pkg/autodetect/detector/go_test.go create mode 100644 pkg/autodetect/detector/php.go create mode 100644 pkg/autodetect/detector/php_test.go diff --git a/.github/workflows/debug.yaml b/.github/workflows/debug.yaml index 2328cd6032f..4b777067d1e 100644 --- a/.github/workflows/debug.yaml +++ b/.github/workflows/debug.yaml @@ -9,7 +9,7 @@ on: default: "ubuntu-latest" type: choice options: - - macos-12 + - macos-latest - ubuntu-latest permissions: diff --git a/docs/app/docs/cli_reference/devbox_completion_zsh.md b/docs/app/docs/cli_reference/devbox_completion_zsh.md index 5c3fa98402f..ac37ffb9118 100644 --- a/docs/app/docs/cli_reference/devbox_completion_zsh.md +++ b/docs/app/docs/cli_reference/devbox_completion_zsh.md @@ -2,8 +2,15 @@ Generate the autocompletion script for the zsh shell. -If shell completion is not already enabled in your environment you will need -to enable it. You can execute the following once: +If you are using Oh My Zsh, just run the following: + +```bash +mkdir -p ~/.oh-my-zsh/completions +devbox completion zsh > ~/.oh-my-zsh/completions/_devbox +``` + +If you are not using Oh My Zsh and shell completion is not already enabled in your environment you will need +to enable it. You can execute the following once: ```bash echo "autoload -U compinit; compinit" >> ~/.zshrc diff --git a/docs/app/docs/configuration.md b/docs/app/docs/configuration.md index dcad736775a..ce5f9a64efd 100644 --- a/docs/app/docs/configuration.md +++ b/docs/app/docs/configuration.md @@ -182,11 +182,11 @@ Currently, you can only set values using string literals, `$PWD`, and `$PATH`. A ### Env From -Env from takes a string or list of strings for loading environment variables into your shells and scripts. Currently it supports loading from two sources: .env files, and Jetify Secrsts. +Env from takes a string for loading environment variables into your shells and scripts. Currently it supports loading from two sources: .env files, and Jetify Secrets. #### .env Files -You can load environment variables from a `.env` file by adding the path to the file in the `env_from` field. This is useful for loading secrets or other sensitive information that you don't want to store in your `devbox.json`. +You can load environment variables from a `.env` file by adding the path to the file in the `env_from` field (the file must end with `.env`). This is useful for loading secrets or other sensitive information that you don't want to store in your `devbox.json`. ```json { @@ -198,15 +198,15 @@ This will load the environment variables from the `.env` file into your shell wh #### Jetify Secrets -You can securely load secrets from Jetify Secrets by running `devbox secrets init` and creating a project in Jetify Cloud. This will add the `jetpack-cloud` field to `env_from` in your project. +You can securely load secrets from Jetify Secrets by running `devbox secrets init` and creating a project in Jetify Cloud. This will add the `jetify-cloud` field to `env_from` in your project. ```json { - "env_from": "jetpack-cloud" + "env_from": "jetify-cloud" } ``` -Note that setting secrets securetly with Jetify Secrets requires a Jetify Cloud account. For more information, see the [Jetify Secrets](/docs/cloud/secrets/) guide. +Note that setting secrets securely with Jetify Secrets requires a Jetify Cloud account. For more information, see the [Jetify Secrets](/docs/cloud/secrets/) guide. ### Shell diff --git a/docs/app/docs/guides/creating_plugins.md b/docs/app/docs/guides/creating_plugins.md index d5000853fea..f3b047f1b3a 100644 --- a/docs/app/docs/guides/creating_plugins.md +++ b/docs/app/docs/guides/creating_plugins.md @@ -219,7 +219,7 @@ The plugin.json below installs MongoDB + the Mongo shell, and sets the environme "version": "0.0.1", "description": "Plugin for the [`mongodb`](https://www.nixhub.io/packages/mongodb) package. This plugin configures MonogoDB to use a local config file and data directory for this project, and configures a mongodb service.", "packages": [ - "mongodb@latest" + "mongodb@latest", "mongosh@latest" ], "env": { diff --git a/docs/app/docs/guides/secrets.md b/docs/app/docs/guides/secrets.md index 10035232a0f..fb670bf9144 100644 --- a/docs/app/docs/guides/secrets.md +++ b/docs/app/docs/guides/secrets.md @@ -28,9 +28,7 @@ For environment variables that you want to keep out of your `devbox.json` file, { "packages": {}, "shell": {}, - "env_from": [ - "path/to/.env" - ] + "env_from": "path/to/.env" } ``` diff --git a/docs/app/docs/quickstart.mdx b/docs/app/docs/quickstart.mdx index 2f043ec227f..9d4a987dec4 100644 --- a/docs/app/docs/quickstart.mdx +++ b/docs/app/docs/quickstart.mdx @@ -21,7 +21,7 @@ Follow the instruction from [the installation guide](./installing_devbox.mdx). :::note If you want to try Devbox before installing it, you can open a cloud shell on your browser using the link below -[![Open In Devspace](https://www.jetify.com/img/devbox/open-in-devbox.svg)](https://www.jetify.com/devbox/templates/tutorial) +[![Open In Devspace](https://www.jetify.com/img/devbox/open-in-devspace.svg)](https://www.jetify.com/devbox/templates/tutorial) ::: ## Create a Development Environment diff --git a/docs/app/docusaurus.config.js b/docs/app/docusaurus.config.js index fcd4c8fb7f9..21cb50b8044 100644 --- a/docs/app/docusaurus.config.js +++ b/docs/app/docusaurus.config.js @@ -66,7 +66,7 @@ const config = { routeBasePath: "cloud", sidebarPath: require.resolve("./cloud_sidebars.js"), }, - ],[ + ], [ "@docusaurus/plugin-content-docs", { id: "nixhub", diff --git a/flake.lock b/flake.lock index f3cfe0f3103..4029d991d3d 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1728888510, - "narHash": "sha256-nsNdSldaAyu6PE3YUA+YQLqUDJh+gRbBooMMekZJwvI=", + "lastModified": 1729880355, + "narHash": "sha256-RP+OQ6koQQLX5nw0NmcDrzvGL8HDLnyXt/jHhL1jwjM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a3c0b3b21515f74fd2665903d4ce6bc4dc81c77c", + "rev": "18536bf04cd71abd345f9579158841376fdd0c5a", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index d4dc96e56b4..00d3c871cf6 100644 --- a/flake.nix +++ b/flake.nix @@ -11,7 +11,7 @@ let pkgs = nixpkgs.legacyPackages.${system}; - lastTag = "0.13.5"; + lastTag = "0.13.6"; revision = if (self ? shortRev) diff --git a/go.mod b/go.mod index 06bd9f1b5f2..223acf9a5aa 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/zealic/go2node v0.1.0 go.jetify.com/typeid v1.2.0 go.jetpack.io/envsec v0.0.16-0.20240604163020-540ad12af899 - go.jetpack.io/pkg v0.0.0-20240815004735-7649b4283d51 + go.jetpack.io/pkg v0.0.0-20241025195518-152e59e26d5d golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 golang.org/x/mod v0.18.0 golang.org/x/oauth2 v0.21.0 diff --git a/go.sum b/go.sum index 93c0ddb1ce3..ee36653737c 100644 --- a/go.sum +++ b/go.sum @@ -381,8 +381,8 @@ go.jetify.com/typeid v1.2.0 h1:Nd1MZZoWe9q4kkh82xZHQbqCzxJX/ZxgK8RjQWxygwk= go.jetify.com/typeid v1.2.0/go.mod h1:CtVGyt2+TSp4Rq5+ARLvGsJqdNypKBAC6INQ9TLPlmk= go.jetpack.io/envsec v0.0.16-0.20240604163020-540ad12af899 h1:TfmHWWhwKu1jGmSLp8Iy0fsNyvqP5cAf7w/vD80ub00= go.jetpack.io/envsec v0.0.16-0.20240604163020-540ad12af899/go.mod h1:LOdrWtfvoV9dPSVHWN0onLSqeYAOKrS7k1AzwpZg0X0= -go.jetpack.io/pkg v0.0.0-20240815004735-7649b4283d51 h1:udl601M1zuLyjqsW4EAJRQ0ruTYq8DXbkcFg27Ydk4I= -go.jetpack.io/pkg v0.0.0-20240815004735-7649b4283d51/go.mod h1:m65l3dl6aFQdVdSvAvih73ZEfs9ndbFGnw03RVC2HFU= +go.jetpack.io/pkg v0.0.0-20241025195518-152e59e26d5d h1:z/GGViwoVX8rrE8LySyL2HyKiqyopA+4RwRHEVFn/QQ= +go.jetpack.io/pkg v0.0.0-20241025195518-152e59e26d5d/go.mod h1:m65l3dl6aFQdVdSvAvih73ZEfs9ndbFGnw03RVC2HFU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/internal/build/build.go b/internal/build/build.go index 9d102c8ca81..dee0ac00db3 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -4,7 +4,10 @@ package build import ( + "fmt" + "log/slog" "os" + "path/filepath" "runtime" "strconv" "sync" @@ -98,3 +101,33 @@ func DashboardHostname() string { } return "https://cloud.jetify.com" } + +// SourceDir searches for the source code directory that built the current +// binary. +func SourceDir() (string, error) { + _, file, _, ok := runtime.Caller(0) + if !ok || file == "" { + return "", fmt.Errorf("build.SourceDir: binary is missing path info") + } + slog.Debug("trying to determine path to devbox source using runtime.Caller", "path", file) + + dir := filepath.Dir(file) + if _, err := os.Stat(dir); err != nil { + if filepath.IsAbs(file) { + return "", fmt.Errorf("build.SourceDir: path to binary source doesn't exist: %v", err) + } + return "", fmt.Errorf("build.SourceDir: binary was built with -trimpath") + } + + for { + _, err := os.Stat(filepath.Join(dir, "go.mod")) + if err == nil { + slog.Debug("found devbox source directory", "path", dir) + return dir, nil + } + if dir == "/" || dir == "." { + return "", fmt.Errorf("build.SourceDir: can't find go.mod in any parent directories of %s", file) + } + dir = filepath.Dir(dir) + } +} diff --git a/internal/devbox/devbox.go b/internal/devbox/devbox.go index d36d74c5036..dc571b431c6 100644 --- a/internal/devbox/devbox.go +++ b/internal/devbox/devbox.go @@ -32,6 +32,7 @@ import ( "go.jetpack.io/devbox/internal/devbox/envpath" "go.jetpack.io/devbox/internal/devbox/generate" "go.jetpack.io/devbox/internal/devconfig" + "go.jetpack.io/devbox/internal/devconfig/configfile" "go.jetpack.io/devbox/internal/devpkg" "go.jetpack.io/devbox/internal/devpkg/pkgtype" "go.jetpack.io/devbox/internal/envir" @@ -373,7 +374,7 @@ func (d *Devbox) EnvExports(ctx context.Context, opts devopt.EnvExportsOpts) (st envStr := exportify(envs) if opts.RunHooks { - hooksStr := ". " + shellgen.ScriptPath(d.ProjectDir(), shellgen.HooksFilename) + hooksStr := ". \"" + shellgen.ScriptPath(d.ProjectDir(), shellgen.HooksFilename) + "\"" envStr = fmt.Sprintf("%s\n%s;\n", envStr, hooksStr) } @@ -1009,9 +1010,9 @@ func (d *Devbox) configEnvs( } } else if d.cfg.Root.EnvFrom != "" { return nil, usererr.New( - "unknown from_env value: %s. Supported value is: %q.", + "unknown env_from value: %s. Supported values are: \"%q\" or a path to a file ending in \".env\"", d.cfg.Root.EnvFrom, - "jetpack-cloud", + configfile.JetifyCloudEnvFromValue, ) } for k, v := range d.cfg.Env() { diff --git a/internal/devconfig/configfile/env.go b/internal/devconfig/configfile/env.go index f6a1a6ff529..83161aed658 100644 --- a/internal/devconfig/configfile/env.go +++ b/internal/devconfig/configfile/env.go @@ -8,9 +8,11 @@ import ( "github.com/hashicorp/go-envparse" ) +var JetifyCloudEnvFromValue = "jetify-cloud" + func (c *ConfigFile) IsEnvsecEnabled() bool { // envsec for legacy. jetpack-cloud for legacy - return c.EnvFrom == "envsec" || c.EnvFrom == "jetpack-cloud" || c.EnvFrom == "jetify-cloud" + return c.EnvFrom == "envsec" || c.EnvFrom == "jetpack-cloud" || c.EnvFrom == JetifyCloudEnvFromValue } func (c *ConfigFile) IsdotEnvEnabled() bool { @@ -25,10 +27,13 @@ func (c *ConfigFile) ParseEnvsFromDotEnv() (map[string]string, error) { if !c.IsdotEnvEnabled() { return nil, fmt.Errorf("env file does not have a .env extension") } - - file, err := os.Open(c.EnvFrom) + envFileAbsPath := c.EnvFrom + if !filepath.IsAbs(c.EnvFrom) { + envFileAbsPath = filepath.Join(filepath.Dir(c.AbsRootPath), c.EnvFrom) + } + file, err := os.Open(envFileAbsPath) if err != nil { - return nil, fmt.Errorf("failed to open file: %s", c.EnvFrom) + return nil, fmt.Errorf("failed to open file: %s", envFileAbsPath) } defer file.Close() diff --git a/internal/shellgen/flake_plan.go b/internal/shellgen/flake_plan.go index 82072e18d10..d38249d905e 100644 --- a/internal/shellgen/flake_plan.go +++ b/internal/shellgen/flake_plan.go @@ -10,6 +10,7 @@ import ( "slices" "strings" + "go.jetpack.io/devbox/internal/build" "go.jetpack.io/devbox/internal/devpkg" "go.jetpack.io/devbox/internal/nix" "go.jetpack.io/devbox/internal/patchpkg" @@ -73,9 +74,11 @@ func (f *flakePlan) needsGlibcPatch() bool { } type glibcPatchFlake struct { - // DevboxExecutable is the absolute path to the Devbox binary to use as - // the flake's builder. It must not be the wrapper script. - DevboxExecutable string + // DevboxFlake provides the devbox binary that will act as the patch + // flake's builder. By default it's set to "github:jetify-com/devbox/" + + // [build.Version]. For dev builds, it's set to the local path to the + // Devbox source code (this Go module) if it's available. + DevboxFlake flake.Ref // NixpkgsGlibcFlakeRef is a flake reference to the nixpkgs flake // containing the new glibc package. @@ -100,31 +103,43 @@ type glibcPatchFlake struct { } func newGlibcPatchFlake(nixpkgsGlibcRev string, packages []*devpkg.Package) (glibcPatchFlake, error) { - // Get the path to the actual devbox binary (not the /usr/bin/devbox - // wrapper) so the flake build can use it. - exe, err := os.Executable() - if err != nil { - return glibcPatchFlake{}, err - } - exe, err = filepath.EvalSymlinks(exe) - if err != nil { - return glibcPatchFlake{}, err + patchFlake := glibcPatchFlake{ + DevboxFlake: flake.Ref{ + Type: flake.TypeGitHub, + Owner: "jetify-com", + Repo: "devbox", + Ref: build.Version, + }, + NixpkgsGlibcFlakeRef: "flake:nixpkgs/" + nixpkgsGlibcRev, } - flake := glibcPatchFlake{ - DevboxExecutable: exe, - NixpkgsGlibcFlakeRef: "flake:nixpkgs/" + nixpkgsGlibcRev, + // In dev builds, use the local Devbox flake for patching packages + // instead of the one on GitHub. Using build.IsDev doesn't work because + // DEVBOX_PROD=1 will attempt to download 0.0.0-dev from GitHub. + if strings.HasPrefix(build.Version, "0.0.0") { + src, err := build.SourceDir() + if err != nil { + slog.Error("can't find the local devbox flake for patching, falling back to the latest github release", "err", err) + patchFlake.DevboxFlake = flake.Ref{ + Type: flake.TypeGitHub, + Owner: "jetify-com", + Repo: "devbox", + } + } else { + patchFlake.DevboxFlake = flake.Ref{Type: flake.TypePath, Path: src} + } } + for _, pkg := range packages { // Check to see if this is a CUDA package. If so, we need to add // it to the flake dependencies so that we can patch other // packages to reference it (like Python). - relAttrPath, err := flake.systemRelativeAttrPath(pkg) + relAttrPath, err := patchFlake.systemRelativeAttrPath(pkg) if err != nil { return glibcPatchFlake{}, err } if strings.HasPrefix(relAttrPath, "cudaPackages") { - if err := flake.addDependency(pkg); err != nil { + if err := patchFlake.addDependency(pkg); err != nil { return glibcPatchFlake{}, err } } @@ -132,11 +147,13 @@ func newGlibcPatchFlake(nixpkgsGlibcRev string, packages []*devpkg.Package) (gli if !pkg.Patch { continue } - if err := flake.addOutput(pkg); err != nil { + if err := patchFlake.addOutput(pkg); err != nil { return glibcPatchFlake{}, err } } - return flake, nil + + slog.Debug("creating new patch flake", "flake", &patchFlake) + return patchFlake, nil } // addInput adds a flake input that provides pkg. @@ -301,5 +318,34 @@ func (g *glibcPatchFlake) writeTo(dir string) error { slog.Debug("error copying system libcuda.so to flake", "dir", dir) } } - return writeFromTemplate(dir, g, "glibc-patch.nix", "flake.nix") + changed, err := writeFromTemplate(dir, g, "glibc-patch.nix", "flake.nix") + if err != nil { + return err + } + if changed { + _ = os.Remove(filepath.Join(dir, "flake.lock")) + } + return nil +} + +func (g *glibcPatchFlake) LogValue() slog.Value { + inputs := make([]slog.Attr, 0, 2+len(g.Inputs)) + inputs = append(inputs, + slog.String("devbox", g.DevboxFlake.String()), + slog.String("nixpkgs-glibc", g.NixpkgsGlibcFlakeRef), + ) + for k, v := range g.Inputs { + inputs = append(inputs, slog.String(k, v)) + } + + var outputs []string + for _, pkg := range g.Outputs.Packages { + for attrPath := range pkg { + outputs = append(outputs, attrPath) + } + } + return slog.GroupValue( + slog.Attr{Key: "inputs", Value: slog.GroupValue(inputs...)}, + slog.Attr{Key: "outputs", Value: slog.AnyValue(outputs)}, + ) } diff --git a/internal/shellgen/generate.go b/internal/shellgen/generate.go index 57d8e37acc9..3fb9a907efc 100644 --- a/internal/shellgen/generate.go +++ b/internal/shellgen/generate.go @@ -37,13 +37,13 @@ func GenerateForPrintEnv(ctx context.Context, devbox devboxer) error { outPath := genPath(devbox) // Preserving shell.nix to avoid breaking old-style .envrc users - err = writeFromTemplate(outPath, plan, "shell.nix", "shell.nix") + _, err = writeFromTemplate(outPath, plan, "shell.nix", "shell.nix") if err != nil { return errors.WithStack(err) } // Gitignore file is added to the .devbox directory - err = writeFromTemplate(filepath.Join(devbox.ProjectDir(), ".devbox"), plan, ".gitignore", ".gitignore") + _, err = writeFromTemplate(filepath.Join(devbox.ProjectDir(), ".devbox"), plan, ".gitignore", ".gitignore") if err != nil { return errors.WithStack(err) } @@ -70,7 +70,7 @@ var ( tmplBuf bytes.Buffer ) -func writeFromTemplate(path string, plan any, tmplName, generatedName string) error { +func writeFromTemplate(path string, plan any, tmplName, generatedName string) (changed bool, err error) { tmplKey := tmplName + ".tmpl" tmpl := tmplCache[tmplKey] if tmpl == nil { @@ -81,64 +81,64 @@ func writeFromTemplate(path string, plan any, tmplName, generatedName string) er glob := "tmpl/" + tmplKey tmpl, err = tmpl.ParseFS(tmplFS, glob) if err != nil { - return redact.Errorf("parse embedded tmplFS glob %q: %v", redact.Safe(glob), redact.Safe(err)) + return false, redact.Errorf("parse embedded tmplFS glob %q: %v", redact.Safe(glob), redact.Safe(err)) } tmplCache[tmplKey] = tmpl } tmplBuf.Reset() if err := tmpl.Execute(&tmplBuf, plan); err != nil { - return redact.Errorf("execute template %s: %v", redact.Safe(tmplKey), err) + return false, redact.Errorf("execute template %s: %v", redact.Safe(tmplKey), err) } // In some circumstances, Nix looks at the mod time of a file when // caching, so we only want to update the file if something has // changed. Blindly overwriting the file could invalidate Nix's cache // every time, slowing down evaluation considerably. - err := overwriteFileIfChanged(filepath.Join(path, generatedName), tmplBuf.Bytes(), 0o644) + changed, err = overwriteFileIfChanged(filepath.Join(path, generatedName), tmplBuf.Bytes(), 0o644) if err != nil { - return redact.Errorf("write %s to file: %v", redact.Safe(tmplName), err) + return changed, redact.Errorf("write %s to file: %v", redact.Safe(tmplName), err) } - return nil + return changed, nil } // overwriteFileIfChanged checks that the contents of f == data, and overwrites // f if they differ. It also ensures that f's permissions are set to perm. -func overwriteFileIfChanged(path string, data []byte, perm os.FileMode) error { +func overwriteFileIfChanged(path string, data []byte, perm os.FileMode) (changed bool, err error) { flag := os.O_RDWR | os.O_CREATE file, err := os.OpenFile(path, flag, perm) if errors.Is(err, os.ErrNotExist) { if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - return err + return false, err } // Definitely a new file if we had to make the directory. - return os.WriteFile(path, data, perm) + return true, os.WriteFile(path, data, perm) } if err != nil { - return err + return false, err } defer file.Close() fi, err := file.Stat() if err != nil || fi.Mode().Perm() != perm { if err := file.Chmod(perm); err != nil { - return err + return false, err } } // Fast path - check if the lengths differ. if err == nil && fi.Size() != int64(len(data)) { - return overwriteFile(file, data, 0) + return true, overwriteFile(file, data, 0) } r := bufio.NewReader(file) for offset := range data { b, err := r.ReadByte() if err != nil || b != data[offset] { - return overwriteFile(file, data, offset) + return true, overwriteFile(file, data, offset) } } - return nil + return false, nil } // overwriteFile truncates f to len(data) and writes data[offset:] beginning at @@ -168,5 +168,9 @@ var templateFuncs = template.FuncMap{ func makeFlakeFile(d devboxer, plan *flakePlan) error { flakeDir := FlakePath(d) - return writeFromTemplate(flakeDir, plan, "flake.nix", "flake.nix") + changed, err := writeFromTemplate(flakeDir, plan, "flake.nix", "flake.nix") + if changed { + _ = os.Remove(filepath.Join(flakeDir, "flake.lock")) + } + return err } diff --git a/internal/shellgen/generate_test.go b/internal/shellgen/generate_test.go index 0643bbd8e01..1801fd73cf6 100644 --- a/internal/shellgen/generate_test.go +++ b/internal/shellgen/generate_test.go @@ -26,14 +26,14 @@ func TestWriteFromTemplate(t *testing.T) { t.Setenv("__DEVBOX_NIX_SYSTEM", "x86_64-linux") dir := filepath.Join(t.TempDir(), "makeme") outPath := filepath.Join(dir, "flake.nix") - err := writeFromTemplate(dir, testFlakeTmplPlan, "flake.nix", "flake.nix") + _, err := writeFromTemplate(dir, testFlakeTmplPlan, "flake.nix", "flake.nix") if err != nil { t.Fatal("got error writing flake template:", err) } cmpGoldenFile(t, outPath, "testdata/flake.nix.golden") t.Run("WriteUnmodified", func(t *testing.T) { - err = writeFromTemplate(dir, testFlakeTmplPlan, "flake.nix", "flake.nix") + _, err = writeFromTemplate(dir, testFlakeTmplPlan, "flake.nix", "flake.nix") if err != nil { t.Fatal("got error writing flake template:", err) } @@ -49,7 +49,7 @@ func TestWriteFromTemplate(t *testing.T) { FlakeInputs: []flakeInput{}, System: "x86_64-linux", } - err = writeFromTemplate(dir, emptyPlan, "flake.nix", "flake.nix") + _, err = writeFromTemplate(dir, emptyPlan, "flake.nix", "flake.nix") if err != nil { t.Fatal("got error writing flake template:", err) } diff --git a/internal/shellgen/tmpl/glibc-patch.nix.tmpl b/internal/shellgen/tmpl/glibc-patch.nix.tmpl index 4d0df7aa04e..b71799e43e5 100644 --- a/internal/shellgen/tmpl/glibc-patch.nix.tmpl +++ b/internal/shellgen/tmpl/glibc-patch.nix.tmpl @@ -2,11 +2,7 @@ description = "Patches packages to use a newer version of glibc"; inputs = { - local-devbox = { - url = "path://{{ .DevboxExecutable }}"; - flake = false; - }; - + devbox.url = "{{ .DevboxFlake }}"; nixpkgs-glibc.url = "{{ .NixpkgsGlibcFlakeRef }}"; {{- range $name, $flakeref := .Inputs }} @@ -14,7 +10,7 @@ {{- end }} }; - outputs = args@{ self, local-devbox, nixpkgs-glibc {{- range $name, $_ := .Inputs -}}, {{ $name }} {{- end }} }: + outputs = args@{ self, devbox, nixpkgs-glibc {{- range $name, $_ := .Inputs -}}, {{ $name }} {{- end }} }: let # Initialize each nixpkgs input into a new attribute set with the # schema "pkgs...". @@ -97,24 +93,9 @@ glibc = if isLinux then nixpkgs-glibc.legacyPackages."${system}".glibc else null; gcc = if isLinux then nixpkgs-glibc.legacyPackages."${system}".stdenv.cc.cc.lib else null; - # Create a package that puts the local devbox binary in the conventional - # bin subdirectory. This also ensures that the executable is named - # "devbox" and not "-source" (which is how Nix names the flake - # input). Invoking it as anything other than "devbox" will break - # testscripts which look at os.Args[0] to decide to run the real - # entrypoint or the test entrypoint. - devbox = derivation { - name = "devbox"; - system = pkg.system; - builder = "${bash}/bin/bash"; - - # exit 0 to work around https://github.com/NixOS/nix/issues/2176 - args = [ "-c" "${coreutils}/bin/mkdir -p $out/bin && ${coreutils}/bin/cp ${local-devbox} $out/bin/devbox && exit 0" ]; - }; - DEVBOX_DEBUG = 1; - src = self; - builder = "${devbox}/bin/devbox"; + + builder = "${devbox.packages.${system}.default}/bin/devbox"; args = [ "patch" "--restore-refs" ] ++ (if glibc != null then [ "--glibc" "${glibc}" ] else [ ]) ++ (if gcc != null then [ "--gcc" "${gcc}" ] else [ ]) ++ diff --git a/internal/telemetry/segment.go b/internal/telemetry/segment.go index ce169364372..b8fbf29a26c 100644 --- a/internal/telemetry/segment.go +++ b/internal/telemetry/segment.go @@ -44,7 +44,7 @@ func newTrackMessage(name string, meta Metadata) *segment.Track { dur = time.Since(meta.EventStart) } uid := userID() - return &segment.Track{ + track := &segment.Track{ MessageId: newEventID(), Type: "track", // Only set anonymous ID if user ID is not set. Otherwise segment will @@ -66,17 +66,30 @@ func newTrackMessage(name string, meta Metadata) *segment.Track { }, }, Properties: segment.Properties{ - "cloud_region": meta.CloudRegion, "command": meta.Command, "command_args": meta.CommandFlags, "duration": dur.Milliseconds(), + "nix_version": nixVersion, "org_id": orgID(), "packages": meta.Packages, "shell": os.Getenv(envir.Shell), "shell_access": shellAccess(), - "nix_version": nixVersion, }, } + + // Property keys match the API events (search "Devspace Created"). + insertEnv := func(envKey, propKey string) { + v, ok := os.LookupEnv(envKey) + if ok { + track.Properties[propKey] = v + } + } + insertEnv("_JETIFY_SANDBOX_ID", "devspace") + insertEnv("_JETIFY_GH_REPO", "repo") + insertEnv("_JETIFY_GIT_REF", "ref") + insertEnv("_JETIFY_GIT_SUBDIR", "subdir") + + return track } // bufferSegmentMessage buffers a Segment message to disk so that Report can diff --git a/pkg/autodetect/autodetect.go b/pkg/autodetect/autodetect.go index 12ef7c8a478..038fd86528c 100644 --- a/pkg/autodetect/autodetect.go +++ b/pkg/autodetect/autodetect.go @@ -41,8 +41,10 @@ func populateConfig(ctx context.Context, path string, config *devconfig.Config) func detectors(path string) []detector.Detector { return []detector.Detector{ - &detector.PythonDetector{Root: path}, + &detector.GoDetector{Root: path}, + &detector.PHPDetector{Root: path}, &detector.PoetryDetector{Root: path}, + &detector.PythonDetector{Root: path}, } } @@ -61,6 +63,13 @@ func relevantDetector(path string) (detector.Detector, error) { relevantScore := 0.0 var mostRelevantDetector detector.Detector for _, detector := range detectors(path) { + if d, ok := detector.(interface { + Init() error + }); ok { + if err := d.Init(); err != nil { + return nil, err + } + } score, err := detector.Relevance(path) if err != nil { return nil, err diff --git a/pkg/autodetect/detector/go.go b/pkg/autodetect/detector/go.go new file mode 100644 index 00000000000..d7c44f007ca --- /dev/null +++ b/pkg/autodetect/detector/go.go @@ -0,0 +1,51 @@ +package detector + +import ( + "context" + "os" + "path/filepath" + "regexp" +) + +type GoDetector struct { + Root string +} + +var _ Detector = &GoDetector{} + +func (d *GoDetector) Relevance(path string) (float64, error) { + goModPath := filepath.Join(d.Root, "go.mod") + _, err := os.Stat(goModPath) + if err == nil { + return 1.0, nil + } + if os.IsNotExist(err) { + return 0, nil + } + return 0, err +} + +func (d *GoDetector) Packages(ctx context.Context) ([]string, error) { + goModPath := filepath.Join(d.Root, "go.mod") + goModContent, err := os.ReadFile(goModPath) + if err != nil { + return nil, err + } + + // Parse the Go version from go.mod + goVersion := parseGoVersion(string(goModContent)) + goVersion = determineBestVersion(ctx, "go", goVersion) + return []string{"go@" + goVersion}, nil +} + +func parseGoVersion(goModContent string) string { + // Use a regular expression to find the Go version directive + re := regexp.MustCompile(`(?m)^go\s+(\d+\.\d+(\.\d+)?)`) + match := re.FindStringSubmatch(goModContent) + + if len(match) >= 2 { + return match[1] + } + + return "" +} diff --git a/pkg/autodetect/detector/go_test.go b/pkg/autodetect/detector/go_test.go new file mode 100644 index 00000000000..758a63ca134 --- /dev/null +++ b/pkg/autodetect/detector/go_test.go @@ -0,0 +1,103 @@ +package detector + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGoDetectorRelevance(t *testing.T) { + tempDir := t.TempDir() + detector := &GoDetector{Root: tempDir} + + t.Run("No go.mod file", func(t *testing.T) { + relevance, err := detector.Relevance(tempDir) + assert.NoError(t, err) + assert.Equal(t, 0.0, relevance) + }) + + t.Run("With go.mod file", func(t *testing.T) { + err := os.WriteFile(filepath.Join(tempDir, "go.mod"), []byte("module example.com"), 0o644) + assert.NoError(t, err) + + relevance, err := detector.Relevance(tempDir) + assert.NoError(t, err) + assert.Equal(t, 1.0, relevance) + }) +} + +func TestGoDetectorPackages(t *testing.T) { + tempDir := t.TempDir() + detector := &GoDetector{Root: tempDir} + + t.Run("No go.mod file", func(t *testing.T) { + packages, err := detector.Packages(context.Background()) + assert.Error(t, err) + assert.Nil(t, packages) + }) + + t.Run("With go.mod file and no version", func(t *testing.T) { + err := os.WriteFile(filepath.Join(tempDir, "go.mod"), []byte("module example.com"), 0o644) + assert.NoError(t, err) + + packages, err := detector.Packages(context.Background()) + assert.NoError(t, err) + assert.Equal(t, []string{"go@latest"}, packages) + }) + + t.Run("With go.mod file and specific version", func(t *testing.T) { + goModContent := ` +module example.com + +go 1.18 +` + err := os.WriteFile(filepath.Join(tempDir, "go.mod"), []byte(goModContent), 0o644) + assert.NoError(t, err) + + packages, err := detector.Packages(context.Background()) + assert.NoError(t, err) + assert.Equal(t, []string{"go@1.18"}, packages) + }) +} + +func TestParseGoVersion(t *testing.T) { + testCases := []struct { + name string + content string + expected string + }{ + { + name: "No version", + content: "module example.com", + expected: "", + }, + { + name: "With version", + content: ` +module example.com + +go 1.18 +`, + expected: "1.18", + }, + { + name: "With patch version", + content: ` +module example.com + +go 1.18.3 +`, + expected: "1.18.3", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + version := parseGoVersion(tc.content) + assert.Equal(t, tc.expected, version) + }) + } +} diff --git a/pkg/autodetect/detector/php.go b/pkg/autodetect/detector/php.go new file mode 100644 index 00000000000..ed01079f50e --- /dev/null +++ b/pkg/autodetect/detector/php.go @@ -0,0 +1,111 @@ +package detector + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "go.jetpack.io/devbox/internal/searcher" +) + +type composerJSON struct { + Require map[string]string `json:"require"` +} + +type PHPDetector struct { + Root string + composerJSON *composerJSON +} + +var _ Detector = &PHPDetector{} + +func (d *PHPDetector) Init() error { + composer, err := loadComposerJSON(d.Root) + if err != nil && !os.IsNotExist(err) { + return err + } + d.composerJSON = composer + return nil +} + +func (d *PHPDetector) Relevance(path string) (float64, error) { + if d.composerJSON == nil { + return 0, nil + } + return 1, nil +} + +func (d *PHPDetector) Packages(ctx context.Context) ([]string, error) { + packages := []string{fmt.Sprintf("php@%s", d.phpVersion(ctx))} + extensions, err := d.phpExtensions(ctx) + if err != nil { + return nil, err + } + packages = append(packages, extensions...) + return packages, nil +} + +func (d *PHPDetector) phpVersion(ctx context.Context) string { + require := d.composerJSON.Require + + if require["php"] == "" { + return "latest" + } + // Remove the caret (^) if present + version := strings.TrimPrefix(require["php"], "^") + + // Extract version in the format x, x.y, or x.y.z + re := regexp.MustCompile(`^(\d+(\.\d+){0,2})`) + match := re.FindString(version) + + return determineBestVersion(ctx, "php", match) +} + +func (d *PHPDetector) phpExtensions(ctx context.Context) ([]string, error) { + resolved, err := searcher.Client().ResolveV2(ctx, "php", d.phpVersion(ctx)) + if err != nil { + return nil, err + } + + // extract major-minor from resolved.Version + re := regexp.MustCompile(`^(\d+)\.(\d+)`) + matches := re.FindStringSubmatch(resolved.Version) + if len(matches) < 3 { + return nil, fmt.Errorf("could not parse PHP version: %s", resolved.Version) + } + majorMinor := matches[1] + matches[2] + + extensions := []string{} + for key := range d.composerJSON.Require { + if strings.HasPrefix(key, "ext-") { + // The way nix versions php extensions is inconsistent. Sometimes the version is the PHP + // version, sometimes it's the extension version. We just use @latest everywhere which in + // practice will just use the version of the extension that exists in the same nixpkgs as + // the php version. + extensions = append( + extensions, + fmt.Sprintf("php%sExtensions.%s@latest", majorMinor, strings.TrimPrefix(key, "ext-")), + ) + } + } + + return extensions, nil +} + +func loadComposerJSON(root string) (*composerJSON, error) { + composerPath := filepath.Join(root, "composer.json") + composerData, err := os.ReadFile(composerPath) + if err != nil { + return nil, err + } + var composer composerJSON + err = json.Unmarshal(composerData, &composer) + if err != nil { + return nil, err + } + return &composer, nil +} diff --git a/pkg/autodetect/detector/php_test.go b/pkg/autodetect/detector/php_test.go new file mode 100644 index 00000000000..1adafd3893d --- /dev/null +++ b/pkg/autodetect/detector/php_test.go @@ -0,0 +1,193 @@ +package detector + +import ( + "context" + "os" + "path/filepath" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPHPDetector_Relevance(t *testing.T) { + tests := []struct { + name string + fs fstest.MapFS + expected float64 + }{ + { + name: "no composer.json", + fs: fstest.MapFS{}, + expected: 0, + }, + { + name: "with composer.json", + fs: fstest.MapFS{ + "composer.json": &fstest.MapFile{ + Data: []byte(`{ + "require": { + "php": "^8.1" + } + }`), + }, + }, + expected: 1, + }, + } + + for _, curTest := range tests { + t.Run(curTest.name, func(t *testing.T) { + dir := t.TempDir() + for name, file := range curTest.fs { + err := os.WriteFile(filepath.Join(dir, name), file.Data, 0o644) + require.NoError(t, err) + } + + d := &PHPDetector{Root: dir} + err := d.Init() + require.NoError(t, err) + + score, err := d.Relevance(dir) + require.NoError(t, err) + assert.Equal(t, curTest.expected, score) + }) + } +} + +func TestPHPDetector_Packages(t *testing.T) { + tests := []struct { + name string + fs fstest.MapFS + expectedPHP string + expectedError bool + }{ + { + name: "no php version specified", + fs: fstest.MapFS{ + "composer.json": &fstest.MapFile{ + Data: []byte(`{ + "require": {} + }`), + }, + }, + expectedPHP: "php@latest", + }, + { + name: "specific php version", + fs: fstest.MapFS{ + "composer.json": &fstest.MapFile{ + Data: []byte(`{ + "require": { + "php": "^8.1" + } + }`), + }, + }, + expectedPHP: "php@8.1", + }, + { + name: "php version with patch", + fs: fstest.MapFS{ + "composer.json": &fstest.MapFile{ + Data: []byte(`{ + "require": { + "php": "^8.1.2" + } + }`), + }, + }, + expectedPHP: "php@8.1.2", + }, + { + name: "invalid composer.json", + fs: fstest.MapFS{ + "composer.json": &fstest.MapFile{ + Data: []byte(`invalid json`), + }, + }, + expectedError: true, + }, + } + + for _, curTest := range tests { + t.Run(curTest.name, func(t *testing.T) { + dir := t.TempDir() + for name, file := range curTest.fs { + err := os.WriteFile(filepath.Join(dir, name), file.Data, 0o644) + require.NoError(t, err) + } + + d := &PHPDetector{Root: dir} + err := d.Init() + if curTest.expectedError { + require.Error(t, err) + return + } + require.NoError(t, err) + + packages, err := d.Packages(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{curTest.expectedPHP}, packages) + }) + } +} + +func TestPHPDetector_PHPExtensions(t *testing.T) { + tests := []struct { + name string + fs fstest.MapFS + expectedExtensions []string + }{ + { + name: "no extensions", + fs: fstest.MapFS{ + "composer.json": &fstest.MapFile{ + Data: []byte(`{ + "require": { + "php": "^8.1" + } + }`), + }, + }, + expectedExtensions: []string{}, + }, + { + name: "multiple extensions", + fs: fstest.MapFS{ + "composer.json": &fstest.MapFile{ + Data: []byte(`{ + "require": { + "php": "^8.1", + "ext-mbstring": "*", + "ext-imagick": "*" + } + }`), + }, + }, + expectedExtensions: []string{ + "php81Extensions.mbstring@latest", + "php81Extensions.imagick@latest", + }, + }, + } + + for _, curTest := range tests { + t.Run(curTest.name, func(t *testing.T) { + dir := t.TempDir() + for name, file := range curTest.fs { + err := os.WriteFile(filepath.Join(dir, name), file.Data, 0o644) + require.NoError(t, err) + } + + d := &PHPDetector{Root: dir} + err := d.Init() + require.NoError(t, err) + + extensions, err := d.phpExtensions(context.Background()) + require.NoError(t, err) + assert.ElementsMatch(t, curTest.expectedExtensions, extensions) + }) + } +} diff --git a/vendor-hash b/vendor-hash index 9ececc1780d..b87e65da4b3 100644 --- a/vendor-hash +++ b/vendor-hash @@ -1 +1 @@ -sha256-rwmNzYzmZqNcNVV4GgqCVLT6ofIkblVCMJHLGwlhcGw= +sha256-js0dxnLBSnfhgjigTmQAh7D9t6ZeSHf7k6Xd3RIBUjo= From 969c4abb48afd6387ef4ee3d47614dc69f581b0b Mon Sep 17 00:00:00 2001 From: John Lago <750845+Lagoja@users.noreply.github.com> Date: Mon, 25 Nov 2024 12:07:45 -0800 Subject: [PATCH 3/3] Fix flake version string issue (#2406) (cherry picked from commit ed490a29207d28569b48570609425ee11de00c24) --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 00d3c871cf6..5b69305eeaa 100644 --- a/flake.nix +++ b/flake.nix @@ -19,7 +19,7 @@ else "${self.dirtyShortRev or "dirty"}"; # Add the commit to the version string for flake builds - version = "${lastTag}-${revision}"; + version = "${lastTag}"; # Run `devbox run update-flake` to update the vendor-hash vendorHash =