diff --git a/.config/brew/Formula/polylint.rb b/.config/brew/Formula/polylint.rb index 99e43c7..c8cc7a7 100644 --- a/.config/brew/Formula/polylint.rb +++ b/.config/brew/Formula/polylint.rb @@ -5,21 +5,21 @@ class Polylint < Formula desc "Polylint: Extensible generic linter" homepage "https://github.com/zph/polylint" - version "0.0.1" + version "0.0.5" license "MIT" on_macos do if Hardware::CPU.intel? - url "https://github.com/zph/polylint/releases/download/v0.0.1/polylint_Darwin_x86_64.tar.gz" - sha256 "258aae10c3667e947882ba58f0577d00f3cbcf071b80cc234a12c6d176db790b" + url "https://github.com/zph/polylint/releases/download/v0.0.5/polylint_darwin_x86_64.tar.gz" + sha256 "80f1447fbcee99c9cee2789c5fe3048379ba60f3dc481516ced70e78016f34a2" def install bin.install "polylint" end end if Hardware::CPU.arm? - url "https://github.com/zph/polylint/releases/download/v0.0.1/polylint_Darwin_arm64.tar.gz" - sha256 "16523d06970e2877a6a78104bf08d92ef71214d29e42accd438f96173dcc7e83" + url "https://github.com/zph/polylint/releases/download/v0.0.5/polylint_darwin_arm64.tar.gz" + sha256 "d7221479c7d0635c9d9960584bb3d68203c3d813094510a2586cb509d6da39ca" def install bin.install "polylint" @@ -29,16 +29,16 @@ def install on_linux do if Hardware::CPU.intel? - url "https://github.com/zph/polylint/releases/download/v0.0.1/polylint_Linux_x86_64.tar.gz" - sha256 "a27ed72eef00d89e8dbb71930b20f333b4d40e66006952e649fdb65a9a61e422" + url "https://github.com/zph/polylint/releases/download/v0.0.5/polylint_linux_x86_64.tar.gz" + sha256 "814fb8775b3ac202ba5f0e67d88c98ca021c98dbed57977ffcfe35c88cb73ee8" def install bin.install "polylint" end end if Hardware::CPU.arm? && Hardware::CPU.is_64_bit? - url "https://github.com/zph/polylint/releases/download/v0.0.1/polylint_Linux_arm64.tar.gz" - sha256 "34bd974561ddd5ef462872454198a6103957373da72d5c7d03efad13abddcc9d" + url "https://github.com/zph/polylint/releases/download/v0.0.5/polylint_linux_arm64.tar.gz" + sha256 "b40274c0fa63aabd0e3a9461a2bd3c2db13acb7b2533be49e7dfc2b4b3dd5263" def install bin.install "polylint" diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 5730f6c..fb39644 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -20,10 +20,10 @@ builds: - env: [] goos: - linux - - windows - darwin + # Note the need for prefixing a leading v ldflags: - - "-s -w -X 'github.com/zph/polylint/cmd.version={{.Version}}' -X 'github.com/zph/polylint/cmd.commit={{.Commit}}' -X 'github.com/zph/polylint/cmd.date={{.Date}}' -X 'github.com/zph/polylint/cmd.builtBy=goreleaser'" + - "-s -w -X 'github.com/zph/polylint/cmd.version=v{{.Version}}' -X 'github.com/zph/polylint/cmd.commit={{.Commit}}' -X 'github.com/zph/polylint/cmd.date={{.Date}}' -X 'github.com/zph/polylint/cmd.builtBy=goreleaser'" archives: - format: tar.gz @@ -36,10 +36,6 @@ archives: {{- else if eq .Arch "386" }}i386 {{- else }}{{ .Arch }}{{ end }} {{- if .Arm }}v{{ .Arm }}{{ end }} - # use zip for windows archives - format_overrides: - - goos: windows - format: zip changelog: sort: asc diff --git a/Makefile b/Makefile index e2fb842..b77b59c 100644 --- a/Makefile +++ b/Makefile @@ -6,14 +6,16 @@ all: build test build: go build -o ${BINARY_PATH} -run: - go build -o ${BINARY_PATH} +run: build ${BINARY_NAME} run ~/src/runbook clean: go clean rm ${BINARY_PATH} +install: build + cp -f ./bin/polylint ~/bin/ + test: go test -v . diff --git a/TODO.md b/TODO.md index 2f5fc91..3ed2ecf 100644 --- a/TODO.md +++ b/TODO.md @@ -40,9 +40,16 @@ - [ ] Configurable logging (log levels for debugging and k/v log values) - [ ] Add `init` command to create a default named config file - [ ] Remove panics that are poor programming style -- [ ] Add testing for config files... lines, path -- [ ] Setup goreleaser for releases +- [x] Add testing for config files... lines, path +- [x] Setup goreleaser for releases - [ ] Setup version bumper - - [ ] Ensure version is embedded into cobra cli and available for --version + - [x] Ensure version is embedded into cobra cli and available for --version - [ ] Add way to pipe to exec process and non-zero exit is a failing to provide builtin, js, exec mechanisms -- [ ] Setup version reading https://goreleaser.com/cookbooks/using-main.version/ +- [x] Setup version reading https://goreleaser.com/cookbooks/using-main.version/ +- [x] Add this as a hermit-package in zph/hermit-packages + +## Performance +- [ ] Setup support for gitignore and global gitignore to avoid reading things like node_modules :yawning_face: +- [ ] Setup parallelism for larger repos + - [ ] Requires reworking how we use goja functions because those VMs are not thread safe + - [ ] Could setup a worker pool of VMs? Or could provision new VM for each fn call.... diff --git a/cmd/root.go b/cmd/root.go index e2fe07f..b318e8f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -12,7 +12,7 @@ import ( ) var ( - version = "v0.0.1" + version = "v100.0.1" commit = "" date = "" ) @@ -57,14 +57,22 @@ func initConfig() { // Search config in home directory with name ".polylint" (without extension). viper.AddConfigPath(".") + viper.AddConfigPath("../.") + viper.AddConfigPath("../../.") viper.AddConfigPath(home) viper.SetConfigType("yaml") viper.SetConfigName(".polylint") } + viper.SetEnvPrefix("POLYLINT") viper.AutomaticEnv() // read in environment variables that match if err := viper.ReadInConfig(); err != nil { - fmt.Fprintln(os.Stderr, "Error reading config file:", viper.ConfigFileUsed()) + // Error means that it's a non-standard configuration file but that's ok + if err != viper.UnsupportedConfigError("polylint") { + fmt.Fprintf(os.Stderr, "Error reading config file: %s\nError: %e", viper.ConfigFileUsed(), err) + } + } else { + fmt.Printf("Using config file: %s\n", viper.ConfigFileUsed()) } } diff --git a/cmd/run.go b/cmd/run.go index ad16b22..51445f8 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" + "github.com/jedib0t/go-pretty/v6/table" "github.com/spf13/cobra" "github.com/spf13/viper" pl "github.com/zph/polylint/pkg" @@ -30,6 +31,7 @@ func Run(cmd *cobra.Command, args []string) (int, []error) { var exitCode int exitCode = 0 var errs []error + var results []pl.FileReport for _, root := range args { configRaw, err := os.ReadFile(viper.ConfigFileUsed()) @@ -49,7 +51,10 @@ func Run(cmd *cobra.Command, args []string) (int, []error) { return err } - // Check if the file has the extension + // Causes 4x slowdown in benchmarks on project with large node_modules folder + // if we use IsRegular() instead of isDir() + // Regular mode = non-dir, non-symlink etcs + // TODO(zph) investigate this issue and find a better solution if !info.IsDir() { content, err := os.ReadFile(path) @@ -63,20 +68,48 @@ func Run(cmd *cobra.Command, args []string) (int, []error) { return fmt.Errorf("error processing file %q: %v", path, err2) } - if len(result.Findings) > 0 { - fmt.Printf("\n%s: violations count %d\n", result.Path, len(result.Findings)) - for idx, finding := range result.Findings { - // TODO: figure out why the rule embedded is wrong - fmt.Printf("%d: Line %3d %30s %20s\n", idx+1, finding.LineNo, finding.RuleId, finding.Rule.Description) - } - exitCode = 1 - } + results = append(results, result) } return nil }) errs = append(errs, err) } + + t := table.NewWriter() + t.SetOutputMirror(os.Stdout) + t.AppendHeader(table.Row{"File", "#", "Scope", "Rule Id", "Recommendation", "Link"}) + + summary := table.NewWriter() + summary.SetOutputMirror(os.Stdout) + summary.AppendHeader(table.Row{"File", "Violations"}) + summary.SortBy([]table.SortBy{ + {Name: "Violations", Mode: table.Dsc}, + }) + for _, result := range results { + if len(result.Findings) > 0 { + summary.AppendRow([]interface{}{result.Path, len(result.Findings)}) + for idx, finding := range result.Findings { + var scope string + if finding.Rule.Scope == "file" || finding.Rule.Scope == "path" { + scope = fmt.Sprintf("%s", finding.Rule.Scope) + } else { + scope = fmt.Sprintf("%s %3d", finding.Rule.Scope, finding.LineNo) + } + t.AppendRow([]interface{}{ + result.Path, idx + 1, scope, finding.RuleId, finding.Rule.Recommendation, finding.Rule.Link, + }) + exitCode += 1 + } + } + } + summary.Render() + t.Render() + + if exitCode > 255 { + exitCode = 255 + } + nonNilErrors := make([]error, 0) for _, e := range errs { if e != nil { diff --git a/examples/simple.yaml b/examples/simple.yaml index 2872be8..46414c2 100644 --- a/examples/simple.yaml +++ b/examples/simple.yaml @@ -71,3 +71,18 @@ rules: scope: path name: fn body: const fn = (path, _i, _l) => path.includes('print') + +- id: no-python-in-path + description: Don't use python here + recommendation: None + severity: low + link: https://examples.com/wiki/no-print-js-path-level + include_paths: '\.py$' + exclude_paths: null + fn: + type: wasm + scope: path + name: path_validator + body: ./plugins/test-plugin.wasm + metadata: + sha256: f2880b9d1a2f70f7eddca65c7aa539483c800653669e5a070b8cb3b11a199eca diff --git a/go.mod b/go.mod index 30c81ca..e7a5bb4 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,11 @@ go 1.21.6 require ( github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 + github.com/extism/go-sdk v1.2.0 + github.com/jedib0t/go-pretty/v6 v6.5.8 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 + go.uber.org/zap v1.27.0 golang.org/x/mod v0.12.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -14,12 +17,15 @@ require ( github.com/dlclark/regexp2 v1.7.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect + github.com/gobwas/glob v0.2.3 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/rivo/uniseg v0.2.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -27,10 +33,10 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.9.0 // indirect + github.com/tetratelabs/wazero v1.3.0 // indirect + go.uber.org/multierr v1.10.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 981b5d7..1c6fed4 100644 --- a/go.sum +++ b/go.sum @@ -15,12 +15,16 @@ github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 h1:O7I1iuzEA7SG+dK8ocO github.com/dop251/goja v0.0.0-20240220182346-e401ed450204/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= +github.com/extism/go-sdk v1.2.0 h1:A0DnIMthdP8h6K9NbRpRs1PIXHOUlb/t/TZWk5eUzx4= +github.com/extism/go-sdk v1.2.0/go.mod h1:xUfKSEQndAvHBc1Ohdre0e+UdnRzUpVfbA8QLcx4fbY= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= @@ -30,6 +34,8 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jedib0t/go-pretty/v6 v6.5.8 h1:8BCzJdSvUbaDuRba4YVh+SKMGcAAKdkcF3SVFbrHAtQ= +github.com/jedib0t/go-pretty/v6 v6.5.8/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -41,6 +47,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= @@ -48,6 +56,8 @@ github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdU github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -71,18 +81,21 @@ github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMV github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tetratelabs/wazero v1.3.0 h1:nqw7zCldxE06B8zSZAY0ACrR9OH5QCcPwYmYlwtcwtE= +github.com/tetratelabs/wazero v1.3.0/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= @@ -101,8 +114,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/main_test.go b/main_test.go index 2127a64..749cc67 100644 --- a/main_test.go +++ b/main_test.go @@ -40,13 +40,13 @@ func TestProcessFile(t *testing.T) { findingsCount int expectedErr error }{ - {"Basic test without ignores", `print("A")`, "example.py", 4, nil}, - {"Basic test for-file ignore", forFileIgnore, "example.py", 2, nil}, - {"Basic test next-line ignore", nextLineIgnore, "example.py", 2, nil}, - {"Basic test next-line ignore shorthand", nextLineIgnoreShorthand, "example.py", 3, nil}, - {"Basic test next-line ignore doesn't apply", nextLineIgnoreDoesntApply, "example.py", 4, nil}, - {"Basic test with faulty ignore statement", fileWithFaultyIgnoreStatement, "example.py", 4, nil}, - {"Basic test with banned filename", nextLineIgnore, "print.py", 3, nil}, + {"Basic test without ignores", `print("A")`, "example.py", 5, nil}, + {"Basic test for-file ignore", forFileIgnore, "example.py", 3, nil}, + {"Basic test next-line ignore", nextLineIgnore, "example.py", 3, nil}, + {"Basic test next-line ignore shorthand", nextLineIgnoreShorthand, "example.py", 4, nil}, + {"Basic test next-line ignore doesn't apply", nextLineIgnoreDoesntApply, "example.py", 5, nil}, + {"Basic test with faulty ignore statement", fileWithFaultyIgnoreStatement, "example.py", 5, nil}, + {"Basic test with banned filename", nextLineIgnore, "print.py", 4, nil}, } for idx, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -86,7 +86,7 @@ func TestConfigFileParsing(t *testing.T) { content string expectedRuleCount int }{ - {"basic config file with 1 rule", simpleConfigFile, 6}, + {"basic config file with 1 rule", simpleConfigFile, 7}, } for idx, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/log.go b/pkg/log.go new file mode 100644 index 0000000..95dc599 --- /dev/null +++ b/pkg/log.go @@ -0,0 +1,16 @@ +package polylint + +import "go.uber.org/zap" + +var logz *zap.SugaredLogger + +func init() { + logger, _ := zap.NewProductionConfig().Build( + zap.WithCaller(true), + ) + + zap.NewAtomicLevelAt(zap.DebugLevel) + defer logger.Sync() // flushes buffer, if any + logz = logger.Sugar() + logz.Debugw("polylint initialized", "version", "0.0.2") +} diff --git a/pkg/processors.go b/pkg/processors.go index de3b85d..f298fa8 100644 --- a/pkg/processors.go +++ b/pkg/processors.go @@ -1,7 +1,9 @@ package polylint import ( + "context" "crypto/sha256" + "encoding/json" "fmt" "io" "log" @@ -13,6 +15,7 @@ import ( "strings" "github.com/dop251/goja" + extism "github.com/extism/go-sdk" "github.com/spf13/viper" "golang.org/x/mod/semver" "gopkg.in/yaml.v2" @@ -41,7 +44,7 @@ func extractIgnoresFromLine(line string, lineNo int, f *FileReport) error { f.Ignores = append(f.Ignores, Ignore{Scope: pathScope, SourceLineNo: lineNo, LineNo: 0, Id: ignore}) } } else { - fmt.Printf("WARNING: directive for polylint not recognized on line %d %s %s\n", lineNo, directive, ignoresStr) + logz.Warnf("WARNING: directive for polylint not recognized on line %d %s %s\n", lineNo, directive, ignoresStr) return nil } } @@ -98,24 +101,27 @@ func LoadConfigFile(content string) (ConfigFile, error) { var config ConfigFile err := yaml.Unmarshal([]byte(content), &rawConfig) if err != nil { - fmt.Printf("Error unmarshalling YAML: %v", err) + logz.Errorf("Error unmarshalling YAML: %v", err) return ConfigFile{}, err } if !strings.HasPrefix(rawConfig.Version, "v") { - fmt.Printf("Error: config file version must start with a 'v' but was %s\n", rawConfig.Version) + logz.Errorf("Error: config file version must start with a 'v' but was %s\n", rawConfig.Version) panic("Invalid version due to semver incompatibility") } if !semver.IsValid(rawConfig.Version) { - fmt.Printf("Error: Config version %s is newer than binary version %s\n", rawConfig.Version, viper.GetString("binary_version")) - fmt.Println(semver.IsValid(rawConfig.Version)) + logz.Errorf("Error: Config version %s is newer than binary version %s\n", rawConfig.Version, viper.GetString("binary_version")) + logz.Errorln(semver.IsValid(rawConfig.Version)) panic("Invalid version due to semver incompatibility") } // If version file is too new for binary version if semver.Compare(rawConfig.Version, viper.GetString("binary_version")) == 1 { - fmt.Printf("Warning: config file version %s is newer than binary version %s\n", rawConfig.Version, viper.GetString("binary_version")) + _ = 1 + // TODO: determine how to handle version for version check when not set in tests + // Ignore for now until we can control the output + //logz.Warnf("Warning: config file version %s is newer than binary version %s\n", rawConfig.Version, viper.GetString("binary_version")) } config.Version = rawConfig.Version @@ -285,6 +291,8 @@ func BuildLineFn(f RawFn) RuleFunc { return buildLineFnBuiltin(f) case "js": return buildJsFn(f) + case "wasm": + return buildWasmFn(f) default: panic(fmt.Sprintf("unknown type %s", f.Type)) } @@ -296,6 +304,8 @@ func BuildFileScopeFn(f RawFn) RuleFunc { return BuildFileFnBuiltin(f) case "js": return BuildFileFnJs(f) + case "wasm": + return buildWasmFn(f) default: panic(fmt.Sprintf("unknown type %s", f.Type)) } @@ -340,6 +350,8 @@ func BuildPathScopeFn(f RawFn) RuleFunc { return BuildPathFnBuiltin(f) case "js": return BuildPathFnJs(f) + case "wasm": + return buildWasmFn(f) default: panic(fmt.Sprintf("unknown type %s", f.Type)) } @@ -394,6 +406,80 @@ func buildJsFn(f RawFn) RuleFunc { return fn } +func buildWasmFn(f RawFn) RuleFunc { + hash, err := f.GetMetadataHash() + if err != nil { + logz.Warnf("Warning: cannot find metadata hash for %s\n", f.Body) + } + var content []byte + + // TODO: handle null case of hash + if hash != "" { + content, err = f.GetWASMFromCache(hash) + } + if err != nil { + if strings.HasPrefix(f.Body, "http") { + content, err = f.GetWASMFromUrl(f.Body) + } else { + content, err = f.GetWASMFromPath(f.Body) + } + } + if err != nil { + panic(err) + } + + ok := f.CheckWASMHash(content, hash) + if !ok && hash != "" { + logz.Errorf("hash mismatch for %s", f.Body) + } + f.WriteWASMToCache(content) + + var location []extism.Wasm + location = append(location, extism.WasmData{ + Data: content, + Hash: hash, + }) + + manifest := extism.Manifest{ + Wasm: location, + } + + ctx := context.Background() + config := extism.PluginConfig{ + EnableWasi: true, + } + + plugin, err := extism.NewPlugin(ctx, manifest, config, []extism.HostFunction{}) + if err != nil { + logz.Errorf("Failed to initialize plugin: %v\n", err) + os.Exit(1) + } + + return func(path string, idx int, line string) bool { + args := RuleFuncArgs{path, idx, line} + b, err := json.Marshal(&args) + if err != nil { + panic(err) + } + + exit, bytes, err := plugin.CallWithContext(ctx, f.Name, b) + if err != nil { + logz.Errorln(err) + os.Exit(int(exit)) + } + var result RuleFuncResult + json.Unmarshal(bytes, &result) + + return result.Value + } +} + +type RuleFuncArgs [3]interface{} + +type RuleFuncResult struct { + Value bool +} + func ProcessFile(content string, path string, cfg ConfigFile) (FileReport, error) { f := FileReport{ Path: path, @@ -405,7 +491,7 @@ func ProcessFile(content string, path string, cfg ConfigFile) (FileReport, error for idx, line := range lines { err := processLine(line, idx, &f) if err != nil { - fmt.Printf("ERROR: %s\n", err) + logz.Errorf("ERROR: %s\n", err) return FileReport{}, err } } diff --git a/pkg/types.go b/pkg/types.go index 2f81294..a0a1593 100644 --- a/pkg/types.go +++ b/pkg/types.go @@ -1,11 +1,18 @@ package polylint import ( + "crypto/sha256" + "fmt" + "io" + "net/http" + "os" + "path" "regexp" ) type SeverityLevel int type Scope string +type FnType string const ( unknownSeverity SeverityLevel = iota @@ -14,6 +21,12 @@ const ( highSeverity ) +const ( + builtinType FnType = "builtin" + jsType FnType = "js" + wasmType FnType = "wasm" +) + const ( unknownScope Scope = "unknown" pathScope Scope = "path" @@ -60,7 +73,7 @@ type Rule struct { } type Fn struct { - Type string + Type FnType Scope Scope Name string Args []any @@ -73,6 +86,100 @@ type RawFn struct { Name string Args []any Body string + + // sha256: sha256 hash in hex form + Metadata map[string]any +} + +func (f RawFn) GetMetadataHash() (string, error) { + hash, ok := f.Metadata["sha256"] + + if !ok { + return "", fmt.Errorf("could not get sha256 hash from metadata for: %s", f.Body) + } + + h, ok := hash.(string) + + if !ok { + return "", fmt.Errorf("could not get sha256 hash from metadata for: %s", f.Body) + } + return h, nil +} + +func (f RawFn) GetWASMFromUrl(url string) ([]byte, error) { + // Get the data + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Write the body to file + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return body, err +} + +func (f RawFn) GetWASMFromPath(path string) ([]byte, error) { + // Create the file + content, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return content, nil +} + +func (f RawFn) WriteWASMToCache(content []byte) (bool, error) { + // Make dir in ~/.local/cache/polylint/cache/SHA256 + h := sha256.New() + + h.Write(content) + + bs := h.Sum(nil) + + hex := fmt.Sprintf("%x", bs) + output_folder, err := f.CacheDirWASM() + if err != nil { + return false, err + } + os.MkdirAll(output_folder, 0755) + os.WriteFile(path.Join(output_folder, hex), content, 0755) + return true, nil +} + +func (f RawFn) CacheDirWASM() (string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return path.Join(home, ".local", "cache", "polylint", "cache"), nil +} + +func (f RawFn) CheckWASMHash(content []byte, hash string) bool { + h := sha256.New() + + h.Write(content) + + bs := h.Sum(nil) + + actual := fmt.Sprintf("%x", bs) + comparison := actual == hash + if !comparison { + logz.Infof("Comparison failed for desired sha256 %s and actual: %s\n", hash, actual) + } + return comparison +} + +func (f RawFn) GetWASMFromCache(hash string) ([]byte, error) { + dir, err := f.CacheDirWASM() + if err != nil { + return nil, err + } + // TODO: move to debug level logging + logz.Debugf("Success fetching file from cache %s\n", hash) + return os.ReadFile(path.Join(dir, hash)) } type RawRule struct { diff --git a/plugins/Makefile b/plugins/Makefile new file mode 100644 index 0000000..688ed3a --- /dev/null +++ b/plugins/Makefile @@ -0,0 +1,4 @@ +build: plugin.wasm + +plugin.wasm: *.js *.d.ts + extism-js plugin.js -i plugin.d.ts -o test-plugin.wasm diff --git a/plugins/plugin.d.ts b/plugins/plugin.d.ts new file mode 100644 index 0000000..1224763 --- /dev/null +++ b/plugins/plugin.d.ts @@ -0,0 +1,5 @@ +declare module 'main' { + export function path_validator(): I32; + export function file_content_validator(): I32; + export function line_validator(): I32; +} diff --git a/plugins/plugin.js b/plugins/plugin.js new file mode 100644 index 0000000..8f38720 --- /dev/null +++ b/plugins/plugin.js @@ -0,0 +1,21 @@ +// TODO: use decorators to simplify the internal fn behaviors? +function path_validator() { + const [path, idx, line] = JSON.parse(Host.inputString()) + if(path.includes(".py")) { + Host.outputString(JSON.stringify({value: true})) + } else { + Host.outputString(JSON.stringify({value: false})) + } +} + +function file_content_validator() { + const [path, idx, file] = JSON.parse(Host.inputString()) + Host.outputString(JSON.stringify({value: false})) +} + +function line_validator() { + const [path, idx, line] = JSON.parse(Host.inputString()) + Host.outputString(JSON.stringify({value: false})) +} + +module.exports = {path_validator, file_content_validator, line_validator} diff --git a/plugins/test-plugin.wasm b/plugins/test-plugin.wasm new file mode 100644 index 0000000..1aea024 Binary files /dev/null and b/plugins/test-plugin.wasm differ