From c1d0766e8b4ebd22d79367a84983634b7694f550 Mon Sep 17 00:00:00 2001 From: Zhongpeng Lin Date: Sat, 4 Mar 2023 20:42:06 -0800 Subject: [PATCH 1/2] Properly shutting down python processes debugging using lifecycle manager in progres --- gazelle/python/BUILD.bazel | 1 + gazelle/python/language.go | 1 + gazelle/python/lifecycle.go | 37 +++++++++++++++++++++++++++++++++++ gazelle/python/parser.go | 14 +++++++------ gazelle/python/std_modules.go | 14 +++++++------ 5 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 gazelle/python/lifecycle.go diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel index ddcad2785d..fcfe81bd61 100644 --- a/gazelle/python/BUILD.bazel +++ b/gazelle/python/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "generate.go", "kinds.go", "language.go", + "lifecycle.go", "parser.go", "resolve.go", "std_modules.go", diff --git a/gazelle/python/language.go b/gazelle/python/language.go index 56eb97b043..568ac9225c 100644 --- a/gazelle/python/language.go +++ b/gazelle/python/language.go @@ -23,6 +23,7 @@ import ( type Python struct { Configurer Resolver + LifeCycleManager } // NewLanguage initializes a new Python that satisfies the language.Language diff --git a/gazelle/python/lifecycle.go b/gazelle/python/lifecycle.go new file mode 100644 index 0000000000..f38383a3f7 --- /dev/null +++ b/gazelle/python/lifecycle.go @@ -0,0 +1,37 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "context" + "github.com/bazelbuild/bazel-gazelle/language" +) + +var _ language.FinishableLanguage = (*LifeCycleManager)(nil) + +type LifeCycleManager struct{} + +func (l *LifeCycleManager) Init(ctx context.Context) { + startParserProcess(ctx) + startStdModuleProcess(ctx) +} + +func (l *LifeCycleManager) DoneGeneratingRules() { + shutdownParserProcess() +} + +func (l *LifeCycleManager) DoneResolvingDeps() { + shutdownStdModuleProcess() +} diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index 33eb6f4b33..7f10a754bf 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -25,7 +25,6 @@ import ( "os/exec" "strings" "sync" - "time" "github.com/bazelbuild/rules_go/go/tools/bazel" "github.com/emirpasic/gods/sets/treeset" @@ -33,20 +32,18 @@ import ( ) var ( - parserStdin io.Writer + parserStdin io.WriteCloser parserStdout io.Reader parserMutex sync.Mutex ) -func init() { +func startParserProcess(ctx context.Context) { parseScriptRunfile, err := bazel.Runfile("python/parse") if err != nil { log.Printf("failed to initialize parser: %v\n", err) os.Exit(1) } - ctx := context.Background() - ctx, parserCancel := context.WithTimeout(ctx, time.Minute*10) cmd := exec.CommandContext(ctx, parseScriptRunfile) cmd.Stderr = os.Stderr @@ -71,7 +68,6 @@ func init() { } go func() { - defer parserCancel() if err := cmd.Wait(); err != nil { log.Printf("failed to wait for parser: %v\n", err) os.Exit(1) @@ -79,6 +75,12 @@ func init() { }() } +func shutdownParserProcess() { + if err := parserStdin.Close(); err != nil { + fmt.Fprintf(os.Stderr, "error closing parser: %v", err) + } +} + // python3Parser implements a parser for Python files that extracts the modules // as seen in the import statements. type python3Parser struct { diff --git a/gazelle/python/std_modules.go b/gazelle/python/std_modules.go index 94ef45666e..c537184c74 100644 --- a/gazelle/python/std_modules.go +++ b/gazelle/python/std_modules.go @@ -25,19 +25,18 @@ import ( "strconv" "strings" "sync" - "time" "github.com/bazelbuild/rules_go/go/tools/bazel" ) var ( - stdModulesStdin io.Writer + stdModulesStdin io.WriteCloser stdModulesStdout io.Reader stdModulesMutex sync.Mutex stdModulesSeen map[string]struct{} ) -func init() { +func startStdModuleProcess(ctx context.Context) { stdModulesSeen = make(map[string]struct{}) stdModulesScriptRunfile, err := bazel.Runfile("python/std_modules") @@ -46,8 +45,6 @@ func init() { os.Exit(1) } - ctx := context.Background() - ctx, stdModulesCancel := context.WithTimeout(ctx, time.Minute*10) cmd := exec.CommandContext(ctx, stdModulesScriptRunfile) cmd.Stderr = os.Stderr @@ -73,7 +70,6 @@ func init() { } go func() { - defer stdModulesCancel() if err := cmd.Wait(); err != nil { log.Printf("failed to wait for std_modules: %v\n", err) os.Exit(1) @@ -81,6 +77,12 @@ func init() { }() } +func shutdownStdModuleProcess() { + if err := stdModulesStdin.Close(); err != nil { + fmt.Fprintf(os.Stderr, "error closing std module: %v", err) + } +} + func isStdModule(m module) (bool, error) { if _, seen := stdModulesSeen[m.Name]; seen { return true, nil From 5d1714c1ce91e14add3af11c5122620978aa6646 Mon Sep 17 00:00:00 2001 From: Zhongpeng Lin Date: Mon, 19 Jun 2023 21:59:58 -0700 Subject: [PATCH 2/2] Embedding BaseLifecycleManager --- gazelle/go.mod | 9 ++++++--- gazelle/go.sum | 22 ++++++++++++---------- gazelle/python/lifecycle.go | 10 +++++----- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/gazelle/go.mod b/gazelle/go.mod index 94f19e801f..1d1cee75f5 100644 --- a/gazelle/go.mod +++ b/gazelle/go.mod @@ -3,7 +3,9 @@ module github.com/bazelbuild/rules_python/gazelle go 1.19 require ( - github.com/bazelbuild/buildtools v0.0.0-20221004120235-7186f635531b + github.com/bazelbuild/bazel-gazelle v0.31.1 + github.com/bazelbuild/buildtools v0.0.0-20230510134650-37bd1811516d + github.com/bazelbuild/rules_go v0.39.1 github.com/bmatcuk/doublestar v1.3.4 github.com/emirpasic/gods v1.18.1 github.com/ghodss/yaml v1.0.0 @@ -12,6 +14,7 @@ require ( require ( github.com/google/go-cmp v0.5.9 // indirect - golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect - golang.org/x/tools v0.1.12 // indirect + golang.org/x/mod v0.10.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/tools v0.9.1 // indirect ) diff --git a/gazelle/go.sum b/gazelle/go.sum index ed8ceae5ec..ba2c8bf688 100644 --- a/gazelle/go.sum +++ b/gazelle/go.sum @@ -1,11 +1,11 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/bazelbuild/bazel-gazelle v0.27.0 h1:+/ZhUxlDy4XnyMIGeKkbRZoIGssy1eO51GijwIvvuwE= -github.com/bazelbuild/bazel-gazelle v0.27.0/go.mod h1:2K6B42/loq8ext4JObmam4gTYx4En1MUSzHFKQF8hPM= -github.com/bazelbuild/buildtools v0.0.0-20221004120235-7186f635531b h1:jhiMzJ+8unnLRtV8rpbWBFE9pFNzIqgUTyZU5aA++w8= -github.com/bazelbuild/buildtools v0.0.0-20221004120235-7186f635531b/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo= -github.com/bazelbuild/rules_go v0.35.0 h1:ViPR65vOrg74JKntAUFY6qZkheBKGB6to7wFd8gCRU4= -github.com/bazelbuild/rules_go v0.35.0/go.mod h1:ahciH68Viyxtm/gvCQplaAiu8buhf/b+gWswcPjFixI= +github.com/bazelbuild/bazel-gazelle v0.31.1 h1:ROyUyUHzoEdvoOs1e0haxJx1l5EjZX6AOqiKdVlaBbg= +github.com/bazelbuild/bazel-gazelle v0.31.1/go.mod h1:Ul0pqz50f5wxz0QNzsZ+mrEu4AVAVJZEB5xLnHgIG9c= +github.com/bazelbuild/buildtools v0.0.0-20230510134650-37bd1811516d h1:Fl1FfItZp34QIQmmDTbZXHB5XA6JfbNNfH7tRRGWvQo= +github.com/bazelbuild/buildtools v0.0.0-20230510134650-37bd1811516d/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo= +github.com/bazelbuild/rules_go v0.39.1 h1:wkJLUDx59dntWMghuL8++GteoU1To6sRoKJXuyFtmf8= +github.com/bazelbuild/rules_go v0.39.1/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU= github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -45,6 +45,8 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -56,15 +58,15 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= diff --git a/gazelle/python/lifecycle.go b/gazelle/python/lifecycle.go index f38383a3f7..592b322a3c 100644 --- a/gazelle/python/lifecycle.go +++ b/gazelle/python/lifecycle.go @@ -19,11 +19,11 @@ import ( "github.com/bazelbuild/bazel-gazelle/language" ) -var _ language.FinishableLanguage = (*LifeCycleManager)(nil) - -type LifeCycleManager struct{} +type LifeCycleManager struct { + language.BaseLifecycleManager +} -func (l *LifeCycleManager) Init(ctx context.Context) { +func (l *LifeCycleManager) Before(ctx context.Context) { startParserProcess(ctx) startStdModuleProcess(ctx) } @@ -32,6 +32,6 @@ func (l *LifeCycleManager) DoneGeneratingRules() { shutdownParserProcess() } -func (l *LifeCycleManager) DoneResolvingDeps() { +func (l *LifeCycleManager) AfterResolvingDeps(ctx context.Context) { shutdownStdModuleProcess() }