diff --git a/builtin/builtin.go b/builtin/builtin.go index a1e10126..83502849 100644 --- a/builtin/builtin.go +++ b/builtin/builtin.go @@ -162,7 +162,16 @@ func init() { "Warning": py.Warning, "ZeroDivisionError": py.ZeroDivisionError, } - py.NewModule("builtins", builtin_doc, methods, globals) + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "builtins", + Doc: builtin_doc, + Flags: py.ShareModule, + }, + Methods: methods, + Globals: globals, + }) } const print_doc = `print(value, ..., sep=' ', end='\\n', file=sys.stdout, flush=False) @@ -178,18 +187,22 @@ func builtin_print(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Obje var ( sepObj py.Object = py.String(" ") endObj py.Object = py.String("\n") - file py.Object = py.MustGetModule("sys").Globals["stdout"] flush py.Object ) + sysModule, err := self.(*py.Module).Context.GetModule("sys") + if err != nil { + return nil, err + } + stdout := sysModule.Globals["stdout"] kwlist := []string{"sep", "end", "file", "flush"} - err := py.ParseTupleAndKeywords(nil, kwargs, "|ssOO:print", kwlist, &sepObj, &endObj, &file, &flush) + err = py.ParseTupleAndKeywords(nil, kwargs, "|ssOO:print", kwlist, &sepObj, &endObj, &stdout, &flush) if err != nil { return nil, err } sep := sepObj.(py.String) end := endObj.(py.String) - write, err := py.GetAttrString(file, "write") + write, err := py.GetAttrString(stdout, "write") if err != nil { return nil, err } @@ -219,7 +232,7 @@ func builtin_print(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Obje } if shouldFlush, _ := py.MakeBool(flush); shouldFlush == py.True { - fflush, err := py.GetAttrString(file, "flush") + fflush, err := py.GetAttrString(stdout, "flush") if err == nil { return py.Call(fflush, nil, nil) } @@ -449,7 +462,7 @@ func builtin___build_class__(self py.Object, args py.Tuple, kwargs py.StringDict } // fmt.Printf("Calling %v with %v and %v\n", fn.Name, fn.Globals, ns) // fmt.Printf("Code = %#v\n", fn.Code) - cell, err = py.VmRun(fn.Globals, ns, fn.Code, fn.Closure) + cell, err = fn.Context.RunCode(fn.Code, fn.Globals, ns, fn.Closure) if err != nil { return nil, err } @@ -750,7 +763,7 @@ func builtin_compile(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Ob } // if dont_inherit.(py.Int) != 0 { - // PyEval_MergeCompilerFlags(&cf) + // PyEval_MergeCompilerFlags(&cf) // } // switch string(startstr.(py.String)) { @@ -782,7 +795,7 @@ func builtin_compile(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Ob return nil, err } // result = py.CompileStringExFlags(str, filename, start[mode], &cf, optimize) - result, err = compile.Compile(str, string(filename.(py.String)), string(startstr.(py.String)), int(supplied_flags.(py.Int)), dont_inherit.(py.Int) != 0) + result, err = compile.Compile(str, string(filename.(py.String)), py.CompileMode(startstr.(py.String)), int(supplied_flags.(py.Int)), dont_inherit.(py.Int) != 0) if err != nil { return nil, err } diff --git a/compile/compile.go b/compile/compile.go index d0f59c45..775717a9 100644 --- a/compile/compile.go +++ b/compile/compile.go @@ -89,33 +89,36 @@ func init() { py.Compile = Compile } -// Compile(source, filename, mode, flags, dont_inherit) -> code object +// Compile(src, srcDesc, compileMode, flags, dont_inherit) -> code object // // Compile the source string (a Python module, statement or expression) -// into a code object that can be executed by exec() or eval(). -// The filename will be used for run-time error messages. -// The mode must be 'exec' to compile a module, 'single' to compile a -// single (interactive) statement, or 'eval' to compile an expression. +// into a code object that can be executed. +// +// srcDesc is used for run-time error messages and is typically a file system pathname, +// +// See py.CompileMode for compile mode options. +// // The flags argument, if present, controls which future statements influence // the compilation of the code. +// // The dont_inherit argument, if non-zero, stops the compilation inheriting // the effects of any future statements in effect in the code calling // compile; if absent or zero these statements do influence the compilation, // in addition to any features explicitly specified. -func Compile(str, filename, mode string, futureFlags int, dont_inherit bool) (py.Object, error) { +func Compile(src, srcDesc string, mode py.CompileMode, futureFlags int, dont_inherit bool) (*py.Code, error) { // Parse Ast - Ast, err := parser.ParseString(str, mode) + Ast, err := parser.ParseString(src, mode) if err != nil { return nil, err } // Make symbol table - SymTable, err := symtable.NewSymTable(Ast, filename) + SymTable, err := symtable.NewSymTable(Ast, srcDesc) if err != nil { return nil, err } c := newCompiler(nil, compilerScopeModule) - c.Filename = filename - err = c.compileAst(Ast, filename, futureFlags, dont_inherit, SymTable) + c.Filename = srcDesc + err = c.compileAst(Ast, srcDesc, futureFlags, dont_inherit, SymTable) if err != nil { return nil, err } diff --git a/compile/compile_data_test.go b/compile/compile_data_test.go index edc53720..f0ee0422 100644 --- a/compile/compile_data_test.go +++ b/compile/compile_data_test.go @@ -12,7 +12,7 @@ import ( var compileTestData = []struct { in string - mode string // exec, eval or single + mode py.CompileMode out *py.Code exceptionType *py.Type errString string diff --git a/compile/compile_test.go b/compile/compile_test.go index 72d4a3f6..aee3caac 100644 --- a/compile/compile_test.go +++ b/compile/compile_test.go @@ -163,7 +163,7 @@ func EqCode(t *testing.T, name string, a, b *py.Code) { func TestCompile(t *testing.T) { for _, test := range compileTestData { // log.Printf(">>> %s", test.in) - codeObj, err := Compile(test.in, "", test.mode, 0, true) + code, err := Compile(test.in, "", test.mode, 0, true) if err != nil { if test.exceptionType == nil { t.Errorf("%s: Got exception %v when not expecting one", test.in, err) @@ -196,17 +196,12 @@ func TestCompile(t *testing.T) { } } else { if test.out == nil { - if codeObj != nil { - t.Errorf("%s: Expecting nil *py.Code but got %T", test.in, codeObj) + if code != nil { + t.Errorf("%s: Expecting nil *py.Code but got %T", test.in, code) } } else { - code, ok := codeObj.(*py.Code) - if !ok { - t.Errorf("%s: Expecting *py.Code but got %T", test.in, codeObj) - } else { - //t.Logf("Testing %q", test.in) - EqCode(t, test.in, test.out, code) - } + //t.Logf("Testing %q", test.in) + EqCode(t, test.in, test.out, code) } } } diff --git a/go.mod b/go.mod index 095bd34f..163319a8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,6 @@ module github.com/go-python/gpython go 1.16 require ( - github.com/gopherjs/gopherwasm v1.0.0 - github.com/peterh/liner v1.1.0 + github.com/gopherjs/gopherwasm v1.1.0 + github.com/peterh/liner v1.2.2 ) diff --git a/go.sum b/go.sum index c17fb31d..58d8376b 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,13 @@ github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSf github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherwasm v1.0.0 h1:32nge/RlujS1Im4HNCJPp0NbBOAeBXFuT1KonUuLl+Y= github.com/gopherjs/gopherwasm v1.0.0/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI= +github.com/gopherjs/gopherwasm v1.1.0 h1:fA2uLoctU5+T3OhOn2vYP0DVT6pxc7xhTlBB1paATqQ= +github.com/gopherjs/gopherwasm v1.1.0/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI= github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/peterh/liner v1.1.0 h1:f+aAedNJA6uk7+6rXsYBnhdo4Xux7ESLe+kcuVUF5os= github.com/peterh/liner v1.1.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw= +github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI= +golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 h1:kwrAHlwJ0DUBZwQ238v+Uod/3eZ8B2K5rYsUHBQvzmI= +golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/importlib/importlib.go b/importlib/importlib.go index e1d307db..58ccb966 100644 --- a/importlib/importlib.go +++ b/importlib/importlib.go @@ -7,15 +7,18 @@ package py import ( - "log" - - "github.com/go-python/gpython/marshal" + "github.com/go-python/gpython/py" ) // Load the frozen module func init() { - _, err := marshal.LoadFrozenModule("importlib", data) - log.Fatalf("Failed to load importlib: %v", err) + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "importlib", + }, + CodeBuf: data, + }) } // Auto-generated by Modules/_freeze_importlib.c diff --git a/main.go b/main.go index 8148056b..9743bc0f 100644 --- a/main.go +++ b/main.go @@ -12,22 +12,13 @@ import ( "runtime" "runtime/pprof" - _ "github.com/go-python/gpython/builtin" + "github.com/go-python/gpython/repl" "github.com/go-python/gpython/repl/cli" - //_ "github.com/go-python/gpython/importlib" - "io/ioutil" "log" "os" - "strings" - "github.com/go-python/gpython/compile" - "github.com/go-python/gpython/marshal" - _ "github.com/go-python/gpython/math" "github.com/go-python/gpython/py" - pysys "github.com/go-python/gpython/sys" - _ "github.com/go-python/gpython/time" - "github.com/go-python/gpython/vm" ) // Globals @@ -48,33 +39,14 @@ Full options: flag.PrintDefaults() } -// Exit with the message -func fatal(message string, args ...interface{}) { - if !strings.HasSuffix(message, "\n") { - message += "\n" - } - syntaxError() - fmt.Fprintf(os.Stderr, message, args...) - os.Exit(1) -} - func main() { flag.Usage = syntaxError flag.Parse() args := flag.Args() - py.MustGetModule("sys").Globals["argv"] = pysys.MakeArgv(args) - if len(args) == 0 { - - fmt.Printf("Python 3.4.0 (%s, %s)\n", commit, date) - fmt.Printf("[Gpython %s]\n", version) - fmt.Printf("- os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) - fmt.Printf("- go version: %s\n", runtime.Version()) - cli.RunREPL() - return - } - prog := args[0] - // fmt.Printf("Running %q\n", prog) + opts := py.DefaultContextOpts() + opts.SysArgs = flag.Args() + ctx := py.NewContext(opts) if *cpuprofile != "" { f, err := os.Create(*cpuprofile) @@ -88,41 +60,23 @@ func main() { defer pprof.StopCPUProfile() } - // FIXME should be using ImportModuleLevelObject() here - f, err := os.Open(prog) - if err != nil { - log.Fatalf("Failed to open %q: %v", prog, err) - } - var obj py.Object - if strings.HasSuffix(prog, ".pyc") { - obj, err = marshal.ReadPyc(f) - if err != nil { - log.Fatalf("Failed to marshal %q: %v", prog, err) - } - } else if strings.HasSuffix(prog, ".py") { - str, err := ioutil.ReadAll(f) - if err != nil { - log.Fatalf("Failed to read %q: %v", prog, err) - } - obj, err = compile.Compile(string(str), prog, "exec", 0, true) + // IF no args, enter REPL mode + if len(args) == 0 { + + fmt.Printf("Python 3.4.0 (%s, %s)\n", commit, date) + fmt.Printf("[Gpython %s]\n", version) + fmt.Printf("- os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) + fmt.Printf("- go version: %s\n", runtime.Version()) + + replCtx := repl.New(ctx) + cli.RunREPL(replCtx) + + } else { + _, err := py.RunFile(ctx, args[0], py.CompileOpts{}, nil) if err != nil { - log.Fatalf("Can't compile %q: %v", prog, err) + py.TracebackDump(err) + log.Fatal(err) } - } else { - log.Fatalf("Can't execute %q", prog) - } - if err = f.Close(); err != nil { - log.Fatalf("Failed to close %q: %v", prog, err) - } - code := obj.(*py.Code) - module := py.NewModule("__main__", "", nil, nil) - module.Globals["__file__"] = py.String(prog) - res, err := vm.Run(module.Globals, module.Globals, code, nil) - if err != nil { - py.TracebackDump(err) - log.Fatal(err) } - // fmt.Printf("Return = %v\n", res) - _ = res } diff --git a/marshal/marshal.go b/marshal/marshal.go index c29c97bf..937008cc 100644 --- a/marshal/marshal.go +++ b/marshal/marshal.go @@ -6,7 +6,6 @@ package marshal import ( - "bytes" "encoding/binary" "errors" "fmt" @@ -15,7 +14,6 @@ import ( "strconv" "github.com/go-python/gpython/py" - "github.com/go-python/gpython/vm" ) const ( @@ -454,23 +452,6 @@ func ReadPyc(r io.Reader) (obj py.Object, err error) { return ReadObject(r) } -// Unmarshals a frozen module -func LoadFrozenModule(name string, data []byte) (*py.Module, error) { - r := bytes.NewBuffer(data) - obj, err := ReadObject(r) - if err != nil { - return nil, err - } - code := obj.(*py.Code) - module := py.NewModule(name, "", nil, nil) - _, err = vm.Run(module.Globals, module.Globals, code, nil) - if err != nil { - py.TracebackDump(err) - return nil, err - } - return module, nil -} - const dump_doc = `dump(value, file[, version]) Write the value on the open file. The value must be a supported type. @@ -634,5 +615,14 @@ func init() { globals := py.StringDict{ "version": py.Int(MARSHAL_VERSION), } - py.NewModule("marshal", module_doc, methods, globals) + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "marshal", + Doc: module_doc, + Flags: py.ShareModule, + }, + Globals: globals, + Methods: methods, + }) } diff --git a/math/math.go b/math/math.go index 88d60fb3..63ecec00 100644 --- a/math/math.go +++ b/math/math.go @@ -1333,9 +1333,17 @@ func init() { py.MustNewMethod("trunc", math_trunc, 0, math_trunc_doc), py.MustNewMethod("to_ulps", math_to_ulps, 0, math_to_ulps_doc), } - globals := py.StringDict{ - "pi": py.Float(math.Pi), - "e": py.Float(math.E), - } - py.NewModule("math", math_doc, methods, globals) + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "math", + Doc: math_doc, + Flags: py.ShareModule, + }, + Methods: methods, + Globals: py.StringDict{ + "pi": py.Float(math.Pi), + "e": py.Float(math.E), + }, + }) } diff --git a/modules/runtime.go b/modules/runtime.go new file mode 100644 index 00000000..91fc7092 --- /dev/null +++ b/modules/runtime.go @@ -0,0 +1,287 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modules + +import ( + "bytes" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "sync" + + "github.com/go-python/gpython/marshal" + "github.com/go-python/gpython/py" + "github.com/go-python/gpython/vm" + + _ "github.com/go-python/gpython/builtin" + _ "github.com/go-python/gpython/math" + _ "github.com/go-python/gpython/sys" + _ "github.com/go-python/gpython/time" +) + +func init() { + // Assign the base-level py.Context creation function while also preventing an import cycle. + py.NewContext = NewContext +} + +// context implements interface py.Context +type context struct { + store *py.ModuleStore + opts py.ContextOpts + closeOnce sync.Once + closing bool + closed bool + running sync.WaitGroup + done chan struct{} +} + +// NewContext creates a new gpython interpreter instance context. +// +// See type Context interface for info. +func NewContext(opts py.ContextOpts) py.Context { + ctx := &context{ + opts: opts, + done: make(chan struct{}), + closing: false, + closed: false, + } + + ctx.store = py.NewModuleStore() + + py.Import(ctx, "builtins", "sys") + + sys_mod := ctx.Store().MustGetModule("sys") + sys_mod.Globals["argv"] = py.NewListFromStrings(opts.SysArgs) + sys_mod.Globals["path"] = py.NewListFromStrings(opts.SysPaths) + + return ctx +} + +// ModuleInit digests a ModuleImpl, compiling and marshalling as needed, creating a new Module instance in this Context. +func (ctx *context) ModuleInit(impl *py.ModuleImpl) (*py.Module, error) { + err := ctx.pushBusy() + defer ctx.popBusy() + if err != nil { + return nil, err + } + + if impl.Code == nil && len(impl.CodeSrc) > 0 { + impl.Code, err = py.Compile(string(impl.CodeSrc), impl.Info.FileDesc, py.ExecMode, 0, true) + if err != nil { + return nil, err + } + } + + if impl.Code == nil && len(impl.CodeBuf) > 0 { + codeBuf := bytes.NewBuffer(impl.CodeBuf) + obj, err := marshal.ReadObject(codeBuf) + if err != nil { + return nil, err + } + impl.Code, _ = obj.(*py.Code) + if impl.Code == nil { + return nil, py.ExceptionNewf(py.AssertionError, "Embedded code did not produce a py.Code object") + } + } + + module, err := ctx.Store().NewModule(ctx, impl) + if err != nil { + return nil, err + } + + if impl.Code != nil { + _, err = ctx.RunCode(impl.Code, module.Globals, module.Globals, nil) + if err != nil { + return nil, err + } + } + + return module, nil +} + +func (ctx *context) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.CompileOut, error) { + err := ctx.pushBusy() + defer ctx.popBusy() + if err != nil { + return py.CompileOut{}, err + } + + tryPaths := defaultPaths + if opts.UseSysPaths { + tryPaths = ctx.Store().MustGetModule("sys").Globals["path"].(*py.List).Items + } + + out := py.CompileOut{} + + err = resolveRunPath(pathname, opts, tryPaths, func(fpath string) (bool, error) { + + stat, err := os.Stat(fpath) + if err == nil && stat.IsDir() { + // FIXME this is a massive simplification! + fpath = path.Join(fpath, "__init__.py") + _, err = os.Stat(fpath) + } + + ext := strings.ToLower(filepath.Ext(fpath)) + if ext == "" && os.IsNotExist(err) { + fpath += ".py" + ext = ".py" + _, err = os.Stat(fpath) + } + + // Keep searching while we get FNFs, stop on an error + if err != nil { + if os.IsNotExist(err) { + return true, nil + } + err = py.ExceptionNewf(py.OSError, "Error accessing %q: %v", fpath, err) + return false, err + } + + switch ext { + case ".py": + var pySrc []byte + pySrc, err = ioutil.ReadFile(fpath) + if err != nil { + return false, py.ExceptionNewf(py.OSError, "Error reading %q: %v", fpath, err) + } + + out.Code, err = py.Compile(string(pySrc), fpath, py.ExecMode, 0, true) + if err != nil { + return false, err + } + out.SrcPathname = fpath + case ".pyc": + file, err := os.Open(fpath) + if err != nil { + return false, py.ExceptionNewf(py.OSError, "Error opening %q: %v", fpath, err) + } + defer file.Close() + codeObj, err := marshal.ReadPyc(file) + if err != nil { + return false, py.ExceptionNewf(py.ImportError, "Failed to marshal %q: %v", fpath, err) + } + out.Code, _ = codeObj.(*py.Code) + out.PycPathname = fpath + } + + out.FileDesc = fpath + return false, nil + }) + + if out.Code == nil && err == nil { + err = py.ExceptionNewf(py.AssertionError, "Missing code object") + } + + if err != nil { + return py.CompileOut{}, err + } + + return out, nil +} + +func (ctx *context) pushBusy() error { + if ctx.closed { + return py.ExceptionNewf(py.RuntimeError, "Context closed") + } + ctx.running.Add(1) + return nil +} + +func (ctx *context) popBusy() { + ctx.running.Done() +} + +// Close -- see type py.Context +func (ctx *context) Close() error { + ctx.closeOnce.Do(func() { + ctx.closing = true + ctx.running.Wait() + ctx.closed = true + + // Give each module a chance to release resources + ctx.store.OnContextClosed() + close(ctx.done) + }) + return nil +} + +// Done -- see type py.Context +func (ctx *context) Done() <-chan struct{} { + return ctx.done +} + +var defaultPaths = []py.Object{ + py.String("."), +} + +func resolveRunPath(runPath string, opts py.CompileOpts, pathObjs []py.Object, tryPath func(pyPath string) (bool, error)) error { + runPath = strings.TrimSuffix(runPath, "/") + + var ( + err error + cwd string + cont = true + ) + + for _, pathObj := range pathObjs { + pathStr, ok := pathObj.(py.String) + if !ok { + continue + } + + // If an absolute path, just try that. + // Otherwise, check from the passed current dir then check from the current working dir. + fpath := path.Join(string(pathStr), runPath) + if filepath.IsAbs(fpath) { + cont, err = tryPath(fpath) + } else { + if len(opts.CurDir) > 0 { + subPath := path.Join(opts.CurDir, fpath) + cont, err = tryPath(subPath) + } + if cont && err == nil { + if cwd == "" { + cwd, _ = os.Getwd() + } + subPath := path.Join(cwd, fpath) + cont, err = tryPath(subPath) + } + } + if !cont { + break + } + } + + if err != nil { + return err + } + + if cont { + return py.ExceptionNewf(py.FileNotFoundError, "Failed to resolve %q", runPath) + } + + return err +} + +func (ctx *context) RunCode(code *py.Code, globals, locals py.StringDict, closure py.Tuple) (py.Object, error) { + err := ctx.pushBusy() + defer ctx.popBusy() + if err != nil { + return nil, err + } + + return vm.EvalCode(ctx, code, globals, locals, nil, nil, nil, nil, closure) +} + +func (ctx *context) GetModule(moduleName string) (*py.Module, error) { + return ctx.store.GetModule(moduleName) +} + +func (ctx *context) Store() *py.ModuleStore { + return ctx.store +} diff --git a/parser/grammar_test.go b/parser/grammar_test.go index d3d48a50..a99beb95 100644 --- a/parser/grammar_test.go +++ b/parser/grammar_test.go @@ -21,7 +21,7 @@ var debugLevel = flag.Int("debugLevel", 0, "Debug level 0-4") func TestGrammar(t *testing.T) { SetDebug(*debugLevel) for _, test := range grammarTestData { - Ast, err := ParseString(test.in, test.mode) + Ast, err := ParseString(test.in, py.CompileMode(test.mode)) if err != nil { if test.exceptionType == nil { t.Errorf("%s: Got exception %v when not expecting one", test.in, err) diff --git a/parser/lexer.go b/parser/lexer.go index 6f9f1726..76847893 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -62,7 +62,7 @@ type yyLex struct { // can be 'exec' if source consists of a sequence of statements, // 'eval' if it consists of a single expression, or 'single' if it // consists of a single interactive statement -func NewLex(r io.Reader, filename string, mode string) (*yyLex, error) { +func NewLex(r io.Reader, filename string, mode py.CompileMode) (*yyLex, error) { x := &yyLex{ reader: bufio.NewReader(r), filename: filename, @@ -70,12 +70,12 @@ func NewLex(r io.Reader, filename string, mode string) (*yyLex, error) { state: readString, } switch mode { - case "exec": + case py.ExecMode: x.queue(FILE_INPUT) x.exec = true - case "eval": + case py.EvalMode: x.queue(EVAL_INPUT) - case "single": + case py.SingleMode: x.queue(SINGLE_INPUT) x.interactive = true default: @@ -933,7 +933,7 @@ func SetDebug(level int) { } // Parse a file -func Parse(in io.Reader, filename string, mode string) (mod ast.Mod, err error) { +func Parse(in io.Reader, filename string, mode py.CompileMode) (mod ast.Mod, err error) { lex, err := NewLex(in, filename, mode) if err != nil { return nil, err @@ -952,12 +952,12 @@ func Parse(in io.Reader, filename string, mode string) (mod ast.Mod, err error) } // Parse a string -func ParseString(in string, mode string) (ast.Ast, error) { +func ParseString(in string, mode py.CompileMode) (ast.Ast, error) { return Parse(bytes.NewBufferString(in), "", mode) } // Lex a file only, returning a sequence of tokens -func Lex(in io.Reader, filename string, mode string) (lts LexTokens, err error) { +func Lex(in io.Reader, filename string, mode py.CompileMode) (lts LexTokens, err error) { lex, err := NewLex(in, filename, mode) if err != nil { return nil, err @@ -984,6 +984,6 @@ func Lex(in io.Reader, filename string, mode string) (lts LexTokens, err error) } // Lex a string -func LexString(in string, mode string) (lts LexTokens, err error) { +func LexString(in string, mode py.CompileMode) (lts LexTokens, err error) { return Lex(bytes.NewBufferString(in), "", mode) } diff --git a/parser/lexer_test.go b/parser/lexer_test.go index 18e20e5a..ab252088 100644 --- a/parser/lexer_test.go +++ b/parser/lexer_test.go @@ -196,7 +196,7 @@ func TestLex(t *testing.T) { for _, test := range []struct { in string errString string - mode string + mode py.CompileMode lts LexTokens }{ {"", "", "exec", LexTokens{ diff --git a/py/dict.go b/py/dict.go index d3df854c..4f277c47 100644 --- a/py/dict.go +++ b/py/dict.go @@ -217,3 +217,9 @@ func (a StringDict) M__contains__(other Object) (Object, error) { } return False, nil } + +func (d StringDict) GetDict() StringDict { + return d +} + +var _ IGetDict = (*StringDict)(nil) diff --git a/py/frame.go b/py/frame.go index f91438a0..ce595d8a 100644 --- a/py/frame.go +++ b/py/frame.go @@ -26,6 +26,7 @@ type TryBlock struct { // A python Frame object type Frame struct { // Back *Frame // previous frame, or nil + Context Context // host module (state) context Code *Code // code segment Builtins StringDict // builtin symbol table Globals StringDict // global symbol table @@ -77,7 +78,7 @@ func (o *Frame) Type() *Type { } // Make a new frame for a code object -func NewFrame(globals, locals StringDict, code *Code, closure Tuple) *Frame { +func NewFrame(ctx Context, globals, locals StringDict, code *Code, closure Tuple) *Frame { nlocals := int(code.Nlocals) ncells := len(code.Cellvars) nfrees := len(code.Freevars) @@ -90,12 +91,13 @@ func NewFrame(globals, locals StringDict, code *Code, closure Tuple) *Frame { cellAndFreeVars := allocation[nlocals:varsize] return &Frame{ + Context: ctx, Globals: globals, Locals: locals, Code: code, LocalVars: localVars, CellAndFreeVars: cellAndFreeVars, - Builtins: Builtins.Globals, + Builtins: ctx.Store().Builtins.Globals, Localsplus: allocation, Stack: make([]Object, 0, code.Stacksize), } diff --git a/py/function.go b/py/function.go index 28eace5d..2c37499d 100644 --- a/py/function.go +++ b/py/function.go @@ -18,6 +18,7 @@ package py // A python Function object type Function struct { Code *Code // A code object, the __code__ attribute + Context Context // Host VM context Globals StringDict // A dictionary (other mappings won't do) Defaults Tuple // NULL or a tuple KwDefaults StringDict // NULL or a dict @@ -26,7 +27,6 @@ type Function struct { Name string // The __name__ attribute, a string object Dict StringDict // The __dict__ attribute, a dict or NULL Weakreflist List // List of weak references - Module Object // The __module__ attribute, can be anything Annotations StringDict // Annotations, a dict or NULL Qualname string // The qualified name } @@ -56,9 +56,8 @@ func (f *Function) GetDict() StringDict { // attribute. qualname should be a unicode object or ""; if "", the // __qualname__ attribute is set to the same value as its __name__ // attribute. -func NewFunction(code *Code, globals StringDict, qualname string) *Function { +func NewFunction(ctx Context, code *Code, globals StringDict, qualname string) *Function { var doc Object - var module Object = None if len(code.Consts) >= 1 { doc = code.Consts[0] if _, ok := doc.(String); !ok { @@ -68,29 +67,24 @@ func NewFunction(code *Code, globals StringDict, qualname string) *Function { doc = None } - // __module__: If module name is in globals, use it. Otherwise, use None. - if moduleobj, ok := globals["__name__"]; ok { - module = moduleobj - } - if qualname == "" { qualname = code.Name } return &Function{ Code: code, + Context: ctx, Qualname: qualname, Globals: globals, Name: code.Name, Doc: doc, - Module: module, Dict: make(StringDict), } } // Call a function func (f *Function) M__call__(args Tuple, kwargs StringDict) (Object, error) { - result, err := VmEvalCodeEx(f.Code, f.Globals, NewStringDict(), args, kwargs, f.Defaults, f.KwDefaults, f.Closure) + result, err := VmEvalCode(f.Context, f.Code, f.Globals, NewStringDict(), args, kwargs, f.Defaults, f.KwDefaults, f.Closure) if err != nil { return nil, err } diff --git a/py/import.go b/py/import.go index 3cd1fd64..0eaae921 100644 --- a/py/import.go +++ b/py/import.go @@ -7,17 +7,19 @@ package py import ( - "io/ioutil" - "os" "path" - "path/filepath" "strings" ) -var ( - // This will become sys.path one day ;-) - modulePath = []string{"", "/usr/lib/python3.4", "/usr/local/lib/python3.4/dist-packages", "/usr/lib/python3/dist-packages"} -) +func Import(ctx Context, names ...string) error { + for _, name := range names { + _, err := ImportModuleLevelObject(ctx, name, nil, nil, nil, 0) + if err != nil { + return err + } + } + return nil +} // The workings of __import__ // @@ -78,9 +80,18 @@ var ( // // Changed in version 3.3: Negative values for level are no longer // supported (which also changes the default value to 0). -func ImportModuleLevelObject(name string, globals, locals StringDict, fromlist Tuple, level int) (Object, error) { +func ImportModuleLevelObject(ctx Context, name string, globals, locals StringDict, fromlist Tuple, level int) (Object, error) { // Module already loaded - return that - if module, ok := modules[name]; ok { + if module, err := ctx.GetModule(name); err == nil { + return module, nil + } + + // See if the module is a registered embeddded module that has not been loaded into this ctx yet. + if impl := GetModuleImpl(name); impl != nil { + module, err := ctx.ModuleInit(impl) + if err != nil { + return nil, err + } return module, nil } @@ -88,74 +99,24 @@ func ImportModuleLevelObject(name string, globals, locals StringDict, fromlist T return nil, ExceptionNewf(SystemError, "Relative import not supported yet") } + // Convert import's dot separators into path seps parts := strings.Split(name, ".") - pathParts := path.Join(parts...) + srcPathname := path.Join(parts...) - for _, mpath := range modulePath { - if mpath == "" { - mpathObj, ok := globals["__file__"] - if !ok { - var err error - mpath, err = os.Getwd() - if err != nil { - return nil, err - } - } else { - mpath = path.Dir(string(mpathObj.(String))) - } - } - fullPath := path.Join(mpath, pathParts) - // FIXME Read pyc/pyo too - fullPath, err := filepath.Abs(fullPath) - if err != nil { - continue - } - if fi, err := os.Stat(fullPath); err == nil && fi.IsDir() { - // FIXME this is a massive simplification! - fullPath = path.Join(fullPath, "__init__.py") - } else { - fullPath += ".py" - } - // Check if file exists - if _, err := os.Stat(fullPath); err == nil { - str, err := ioutil.ReadFile(fullPath) - if err != nil { - return nil, ExceptionNewf(OSError, "Couldn't read %q: %v", fullPath, err) - } - codeObj, err := Compile(string(str), fullPath, "exec", 0, true) - if err != nil { - return nil, err - } - code, ok := codeObj.(*Code) - if !ok { - return nil, ExceptionNewf(ImportError, "Compile didn't return code object") - } - module := NewModule(name, "", nil, nil) - _, err = VmRun(module.Globals, module.Globals, code, nil) - if err != nil { - return nil, err - } - module.Globals["__file__"] = String(fullPath) - return module, nil - } + opts := CompileOpts{ + UseSysPaths: true, } - return nil, ExceptionNewf(ImportError, "No module named '%s'", name) - - // Convert to absolute path if relative - // Use __file__ from globals to work out what we are relative to - - // '' in path seems to mean use the current __file__ - - // Find a valid path which we need to check for the correct __init__.py in subdirectories etc - - // Look for .py and .pyc files - - // Make absolute module path too if we can for sys.modules - //How do we uniquely identify modules? + if fromFile, ok := globals["__file__"]; ok { + opts.CurDir = path.Dir(string(fromFile.(String))) + } - // SystemError: Parent module '' not loaded, cannot perform relative import + module, err := RunFile(ctx, srcPathname, opts, name) + if err != nil { + return nil, err + } + return module, nil } // Straight port of the python code @@ -163,7 +124,7 @@ func ImportModuleLevelObject(name string, globals, locals StringDict, fromlist T // This calls functins from _bootstrap.py which is a frozen module // // Too much functionality for the moment -func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Object, level int) (Object, error) { +func XImportModuleLevelObject(ctx Context, nameObj, given_globals, locals, given_fromlist Object, level int) (Object, error) { var abs_name string var builtins_import Object var final_mod Object @@ -175,6 +136,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj var ok bool var name string var err error + store := ctx.Store() // Make sure to use default values so as to not have // PyObject_CallMethodObjArgs() truncate the parameter list because of a @@ -239,7 +201,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj } } - if _, ok = modules[string(Package)]; !ok { + if _, err = ctx.GetModule(string(Package)); err != nil { return nil, ExceptionNewf(SystemError, "Parent module %q not loaded, cannot perform relative import", Package) } } else { // level == 0 */ @@ -277,16 +239,16 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj // From this point forward, goto error_with_unlock! builtins_import, ok = globals["__import__"] if !ok { - builtins_import, ok = Builtins.Globals["__import__"] + builtins_import, ok = store.Builtins.Globals["__import__"] if !ok { return nil, ExceptionNewf(ImportError, "__import__ not found") } } - mod, ok = modules[abs_name] - if mod == None { + mod, err = ctx.GetModule(abs_name) + if err != nil || mod == None { return nil, ExceptionNewf(ImportError, "import of %q halted; None in sys.modules", abs_name) - } else if ok { + } else if err == nil { var value Object var err error initializing := false @@ -306,7 +268,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj } if initializing { // _bootstrap._lock_unlock_module() releases the import lock */ - value, err = Importlib.Call("_lock_unlock_module", Tuple{String(abs_name)}, nil) + _, err = store.Importlib.Call("_lock_unlock_module", Tuple{String(abs_name)}, nil) if err != nil { return nil, err } @@ -318,7 +280,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj } } else { // _bootstrap._find_and_load() releases the import lock - mod, err = Importlib.Call("_find_and_load", Tuple{String(abs_name), builtins_import}, nil) + mod, err = store.Importlib.Call("_find_and_load", Tuple{String(abs_name), builtins_import}, nil) if err != nil { return nil, err } @@ -345,8 +307,8 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj cut_off := len(name) - len(front) abs_name_len := len(abs_name) to_return := abs_name[:abs_name_len-cut_off] - final_mod, ok = modules[to_return] - if !ok { + final_mod, err = ctx.GetModule(to_return) + if err != nil { return nil, ExceptionNewf(KeyError, "%q not in sys.modules as expected", to_return) } } @@ -354,7 +316,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj final_mod = mod } } else { - final_mod, err = Importlib.Call("_handle_fromlist", Tuple{mod, fromlist, builtins_import}, nil) + final_mod, err = store.Importlib.Call("_handle_fromlist", Tuple{mod, fromlist, builtins_import}, nil) if err != nil { return nil, err } @@ -376,7 +338,7 @@ error: } // The actual import code -func BuiltinImport(self Object, args Tuple, kwargs StringDict, currentGlobal StringDict) (Object, error) { +func BuiltinImport(ctx Context, self Object, args Tuple, kwargs StringDict, currentGlobal StringDict) (Object, error) { kwlist := []string{"name", "globals", "locals", "fromlist", "level"} var name Object var globals Object = currentGlobal @@ -391,5 +353,5 @@ func BuiltinImport(self Object, args Tuple, kwargs StringDict, currentGlobal Str if fromlist == None { fromlist = Tuple{} } - return ImportModuleLevelObject(string(name.(String)), globals.(StringDict), locals.(StringDict), fromlist.(Tuple), int(level.(Int))) + return ImportModuleLevelObject(ctx, string(name.(String)), globals.(StringDict), locals.(StringDict), fromlist.(Tuple), int(level.(Int))) } diff --git a/py/int.go b/py/int.go index c183991d..d3c84ab8 100644 --- a/py/int.go +++ b/py/int.go @@ -254,8 +254,7 @@ func (a Int) M__pos__() (Object, error) { func (a Int) M__abs__() (Object, error) { if a == IntMin { - abig, _ := ConvertToBigInt(a) - return abig.M__abs__() + return a.M__neg__() } if a < 0 { return -a, nil diff --git a/py/list.go b/py/list.go index eee7bf3e..28a118a1 100644 --- a/py/list.go +++ b/py/list.go @@ -41,8 +41,8 @@ func init() { ListType.Dict["sort"] = MustNewMethod("sort", func(self Object, args Tuple, kwargs StringDict) (Object, error) { const funcName = "sort" - var l *List - if self == None { + l, isList := self.(*List) + if !isList { // method called using `list.sort([], **kwargs)` var o Object err := UnpackTuple(args, nil, funcName, 1, 1, &o) @@ -60,7 +60,6 @@ func init() { if err != nil { return nil, err } - l = self.(*List) } err := SortInPlace(l, kwargs, funcName) if err != nil { @@ -121,6 +120,15 @@ func NewListFromItems(items []Object) *List { return l } +// Makes an argv into a tuple +func NewListFromStrings(items []string) *List { + l := NewListSized(len(items)) + for i, v := range items { + l.Items[i] = String(v) + } + return l +} + // Copy a list object func (l *List) Copy() *List { return NewListFromItems(l.Items) @@ -141,6 +149,13 @@ func (l *List) Extend(items []Object) { l.Items = append(l.Items, items...) } +// Extend the list with strings +func (l *List) ExtendWithStrings(items []string) { + for _, item := range items { + l.Items = append(l.Items, Object(String(item))) + } +} + // Extends the list with the sequence passed in func (l *List) ExtendSequence(seq Object) error { return Iterate(seq, func(item Object) bool { diff --git a/py/method.go b/py/method.go index 28e8c1d5..c8b0ab03 100644 --- a/py/method.go +++ b/py/method.go @@ -70,6 +70,8 @@ type Method struct { Flags int // Go function implementation method interface{} + // Parent module of this method + Module *Module } // Internal method types implemented within eval.go @@ -231,7 +233,7 @@ func newBoundMethod(name string, fn interface{}) (Object, error) { // Call a method func (m *Method) M__call__(args Tuple, kwargs StringDict) (Object, error) { - self := None // FIXME should be the module + self := Object(m.Module) if kwargs != nil { return m.CallWithKeywords(self, args, kwargs) } diff --git a/py/module.go b/py/module.go index ae2a2171..eccd12a7 100644 --- a/py/module.go +++ b/py/module.go @@ -6,24 +6,93 @@ package py -import "fmt" +import ( + "fmt" + "sync" +) + +type ModuleFlags int32 + +const ( + // ShareModule signals that an embedded module is threadsafe and read-only, meaninging it could be shared across multiple py.Context instances (for efficiency). + // Otherwise, ModuleImpl will create a separate py.Module instance for each py.Context that imports it. + // This should be used with extreme caution since any module mutation (write) means possible cross-context data corruption. + ShareModule ModuleFlags = 0x01 + + MainModuleName = "__main__" +) + +// ModuleInfo contains info and about a module and can specify flags that affect how it is imported into a py.Context +type ModuleInfo struct { + Name string // __name__ (if nil, "__main__" is used) + Doc string // __doc__ + FileDesc string // __file__ + Flags ModuleFlags +} -var ( +// ModuleImpl is used for modules that are ready to be imported into a py.Context. +// The model is that a ModuleImpl is read-only and instantiates a Module into a py.Context when imported. +// +// By convention, .Code is executed when a module instance is initialized. If nil, +// then .CodeBuf or .CodeSrc will be auto-compiled to set .Code. +type ModuleImpl struct { + Info ModuleInfo + Methods []*Method // Module-bound global method functions + Globals StringDict // Module-bound global variables + CodeSrc string // Module code body (source code to be compiled) + CodeBuf []byte // Module code body (serialized py.Code object) + Code *Code // Module code body + OnContextClosed func(*Module) // Callback for when a py.Context is closing to release resources +} + +// ModuleStore is a container of Module imported into an owning py.Context. +type ModuleStore struct { // Registry of installed modules - modules = make(map[string]*Module) + modules map[string]*Module // Builtin module Builtins *Module // this should be the frozen module importlib/_bootstrap.py generated // by Modules/_freeze_importlib.c into Python/importlib.h Importlib *Module -) +} + +func RegisterModule(module *ModuleImpl) { + gRuntime.RegisterModule(module) +} -// A python Module object +func GetModuleImpl(moduleName string) *ModuleImpl { + gRuntime.mu.RLock() + defer gRuntime.mu.RUnlock() + impl := gRuntime.ModuleImpls[moduleName] + return impl +} + +type Runtime struct { + mu sync.RWMutex + ModuleImpls map[string]*ModuleImpl +} + +var gRuntime = Runtime{ + ModuleImpls: make(map[string]*ModuleImpl), +} + +func (rt *Runtime) RegisterModule(impl *ModuleImpl) { + rt.mu.Lock() + defer rt.mu.Unlock() + rt.ModuleImpls[impl.Info.Name] = impl +} + +func NewModuleStore() *ModuleStore { + return &ModuleStore{ + modules: make(map[string]*Module), + } +} + +// Module is a runtime instance of a ModuleImpl bound to the py.Context that imported it. type Module struct { - Name string - Doc string - Globals StringDict - // dict Dict + ModuleImpl *ModuleImpl // Parent implementation of this Module instance + Globals StringDict // Initialized from ModuleImpl.Globals + Context Context // Parent context that "owns" this Module instance } var ModuleType = NewType("module", "module object") @@ -34,7 +103,11 @@ func (o *Module) Type() *Type { } func (m *Module) M__repr__() (Object, error) { - return String(fmt.Sprintf("", m.Name)), nil + name, ok := m.Globals["__name__"].(String) + if !ok { + name = "???" + } + return String(fmt.Sprintf("", string(name))), nil } // Get the Dict @@ -42,60 +115,82 @@ func (m *Module) GetDict() StringDict { return m.Globals } -// Define a new module -func NewModule(name, doc string, methods []*Method, globals StringDict) *Module { +// Calls a named method of a module +func (m *Module) Call(name string, args Tuple, kwargs StringDict) (Object, error) { + attr, err := GetAttrString(m, name) + if err != nil { + return nil, err + } + return Call(attr, args, kwargs) +} + +// Interfaces +var _ IGetDict = (*Module)(nil) + +// NewModule adds a new Module instance to this ModuleStore. +// Each given Method prototype is used to create a new "live" Method bound this the newly created Module. +// This func also sets appropriate module global attribs based on the given ModuleInfo (e.g. __name__). +func (store *ModuleStore) NewModule(ctx Context, impl *ModuleImpl) (*Module, error) { + name := impl.Info.Name + if name == "" { + name = MainModuleName + } m := &Module{ - Name: name, - Doc: doc, - Globals: globals.Copy(), + ModuleImpl: impl, + Globals: impl.Globals.Copy(), + Context: ctx, } // Insert the methods into the module dictionary - for _, method := range methods { - m.Globals[method.Name] = method + // Copy each method an insert each "live" with a ptr back to the module (which can also lead us to the host Context) + for _, method := range impl.Methods { + methodInst := new(Method) + *methodInst = *method + methodInst.Module = m + m.Globals[method.Name] = methodInst } // Set some module globals m.Globals["__name__"] = String(name) - m.Globals["__doc__"] = String(doc) + m.Globals["__doc__"] = String(impl.Info.Doc) m.Globals["__package__"] = None + if len(impl.Info.FileDesc) > 0 { + m.Globals["__file__"] = String(impl.Info.FileDesc) + } // Register the module - modules[name] = m + store.modules[name] = m // Make a note of some modules switch name { case "builtins": - Builtins = m + store.Builtins = m case "importlib": - Importlib = m + store.Importlib = m } - // fmt.Printf("Registering module %q\n", name) - return m + // fmt.Printf("Registered module %q\n", moduleName) + return m, nil } // Gets a module -func GetModule(name string) (*Module, error) { - m, ok := modules[name] +func (store *ModuleStore) GetModule(name string) (*Module, error) { + m, ok := store.modules[name] if !ok { - return nil, ExceptionNewf(ImportError, "Module %q not found", name) + return nil, ExceptionNewf(ImportError, "Module '%s' not found", name) } return m, nil } // Gets a module or panics -func MustGetModule(name string) *Module { - m, err := GetModule(name) +func (store *ModuleStore) MustGetModule(name string) *Module { + m, err := store.GetModule(name) if err != nil { panic(err) } return m } -// Calls a named method of a module -func (m *Module) Call(name string, args Tuple, kwargs StringDict) (Object, error) { - attr, err := GetAttrString(m, name) - if err != nil { - return nil, err +// OnContextClosed signals all module instances that the parent py.Context has closed +func (store *ModuleStore) OnContextClosed() { + for _, m := range store.modules { + if m.ModuleImpl.OnContextClosed != nil { + m.ModuleImpl.OnContextClosed(m) + } } - return Call(attr, args, kwargs) } - -// Interfaces -var _ IGetDict = (*Module)(nil) diff --git a/py/py.go b/py/py.go index 0c48ee7b..59d0737a 100644 --- a/py/py.go +++ b/py/py.go @@ -24,15 +24,10 @@ type IGoInt64 interface { GoInt64() (int64, error) } -// Some well known objects var ( // Set in vm/eval.go - to avoid circular import - VmRun func(globals, locals StringDict, code *Code, closure Tuple) (res Object, err error) - VmRunFrame func(frame *Frame) (res Object, err error) - VmEvalCodeEx func(co *Code, globals, locals StringDict, args []Object, kws StringDict, defs []Object, kwdefs StringDict, closure Tuple) (retval Object, err error) - - // See compile/compile.go - set to avoid circular import - Compile func(str, filename, mode string, flags int, dont_inherit bool) (Object, error) + VmEvalCode func(ctx Context, code *Code, globals, locals StringDict, args []Object, kws StringDict, defs []Object, kwdefs StringDict, closure Tuple) (retval Object, err error) + VmRunFrame func(frame *Frame) (res Object, err error) ) // Called to create a new instance of class cls. __new__() is a static method (special-cased so you need not declare it as such) that takes the class of which an instance was requested as its first argument. The remaining arguments are those passed to the object constructor expression (the call to the class). The return value of __new__() should be the new object instance (usually an instance of cls). diff --git a/py/run.go b/py/run.go new file mode 100644 index 00000000..5659232b --- /dev/null +++ b/py/run.go @@ -0,0 +1,179 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +type CompileMode string + +const ( + ExecMode CompileMode = "exec" // Compile a module + EvalMode CompileMode = "eval" // Compile an expression + SingleMode CompileMode = "single" // Compile a single (interactive) statement +) + +// Context is a gpython environment instance container, providing a high-level mechanism +// for multiple python interpreters to run concurrently without restriction. +// +// Context instances maintain completely independent environments, namely the modules that +// have been imported and their state. Modules imported into a Context are instanced +// from a parent ModuleImpl. For example, since Contexts each have their +// own sys module instance, each can set sys.path differently and independently. +// +// If you access a Context from multiple groutines, you are responsible that access is not concurrent, +// with the exception of Close() and Done(). +// +// See examples/multi-context and examples/embedding. +type Context interface { + + // Resolves then compiles (if applicable) the given file system pathname into a py.Code ready to be executed. + ResolveAndCompile(pathname string, opts CompileOpts) (CompileOut, error) + + // Creates a new py.Module instance and initializes ModuleImpl's code in the new module (if applicable). + ModuleInit(impl *ModuleImpl) (*Module, error) + + // RunCode is a lower-level invocation to execute the given py.Code. + // Blocks until execution is complete. + RunCode(code *Code, globals, locals StringDict, closure Tuple) (result Object, err error) + + // Returns the named module for this context (or an error if not found) + GetModule(moduleName string) (*Module, error) + + // Gereric access to this context's modules / state. + Store() *ModuleStore + + // Close signals this context is about to go out of scope and any internal resources should be released. + // Code execution on a py.Context that has been closed will result in an error. + Close() error + + // Done returns a signal that can be used to detect when this Context has fully closed / completed. + // If Close() is called while execution in progress, Done() will not signal until execution is complete. + Done() <-chan struct{} +} + +// CompileOpts specifies options for high-level compilation. +type CompileOpts struct { + UseSysPaths bool // If set, sys.path will be used to resolve relative pathnames + CurDir string // If non-empty, this is the path of the current working directory. If empty, os.Getwd() is used. +} + +// CompileOut the output of high-level compilation -- e.g. ResolveAndCompile() +type CompileOut struct { + SrcPathname string // Resolved pathname the .py file that was compiled (if applicable) + PycPathname string // Pathname of the .pyc file read and/or written (if applicable) + FileDesc string // Pathname to be used for a a module's "__file__" attrib + Code *Code // Read/Output code object ready for execution +} + +// DefaultCoreSysPaths specify default search paths for module sys +// This can be changed during runtime and plays nice with others using DefaultContextOpts() +var DefaultCoreSysPaths = []string{ + ".", + "lib", +} + +// DefaultAuxSysPaths are secondary default search paths for module sys. +// This can be changed during runtime and plays nice with others using DefaultContextOpts() +// They are separated from the default core paths since they the more likley thing you will want to completely replace when using gpython. +var DefaultAuxSysPaths = []string{ + "/usr/lib/python3.4", + "/usr/local/lib/python3.4/dist-packages", + "/usr/lib/python3/dist-packages", +} + +// ContextOpts specifies fundamental environment and input settings for creating a new py.Context +type ContextOpts struct { + SysArgs []string // sys.argv initializer + SysPaths []string // sys.path initializer +} + +var ( + // DefaultContextOpts should be the default opts created for py.NewContext. + // Calling this ensure that you future proof you code for suggested/default settings. + DefaultContextOpts = func() ContextOpts { + opts := ContextOpts{ + SysPaths: DefaultCoreSysPaths, + } + opts.SysPaths = append(opts.SysPaths, DefaultAuxSysPaths...) + return opts + } + + // NewContext is a high-level call to create a new gpython interpreter context. + // See type Context interface. + NewContext func(opts ContextOpts) Context + + // Compiles a python buffer into a py.Code object. + // Returns a py.Code object or otherwise an error. + Compile func(src, srcDesc string, mode CompileMode, flags int, dont_inherit bool) (*Code, error) +) + +// RunFile resolves the given pathname, compiles as needed, executes the code in the given module, and returns the Module to indicate success. +// +// See RunCode() for description of inModule. +func RunFile(ctx Context, pathname string, opts CompileOpts, inModule interface{}) (*Module, error) { + out, err := ctx.ResolveAndCompile(pathname, opts) + if err != nil { + return nil, err + } + + return RunCode(ctx, out.Code, out.FileDesc, inModule) +} + +// RunSrc compiles the given python buffer and executes it within the given module and returns the Module to indicate success. +// +// See RunCode() for description of inModule. +func RunSrc(ctx Context, pySrc string, pySrcDesc string, inModule interface{}) (*Module, error) { + if pySrcDesc == "" { + pySrcDesc = "" + } + code, err := Compile(pySrc+"\n", pySrcDesc, SingleMode, 0, true) + if err != nil { + return nil, err + } + + return RunCode(ctx, code, pySrcDesc, inModule) +} + +// RunCode executes the given code object within the given module and returns the Module to indicate success. +// If inModule is a *Module, then the code is run in that module. +// If inModule is nil, the code is run in a new __main__ module (and the new Module is returned). +// If inModule is a string, the code is run in a new module with the given name (and the new Module is returned). +func RunCode(ctx Context, code *Code, codeDesc string, inModule interface{}) (*Module, error) { + var ( + module *Module + moduleName string + err error + ) + + createNew := false + switch mod := inModule.(type) { + + case string: + moduleName = mod + createNew = true + case nil: + createNew = true + case *Module: + _, err = ctx.RunCode(code, mod.Globals, mod.Globals, nil) + module = mod + default: + err = ExceptionNewf(TypeError, "unsupported module type: %v", inModule) + } + + if err == nil && createNew { + moduleImpl := ModuleImpl{ + Info: ModuleInfo{ + Name: moduleName, + FileDesc: codeDesc, + }, + Code: code, + } + module, err = ctx.ModuleInit(&moduleImpl) + } + + if err != nil { + return nil, err + } + + return module, nil +} diff --git a/py/traceback.go b/py/traceback.go index 7cad8f04..bf7ba6db 100644 --- a/py/traceback.go +++ b/py/traceback.go @@ -52,7 +52,7 @@ RuntimeError: this is the error message func (tb *Traceback) TracebackDump(w io.Writer) { for ; tb != nil; tb = tb.Next { fmt.Fprintf(w, " File %q, line %d, in %s\n", tb.Frame.Code.Filename, tb.Lineno, tb.Frame.Code.Name) - fmt.Fprintf(w, " %s\n", "FIXME line of source goes here") + //fmt.Fprintf(w, " %s\n", "FIXME line of source goes here") } } diff --git a/pytest/pytest.go b/pytest/pytest.go index 7dbd6f3d..c7af7737 100644 --- a/pytest/pytest.go +++ b/pytest/pytest.go @@ -11,13 +11,14 @@ import ( "strings" "testing" - _ "github.com/go-python/gpython/builtin" + _ "github.com/go-python/gpython/modules" + "github.com/go-python/gpython/compile" "github.com/go-python/gpython/py" - _ "github.com/go-python/gpython/sys" - "github.com/go-python/gpython/vm" ) +var gContext = py.NewContext(py.DefaultContextOpts()) + // Compile the program in the file prog to code in the module that is returned func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) { f, err := os.Open(prog) @@ -34,27 +35,32 @@ func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) { if err != nil { t.Fatalf("%s: ReadAll failed: %v", prog, err) } + return CompileSrc(t, gContext, string(str), prog) +} - obj, err := compile.Compile(string(str), prog, "exec", 0, true) +func CompileSrc(t testing.TB, ctx py.Context, pySrc string, prog string) (*py.Module, *py.Code) { + code, err := compile.Compile(string(pySrc), prog, py.ExecMode, 0, true) if err != nil { t.Fatalf("%s: Compile failed: %v", prog, err) } - code := obj.(*py.Code) - module := py.NewModule("__main__", "", nil, nil) - module.Globals["__file__"] = py.String(prog) + module, err := ctx.Store().NewModule(ctx, &py.ModuleImpl{ + Info: py.ModuleInfo{ + FileDesc: prog, + }, + }) + if err != nil { + t.Fatalf("%s: NewModule failed: %v", prog, err) + } + return module, code } // Run the code in the module func run(t testing.TB, module *py.Module, code *py.Code) { - _, err := vm.Run(module.Globals, module.Globals, code, nil) + _, err := gContext.RunCode(code, module.Globals, module.Globals, nil) if err != nil { - if wantErr, ok := module.Globals["err"]; ok { - wantErrObj, ok := wantErr.(py.Object) - if !ok { - t.Fatalf("want err is not py.Object: %#v", wantErr) - } + if wantErrObj, ok := module.Globals["err"]; ok { gotExc, ok := err.(py.ExceptionInfo) if !ok { t.Fatalf("got err is not ExceptionInfo: %#v", err) diff --git a/repl/cli/cli.go b/repl/cli/cli.go index 2e1da88a..90f1463b 100644 --- a/repl/cli/cli.go +++ b/repl/cli/cli.go @@ -119,10 +119,12 @@ func (rl *readline) Print(out string) { } // RunREPL starts the REPL loop -func RunREPL() { - repl := repl.New() - rl := newReadline(repl) - repl.SetUI(rl) +func RunREPL(replCtx *repl.REPL) { + if replCtx == nil { + replCtx = repl.New(nil) + } + rl := newReadline(replCtx) + replCtx.SetUI(rl) defer rl.Close() err := rl.ReadHistory() if err != nil { diff --git a/repl/repl.go b/repl/repl.go index d7fd9600..f6639b25 100644 --- a/repl/repl.go +++ b/repl/repl.go @@ -10,7 +10,6 @@ import ( "sort" "strings" - "github.com/go-python/gpython/compile" "github.com/go-python/gpython/py" "github.com/go-python/gpython/vm" ) @@ -23,7 +22,8 @@ const ( // Repl state type REPL struct { - module *py.Module + Context py.Context + Module *py.Module prog string continuation bool previous string @@ -39,15 +39,23 @@ type UI interface { Print(string) } -// New create a new REPL and initialises the state machine -func New() *REPL { +// New create a new REPL and initializes the state machine +func New(ctx py.Context) *REPL { + if ctx == nil { + ctx = py.NewContext(py.DefaultContextOpts()) + } + r := &REPL{ - module: py.NewModule("__main__", "", nil, nil), + Context: ctx, prog: "", continuation: false, previous: "", } - r.module.Globals["__file__"] = py.String(r.prog) + r.Module, _ = ctx.ModuleInit(&py.ModuleImpl{ + Info: py.ModuleInfo{ + FileDesc: r.prog, + }, + }) return r } @@ -76,7 +84,7 @@ func (r *REPL) Run(line string) { if toCompile == "" { return } - obj, err := compile.Compile(toCompile+"\n", r.prog, "single", 0, true) + code, err := py.Compile(toCompile+"\n", r.prog, py.SingleMode, 0, true) if err != nil { // Detect that we should start a continuation line // FIXME detect EOF properly! @@ -99,8 +107,7 @@ func (r *REPL) Run(line string) { r.term.Print(fmt.Sprintf("Compile error: %v", err)) return } - code := obj.(*py.Code) - _, err = vm.Run(r.module.Globals, r.module.Globals, code, nil) + _, err = r.Context.RunCode(code, r.Module.Globals, r.Module.Globals, nil) if err != nil { py.TracebackDump(err) } @@ -129,8 +136,8 @@ func (r *REPL) Completer(line string, pos int) (head string, completions []strin } } } - match(r.module.Globals) - match(py.Builtins.Globals) + match(r.Module.Globals) + match(r.Context.Store().Builtins.Globals) sort.Strings(completions) return head, completions, tail } diff --git a/repl/repl_test.go b/repl/repl_test.go index a98ce9a9..9c64879e 100644 --- a/repl/repl_test.go +++ b/repl/repl_test.go @@ -6,10 +6,7 @@ import ( "testing" // import required modules - _ "github.com/go-python/gpython/builtin" - _ "github.com/go-python/gpython/math" - _ "github.com/go-python/gpython/sys" - _ "github.com/go-python/gpython/time" + _ "github.com/go-python/gpython/modules" ) type replTest struct { @@ -38,7 +35,7 @@ func (rt *replTest) assert(t *testing.T, what, wantPrompt, wantOut string) { } func TestREPL(t *testing.T) { - r := New() + r := New(nil) rt := &replTest{} r.SetUI(rt) @@ -78,7 +75,7 @@ func TestREPL(t *testing.T) { } func TestCompleter(t *testing.T) { - r := New() + r := New(nil) rt := &replTest{} r.SetUI(rt) diff --git a/repl/web/main.go b/repl/web/main.go index bad66f16..0be6f3cd 100644 --- a/repl/web/main.go +++ b/repl/web/main.go @@ -4,6 +4,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build js // +build js package main @@ -15,11 +16,9 @@ import ( "github.com/gopherjs/gopherwasm/js" // gopherjs to wasm converter shim // import required modules - _ "github.com/go-python/gpython/builtin" - _ "github.com/go-python/gpython/math" + _ "github.com/go-python/gpython/modules" + "github.com/go-python/gpython/repl" - _ "github.com/go-python/gpython/sys" - _ "github.com/go-python/gpython/time" ) // Implement the replUI interface @@ -77,7 +76,7 @@ func main() { node.Get("classList").Call("add", "active") // Make a repl referring to an empty term for the moment - REPL := repl.New() + REPL := repl.New(nil) cb := js.NewCallback(func(args []js.Value) { REPL.Run(args[0].String()) }) diff --git a/symtable/symtable_data_test.go b/symtable/symtable_data_test.go index b755841d..2ac2754a 100644 --- a/symtable/symtable_data_test.go +++ b/symtable/symtable_data_test.go @@ -12,7 +12,7 @@ import ( var symtableTestData = []struct { in string - mode string // exec, eval or single + mode py.CompileMode out *SymTable exceptionType *py.Type errString string diff --git a/sys/sys.go b/sys/sys.go index fcd1f0df..b8aa3dc2 100644 --- a/sys/sys.go +++ b/sys/sys.go @@ -652,18 +652,27 @@ func init() { py.MustNewMethod("call_tracing", sys_call_tracing, 0, call_tracing_doc), py.MustNewMethod("_debugmallocstats", sys_debugmallocstats, 0, debugmallocstats_doc), } - argv := MakeArgv(os.Args[1:]) - stdin, stdout, stderr := &py.File{os.Stdin, py.FileRead}, - &py.File{os.Stdout, py.FileWrite}, - &py.File{os.Stderr, py.FileWrite} + + stdin := &py.File{File: os.Stdin, FileMode: py.FileRead} + stdout := &py.File{File: os.Stdout, FileMode: py.FileWrite} + stderr := &py.File{File: os.Stderr, FileMode: py.FileWrite} + + executable, err := os.Executable() + if err != nil { + panic(err) + } + globals := py.StringDict{ - "argv": argv, + "path": py.NewList(), + "argv": py.NewListFromStrings(os.Args[1:]), "stdin": stdin, "stdout": stdout, "stderr": stderr, "__stdin__": stdin, "__stdout__": stdout, "__stderr__": stderr, + "executable": py.String(executable), + //"version": py.Int(MARSHAL_VERSION), // /* stdin/stdout/stderr are now set by pythonrun.c */ @@ -787,14 +796,14 @@ func init() { // SET_SYS_FROM_STRING("thread_info", PyThread_GetInfo()); // #endif } - py.NewModule("sys", module_doc, methods, globals) -} -// Makes an argv into a tuple -func MakeArgv(pyargs []string) py.Object { - argv := py.NewListSized(len(pyargs)) - for i, v := range pyargs { - argv.Items[i] = py.String(v) - } - return argv + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "sys", + Doc: module_doc, + }, + Methods: methods, + Globals: globals, + }) + } diff --git a/time/time.go b/time/time.go index c028966a..d783ae8f 100644 --- a/time/time.go +++ b/time/time.go @@ -1007,10 +1007,16 @@ func init() { py.MustNewMethod("perf_counter", time_perf_counter, 0, perf_counter_doc), py.MustNewMethod("get_clock_info", time_get_clock_info, 0, get_clock_info_doc), } - globals := py.StringDict{ - //"version": py.Int(MARSHAL_VERSION), - } - py.NewModule("time", module_doc, methods, globals) + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "time", + Doc: module_doc, + }, + Methods: methods, + Globals: py.StringDict{ + }, + }) } diff --git a/vm/builtin.go b/vm/builtin.go index 3fcdccac..6466b31f 100644 --- a/vm/builtin.go +++ b/vm/builtin.go @@ -12,13 +12,13 @@ import ( "github.com/go-python/gpython/py" ) -func builtinEvalOrExec(self py.Object, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict, mode string) (py.Object, error) { +func builtinEvalOrExec(ctx py.Context, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict, mode py.CompileMode) (py.Object, error) { var ( cmd py.Object globals py.Object = py.None locals py.Object = py.None ) - err := py.UnpackTuple(args, kwargs, mode, 1, 3, &cmd, &globals, &locals) + err := py.UnpackTuple(args, kwargs, string(mode), 1, 3, &cmd, &globals, &locals) if err != nil { return nil, err } @@ -60,24 +60,23 @@ func builtinEvalOrExec(self py.Object, args py.Tuple, kwargs, currentLocals, cur } if code == nil { codeStr = strings.TrimLeft(codeStr, " \t") - obj, err := py.Compile(codeStr, "", mode, 0, true) + code, err = py.Compile(codeStr, "", mode, 0, true) if err != nil { return nil, err } - code = obj.(*py.Code) } if code.GetNumFree() > 0 { return nil, py.ExceptionNewf(py.TypeError, "code passed to %s() may not contain free variables", mode) } - return EvalCode(code, globalsDict, localsDict) + return ctx.RunCode(code, globalsDict, localsDict, nil) } -func builtinEval(self py.Object, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { - return builtinEvalOrExec(self, args, kwargs, currentLocals, currentGlobals, builtins, "eval") +func builtinEval(ctx py.Context, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { + return builtinEvalOrExec(ctx, args, kwargs, currentLocals, currentGlobals, builtins, py.EvalMode) } -func builtinExec(self py.Object, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { - _, err := builtinEvalOrExec(self, args, kwargs, currentLocals, currentGlobals, builtins, "exec") +func builtinExec(ctx py.Context, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { + _, err := builtinEvalOrExec(ctx, args, kwargs, currentLocals, currentGlobals, builtins, py.ExecMode) if err != nil { return nil, err } diff --git a/vm/eval.go b/vm/eval.go index f8e2ef8f..94bf6d1e 100644 --- a/vm/eval.go +++ b/vm/eval.go @@ -767,7 +767,7 @@ func do_END_FINALLY(vm *Vm, arg int32) error { // Loads the __build_class__ helper function to the stack which // creates a new class object. func do_LOAD_BUILD_CLASS(vm *Vm, arg int32) error { - vm.PUSH(py.Builtins.Globals["__build_class__"]) + vm.PUSH(vm.context.Store().Builtins.Globals["__build_class__"]) return nil } @@ -1435,7 +1435,7 @@ func _make_function(vm *Vm, argc int32, opcode OpCode) { num_annotations := (argc >> 16) & 0x7fff qualname := vm.POP() code := vm.POP() - function := py.NewFunction(code.(*py.Code), vm.frame.Globals, string(qualname.(py.String))) + function := py.NewFunction(vm.context, code.(*py.Code), vm.frame.Globals, string(qualname.(py.String))) if opcode == MAKE_CLOSURE { function.Closure = vm.POP().(py.Tuple) @@ -1579,7 +1579,7 @@ func EvalGetFuncDesc(fn py.Object) string { } } -// As py.Call but takes an intepreter Frame object +// As py.Call but takes an interpreter Frame object // // Used to implement some interpreter magic like locals(), globals() etc func callInternal(fn py.Object, args py.Tuple, kwargs py.StringDict, f *py.Frame) (py.Object, error) { @@ -1592,13 +1592,13 @@ func callInternal(fn py.Object, args py.Tuple, kwargs py.StringDict, f *py.Frame f.FastToLocals() return f.Locals, nil case py.InternalMethodImport: - return py.BuiltinImport(nil, args, kwargs, f.Globals) + return py.BuiltinImport(f.Context, nil, args, kwargs, f.Globals) case py.InternalMethodEval: f.FastToLocals() - return builtinEval(nil, args, kwargs, f.Locals, f.Globals, f.Builtins) + return builtinEval(f.Context, args, kwargs, f.Locals, f.Globals, f.Builtins) case py.InternalMethodExec: f.FastToLocals() - return builtinExec(nil, args, kwargs, f.Locals, f.Globals, f.Builtins) + return builtinExec(f.Context, args, kwargs, f.Locals, f.Globals, f.Builtins) default: return nil, py.ExceptionNewf(py.SystemError, "Internal method %v not found", x) } @@ -1731,7 +1731,8 @@ func (vm *Vm) UnwindExceptHandler(frame *py.Frame, block *py.TryBlock) { // This is the equivalent of PyEval_EvalFrame func RunFrame(frame *py.Frame) (res py.Object, err error) { var vm = Vm{ - frame: frame, + frame: frame, + context: frame.Context, } // FIXME need to do this to save the old exeption when we @@ -2033,7 +2034,14 @@ func tooManyPositional(co *py.Code, given, defcount int, fastlocals []py.Object) chooseString(given == 1 && kwonly_given == 0, "was", "were")) } -func EvalCodeEx(co *py.Code, globals, locals py.StringDict, args []py.Object, kws py.StringDict, defs []py.Object, kwdefs py.StringDict, closure py.Tuple) (retval py.Object, err error) { +// EvalCode runs a new virtual machine on a Code object. +// +// Any parameters are expected to have been decoded into locals +// +// Returns an Object and an error. The error will be a py.ExceptionInfo +// +// This is the equivalent of PyEval_EvalCode with closure support +func EvalCode(ctx py.Context, co *py.Code, globals, locals py.StringDict, args []py.Object, kws py.StringDict, defs []py.Object, kwdefs py.StringDict, closure py.Tuple) (retval py.Object, err error) { total_args := int(co.Argcount + co.Kwonlyargcount) n := len(args) var kwdict py.StringDict @@ -2045,7 +2053,7 @@ func EvalCodeEx(co *py.Code, globals, locals py.StringDict, args []py.Object, kw //assert(tstate != nil) //assert(globals != nil) // f = PyFrame_New(tstate, co, globals, locals) - f := py.NewFrame(globals, locals, co, closure) // FIXME extra closure parameter? + f := py.NewFrame(ctx, globals, locals, co, closure) // FIXME extra closure parameter? fastlocals := f.Localsplus freevars := f.CellAndFreeVars @@ -2162,34 +2170,8 @@ func EvalCodeEx(co *py.Code, globals, locals py.StringDict, args []py.Object, kw return RunFrame(f) } -func EvalCode(co *py.Code, globals, locals py.StringDict) (py.Object, error) { - return EvalCodeEx(co, - globals, locals, - nil, - nil, - nil, - nil, nil) -} - -// Run the virtual machine on a Code object -// -// Any parameters are expected to have been decoded into locals -// -// Returns an Object and an error. The error will be a py.ExceptionInfo -// -// This is the equivalent of PyEval_EvalCode with closure support -func Run(globals, locals py.StringDict, code *py.Code, closure py.Tuple) (res py.Object, err error) { - return EvalCodeEx(code, - globals, locals, - nil, - nil, - nil, - nil, closure) -} - // Write the py global to avoid circular import func init() { - py.VmRun = Run + py.VmEvalCode = EvalCode py.VmRunFrame = RunFrame - py.VmEvalCodeEx = EvalCodeEx } diff --git a/vm/vm.go b/vm/vm.go index 4eb3cb20..9abc45ea 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -41,4 +41,6 @@ type Vm struct { curexc py.ExceptionInfo // Previous exception type, value and traceback exc py.ExceptionInfo + // VM access to state / modules + context py.Context } diff --git a/vm/vm_test.go b/vm/vm_test.go index f1686207..fd5fba24 100644 --- a/vm/vm_test.go +++ b/vm/vm_test.go @@ -5,8 +5,13 @@ package vm_test import ( + "fmt" + "strconv" + "strings" + "sync" "testing" + "github.com/go-python/gpython/py" "github.com/go-python/gpython/pytest" ) @@ -17,3 +22,73 @@ func TestVm(t *testing.T) { func BenchmarkVM(b *testing.B) { pytest.RunBenchmarks(b, "benchmarks") } + +var jobSrcTemplate = ` + +doc="multi py.Context text" +WORKER_ID = "{{WORKER_ID}}" +def fib(n): + if n == 0: + return 0 + elif n == 1: + return 1 + return fib(n - 2) + fib(n - 1) + +x = {{FIB_TO}} +fx = fib(x) +print("%s says fib(%d) is %d" % (WORKER_ID, x, fx)) +` + +type worker struct { + name string + ctx py.Context +} + +// run loads a specific worker with a specific work load +func (w *worker) run(b testing.TB, pySrc string, countUpto int) { + pySrc = strings.Replace(pySrc, "{{WORKER_ID}}", w.name, -1) + pySrc = strings.Replace(pySrc, "{{FIB_TO}}", strconv.Itoa(countUpto), -1) + + module, code := pytest.CompileSrc(b, w.ctx, pySrc, w.name) + _, err := w.ctx.RunCode(code, module.Globals, module.Globals, nil) + if err != nil { + b.Fatal(err) + } +} + +func BenchmarkContext(b *testing.B) { + numWorkers := 4 + workersRunning := sync.WaitGroup{} + + numJobs := 35 + fmt.Printf("Starting %d workers to process %d jobs...\n", numWorkers, numJobs) + + jobPipe := make(chan int) + go func() { + for i := 0; i < numJobs; i++ { + jobPipe <- i + 1 + } + close(jobPipe) + }() + + workers := make([]worker, numWorkers) + for i := 0; i < numWorkers; i++ { + + workers[i] = worker{ + name: fmt.Sprintf("Worker #%d", i+1), + ctx: py.NewContext(py.DefaultContextOpts()), + } + + workersRunning.Add(1) + w := workers[i] + go func() { + for jobID := range jobPipe { + w.run(b, jobSrcTemplate, jobID) + //fmt.Printf("### %s finished job %v ###\n", w.name, jobID) + } + workersRunning.Done() + }() + } + + workersRunning.Wait() +}