From fc66b7f649de357b91ba69b15d09cdc4574e2b1d Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Tue, 7 Dec 2021 21:07:24 -0600 Subject: [PATCH 01/79] py.CompileMode now a string enum --- compile/compile.go | 3 +- compile/compile_data_test.go | 2 +- parser/grammar_test.go | 2 +- parser/lexer.go | 16 +++---- parser/lexer_test.go | 2 +- py/run.go | 79 ++++++++++++++++++++++++++++++++++ symtable/symtable_data_test.go | 2 +- 7 files changed, 92 insertions(+), 14 deletions(-) mode change 100644 => 100755 compile/compile_data_test.go mode change 100644 => 100755 parser/grammar_test.go mode change 100644 => 100755 parser/lexer.go mode change 100644 => 100755 parser/lexer_test.go create mode 100755 py/run.go mode change 100644 => 100755 symtable/symtable_data_test.go diff --git a/compile/compile.go b/compile/compile.go index dc8851cf..d3750cb1 100644 --- a/compile/compile.go +++ b/compile/compile.go @@ -102,7 +102,7 @@ func init() { // the effects of any future statements in effect in the code calling // compile; if absent or zero these statements do influence the compilation, // in addition to any features explicitly specified. -func Compile(str, filename, mode string, futureFlags int, dont_inherit bool) (py.Object, error) { +func Compile(str, filename string, mode py.CompileMode, futureFlags int, dont_inherit bool) (py.Object, error) { // Parse Ast Ast, err := parser.ParseString(str, mode) if err != nil { @@ -1342,7 +1342,6 @@ func (c *compiler) NameOp(name string, ctx ast.ExprContext) { default: panic("NameOp: ctx invalid for name variable") } - break } if op == 0 { panic("NameOp: Op not set") diff --git a/compile/compile_data_test.go b/compile/compile_data_test.go old mode 100644 new mode 100755 index edc53720..f0ee0422 --- a/compile/compile_data_test.go +++ b/compile/compile_data_test.go @@ -12,7 +12,7 @@ import ( var compileTestData = []struct { in string - mode string // exec, eval or single + mode py.CompileMode out *py.Code exceptionType *py.Type errString string diff --git a/parser/grammar_test.go b/parser/grammar_test.go old mode 100644 new mode 100755 index d3d48a50..a99beb95 --- a/parser/grammar_test.go +++ b/parser/grammar_test.go @@ -21,7 +21,7 @@ var debugLevel = flag.Int("debugLevel", 0, "Debug level 0-4") func TestGrammar(t *testing.T) { SetDebug(*debugLevel) for _, test := range grammarTestData { - Ast, err := ParseString(test.in, test.mode) + Ast, err := ParseString(test.in, py.CompileMode(test.mode)) if err != nil { if test.exceptionType == nil { t.Errorf("%s: Got exception %v when not expecting one", test.in, err) diff --git a/parser/lexer.go b/parser/lexer.go old mode 100644 new mode 100755 index 6f9f1726..76847893 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -62,7 +62,7 @@ type yyLex struct { // can be 'exec' if source consists of a sequence of statements, // 'eval' if it consists of a single expression, or 'single' if it // consists of a single interactive statement -func NewLex(r io.Reader, filename string, mode string) (*yyLex, error) { +func NewLex(r io.Reader, filename string, mode py.CompileMode) (*yyLex, error) { x := &yyLex{ reader: bufio.NewReader(r), filename: filename, @@ -70,12 +70,12 @@ func NewLex(r io.Reader, filename string, mode string) (*yyLex, error) { state: readString, } switch mode { - case "exec": + case py.ExecMode: x.queue(FILE_INPUT) x.exec = true - case "eval": + case py.EvalMode: x.queue(EVAL_INPUT) - case "single": + case py.SingleMode: x.queue(SINGLE_INPUT) x.interactive = true default: @@ -933,7 +933,7 @@ func SetDebug(level int) { } // Parse a file -func Parse(in io.Reader, filename string, mode string) (mod ast.Mod, err error) { +func Parse(in io.Reader, filename string, mode py.CompileMode) (mod ast.Mod, err error) { lex, err := NewLex(in, filename, mode) if err != nil { return nil, err @@ -952,12 +952,12 @@ func Parse(in io.Reader, filename string, mode string) (mod ast.Mod, err error) } // Parse a string -func ParseString(in string, mode string) (ast.Ast, error) { +func ParseString(in string, mode py.CompileMode) (ast.Ast, error) { return Parse(bytes.NewBufferString(in), "", mode) } // Lex a file only, returning a sequence of tokens -func Lex(in io.Reader, filename string, mode string) (lts LexTokens, err error) { +func Lex(in io.Reader, filename string, mode py.CompileMode) (lts LexTokens, err error) { lex, err := NewLex(in, filename, mode) if err != nil { return nil, err @@ -984,6 +984,6 @@ func Lex(in io.Reader, filename string, mode string) (lts LexTokens, err error) } // Lex a string -func LexString(in string, mode string) (lts LexTokens, err error) { +func LexString(in string, mode py.CompileMode) (lts LexTokens, err error) { return Lex(bytes.NewBufferString(in), "", mode) } diff --git a/parser/lexer_test.go b/parser/lexer_test.go old mode 100644 new mode 100755 index 18e20e5a..ab252088 --- a/parser/lexer_test.go +++ b/parser/lexer_test.go @@ -196,7 +196,7 @@ func TestLex(t *testing.T) { for _, test := range []struct { in string errString string - mode string + mode py.CompileMode lts LexTokens }{ {"", "", "exec", LexTokens{ diff --git a/py/run.go b/py/run.go new file mode 100755 index 00000000..68af8df3 --- /dev/null +++ b/py/run.go @@ -0,0 +1,79 @@ +package py + +type CompileMode string + +const ( + ExecMode CompileMode = "exec" + EvalMode CompileMode = "eval" + SingleMode CompileMode = "single" +) + +type RunFlags int32 + +const ( + // RunOpts.FilePath is intelligently interepreted to load the appropriate pyc object (otherwise new code is generated from the implied .py file) + SmartCodeAcquire RunFlags = 0x01 +) + +// type Sys interface { +// ResetArfs +// } + +// Rename to Ctx? +type Ctx interface { + // These each initiate blocking execution. + //RunAsModule(opts RunOpts) (*Module, error) + //RunCodeAsModule(code *Code, moduleName string, fileDesc string) (*Module, error) + Run(globals, locals StringDict, code *Code, closure Tuple) (res Object, err error) + RunFrame(frame *Frame) (res Object, err error) + EvalCodeEx(co *Code, globals, locals StringDict, args []Object, kws StringDict, defs []Object, kwdefs StringDict, closure Tuple) (retval Object, err error) + + // // Execution of any of the above will stop when the next opcode runs + // SignalHalt() + + //Sys() Sys + + GetModule(moduleName string) (*Module, error) + Store() *Store +} + +const ( + SrcFileExt = ".py" + CodeFileExt = ".pyc" + + DefaultModuleName = "__main__" +) + +type StdLib int32 + +const ( + Lib_sys StdLib = 1 << iota + Lib_time + + CoreLibs = Lib_sys | Lib_time +) + +type RunParams struct { + ModuleInfo ModuleInfo // Newly created module to execute within (if ModuleInfo.Name is nil, then "__main__" is used) + NoopOnFNF bool // If set and Pathname did not resolve, (nil, nil) is returned (vs an error) + Silent bool // If set and an error occurs, no error info is printed + Pathname string +} + +var DefaultCtxOpts = CtxOpts{} + +type CtxOpts struct { + Args []string + SetSysArgs bool +} + +// Some well known objects +var ( + NewCtx func(opts CtxOpts) Ctx + + // // Called at least once before using gpython; multiple calls to it have no effect. + // // Called each time NewCtx is called + // Init func() + Compile func(str, filename string, mode CompileMode, flags int, dont_inherit bool) (Object, error) + Run func(ctx Ctx, params RunParams) (*Module, error) +) diff --git a/symtable/symtable_data_test.go b/symtable/symtable_data_test.go old mode 100644 new mode 100755 index b755841d..2ac2754a --- a/symtable/symtable_data_test.go +++ b/symtable/symtable_data_test.go @@ -12,7 +12,7 @@ import ( var symtableTestData = []struct { in string - mode string // exec, eval or single + mode py.CompileMode out *SymTable exceptionType *py.Type errString string From d10c35f38f8b1fdd80dc1880ca12f325b91ec777 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Tue, 7 Dec 2021 21:19:29 -0600 Subject: [PATCH 02/79] addressed benign Go warnings --- py/bytes.go | 4 ++-- py/code.go | 2 +- py/range.go | 1 - py/string.go | 5 ++--- py/traceback.go | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) mode change 100644 => 100755 py/bytes.go mode change 100644 => 100755 py/code.go mode change 100644 => 100755 py/range.go mode change 100644 => 100755 py/string.go mode change 100644 => 100755 py/traceback.go diff --git a/py/bytes.go b/py/bytes.go old mode 100644 new mode 100755 index b9ae3abe..2c653455 --- a/py/bytes.go +++ b/py/bytes.go @@ -214,14 +214,14 @@ func (a Bytes) M__le__(other Object) (Object, error) { func (a Bytes) M__eq__(other Object) (Object, error) { if b, ok := convertToBytes(other); ok { - return NewBool(bytes.Compare(a, b) == 0), nil + return NewBool(bytes.Equal(a, b)), nil } return NotImplemented, nil } func (a Bytes) M__ne__(other Object) (Object, error) { if b, ok := convertToBytes(other); ok { - return NewBool(bytes.Compare(a, b) != 0), nil + return NewBool(!bytes.Equal(a, b)), nil } return NotImplemented, nil } diff --git a/py/code.go b/py/code.go old mode 100644 new mode 100755 index 355e88e0..09027497 --- a/py/code.go +++ b/py/code.go @@ -97,7 +97,7 @@ const NAME_CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvw // all_name_chars(s): true iff all chars in s are valid NAME_CHARS func all_name_chars(s String) bool { for _, c := range s { - if strings.IndexRune(NAME_CHARS, c) < 0 { + if !strings.ContainsRune(NAME_CHARS, c) { return false } } diff --git a/py/range.go b/py/range.go old mode 100644 new mode 100755 index 1f0513f8..1b261ccd --- a/py/range.go +++ b/py/range.go @@ -146,7 +146,6 @@ func computeRangeLength(start, stop, step Int) Int { if step > 0 { lo = start hi = stop - step = step } else { lo = stop hi = start diff --git a/py/string.go b/py/string.go old mode 100644 new mode 100755 index b985621d..0f9ddc28 --- a/py/string.go +++ b/py/string.go @@ -94,8 +94,7 @@ func StringEscape(a String, ascii bool) string { func fieldsN(s string, n int) []string { out := []string{} cur := []rune{} - r := []rune(s) - for _, c := range r { + for _, c := range s { //until we have covered the first N elements, multiple white-spaces are 'merged' if n < 0 || len(out) < n { if unicode.IsSpace(c) { @@ -139,7 +138,7 @@ func init() { maxSplit = int(m) } } - valArray := []string{} + var valArray []string if valStr, ok := value.(String); ok { valArray = strings.SplitN(string(selfStr), string(valStr), maxSplit+1) } else if _, ok := value.(NoneType); ok { diff --git a/py/traceback.go b/py/traceback.go old mode 100644 new mode 100755 index 7cad8f04..bf7ba6db --- a/py/traceback.go +++ b/py/traceback.go @@ -52,7 +52,7 @@ RuntimeError: this is the error message func (tb *Traceback) TracebackDump(w io.Writer) { for ; tb != nil; tb = tb.Next { fmt.Fprintf(w, " File %q, line %d, in %s\n", tb.Frame.Code.Filename, tb.Lineno, tb.Frame.Code.Name) - fmt.Fprintf(w, " %s\n", "FIXME line of source goes here") + //fmt.Fprintf(w, " %s\n", "FIXME line of source goes here") } } From 2161b71f5e4b4f6c0b95e8069ed7b03825a06f46 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Tue, 7 Dec 2021 21:20:33 -0600 Subject: [PATCH 03/79] make Float implement M__index__ --- py/float.go | 5 +++++ 1 file changed, 5 insertions(+) mode change 100644 => 100755 py/float.go diff --git a/py/float.go b/py/float.go old mode 100644 new mode 100755 index 4f47759b..66d1f78f --- a/py/float.go +++ b/py/float.go @@ -309,6 +309,10 @@ func (a Float) M__bool__() (Object, error) { return NewBool(a != 0), nil } +func (a Float) M__index__() (Int, error) { + return Int(a), nil +} + func (a Float) M__int__() (Object, error) { if a >= IntMin && a <= IntMax { return Int(a), nil @@ -413,3 +417,4 @@ var _ floatArithmetic = Float(0) var _ conversionBetweenTypes = Float(0) var _ I__bool__ = Float(0) var _ richComparison = Float(0) +var _ I__index__ = Float(0) From 247ec429581d46a3dd6a3659f16e070c774fb11a Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Tue, 7 Dec 2021 21:20:57 -0600 Subject: [PATCH 04/79] added string helper functions --- py/list.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) mode change 100644 => 100755 py/list.go diff --git a/py/list.go b/py/list.go old mode 100644 new mode 100755 index eee7bf3e..cae142e1 --- a/py/list.go +++ b/py/list.go @@ -121,6 +121,16 @@ func NewListFromItems(items []Object) *List { return l } +// Makes an argv into a tuple +func NewListFromStrings(items []string) *List { + l := NewListSized(len(items)) + for i, v := range items { + l.Items[i] = String(v) + } + return l +} + + // Copy a list object func (l *List) Copy() *List { return NewListFromItems(l.Items) @@ -141,6 +151,13 @@ func (l *List) Extend(items []Object) { l.Items = append(l.Items, items...) } +// Extend the list with strings +func (l *List) ExtendWithStrings(items []string) { + for _, item := range items { + l.Items = append(l.Items, Object(String(item))) + } +} + // Extends the list with the sequence passed in func (l *List) ExtendSequence(seq Object) error { return Iterate(seq, func(item Object) bool { From 9a75271d5062d6851809b2b02ceeedeb83838169 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Wed, 8 Dec 2021 09:23:33 -0600 Subject: [PATCH 05/79] updated to 1.15 --- go.mod | 7 ++++--- go.sum | 15 ++++++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) mode change 100644 => 100755 go.mod mode change 100644 => 100755 go.sum diff --git a/go.mod b/go.mod old mode 100644 new mode 100755 index 4cda1cf1..4a927986 --- a/go.mod +++ b/go.mod @@ -1,8 +1,9 @@ module github.com/go-python/gpython -go 1.12 +go 1.15 require ( - github.com/gopherjs/gopherwasm v1.0.0 // indirect - github.com/peterh/liner v1.1.0 + github.com/gopherjs/gopherwasm v1.1.0 + github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/peterh/liner v1.2.1 ) diff --git a/go.sum b/go.sum old mode 100644 new mode 100755 index c17fb31d..a53dc4e1 --- a/go.sum +++ b/go.sum @@ -1,8 +1,13 @@ +github.com/go-python/py v0.0.0-20150724143143-adb74cbb1b5b h1:vTYiMD/1XTeL0SwwyVG30JveBV9ZMgJEZBwDcixC/oM= +github.com/go-python/py v0.0.0-20150724143143-adb74cbb1b5b/go.mod h1:FDA9/ZAzj70a4RlKS443aRrgA+IW1WLOWAFKZZ8tHXk= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSfgRJKqIzapE78onvvTbdi1rMkU00lZw= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherwasm v1.0.0 h1:32nge/RlujS1Im4HNCJPp0NbBOAeBXFuT1KonUuLl+Y= -github.com/gopherjs/gopherwasm v1.0.0/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI= -github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= +github.com/gopherjs/gopherwasm v1.1.0 h1:fA2uLoctU5+T3OhOn2vYP0DVT6pxc7xhTlBB1paATqQ= +github.com/gopherjs/gopherwasm v1.1.0/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/peterh/liner v1.1.0 h1:f+aAedNJA6uk7+6rXsYBnhdo4Xux7ESLe+kcuVUF5os= -github.com/peterh/liner v1.1.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/peterh/liner v1.2.1 h1:O4BlKaq/LWu6VRWmol4ByWfzx6MfXc5Op5HETyIy5yg= +github.com/peterh/liner v1.2.1/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= From 90a8988f2f64f8b1acc246d19cd00f4be92d8c00 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Wed, 8 Dec 2021 20:54:21 -0600 Subject: [PATCH 06/79] new Ctx-aware module model --- builtin/builtin.go | 37 ++++++++---- importlib/importlib.go | 12 ++-- marshal/marshal.go | 33 ++++++++--- math/math.go | 19 ++++-- py/method.go | 6 +- py/module.go | 128 +++++++++++++++++++++++++++++++++-------- sys/sys.go | 44 +++++++++----- time/time.go | 14 +++-- 8 files changed, 221 insertions(+), 72 deletions(-) mode change 100644 => 100755 builtin/builtin.go mode change 100644 => 100755 importlib/importlib.go mode change 100644 => 100755 marshal/marshal.go mode change 100644 => 100755 math/math.go mode change 100644 => 100755 py/module.go mode change 100644 => 100755 sys/sys.go mode change 100644 => 100755 time/time.go diff --git a/builtin/builtin.go b/builtin/builtin.go old mode 100644 new mode 100755 index 8f4ef674..f83b3c6d --- a/builtin/builtin.go +++ b/builtin/builtin.go @@ -162,7 +162,17 @@ func init() { "Warning": py.Warning, "ZeroDivisionError": py.ZeroDivisionError, } - py.NewModule("builtins", builtin_doc, methods, globals) + + py.RegisterModule(&py.StaticModule{ + Info: py.ModuleInfo{ + Name: "builtins", + Doc: builtin_doc, + Flags: py.ShareModule, + }, + Methods: methods, + Globals: globals, + }) + } const print_doc = `print(value, ..., sep=' ', end='\\n', file=sys.stdout, flush=False) @@ -178,18 +188,22 @@ func builtin_print(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Obje var ( sepObj py.Object = py.String(" ") endObj py.Object = py.String("\n") - file py.Object = py.MustGetModule("sys").Globals["stdout"] flush py.Object ) + sysModule, err := self.(*py.Module).Ctx.GetModule("sys") + if err != nil { + return nil, err + } + stdout := sysModule.Globals["stdout"] kwlist := []string{"sep", "end", "file", "flush"} - err := py.ParseTupleAndKeywords(nil, kwargs, "|ssOO:print", kwlist, &sepObj, &endObj, &file, &flush) + err = py.ParseTupleAndKeywords(nil, kwargs, "|ssOO:print", kwlist, &sepObj, &endObj, &stdout, &flush) if err != nil { return nil, err } sep := sepObj.(py.String) end := endObj.(py.String) - write, err := py.GetAttrString(file, "write") + write, err := py.GetAttrString(stdout, "write") if err != nil { return nil, err } @@ -219,7 +233,7 @@ func builtin_print(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Obje } if shouldFlush, _ := py.MakeBool(flush); shouldFlush == py.True { - fflush, err := py.GetAttrString(file, "flush") + fflush, err := py.GetAttrString(stdout, "flush") if err == nil { return py.Call(fflush, nil, nil) } @@ -449,7 +463,7 @@ func builtin___build_class__(self py.Object, args py.Tuple, kwargs py.StringDict } // fmt.Printf("Calling %v with %v and %v\n", fn.Name, fn.Globals, ns) // fmt.Printf("Code = %#v\n", fn.Code) - cell, err = py.VmRun(fn.Globals, ns, fn.Code, fn.Closure) + cell, err = fn.Ctx.Run(fn.Globals, ns, fn.Code, fn.Closure) if err != nil { return nil, err } @@ -749,9 +763,9 @@ func builtin_compile(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Ob return nil, py.ExceptionNewf(py.ValueError, "compile(): invalid optimize value") } - if dont_inherit.(py.Int) != 0 { - // PyEval_MergeCompilerFlags(&cf) - } + // if dont_inherit.(py.Int) != 0 { + // PyEval_MergeCompilerFlags(&cf) + // } // switch string(startstr.(py.String)) { // case "exec": @@ -782,7 +796,7 @@ func builtin_compile(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Ob return nil, err } // result = py.CompileStringExFlags(str, filename, start[mode], &cf, optimize) - result, err = compile.Compile(str, string(filename.(py.String)), string(startstr.(py.String)), int(supplied_flags.(py.Int)), dont_inherit.(py.Int) != 0) + result, err = compile.Compile(str, string(filename.(py.String)), py.CompileMode(startstr.(py.String)), int(supplied_flags.(py.Int)), dont_inherit.(py.Int) != 0) if err != nil { return nil, err } @@ -882,9 +896,8 @@ or ... etc. ` func isinstance(obj py.Object, classOrTuple py.Object) (py.Bool, error) { - switch classOrTuple.(type) { + switch class_tuple := classOrTuple.(type) { case py.Tuple: - var class_tuple = classOrTuple.(py.Tuple) for idx := range class_tuple { res, _ := isinstance(obj, class_tuple[idx]) if res { diff --git a/importlib/importlib.go b/importlib/importlib.go old mode 100644 new mode 100755 index e1d307db..a6f1ad3f --- a/importlib/importlib.go +++ b/importlib/importlib.go @@ -7,15 +7,19 @@ package py import ( - "log" - "github.com/go-python/gpython/marshal" + "github.com/go-python/gpython/py" ) // Load the frozen module func init() { - _, err := marshal.LoadFrozenModule("importlib", data) - log.Fatalf("Failed to load importlib: %v", err) + + py.RegisterModule(&marshal.FrozenModule{ + Info: py.ModuleInfo{ + Name: "importlib", + }, + Code: data, + }) } // Auto-generated by Modules/_freeze_importlib.c diff --git a/marshal/marshal.go b/marshal/marshal.go old mode 100644 new mode 100755 index dd039d5c..85e7e059 --- a/marshal/marshal.go +++ b/marshal/marshal.go @@ -15,7 +15,6 @@ import ( "strconv" "github.com/go-python/gpython/py" - "github.com/go-python/gpython/vm" ) const ( @@ -448,22 +447,31 @@ func ReadPyc(r io.Reader) (obj py.Object, err error) { } // FIXME do something with timestamp & length? if header.Magic>>16 != 0x0a0d { - return nil, errors.New("Bad magic in .pyc file") + return nil, errors.New("bad magic in .pyc file") } // fmt.Printf("header = %v\n", header) return ReadObject(r) } -// Unmarshals a frozen module -func LoadFrozenModule(name string, data []byte) (*py.Module, error) { - r := bytes.NewBuffer(data) +// Set for straight-forward modules that are threadsafe, stateless, and/or should be shared across multiple py.Ctx instances (for efficiency). +type FrozenModule struct { + Info py.ModuleInfo + Code []byte +} + +func (mod *FrozenModule) ModuleInfo() py.ModuleInfo { + return mod.Info +} + +func (mod *FrozenModule) ModuleInit(ctx py.Ctx) (*py.Module, error) { + r := bytes.NewBuffer(mod.Code) obj, err := ReadObject(r) if err != nil { return nil, err } code := obj.(*py.Code) - module := py.NewModule(name, "", nil, nil) - _, err = vm.Run(module.Globals, module.Globals, code, nil) + module := ctx.Store().NewModule(ctx, mod.Info, nil, nil) + _, err = ctx.Run(module.Globals, module.Globals, code, nil) if err != nil { py.TracebackDump(err) return nil, err @@ -634,5 +642,14 @@ func init() { globals := py.StringDict{ "version": py.Int(MARSHAL_VERSION), } - py.NewModule("marshal", module_doc, methods, globals) + + py.RegisterModule(&py.StaticModule{ + Info: py.ModuleInfo{ + Name: "marshal", + Doc: module_doc, + Flags: py.ShareModule, + }, + Globals: globals, + Methods: methods, + }) } diff --git a/math/math.go b/math/math.go old mode 100644 new mode 100755 index 88d60fb3..867cf25c --- a/math/math.go +++ b/math/math.go @@ -1333,9 +1333,18 @@ func init() { py.MustNewMethod("trunc", math_trunc, 0, math_trunc_doc), py.MustNewMethod("to_ulps", math_to_ulps, 0, math_to_ulps_doc), } - globals := py.StringDict{ - "pi": py.Float(math.Pi), - "e": py.Float(math.E), - } - py.NewModule("math", math_doc, methods, globals) + + py.RegisterModule(&py.StaticModule{ + Info: py.ModuleInfo{ + Name: "math", + Doc: math_doc, + Flags: py.ShareModule, + }, + Methods: methods, + Globals: py.StringDict{ + "pi": py.Float(math.Pi), + "e": py.Float(math.E), + }, + }) + } diff --git a/py/method.go b/py/method.go index 1f8ecb72..c8b0ab03 100644 --- a/py/method.go +++ b/py/method.go @@ -70,6 +70,8 @@ type Method struct { Flags int // Go function implementation method interface{} + // Parent module of this method + Module *Module } // Internal method types implemented within eval.go @@ -224,14 +226,14 @@ func newBoundMethod(name string, fn interface{}) (Object, error) { return f(a, b, c) } default: - return nil, fmt.Errorf("Unknown bound method type for %q: %T", name, fn) + return nil, fmt.Errorf("unknown bound method type for %q: %T", name, fn) } return m, nil } // Call a method func (m *Method) M__call__(args Tuple, kwargs StringDict) (Object, error) { - self := None // FIXME should be the module + self := Object(m.Module) if kwargs != nil { return m.CallWithKeywords(self, args, kwargs) } diff --git a/py/module.go b/py/module.go old mode 100644 new mode 100755 index ae2a2171..f8705277 --- a/py/module.go +++ b/py/module.go @@ -6,24 +6,94 @@ package py -import "fmt" +import ( + "fmt" + "sync" +) + +type ModuleFlags int32 -var ( +const ( + // Set for modules that are threadsafe, stateless, and/or can be shared across multiple py.Ctx instances (for efficiency). + // Otherwise, a separate module instance is created for each py.Ctx that imports it. + ShareModule ModuleFlags = 0x01 // @@TODO +) + +type ModuleInfo struct { + Name string + Doc string + FileDesc string + Flags ModuleFlags +} + +type ModuleImpl interface { + ModuleInfo() ModuleInfo + ModuleInit(ctx Ctx) (*Module, error) +} + +// Set for straight-forward modules that are threadsafe, stateless, and/or should be shared across multiple py.Ctx instances (for efficiency). +type StaticModule struct { + Info ModuleInfo + Methods []*Method + Globals StringDict +} + +func (mod *StaticModule) ModuleInfo() ModuleInfo { + return mod.Info +} + +func (mod *StaticModule) ModuleInit(ctx Ctx) (*Module, error) { + return ctx.Store().NewModule(ctx, mod.Info, mod.Methods, mod.Globals), nil +} + +type Store struct { // Registry of installed modules - modules = make(map[string]*Module) + modules map[string]*Module // Builtin module Builtins *Module // this should be the frozen module importlib/_bootstrap.py generated // by Modules/_freeze_importlib.c into Python/importlib.h Importlib *Module -) +} -// A python Module object +func RegisterModule(module ModuleImpl) { + gRuntime.RegisterModule(module) +} + +func GetModuleImpl(moduleName string) ModuleImpl { + gRuntime.mu.RLock() + defer gRuntime.mu.RUnlock() + impl := gRuntime.ModuleImpls[moduleName] + return impl +} + +type Runtime struct { + mu sync.RWMutex + ModuleImpls map[string]ModuleImpl +} + +var gRuntime = Runtime{ + ModuleImpls: make(map[string]ModuleImpl), +} + +func (rt *Runtime) RegisterModule(module ModuleImpl) { + rt.mu.Lock() + defer rt.mu.Unlock() + rt.ModuleImpls[module.ModuleInfo().Name] = module +} + +func NewStore() *Store { + return &Store{ + modules: make(map[string]*Module), + } +} + +// A python Module object that has been initted for a given py.Ctx type Module struct { - Name string - Doc string + ModuleInfo + Globals StringDict - // dict Dict + Ctx Ctx } var ModuleType = NewType("module", "module object") @@ -43,45 +113,55 @@ func (m *Module) GetDict() StringDict { } // Define a new module -func NewModule(name, doc string, methods []*Method, globals StringDict) *Module { +func (store *Store) NewModule(ctx Ctx, info ModuleInfo, methods []*Method, globals StringDict) *Module { + if info.Name == "" { + info.Name = "__main__" + } m := &Module{ - Name: name, - Doc: doc, - Globals: globals.Copy(), + ModuleInfo: info, + Globals: globals.Copy(), + Ctx: ctx, } // Insert the methods into the module dictionary + // Copy each method an insert each "live" with a ptr back to the module (which can also lead us to the host Ctx) for _, method := range methods { - m.Globals[method.Name] = method + methodInst := new(Method) + *methodInst = *method + methodInst.Module = m + m.Globals[method.Name] = methodInst } // Set some module globals - m.Globals["__name__"] = String(name) - m.Globals["__doc__"] = String(doc) + m.Globals["__name__"] = String(info.Name) + m.Globals["__doc__"] = String(info.Doc) m.Globals["__package__"] = None + if len(info.FileDesc) > 0 { + m.Globals["__file__"] = String(info.FileDesc) + } // Register the module - modules[name] = m + store.modules[info.Name] = m // Make a note of some modules - switch name { + switch info.Name { case "builtins": - Builtins = m + store.Builtins = m case "importlib": - Importlib = m + store.Importlib = m } // fmt.Printf("Registering module %q\n", name) return m } // Gets a module -func GetModule(name string) (*Module, error) { - m, ok := modules[name] +func (store *Store) GetModule(name string) (*Module, error) { + m, ok := store.modules[name] if !ok { - return nil, ExceptionNewf(ImportError, "Module %q not found", name) + return nil, ExceptionNewf(ImportError, "Module '%q' not found", name) } return m, nil } // Gets a module or panics -func MustGetModule(name string) *Module { - m, err := GetModule(name) +func (store *Store) MustGetModule(name string) *Module { + m, err := store.GetModule(name) if err != nil { panic(err) } diff --git a/sys/sys.go b/sys/sys.go old mode 100644 new mode 100755 index fcd1f0df..d02350fc --- a/sys/sys.go +++ b/sys/sys.go @@ -23,6 +23,15 @@ import ( "github.com/go-python/gpython/py" ) +// Alter this list before the sys module starts to alter the factory path set +var SysPathInitializer = []string{ + "./", // Denotes the dir of the current file (or cwd if __file__ is not set) + "./lib", + "/usr/lib/python3.4", + "/usr/local/lib/python3.4/dist-packages", + "/usr/lib/python3/dist-packages", +} + const module_doc = `This module provides access to some objects used or maintained by the interpreter and to functions that interact strongly with the interpreter. @@ -652,12 +661,13 @@ func init() { py.MustNewMethod("call_tracing", sys_call_tracing, 0, call_tracing_doc), py.MustNewMethod("_debugmallocstats", sys_debugmallocstats, 0, debugmallocstats_doc), } - argv := MakeArgv(os.Args[1:]) - stdin, stdout, stderr := &py.File{os.Stdin, py.FileRead}, - &py.File{os.Stdout, py.FileWrite}, - &py.File{os.Stderr, py.FileWrite} + + stdin := &py.File{File: os.Stdin, FileMode: py.FileRead} + stdout := &py.File{File: os.Stdout, FileMode: py.FileWrite} + stderr := &py.File{File: os.Stderr, FileMode: py.FileWrite} + globals := py.StringDict{ - "argv": argv, + "argv": py.NewListFromStrings(os.Args[1:]), "stdin": stdin, "stdout": stdout, "stderr": stderr, @@ -787,14 +797,22 @@ func init() { // SET_SYS_FROM_STRING("thread_info", PyThread_GetInfo()); // #endif } - py.NewModule("sys", module_doc, methods, globals) -} -// Makes an argv into a tuple -func MakeArgv(pyargs []string) py.Object { - argv := py.NewListSized(len(pyargs)) - for i, v := range pyargs { - argv.Items[i] = py.String(v) + executable, err := os.Executable() + if err != nil { + panic(err) } - return argv + + globals["executable"] = py.String(executable) + globals["path"] = py.NewListFromStrings(SysPathInitializer) + + py.RegisterModule(&py.StaticModule{ + Info: py.ModuleInfo{ + Name: "sys", + Doc: module_doc, + }, + Methods: methods, + Globals: globals, + }) + } diff --git a/time/time.go b/time/time.go old mode 100644 new mode 100755 index 88deb94a..d8397988 --- a/time/time.go +++ b/time/time.go @@ -997,10 +997,16 @@ func init() { py.MustNewMethod("perf_counter", time_perf_counter, 0, perf_counter_doc), py.MustNewMethod("get_clock_info", time_get_clock_info, 0, get_clock_info_doc), } - globals := py.StringDict{ - //"version": py.Int(MARSHAL_VERSION), - } - py.NewModule("time", module_doc, methods, globals) + + py.RegisterModule(&py.StaticModule{ + Info: py.ModuleInfo{ + Name: "time", + Doc: module_doc, + }, + Methods: methods, + Globals: py.StringDict{ + }, + }) } From c51644ee24d128e876be407ec7c78262789e24cb Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Thu, 9 Dec 2021 16:29:46 -0600 Subject: [PATCH 07/79] py.Ctx model allows concurrent execution --- builtin/builtin.go | 2 +- main.go | 88 +++++++------------------- marshal/marshal.go | 3 +- modules/runtime.go | 150 +++++++++++++++++++++++++++++++++++++++++++++ py/frame.go | 6 +- py/function.go | 14 ++--- py/import.go | 121 +++++++++++++++--------------------- py/py.go | 9 +-- py/run.go | 27 ++++---- pytest/pytest.go | 36 ++++------- repl/repl.go | 24 +++++--- vm/builtin.go | 14 ++--- vm/eval.go | 57 +++++++---------- vm/vm.go | 2 + 14 files changed, 301 insertions(+), 252 deletions(-) mode change 100644 => 100755 main.go create mode 100755 modules/runtime.go mode change 100644 => 100755 py/frame.go mode change 100644 => 100755 py/function.go mode change 100644 => 100755 py/import.go mode change 100644 => 100755 py/py.go mode change 100644 => 100755 repl/repl.go mode change 100644 => 100755 vm/builtin.go mode change 100644 => 100755 vm/eval.go mode change 100644 => 100755 vm/vm.go diff --git a/builtin/builtin.go b/builtin/builtin.go index f83b3c6d..ac80ef57 100755 --- a/builtin/builtin.go +++ b/builtin/builtin.go @@ -463,7 +463,7 @@ func builtin___build_class__(self py.Object, args py.Tuple, kwargs py.StringDict } // fmt.Printf("Calling %v with %v and %v\n", fn.Name, fn.Globals, ns) // fmt.Printf("Code = %#v\n", fn.Code) - cell, err = fn.Ctx.Run(fn.Globals, ns, fn.Code, fn.Closure) + cell, err = fn.Ctx.RunCode(fn.Code, fn.Globals, ns, fn.Closure) if err != nil { return nil, err } diff --git a/main.go b/main.go old mode 100644 new mode 100755 index 8148056b..8c3fc908 --- a/main.go +++ b/main.go @@ -12,22 +12,13 @@ import ( "runtime" "runtime/pprof" - _ "github.com/go-python/gpython/builtin" + "github.com/go-python/gpython/repl" "github.com/go-python/gpython/repl/cli" - //_ "github.com/go-python/gpython/importlib" - "io/ioutil" "log" "os" - "strings" - "github.com/go-python/gpython/compile" - "github.com/go-python/gpython/marshal" - _ "github.com/go-python/gpython/math" "github.com/go-python/gpython/py" - pysys "github.com/go-python/gpython/sys" - _ "github.com/go-python/gpython/time" - "github.com/go-python/gpython/vm" ) // Globals @@ -48,34 +39,15 @@ Full options: flag.PrintDefaults() } -// Exit with the message -func fatal(message string, args ...interface{}) { - if !strings.HasSuffix(message, "\n") { - message += "\n" - } - syntaxError() - fmt.Fprintf(os.Stderr, message, args...) - os.Exit(1) -} - func main() { flag.Usage = syntaxError flag.Parse() args := flag.Args() - py.MustGetModule("sys").Globals["argv"] = pysys.MakeArgv(args) - if len(args) == 0 { - - fmt.Printf("Python 3.4.0 (%s, %s)\n", commit, date) - fmt.Printf("[Gpython %s]\n", version) - fmt.Printf("- os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) - fmt.Printf("- go version: %s\n", runtime.Version()) - - cli.RunREPL() - return - } - prog := args[0] - // fmt.Printf("Running %q\n", prog) - + + opts := py.DefaultCtxOpts + opts.Args = flag.Args() + ctx := py.NewCtx(opts) + if *cpuprofile != "" { f, err := os.Create(*cpuprofile) if err != nil { @@ -87,42 +59,24 @@ func main() { } defer pprof.StopCPUProfile() } + + // IF no args, enter REPL mode + if len(args) == 0 { - // FIXME should be using ImportModuleLevelObject() here - f, err := os.Open(prog) - if err != nil { - log.Fatalf("Failed to open %q: %v", prog, err) - } - var obj py.Object - if strings.HasSuffix(prog, ".pyc") { - obj, err = marshal.ReadPyc(f) - if err != nil { - log.Fatalf("Failed to marshal %q: %v", prog, err) - } - } else if strings.HasSuffix(prog, ".py") { - str, err := ioutil.ReadAll(f) - if err != nil { - log.Fatalf("Failed to read %q: %v", prog, err) - } - obj, err = compile.Compile(string(str), prog, "exec", 0, true) + fmt.Printf("Python 3.4.0 (%s, %s)\n", commit, date) + fmt.Printf("[Gpython %s]\n", version) + fmt.Printf("- os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) + fmt.Printf("- go version: %s\n", runtime.Version()) + + replCtx := repl.New(ctx) + cli.RunREPL(replCtx) + + } else { + _, err := ctx.RunFile(args[0], py.RunOpts{}) if err != nil { - log.Fatalf("Can't compile %q: %v", prog, err) + py.TracebackDump(err) + log.Fatal(err) } - } else { - log.Fatalf("Can't execute %q", prog) - } - if err = f.Close(); err != nil { - log.Fatalf("Failed to close %q: %v", prog, err) - } - code := obj.(*py.Code) - module := py.NewModule("__main__", "", nil, nil) - module.Globals["__file__"] = py.String(prog) - res, err := vm.Run(module.Globals, module.Globals, code, nil) - if err != nil { - py.TracebackDump(err) - log.Fatal(err) } - // fmt.Printf("Return = %v\n", res) - _ = res } diff --git a/marshal/marshal.go b/marshal/marshal.go index 85e7e059..6dbb9166 100755 --- a/marshal/marshal.go +++ b/marshal/marshal.go @@ -471,9 +471,8 @@ func (mod *FrozenModule) ModuleInit(ctx py.Ctx) (*py.Module, error) { } code := obj.(*py.Code) module := ctx.Store().NewModule(ctx, mod.Info, nil, nil) - _, err = ctx.Run(module.Globals, module.Globals, code, nil) + _, err = ctx.RunCode(code, module.Globals, module.Globals, nil) if err != nil { - py.TracebackDump(err) return nil, err } return module, nil diff --git a/modules/runtime.go b/modules/runtime.go new file mode 100755 index 00000000..ebb8fb9c --- /dev/null +++ b/modules/runtime.go @@ -0,0 +1,150 @@ +package modules + +import ( + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + + "github.com/go-python/gpython/marshal" + "github.com/go-python/gpython/py" + "github.com/go-python/gpython/vm" + + _ "github.com/go-python/gpython/builtin" + _ "github.com/go-python/gpython/math" + _ "github.com/go-python/gpython/sys" + _ "github.com/go-python/gpython/time" +) + +func init() { + py.NewCtx = NewCtx +} + +var defaultPaths = []py.Object{ + py.String("."), +} + +func (ctx *ctx) RunFile(runPath string, opts py.RunOpts) (*py.Module, error) { + + tryPaths := defaultPaths + if opts.UseSysPaths { + tryPaths = ctx.Store().MustGetModule("sys").Globals["path"].(*py.List).Items + } + for _, pathObj := range tryPaths { + pathStr, ok := pathObj.(py.String) + if !ok { + continue + } + fpath := path.Join(string(pathStr), runPath) + if !filepath.IsAbs(fpath) { + if opts.CurDir == "" { + opts.CurDir, _ = os.Getwd() + } + fpath = path.Join(opts.CurDir, fpath) + } + + if fpath[len(fpath)-1] == '/' { + fpath = fpath[:len(fpath)-1] + } + + stat, err := os.Stat(fpath) + if err == nil && stat.IsDir() { + // FIXME this is a massive simplification! + fpath = path.Join(fpath, "__init__.py") + _, err = os.Stat(fpath) + } + + ext := strings.ToLower(filepath.Ext(fpath)) + if ext == "" && os.IsNotExist(err) { + fpath += ".py" + ext = ".py" + _, err = os.Stat(fpath) + } + + if err != nil { + if os.IsNotExist(err) { + continue + } + err = py.ExceptionNewf(py.OSError, "Error accessing %q: %v", fpath, err) + return nil, err + } + + var codeObj py.Object + if ext == ".py" { + var pySrc []byte + pySrc, err = ioutil.ReadFile(fpath) + if err != nil { + return nil, py.ExceptionNewf(py.OSError, "Error reading %q: %v", fpath, err) + } + + codeObj, err = py.Compile(string(pySrc), fpath, py.ExecMode, 0, true) + if err != nil { + return nil, err + } + } else if ext == ".pyc" { + var file *os.File + file, err = os.Open(fpath) + if err != nil { + return nil, py.ExceptionNewf(py.OSError, "Error opening %q: %v", fpath, err) + } + defer file.Close() + codeObj, err = marshal.ReadPyc(file) + if err != nil { + return nil, py.ExceptionNewf(py.ImportError, "Failed to marshal %q: %v", fpath, err) + } + } + + var code *py.Code + if codeObj != nil { + code, _ = codeObj.(*py.Code) + } + if code == nil { + return nil, py.ExceptionNewf(py.AssertionError, "Missing code object") + } + + if opts.HostModule == nil { + opts.HostModule = ctx.Store().NewModule(ctx, py.ModuleInfo{ + Name: opts.ModuleName, + FileDesc: fpath, + }, nil, nil) + } + + _, err = vm.EvalCode(ctx, code, opts.HostModule.Globals, opts.HostModule.Globals, nil, nil, nil, nil, nil) + return opts.HostModule, err + } + + return nil, py.ExceptionNewf(py.FileNotFoundError, "Failed to resolve %q", runPath) +} + +func (ctx *ctx) RunCode(code *py.Code, globals, locals py.StringDict, closure py.Tuple) (py.Object, error) { + return vm.EvalCode(ctx, code, globals, locals, nil, nil, nil, nil, closure) +} + +func (ctx *ctx) GetModule(moduleName string) (*py.Module, error) { + return ctx.store.GetModule(moduleName) +} + +func (ctx *ctx) Store() *py.Store { + return ctx.store +} + +func NewCtx(opts py.CtxOpts) py.Ctx { + ctx := &ctx{ + opts: opts, + } + + ctx.store = py.NewStore() + + py.Import(ctx, "builtins", "sys") + if opts.SetSysArgs { + ctx.Store().MustGetModule("sys").Globals["argv"] = py.NewListFromStrings(opts.Args) + } + + return ctx +} + +type ctx struct { + store *py.Store + opts py.CtxOpts +} diff --git a/py/frame.go b/py/frame.go old mode 100644 new mode 100755 index f91438a0..ae39c018 --- a/py/frame.go +++ b/py/frame.go @@ -26,6 +26,7 @@ type TryBlock struct { // A python Frame object type Frame struct { // Back *Frame // previous frame, or nil + Ctx Ctx Code *Code // code segment Builtins StringDict // builtin symbol table Globals StringDict // global symbol table @@ -77,7 +78,7 @@ func (o *Frame) Type() *Type { } // Make a new frame for a code object -func NewFrame(globals, locals StringDict, code *Code, closure Tuple) *Frame { +func NewFrame(ctx Ctx, globals, locals StringDict, code *Code, closure Tuple) *Frame { nlocals := int(code.Nlocals) ncells := len(code.Cellvars) nfrees := len(code.Freevars) @@ -90,12 +91,13 @@ func NewFrame(globals, locals StringDict, code *Code, closure Tuple) *Frame { cellAndFreeVars := allocation[nlocals:varsize] return &Frame{ + Ctx: ctx, Globals: globals, Locals: locals, Code: code, LocalVars: localVars, CellAndFreeVars: cellAndFreeVars, - Builtins: Builtins.Globals, + Builtins: ctx.Store().Builtins.Globals, Localsplus: allocation, Stack: make([]Object, 0, code.Stacksize), } diff --git a/py/function.go b/py/function.go old mode 100644 new mode 100755 index 28eace5d..c8c9cb37 --- a/py/function.go +++ b/py/function.go @@ -18,6 +18,7 @@ package py // A python Function object type Function struct { Code *Code // A code object, the __code__ attribute + Ctx Ctx // Host VM context Globals StringDict // A dictionary (other mappings won't do) Defaults Tuple // NULL or a tuple KwDefaults StringDict // NULL or a dict @@ -26,7 +27,6 @@ type Function struct { Name string // The __name__ attribute, a string object Dict StringDict // The __dict__ attribute, a dict or NULL Weakreflist List // List of weak references - Module Object // The __module__ attribute, can be anything Annotations StringDict // Annotations, a dict or NULL Qualname string // The qualified name } @@ -56,9 +56,8 @@ func (f *Function) GetDict() StringDict { // attribute. qualname should be a unicode object or ""; if "", the // __qualname__ attribute is set to the same value as its __name__ // attribute. -func NewFunction(code *Code, globals StringDict, qualname string) *Function { +func NewFunction(code *Code, ctx Ctx, globals StringDict, qualname string) *Function { var doc Object - var module Object = None if len(code.Consts) >= 1 { doc = code.Consts[0] if _, ok := doc.(String); !ok { @@ -68,29 +67,24 @@ func NewFunction(code *Code, globals StringDict, qualname string) *Function { doc = None } - // __module__: If module name is in globals, use it. Otherwise, use None. - if moduleobj, ok := globals["__name__"]; ok { - module = moduleobj - } - if qualname == "" { qualname = code.Name } return &Function{ Code: code, + Ctx: ctx, Qualname: qualname, Globals: globals, Name: code.Name, Doc: doc, - Module: module, Dict: make(StringDict), } } // Call a function func (f *Function) M__call__(args Tuple, kwargs StringDict) (Object, error) { - result, err := VmEvalCodeEx(f.Code, f.Globals, NewStringDict(), args, kwargs, f.Defaults, f.KwDefaults, f.Closure) + result, err := VmEvalCode(f.Ctx, f.Code, f.Globals, NewStringDict(), args, kwargs, f.Defaults, f.KwDefaults, f.Closure) if err != nil { return nil, err } diff --git a/py/import.go b/py/import.go old mode 100644 new mode 100755 index 3cd1fd64..009976f4 --- a/py/import.go +++ b/py/import.go @@ -7,17 +7,19 @@ package py import ( - "io/ioutil" - "os" "path" - "path/filepath" "strings" ) -var ( - // This will become sys.path one day ;-) - modulePath = []string{"", "/usr/lib/python3.4", "/usr/local/lib/python3.4/dist-packages", "/usr/lib/python3/dist-packages"} -) +func Import(ctx Ctx, names ...string) error { + for _, name := range names { + _, err := ImportModuleLevelObject(ctx, name, nil, nil, nil, 0) + if err != nil { + return err + } + } + return nil +} // The workings of __import__ // @@ -78,9 +80,18 @@ var ( // // Changed in version 3.3: Negative values for level are no longer // supported (which also changes the default value to 0). -func ImportModuleLevelObject(name string, globals, locals StringDict, fromlist Tuple, level int) (Object, error) { +func ImportModuleLevelObject(ctx Ctx, name string, globals, locals StringDict, fromlist Tuple, level int) (Object, error) { // Module already loaded - return that - if module, ok := modules[name]; ok { + if module, err := ctx.GetModule(name); err == nil { + return module, nil + } + + // See if the module is a registered embeddded module that has not been loaded into this ctx yet. + if impl := GetModuleImpl(name); impl != nil { + module, err := impl.ModuleInit(ctx) + if err != nil { + return nil, err + } return module, nil } @@ -88,58 +99,27 @@ func ImportModuleLevelObject(name string, globals, locals StringDict, fromlist T return nil, ExceptionNewf(SystemError, "Relative import not supported yet") } + // Convert import's dot separators into path seps parts := strings.Split(name, ".") pathParts := path.Join(parts...) - for _, mpath := range modulePath { - if mpath == "" { - mpathObj, ok := globals["__file__"] - if !ok { - var err error - mpath, err = os.Getwd() - if err != nil { - return nil, err - } - } else { - mpath = path.Dir(string(mpathObj.(String))) - } - } - fullPath := path.Join(mpath, pathParts) - // FIXME Read pyc/pyo too - fullPath, err := filepath.Abs(fullPath) - if err != nil { - continue - } - if fi, err := os.Stat(fullPath); err == nil && fi.IsDir() { - // FIXME this is a massive simplification! - fullPath = path.Join(fullPath, "__init__.py") - } else { - fullPath += ".py" - } - // Check if file exists - if _, err := os.Stat(fullPath); err == nil { - str, err := ioutil.ReadFile(fullPath) - if err != nil { - return nil, ExceptionNewf(OSError, "Couldn't read %q: %v", fullPath, err) - } - codeObj, err := Compile(string(str), fullPath, "exec", 0, true) - if err != nil { - return nil, err - } - code, ok := codeObj.(*Code) - if !ok { - return nil, ExceptionNewf(ImportError, "Compile didn't return code object") - } - module := NewModule(name, "", nil, nil) - _, err = VmRun(module.Globals, module.Globals, code, nil) - if err != nil { - return nil, err - } - module.Globals["__file__"] = String(fullPath) - return module, nil - } + opts := RunOpts{ + ModuleName: name, + UseSysPaths: true, + } + + if fromFile, ok := globals["__file__"]; ok { + opts.CurDir = path.Dir(string(fromFile.(String))) + } + + module, err := ctx.RunFile(pathParts, opts) + if err != nil { + return nil, err } - return nil, ExceptionNewf(ImportError, "No module named '%s'", name) + + return module, nil + + //return nil, ExceptionNewf(ImportError, "No module named '%s'", name) // Convert to absolute path if relative // Use __file__ from globals to work out what we are relative to @@ -163,7 +143,7 @@ func ImportModuleLevelObject(name string, globals, locals StringDict, fromlist T // This calls functins from _bootstrap.py which is a frozen module // // Too much functionality for the moment -func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Object, level int) (Object, error) { +func XImportModuleLevelObject(ctx Ctx, nameObj, given_globals, locals, given_fromlist Object, level int) (Object, error) { var abs_name string var builtins_import Object var final_mod Object @@ -175,6 +155,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj var ok bool var name string var err error + store := ctx.Store() // Make sure to use default values so as to not have // PyObject_CallMethodObjArgs() truncate the parameter list because of a @@ -239,7 +220,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj } } - if _, ok = modules[string(Package)]; !ok { + if _, err = ctx.GetModule(string(Package)); err != nil { return nil, ExceptionNewf(SystemError, "Parent module %q not loaded, cannot perform relative import", Package) } } else { // level == 0 */ @@ -277,16 +258,16 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj // From this point forward, goto error_with_unlock! builtins_import, ok = globals["__import__"] if !ok { - builtins_import, ok = Builtins.Globals["__import__"] + builtins_import, ok = store.Builtins.Globals["__import__"] if !ok { return nil, ExceptionNewf(ImportError, "__import__ not found") } } - mod, ok = modules[abs_name] - if mod == None { + mod, err = ctx.GetModule(abs_name) + if err != nil || mod == None { return nil, ExceptionNewf(ImportError, "import of %q halted; None in sys.modules", abs_name) - } else if ok { + } else if err == nil { var value Object var err error initializing := false @@ -306,7 +287,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj } if initializing { // _bootstrap._lock_unlock_module() releases the import lock */ - value, err = Importlib.Call("_lock_unlock_module", Tuple{String(abs_name)}, nil) + _, err = store.Importlib.Call("_lock_unlock_module", Tuple{String(abs_name)}, nil) if err != nil { return nil, err } @@ -318,7 +299,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj } } else { // _bootstrap._find_and_load() releases the import lock - mod, err = Importlib.Call("_find_and_load", Tuple{String(abs_name), builtins_import}, nil) + mod, err = store.Importlib.Call("_find_and_load", Tuple{String(abs_name), builtins_import}, nil) if err != nil { return nil, err } @@ -345,8 +326,8 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj cut_off := len(name) - len(front) abs_name_len := len(abs_name) to_return := abs_name[:abs_name_len-cut_off] - final_mod, ok = modules[to_return] - if !ok { + final_mod, err = ctx.GetModule(to_return) + if err != nil { return nil, ExceptionNewf(KeyError, "%q not in sys.modules as expected", to_return) } } @@ -354,7 +335,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj final_mod = mod } } else { - final_mod, err = Importlib.Call("_handle_fromlist", Tuple{mod, fromlist, builtins_import}, nil) + final_mod, err = store.Importlib.Call("_handle_fromlist", Tuple{mod, fromlist, builtins_import}, nil) if err != nil { return nil, err } @@ -376,7 +357,7 @@ error: } // The actual import code -func BuiltinImport(self Object, args Tuple, kwargs StringDict, currentGlobal StringDict) (Object, error) { +func BuiltinImport(ctx Ctx, self Object, args Tuple, kwargs StringDict, currentGlobal StringDict) (Object, error) { kwlist := []string{"name", "globals", "locals", "fromlist", "level"} var name Object var globals Object = currentGlobal @@ -391,5 +372,5 @@ func BuiltinImport(self Object, args Tuple, kwargs StringDict, currentGlobal Str if fromlist == None { fromlist = Tuple{} } - return ImportModuleLevelObject(string(name.(String)), globals.(StringDict), locals.(StringDict), fromlist.(Tuple), int(level.(Int))) + return ImportModuleLevelObject(ctx, string(name.(String)), globals.(StringDict), locals.(StringDict), fromlist.(Tuple), int(level.(Int))) } diff --git a/py/py.go b/py/py.go old mode 100644 new mode 100755 index 0c48ee7b..d9fd3351 --- a/py/py.go +++ b/py/py.go @@ -24,15 +24,10 @@ type IGoInt64 interface { GoInt64() (int64, error) } -// Some well known objects var ( // Set in vm/eval.go - to avoid circular import - VmRun func(globals, locals StringDict, code *Code, closure Tuple) (res Object, err error) - VmRunFrame func(frame *Frame) (res Object, err error) - VmEvalCodeEx func(co *Code, globals, locals StringDict, args []Object, kws StringDict, defs []Object, kwdefs StringDict, closure Tuple) (retval Object, err error) - - // See compile/compile.go - set to avoid circular import - Compile func(str, filename, mode string, flags int, dont_inherit bool) (Object, error) + VmEvalCode func(ctx Ctx, code *Code, globals, locals StringDict, args []Object, kws StringDict, defs []Object, kwdefs StringDict, closure Tuple) (retval Object, err error) + VmRunFrame func(frame *Frame) (res Object, err error) ) // Called to create a new instance of class cls. __new__() is a static method (special-cased so you need not declare it as such) that takes the class of which an instance was requested as its first argument. The remaining arguments are those passed to the object constructor expression (the call to the class). The return value of __new__() should be the new object instance (usually an instance of cls). diff --git a/py/run.go b/py/run.go index 68af8df3..8c6f276a 100755 --- a/py/run.go +++ b/py/run.go @@ -21,12 +21,12 @@ const ( // Rename to Ctx? type Ctx interface { + // These each initiate blocking execution. - //RunAsModule(opts RunOpts) (*Module, error) - //RunCodeAsModule(code *Code, moduleName string, fileDesc string) (*Module, error) - Run(globals, locals StringDict, code *Code, closure Tuple) (res Object, err error) - RunFrame(frame *Frame) (res Object, err error) - EvalCodeEx(co *Code, globals, locals StringDict, args []Object, kws StringDict, defs []Object, kwdefs StringDict, closure Tuple) (retval Object, err error) + RunCode(code *Code, globals, locals StringDict, closure Tuple) (res Object, err error) + + // Runs a given file in the given host module. + RunFile(runPath string, opts RunOpts) (*Module, error) // // Execution of any of the above will stop when the next opcode runs // SignalHalt() @@ -53,11 +53,11 @@ const ( CoreLibs = Lib_sys | Lib_time ) -type RunParams struct { - ModuleInfo ModuleInfo // Newly created module to execute within (if ModuleInfo.Name is nil, then "__main__" is used) - NoopOnFNF bool // If set and Pathname did not resolve, (nil, nil) is returned (vs an error) - Silent bool // If set and an error occurs, no error info is printed - Pathname string +type RunOpts struct { + HostModule *Module // Host module to execute within (if nil, a new module is created) + ModuleName string // If HostModule == nil, this is the name of the newly created module. If nil, "__main__" is used. + CurDir string // If non-nil, this is the path of the current working directory. If nil, os.Getwd() is used + UseSysPaths bool } var DefaultCtxOpts = CtxOpts{} @@ -67,13 +67,8 @@ type CtxOpts struct { SetSysArgs bool } -// Some well known objects +// High-level entry points var ( NewCtx func(opts CtxOpts) Ctx - - // // Called at least once before using gpython; multiple calls to it have no effect. - // // Called each time NewCtx is called - // Init func() Compile func(str, filename string, mode CompileMode, flags int, dont_inherit bool) (Object, error) - Run func(ctx Ctx, params RunParams) (*Module, error) ) diff --git a/pytest/pytest.go b/pytest/pytest.go index 7dbd6f3d..ad159fec 100644 --- a/pytest/pytest.go +++ b/pytest/pytest.go @@ -11,13 +11,14 @@ import ( "strings" "testing" - _ "github.com/go-python/gpython/builtin" + _ "github.com/go-python/gpython/modules" + "github.com/go-python/gpython/compile" "github.com/go-python/gpython/py" - _ "github.com/go-python/gpython/sys" - "github.com/go-python/gpython/vm" ) +var gCtx = py.NewCtx(py.DefaultCtxOpts) + // Compile the program in the file prog to code in the module that is returned func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) { f, err := os.Open(prog) @@ -35,39 +36,24 @@ func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) { t.Fatalf("%s: ReadAll failed: %v", prog, err) } - obj, err := compile.Compile(string(str), prog, "exec", 0, true) + obj, err := compile.Compile(string(str), prog, py.ExecMode, 0, true) if err != nil { t.Fatalf("%s: Compile failed: %v", prog, err) } code := obj.(*py.Code) - module := py.NewModule("__main__", "", nil, nil) - module.Globals["__file__"] = py.String(prog) + module := gCtx.Store().NewModule(gCtx, py.ModuleInfo{ + FileDesc: prog, + }, nil, nil) return module, code } // Run the code in the module func run(t testing.TB, module *py.Module, code *py.Code) { - _, err := vm.Run(module.Globals, module.Globals, code, nil) + _, err := gCtx.RunCode(code, module.Globals, module.Globals, nil) if err != nil { - if wantErr, ok := module.Globals["err"]; ok { - wantErrObj, ok := wantErr.(py.Object) - if !ok { - t.Fatalf("want err is not py.Object: %#v", wantErr) - } - gotExc, ok := err.(py.ExceptionInfo) - if !ok { - t.Fatalf("got err is not ExceptionInfo: %#v", err) - } - if gotExc.Value.Type() != wantErrObj.Type() { - t.Fatalf("Want exception %v got %v", wantErrObj, gotExc.Value) - } - // t.Logf("matched exception") - return - } else { - py.TracebackDump(err) - t.Fatalf("Run failed: %v at %q", err, module.Globals["doc"]) - } + py.TracebackDump(err) + t.Fatalf("Run failed: %v at %q", err, module.Globals["doc"]) } // t.Logf("%s: Return = %v", prog, res) diff --git a/repl/repl.go b/repl/repl.go old mode 100644 new mode 100755 index d7fd9600..a7b9da3c --- a/repl/repl.go +++ b/repl/repl.go @@ -10,7 +10,6 @@ import ( "sort" "strings" - "github.com/go-python/gpython/compile" "github.com/go-python/gpython/py" "github.com/go-python/gpython/vm" ) @@ -23,7 +22,8 @@ const ( // Repl state type REPL struct { - module *py.Module + Ctx py.Ctx + Module *py.Module prog string continuation bool previous string @@ -40,14 +40,20 @@ type UI interface { } // New create a new REPL and initialises the state machine -func New() *REPL { +func New(ctx py.Ctx) *REPL { + if ctx == nil { + ctx = py.NewCtx(py.DefaultCtxOpts) + } + r := &REPL{ - module: py.NewModule("__main__", "", nil, nil), + Ctx: ctx, prog: "", continuation: false, previous: "", } - r.module.Globals["__file__"] = py.String(r.prog) + r.Module = ctx.Store().NewModule(ctx, py.ModuleInfo{ + FileDesc: r.prog, + }, nil, nil) return r } @@ -76,7 +82,7 @@ func (r *REPL) Run(line string) { if toCompile == "" { return } - obj, err := compile.Compile(toCompile+"\n", r.prog, "single", 0, true) + obj, err := py.Compile(toCompile+"\n", r.prog, py.SingleMode, 0, true) if err != nil { // Detect that we should start a continuation line // FIXME detect EOF properly! @@ -100,7 +106,7 @@ func (r *REPL) Run(line string) { return } code := obj.(*py.Code) - _, err = vm.Run(r.module.Globals, r.module.Globals, code, nil) + _, err = r.Ctx.RunCode(code, r.Module.Globals, r.Module.Globals, nil) if err != nil { py.TracebackDump(err) } @@ -129,8 +135,8 @@ func (r *REPL) Completer(line string, pos int) (head string, completions []strin } } } - match(r.module.Globals) - match(py.Builtins.Globals) + match(r.Module.Globals) + match(r.Ctx.Store().Builtins.Globals) sort.Strings(completions) return head, completions, tail } diff --git a/vm/builtin.go b/vm/builtin.go old mode 100644 new mode 100755 index 3fcdccac..e07343a4 --- a/vm/builtin.go +++ b/vm/builtin.go @@ -12,13 +12,13 @@ import ( "github.com/go-python/gpython/py" ) -func builtinEvalOrExec(self py.Object, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict, mode string) (py.Object, error) { +func builtinEvalOrExec(ctx py.Ctx, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict, mode py.CompileMode) (py.Object, error) { var ( cmd py.Object globals py.Object = py.None locals py.Object = py.None ) - err := py.UnpackTuple(args, kwargs, mode, 1, 3, &cmd, &globals, &locals) + err := py.UnpackTuple(args, kwargs, string(mode), 1, 3, &cmd, &globals, &locals) if err != nil { return nil, err } @@ -69,15 +69,15 @@ func builtinEvalOrExec(self py.Object, args py.Tuple, kwargs, currentLocals, cur if code.GetNumFree() > 0 { return nil, py.ExceptionNewf(py.TypeError, "code passed to %s() may not contain free variables", mode) } - return EvalCode(code, globalsDict, localsDict) + return ctx.RunCode(code, globalsDict, localsDict, nil) } -func builtinEval(self py.Object, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { - return builtinEvalOrExec(self, args, kwargs, currentLocals, currentGlobals, builtins, "eval") +func builtinEval(ctx py.Ctx, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { + return builtinEvalOrExec(ctx, args, kwargs, currentLocals, currentGlobals, builtins, py.EvalMode) } -func builtinExec(self py.Object, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { - _, err := builtinEvalOrExec(self, args, kwargs, currentLocals, currentGlobals, builtins, "exec") +func builtinExec(ctx py.Ctx, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { + _, err := builtinEvalOrExec(ctx, args, kwargs, currentLocals, currentGlobals, builtins, py.ExecMode) if err != nil { return nil, err } diff --git a/vm/eval.go b/vm/eval.go old mode 100644 new mode 100755 index e98392a9..6b6f0788 --- a/vm/eval.go +++ b/vm/eval.go @@ -767,7 +767,7 @@ func do_END_FINALLY(vm *Vm, arg int32) error { // Loads the __build_class__ helper function to the stack which // creates a new class object. func do_LOAD_BUILD_CLASS(vm *Vm, arg int32) error { - vm.PUSH(py.Builtins.Globals["__build_class__"]) + vm.PUSH(vm.ctx.Store().Builtins.Globals["__build_class__"]) return nil } @@ -1087,9 +1087,11 @@ func do_IMPORT_NAME(vm *Vm, namei int32) error { } v := vm.POP() u := vm.TOP() - var locals py.Object = vm.frame.Locals - if locals == nil { + var locals py.Object + if vm.frame.Locals == nil { locals = py.None + } else { + locals = vm.frame.Locals } var args py.Tuple if _, ok := u.(py.Int); ok { @@ -1435,7 +1437,7 @@ func _make_function(vm *Vm, argc int32, opcode OpCode) { num_annotations := (argc >> 16) & 0x7fff qualname := vm.POP() code := vm.POP() - function := py.NewFunction(code.(*py.Code), vm.frame.Globals, string(qualname.(py.String))) + function := py.NewFunction(code.(*py.Code), vm.ctx, vm.frame.Globals, string(qualname.(py.String))) if opcode == MAKE_CLOSURE { function.Closure = vm.POP().(py.Tuple) @@ -1579,7 +1581,7 @@ func EvalGetFuncDesc(fn py.Object) string { } } -// As py.Call but takes an intepreter Frame object +// As py.Call but takes an interpreter Frame object // // Used to implement some interpreter magic like locals(), globals() etc func callInternal(fn py.Object, args py.Tuple, kwargs py.StringDict, f *py.Frame) (py.Object, error) { @@ -1592,13 +1594,13 @@ func callInternal(fn py.Object, args py.Tuple, kwargs py.StringDict, f *py.Frame f.FastToLocals() return f.Locals, nil case py.InternalMethodImport: - return py.BuiltinImport(nil, args, kwargs, f.Globals) + return py.BuiltinImport(f.Ctx, nil, args, kwargs, f.Globals) case py.InternalMethodEval: f.FastToLocals() - return builtinEval(nil, args, kwargs, f.Locals, f.Globals, f.Builtins) + return builtinEval(f.Ctx.Store().Importlib.Ctx, args, kwargs, f.Locals, f.Globals, f.Builtins) case py.InternalMethodExec: f.FastToLocals() - return builtinExec(nil, args, kwargs, f.Locals, f.Globals, f.Builtins) + return builtinExec(f.Ctx, args, kwargs, f.Locals, f.Globals, f.Builtins) default: return nil, py.ExceptionNewf(py.SystemError, "Internal method %v not found", x) } @@ -1732,6 +1734,7 @@ func (vm *Vm) UnwindExceptHandler(frame *py.Frame, block *py.TryBlock) { func RunFrame(frame *py.Frame) (res py.Object, err error) { var vm = Vm{ frame: frame, + ctx: frame.Ctx, } // FIXME need to do this to save the old exeption when we @@ -2033,7 +2036,14 @@ func tooManyPositional(co *py.Code, given, defcount int, fastlocals []py.Object) chooseString(given == 1 && kwonly_given == 0, "was", "were")) } -func EvalCodeEx(co *py.Code, globals, locals py.StringDict, args []py.Object, kws py.StringDict, defs []py.Object, kwdefs py.StringDict, closure py.Tuple) (retval py.Object, err error) { +// Run a new virtual machine on a Code object +// +// Any parameters are expected to have been decoded into locals +// +// Returns an Object and an error. The error will be a py.ExceptionInfo +// +// This is the equivalent of PyEval_EvalCode with closure support +func EvalCode(ctx py.Ctx, co *py.Code, globals, locals py.StringDict, args []py.Object, kws py.StringDict, defs []py.Object, kwdefs py.StringDict, closure py.Tuple) (retval py.Object, err error) { total_args := int(co.Argcount + co.Kwonlyargcount) n := len(args) var kwdict py.StringDict @@ -2045,7 +2055,7 @@ func EvalCodeEx(co *py.Code, globals, locals py.StringDict, args []py.Object, kw //assert(tstate != nil) //assert(globals != nil) // f = PyFrame_New(tstate, co, globals, locals) - f := py.NewFrame(globals, locals, co, closure) // FIXME extra closure parameter? + f := py.NewFrame(ctx, globals, locals, co, closure) // FIXME extra closure parameter? fastlocals := f.Localsplus freevars := f.CellAndFreeVars @@ -2162,34 +2172,9 @@ func EvalCodeEx(co *py.Code, globals, locals py.StringDict, args []py.Object, kw return RunFrame(f) } -func EvalCode(co *py.Code, globals, locals py.StringDict) (py.Object, error) { - return EvalCodeEx(co, - globals, locals, - nil, - nil, - nil, - nil, nil) -} - -// Run the virtual machine on a Code object -// -// Any parameters are expected to have been decoded into locals -// -// Returns an Object and an error. The error will be a py.ExceptionInfo -// -// This is the equivalent of PyEval_EvalCode with closure support -func Run(globals, locals py.StringDict, code *py.Code, closure py.Tuple) (res py.Object, err error) { - return EvalCodeEx(code, - globals, locals, - nil, - nil, - nil, - nil, closure) -} // Write the py global to avoid circular import func init() { - py.VmRun = Run + py.VmEvalCode = EvalCode py.VmRunFrame = RunFrame - py.VmEvalCodeEx = EvalCodeEx } diff --git a/vm/vm.go b/vm/vm.go old mode 100644 new mode 100755 index 4eb3cb20..8df37c9e --- a/vm/vm.go +++ b/vm/vm.go @@ -41,4 +41,6 @@ type Vm struct { curexc py.ExceptionInfo // Previous exception type, value and traceback exc py.ExceptionInfo + // This vm's access to persistent state and modules + ctx py.Ctx } From 7f7d213375c0bb1ed7d47c6b5d47b11240d72c42 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Thu, 9 Dec 2021 16:31:05 -0600 Subject: [PATCH 08/79] repl.New() now accepts an existing py.Ctx --- repl/cli/cli.go | 10 ++++++---- repl/repl_test.go | 4 ++-- repl/web/main.go | 3 ++- 3 files changed, 10 insertions(+), 7 deletions(-) mode change 100644 => 100755 repl/cli/cli.go mode change 100644 => 100755 repl/repl_test.go diff --git a/repl/cli/cli.go b/repl/cli/cli.go old mode 100644 new mode 100755 index 2e1da88a..90f1463b --- a/repl/cli/cli.go +++ b/repl/cli/cli.go @@ -119,10 +119,12 @@ func (rl *readline) Print(out string) { } // RunREPL starts the REPL loop -func RunREPL() { - repl := repl.New() - rl := newReadline(repl) - repl.SetUI(rl) +func RunREPL(replCtx *repl.REPL) { + if replCtx == nil { + replCtx = repl.New(nil) + } + rl := newReadline(replCtx) + replCtx.SetUI(rl) defer rl.Close() err := rl.ReadHistory() if err != nil { diff --git a/repl/repl_test.go b/repl/repl_test.go old mode 100644 new mode 100755 index a98ce9a9..39715a9b --- a/repl/repl_test.go +++ b/repl/repl_test.go @@ -38,7 +38,7 @@ func (rt *replTest) assert(t *testing.T, what, wantPrompt, wantOut string) { } func TestREPL(t *testing.T) { - r := New() + r := New(nil) rt := &replTest{} r.SetUI(rt) @@ -78,7 +78,7 @@ func TestREPL(t *testing.T) { } func TestCompleter(t *testing.T) { - r := New() + r := New(nil) rt := &replTest{} r.SetUI(rt) diff --git a/repl/web/main.go b/repl/web/main.go index bad66f16..16a2eaaa 100644 --- a/repl/web/main.go +++ b/repl/web/main.go @@ -4,6 +4,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build js // +build js package main @@ -77,7 +78,7 @@ func main() { node.Get("classList").Call("add", "active") // Make a repl referring to an empty term for the moment - REPL := repl.New() + REPL := repl.New(nil) cb := js.NewCallback(func(args []js.Value) { REPL.Run(args[0].String()) }) From 708bb7ee82b2d28805c2ee18f6ce05c2ed6dfdb9 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Thu, 9 Dec 2021 16:42:12 -0600 Subject: [PATCH 09/79] fixed typo in callInternal() --- vm/eval.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/eval.go b/vm/eval.go index 6b6f0788..b0c430c1 100755 --- a/vm/eval.go +++ b/vm/eval.go @@ -1597,7 +1597,7 @@ func callInternal(fn py.Object, args py.Tuple, kwargs py.StringDict, f *py.Frame return py.BuiltinImport(f.Ctx, nil, args, kwargs, f.Globals) case py.InternalMethodEval: f.FastToLocals() - return builtinEval(f.Ctx.Store().Importlib.Ctx, args, kwargs, f.Locals, f.Globals, f.Builtins) + return builtinEval(f.Ctx, args, kwargs, f.Locals, f.Globals, f.Builtins) case py.InternalMethodExec: f.FastToLocals() return builtinExec(f.Ctx, args, kwargs, f.Locals, f.Globals, f.Builtins) From a244256fec0fc950363c1e764056f3006f12e23c Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Thu, 9 Dec 2021 16:45:40 -0600 Subject: [PATCH 10/79] unbroke test compatibilty --- pytest/pytest.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pytest/pytest.go b/pytest/pytest.go index ad159fec..1126446f 100644 --- a/pytest/pytest.go +++ b/pytest/pytest.go @@ -52,8 +52,20 @@ func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) { func run(t testing.TB, module *py.Module, code *py.Code) { _, err := gCtx.RunCode(code, module.Globals, module.Globals, nil) if err != nil { - py.TracebackDump(err) - t.Fatalf("Run failed: %v at %q", err, module.Globals["doc"]) + if wantErrObj, ok := module.Globals["err"]; ok { + gotExc, ok := err.(py.ExceptionInfo) + if !ok { + t.Fatalf("got err is not ExceptionInfo: %#v", err) + } + if gotExc.Value.Type() != wantErrObj.Type() { + t.Fatalf("Want exception %v got %v", wantErrObj, gotExc.Value) + } + // t.Logf("matched exception") + return + } else { + py.TracebackDump(err) + t.Fatalf("Run failed: %v at %q", err, module.Globals["doc"]) + } } // t.Logf("%s: Return = %v", prog, res) From 75d874323cff1fb50f8a1b9c65759f7c07347b85 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Fri, 10 Dec 2021 12:12:46 -0600 Subject: [PATCH 11/79] cleaned up py.Ctx initialization wrt sys and DefaultCtxOpts() --- main.go | 4 ++-- modules/runtime.go | 7 ++++--- py/run.go | 41 ++++++++++++++++++++++++++++++----------- pytest/pytest.go | 9 ++++++--- repl/repl.go | 2 +- sys/sys.go | 31 +++++++++++-------------------- 6 files changed, 54 insertions(+), 40 deletions(-) diff --git a/main.go b/main.go index 8c3fc908..b3a51fb3 100755 --- a/main.go +++ b/main.go @@ -44,8 +44,8 @@ func main() { flag.Parse() args := flag.Args() - opts := py.DefaultCtxOpts - opts.Args = flag.Args() + opts := py.DefaultCtxOpts() + opts.SysArgs = flag.Args() ctx := py.NewCtx(opts) if *cpuprofile != "" { diff --git a/modules/runtime.go b/modules/runtime.go index ebb8fb9c..5d1c5782 100755 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -137,9 +137,10 @@ func NewCtx(opts py.CtxOpts) py.Ctx { ctx.store = py.NewStore() py.Import(ctx, "builtins", "sys") - if opts.SetSysArgs { - ctx.Store().MustGetModule("sys").Globals["argv"] = py.NewListFromStrings(opts.Args) - } + + sys_mod := ctx.Store().MustGetModule("sys") + sys_mod.Globals["argv"] = py.NewListFromStrings(opts.SysArgs) + sys_mod.Globals["path"] = py.NewListFromStrings(opts.SysPaths) return ctx } diff --git a/py/run.go b/py/run.go index 8c6f276a..23503e14 100755 --- a/py/run.go +++ b/py/run.go @@ -15,11 +15,10 @@ const ( SmartCodeAcquire RunFlags = 0x01 ) -// type Sys interface { -// ResetArfs -// } - -// Rename to Ctx? +// Ctx is gpython virtual environment instance ("context"), providing a mechanism +// for multple gpython interpreters to run concurrently without restriction. +// +// See BenchmarkCtx() in vm/vm_test.go type Ctx interface { // These each initiate blocking execution. @@ -28,12 +27,14 @@ type Ctx interface { // Runs a given file in the given host module. RunFile(runPath string, opts RunOpts) (*Module, error) - // // Execution of any of the above will stop when the next opcode runs + // Execution of any of the above will stop when the next opcode runs + // @@TODO // SignalHalt() - //Sys() Sys - + // Returns the named module for this context (or an error if not found) GetModule(moduleName string) (*Module, error) + + // WIP Store() *Store } @@ -60,15 +61,33 @@ type RunOpts struct { UseSysPaths bool } -var DefaultCtxOpts = CtxOpts{} +// Can be changed during runtime and will \play nice with others using DefaultCtxOpts() +var CoreSysPaths = []string{ + ".", + "lib", +} + +// Can be changed during runtime and will \play nice with others using DefaultCtxOpts() +var AuxSysPaths = []string{ + "/usr/lib/python3.4", + "/usr/local/lib/python3.4/dist-packages", + "/usr/lib/python3/dist-packages", +} type CtxOpts struct { - Args []string - SetSysArgs bool + SysArgs []string // sys.argv initializer + SysPaths []string // sys.path initializer } // High-level entry points var ( + DefaultCtxOpts = func() CtxOpts { + opts := CtxOpts{ + SysPaths: CoreSysPaths, + } + opts.SysPaths = append(opts.SysPaths, AuxSysPaths...) + return opts + } NewCtx func(opts CtxOpts) Ctx Compile func(str, filename string, mode CompileMode, flags int, dont_inherit bool) (Object, error) ) diff --git a/pytest/pytest.go b/pytest/pytest.go index 1126446f..0b9de29a 100644 --- a/pytest/pytest.go +++ b/pytest/pytest.go @@ -17,7 +17,7 @@ import ( "github.com/go-python/gpython/py" ) -var gCtx = py.NewCtx(py.DefaultCtxOpts) +var gCtx = py.NewCtx(py.DefaultCtxOpts()) // Compile the program in the file prog to code in the module that is returned func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) { @@ -35,14 +35,17 @@ func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) { if err != nil { t.Fatalf("%s: ReadAll failed: %v", prog, err) } + return CompileSrc(t, gCtx, string(str), prog) +} - obj, err := compile.Compile(string(str), prog, py.ExecMode, 0, true) +func CompileSrc(t testing.TB, ctx py.Ctx, pySrc string, prog string) (*py.Module, *py.Code) { + obj, err := compile.Compile(string(pySrc), prog, py.ExecMode, 0, true) if err != nil { t.Fatalf("%s: Compile failed: %v", prog, err) } code := obj.(*py.Code) - module := gCtx.Store().NewModule(gCtx, py.ModuleInfo{ + module := ctx.Store().NewModule(ctx, py.ModuleInfo{ FileDesc: prog, }, nil, nil) return module, code diff --git a/repl/repl.go b/repl/repl.go index a7b9da3c..5fb19add 100755 --- a/repl/repl.go +++ b/repl/repl.go @@ -42,7 +42,7 @@ type UI interface { // New create a new REPL and initialises the state machine func New(ctx py.Ctx) *REPL { if ctx == nil { - ctx = py.NewCtx(py.DefaultCtxOpts) + ctx = py.NewCtx(py.DefaultCtxOpts()) } r := &REPL{ diff --git a/sys/sys.go b/sys/sys.go index d02350fc..c401214f 100755 --- a/sys/sys.go +++ b/sys/sys.go @@ -23,15 +23,6 @@ import ( "github.com/go-python/gpython/py" ) -// Alter this list before the sys module starts to alter the factory path set -var SysPathInitializer = []string{ - "./", // Denotes the dir of the current file (or cwd if __file__ is not set) - "./lib", - "/usr/lib/python3.4", - "/usr/local/lib/python3.4/dist-packages", - "/usr/lib/python3/dist-packages", -} - const module_doc = `This module provides access to some objects used or maintained by the interpreter and to functions that interact strongly with the interpreter. @@ -666,7 +657,13 @@ func init() { stdout := &py.File{File: os.Stdout, FileMode: py.FileWrite} stderr := &py.File{File: os.Stderr, FileMode: py.FileWrite} + executable, err := os.Executable() + if err != nil { + panic(err) + } + globals := py.StringDict{ + "path": py.NewList(), "argv": py.NewListFromStrings(os.Args[1:]), "stdin": stdin, "stdout": stdout, @@ -674,6 +671,8 @@ func init() { "__stdin__": stdin, "__stdout__": stdout, "__stderr__": stderr, + "executable": py.String(executable), + //"version": py.Int(MARSHAL_VERSION), // /* stdin/stdout/stderr are now set by pythonrun.c */ @@ -798,21 +797,13 @@ func init() { // #endif } - executable, err := os.Executable() - if err != nil { - panic(err) - } - - globals["executable"] = py.String(executable) - globals["path"] = py.NewListFromStrings(SysPathInitializer) - py.RegisterModule(&py.StaticModule{ Info: py.ModuleInfo{ - Name: "sys", - Doc: module_doc, + Name: "sys", + Doc: module_doc, }, Methods: methods, Globals: globals, }) - + } From d5fb8e86ba35b899287e9186340bc031e2150f4e Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Fri, 10 Dec 2021 12:13:22 -0600 Subject: [PATCH 12/79] added multi Ctx benchmark --- vm/vm_test.go | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/vm/vm_test.go b/vm/vm_test.go index f1686207..3949d555 100644 --- a/vm/vm_test.go +++ b/vm/vm_test.go @@ -5,8 +5,13 @@ package vm_test import ( + "fmt" + "strconv" + "strings" + "sync" "testing" + "github.com/go-python/gpython/py" "github.com/go-python/gpython/pytest" ) @@ -17,3 +22,72 @@ func TestVm(t *testing.T) { func BenchmarkVM(b *testing.B) { pytest.RunBenchmarks(b, "benchmarks") } + +var jobSrcTemplate = ` + +doc="multi py.Ctx text" +WORKER_ID = "{{WORKER_ID}}" +def fib(n): + if n == 0: + return 0 + elif n == 1: + return 1 + return fib(n - 2) + fib(n - 1) + +x = {{FIB_TO}} +fx = fib(x) +print("%s says fib(%d) is %d" % (WORKER_ID, x, fx)) +` + +type worker struct { + name string + ctx py.Ctx +} + +func (w *worker) run(b testing.TB, pySrc string, countUpto int) { + pySrc = strings.Replace(pySrc, "{{WORKER_ID}}", w.name, -1) + pySrc = strings.Replace(pySrc, "{{FIB_TO}}", strconv.Itoa(countUpto), -1) + + module, code := pytest.CompileSrc(b, w.ctx, pySrc, w.name) + _, err := w.ctx.RunCode(code, module.Globals, module.Globals, nil) + if err != nil { + b.Fatal(err) + } +} + +func BenchmarkCtx(b *testing.B) { + numWorkers := 4 + workersRunning := sync.WaitGroup{} + + numJobs := 35 + fmt.Printf("Starting %d workers to process %d jobs...\n", numWorkers, numJobs) + + jobPipe := make(chan int) + go func() { + for i := 1; i <= numJobs; i++ { + jobPipe <- i + } + close(jobPipe) + }() + + workers := make([]worker, numWorkers) + for i := 0; i < numWorkers; i++ { + + workers[i] = worker{ + name: fmt.Sprintf("Worker #%d", i+1), + ctx: py.NewCtx(py.DefaultCtxOpts()), + } + + workersRunning.Add(1) + w := workers[i] + go func() { + for jobID := range jobPipe { + w.run(b, jobSrcTemplate, jobID) + //fmt.Printf("### %s finished job %v ###\n", w.name, jobID) + } + workersRunning.Done() + }() + } + + workersRunning.Wait() +} From 269bdc0df13165bc6af3f8471a645edc5e790b57 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Fri, 10 Dec 2021 12:13:34 -0600 Subject: [PATCH 13/79] WIP --- py/util.go | 147 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100755 py/util.go diff --git a/py/util.go b/py/util.go new file mode 100755 index 00000000..767c643f --- /dev/null +++ b/py/util.go @@ -0,0 +1,147 @@ +package py + +import ( + "errors" + "strconv" +) + +func GetLen(obj Object) (Int, error) { + getlen, ok := obj.(I__len__) + if !ok { + return 0, nil + } + + lenObj, err := getlen.M__len__() + if err != nil { + return 0, err + } + + return GetInt(lenObj) +} + +func GetInt(obj Object) (Int, error) { + toIdx, ok := obj.(I__index__) + if !ok { + _, err := cantConvert(obj, "int") + return 0, err + } + + return toIdx.M__index__() +} + +func LoadTuple(args Tuple, vars []interface{}) error { + + if len(args) > len(vars) { + return ExceptionNewf(RuntimeError, "%d args given, expected %d", len(args), len(vars)) + } + + if len(vars) > len(args) { + vars = vars[:len(args)] + } + + for i, rval := range vars { + err := loadValue(args[i], rval) + if err == ErrUnsupportedObjType { + return ExceptionNewf(TypeError, "arg %d has unsupported object type: %s", i, args[i].Type().Name) + } + } + + return nil +} + +var ( + ErrUnsupportedObjType = errors.New("unsupported obj type") +) + +func LoadAttr(obj Object, attrName string, data interface{}) error { + attr, err := GetAttrString(obj, attrName) + if err != nil { + return err + } + err = loadValue(attr, data) + if err == ErrUnsupportedObjType { + return ExceptionNewf(TypeError, "attribute \"%s\" has unsupported object type: %s", attrName, attr.Type().Name) + } + return nil +} + +func loadValue(src Object, data interface{}) error { + + var ( + v_str string + v_float float64 + v_int int64 + ) + + if b, ok := src.(Bool); ok { + if b { + v_int = 1 + v_float = 1 + v_str = "True" + } else { + v_str = "False" + } + } else if val, ok := src.(Int); ok { + v_int = int64(val) + v_float = float64(val) + } else if val, ok := src.(Float); ok { + v_int = int64(val) + v_float = float64(val) + } else if val, ok := src.(String); ok { + v_str = string(val) + intval, _ := strconv.Atoi(v_str) + v_int = int64(intval) + } else if _, ok := src.(NoneType); ok { + // No-op + } else { + return ErrUnsupportedObjType + } + + switch dst := data.(type) { + case *bool: + *dst = v_int != 0 + case *int8: + *dst = int8(v_int) + case *uint8: + *dst = uint8(v_int) + case *int16: + *dst = int16(v_int) + case *uint16: + *dst = uint16(v_int) + case *int32: + *dst = int32(v_int) + case *uint32: + *dst = uint32(v_int) + case *int: + *dst = int(v_int) + case *uint: + *dst = uint(v_int) + case *int64: + *dst = v_int + case *uint64: + *dst = uint64(v_int) + case *float32: + *dst = float32(v_float) + case *float64: + *dst = v_float + case *string: + *dst = v_str + + // case []uint64: + // for i := range data { + // dst[i] = order.Uint64(bs[8*i:]) + // } + // case []float32: + // for i := range data { + // dst[i] = math.Float32frombits(order.Uint32(bs[4*i:])) + // } + // case []float64: + // for i := range data { + // dst[i] = math.Float64frombits(order.Uint64(bs[8*i:])) + // } + + default: + return ExceptionNewf(NotImplementedError, "%s", "unsupported Go data type") + } + return nil +} From 4db9c657df8b5a9792bea0dfc04d38c7681b62ce Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Fri, 10 Dec 2021 12:32:40 -0600 Subject: [PATCH 14/79] fixed py.NewCtx not being set --- repl/repl_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/repl/repl_test.go b/repl/repl_test.go index 39715a9b..9c64879e 100755 --- a/repl/repl_test.go +++ b/repl/repl_test.go @@ -6,10 +6,7 @@ import ( "testing" // import required modules - _ "github.com/go-python/gpython/builtin" - _ "github.com/go-python/gpython/math" - _ "github.com/go-python/gpython/sys" - _ "github.com/go-python/gpython/time" + _ "github.com/go-python/gpython/modules" ) type replTest struct { From 53d05dbde6906528c68b0da36cdb664361376174 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Fri, 10 Dec 2021 12:52:14 -0600 Subject: [PATCH 15/79] fixed List.sort() detection of instance invocation --- py/list.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py/list.go b/py/list.go index cae142e1..10141a18 100755 --- a/py/list.go +++ b/py/list.go @@ -41,8 +41,8 @@ func init() { ListType.Dict["sort"] = MustNewMethod("sort", func(self Object, args Tuple, kwargs StringDict) (Object, error) { const funcName = "sort" - var l *List - if self == None { + l, isList := self.(*List) + if !isList { // method called using `list.sort([], **kwargs)` var o Object err := UnpackTuple(args, nil, funcName, 1, 1, &o) From 7fd1c29ec59a7019e3dac647e0a86f8035e7e944 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Sat, 11 Dec 2021 16:04:54 -0600 Subject: [PATCH 16/79] separated path resolution from path evaluation --- modules/runtime.go | 115 +++++++++++++++++++++++++++++++-------------- 1 file changed, 80 insertions(+), 35 deletions(-) diff --git a/modules/runtime.go b/modules/runtime.go index 5d1c5782..b247a43c 100755 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -25,28 +25,68 @@ var defaultPaths = []py.Object{ py.String("."), } -func (ctx *ctx) RunFile(runPath string, opts py.RunOpts) (*py.Module, error) { +func resolveRunPath(runPath string, opts py.RunOpts, pathObjs []py.Object, tryPath func(pyPath string) (bool, error)) error { + var cwd string - tryPaths := defaultPaths - if opts.UseSysPaths { - tryPaths = ctx.Store().MustGetModule("sys").Globals["path"].(*py.List).Items + // Remove trailing slash if present + if runPath[len(runPath)-1] == '/' { + runPath = runPath[:len(runPath)-1] } - for _, pathObj := range tryPaths { + + var err error + + cont := true + + for _, pathObj := range pathObjs { pathStr, ok := pathObj.(py.String) if !ok { continue } + + // If an absolute path, just try that. + // Otherwise, check from the passed current dir then check from the current working dir. fpath := path.Join(string(pathStr), runPath) - if !filepath.IsAbs(fpath) { - if opts.CurDir == "" { - opts.CurDir, _ = os.Getwd() + if filepath.IsAbs(fpath) { + cont, err = tryPath(fpath) + } else { + if len(opts.CurDir) > 0 { + subPath := path.Join(opts.CurDir, fpath) + cont, err = tryPath(subPath) + } + if cont && err == nil { + if len(cwd) == 0 { + cwd, _ = os.Getwd() + } + subPath := path.Join(cwd, fpath) + cont, err = tryPath(subPath) } - fpath = path.Join(opts.CurDir, fpath) } - - if fpath[len(fpath)-1] == '/' { - fpath = fpath[:len(fpath)-1] + if !cont { + break } + } + + if err != nil { + return err + } + + if cont { + return py.ExceptionNewf(py.FileNotFoundError, "Failed to resolve %q", runPath) + } + + return err +} + +func (ctx *ctx) RunFile(runPath string, opts py.RunOpts) (*py.Module, error) { + tryPaths := defaultPaths + if opts.UseSysPaths { + tryPaths = ctx.Store().MustGetModule("sys").Globals["path"].(*py.List).Items + } + + var codeObj py.Object + var fileDesc string + + err := resolveRunPath(runPath, opts, tryPaths, func(fpath string) (bool, error) { stat, err := os.Stat(fpath) if err == nil && stat.IsDir() { @@ -62,59 +102,64 @@ func (ctx *ctx) RunFile(runPath string, opts py.RunOpts) (*py.Module, error) { _, err = os.Stat(fpath) } + // Keep searching while we get FNFs, stop on an error if err != nil { if os.IsNotExist(err) { - continue + return true, nil } err = py.ExceptionNewf(py.OSError, "Error accessing %q: %v", fpath, err) - return nil, err + return false, err } - var codeObj py.Object if ext == ".py" { var pySrc []byte pySrc, err = ioutil.ReadFile(fpath) if err != nil { - return nil, py.ExceptionNewf(py.OSError, "Error reading %q: %v", fpath, err) + return false, py.ExceptionNewf(py.OSError, "Error reading %q: %v", fpath, err) } codeObj, err = py.Compile(string(pySrc), fpath, py.ExecMode, 0, true) if err != nil { - return nil, err + return false, err } } else if ext == ".pyc" { var file *os.File file, err = os.Open(fpath) if err != nil { - return nil, py.ExceptionNewf(py.OSError, "Error opening %q: %v", fpath, err) + return false, py.ExceptionNewf(py.OSError, "Error opening %q: %v", fpath, err) } defer file.Close() codeObj, err = marshal.ReadPyc(file) if err != nil { - return nil, py.ExceptionNewf(py.ImportError, "Failed to marshal %q: %v", fpath, err) + return false, py.ExceptionNewf(py.ImportError, "Failed to marshal %q: %v", fpath, err) } } - var code *py.Code - if codeObj != nil { - code, _ = codeObj.(*py.Code) - } - if code == nil { - return nil, py.ExceptionNewf(py.AssertionError, "Missing code object") - } + fileDesc = fpath + return false, nil + }) - if opts.HostModule == nil { - opts.HostModule = ctx.Store().NewModule(ctx, py.ModuleInfo{ - Name: opts.ModuleName, - FileDesc: fpath, - }, nil, nil) - } + if err != nil { + return nil, err + } + + var code *py.Code + if codeObj != nil { + code, _ = codeObj.(*py.Code) + } + if code == nil { + return nil, py.ExceptionNewf(py.AssertionError, "Missing code object") + } - _, err = vm.EvalCode(ctx, code, opts.HostModule.Globals, opts.HostModule.Globals, nil, nil, nil, nil, nil) - return opts.HostModule, err + if opts.HostModule == nil { + opts.HostModule = ctx.Store().NewModule(ctx, py.ModuleInfo{ + Name: opts.ModuleName, + FileDesc: fileDesc, + }, nil, nil) } - return nil, py.ExceptionNewf(py.FileNotFoundError, "Failed to resolve %q", runPath) + _, err = vm.EvalCode(ctx, code, opts.HostModule.Globals, opts.HostModule.Globals, nil, nil, nil, nil, nil) + return opts.HostModule, err } func (ctx *ctx) RunCode(code *py.Code, globals, locals py.StringDict, closure py.Tuple) (py.Object, error) { From 8f3a1040b8a79bc8c641732c5423a41815de6daf Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Sat, 18 Dec 2021 17:24:17 -0600 Subject: [PATCH 17/79] removed cruft --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index a53dc4e1..6ee05de2 100755 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/go-python/py v0.0.0-20150724143143-adb74cbb1b5b h1:vTYiMD/1XTeL0SwwyVG30JveBV9ZMgJEZBwDcixC/oM= -github.com/go-python/py v0.0.0-20150724143143-adb74cbb1b5b/go.mod h1:FDA9/ZAzj70a4RlKS443aRrgA+IW1WLOWAFKZZ8tHXk= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSfgRJKqIzapE78onvvTbdi1rMkU00lZw= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherwasm v1.1.0 h1:fA2uLoctU5+T3OhOn2vYP0DVT6pxc7xhTlBB1paATqQ= From 5ebaf66936cc1f96e34a348cf7066c0369d9f7c7 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Sat, 18 Dec 2021 17:27:55 -0600 Subject: [PATCH 18/79] place ctx at top --- modules/runtime.go | 137 +++++++++++++++++++++++---------------------- 1 file changed, 70 insertions(+), 67 deletions(-) diff --git a/modules/runtime.go b/modules/runtime.go index b247a43c..bc179473 100755 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -18,63 +18,31 @@ import ( ) func init() { + // Assign the base-level py.Ctx creation function while also preventing an import cycle. py.NewCtx = NewCtx } -var defaultPaths = []py.Object{ - py.String("."), +// ctx implements py.Ctx +type ctx struct { + store *py.Store + opts py.CtxOpts } -func resolveRunPath(runPath string, opts py.RunOpts, pathObjs []py.Object, tryPath func(pyPath string) (bool, error)) error { - var cwd string - - // Remove trailing slash if present - if runPath[len(runPath)-1] == '/' { - runPath = runPath[:len(runPath)-1] +// See py.Ctx interface +func NewCtx(opts py.CtxOpts) py.Ctx { + ctx := &ctx{ + opts: opts, } - var err error - - cont := true - - for _, pathObj := range pathObjs { - pathStr, ok := pathObj.(py.String) - if !ok { - continue - } - - // If an absolute path, just try that. - // Otherwise, check from the passed current dir then check from the current working dir. - fpath := path.Join(string(pathStr), runPath) - if filepath.IsAbs(fpath) { - cont, err = tryPath(fpath) - } else { - if len(opts.CurDir) > 0 { - subPath := path.Join(opts.CurDir, fpath) - cont, err = tryPath(subPath) - } - if cont && err == nil { - if len(cwd) == 0 { - cwd, _ = os.Getwd() - } - subPath := path.Join(cwd, fpath) - cont, err = tryPath(subPath) - } - } - if !cont { - break - } - } + ctx.store = py.NewStore() - if err != nil { - return err - } + py.Import(ctx, "builtins", "sys") - if cont { - return py.ExceptionNewf(py.FileNotFoundError, "Failed to resolve %q", runPath) - } + sys_mod := ctx.Store().MustGetModule("sys") + sys_mod.Globals["argv"] = py.NewListFromStrings(opts.SysArgs) + sys_mod.Globals["path"] = py.NewListFromStrings(opts.SysPaths) - return err + return ctx } func (ctx *ctx) RunFile(runPath string, opts py.RunOpts) (*py.Module, error) { @@ -162,35 +130,70 @@ func (ctx *ctx) RunFile(runPath string, opts py.RunOpts) (*py.Module, error) { return opts.HostModule, err } -func (ctx *ctx) RunCode(code *py.Code, globals, locals py.StringDict, closure py.Tuple) (py.Object, error) { - return vm.EvalCode(ctx, code, globals, locals, nil, nil, nil, nil, closure) +var defaultPaths = []py.Object{ + py.String("."), } -func (ctx *ctx) GetModule(moduleName string) (*py.Module, error) { - return ctx.store.GetModule(moduleName) -} +func resolveRunPath(runPath string, opts py.RunOpts, pathObjs []py.Object, tryPath func(pyPath string) (bool, error)) error { + var cwd string -func (ctx *ctx) Store() *py.Store { - return ctx.store -} + // Remove trailing slash if present + if runPath[len(runPath)-1] == '/' { + runPath = runPath[:len(runPath)-1] + } -func NewCtx(opts py.CtxOpts) py.Ctx { - ctx := &ctx{ - opts: opts, + var err error + + cont := true + + for _, pathObj := range pathObjs { + pathStr, ok := pathObj.(py.String) + if !ok { + continue + } + + // If an absolute path, just try that. + // Otherwise, check from the passed current dir then check from the current working dir. + fpath := path.Join(string(pathStr), runPath) + if filepath.IsAbs(fpath) { + cont, err = tryPath(fpath) + } else { + if len(opts.CurDir) > 0 { + subPath := path.Join(opts.CurDir, fpath) + cont, err = tryPath(subPath) + } + if cont && err == nil { + if len(cwd) == 0 { + cwd, _ = os.Getwd() + } + subPath := path.Join(cwd, fpath) + cont, err = tryPath(subPath) + } + } + if !cont { + break + } } - ctx.store = py.NewStore() + if err != nil { + return err + } - py.Import(ctx, "builtins", "sys") + if cont { + return py.ExceptionNewf(py.FileNotFoundError, "Failed to resolve %q", runPath) + } - sys_mod := ctx.Store().MustGetModule("sys") - sys_mod.Globals["argv"] = py.NewListFromStrings(opts.SysArgs) - sys_mod.Globals["path"] = py.NewListFromStrings(opts.SysPaths) + return err +} - return ctx +func (ctx *ctx) RunCode(code *py.Code, globals, locals py.StringDict, closure py.Tuple) (py.Object, error) { + return vm.EvalCode(ctx, code, globals, locals, nil, nil, nil, nil, closure) } -type ctx struct { - store *py.Store - opts py.CtxOpts +func (ctx *ctx) GetModule(moduleName string) (*py.Module, error) { + return ctx.store.GetModule(moduleName) +} + +func (ctx *ctx) Store() *py.Store { + return ctx.store } From d423190e5a1b0f5c47f70df186d4667cb9bf4705 Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Sat, 18 Dec 2021 17:38:40 -0600 Subject: [PATCH 19/79] err msg tweak Co-authored-by: Sebastien Binet --- py/module.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/module.go b/py/module.go index f8705277..8f7db2c0 100755 --- a/py/module.go +++ b/py/module.go @@ -154,7 +154,7 @@ func (store *Store) NewModule(ctx Ctx, info ModuleInfo, methods []*Method, globa func (store *Store) GetModule(name string) (*Module, error) { m, ok := store.modules[name] if !ok { - return nil, ExceptionNewf(ImportError, "Module '%q' not found", name) + return nil, ExceptionNewf(ImportError, "Module '%s' not found", name) } return m, nil } From b33c2669dece47d86ed2884ca429b7862aad7b30 Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Sat, 18 Dec 2021 17:39:38 -0600 Subject: [PATCH 20/79] typo fix Co-authored-by: Sebastien Binet --- py/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/run.go b/py/run.go index 23503e14..18120701 100755 --- a/py/run.go +++ b/py/run.go @@ -16,7 +16,7 @@ const ( ) // Ctx is gpython virtual environment instance ("context"), providing a mechanism -// for multple gpython interpreters to run concurrently without restriction. +// for multiple gpython interpreters to run concurrently without restriction. // // See BenchmarkCtx() in vm/vm_test.go type Ctx interface { From efca7611ef771a8056d667a5f05a9758d97105cc Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Sat, 18 Dec 2021 17:50:07 -0600 Subject: [PATCH 21/79] for loops we can all agree on Co-authored-by: Sebastien Binet --- vm/vm_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/vm_test.go b/vm/vm_test.go index 3949d555..19c3dc46 100644 --- a/vm/vm_test.go +++ b/vm/vm_test.go @@ -64,8 +64,8 @@ func BenchmarkCtx(b *testing.B) { jobPipe := make(chan int) go func() { - for i := 1; i <= numJobs; i++ { - jobPipe <- i + for i := 0; i < numJobs; i++ { + jobPipe <- i+1 } close(jobPipe) }() From 1569475db8be754e9d519c18d0dfe265810b03d2 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Sat, 18 Dec 2021 17:51:29 -0600 Subject: [PATCH 22/79] removed outdated comments --- py/import.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/py/import.go b/py/import.go index 009976f4..1f3293ee 100755 --- a/py/import.go +++ b/py/import.go @@ -118,24 +118,6 @@ func ImportModuleLevelObject(ctx Ctx, name string, globals, locals StringDict, f } return module, nil - - //return nil, ExceptionNewf(ImportError, "No module named '%s'", name) - - // Convert to absolute path if relative - // Use __file__ from globals to work out what we are relative to - - // '' in path seems to mean use the current __file__ - - // Find a valid path which we need to check for the correct __init__.py in subdirectories etc - - // Look for .py and .pyc files - - // Make absolute module path too if we can for sys.modules - - //How do we uniquely identify modules? - - // SystemError: Parent module '' not loaded, cannot perform relative import - } // Straight port of the python code From 6372aeed63cec9edaca2c4857921379b2280cd06 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Sat, 18 Dec 2021 18:00:36 -0600 Subject: [PATCH 23/79] added info for Frame.Ctx --- py/frame.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/frame.go b/py/frame.go index ae39c018..68b07aba 100755 --- a/py/frame.go +++ b/py/frame.go @@ -26,7 +26,7 @@ type TryBlock struct { // A python Frame object type Frame struct { // Back *Frame // previous frame, or nil - Ctx Ctx + Ctx Ctx // host module (state) context Code *Code // code segment Builtins StringDict // builtin symbol table Globals StringDict // global symbol table From 4af4f905d3125c498d854e6cd7b277644b541fff Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Sat, 18 Dec 2021 18:11:53 -0600 Subject: [PATCH 24/79] removed unnecessary assignment --- py/list.go | 1 - 1 file changed, 1 deletion(-) diff --git a/py/list.go b/py/list.go index 10141a18..5ea84ed7 100755 --- a/py/list.go +++ b/py/list.go @@ -60,7 +60,6 @@ func init() { if err != nil { return nil, err } - l = self.(*List) } err := SortInPlace(l, kwargs, funcName) if err != nil { From aec59f8309567cf23a8387ffc06ec69327185ea1 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Sat, 18 Dec 2021 18:13:56 -0600 Subject: [PATCH 25/79] copyright and formatting --- modules/runtime.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/runtime.go b/modules/runtime.go index bc179473..e3583ef9 100755 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -1,3 +1,7 @@ +// Copyright 2021 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package modules import ( From 74f1a190559268fbcd5174ad6d8ed5b8d93c9cd9 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Sat, 18 Dec 2021 18:31:16 -0600 Subject: [PATCH 26/79] Ctx and NewCtx docs --- py/run.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/py/run.go b/py/run.go index 18120701..36f92a34 100755 --- a/py/run.go +++ b/py/run.go @@ -18,6 +18,9 @@ const ( // Ctx is gpython virtual environment instance ("context"), providing a mechanism // for multiple gpython interpreters to run concurrently without restriction. // +// In general, one creates a py.Ctx (via py.NewCtx) for each concurrent goroutine to be running an interpreter. +// In other words, ensure that a py.Ctx is never concurrently accessed across goroutines. +// // See BenchmarkCtx() in vm/vm_test.go type Ctx interface { @@ -34,7 +37,7 @@ type Ctx interface { // Returns the named module for this context (or an error if not found) GetModule(moduleName string) (*Module, error) - // WIP + // Gereric access to this context's modules / state. Store() *Store } @@ -79,8 +82,9 @@ type CtxOpts struct { SysPaths []string // sys.path initializer } -// High-level entry points var ( + // DefaultCtxOpts should be default opts created for py.NewCtx. + // Calling this ensure that you future proof you code for suggested/default settings. DefaultCtxOpts = func() CtxOpts { opts := CtxOpts{ SysPaths: CoreSysPaths, @@ -88,6 +92,12 @@ var ( opts.SysPaths = append(opts.SysPaths, AuxSysPaths...) return opts } - NewCtx func(opts CtxOpts) Ctx + + // NewCtx is a high-level call to create a new gpython interpreter context. + // It allows you specify default settings, sys search paths, and is the foundational object for concurrent interpreter execution. + NewCtx func(opts CtxOpts) Ctx + + // Compile is a high-level call to compile a python buffer into a py.Code object. + // Returns a py.Code object or otherwise an error. Compile func(str, filename string, mode CompileMode, flags int, dont_inherit bool) (Object, error) ) From 8fa6622f43feb6219b4b3998e97efb653105f50b Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Sat, 18 Dec 2021 19:01:21 -0600 Subject: [PATCH 27/79] switch cleanup --- py/util.go | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/py/util.go b/py/util.go index 767c643f..5fbee64f 100755 --- a/py/util.go +++ b/py/util.go @@ -66,34 +66,36 @@ func LoadAttr(obj Object, attrName string, data interface{}) error { } func loadValue(src Object, data interface{}) error { - var ( v_str string v_float float64 v_int int64 ) - if b, ok := src.(Bool); ok { - if b { + haveStr := false + + switch v := src.(type) { + case Bool: + if v { v_int = 1 v_float = 1 v_str = "True" } else { v_str = "False" } - } else if val, ok := src.(Int); ok { - v_int = int64(val) - v_float = float64(val) - } else if val, ok := src.(Float); ok { - v_int = int64(val) - v_float = float64(val) - } else if val, ok := src.(String); ok { - v_str = string(val) - intval, _ := strconv.Atoi(v_str) - v_int = int64(intval) - } else if _, ok := src.(NoneType); ok { + haveStr = true + case Int: + v_int = int64(v) + v_float = float64(v) + case Float: + v_int = int64(v) + v_float = float64(v) + case String: + v_str = string(v) + haveStr = true + case NoneType: // No-op - } else { + default: return ErrUnsupportedObjType } @@ -121,8 +123,14 @@ func loadValue(src Object, data interface{}) error { case *uint64: *dst = uint64(v_int) case *float32: + if haveStr { + v_float, _ = strconv.ParseFloat(v_str, 32) + } *dst = float32(v_float) case *float64: + if haveStr { + v_float, _ = strconv.ParseFloat(v_str, 64) + } *dst = v_float case *string: *dst = v_str From 26fb9aaa60775ad51c405e86ce0f7fd54c4d9e06 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 20 Dec 2021 01:57:07 -0600 Subject: [PATCH 28/79] leaner ModuleImpl schema --- builtin/builtin.go | 2 +- importlib/importlib.go | 5 ++- marshal/marshal.go | 28 +-------------- math/math.go | 2 +- modules/runtime.go | 78 ++++++++++++++++++++++++++++-------------- py/import.go | 9 +++-- py/module.go | 50 +++++++++++++-------------- sys/sys.go | 2 +- time/time.go | 2 +- 9 files changed, 87 insertions(+), 91 deletions(-) diff --git a/builtin/builtin.go b/builtin/builtin.go index ac80ef57..14ab7e23 100755 --- a/builtin/builtin.go +++ b/builtin/builtin.go @@ -163,7 +163,7 @@ func init() { "ZeroDivisionError": py.ZeroDivisionError, } - py.RegisterModule(&py.StaticModule{ + py.RegisterModule(&py.ModuleImpl{ Info: py.ModuleInfo{ Name: "builtins", Doc: builtin_doc, diff --git a/importlib/importlib.go b/importlib/importlib.go index a6f1ad3f..b70d330d 100755 --- a/importlib/importlib.go +++ b/importlib/importlib.go @@ -7,18 +7,17 @@ package py import ( - "github.com/go-python/gpython/marshal" "github.com/go-python/gpython/py" ) // Load the frozen module func init() { - py.RegisterModule(&marshal.FrozenModule{ + py.RegisterModule(&py.ModuleImpl{ Info: py.ModuleInfo{ Name: "importlib", }, - Code: data, + CodeBuf: data, }) } diff --git a/marshal/marshal.go b/marshal/marshal.go index 6dbb9166..937008cc 100755 --- a/marshal/marshal.go +++ b/marshal/marshal.go @@ -6,7 +6,6 @@ package marshal import ( - "bytes" "encoding/binary" "errors" "fmt" @@ -453,31 +452,6 @@ func ReadPyc(r io.Reader) (obj py.Object, err error) { return ReadObject(r) } -// Set for straight-forward modules that are threadsafe, stateless, and/or should be shared across multiple py.Ctx instances (for efficiency). -type FrozenModule struct { - Info py.ModuleInfo - Code []byte -} - -func (mod *FrozenModule) ModuleInfo() py.ModuleInfo { - return mod.Info -} - -func (mod *FrozenModule) ModuleInit(ctx py.Ctx) (*py.Module, error) { - r := bytes.NewBuffer(mod.Code) - obj, err := ReadObject(r) - if err != nil { - return nil, err - } - code := obj.(*py.Code) - module := ctx.Store().NewModule(ctx, mod.Info, nil, nil) - _, err = ctx.RunCode(code, module.Globals, module.Globals, nil) - if err != nil { - return nil, err - } - return module, nil -} - const dump_doc = `dump(value, file[, version]) Write the value on the open file. The value must be a supported type. @@ -642,7 +616,7 @@ func init() { "version": py.Int(MARSHAL_VERSION), } - py.RegisterModule(&py.StaticModule{ + py.RegisterModule(&py.ModuleImpl{ Info: py.ModuleInfo{ Name: "marshal", Doc: module_doc, diff --git a/math/math.go b/math/math.go index 867cf25c..0e6790fd 100755 --- a/math/math.go +++ b/math/math.go @@ -1334,7 +1334,7 @@ func init() { py.MustNewMethod("to_ulps", math_to_ulps, 0, math_to_ulps_doc), } - py.RegisterModule(&py.StaticModule{ + py.RegisterModule(&py.ModuleImpl{ Info: py.ModuleInfo{ Name: "math", Doc: math_doc, diff --git a/modules/runtime.go b/modules/runtime.go index e3583ef9..c3d40c14 100755 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -5,6 +5,7 @@ package modules import ( + "bytes" "io/ioutil" "os" "path" @@ -49,16 +50,52 @@ func NewCtx(opts py.CtxOpts) py.Ctx { return ctx } -func (ctx *ctx) RunFile(runPath string, opts py.RunOpts) (*py.Module, error) { +func (ctx *ctx) ModuleInit(impl *py.ModuleImpl) (*py.Module, error) { + var err error + + if impl.Code == nil && len(impl.CodeSrc) > 0 { + impl.Code, err = py.Compile(string(impl.CodeSrc), impl.Info.FileDesc, py.ExecMode, 0, true) + if err != nil { + return nil, err + } + } + + if impl.Code == nil && len(impl.CodeBuf) > 0 { + codeBuf := bytes.NewBuffer(impl.CodeBuf) + obj, err := marshal.ReadObject(codeBuf) + if err != nil { + return nil, err + } + impl.Code, _ = obj.(*py.Code) + if impl.Code == nil { + return nil, py.ExceptionNewf(py.AssertionError, "Embedded code did not produce a py.Code object") + } + } + + module, err := ctx.Store().NewModule(ctx, impl.Info, impl.Methods, impl.Globals) + if err != nil { + return nil, err + } + + if impl.Code != nil { + _, err = ctx.RunCode(impl.Code, module.Globals, module.Globals, nil) + if err != nil { + return nil, err + } + } + + return module, nil +} + +func (ctx *ctx) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.CompileOut, error) { tryPaths := defaultPaths if opts.UseSysPaths { tryPaths = ctx.Store().MustGetModule("sys").Globals["path"].(*py.List).Items } - var codeObj py.Object - var fileDesc string + out := py.CompileOut{} - err := resolveRunPath(runPath, opts, tryPaths, func(fpath string) (bool, error) { + err := resolveRunPath(pathname, opts, tryPaths, func(fpath string) (bool, error) { stat, err := os.Stat(fpath) if err == nil && stat.IsDir() { @@ -90,10 +127,11 @@ func (ctx *ctx) RunFile(runPath string, opts py.RunOpts) (*py.Module, error) { return false, py.ExceptionNewf(py.OSError, "Error reading %q: %v", fpath, err) } - codeObj, err = py.Compile(string(pySrc), fpath, py.ExecMode, 0, true) + out.Code, err = py.Compile(string(pySrc), fpath, py.ExecMode, 0, true) if err != nil { return false, err } + out.SrcPathname = fpath } else if ext == ".pyc" { var file *os.File file, err = os.Open(fpath) @@ -101,44 +139,34 @@ func (ctx *ctx) RunFile(runPath string, opts py.RunOpts) (*py.Module, error) { return false, py.ExceptionNewf(py.OSError, "Error opening %q: %v", fpath, err) } defer file.Close() - codeObj, err = marshal.ReadPyc(file) + codeObj, err := marshal.ReadPyc(file) if err != nil { return false, py.ExceptionNewf(py.ImportError, "Failed to marshal %q: %v", fpath, err) } + out.Code, _ = codeObj.(*py.Code) + out.PycPathname = fpath } - fileDesc = fpath + out.FileDesc = fpath return false, nil }) - if err != nil { - return nil, err - } - - var code *py.Code - if codeObj != nil { - code, _ = codeObj.(*py.Code) - } - if code == nil { - return nil, py.ExceptionNewf(py.AssertionError, "Missing code object") + if out.Code == nil && err == nil { + err = py.ExceptionNewf(py.AssertionError, "Missing code object") } - if opts.HostModule == nil { - opts.HostModule = ctx.Store().NewModule(ctx, py.ModuleInfo{ - Name: opts.ModuleName, - FileDesc: fileDesc, - }, nil, nil) + if err != nil { + return py.CompileOut{}, err } - _, err = vm.EvalCode(ctx, code, opts.HostModule.Globals, opts.HostModule.Globals, nil, nil, nil, nil, nil) - return opts.HostModule, err + return out, nil } var defaultPaths = []py.Object{ py.String("."), } -func resolveRunPath(runPath string, opts py.RunOpts, pathObjs []py.Object, tryPath func(pyPath string) (bool, error)) error { +func resolveRunPath(runPath string, opts py.CompileOpts, pathObjs []py.Object, tryPath func(pyPath string) (bool, error)) error { var cwd string // Remove trailing slash if present diff --git a/py/import.go b/py/import.go index 1f3293ee..65a2e67d 100755 --- a/py/import.go +++ b/py/import.go @@ -88,7 +88,7 @@ func ImportModuleLevelObject(ctx Ctx, name string, globals, locals StringDict, f // See if the module is a registered embeddded module that has not been loaded into this ctx yet. if impl := GetModuleImpl(name); impl != nil { - module, err := impl.ModuleInit(ctx) + module, err := ctx.ModuleInit(impl) if err != nil { return nil, err } @@ -101,10 +101,9 @@ func ImportModuleLevelObject(ctx Ctx, name string, globals, locals StringDict, f // Convert import's dot separators into path seps parts := strings.Split(name, ".") - pathParts := path.Join(parts...) + srcPathname := path.Join(parts...) - opts := RunOpts{ - ModuleName: name, + opts := CompileOpts{ UseSysPaths: true, } @@ -112,7 +111,7 @@ func ImportModuleLevelObject(ctx Ctx, name string, globals, locals StringDict, f opts.CurDir = path.Dir(string(fromFile.(String))) } - module, err := ctx.RunFile(pathParts, opts) + module, err := RunInNewModule(ctx, srcPathname, opts, name) if err != nil { return nil, err } diff --git a/py/module.go b/py/module.go index 8f7db2c0..836a2bd2 100755 --- a/py/module.go +++ b/py/module.go @@ -17,33 +17,29 @@ const ( // Set for modules that are threadsafe, stateless, and/or can be shared across multiple py.Ctx instances (for efficiency). // Otherwise, a separate module instance is created for each py.Ctx that imports it. ShareModule ModuleFlags = 0x01 // @@TODO + + MainModuleName = "__main__" ) +// ModuleInfo contains info and about a module and can specify flags that affect how it is imported into a py.Ctx type ModuleInfo struct { - Name string - Doc string - FileDesc string + Name string // __name__ (if nil, "__main__" is used) + Doc string // __doc__ + FileDesc string // __file__ Flags ModuleFlags } -type ModuleImpl interface { - ModuleInfo() ModuleInfo - ModuleInit(ctx Ctx) (*Module, error) -} - -// Set for straight-forward modules that are threadsafe, stateless, and/or should be shared across multiple py.Ctx instances (for efficiency). -type StaticModule struct { +// ModuleImpl is used for modules that are ready to be imported into a py.Ctx. +// If a module is threadsafe and stateless it can be shared across multiple py.Ctx instances (for efficiency). +// By convention, .Code is executed when a module instance is initialized. +// If .Code == nil, then .CodeBuf or .CodeSrc will be auto-compiled to set .Code. +type ModuleImpl struct { Info ModuleInfo Methods []*Method Globals StringDict -} - -func (mod *StaticModule) ModuleInfo() ModuleInfo { - return mod.Info -} - -func (mod *StaticModule) ModuleInit(ctx Ctx) (*Module, error) { - return ctx.Store().NewModule(ctx, mod.Info, mod.Methods, mod.Globals), nil + CodeSrc string // Module code body (py source code to be compiled) + CodeBuf []byte // Module code body (serialized py.Code object) + Code *Code // Module code body } type Store struct { @@ -56,11 +52,11 @@ type Store struct { Importlib *Module } -func RegisterModule(module ModuleImpl) { +func RegisterModule(module *ModuleImpl) { gRuntime.RegisterModule(module) } -func GetModuleImpl(moduleName string) ModuleImpl { +func GetModuleImpl(moduleName string) *ModuleImpl { gRuntime.mu.RLock() defer gRuntime.mu.RUnlock() impl := gRuntime.ModuleImpls[moduleName] @@ -69,17 +65,17 @@ func GetModuleImpl(moduleName string) ModuleImpl { type Runtime struct { mu sync.RWMutex - ModuleImpls map[string]ModuleImpl + ModuleImpls map[string]*ModuleImpl } var gRuntime = Runtime{ - ModuleImpls: make(map[string]ModuleImpl), + ModuleImpls: make(map[string]*ModuleImpl), } -func (rt *Runtime) RegisterModule(module ModuleImpl) { +func (rt *Runtime) RegisterModule(impl *ModuleImpl) { rt.mu.Lock() defer rt.mu.Unlock() - rt.ModuleImpls[module.ModuleInfo().Name] = module + rt.ModuleImpls[impl.Info.Name] = impl } func NewStore() *Store { @@ -113,9 +109,9 @@ func (m *Module) GetDict() StringDict { } // Define a new module -func (store *Store) NewModule(ctx Ctx, info ModuleInfo, methods []*Method, globals StringDict) *Module { +func (store *Store) NewModule(ctx Ctx, info ModuleInfo, methods []*Method, globals StringDict) (*Module, error) { if info.Name == "" { - info.Name = "__main__" + info.Name = MainModuleName } m := &Module{ ModuleInfo: info, @@ -147,7 +143,7 @@ func (store *Store) NewModule(ctx Ctx, info ModuleInfo, methods []*Method, globa store.Importlib = m } // fmt.Printf("Registering module %q\n", name) - return m + return m, nil } // Gets a module diff --git a/sys/sys.go b/sys/sys.go index c401214f..b8aa3dc2 100755 --- a/sys/sys.go +++ b/sys/sys.go @@ -797,7 +797,7 @@ func init() { // #endif } - py.RegisterModule(&py.StaticModule{ + py.RegisterModule(&py.ModuleImpl{ Info: py.ModuleInfo{ Name: "sys", Doc: module_doc, diff --git a/time/time.go b/time/time.go index d8397988..769bbe99 100755 --- a/time/time.go +++ b/time/time.go @@ -998,7 +998,7 @@ func init() { py.MustNewMethod("get_clock_info", time_get_clock_info, 0, get_clock_info_doc), } - py.RegisterModule(&py.StaticModule{ + py.RegisterModule(&py.ModuleImpl{ Info: py.ModuleInfo{ Name: "time", Doc: module_doc, From 8614aeb26f7075c33bd81a75e34235d2fe0f3fbf Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 20 Dec 2021 01:57:21 -0600 Subject: [PATCH 29/79] added GetDict() impl --- py/dict.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/py/dict.go b/py/dict.go index d3df854c..4f277c47 100644 --- a/py/dict.go +++ b/py/dict.go @@ -217,3 +217,9 @@ func (a StringDict) M__contains__(other Object) (Object, error) { } return False, nil } + +func (d StringDict) GetDict() StringDict { + return d +} + +var _ IGetDict = (*StringDict)(nil) From 86531c681d6f84086c174461e388cc3e5725d7d9 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 20 Dec 2021 02:08:30 -0600 Subject: [PATCH 30/79] Compile() now returns py.Code (vs py.Object) --- compile/compile.go | 23 +++++++++++++---------- compile/compile_test.go | 15 +++++---------- pytest/pytest.go | 9 ++++++--- vm/builtin.go | 3 +-- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/compile/compile.go b/compile/compile.go index d3750cb1..775717a9 100644 --- a/compile/compile.go +++ b/compile/compile.go @@ -89,33 +89,36 @@ func init() { py.Compile = Compile } -// Compile(source, filename, mode, flags, dont_inherit) -> code object +// Compile(src, srcDesc, compileMode, flags, dont_inherit) -> code object // // Compile the source string (a Python module, statement or expression) -// into a code object that can be executed by exec() or eval(). -// The filename will be used for run-time error messages. -// The mode must be 'exec' to compile a module, 'single' to compile a -// single (interactive) statement, or 'eval' to compile an expression. +// into a code object that can be executed. +// +// srcDesc is used for run-time error messages and is typically a file system pathname, +// +// See py.CompileMode for compile mode options. +// // The flags argument, if present, controls which future statements influence // the compilation of the code. +// // The dont_inherit argument, if non-zero, stops the compilation inheriting // the effects of any future statements in effect in the code calling // compile; if absent or zero these statements do influence the compilation, // in addition to any features explicitly specified. -func Compile(str, filename string, mode py.CompileMode, futureFlags int, dont_inherit bool) (py.Object, error) { +func Compile(src, srcDesc string, mode py.CompileMode, futureFlags int, dont_inherit bool) (*py.Code, error) { // Parse Ast - Ast, err := parser.ParseString(str, mode) + Ast, err := parser.ParseString(src, mode) if err != nil { return nil, err } // Make symbol table - SymTable, err := symtable.NewSymTable(Ast, filename) + SymTable, err := symtable.NewSymTable(Ast, srcDesc) if err != nil { return nil, err } c := newCompiler(nil, compilerScopeModule) - c.Filename = filename - err = c.compileAst(Ast, filename, futureFlags, dont_inherit, SymTable) + c.Filename = srcDesc + err = c.compileAst(Ast, srcDesc, futureFlags, dont_inherit, SymTable) if err != nil { return nil, err } diff --git a/compile/compile_test.go b/compile/compile_test.go index 72d4a3f6..aee3caac 100644 --- a/compile/compile_test.go +++ b/compile/compile_test.go @@ -163,7 +163,7 @@ func EqCode(t *testing.T, name string, a, b *py.Code) { func TestCompile(t *testing.T) { for _, test := range compileTestData { // log.Printf(">>> %s", test.in) - codeObj, err := Compile(test.in, "", test.mode, 0, true) + code, err := Compile(test.in, "", test.mode, 0, true) if err != nil { if test.exceptionType == nil { t.Errorf("%s: Got exception %v when not expecting one", test.in, err) @@ -196,17 +196,12 @@ func TestCompile(t *testing.T) { } } else { if test.out == nil { - if codeObj != nil { - t.Errorf("%s: Expecting nil *py.Code but got %T", test.in, codeObj) + if code != nil { + t.Errorf("%s: Expecting nil *py.Code but got %T", test.in, code) } } else { - code, ok := codeObj.(*py.Code) - if !ok { - t.Errorf("%s: Expecting *py.Code but got %T", test.in, codeObj) - } else { - //t.Logf("Testing %q", test.in) - EqCode(t, test.in, test.out, code) - } + //t.Logf("Testing %q", test.in) + EqCode(t, test.in, test.out, code) } } } diff --git a/pytest/pytest.go b/pytest/pytest.go index 0b9de29a..e42c42d8 100644 --- a/pytest/pytest.go +++ b/pytest/pytest.go @@ -39,15 +39,18 @@ func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) { } func CompileSrc(t testing.TB, ctx py.Ctx, pySrc string, prog string) (*py.Module, *py.Code) { - obj, err := compile.Compile(string(pySrc), prog, py.ExecMode, 0, true) + code, err := compile.Compile(string(pySrc), prog, py.ExecMode, 0, true) if err != nil { t.Fatalf("%s: Compile failed: %v", prog, err) } - code := obj.(*py.Code) - module := ctx.Store().NewModule(ctx, py.ModuleInfo{ + module, err := ctx.Store().NewModule(ctx, py.ModuleInfo{ FileDesc: prog, }, nil, nil) + if err != nil { + t.Fatalf("%s: NewModule failed: %v", prog, err) + } + return module, code } diff --git a/vm/builtin.go b/vm/builtin.go index e07343a4..6c063b37 100755 --- a/vm/builtin.go +++ b/vm/builtin.go @@ -60,11 +60,10 @@ func builtinEvalOrExec(ctx py.Ctx, args py.Tuple, kwargs, currentLocals, current } if code == nil { codeStr = strings.TrimLeft(codeStr, " \t") - obj, err := py.Compile(codeStr, "", mode, 0, true) + code, err = py.Compile(codeStr, "", mode, 0, true) if err != nil { return nil, err } - code = obj.(*py.Code) } if code.GetNumFree() > 0 { return nil, py.ExceptionNewf(py.TypeError, "code passed to %s() may not contain free variables", mode) From f71b961f6f9d285b6177deae1caa435b7c59a460 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 20 Dec 2021 02:09:48 -0600 Subject: [PATCH 31/79] tighter py.Ctx invocation patterns --- main.go | 2 +- py/run.go | 77 +++++++++++++++++++++++++++++++++++++++------------- repl/repl.go | 13 +++++---- 3 files changed, 66 insertions(+), 26 deletions(-) diff --git a/main.go b/main.go index b3a51fb3..17b499e1 100755 --- a/main.go +++ b/main.go @@ -72,7 +72,7 @@ func main() { cli.RunREPL(replCtx) } else { - _, err := ctx.RunFile(args[0], py.RunOpts{}) + err := py.RunFile(ctx, args[0], py.CompileOpts{}) if err != nil { py.TracebackDump(err) log.Fatal(err) diff --git a/py/run.go b/py/run.go index 36f92a34..0e4a27c5 100755 --- a/py/run.go +++ b/py/run.go @@ -3,9 +3,9 @@ package py type CompileMode string const ( - ExecMode CompileMode = "exec" - EvalMode CompileMode = "eval" - SingleMode CompileMode = "single" + ExecMode CompileMode = "exec" // Compile a module + EvalMode CompileMode = "eval" // Compile an expression + SingleMode CompileMode = "single" // Compile a single (interactive) statement ) type RunFlags int32 @@ -21,14 +21,20 @@ const ( // In general, one creates a py.Ctx (via py.NewCtx) for each concurrent goroutine to be running an interpreter. // In other words, ensure that a py.Ctx is never concurrently accessed across goroutines. // -// See BenchmarkCtx() in vm/vm_test.go +// RunFile() and RunCode() block until code execution is complete. +// In the future, they will abort early if the parent associated py.Ctx is signaled. +// +// See examples/multi-ctx type Ctx interface { - // These each initiate blocking execution. - RunCode(code *Code, globals, locals StringDict, closure Tuple) (res Object, err error) + // Resolves then compiles (if applicable) the given file system pathname into a py.Code ready to be executed. + ResolveAndCompile(pathname string, opts CompileOpts) (CompileOut, error) + + // Creates a new py.Module instance and initializes ModuleImpl's code in the new module (if applicable). + ModuleInit(impl *ModuleImpl) (*Module, error) - // Runs a given file in the given host module. - RunFile(runPath string, opts RunOpts) (*Module, error) + // RunCode is a lower-level invocation to execute the given py.Code. + RunCode(code *Code, globals, locals StringDict, closure Tuple) (result Object, err error) // Execution of any of the above will stop when the next opcode runs // @@TODO @@ -44,8 +50,6 @@ type Ctx interface { const ( SrcFileExt = ".py" CodeFileExt = ".pyc" - - DefaultModuleName = "__main__" ) type StdLib int32 @@ -57,11 +61,16 @@ const ( CoreLibs = Lib_sys | Lib_time ) -type RunOpts struct { - HostModule *Module // Host module to execute within (if nil, a new module is created) - ModuleName string // If HostModule == nil, this is the name of the newly created module. If nil, "__main__" is used. - CurDir string // If non-nil, this is the path of the current working directory. If nil, os.Getwd() is used - UseSysPaths bool +type CompileOpts struct { + UseSysPaths bool // If set, sys.path will be used to resolve relative pathnames + CurDir string // If non-nil, this is the path of the current working directory. If nil, os.Getwd() is used. +} + +type CompileOut struct { + SrcPathname string // Resolved pathname the .py file that was compiled (if applicable) + PycPathname string // Pathname of the .pyc file read and/or written (if applicable) + FileDesc string // Pathname to be used for a a module's "__file__" attrib + Code *Code // Read/Output code object ready for execution } // Can be changed during runtime and will \play nice with others using DefaultCtxOpts() @@ -83,7 +92,7 @@ type CtxOpts struct { } var ( - // DefaultCtxOpts should be default opts created for py.NewCtx. + // DefaultCtxOpts should be the default opts created for py.NewCtx. // Calling this ensure that you future proof you code for suggested/default settings. DefaultCtxOpts = func() CtxOpts { opts := CtxOpts{ @@ -94,10 +103,40 @@ var ( } // NewCtx is a high-level call to create a new gpython interpreter context. - // It allows you specify default settings, sys search paths, and is the foundational object for concurrent interpreter execution. + // See type Ctx interface. NewCtx func(opts CtxOpts) Ctx - // Compile is a high-level call to compile a python buffer into a py.Code object. + // Compiles a python buffer into a py.Code object. // Returns a py.Code object or otherwise an error. - Compile func(str, filename string, mode CompileMode, flags int, dont_inherit bool) (Object, error) + Compile func(src, srcDesc string, mode CompileMode, flags int, dont_inherit bool) (*Code, error) ) + +// Convenience function that resolves then executes the given file pathname in a new module. +// If moduleName is nil, "__main__" is used +func RunInNewModule( + ctx Ctx, + pathname string, + opts CompileOpts, + moduleName string, +) (*Module, error) { + out, err := ctx.ResolveAndCompile(pathname, opts) + if err != nil { + return nil, err + } + + moduleImpl := ModuleImpl{ + Info: ModuleInfo{ + Name: moduleName, + FileDesc: out.FileDesc, + }, + Code: out.Code, + } + + return ctx.ModuleInit(&moduleImpl) +} + +// Convenience function that resolves then executes the given file pathname in a new __main__ module +func RunFile(ctx Ctx, pathname string, opts CompileOpts) error { + _, err := RunInNewModule(ctx, pathname, opts, "") + return err +} diff --git a/repl/repl.go b/repl/repl.go index 5fb19add..f65e560f 100755 --- a/repl/repl.go +++ b/repl/repl.go @@ -39,7 +39,7 @@ type UI interface { Print(string) } -// New create a new REPL and initialises the state machine +// New create a new REPL and initializes the state machine func New(ctx py.Ctx) *REPL { if ctx == nil { ctx = py.NewCtx(py.DefaultCtxOpts()) @@ -51,9 +51,11 @@ func New(ctx py.Ctx) *REPL { continuation: false, previous: "", } - r.Module = ctx.Store().NewModule(ctx, py.ModuleInfo{ - FileDesc: r.prog, - }, nil, nil) + r.Module, _ = ctx.ModuleInit(&py.ModuleImpl{ + Info: py.ModuleInfo{ + FileDesc: r.prog, + }, + }) return r } @@ -82,7 +84,7 @@ func (r *REPL) Run(line string) { if toCompile == "" { return } - obj, err := py.Compile(toCompile+"\n", r.prog, py.SingleMode, 0, true) + code, err := py.Compile(toCompile+"\n", r.prog, py.SingleMode, 0, true) if err != nil { // Detect that we should start a continuation line // FIXME detect EOF properly! @@ -105,7 +107,6 @@ func (r *REPL) Run(line string) { r.term.Print(fmt.Sprintf("Compile error: %v", err)) return } - code := obj.(*py.Code) _, err = r.Ctx.RunCode(code, r.Module.Globals, r.Module.Globals, nil) if err != nil { py.TracebackDump(err) From 77a795416a374054bdcdbf8e1572c8d2ca390c39 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 20 Dec 2021 02:11:54 -0600 Subject: [PATCH 32/79] WIP --- examples/multi-ctx/main.go | 133 +++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 examples/multi-ctx/main.go diff --git a/examples/multi-ctx/main.go b/examples/multi-ctx/main.go new file mode 100644 index 00000000..50f0ef50 --- /dev/null +++ b/examples/multi-ctx/main.go @@ -0,0 +1,133 @@ +package main + +import ( + "fmt" + "log" + "runtime" + "strings" + "sync" + "time" + + _ "github.com/go-python/gpython/modules" + "github.com/go-python/gpython/py" +) + +func main() { + + // The total job count implies a fixed amount of work. + // The number of workers is how many py.Ctx (in concurrent goroutines) to pull jobs off the queue. + // One worker does all the work serially while N number of workers will (ideally) divide that up. + totalJobs := 20 + + for i := 0; i < 10; i++ { + numWorkers := i + 1 + elapsed := RunMultiPi(numWorkers, totalJobs) + fmt.Printf("=====> %2d worker(s): %v\n\n", numWorkers, elapsed) + + // Give each trial a fresh start + runtime.GC() + } + +} + + +var jobScript = ` +pi = chud.pi_chudnovsky_bs(numDigits) +last_5 = pi % 100000 +print("%s: last 5 digits of %d is %d (job #%0d)" % (WORKER_ID, numDigits, last_5, jobID)) +` + +var jobSrcTemplate = ` +import pi_chudnovsky_bs as chud + +WORKER_ID = "{{WORKER_ID}}" + +print("%s ready!" % (WORKER_ID)) +` + +type worker struct { + name string + ctx py.Ctx + main *py.Module + job *py.Code +} + +func (w *worker) compileTemplate(pySrc string) { + pySrc = strings.Replace(pySrc, "{{WORKER_ID}}", w.name, -1) + + mainImpl := py.ModuleImpl{ + CodeSrc: pySrc, + } + + var err error + w.main, err = w.ctx.ModuleInit(&mainImpl) + if err != nil { + log.Fatal(err) + } +} + +func RunMultiPi(numWorkers, numTimes int) time.Duration { + workersRunning := sync.WaitGroup{} + + fmt.Printf("Starting %d worker(s) to calculate %d jobs...\n", numWorkers, numTimes) + + jobPipe := make(chan int) + go func() { + for i := 0; i < numTimes; i++ { + jobPipe <- i+1 + } + close(jobPipe) + }() + + // Note that py.Code can be shared (accessed concurrently) since it is an inherently read-only object + jobCode, err := py.Compile(jobScript, "", py.ExecMode, 0, true) + if err != nil { + log.Fatal("jobScript failed to comple") + } + + workers := make([]worker, numWorkers) + for i := 0; i < numWorkers; i++ { + + opts := py.DefaultCtxOpts() + + // Make sure our import statement will find pi_chudnovsky_bs + opts.SysPaths = append(opts.SysPaths, "..") + + workers[i] = worker{ + name: fmt.Sprintf("Worker #%d", i+1), + ctx: py.NewCtx(opts), + job: jobCode, + } + + workersRunning.Add(1) + } + + startTime := time.Now() + + for i := range workers { + w := workers[i] + go func() { + + // Compiling can be concurrent since there is no associated py.Ctx + w.compileTemplate(jobSrcTemplate) + + for jobID := range jobPipe { + numDigits := 100000 + if jobID % 2 == 0 { + numDigits *= 10 + } + py.SetAttrString(w.main.Globals, "numDigits", py.Int(numDigits)) + py.SetAttrString(w.main.Globals, "jobID", py.Int(jobID)) + w.ctx.RunCode(jobCode, w.main.Globals, w.main.Globals, nil) + } + workersRunning.Done() + }() + } + + workersRunning.Wait() + + return time.Since(startTime) +} + + + From 6bba7da52e22bb57f3454d771c14dd5e6530085a Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 20 Dec 2021 02:40:44 -0600 Subject: [PATCH 33/79] reverted to 644 --- ast/asdl_go.py | 0 ast/asttest.py | 0 builtin/builtin.go | 0 compile/compile_data_test.go | 0 compile/diffdis.py | 0 compile/make_compile_test.py | 0 examples/pystone.py | 0 go.mod | 0 go.sum | 0 importlib/importlib.go | 0 main.go | 0 marshal/marshal.go | 0 math/math.go | 0 modules/runtime.go | 0 parser/grammar_test.go | 0 parser/lexer.go | 0 parser/lexer_test.go | 0 py/bytes.go | 0 py/code.go | 0 py/float.go | 0 py/frame.go | 0 py/function.go | 0 py/import.go | 0 py/list.go | 0 py/module.go | 0 py/py.go | 0 py/range.go | 0 py/run.go | 0 py/string.go | 0 py/traceback.go | 0 py/util.go | 0 repl/cli/cli.go | 0 repl/repl.go | 0 repl/repl_test.go | 0 sys/sys.go | 0 time/time.go | 0 vm/builtin.go | 0 vm/eval.go | 0 vm/vm.go | 0 39 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 ast/asdl_go.py mode change 100755 => 100644 ast/asttest.py mode change 100755 => 100644 builtin/builtin.go mode change 100755 => 100644 compile/compile_data_test.go mode change 100755 => 100644 compile/diffdis.py mode change 100755 => 100644 compile/make_compile_test.py mode change 100755 => 100644 examples/pystone.py mode change 100755 => 100644 go.mod mode change 100755 => 100644 go.sum mode change 100755 => 100644 importlib/importlib.go mode change 100755 => 100644 main.go mode change 100755 => 100644 marshal/marshal.go mode change 100755 => 100644 math/math.go mode change 100755 => 100644 modules/runtime.go mode change 100755 => 100644 parser/grammar_test.go mode change 100755 => 100644 parser/lexer.go mode change 100755 => 100644 parser/lexer_test.go mode change 100755 => 100644 py/bytes.go mode change 100755 => 100644 py/code.go mode change 100755 => 100644 py/float.go mode change 100755 => 100644 py/frame.go mode change 100755 => 100644 py/function.go mode change 100755 => 100644 py/import.go mode change 100755 => 100644 py/list.go mode change 100755 => 100644 py/module.go mode change 100755 => 100644 py/py.go mode change 100755 => 100644 py/range.go mode change 100755 => 100644 py/run.go mode change 100755 => 100644 py/string.go mode change 100755 => 100644 py/traceback.go mode change 100755 => 100644 py/util.go mode change 100755 => 100644 repl/cli/cli.go mode change 100755 => 100644 repl/repl.go mode change 100755 => 100644 repl/repl_test.go mode change 100755 => 100644 sys/sys.go mode change 100755 => 100644 time/time.go mode change 100755 => 100644 vm/builtin.go mode change 100755 => 100644 vm/eval.go mode change 100755 => 100644 vm/vm.go diff --git a/ast/asdl_go.py b/ast/asdl_go.py old mode 100755 new mode 100644 diff --git a/ast/asttest.py b/ast/asttest.py old mode 100755 new mode 100644 diff --git a/builtin/builtin.go b/builtin/builtin.go old mode 100755 new mode 100644 diff --git a/compile/compile_data_test.go b/compile/compile_data_test.go old mode 100755 new mode 100644 diff --git a/compile/diffdis.py b/compile/diffdis.py old mode 100755 new mode 100644 diff --git a/compile/make_compile_test.py b/compile/make_compile_test.py old mode 100755 new mode 100644 diff --git a/examples/pystone.py b/examples/pystone.py old mode 100755 new mode 100644 diff --git a/go.mod b/go.mod old mode 100755 new mode 100644 diff --git a/go.sum b/go.sum old mode 100755 new mode 100644 diff --git a/importlib/importlib.go b/importlib/importlib.go old mode 100755 new mode 100644 diff --git a/main.go b/main.go old mode 100755 new mode 100644 diff --git a/marshal/marshal.go b/marshal/marshal.go old mode 100755 new mode 100644 diff --git a/math/math.go b/math/math.go old mode 100755 new mode 100644 diff --git a/modules/runtime.go b/modules/runtime.go old mode 100755 new mode 100644 diff --git a/parser/grammar_test.go b/parser/grammar_test.go old mode 100755 new mode 100644 diff --git a/parser/lexer.go b/parser/lexer.go old mode 100755 new mode 100644 diff --git a/parser/lexer_test.go b/parser/lexer_test.go old mode 100755 new mode 100644 diff --git a/py/bytes.go b/py/bytes.go old mode 100755 new mode 100644 diff --git a/py/code.go b/py/code.go old mode 100755 new mode 100644 diff --git a/py/float.go b/py/float.go old mode 100755 new mode 100644 diff --git a/py/frame.go b/py/frame.go old mode 100755 new mode 100644 diff --git a/py/function.go b/py/function.go old mode 100755 new mode 100644 diff --git a/py/import.go b/py/import.go old mode 100755 new mode 100644 diff --git a/py/list.go b/py/list.go old mode 100755 new mode 100644 diff --git a/py/module.go b/py/module.go old mode 100755 new mode 100644 diff --git a/py/py.go b/py/py.go old mode 100755 new mode 100644 diff --git a/py/range.go b/py/range.go old mode 100755 new mode 100644 diff --git a/py/run.go b/py/run.go old mode 100755 new mode 100644 diff --git a/py/string.go b/py/string.go old mode 100755 new mode 100644 diff --git a/py/traceback.go b/py/traceback.go old mode 100755 new mode 100644 diff --git a/py/util.go b/py/util.go old mode 100755 new mode 100644 diff --git a/repl/cli/cli.go b/repl/cli/cli.go old mode 100755 new mode 100644 diff --git a/repl/repl.go b/repl/repl.go old mode 100755 new mode 100644 diff --git a/repl/repl_test.go b/repl/repl_test.go old mode 100755 new mode 100644 diff --git a/sys/sys.go b/sys/sys.go old mode 100755 new mode 100644 diff --git a/time/time.go b/time/time.go old mode 100755 new mode 100644 diff --git a/vm/builtin.go b/vm/builtin.go old mode 100755 new mode 100644 diff --git a/vm/eval.go b/vm/eval.go old mode 100755 new mode 100644 diff --git a/vm/vm.go b/vm/vm.go old mode 100755 new mode 100644 From 46d1052bf621bcc23b8f580ce9234f7aa224df5d Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 20 Dec 2021 02:54:16 -0600 Subject: [PATCH 34/79] chmod 644 --- symtable/symtable_data_test.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 symtable/symtable_data_test.go diff --git a/symtable/symtable_data_test.go b/symtable/symtable_data_test.go old mode 100755 new mode 100644 From 66fad235819fb9cfbe93d1714ff0ed688d2a85e8 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Sun, 26 Dec 2021 21:42:58 -0600 Subject: [PATCH 35/79] enhanced RunFile() to enclose RunInNewModule() --- main.go | 2 +- py/import.go | 2 +- py/run.go | 56 +++++++++++++++++++++++++++++++++------------------- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/main.go b/main.go index 17b499e1..c4cdc98f 100644 --- a/main.go +++ b/main.go @@ -72,7 +72,7 @@ func main() { cli.RunREPL(replCtx) } else { - err := py.RunFile(ctx, args[0], py.CompileOpts{}) + _, err := py.RunFile(ctx, args[0], py.CompileOpts{}, nil) if err != nil { py.TracebackDump(err) log.Fatal(err) diff --git a/py/import.go b/py/import.go index 65a2e67d..454473d3 100644 --- a/py/import.go +++ b/py/import.go @@ -111,7 +111,7 @@ func ImportModuleLevelObject(ctx Ctx, name string, globals, locals StringDict, f opts.CurDir = path.Dir(string(fromFile.(String))) } - module, err := RunInNewModule(ctx, srcPathname, opts, name) + module, err := RunFile(ctx, srcPathname, opts, name) if err != nil { return nil, err } diff --git a/py/run.go b/py/run.go index 0e4a27c5..42cb9c91 100644 --- a/py/run.go +++ b/py/run.go @@ -111,32 +111,48 @@ var ( Compile func(src, srcDesc string, mode CompileMode, flags int, dont_inherit bool) (*Code, error) ) -// Convenience function that resolves then executes the given file pathname in a new module. -// If moduleName is nil, "__main__" is used -func RunInNewModule( - ctx Ctx, - pathname string, - opts CompileOpts, - moduleName string, -) (*Module, error) { +// Resolves the given pathname, compiles as needed, and runs that code in the given module, returning the Module to indicate success. +// If inModule is a *Module, then the code is run in that module. +// If inModule is nil, the code is run in a new __main__ module (and the new Module is returned). +// If inModule is a string, the code is run in a new module with the given name (and the new Module is returned). +func RunFile(ctx Ctx, pathname string, opts CompileOpts, inModule interface{}) (*Module, error) { out, err := ctx.ResolveAndCompile(pathname, opts) if err != nil { return nil, err } - moduleImpl := ModuleImpl{ - Info: ModuleInfo{ - Name: moduleName, - FileDesc: out.FileDesc, - }, - Code: out.Code, + var moduleName string + createNew := false + var module *Module + + switch mod := inModule.(type) { + + case string: + moduleName = mod + createNew = true + case nil: + createNew = true + case *Module: + _, err = ctx.RunCode(out.Code, mod.Globals, mod.Globals, nil) + module = mod + default: + err = ExceptionNewf(TypeError, "unsupported module type: %v", inModule) } - return ctx.ModuleInit(&moduleImpl) -} + if err == nil && createNew { + moduleImpl := ModuleImpl{ + Info: ModuleInfo{ + Name: moduleName, + FileDesc: out.FileDesc, + }, + Code: out.Code, + } + module, err = ctx.ModuleInit(&moduleImpl) + } + + if err != nil { + return nil, err + } -// Convenience function that resolves then executes the given file pathname in a new __main__ module -func RunFile(ctx Ctx, pathname string, opts CompileOpts) error { - _, err := RunInNewModule(ctx, pathname, opts, "") - return err + return module, nil } From 95afd7bb00f87c2291253b7207d9ca4feec61be0 Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Mon, 10 Jan 2022 10:18:33 -0600 Subject: [PATCH 36/79] Update examples/multi-ctx/main.go Co-authored-by: Sebastien Binet --- examples/multi-ctx/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/multi-ctx/main.go b/examples/multi-ctx/main.go index 50f0ef50..4d05e731 100644 --- a/examples/multi-ctx/main.go +++ b/examples/multi-ctx/main.go @@ -67,7 +67,7 @@ func (w *worker) compileTemplate(pySrc string) { } func RunMultiPi(numWorkers, numTimes int) time.Duration { - workersRunning := sync.WaitGroup{} + var workersRunning sync.WaitGroup fmt.Printf("Starting %d worker(s) to calculate %d jobs...\n", numWorkers, numTimes) From 319edb7363892eeeacd1e1e65e595adcb994585c Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Mon, 10 Jan 2022 10:18:43 -0600 Subject: [PATCH 37/79] Update modules/runtime.go Co-authored-by: Sebastien Binet --- modules/runtime.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/runtime.go b/modules/runtime.go index c3d40c14..7296b8e4 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -120,7 +120,8 @@ func (ctx *ctx) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.Comp return false, err } - if ext == ".py" { + switch ext { + case ".py": var pySrc []byte pySrc, err = ioutil.ReadFile(fpath) if err != nil { From 06a2eaec5be46861a6534469c96acffe59425090 Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Mon, 10 Jan 2022 10:19:00 -0600 Subject: [PATCH 38/79] Update modules/runtime.go Co-authored-by: Sebastien Binet --- modules/runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/runtime.go b/modules/runtime.go index 7296b8e4..ba80b6f9 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -133,7 +133,7 @@ func (ctx *ctx) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.Comp return false, err } out.SrcPathname = fpath - } else if ext == ".pyc" { + case ".pyc": var file *os.File file, err = os.Open(fpath) if err != nil { From 8d4d586555e4d138a11b2af97186693eb0ece549 Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Mon, 10 Jan 2022 10:19:29 -0600 Subject: [PATCH 39/79] Update modules/runtime.go Co-authored-by: Sebastien Binet --- modules/runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/runtime.go b/modules/runtime.go index ba80b6f9..3ab6779e 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -137,7 +137,7 @@ func (ctx *ctx) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.Comp var file *os.File file, err = os.Open(fpath) if err != nil { - return false, py.ExceptionNewf(py.OSError, "Error opening %q: %v", fpath, err) + return false, py.ExceptionNewf(py.OSError, "Error opening %q: %w", fpath, err) } defer file.Close() codeObj, err := marshal.ReadPyc(file) From d475e673fc2cb34bd6eead59420a812d1f04e40f Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Mon, 10 Jan 2022 10:19:44 -0600 Subject: [PATCH 40/79] Update vm/eval.go Co-authored-by: Sebastien Binet --- vm/eval.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/eval.go b/vm/eval.go index b0c430c1..23418e0b 100644 --- a/vm/eval.go +++ b/vm/eval.go @@ -2036,7 +2036,7 @@ func tooManyPositional(co *py.Code, given, defcount int, fastlocals []py.Object) chooseString(given == 1 && kwonly_given == 0, "was", "were")) } -// Run a new virtual machine on a Code object +// EvalCode runs a new virtual machine on a Code object. // // Any parameters are expected to have been decoded into locals // From 6afac99ef3431cf05c56979aa70dee66d99d0932 Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Mon, 10 Jan 2022 10:19:59 -0600 Subject: [PATCH 41/79] Update py/run.go Co-authored-by: Sebastien Binet --- py/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/run.go b/py/run.go index 42cb9c91..f16056fb 100644 --- a/py/run.go +++ b/py/run.go @@ -11,7 +11,7 @@ const ( type RunFlags int32 const ( - // RunOpts.FilePath is intelligently interepreted to load the appropriate pyc object (otherwise new code is generated from the implied .py file) + // RunOpts.FilePath is intelligently interpreted to load the appropriate pyc object (otherwise new code is generated from the implied .py file) SmartCodeAcquire RunFlags = 0x01 ) From 931c346f029df10613e9600739165cf3854d8bae Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Mon, 10 Jan 2022 10:20:43 -0600 Subject: [PATCH 42/79] Update modules/runtime.go Co-authored-by: Sebastien Binet --- modules/runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/runtime.go b/modules/runtime.go index 3ab6779e..a5b62a63 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -142,7 +142,7 @@ func (ctx *ctx) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.Comp defer file.Close() codeObj, err := marshal.ReadPyc(file) if err != nil { - return false, py.ExceptionNewf(py.ImportError, "Failed to marshal %q: %v", fpath, err) + return false, py.ExceptionNewf(py.ImportError, "Failed to marshal %q: %w", fpath, err) } out.Code, _ = codeObj.(*py.Code) out.PycPathname = fpath From a7ce4bb37f4016c807ecda3d2e01d68e6187ae3d Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Mon, 10 Jan 2022 10:20:56 -0600 Subject: [PATCH 43/79] Update modules/runtime.go Co-authored-by: Sebastien Binet --- modules/runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/runtime.go b/modules/runtime.go index a5b62a63..48722f7a 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -125,7 +125,7 @@ func (ctx *ctx) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.Comp var pySrc []byte pySrc, err = ioutil.ReadFile(fpath) if err != nil { - return false, py.ExceptionNewf(py.OSError, "Error reading %q: %v", fpath, err) + return false, py.ExceptionNewf(py.OSError, "Error reading %q: %w", fpath, err) } out.Code, err = py.Compile(string(pySrc), fpath, py.ExecMode, 0, true) From c1f8809c66fabaa5b3c91c59631f8880123d0195 Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Mon, 10 Jan 2022 10:21:12 -0600 Subject: [PATCH 44/79] Update modules/runtime.go Co-authored-by: Sebastien Binet --- modules/runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/runtime.go b/modules/runtime.go index 48722f7a..bf08a6e5 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -116,7 +116,7 @@ func (ctx *ctx) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.Comp if os.IsNotExist(err) { return true, nil } - err = py.ExceptionNewf(py.OSError, "Error accessing %q: %v", fpath, err) + err = py.ExceptionNewf(py.OSError, "Error accessing %q: %w", fpath, err) return false, err } From 9f02cf1aaf60c5fd553fb533b9cad56fbbd4aefc Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Mon, 10 Jan 2022 10:21:36 -0600 Subject: [PATCH 45/79] Update modules/runtime.go Co-authored-by: Sebastien Binet --- modules/runtime.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/runtime.go b/modules/runtime.go index bf08a6e5..cac2fb2a 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -170,10 +170,7 @@ var defaultPaths = []py.Object{ func resolveRunPath(runPath string, opts py.CompileOpts, pathObjs []py.Object, tryPath func(pyPath string) (bool, error)) error { var cwd string - // Remove trailing slash if present - if runPath[len(runPath)-1] == '/' { - runPath = runPath[:len(runPath)-1] - } + runPath = strings.TrimSuffix(runPath, "/") var err error From 85e3a4a21311968f4d9303d2dbebb56aad96ac7b Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Mon, 10 Jan 2022 10:21:56 -0600 Subject: [PATCH 46/79] Update modules/runtime.go Co-authored-by: Sebastien Binet --- modules/runtime.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/runtime.go b/modules/runtime.go index cac2fb2a..5c6a039c 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -172,9 +172,11 @@ func resolveRunPath(runPath string, opts py.CompileOpts, pathObjs []py.Object, t runPath = strings.TrimSuffix(runPath, "/") - var err error - - cont := true + var ( + err error + cwd string + cont = true + ) for _, pathObj := range pathObjs { pathStr, ok := pathObj.(py.String) From fbec34e90e22f0f2e90966afb4c3ffe49830c36e Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Mon, 10 Jan 2022 10:22:16 -0600 Subject: [PATCH 47/79] Update modules/runtime.go Co-authored-by: Sebastien Binet --- modules/runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/runtime.go b/modules/runtime.go index 5c6a039c..331c1685 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -195,7 +195,7 @@ func resolveRunPath(runPath string, opts py.CompileOpts, pathObjs []py.Object, t cont, err = tryPath(subPath) } if cont && err == nil { - if len(cwd) == 0 { + if cwd == "" { cwd, _ = os.Getwd() } subPath := path.Join(cwd, fpath) From dbd54d8fac83a447d8a5402fcca78d3b0b49d006 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 10 Jan 2022 10:24:31 -0600 Subject: [PATCH 48/79] code cleanuo --- modules/runtime.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/runtime.go b/modules/runtime.go index 331c1685..978099bd 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -168,8 +168,6 @@ var defaultPaths = []py.Object{ } func resolveRunPath(runPath string, opts py.CompileOpts, pathObjs []py.Object, tryPath func(pyPath string) (bool, error)) error { - var cwd string - runPath = strings.TrimSuffix(runPath, "/") var ( From 71a5604b9efabd2d40d3fb654ae2adb1e44cac00 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 10 Jan 2022 10:28:34 -0600 Subject: [PATCH 49/79] Module doc str --- py/module.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/module.go b/py/module.go index 836a2bd2..96b99fd5 100644 --- a/py/module.go +++ b/py/module.go @@ -84,7 +84,7 @@ func NewStore() *Store { } } -// A python Module object that has been initted for a given py.Ctx +// Module is a runtime instance of a ModuleImpl bound the py.Ctx that imported it. type Module struct { ModuleInfo From 634823cbceb7b6e94f726cce467f5f458ede644c Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 10 Jan 2022 10:33:37 -0600 Subject: [PATCH 50/79] consistent arg order of NewFunction --- py/function.go | 2 +- vm/eval.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/py/function.go b/py/function.go index c8c9cb37..ba9d162e 100644 --- a/py/function.go +++ b/py/function.go @@ -56,7 +56,7 @@ func (f *Function) GetDict() StringDict { // attribute. qualname should be a unicode object or ""; if "", the // __qualname__ attribute is set to the same value as its __name__ // attribute. -func NewFunction(code *Code, ctx Ctx, globals StringDict, qualname string) *Function { +func NewFunction(ctx Ctx, code *Code, globals StringDict, qualname string) *Function { var doc Object if len(code.Consts) >= 1 { doc = code.Consts[0] diff --git a/vm/eval.go b/vm/eval.go index 23418e0b..62ce6511 100644 --- a/vm/eval.go +++ b/vm/eval.go @@ -1437,7 +1437,7 @@ func _make_function(vm *Vm, argc int32, opcode OpCode) { num_annotations := (argc >> 16) & 0x7fff qualname := vm.POP() code := vm.POP() - function := py.NewFunction(code.(*py.Code), vm.ctx, vm.frame.Globals, string(qualname.(py.String))) + function := py.NewFunction(vm.ctx, code.(*py.Code), vm.frame.Globals, string(qualname.(py.String))) if opcode == MAKE_CLOSURE { function.Closure = vm.POP().(py.Tuple) From 6a0c11edd35783d8dbda100cc33bebc88dae1226 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 10 Jan 2022 10:41:30 -0600 Subject: [PATCH 51/79] license and formatting --- examples/multi-ctx/main.go | 106 ++++++++++++++++++------------------- py/run.go | 4 ++ py/util.go | 4 ++ 3 files changed, 61 insertions(+), 53 deletions(-) diff --git a/examples/multi-ctx/main.go b/examples/multi-ctx/main.go index 4d05e731..b35d581e 100644 --- a/examples/multi-ctx/main.go +++ b/examples/multi-ctx/main.go @@ -1,3 +1,7 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package main import ( @@ -15,21 +19,20 @@ import ( func main() { // The total job count implies a fixed amount of work. - // The number of workers is how many py.Ctx (in concurrent goroutines) to pull jobs off the queue. - // One worker does all the work serially while N number of workers will (ideally) divide that up. - totalJobs := 20 - - for i := 0; i < 10; i++ { - numWorkers := i + 1 - elapsed := RunMultiPi(numWorkers, totalJobs) - fmt.Printf("=====> %2d worker(s): %v\n\n", numWorkers, elapsed) - - // Give each trial a fresh start - runtime.GC() - } - -} + // The number of workers is how many py.Ctx (in concurrent goroutines) to pull jobs off the queue. + // One worker does all the work serially while N number of workers will (ideally) divide that up. + totalJobs := 20 + + for i := 0; i < 10; i++ { + numWorkers := i + 1 + elapsed := RunMultiPi(numWorkers, totalJobs) + fmt.Printf("=====> %2d worker(s): %v\n\n", numWorkers, elapsed) + // Give each trial a fresh start + runtime.GC() + } + +} var jobScript = ` pi = chud.pi_chudnovsky_bs(numDigits) @@ -54,16 +57,16 @@ type worker struct { func (w *worker) compileTemplate(pySrc string) { pySrc = strings.Replace(pySrc, "{{WORKER_ID}}", w.name, -1) - - mainImpl := py.ModuleImpl{ - CodeSrc: pySrc, - } - - var err error - w.main, err = w.ctx.ModuleInit(&mainImpl) - if err != nil { - log.Fatal(err) - } + + mainImpl := py.ModuleImpl{ + CodeSrc: pySrc, + } + + var err error + w.main, err = w.ctx.ModuleInit(&mainImpl) + if err != nil { + log.Fatal(err) + } } func RunMultiPi(numWorkers, numTimes int) time.Duration { @@ -74,60 +77,57 @@ func RunMultiPi(numWorkers, numTimes int) time.Duration { jobPipe := make(chan int) go func() { for i := 0; i < numTimes; i++ { - jobPipe <- i+1 + jobPipe <- i + 1 } close(jobPipe) }() - + // Note that py.Code can be shared (accessed concurrently) since it is an inherently read-only object jobCode, err := py.Compile(jobScript, "", py.ExecMode, 0, true) if err != nil { - log.Fatal("jobScript failed to comple") + log.Fatal("jobScript failed to comple") } workers := make([]worker, numWorkers) for i := 0; i < numWorkers; i++ { - opts := py.DefaultCtxOpts() - - // Make sure our import statement will find pi_chudnovsky_bs - opts.SysPaths = append(opts.SysPaths, "..") + opts := py.DefaultCtxOpts() + + // Make sure our import statement will find pi_chudnovsky_bs + opts.SysPaths = append(opts.SysPaths, "..") workers[i] = worker{ name: fmt.Sprintf("Worker #%d", i+1), ctx: py.NewCtx(opts), - job: jobCode, + job: jobCode, } - + workersRunning.Add(1) - } - - startTime := time.Now() - + } + + startTime := time.Now() + for i := range workers { - w := workers[i] + w := workers[i] go func() { - - // Compiling can be concurrent since there is no associated py.Ctx - w.compileTemplate(jobSrcTemplate) - + + // Compiling can be concurrent since there is no associated py.Ctx + w.compileTemplate(jobSrcTemplate) + for jobID := range jobPipe { - numDigits := 100000 - if jobID % 2 == 0 { - numDigits *= 10 - } - py.SetAttrString(w.main.Globals, "numDigits", py.Int(numDigits)) - py.SetAttrString(w.main.Globals, "jobID", py.Int(jobID)) - w.ctx.RunCode(jobCode, w.main.Globals, w.main.Globals, nil) + numDigits := 100000 + if jobID%2 == 0 { + numDigits *= 10 + } + py.SetAttrString(w.main.Globals, "numDigits", py.Int(numDigits)) + py.SetAttrString(w.main.Globals, "jobID", py.Int(jobID)) + w.ctx.RunCode(jobCode, w.main.Globals, w.main.Globals, nil) } workersRunning.Done() }() } workersRunning.Wait() - + return time.Since(startTime) } - - - diff --git a/py/run.go b/py/run.go index f16056fb..4236a5cf 100644 --- a/py/run.go +++ b/py/run.go @@ -1,3 +1,7 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package py type CompileMode string diff --git a/py/util.go b/py/util.go index 5fbee64f..148c1d25 100644 --- a/py/util.go +++ b/py/util.go @@ -1,3 +1,7 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package py import ( From 6242836f2110df2bbd32710e82873bb6b171e241 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 10 Jan 2022 11:03:11 -0600 Subject: [PATCH 52/79] comment and cruft cleanup --- examples/multi-ctx/main.go | 2 +- py/run.go | 43 +++++++++++++------------------------- 2 files changed, 15 insertions(+), 30 deletions(-) diff --git a/examples/multi-ctx/main.go b/examples/multi-ctx/main.go index b35d581e..3bb7ac9c 100644 --- a/examples/multi-ctx/main.go +++ b/examples/multi-ctx/main.go @@ -20,7 +20,7 @@ func main() { // The total job count implies a fixed amount of work. // The number of workers is how many py.Ctx (in concurrent goroutines) to pull jobs off the queue. - // One worker does all the work serially while N number of workers will (ideally) divide that up. + // One worker does all the work serially while N number of workers will (ideally) divides up. totalJobs := 20 for i := 0; i < 10; i++ { diff --git a/py/run.go b/py/run.go index 4236a5cf..0d729e43 100644 --- a/py/run.go +++ b/py/run.go @@ -12,13 +12,6 @@ const ( SingleMode CompileMode = "single" // Compile a single (interactive) statement ) -type RunFlags int32 - -const ( - // RunOpts.FilePath is intelligently interpreted to load the appropriate pyc object (otherwise new code is generated from the implied .py file) - SmartCodeAcquire RunFlags = 0x01 -) - // Ctx is gpython virtual environment instance ("context"), providing a mechanism // for multiple gpython interpreters to run concurrently without restriction. // @@ -51,25 +44,13 @@ type Ctx interface { Store() *Store } -const ( - SrcFileExt = ".py" - CodeFileExt = ".pyc" -) - -type StdLib int32 - -const ( - Lib_sys StdLib = 1 << iota - Lib_time - - CoreLibs = Lib_sys | Lib_time -) - +// CompileOpts specifies options for high-level compilation. type CompileOpts struct { UseSysPaths bool // If set, sys.path will be used to resolve relative pathnames - CurDir string // If non-nil, this is the path of the current working directory. If nil, os.Getwd() is used. + CurDir string // If non-empty, this is the path of the current working directory. If empty, os.Getwd() is used. } +// CompileOut is the output object of high-level compilation, e.g. ResolveAndCompile() type CompileOut struct { SrcPathname string // Resolved pathname the .py file that was compiled (if applicable) PycPathname string // Pathname of the .pyc file read and/or written (if applicable) @@ -77,19 +58,23 @@ type CompileOut struct { Code *Code // Read/Output code object ready for execution } -// Can be changed during runtime and will \play nice with others using DefaultCtxOpts() -var CoreSysPaths = []string{ +// DefaultCoreSysPaths specify default search paths for module sys +// This can be changed during runtime and plays nice with others using DefaultCtxOpts() +var DefaultCoreSysPaths = []string{ ".", "lib", } -// Can be changed during runtime and will \play nice with others using DefaultCtxOpts() -var AuxSysPaths = []string{ +// DefaultAuxSysPaths are secondary default search paths for module sys. +// This can be changed during runtime and plays nice with others using DefaultCtxOpts() +// They are separated from the default core paths since they the more likley thing you will want to completely replace when using gpython. +var DefaultAuxSysPaths = []string{ "/usr/lib/python3.4", "/usr/local/lib/python3.4/dist-packages", "/usr/lib/python3/dist-packages", } +// CtxOpts specifies fundamental environment and input settings for creating a new py.Ctx type CtxOpts struct { SysArgs []string // sys.argv initializer SysPaths []string // sys.path initializer @@ -100,9 +85,9 @@ var ( // Calling this ensure that you future proof you code for suggested/default settings. DefaultCtxOpts = func() CtxOpts { opts := CtxOpts{ - SysPaths: CoreSysPaths, + SysPaths: DefaultCoreSysPaths, } - opts.SysPaths = append(opts.SysPaths, AuxSysPaths...) + opts.SysPaths = append(opts.SysPaths, DefaultAuxSysPaths...) return opts } @@ -115,7 +100,7 @@ var ( Compile func(src, srcDesc string, mode CompileMode, flags int, dont_inherit bool) (*Code, error) ) -// Resolves the given pathname, compiles as needed, and runs that code in the given module, returning the Module to indicate success. +// RunFile resolves the given pathname, compiles as needed, and runs that code in the given module, returning the Module to indicate success. // If inModule is a *Module, then the code is run in that module. // If inModule is nil, the code is run in a new __main__ module (and the new Module is returned). // If inModule is a string, the code is run in a new module with the given name (and the new Module is returned). From 61c783008031d7683679481b7cf2ca8863e0dbce Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 10 Jan 2022 11:22:56 -0600 Subject: [PATCH 53/79] renamed Store to ModuleStore + docs --- modules/runtime.go | 6 +++--- py/module.go | 19 +++++++++++-------- py/run.go | 2 +- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/modules/runtime.go b/modules/runtime.go index 978099bd..bd236e41 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -29,7 +29,7 @@ func init() { // ctx implements py.Ctx type ctx struct { - store *py.Store + store *py.ModuleStore opts py.CtxOpts } @@ -39,7 +39,7 @@ func NewCtx(opts py.CtxOpts) py.Ctx { opts: opts, } - ctx.store = py.NewStore() + ctx.store = py.NewModuleStore() py.Import(ctx, "builtins", "sys") @@ -224,6 +224,6 @@ func (ctx *ctx) GetModule(moduleName string) (*py.Module, error) { return ctx.store.GetModule(moduleName) } -func (ctx *ctx) Store() *py.Store { +func (ctx *ctx) Store() *py.ModuleStore { return ctx.store } diff --git a/py/module.go b/py/module.go index 96b99fd5..9e809af3 100644 --- a/py/module.go +++ b/py/module.go @@ -42,7 +42,8 @@ type ModuleImpl struct { Code *Code // Module code body } -type Store struct { +// ModuleStore is a container of Module imported into an owning py.Ctx. +type ModuleStore struct { // Registry of installed modules modules map[string]*Module // Builtin module @@ -78,13 +79,13 @@ func (rt *Runtime) RegisterModule(impl *ModuleImpl) { rt.ModuleImpls[impl.Info.Name] = impl } -func NewStore() *Store { - return &Store{ +func NewModuleStore() *ModuleStore { + return &ModuleStore{ modules: make(map[string]*Module), } } -// Module is a runtime instance of a ModuleImpl bound the py.Ctx that imported it. +// Module is a runtime instance of a ModuleImpl bound to the py.Ctx that imported it. type Module struct { ModuleInfo @@ -108,8 +109,10 @@ func (m *Module) GetDict() StringDict { return m.Globals } -// Define a new module -func (store *Store) NewModule(ctx Ctx, info ModuleInfo, methods []*Method, globals StringDict) (*Module, error) { +// NewModule adds a new Module instance to this ModuleStore. +// Each given Method prototype is used to create a new "live" Method bound this the newly created Module. +// This func also sets appropriate module global attribs based on the given ModuleInfo (e.g. __name__). +func (store *ModuleStore) NewModule(ctx Ctx, info ModuleInfo, methods []*Method, globals StringDict) (*Module, error) { if info.Name == "" { info.Name = MainModuleName } @@ -147,7 +150,7 @@ func (store *Store) NewModule(ctx Ctx, info ModuleInfo, methods []*Method, globa } // Gets a module -func (store *Store) GetModule(name string) (*Module, error) { +func (store *ModuleStore) GetModule(name string) (*Module, error) { m, ok := store.modules[name] if !ok { return nil, ExceptionNewf(ImportError, "Module '%s' not found", name) @@ -156,7 +159,7 @@ func (store *Store) GetModule(name string) (*Module, error) { } // Gets a module or panics -func (store *Store) MustGetModule(name string) *Module { +func (store *ModuleStore) MustGetModule(name string) *Module { m, err := store.GetModule(name) if err != nil { panic(err) diff --git a/py/run.go b/py/run.go index 0d729e43..7d5ac0ca 100644 --- a/py/run.go +++ b/py/run.go @@ -41,7 +41,7 @@ type Ctx interface { GetModule(moduleName string) (*Module, error) // Gereric access to this context's modules / state. - Store() *Store + Store() *ModuleStore } // CompileOpts specifies options for high-level compilation. From bb98aa8bdf35ac6ff69f4785f4a1bd46793c9c9f Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 10 Jan 2022 11:34:46 -0600 Subject: [PATCH 54/79] added import docs --- examples/multi-ctx/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/multi-ctx/main.go b/examples/multi-ctx/main.go index 3bb7ac9c..dce03736 100644 --- a/examples/multi-ctx/main.go +++ b/examples/multi-ctx/main.go @@ -12,7 +12,12 @@ import ( "sync" "time" + // This initializes gpython for runtime execution and is critical. + // It defines forward-declared symbols and registers native built-in modules, such as sys and time. _ "github.com/go-python/gpython/modules" + + // This is the primary import for gpython. + // It contains all symbols needed to fully compile and run python. "github.com/go-python/gpython/py" ) From 36b1018a0c41b7e9f481d1e0a828a5f14cfbb1d2 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 10 Jan 2022 11:38:33 -0600 Subject: [PATCH 55/79] shadowing cleanup --- modules/runtime.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/runtime.go b/modules/runtime.go index bd236e41..85442f1f 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -134,8 +134,7 @@ func (ctx *ctx) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.Comp } out.SrcPathname = fpath case ".pyc": - var file *os.File - file, err = os.Open(fpath) + file, err := os.Open(fpath) if err != nil { return false, py.ExceptionNewf(py.OSError, "Error opening %q: %w", fpath, err) } @@ -171,8 +170,8 @@ func resolveRunPath(runPath string, opts py.CompileOpts, pathObjs []py.Object, t runPath = strings.TrimSuffix(runPath, "/") var ( - err error - cwd string + err error + cwd string cont = true ) From f2602bc29c7ce5b91e797bce4d64917eaa7c90e2 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 10 Jan 2022 11:39:58 -0600 Subject: [PATCH 56/79] docs tweaks --- py/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/run.go b/py/run.go index 7d5ac0ca..d1a53f58 100644 --- a/py/run.go +++ b/py/run.go @@ -50,7 +50,7 @@ type CompileOpts struct { CurDir string // If non-empty, this is the path of the current working directory. If empty, os.Getwd() is used. } -// CompileOut is the output object of high-level compilation, e.g. ResolveAndCompile() +// CompileOut the output of high-level compilation -- e.g. ResolveAndCompile() type CompileOut struct { SrcPathname string // Resolved pathname the .py file that was compiled (if applicable) PycPathname string // Pathname of the .pyc file read and/or written (if applicable) From e25e3bb6b0ef05e88e1313b2a572caa452faa155 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 10 Jan 2022 11:45:23 -0600 Subject: [PATCH 57/79] code cleanup --- vm/eval.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/vm/eval.go b/vm/eval.go index 62ce6511..44d8ce62 100644 --- a/vm/eval.go +++ b/vm/eval.go @@ -1087,10 +1087,8 @@ func do_IMPORT_NAME(vm *Vm, namei int32) error { } v := vm.POP() u := vm.TOP() - var locals py.Object - if vm.frame.Locals == nil { - locals = py.None - } else { + var locals py.Object = py.None + if vm.frame.Locals != nil { locals = vm.frame.Locals } var args py.Tuple @@ -1734,7 +1732,7 @@ func (vm *Vm) UnwindExceptHandler(frame *py.Frame, block *py.TryBlock) { func RunFrame(frame *py.Frame) (res py.Object, err error) { var vm = Vm{ frame: frame, - ctx: frame.Ctx, + ctx: frame.Ctx, } // FIXME need to do this to save the old exeption when we @@ -2172,7 +2170,6 @@ func EvalCode(ctx py.Ctx, co *py.Code, globals, locals py.StringDict, args []py. return RunFrame(f) } - // Write the py global to avoid circular import func init() { py.VmEvalCode = EvalCode From 9421b6b16b3c63c207f8c0b25bb83410f41ccb1e Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 10 Jan 2022 11:48:44 -0600 Subject: [PATCH 58/79] synced with gpython --- go.mod | 7 +++---- go.sum | 13 +++++-------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 4a927986..095bd34f 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,8 @@ module github.com/go-python/gpython -go 1.15 +go 1.16 require ( - github.com/gopherjs/gopherwasm v1.1.0 - github.com/mattn/go-runewidth v0.0.13 // indirect - github.com/peterh/liner v1.2.1 + github.com/gopherjs/gopherwasm v1.0.0 + github.com/peterh/liner v1.1.0 ) diff --git a/go.sum b/go.sum index 6ee05de2..c17fb31d 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,8 @@ github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSfgRJKqIzapE78onvvTbdi1rMkU00lZw= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherwasm v1.1.0 h1:fA2uLoctU5+T3OhOn2vYP0DVT6pxc7xhTlBB1paATqQ= -github.com/gopherjs/gopherwasm v1.1.0/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI= +github.com/gopherjs/gopherwasm v1.0.0 h1:32nge/RlujS1Im4HNCJPp0NbBOAeBXFuT1KonUuLl+Y= +github.com/gopherjs/gopherwasm v1.0.0/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI= +github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/peterh/liner v1.2.1 h1:O4BlKaq/LWu6VRWmol4ByWfzx6MfXc5Op5HETyIy5yg= -github.com/peterh/liner v1.2.1/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/peterh/liner v1.1.0 h1:f+aAedNJA6uk7+6rXsYBnhdo4Xux7ESLe+kcuVUF5os= +github.com/peterh/liner v1.1.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= From 96515ea603508258e6ec8af97881a4aef17ac217 Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Mon, 10 Jan 2022 12:10:07 -0600 Subject: [PATCH 59/79] Update modules/runtime.go Co-authored-by: Sebastien Binet --- modules/runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/runtime.go b/modules/runtime.go index 85442f1f..1b4967bb 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -116,7 +116,7 @@ func (ctx *ctx) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.Comp if os.IsNotExist(err) { return true, nil } - err = py.ExceptionNewf(py.OSError, "Error accessing %q: %w", fpath, err) + err = py.ExceptionNewf(py.OSError, "Error accessing %q: %v", fpath, err) return false, err } From f62f2b71ad64b171e045fe0b4699be6af776e771 Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Mon, 10 Jan 2022 12:10:14 -0600 Subject: [PATCH 60/79] Update modules/runtime.go Co-authored-by: Sebastien Binet --- modules/runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/runtime.go b/modules/runtime.go index 1b4967bb..5ce39c81 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -141,7 +141,7 @@ func (ctx *ctx) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.Comp defer file.Close() codeObj, err := marshal.ReadPyc(file) if err != nil { - return false, py.ExceptionNewf(py.ImportError, "Failed to marshal %q: %w", fpath, err) + return false, py.ExceptionNewf(py.ImportError, "Failed to marshal %q: %v", fpath, err) } out.Code, _ = codeObj.(*py.Code) out.PycPathname = fpath From 3bdba62f7be8f1d82c6216a5807f082f7634507c Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 10 Jan 2022 15:34:56 -0600 Subject: [PATCH 61/79] reverted from %w to %v (linux CI compile failure) --- modules/runtime.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/runtime.go b/modules/runtime.go index 5ce39c81..26c6791e 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -125,7 +125,7 @@ func (ctx *ctx) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.Comp var pySrc []byte pySrc, err = ioutil.ReadFile(fpath) if err != nil { - return false, py.ExceptionNewf(py.OSError, "Error reading %q: %w", fpath, err) + return false, py.ExceptionNewf(py.OSError, "Error reading %q: %v", fpath, err) } out.Code, err = py.Compile(string(pySrc), fpath, py.ExecMode, 0, true) @@ -136,7 +136,7 @@ func (ctx *ctx) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.Comp case ".pyc": file, err := os.Open(fpath) if err != nil { - return false, py.ExceptionNewf(py.OSError, "Error opening %q: %w", fpath, err) + return false, py.ExceptionNewf(py.OSError, "Error opening %q: %v", fpath, err) } defer file.Close() codeObj, err := marshal.ReadPyc(file) From 8033615317b96cab0fe92cb15a2c394d09f047cf Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 10 Jan 2022 15:59:24 -0600 Subject: [PATCH 62/79] renamed py.Ctx to py.Context --- builtin/builtin.go | 6 +++--- examples/multi-ctx/main.go | 10 +++++----- main.go | 12 ++++++------ modules/runtime.go | 26 +++++++++++++------------- py/frame.go | 6 +++--- py/function.go | 8 ++++---- py/import.go | 8 ++++---- py/module.go | 22 +++++++++++----------- py/py.go | 2 +- py/run.go | 32 ++++++++++++++++---------------- pytest/pytest.go | 10 +++++----- repl/repl.go | 14 +++++++------- vm/builtin.go | 6 +++--- vm/eval.go | 16 ++++++++-------- vm/vm.go | 2 +- vm/vm_test.go | 10 +++++----- 16 files changed, 95 insertions(+), 95 deletions(-) diff --git a/builtin/builtin.go b/builtin/builtin.go index 14ab7e23..2d8d7a9b 100644 --- a/builtin/builtin.go +++ b/builtin/builtin.go @@ -172,7 +172,7 @@ func init() { Methods: methods, Globals: globals, }) - + } const print_doc = `print(value, ..., sep=' ', end='\\n', file=sys.stdout, flush=False) @@ -190,7 +190,7 @@ func builtin_print(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Obje endObj py.Object = py.String("\n") flush py.Object ) - sysModule, err := self.(*py.Module).Ctx.GetModule("sys") + sysModule, err := self.(*py.Module).Context.GetModule("sys") if err != nil { return nil, err } @@ -463,7 +463,7 @@ func builtin___build_class__(self py.Object, args py.Tuple, kwargs py.StringDict } // fmt.Printf("Calling %v with %v and %v\n", fn.Name, fn.Globals, ns) // fmt.Printf("Code = %#v\n", fn.Code) - cell, err = fn.Ctx.RunCode(fn.Code, fn.Globals, ns, fn.Closure) + cell, err = fn.Context.RunCode(fn.Code, fn.Globals, ns, fn.Closure) if err != nil { return nil, err } diff --git a/examples/multi-ctx/main.go b/examples/multi-ctx/main.go index dce03736..e9f72212 100644 --- a/examples/multi-ctx/main.go +++ b/examples/multi-ctx/main.go @@ -24,7 +24,7 @@ import ( func main() { // The total job count implies a fixed amount of work. - // The number of workers is how many py.Ctx (in concurrent goroutines) to pull jobs off the queue. + // The number of workers is how many py.Context (in concurrent goroutines) to pull jobs off the queue. // One worker does all the work serially while N number of workers will (ideally) divides up. totalJobs := 20 @@ -55,7 +55,7 @@ print("%s ready!" % (WORKER_ID)) type worker struct { name string - ctx py.Ctx + ctx py.Context main *py.Module job *py.Code } @@ -96,14 +96,14 @@ func RunMultiPi(numWorkers, numTimes int) time.Duration { workers := make([]worker, numWorkers) for i := 0; i < numWorkers; i++ { - opts := py.DefaultCtxOpts() + opts := py.DefaultContextOpts() // Make sure our import statement will find pi_chudnovsky_bs opts.SysPaths = append(opts.SysPaths, "..") workers[i] = worker{ name: fmt.Sprintf("Worker #%d", i+1), - ctx: py.NewCtx(opts), + ctx: py.NewContext(opts), job: jobCode, } @@ -116,7 +116,7 @@ func RunMultiPi(numWorkers, numTimes int) time.Duration { w := workers[i] go func() { - // Compiling can be concurrent since there is no associated py.Ctx + // Compiling can be concurrent since there is no associated py.Context w.compileTemplate(jobSrcTemplate) for jobID := range jobPipe { diff --git a/main.go b/main.go index c4cdc98f..9743bc0f 100644 --- a/main.go +++ b/main.go @@ -43,11 +43,11 @@ func main() { flag.Usage = syntaxError flag.Parse() args := flag.Args() - - opts := py.DefaultCtxOpts() + + opts := py.DefaultContextOpts() opts.SysArgs = flag.Args() - ctx := py.NewCtx(opts) - + ctx := py.NewContext(opts) + if *cpuprofile != "" { f, err := os.Create(*cpuprofile) if err != nil { @@ -59,7 +59,7 @@ func main() { } defer pprof.StopCPUProfile() } - + // IF no args, enter REPL mode if len(args) == 0 { @@ -70,7 +70,7 @@ func main() { replCtx := repl.New(ctx) cli.RunREPL(replCtx) - + } else { _, err := py.RunFile(ctx, args[0], py.CompileOpts{}, nil) if err != nil { diff --git a/modules/runtime.go b/modules/runtime.go index 26c6791e..f42a611a 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -23,19 +23,19 @@ import ( ) func init() { - // Assign the base-level py.Ctx creation function while also preventing an import cycle. - py.NewCtx = NewCtx + // Assign the base-level py.Context creation function while also preventing an import cycle. + py.NewContext = NewContext } -// ctx implements py.Ctx -type ctx struct { +// context implements py.Context +type context struct { store *py.ModuleStore - opts py.CtxOpts + opts py.ContextOpts } -// See py.Ctx interface -func NewCtx(opts py.CtxOpts) py.Ctx { - ctx := &ctx{ +// See py.Context interface +func NewContext(opts py.ContextOpts) py.Context { + ctx := &context{ opts: opts, } @@ -50,7 +50,7 @@ func NewCtx(opts py.CtxOpts) py.Ctx { return ctx } -func (ctx *ctx) ModuleInit(impl *py.ModuleImpl) (*py.Module, error) { +func (ctx *context) ModuleInit(impl *py.ModuleImpl) (*py.Module, error) { var err error if impl.Code == nil && len(impl.CodeSrc) > 0 { @@ -87,7 +87,7 @@ func (ctx *ctx) ModuleInit(impl *py.ModuleImpl) (*py.Module, error) { return module, nil } -func (ctx *ctx) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.CompileOut, error) { +func (ctx *context) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.CompileOut, error) { tryPaths := defaultPaths if opts.UseSysPaths { tryPaths = ctx.Store().MustGetModule("sys").Globals["path"].(*py.List).Items @@ -215,14 +215,14 @@ func resolveRunPath(runPath string, opts py.CompileOpts, pathObjs []py.Object, t return err } -func (ctx *ctx) RunCode(code *py.Code, globals, locals py.StringDict, closure py.Tuple) (py.Object, error) { +func (ctx *context) RunCode(code *py.Code, globals, locals py.StringDict, closure py.Tuple) (py.Object, error) { return vm.EvalCode(ctx, code, globals, locals, nil, nil, nil, nil, closure) } -func (ctx *ctx) GetModule(moduleName string) (*py.Module, error) { +func (ctx *context) GetModule(moduleName string) (*py.Module, error) { return ctx.store.GetModule(moduleName) } -func (ctx *ctx) Store() *py.ModuleStore { +func (ctx *context) Store() *py.ModuleStore { return ctx.store } diff --git a/py/frame.go b/py/frame.go index 68b07aba..ce595d8a 100644 --- a/py/frame.go +++ b/py/frame.go @@ -26,7 +26,7 @@ type TryBlock struct { // A python Frame object type Frame struct { // Back *Frame // previous frame, or nil - Ctx Ctx // host module (state) context + Context Context // host module (state) context Code *Code // code segment Builtins StringDict // builtin symbol table Globals StringDict // global symbol table @@ -78,7 +78,7 @@ func (o *Frame) Type() *Type { } // Make a new frame for a code object -func NewFrame(ctx Ctx, globals, locals StringDict, code *Code, closure Tuple) *Frame { +func NewFrame(ctx Context, globals, locals StringDict, code *Code, closure Tuple) *Frame { nlocals := int(code.Nlocals) ncells := len(code.Cellvars) nfrees := len(code.Freevars) @@ -91,7 +91,7 @@ func NewFrame(ctx Ctx, globals, locals StringDict, code *Code, closure Tuple) *F cellAndFreeVars := allocation[nlocals:varsize] return &Frame{ - Ctx: ctx, + Context: ctx, Globals: globals, Locals: locals, Code: code, diff --git a/py/function.go b/py/function.go index ba9d162e..2c37499d 100644 --- a/py/function.go +++ b/py/function.go @@ -18,7 +18,7 @@ package py // A python Function object type Function struct { Code *Code // A code object, the __code__ attribute - Ctx Ctx // Host VM context + Context Context // Host VM context Globals StringDict // A dictionary (other mappings won't do) Defaults Tuple // NULL or a tuple KwDefaults StringDict // NULL or a dict @@ -56,7 +56,7 @@ func (f *Function) GetDict() StringDict { // attribute. qualname should be a unicode object or ""; if "", the // __qualname__ attribute is set to the same value as its __name__ // attribute. -func NewFunction(ctx Ctx, code *Code, globals StringDict, qualname string) *Function { +func NewFunction(ctx Context, code *Code, globals StringDict, qualname string) *Function { var doc Object if len(code.Consts) >= 1 { doc = code.Consts[0] @@ -73,7 +73,7 @@ func NewFunction(ctx Ctx, code *Code, globals StringDict, qualname string) *Func return &Function{ Code: code, - Ctx: ctx, + Context: ctx, Qualname: qualname, Globals: globals, Name: code.Name, @@ -84,7 +84,7 @@ func NewFunction(ctx Ctx, code *Code, globals StringDict, qualname string) *Func // Call a function func (f *Function) M__call__(args Tuple, kwargs StringDict) (Object, error) { - result, err := VmEvalCode(f.Ctx, f.Code, f.Globals, NewStringDict(), args, kwargs, f.Defaults, f.KwDefaults, f.Closure) + result, err := VmEvalCode(f.Context, f.Code, f.Globals, NewStringDict(), args, kwargs, f.Defaults, f.KwDefaults, f.Closure) if err != nil { return nil, err } diff --git a/py/import.go b/py/import.go index 454473d3..0eaae921 100644 --- a/py/import.go +++ b/py/import.go @@ -11,7 +11,7 @@ import ( "strings" ) -func Import(ctx Ctx, names ...string) error { +func Import(ctx Context, names ...string) error { for _, name := range names { _, err := ImportModuleLevelObject(ctx, name, nil, nil, nil, 0) if err != nil { @@ -80,7 +80,7 @@ func Import(ctx Ctx, names ...string) error { // // Changed in version 3.3: Negative values for level are no longer // supported (which also changes the default value to 0). -func ImportModuleLevelObject(ctx Ctx, name string, globals, locals StringDict, fromlist Tuple, level int) (Object, error) { +func ImportModuleLevelObject(ctx Context, name string, globals, locals StringDict, fromlist Tuple, level int) (Object, error) { // Module already loaded - return that if module, err := ctx.GetModule(name); err == nil { return module, nil @@ -124,7 +124,7 @@ func ImportModuleLevelObject(ctx Ctx, name string, globals, locals StringDict, f // This calls functins from _bootstrap.py which is a frozen module // // Too much functionality for the moment -func XImportModuleLevelObject(ctx Ctx, nameObj, given_globals, locals, given_fromlist Object, level int) (Object, error) { +func XImportModuleLevelObject(ctx Context, nameObj, given_globals, locals, given_fromlist Object, level int) (Object, error) { var abs_name string var builtins_import Object var final_mod Object @@ -338,7 +338,7 @@ error: } // The actual import code -func BuiltinImport(ctx Ctx, self Object, args Tuple, kwargs StringDict, currentGlobal StringDict) (Object, error) { +func BuiltinImport(ctx Context, self Object, args Tuple, kwargs StringDict, currentGlobal StringDict) (Object, error) { kwlist := []string{"name", "globals", "locals", "fromlist", "level"} var name Object var globals Object = currentGlobal diff --git a/py/module.go b/py/module.go index 9e809af3..248d1168 100644 --- a/py/module.go +++ b/py/module.go @@ -14,14 +14,14 @@ import ( type ModuleFlags int32 const ( - // Set for modules that are threadsafe, stateless, and/or can be shared across multiple py.Ctx instances (for efficiency). - // Otherwise, a separate module instance is created for each py.Ctx that imports it. + // Set for modules that are threadsafe, stateless, and/or can be shared across multiple py.Context instances (for efficiency). + // Otherwise, a separate module instance is created for each py.Context that imports it. ShareModule ModuleFlags = 0x01 // @@TODO MainModuleName = "__main__" ) -// ModuleInfo contains info and about a module and can specify flags that affect how it is imported into a py.Ctx +// ModuleInfo contains info and about a module and can specify flags that affect how it is imported into a py.Context type ModuleInfo struct { Name string // __name__ (if nil, "__main__" is used) Doc string // __doc__ @@ -29,8 +29,8 @@ type ModuleInfo struct { Flags ModuleFlags } -// ModuleImpl is used for modules that are ready to be imported into a py.Ctx. -// If a module is threadsafe and stateless it can be shared across multiple py.Ctx instances (for efficiency). +// ModuleImpl is used for modules that are ready to be imported into a py.Context. +// If a module is threadsafe and stateless it can be shared across multiple py.Context instances (for efficiency). // By convention, .Code is executed when a module instance is initialized. // If .Code == nil, then .CodeBuf or .CodeSrc will be auto-compiled to set .Code. type ModuleImpl struct { @@ -42,7 +42,7 @@ type ModuleImpl struct { Code *Code // Module code body } -// ModuleStore is a container of Module imported into an owning py.Ctx. +// ModuleStore is a container of Module imported into an owning py.Context. type ModuleStore struct { // Registry of installed modules modules map[string]*Module @@ -85,12 +85,12 @@ func NewModuleStore() *ModuleStore { } } -// Module is a runtime instance of a ModuleImpl bound to the py.Ctx that imported it. +// Module is a runtime instance of a ModuleImpl bound to the py.Context that imported it. type Module struct { ModuleInfo Globals StringDict - Ctx Ctx + Context Context } var ModuleType = NewType("module", "module object") @@ -112,17 +112,17 @@ func (m *Module) GetDict() StringDict { // NewModule adds a new Module instance to this ModuleStore. // Each given Method prototype is used to create a new "live" Method bound this the newly created Module. // This func also sets appropriate module global attribs based on the given ModuleInfo (e.g. __name__). -func (store *ModuleStore) NewModule(ctx Ctx, info ModuleInfo, methods []*Method, globals StringDict) (*Module, error) { +func (store *ModuleStore) NewModule(ctx Context, info ModuleInfo, methods []*Method, globals StringDict) (*Module, error) { if info.Name == "" { info.Name = MainModuleName } m := &Module{ ModuleInfo: info, Globals: globals.Copy(), - Ctx: ctx, + Context: ctx, } // Insert the methods into the module dictionary - // Copy each method an insert each "live" with a ptr back to the module (which can also lead us to the host Ctx) + // Copy each method an insert each "live" with a ptr back to the module (which can also lead us to the host Context) for _, method := range methods { methodInst := new(Method) *methodInst = *method diff --git a/py/py.go b/py/py.go index d9fd3351..59d0737a 100644 --- a/py/py.go +++ b/py/py.go @@ -26,7 +26,7 @@ type IGoInt64 interface { var ( // Set in vm/eval.go - to avoid circular import - VmEvalCode func(ctx Ctx, code *Code, globals, locals StringDict, args []Object, kws StringDict, defs []Object, kwdefs StringDict, closure Tuple) (retval Object, err error) + VmEvalCode func(ctx Context, code *Code, globals, locals StringDict, args []Object, kws StringDict, defs []Object, kwdefs StringDict, closure Tuple) (retval Object, err error) VmRunFrame func(frame *Frame) (res Object, err error) ) diff --git a/py/run.go b/py/run.go index d1a53f58..1d4fe73a 100644 --- a/py/run.go +++ b/py/run.go @@ -12,17 +12,17 @@ const ( SingleMode CompileMode = "single" // Compile a single (interactive) statement ) -// Ctx is gpython virtual environment instance ("context"), providing a mechanism +// Context is gpython virtual environment instance ("context"), providing a mechanism // for multiple gpython interpreters to run concurrently without restriction. // -// In general, one creates a py.Ctx (via py.NewCtx) for each concurrent goroutine to be running an interpreter. -// In other words, ensure that a py.Ctx is never concurrently accessed across goroutines. +// In general, one creates a py.Context (via py.NewContext) for each concurrent goroutine to be running an interpreter. +// In other words, ensure that a py.Context is never concurrently accessed across goroutines. // // RunFile() and RunCode() block until code execution is complete. -// In the future, they will abort early if the parent associated py.Ctx is signaled. +// In the future, they will abort early if the parent associated py.Context is signaled. // // See examples/multi-ctx -type Ctx interface { +type Context interface { // Resolves then compiles (if applicable) the given file system pathname into a py.Code ready to be executed. ResolveAndCompile(pathname string, opts CompileOpts) (CompileOut, error) @@ -59,14 +59,14 @@ type CompileOut struct { } // DefaultCoreSysPaths specify default search paths for module sys -// This can be changed during runtime and plays nice with others using DefaultCtxOpts() +// This can be changed during runtime and plays nice with others using DefaultContextOpts() var DefaultCoreSysPaths = []string{ ".", "lib", } // DefaultAuxSysPaths are secondary default search paths for module sys. -// This can be changed during runtime and plays nice with others using DefaultCtxOpts() +// This can be changed during runtime and plays nice with others using DefaultContextOpts() // They are separated from the default core paths since they the more likley thing you will want to completely replace when using gpython. var DefaultAuxSysPaths = []string{ "/usr/lib/python3.4", @@ -74,26 +74,26 @@ var DefaultAuxSysPaths = []string{ "/usr/lib/python3/dist-packages", } -// CtxOpts specifies fundamental environment and input settings for creating a new py.Ctx -type CtxOpts struct { +// ContextOpts specifies fundamental environment and input settings for creating a new py.Context +type ContextOpts struct { SysArgs []string // sys.argv initializer SysPaths []string // sys.path initializer } var ( - // DefaultCtxOpts should be the default opts created for py.NewCtx. + // DefaultContextOpts should be the default opts created for py.NewContext. // Calling this ensure that you future proof you code for suggested/default settings. - DefaultCtxOpts = func() CtxOpts { - opts := CtxOpts{ + DefaultContextOpts = func() ContextOpts { + opts := ContextOpts{ SysPaths: DefaultCoreSysPaths, } opts.SysPaths = append(opts.SysPaths, DefaultAuxSysPaths...) return opts } - // NewCtx is a high-level call to create a new gpython interpreter context. - // See type Ctx interface. - NewCtx func(opts CtxOpts) Ctx + // NewContext is a high-level call to create a new gpython interpreter context. + // See type Context interface. + NewContext func(opts ContextOpts) Context // Compiles a python buffer into a py.Code object. // Returns a py.Code object or otherwise an error. @@ -104,7 +104,7 @@ var ( // If inModule is a *Module, then the code is run in that module. // If inModule is nil, the code is run in a new __main__ module (and the new Module is returned). // If inModule is a string, the code is run in a new module with the given name (and the new Module is returned). -func RunFile(ctx Ctx, pathname string, opts CompileOpts, inModule interface{}) (*Module, error) { +func RunFile(ctx Context, pathname string, opts CompileOpts, inModule interface{}) (*Module, error) { out, err := ctx.ResolveAndCompile(pathname, opts) if err != nil { return nil, err diff --git a/pytest/pytest.go b/pytest/pytest.go index e42c42d8..6877d966 100644 --- a/pytest/pytest.go +++ b/pytest/pytest.go @@ -17,7 +17,7 @@ import ( "github.com/go-python/gpython/py" ) -var gCtx = py.NewCtx(py.DefaultCtxOpts()) +var gContext = py.NewContext(py.DefaultContextOpts()) // Compile the program in the file prog to code in the module that is returned func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) { @@ -35,10 +35,10 @@ func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) { if err != nil { t.Fatalf("%s: ReadAll failed: %v", prog, err) } - return CompileSrc(t, gCtx, string(str), prog) + return CompileSrc(t, gContext, string(str), prog) } -func CompileSrc(t testing.TB, ctx py.Ctx, pySrc string, prog string) (*py.Module, *py.Code) { +func CompileSrc(t testing.TB, ctx py.Context, pySrc string, prog string) (*py.Module, *py.Code) { code, err := compile.Compile(string(pySrc), prog, py.ExecMode, 0, true) if err != nil { t.Fatalf("%s: Compile failed: %v", prog, err) @@ -50,13 +50,13 @@ func CompileSrc(t testing.TB, ctx py.Ctx, pySrc string, prog string) (*py.Module if err != nil { t.Fatalf("%s: NewModule failed: %v", prog, err) } - + return module, code } // Run the code in the module func run(t testing.TB, module *py.Module, code *py.Code) { - _, err := gCtx.RunCode(code, module.Globals, module.Globals, nil) + _, err := gContext.RunCode(code, module.Globals, module.Globals, nil) if err != nil { if wantErrObj, ok := module.Globals["err"]; ok { gotExc, ok := err.(py.ExceptionInfo) diff --git a/repl/repl.go b/repl/repl.go index f65e560f..f6639b25 100644 --- a/repl/repl.go +++ b/repl/repl.go @@ -22,7 +22,7 @@ const ( // Repl state type REPL struct { - Ctx py.Ctx + Context py.Context Module *py.Module prog string continuation bool @@ -40,13 +40,13 @@ type UI interface { } // New create a new REPL and initializes the state machine -func New(ctx py.Ctx) *REPL { +func New(ctx py.Context) *REPL { if ctx == nil { - ctx = py.NewCtx(py.DefaultCtxOpts()) + ctx = py.NewContext(py.DefaultContextOpts()) } - + r := &REPL{ - Ctx: ctx, + Context: ctx, prog: "", continuation: false, previous: "", @@ -107,7 +107,7 @@ func (r *REPL) Run(line string) { r.term.Print(fmt.Sprintf("Compile error: %v", err)) return } - _, err = r.Ctx.RunCode(code, r.Module.Globals, r.Module.Globals, nil) + _, err = r.Context.RunCode(code, r.Module.Globals, r.Module.Globals, nil) if err != nil { py.TracebackDump(err) } @@ -137,7 +137,7 @@ func (r *REPL) Completer(line string, pos int) (head string, completions []strin } } match(r.Module.Globals) - match(r.Ctx.Store().Builtins.Globals) + match(r.Context.Store().Builtins.Globals) sort.Strings(completions) return head, completions, tail } diff --git a/vm/builtin.go b/vm/builtin.go index 6c063b37..6466b31f 100644 --- a/vm/builtin.go +++ b/vm/builtin.go @@ -12,7 +12,7 @@ import ( "github.com/go-python/gpython/py" ) -func builtinEvalOrExec(ctx py.Ctx, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict, mode py.CompileMode) (py.Object, error) { +func builtinEvalOrExec(ctx py.Context, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict, mode py.CompileMode) (py.Object, error) { var ( cmd py.Object globals py.Object = py.None @@ -71,11 +71,11 @@ func builtinEvalOrExec(ctx py.Ctx, args py.Tuple, kwargs, currentLocals, current return ctx.RunCode(code, globalsDict, localsDict, nil) } -func builtinEval(ctx py.Ctx, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { +func builtinEval(ctx py.Context, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { return builtinEvalOrExec(ctx, args, kwargs, currentLocals, currentGlobals, builtins, py.EvalMode) } -func builtinExec(ctx py.Ctx, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { +func builtinExec(ctx py.Context, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { _, err := builtinEvalOrExec(ctx, args, kwargs, currentLocals, currentGlobals, builtins, py.ExecMode) if err != nil { return nil, err diff --git a/vm/eval.go b/vm/eval.go index 44d8ce62..94bf6d1e 100644 --- a/vm/eval.go +++ b/vm/eval.go @@ -767,7 +767,7 @@ func do_END_FINALLY(vm *Vm, arg int32) error { // Loads the __build_class__ helper function to the stack which // creates a new class object. func do_LOAD_BUILD_CLASS(vm *Vm, arg int32) error { - vm.PUSH(vm.ctx.Store().Builtins.Globals["__build_class__"]) + vm.PUSH(vm.context.Store().Builtins.Globals["__build_class__"]) return nil } @@ -1435,7 +1435,7 @@ func _make_function(vm *Vm, argc int32, opcode OpCode) { num_annotations := (argc >> 16) & 0x7fff qualname := vm.POP() code := vm.POP() - function := py.NewFunction(vm.ctx, code.(*py.Code), vm.frame.Globals, string(qualname.(py.String))) + function := py.NewFunction(vm.context, code.(*py.Code), vm.frame.Globals, string(qualname.(py.String))) if opcode == MAKE_CLOSURE { function.Closure = vm.POP().(py.Tuple) @@ -1592,13 +1592,13 @@ func callInternal(fn py.Object, args py.Tuple, kwargs py.StringDict, f *py.Frame f.FastToLocals() return f.Locals, nil case py.InternalMethodImport: - return py.BuiltinImport(f.Ctx, nil, args, kwargs, f.Globals) + return py.BuiltinImport(f.Context, nil, args, kwargs, f.Globals) case py.InternalMethodEval: f.FastToLocals() - return builtinEval(f.Ctx, args, kwargs, f.Locals, f.Globals, f.Builtins) + return builtinEval(f.Context, args, kwargs, f.Locals, f.Globals, f.Builtins) case py.InternalMethodExec: f.FastToLocals() - return builtinExec(f.Ctx, args, kwargs, f.Locals, f.Globals, f.Builtins) + return builtinExec(f.Context, args, kwargs, f.Locals, f.Globals, f.Builtins) default: return nil, py.ExceptionNewf(py.SystemError, "Internal method %v not found", x) } @@ -1731,8 +1731,8 @@ func (vm *Vm) UnwindExceptHandler(frame *py.Frame, block *py.TryBlock) { // This is the equivalent of PyEval_EvalFrame func RunFrame(frame *py.Frame) (res py.Object, err error) { var vm = Vm{ - frame: frame, - ctx: frame.Ctx, + frame: frame, + context: frame.Context, } // FIXME need to do this to save the old exeption when we @@ -2041,7 +2041,7 @@ func tooManyPositional(co *py.Code, given, defcount int, fastlocals []py.Object) // Returns an Object and an error. The error will be a py.ExceptionInfo // // This is the equivalent of PyEval_EvalCode with closure support -func EvalCode(ctx py.Ctx, co *py.Code, globals, locals py.StringDict, args []py.Object, kws py.StringDict, defs []py.Object, kwdefs py.StringDict, closure py.Tuple) (retval py.Object, err error) { +func EvalCode(ctx py.Context, co *py.Code, globals, locals py.StringDict, args []py.Object, kws py.StringDict, defs []py.Object, kwdefs py.StringDict, closure py.Tuple) (retval py.Object, err error) { total_args := int(co.Argcount + co.Kwonlyargcount) n := len(args) var kwdict py.StringDict diff --git a/vm/vm.go b/vm/vm.go index 8df37c9e..9c122f32 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -42,5 +42,5 @@ type Vm struct { // Previous exception type, value and traceback exc py.ExceptionInfo // This vm's access to persistent state and modules - ctx py.Ctx + context py.Context } diff --git a/vm/vm_test.go b/vm/vm_test.go index 19c3dc46..a47e20dc 100644 --- a/vm/vm_test.go +++ b/vm/vm_test.go @@ -25,7 +25,7 @@ func BenchmarkVM(b *testing.B) { var jobSrcTemplate = ` -doc="multi py.Ctx text" +doc="multi py.Context text" WORKER_ID = "{{WORKER_ID}}" def fib(n): if n == 0: @@ -41,7 +41,7 @@ print("%s says fib(%d) is %d" % (WORKER_ID, x, fx)) type worker struct { name string - ctx py.Ctx + ctx py.Context } func (w *worker) run(b testing.TB, pySrc string, countUpto int) { @@ -55,7 +55,7 @@ func (w *worker) run(b testing.TB, pySrc string, countUpto int) { } } -func BenchmarkCtx(b *testing.B) { +func BenchmarkContext(b *testing.B) { numWorkers := 4 workersRunning := sync.WaitGroup{} @@ -65,7 +65,7 @@ func BenchmarkCtx(b *testing.B) { jobPipe := make(chan int) go func() { for i := 0; i < numJobs; i++ { - jobPipe <- i+1 + jobPipe <- i + 1 } close(jobPipe) }() @@ -75,7 +75,7 @@ func BenchmarkCtx(b *testing.B) { workers[i] = worker{ name: fmt.Sprintf("Worker #%d", i+1), - ctx: py.NewCtx(py.DefaultCtxOpts()), + ctx: py.NewContext(py.DefaultContextOpts()), } workersRunning.Add(1) From 681abf2de628248df3cbba21d4e29b80e21e2232 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Tue, 18 Jan 2022 13:28:20 -0600 Subject: [PATCH 63/79] added Context.Close() and ModuleImpl.OnContextClosed() --- modules/runtime.go | 67 +++++++++++++++++++++++++++++++---- py/module.go | 87 +++++++++++++++++++++++++++------------------- py/run.go | 56 +++++++++++++++++++++-------- py/util.go | 8 ++--- pytest/pytest.go | 8 +++-- 5 files changed, 163 insertions(+), 63 deletions(-) diff --git a/modules/runtime.go b/modules/runtime.go index f42a611a..9b8d2ada 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -11,6 +11,7 @@ import ( "path" "path/filepath" "strings" + "sync" "github.com/go-python/gpython/marshal" "github.com/go-python/gpython/py" @@ -29,14 +30,22 @@ func init() { // context implements py.Context type context struct { - store *py.ModuleStore - opts py.ContextOpts + store *py.ModuleStore + opts py.ContextOpts + closeOnce sync.Once + closing bool + closed bool + running sync.WaitGroup + done chan struct{} } // See py.Context interface func NewContext(opts py.ContextOpts) py.Context { ctx := &context{ - opts: opts, + opts: opts, + done: make(chan struct{}), + closing: false, + closed: false, } ctx.store = py.NewModuleStore() @@ -51,7 +60,11 @@ func NewContext(opts py.ContextOpts) py.Context { } func (ctx *context) ModuleInit(impl *py.ModuleImpl) (*py.Module, error) { - var err error + err := ctx.PushBusy() + defer ctx.PopBusy() + if err != nil { + return nil, err + } if impl.Code == nil && len(impl.CodeSrc) > 0 { impl.Code, err = py.Compile(string(impl.CodeSrc), impl.Info.FileDesc, py.ExecMode, 0, true) @@ -72,7 +85,7 @@ func (ctx *context) ModuleInit(impl *py.ModuleImpl) (*py.Module, error) { } } - module, err := ctx.Store().NewModule(ctx, impl.Info, impl.Methods, impl.Globals) + module, err := ctx.Store().NewModule(ctx, impl) if err != nil { return nil, err } @@ -88,6 +101,12 @@ func (ctx *context) ModuleInit(impl *py.ModuleImpl) (*py.Module, error) { } func (ctx *context) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.CompileOut, error) { + err := ctx.PushBusy() + defer ctx.PopBusy() + if err != nil { + return py.CompileOut{}, err + } + tryPaths := defaultPaths if opts.UseSysPaths { tryPaths = ctx.Store().MustGetModule("sys").Globals["path"].(*py.List).Items @@ -95,7 +114,7 @@ func (ctx *context) ResolveAndCompile(pathname string, opts py.CompileOpts) (py. out := py.CompileOut{} - err := resolveRunPath(pathname, opts, tryPaths, func(fpath string) (bool, error) { + err = resolveRunPath(pathname, opts, tryPaths, func(fpath string) (bool, error) { stat, err := os.Stat(fpath) if err == nil && stat.IsDir() { @@ -162,6 +181,36 @@ func (ctx *context) ResolveAndCompile(pathname string, opts py.CompileOpts) (py. return out, nil } +func (ctx *context) PushBusy() error { + if ctx.closed { + return py.ExceptionNewf(py.RuntimeError, "Context closed") + } + ctx.running.Add(1) + return nil +} + +func (ctx *context) PopBusy() { + ctx.running.Done() +} + +// Close -- see type py.Context +func (ctx *context) Close() { + ctx.closeOnce.Do(func() { + ctx.closing = true + ctx.running.Wait() + ctx.closed = true + + // Give each module a chance to release resources + ctx.store.OnContextClosed() + close(ctx.done) + }) +} + +// Done -- see type py.Context +func (ctx *context) Done() <-chan struct{} { + return ctx.done +} + var defaultPaths = []py.Object{ py.String("."), } @@ -216,6 +265,12 @@ func resolveRunPath(runPath string, opts py.CompileOpts, pathObjs []py.Object, t } func (ctx *context) RunCode(code *py.Code, globals, locals py.StringDict, closure py.Tuple) (py.Object, error) { + err := ctx.PushBusy() + defer ctx.PopBusy() + if err != nil { + return nil, err + } + return vm.EvalCode(ctx, code, globals, locals, nil, nil, nil, nil, closure) } diff --git a/py/module.go b/py/module.go index 248d1168..e37ec4d2 100644 --- a/py/module.go +++ b/py/module.go @@ -14,9 +14,10 @@ import ( type ModuleFlags int32 const ( - // Set for modules that are threadsafe, stateless, and/or can be shared across multiple py.Context instances (for efficiency). - // Otherwise, a separate module instance is created for each py.Context that imports it. - ShareModule ModuleFlags = 0x01 // @@TODO + // ShareModule signals that an embedded module is threadsafe and read-only, meaninging it could be shared across multiple py.Context instances (for efficiency). + // Otherwise, ModuleImpl will create a separate py.Module instance for each py.Context that imports it. + // This should be used with extreme caution since any module mutation (write) means possible cross-context data corruption. + ShareModule ModuleFlags = 0x01 MainModuleName = "__main__" ) @@ -34,12 +35,13 @@ type ModuleInfo struct { // By convention, .Code is executed when a module instance is initialized. // If .Code == nil, then .CodeBuf or .CodeSrc will be auto-compiled to set .Code. type ModuleImpl struct { - Info ModuleInfo - Methods []*Method - Globals StringDict - CodeSrc string // Module code body (py source code to be compiled) - CodeBuf []byte // Module code body (serialized py.Code object) - Code *Code // Module code body + Info ModuleInfo + Methods []*Method // Module-bound global method functions + Globals StringDict // Module-bound global variables + CodeSrc string // Module code body (source code to be compiled) + CodeBuf []byte // Module code body (serialized py.Code object) + Code *Code // Module code body + OnContextClosed func(*Module) // Callback for when a py.Context is closing to release resources } // ModuleStore is a container of Module imported into an owning py.Context. @@ -87,10 +89,9 @@ func NewModuleStore() *ModuleStore { // Module is a runtime instance of a ModuleImpl bound to the py.Context that imported it. type Module struct { - ModuleInfo - - Globals StringDict - Context Context + ModuleImpl *ModuleImpl // Parent implementation of this Module instance + Globals StringDict // Initialized from ModuleImpl.Globals + Context Context // Parent context that "owns" this Module instance } var ModuleType = NewType("module", "module object") @@ -101,7 +102,11 @@ func (o *Module) Type() *Type { } func (m *Module) M__repr__() (Object, error) { - return String(fmt.Sprintf("", m.Name)), nil + name, ok := m.Globals["__name__"].(String) + if !ok { + name = "???" + } + return String(fmt.Sprintf("", string(name))), nil } // Get the Dict @@ -109,43 +114,56 @@ func (m *Module) GetDict() StringDict { return m.Globals } +// Calls a named method of a module +func (m *Module) Call(name string, args Tuple, kwargs StringDict) (Object, error) { + attr, err := GetAttrString(m, name) + if err != nil { + return nil, err + } + return Call(attr, args, kwargs) +} + +// Interfaces +var _ IGetDict = (*Module)(nil) + // NewModule adds a new Module instance to this ModuleStore. // Each given Method prototype is used to create a new "live" Method bound this the newly created Module. // This func also sets appropriate module global attribs based on the given ModuleInfo (e.g. __name__). -func (store *ModuleStore) NewModule(ctx Context, info ModuleInfo, methods []*Method, globals StringDict) (*Module, error) { - if info.Name == "" { - info.Name = MainModuleName +func (store *ModuleStore) NewModule(ctx Context, impl *ModuleImpl) (*Module, error) { + name := impl.Info.Name + if name == "" { + name = MainModuleName } m := &Module{ - ModuleInfo: info, - Globals: globals.Copy(), + ModuleImpl: impl, + Globals: impl.Globals.Copy(), Context: ctx, } // Insert the methods into the module dictionary // Copy each method an insert each "live" with a ptr back to the module (which can also lead us to the host Context) - for _, method := range methods { + for _, method := range impl.Methods { methodInst := new(Method) *methodInst = *method methodInst.Module = m m.Globals[method.Name] = methodInst } // Set some module globals - m.Globals["__name__"] = String(info.Name) - m.Globals["__doc__"] = String(info.Doc) + m.Globals["__name__"] = String(name) + m.Globals["__doc__"] = String(impl.Info.Doc) m.Globals["__package__"] = None - if len(info.FileDesc) > 0 { - m.Globals["__file__"] = String(info.FileDesc) + if len(impl.Info.FileDesc) > 0 { + m.Globals["__file__"] = String(impl.Info.FileDesc) } // Register the module - store.modules[info.Name] = m + store.modules[name] = m // Make a note of some modules - switch info.Name { + switch name { case "builtins": store.Builtins = m case "importlib": store.Importlib = m } - // fmt.Printf("Registering module %q\n", name) + // fmt.Printf("Registered module %q\n", moduleName) return m, nil } @@ -167,14 +185,11 @@ func (store *ModuleStore) MustGetModule(name string) *Module { return m } -// Calls a named method of a module -func (m *Module) Call(name string, args Tuple, kwargs StringDict) (Object, error) { - attr, err := GetAttrString(m, name) - if err != nil { - return nil, err +// OnContextClosed signals all module instances that the parent py.Context has closed +func (store *ModuleStore) OnContextClosed() { + for _, m := range store.modules { + if m.ModuleImpl.OnContextClosed != nil { + m.ModuleImpl.OnContextClosed(m) + } } - return Call(attr, args, kwargs) } - -// Interfaces -var _ IGetDict = (*Module)(nil) diff --git a/py/run.go b/py/run.go index 1d4fe73a..1a7510c1 100644 --- a/py/run.go +++ b/py/run.go @@ -33,15 +33,18 @@ type Context interface { // RunCode is a lower-level invocation to execute the given py.Code. RunCode(code *Code, globals, locals StringDict, closure Tuple) (result Object, err error) - // Execution of any of the above will stop when the next opcode runs - // @@TODO - // SignalHalt() - // Returns the named module for this context (or an error if not found) GetModule(moduleName string) (*Module, error) // Gereric access to this context's modules / state. Store() *ModuleStore + + // Close signals that is context is about to go out of scope and any internal resources should be released. + // Operations on a py.Context that have closed will generally result in an error. + Close() + + // Done returns a signal that can be used to detect when this Context has fully closed / completed. + Done() <-chan struct{} } // CompileOpts specifies options for high-level compilation. @@ -100,20 +103,45 @@ var ( Compile func(src, srcDesc string, mode CompileMode, flags int, dont_inherit bool) (*Code, error) ) -// RunFile resolves the given pathname, compiles as needed, and runs that code in the given module, returning the Module to indicate success. -// If inModule is a *Module, then the code is run in that module. -// If inModule is nil, the code is run in a new __main__ module (and the new Module is returned). -// If inModule is a string, the code is run in a new module with the given name (and the new Module is returned). +// RunFile resolves the given pathname, compiles as needed, executes the code in the given module, and returns the Module to indicate success. +// +// See RunCode() for description of inModule. func RunFile(ctx Context, pathname string, opts CompileOpts, inModule interface{}) (*Module, error) { out, err := ctx.ResolveAndCompile(pathname, opts) if err != nil { return nil, err } - var moduleName string - createNew := false - var module *Module + return RunCode(ctx, out.Code, out.FileDesc, inModule) +} + +// RunSrc compiles the given python buffer and executes it within the given module and returns the Module to indicate success. +// +// See RunCode() for description of inModule. +func RunSrc(ctx Context, pySrc string, pySrcDesc string, inModule interface{}) (*Module, error) { + if pySrcDesc == "" { + pySrcDesc = "" + } + code, err := Compile(pySrc+"\n", pySrcDesc, SingleMode, 0, true) + if err != nil { + return nil, err + } + + return RunCode(ctx, code, pySrcDesc, inModule) +} +// RunCode executes the given code object within the given module and returns the Module to indicate success. +// If inModule is a *Module, then the code is run in that module. +// If inModule is nil, the code is run in a new __main__ module (and the new Module is returned). +// If inModule is a string, the code is run in a new module with the given name (and the new Module is returned). +func RunCode(ctx Context, code *Code, codeDesc string, inModule interface{}) (*Module, error) { + var ( + module *Module + moduleName string + err error + ) + + createNew := false switch mod := inModule.(type) { case string: @@ -122,7 +150,7 @@ func RunFile(ctx Context, pathname string, opts CompileOpts, inModule interface{ case nil: createNew = true case *Module: - _, err = ctx.RunCode(out.Code, mod.Globals, mod.Globals, nil) + _, err = ctx.RunCode(code, mod.Globals, mod.Globals, nil) module = mod default: err = ExceptionNewf(TypeError, "unsupported module type: %v", inModule) @@ -132,9 +160,9 @@ func RunFile(ctx Context, pathname string, opts CompileOpts, inModule interface{ moduleImpl := ModuleImpl{ Info: ModuleInfo{ Name: moduleName, - FileDesc: out.FileDesc, + FileDesc: codeDesc, }, - Code: out.Code, + Code: code, } module, err = ctx.ModuleInit(&moduleImpl) } diff --git a/py/util.go b/py/util.go index 148c1d25..cee748a7 100644 --- a/py/util.go +++ b/py/util.go @@ -9,6 +9,10 @@ import ( "strconv" ) +var ( + ErrUnsupportedObjType = errors.New("unsupported obj type") +) + func GetLen(obj Object) (Int, error) { getlen, ok := obj.(I__len__) if !ok { @@ -53,10 +57,6 @@ func LoadTuple(args Tuple, vars []interface{}) error { return nil } -var ( - ErrUnsupportedObjType = errors.New("unsupported obj type") -) - func LoadAttr(obj Object, attrName string, data interface{}) error { attr, err := GetAttrString(obj, attrName) if err != nil { diff --git a/pytest/pytest.go b/pytest/pytest.go index 6877d966..c7af7737 100644 --- a/pytest/pytest.go +++ b/pytest/pytest.go @@ -44,9 +44,11 @@ func CompileSrc(t testing.TB, ctx py.Context, pySrc string, prog string) (*py.Mo t.Fatalf("%s: Compile failed: %v", prog, err) } - module, err := ctx.Store().NewModule(ctx, py.ModuleInfo{ - FileDesc: prog, - }, nil, nil) + module, err := ctx.Store().NewModule(ctx, &py.ModuleImpl{ + Info: py.ModuleInfo{ + FileDesc: prog, + }, + }) if err != nil { t.Fatalf("%s: NewModule failed: %v", prog, err) } From 7508556c45543ba96616b23dbb953d09234e0885 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Wed, 19 Jan 2022 12:30:15 -0600 Subject: [PATCH 64/79] docs and LoadIntsFromList --- py/util.go | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/py/util.go b/py/util.go index cee748a7..3555b001 100644 --- a/py/util.go +++ b/py/util.go @@ -13,6 +13,7 @@ var ( ErrUnsupportedObjType = errors.New("unsupported obj type") ) +// GetInt is a high-level convenience function that gets the length of the given Object. func GetLen(obj Object) (Int, error) { getlen, ok := obj.(I__len__) if !ok { @@ -27,6 +28,7 @@ func GetLen(obj Object) (Int, error) { return GetInt(lenObj) } +// GetInt is a high-level convenience function that converts the given value to an int. func GetInt(obj Object) (Int, error) { toIdx, ok := obj.(I__index__) if !ok { @@ -37,6 +39,7 @@ func GetInt(obj Object) (Int, error) { return toIdx.M__index__() } +// LoadTuple attempts to convert each element of the given list and store into each destination value (based on its type). func LoadTuple(args Tuple, vars []interface{}) error { if len(args) > len(vars) { @@ -57,18 +60,54 @@ func LoadTuple(args Tuple, vars []interface{}) error { return nil } -func LoadAttr(obj Object, attrName string, data interface{}) error { +// LoadAttr gets the named attrib and attempts to store it into the given destination value (based on its type). +func LoadAttr(obj Object, attrName string, dst interface{}) error { attr, err := GetAttrString(obj, attrName) if err != nil { return err } - err = loadValue(attr, data) + err = loadValue(attr, dst) if err == ErrUnsupportedObjType { return ExceptionNewf(TypeError, "attribute \"%s\" has unsupported object type: %s", attrName, attr.Type().Name) } return nil } +// LoadIntsFromList extracts a list of ints contained given a py.List or py.Tuple +func LoadIntsFromList(list Object) ([]int64, error) { + N, err := GetLen(list) + if err != nil { + return nil, err + } + + getter, ok := list.(I__getitem__) + if !ok { + return nil, nil + } + + var intList []int64 + if ok && N > 0 { + intList = make([]int64, N) + + var intVal Int + for i := Int(0); i < N; i++ { + item, err := getter.M__getitem__(i) + if err != nil { + return nil, err + } + + intVal, err = GetInt(item) + if err != nil { + return nil, err + } + + intList[i] = int64(intVal) + } + } + + return intList, nil +} + func loadValue(src Object, data interface{}) error { var ( v_str string @@ -104,6 +143,8 @@ func loadValue(src Object, data interface{}) error { } switch dst := data.(type) { + case *Int: + *dst = Int(v_int) case *bool: *dst = v_int != 0 case *int8: @@ -136,9 +177,15 @@ func loadValue(src Object, data interface{}) error { v_float, _ = strconv.ParseFloat(v_str, 64) } *dst = v_float + case *Float: + if haveStr { + v_float, _ = strconv.ParseFloat(v_str, 64) + } + *dst = Float(v_float) case *string: *dst = v_str - + case *String: + *dst = String(v_str) // case []uint64: // for i := range data { // dst[i] = order.Uint64(bs[8*i:]) From 99bcb409dfcace47bdae7983d9b19687c58c0b6f Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Fri, 21 Jan 2022 12:38:26 -0600 Subject: [PATCH 65/79] rename edits --- examples/{multi-ctx => multi-context}/main.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/{multi-ctx => multi-context}/main.go (100%) diff --git a/examples/multi-ctx/main.go b/examples/multi-context/main.go similarity index 100% rename from examples/multi-ctx/main.go rename to examples/multi-context/main.go From 0cc76503278c64160b14b7c735d43ac40a39bad1 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Fri, 21 Jan 2022 12:43:07 -0600 Subject: [PATCH 66/79] push/popBusy now private --- modules/runtime.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/modules/runtime.go b/modules/runtime.go index 9b8d2ada..d26632a6 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -60,8 +60,8 @@ func NewContext(opts py.ContextOpts) py.Context { } func (ctx *context) ModuleInit(impl *py.ModuleImpl) (*py.Module, error) { - err := ctx.PushBusy() - defer ctx.PopBusy() + err := ctx.pushBusy() + defer ctx.popBusy() if err != nil { return nil, err } @@ -101,8 +101,8 @@ func (ctx *context) ModuleInit(impl *py.ModuleImpl) (*py.Module, error) { } func (ctx *context) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.CompileOut, error) { - err := ctx.PushBusy() - defer ctx.PopBusy() + err := ctx.pushBusy() + defer ctx.popBusy() if err != nil { return py.CompileOut{}, err } @@ -181,7 +181,7 @@ func (ctx *context) ResolveAndCompile(pathname string, opts py.CompileOpts) (py. return out, nil } -func (ctx *context) PushBusy() error { +func (ctx *context) pushBusy() error { if ctx.closed { return py.ExceptionNewf(py.RuntimeError, "Context closed") } @@ -189,12 +189,12 @@ func (ctx *context) PushBusy() error { return nil } -func (ctx *context) PopBusy() { +func (ctx *context) popBusy() { ctx.running.Done() } // Close -- see type py.Context -func (ctx *context) Close() { +func (ctx *context) Close() error { ctx.closeOnce.Do(func() { ctx.closing = true ctx.running.Wait() @@ -204,6 +204,7 @@ func (ctx *context) Close() { ctx.store.OnContextClosed() close(ctx.done) }) + return nil } // Done -- see type py.Context @@ -265,8 +266,8 @@ func resolveRunPath(runPath string, opts py.CompileOpts, pathObjs []py.Object, t } func (ctx *context) RunCode(code *py.Code, globals, locals py.StringDict, closure py.Tuple) (py.Object, error) { - err := ctx.PushBusy() - defer ctx.PopBusy() + err := ctx.pushBusy() + defer ctx.popBusy() if err != nil { return nil, err } From 9afb2d3cd6133c9c028a01b191d334d4e39b8cd8 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Fri, 21 Jan 2022 12:56:33 -0600 Subject: [PATCH 67/79] doc edits --- py/run.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/py/run.go b/py/run.go index 1a7510c1..e99880bf 100644 --- a/py/run.go +++ b/py/run.go @@ -12,16 +12,18 @@ const ( SingleMode CompileMode = "single" // Compile a single (interactive) statement ) -// Context is gpython virtual environment instance ("context"), providing a mechanism +// Context is a gpython environment instance container, providing a high-level mechanism // for multiple gpython interpreters to run concurrently without restriction. // -// In general, one creates a py.Context (via py.NewContext) for each concurrent goroutine to be running an interpreter. -// In other words, ensure that a py.Context is never concurrently accessed across goroutines. +// Context instances maintain completely independent environments, namely the modules that +// have been imported and their state. Modules imported into a Context are instanced +// from a parent ModuleImpl. For example, since Contexts each have their +// own sys module instance, each can set sys.path differently and independently. // -// RunFile() and RunCode() block until code execution is complete. -// In the future, they will abort early if the parent associated py.Context is signaled. +// If you access a Context from multiple groutines, you are responsible that access is not concurrent, +// with the exception of Close() and Done(). // -// See examples/multi-ctx +// See examples/multi-context and examples/embedding. type Context interface { // Resolves then compiles (if applicable) the given file system pathname into a py.Code ready to be executed. @@ -31,6 +33,7 @@ type Context interface { ModuleInit(impl *ModuleImpl) (*Module, error) // RunCode is a lower-level invocation to execute the given py.Code. + // Blocks until execution is complete. RunCode(code *Code, globals, locals StringDict, closure Tuple) (result Object, err error) // Returns the named module for this context (or an error if not found) @@ -39,11 +42,12 @@ type Context interface { // Gereric access to this context's modules / state. Store() *ModuleStore - // Close signals that is context is about to go out of scope and any internal resources should be released. - // Operations on a py.Context that have closed will generally result in an error. - Close() + // Close signals this context is about to go out of scope and any internal resources should be released. + // Code execution on a py.Context that has been closed will result in an error. + Close() error // Done returns a signal that can be used to detect when this Context has fully closed / completed. + // If Close() is called while execution in progress, Done() will not signal until execution is complete. Done() <-chan struct{} } From 7d2a8d9a28639565862ca5ff915f3697d294e6cb Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 24 Jan 2022 23:31:06 -0600 Subject: [PATCH 68/79] fixed kwarg issues in ParseTupleAndKeywords() and builtin_print test --- builtin/tests/builtin.py | 27 +++++----- py/args.go | 104 +++++++++++++++++++++++---------------- 2 files changed, 77 insertions(+), 54 deletions(-) diff --git a/builtin/tests/builtin.py b/builtin/tests/builtin.py index e88db3a0..e238c89c 100644 --- a/builtin/tests/builtin.py +++ b/builtin/tests/builtin.py @@ -299,40 +299,43 @@ def gen2(): doc="print" ok = False try: - print("hello", sep=1) + print("hello", sep=1, end="!") except TypeError as e: - #if e.args[0] != "sep must be None or a string, not int": - # raise + if e.args[0] != "print() argument 1 must be str, not int": + raise ok = True assert ok, "TypeError not raised" try: - print("hello", sep=" ", end=1) + print("hello", sep=",", end=1) except TypeError as e: - #if e.args[0] != "end must be None or a string, not int": - # raise + if e.args[0] != "print() argument 2 must be str, not int": + raise ok = True assert ok, "TypeError not raised" try: - print("hello", sep=" ", end="\n", file=1) + print("hello", sep=",", end="!", file=1) except AttributeError as e: - #if e.args[0] != "'int' object has no attribute 'write'": - # raise + if e.args[0] != "'int' has no attribute 'write'": + raise ok = True assert ok, "AttributeError not raised" with open("testfile", "w") as f: - print("hello", "world", sep=" ", end="\n", file=f) + print("hello", "world", sep=", ", end="!\n", file=f) + print("hells", "bells", end="...", file=f, sep="_") + print(" ~", "Brother ", "Foo", "bar", file=f, end="", sep="") with open("testfile", "r") as f: - assert f.read() == "hello world\n" + assert f.read() == "hello, world!\nhells_bells... ~Brother Foobar" with open("testfile", "w") as f: print(1,2,3,sep=",",end=",\n", file=f) + print("4","5", file=f, end="!", sep=",") with open("testfile", "r") as f: - assert f.read() == "1,2,3,\n" + assert f.read() == "1,2,3,\n4,5!" doc="round" assert round(1.1) == 1.0 diff --git a/py/args.go b/py/args.go index f38fdfde..5209a3d3 100644 --- a/py/args.go +++ b/py/args.go @@ -368,9 +368,7 @@ // $ // // PyArg_ParseTupleAndKeywords() only: Indicates that the remaining -// arguments in the Python argument list are keyword-only. Currently, -// all keyword-only arguments must also be optional arguments, so | -// must always be specified before $ in the format string. +// arguments in the Python argument list are keyword-only. // // New in version 3.3. // @@ -416,17 +414,13 @@ func ParseTupleAndKeywords(args Tuple, kwargs StringDict, format string, kwlist if kwlist != nil && len(results) != len(kwlist) { return ExceptionNewf(TypeError, "Internal error: supply the same number of results and kwlist") } - min, max, name, ops := parseFormat(format) - keywordOnly := false - err := checkNumberOfArgs(name, len(args)+len(kwargs), len(results), min, max) + var opsBuf [16]formatOp + min, name, kwOnly_i, ops := parseFormat(format, opsBuf[:0]) + err := checkNumberOfArgs(name, len(args)+len(kwargs), len(results), min, len(ops)) if err != nil { return err } - if len(ops) > 0 && ops[0] == "$" { - keywordOnly = true - ops = ops[1:] - } // Check all the kwargs are in kwlist // O(N^2) Slow but kwlist is usually short for kwargName := range kwargs { @@ -439,46 +433,60 @@ func ParseTupleAndKeywords(args Tuple, kwargs StringDict, format string, kwlist found: } - // Create args tuple with all the arguments we have in - args = args.Copy() - for i, kw := range kwlist { - if value, ok := kwargs[kw]; ok { - if len(args) > i { + // Walk through all the results we want + for i, op := range ops { + + var ( + arg Object + kw string + ) + if i < len(kwlist) { + kw = kwlist[i] + arg = kwargs[kw] + } + + // Consume ordered args first -- they should not require keyword only or also be specified via keyword + if i < len(args) { + if i >= kwOnly_i { + return ExceptionNewf(TypeError, "%s() specifies argument '%s' that is keyword only", name, kw) + } + if arg != nil { return ExceptionNewf(TypeError, "%s() got multiple values for argument '%s'", name, kw) } - args = append(args, value) - } else if keywordOnly { - args = append(args, nil) + arg = args[i] } - } - for i, arg := range args { - op := ops[i] + + // Unspecified args retain their default value + if arg == nil { + continue + } + result := results[i] - switch op { - case "O": + switch op.code { + case 'O': *result = arg - case "Z", "z": + case 'Z', 'z': if _, ok := arg.(NoneType); ok { *result = arg break } fallthrough - case "U", "s": + case 'U', 's': if _, ok := arg.(String); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be str, not %s", name, i+1, arg.Type().Name) } *result = arg - case "i": + case 'i': if _, ok := arg.(Int); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be int, not %s", name, i+1, arg.Type().Name) } *result = arg - case "p": + case 'p': if _, ok := arg.(Bool); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be bool, not %s", name, i+1, arg.Type().Name) } *result = arg - case "d": + case 'd': switch x := arg.(type) { case Int: *result = Float(x) @@ -500,30 +508,42 @@ func ParseTuple(args Tuple, format string, results ...*Object) error { return ParseTupleAndKeywords(args, nil, format, nil, results...) } +type formatOp struct { + code byte + modifier byte +} + // Parse the format -func parseFormat(format string) (min, max int, name string, ops []string) { +func parseFormat(format string, in []formatOp) (min int, name string, kwOnly_i int, ops []formatOp) { name = "function" min = -1 - for format != "" { - op := string(format[0]) - format = format[1:] - if len(format) > 1 && (format[1] == '*' || format[1] == '#') { - op += string(format[0]) - format = format[1:] + kwOnly_i = 0xFFFF + ops = in[:0] + + N := len(format) + for i := 0; i < N; { + op := formatOp{code: format[i]} + i++ + if i < N { + if mod := format[i]; mod == '*' || mod == '#' { + op.modifier = mod + i++ + } } - switch op { - case ":", ";": - name = format - format = "" - case "|": + switch op.code { + case ':', ';': + name = format[i:] + i = N + case '$': + kwOnly_i = len(ops) + case '|': min = len(ops) default: ops = append(ops, op) } } - max = len(ops) if min < 0 { - min = max + min = len(ops) } return } From ba7db80d09226467ec635d00dcc182692302bb99 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Mon, 24 Jan 2022 23:42:12 -0600 Subject: [PATCH 69/79] added Flush kwarg to print test --- builtin/tests/builtin.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/builtin/tests/builtin.py b/builtin/tests/builtin.py index e238c89c..ae08e336 100644 --- a/builtin/tests/builtin.py +++ b/builtin/tests/builtin.py @@ -323,16 +323,16 @@ def gen2(): assert ok, "AttributeError not raised" with open("testfile", "w") as f: - print("hello", "world", sep=", ", end="!\n", file=f) - print("hells", "bells", end="...", file=f, sep="_") + print("hello", "world", end="!\n", file=f, sep=", ") + print("hells", "bells", end="...", file=f) print(" ~", "Brother ", "Foo", "bar", file=f, end="", sep="") with open("testfile", "r") as f: - assert f.read() == "hello, world!\nhells_bells... ~Brother Foobar" + assert f.read() == "hello, world!\nhells bells... ~Brother Foobar" with open("testfile", "w") as f: - print(1,2,3,sep=",",end=",\n", file=f) - print("4","5", file=f, end="!", sep=",") + print(1,2,3,sep=",", flush=False, end=",\n", file=f) + print("4",5, file=f, end="!", flush=True, sep=",") with open("testfile", "r") as f: assert f.read() == "1,2,3,\n4,5!" From 61e42c8b6eddecce05958f913a231df27e2650dc Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Wed, 26 Jan 2022 10:37:28 -0600 Subject: [PATCH 70/79] added embedding example --- examples/embedding/README.md | 102 +++++++++++++++ examples/embedding/lib/REPL-startup.py | 8 ++ examples/embedding/lib/mylib.py | 53 ++++++++ examples/embedding/main.go | 51 ++++++++ examples/embedding/mylib-demo.py | 15 +++ examples/embedding/mylib.module.go | 174 +++++++++++++++++++++++++ 6 files changed, 403 insertions(+) create mode 100644 examples/embedding/README.md create mode 100644 examples/embedding/lib/REPL-startup.py create mode 100644 examples/embedding/lib/mylib.py create mode 100644 examples/embedding/main.go create mode 100644 examples/embedding/mylib-demo.py create mode 100644 examples/embedding/mylib.module.go diff --git a/examples/embedding/README.md b/examples/embedding/README.md new file mode 100644 index 00000000..15cf6f3d --- /dev/null +++ b/examples/embedding/README.md @@ -0,0 +1,102 @@ +## Embedding gpython + +This example demonstrates how embed gpython into a Go application. + + +### Why embed gpython? + +Embedding a highly capable and familiar "interpreted" language allows your users +to easily augment app behavior, configuration, and customization -- all post-deployment. + +Have you ever found an exciting software project but lose interest when you discover that you +also need to learn an esoteric language schema? In an era of limited attention span, +most people are generally turned off if they have to learn a new language in addition to learning +to use your app. + +If you consider [why use Python](https://www.stxnext.com/what-is-python-used-for/), then perhaps also +consider your users could immediately feel interested to hear that your software offers +an additional familiar dimension of value. + +Python is widespread in finance, sciences of all kinds, hobbyist programming and is often +endearingly regarded as most popular programming language for non-developers. +If your application can be driven by embedded Python, then chances are others will +feel excited and empowered that your project can be used out of the box +with positive feelings of being in familiar territory. + +### But what about the lack of python modules? + +There are only be a small number of native modules available, but don't forget you have the entire +Go standard library and *any* Go package you can name at your fingertips to expose! +This plus multi-context capability gives gpython enormous potential on how it can +serve your project. + +So basically, gpython is only off the table if you need to run python that makes heavy use of +modules that are only available in CPython. + +### Packing List + +| | | +|---------------------- | ------------------------------------------------------------------| +| `main.go` | if no args, runs in REPL mode, otherwise runs the given file | +| `lib/mylib.py` | models a library that your application would expose | +| `lib/REPL-startup.py` | invoked by `main.go` when starting REPL mode | +| `mylib-demo.py` | models a user-authored script that consumes `mylib` | +| `mylib.module.go` | Go implementation of `mylib_go` consumed by `mylib` | + + +### Invoking a Python Script + +```bash +$ cd examples/embedding/ +$ go build . +$ ./embedding mylib-demo.py +``` +``` +Welcome to a gpython embedded example, + where your wildest Go-based python dreams come true! + +========================================================== + Python 3.4 (github.com/go-python/gpython) + go1.17.6 on darwin amd64 +========================================================== + +Spring Break itinerary: + Stop 1: Miami, Florida | 7 nights + Stop 2: Mallorca, Spain | 3 nights + Stop 3: Ibiza, Spain | 14 nights + Stop 4: Monaco | 12 nights +### Made with Vacaton 1.0 by Fletch F. Fletcher + +I bet Monaco will be the best! +``` + +### REPL Mode + +```bash +$ ./embedding +``` +``` +======= Entering REPL mode, press Ctrl+D to exit ======= + +========================================================== + Python 3.4 (github.com/go-python/gpython) + go1.17.6 on darwin amd64 +========================================================== + +>>> v = Vacation("Spring Break", Stop("Florida", 3), Stop("Nice", 7)) +>>> print(str(v)) +Spring Break, 2 stop(s) +>>> v.PrintItinerary() +Spring Break itinerary: + Stop 1: Florida | 3 nights + Stop 2: Nice | 7 nights +### Made with Vacaton 1.0 by Fletch F. Fletcher +``` + +## Takeways + + - `main.go` demonstrates high-level convenience functions such as `py.RunFile()`. + - Embedding any Go `struct` only requires that it implements `py.Object`, which is a single function: + `Type() *py.Type` + - See [py/run.go](https://github.com/go-python/gpython/tree/master/py/run.go) for more about interpreter instances and `py.Context` + - There are many helper functions available for you in [py/util.go](https://github.com/go-python/gpython/tree/master/py/util.go) and your contributions are welcome! diff --git a/examples/embedding/lib/REPL-startup.py b/examples/embedding/lib/REPL-startup.py new file mode 100644 index 00000000..6b27c415 --- /dev/null +++ b/examples/embedding/lib/REPL-startup.py @@ -0,0 +1,8 @@ + + +# This file is called from main.go when in REPL mode + +# This is here to demonstrate making life easier for your users in REPL mode +# by doing pre-setup here so they don't have to import every time they start. +from mylib import * + diff --git a/examples/embedding/lib/mylib.py b/examples/embedding/lib/mylib.py new file mode 100644 index 00000000..4c279c78 --- /dev/null +++ b/examples/embedding/lib/mylib.py @@ -0,0 +1,53 @@ +import mylib_go as _go + +PY_VERSION = _go.PY_VERSION +GO_VERSION = _go.GO_VERSION + + +print(''' +========================================================== + %s + %s +========================================================== +''' % (PY_VERSION, GO_VERSION)) + + +def Stop(location, num_nights = 2): + return _go.VacationStop_new(location, num_nights) + + +class Vacation: + + def __init__(self, tripName, *stops): + self._v, self._libVers = _go.Vacation_new() + self.tripName = tripName + self.AddStops(*stops) + + def __str__(self): + return "%s, %d stop(s)" % (self.tripName, self.NumStops()) + + def NumStops(self): + return self._v.num_stops() + + def GetStop(self, stop_num): + return self._v.get_stop(stop_num) + + def AddStops(self, *stops): + self._v.add_stops(stops) + + def PrintItinerary(self): + print(self.tripName, "itinerary:") + i = 1 + while 1: + + try: + stop = self.GetStop(i) + except IndexError: + break + + print(" Stop %d: %s" % (i, str(stop))) + i += 1 + + print("### Made with %s " % self._libVers) + + \ No newline at end of file diff --git a/examples/embedding/main.go b/examples/embedding/main.go new file mode 100644 index 00000000..faa5120e --- /dev/null +++ b/examples/embedding/main.go @@ -0,0 +1,51 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "fmt" + + // This initializes gpython for runtime execution and is essential. + // It defines forward-declared symbols and registers native built-in modules, such as sys and time. + _ "github.com/go-python/gpython/modules" + + // Commonly consumed gpython + "github.com/go-python/gpython/py" + "github.com/go-python/gpython/repl" + "github.com/go-python/gpython/repl/cli" +) + +func main() { + flag.Parse() + runWithFile(flag.Arg(0)) +} + +func runWithFile(pyFile string) error { + + // See type Context interface and related docs + ctx := py.NewContext(py.DefaultContextOpts()) + + var err error + if len(pyFile) == 0 { + replCtx := repl.New(ctx) + + fmt.Print("\n======= Entering REPL mode, press Ctrl+D to exit =======\n") + + _, err = py.RunFile(ctx, "lib/REPL-startup.py", py.CompileOpts{}, replCtx.Module) + if err == nil { + cli.RunREPL(replCtx) + } + + } else { + _, err = py.RunFile(ctx, pyFile, py.CompileOpts{}, nil) + } + + if err != nil { + py.TracebackDump(err) + } + + return err +} diff --git a/examples/embedding/mylib-demo.py b/examples/embedding/mylib-demo.py new file mode 100644 index 00000000..4a461023 --- /dev/null +++ b/examples/embedding/mylib-demo.py @@ -0,0 +1,15 @@ +print(''' +Welcome to a gpython embedded example, + where your wildest Go-based python dreams come true!''') + +# This is a model for a public/user-side script that you or users would maintain, +# offering an open canvas to drive app behavior, customization, or anything you can dream up. +# +# Modules you offer for consumption can also serve to document such things. +from mylib import * + +springBreak = Vacation("Spring Break", Stop("Miami, Florida", 7), Stop("Mallorca, Spain", 3)) +springBreak.AddStops(Stop("Ibiza, Spain", 14), Stop("Monaco", 12)) +springBreak.PrintItinerary() + +print("\nI bet %s will be the best!\n" % springBreak.GetStop(4).Get()[0]) diff --git a/examples/embedding/mylib.module.go b/examples/embedding/mylib.module.go new file mode 100644 index 00000000..9589852b --- /dev/null +++ b/examples/embedding/mylib.module.go @@ -0,0 +1,174 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "runtime" + + "github.com/go-python/gpython/py" +) + +// Embedded gpython type delcarations are the bridge between gpython and embedded Go types. +var ( + PyVacationStopType = py.NewType("Stop", "") + PyVacationType = py.NewType("Vacation", "") +) + +// init is where you register your embedded module and attach methods to your embedded class types. +func init() { + + // Attach embedded methods to your embedded python types... + PyVacationStopType.Dict["Set"] = py.MustNewMethod("Set", VacationStop_Set, 0, "") + PyVacationStopType.Dict["Get"] = py.MustNewMethod("Get", VacationStop_Get, 0, "") + PyVacationType.Dict["add_stops"] = py.MustNewMethod("Vacation.add_stops", Vacation_add_stops, 0, "") + PyVacationType.Dict["num_stops"] = py.MustNewMethod("Vacation.num_stops", Vacation_num_stops, 0, "") + PyVacationType.Dict["get_stop"] = py.MustNewMethod("Vacation.get_stop", Vacation_get_stop, 0, "") + + // Bind methods attached at the module (global) level. + // When these are invoked, the first py.Object param (typically "self") is the bound *Module instance. + methods := []*py.Method{ + py.MustNewMethod("VacationStop_new", VacationStop_new, 0, ""), + py.MustNewMethod("Vacation_new", Vacation_new, 0, ""), + } + + // Register a ModuleImpl instance used by the gpython runtime to instantiate new py.Module when first imported. + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "mylib_go", + Doc: "Example embedded python module", + }, + Methods: methods, + Globals: py.StringDict{ + "PY_VERSION": py.String("Python 3.4 (github.com/go-python/gpython)"), + "GO_VERSION": py.String(fmt.Sprintf("%s on %s %s", runtime.Version(), runtime.GOOS, runtime.GOARCH)), + "MYLIB_VERS": py.String("Vacaton 1.0 by Fletch F. Fletcher"), + }, + }) +} + +// VacationStop is an example Go struct to embed. +type VacationStop struct { + Desc py.String + NumNights py.Int +} + +// Type comprises the py.Object interface, allowing a Go struct to be cast as a py.Object. +// Instance methods of an type are then attached to this type object +func (stop *VacationStop) Type() *py.Type { + return PyVacationStopType +} + +func (stop *VacationStop) M__str__() (py.Object, error) { + line := fmt.Sprintf(" %-16v | %2v nights", stop.Desc, stop.NumNights) + return py.String(line), nil +} + +func (stop *VacationStop) M__repr__() (py.Object, error) { + return stop.M__str__() +} + +func VacationStop_new(module py.Object, args py.Tuple) (py.Object, error) { + stop := &VacationStop{} + VacationStop_Set(stop, args) + return stop, nil +} + +// VacationStop_Set is an embedded instance moethod of VacationStop +func VacationStop_Set(self py.Object, args py.Tuple) (py.Object, error) { + stop := self.(*VacationStop) + + // Check out other convenience functions in py/util.go + // Also available is py.ParseTuple(args, "si", ...) + err := py.LoadTuple(args, []interface{}{&stop.Desc, &stop.NumNights}) + if err != nil { + return nil, err + } + + /* Alternative util func is ParseTuple(): + var desc, nights py.Object + err := py.ParseTuple(args, "si", &desc, &nights) + if err != nil { + return nil, err + } + stop.Desc = desc.(py.String) + stop.NumNights = desc.(py.Int) + */ + + return py.None, nil +} + +// VacationStop_Get is an embedded instance method of VacationStop +func VacationStop_Get(self py.Object, args py.Tuple) (py.Object, error) { + stop := self.(*VacationStop) + + return py.Tuple{ + py.Object(stop.Desc), + py.String(stop.NumNights), + }, nil +} + +type Vacation struct { + Stops []*VacationStop + MadeBy string +} + +func (v *Vacation) Type() *py.Type { + return PyVacationType +} + +func Vacation_new(module py.Object, args py.Tuple) (py.Object, error) { + v := &Vacation{} + + // For Module-bound methods, we have easy access to the parent Module + py.LoadAttr(module, "MYLIB_VERS", &v.MadeBy) + + ret := py.Tuple{ + v, + py.String(v.MadeBy), + } + return ret, nil +} + +func Vacation_num_stops(self py.Object, args py.Tuple) (py.Object, error) { + v := self.(*Vacation) + return py.Int(len(v.Stops)), nil +} + +func Vacation_get_stop(self py.Object, args py.Tuple) (py.Object, error) { + v := self.(*Vacation) + + // Check out other convenience functions in py/util.go + // If you would like to be a contributor for gpython, improving these or adding more is a great place to start! + stopNum, err := py.GetInt(args[0]) + if err != nil { + return nil, err + } + + if stopNum < 1 || int(stopNum) > len(v.Stops) { + return nil, py.ExceptionNewf(py.IndexError, "invalid stop index") + } + + return py.Object(v.Stops[stopNum-1]), nil +} + +func Vacation_add_stops(self py.Object, args py.Tuple) (py.Object, error) { + v := self.(*Vacation) + srcStops, ok := args[0].(py.Tuple) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "expected Tuple, got %T", args[0]) + } + + for _, arg := range srcStops { + stop, ok := arg.(*VacationStop) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "expected Stop, got %T", arg) + } + + v.Stops = append(v.Stops, stop) + } + + return py.None, nil +} From 168c337356ede027b915a76d6d306a90fe876f28 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Wed, 26 Jan 2022 10:38:01 -0600 Subject: [PATCH 71/79] main README makeover --- .gitignore | 3 ++ .vscode/launch.json | 26 ++++++++++++++ README.md | 84 +++++++++++++++++++++++---------------------- 3 files changed, 72 insertions(+), 41 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.gitignore b/.gitignore index a3eaead1..1981ae0a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ __pycache__ cover.out /junk /dist + +examples/embedding/embedding +builtin/testfile \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..b0d10522 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,26 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch file", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${file}" + }, + { + "name": "examples/embedding", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/examples/embedding", + "cwd": "${workspaceFolder}/examples/embedding", + "args": [ + "mylib-demo.py" + ], + }, + ] +} \ No newline at end of file diff --git a/README.md b/README.md index a5068a0f..19a6c3b7 100644 --- a/README.md +++ b/README.md @@ -5,18 +5,20 @@ [![GoDoc](https://godoc.org/github.com/go-python/gpython?status.svg)](https://godoc.org/github.com/go-python/gpython) [![License](https://img.shields.io/badge/License-BSD--3-blue.svg)](https://github.com/go-python/gpython/blob/master/LICENSE) -gpython is a part re-implementation / part port of the Python 3.4 -interpreter to the Go language, "batteries not included". - -It includes: - - * runtime - using compatible byte code to python3.4 - * lexer - * parser - * compiler +gpython is a part re-implementation, part port of the Python 3.4 +interpreter in Go. Although there are many areas of improvement, +it stands as an noteworthy achievement in capability and potential. + +gpython includes: + + * lexer, parser, and compiler + * runtime and high-level convenience functions + * multi-context interpreter instancing + * easy embedding into your Go application * interactive mode (REPL) ([try online!](https://gpython.org)) -It does not include very many python modules as many of the core + +gpython does not include many python modules as many of the core modules are written in C not python. The converted modules are: * builtins @@ -27,53 +29,53 @@ modules are written in C not python. The converted modules are: ## Install -Gpython is a Go program and comes as a single binary file. - -Download the relevant binary from here: https://github.com/go-python/gpython/releases +Download directly from the [releases page](https://github.com/go-python/gpython/releases) -Or alternatively if you have Go installed use +Or if you have Go installed: - go get github.com/go-python/gpython - -and this will build the binary in `$GOPATH/bin`. You can then modify -the source and submit patches. + go install github.com/go-python/gpython ## Objectives -Gpython was written as a learning experiment to investigate how hard +gpython started as an experiment to investigate how hard porting Python to Go might be. It turns out that all those C modules -are a significant barrier to making a fully functional port. +are a significant barrier to making gpython a complete replacement +to CPython. -## Status +However, to those that want to embed a highly popular and known language +into their Go application, gpython could be a great choice over less +capable (or lesser known) alternatives. Does the world really need +any more interpreted languages? -The project works well enough to parse all the code in the python 3.4 -distribution and to compile and run python 3 programs which don't -depend on a module gpython doesn't support. +## Status -See the examples directory for some python programs which run with -gpython. +gpython currently: + - Parses all the code in the Python 3.4 distribution + - Runs Python 3 for the modules that are currently supported + - Supports concurrent multi-interpreter execution ("multi-context") Speed hasn't been a goal of the conversions however it runs pystone at -about 20% of the speed of cpython. The pi test runs quicker under -gpython as I think the Go long integer primitives are faster than the +about 20% of the speed of CPython. The [π test](https://github.com/go-python/gpython/tree/master/examples/pi_chudnovsky_bs.py) runs quicker under +gpython as the Go long integer primitives are likely faster than the Python ones. -There are many directions this project could go in. I think the most -profitable would be to re-use the -[grumpy](https://github.com/grumpyhome/grumpy) runtime (which would mean -changing the object model). This would give access to the C modules -that need to be ported and would give grumpy access to a compiler and -interpreter (gpython does support `eval` for instance). +@ncw started gpython it in 2013 and work on is sporadic. If you or someone +you know would be interested to take it futher, it would be much appreciated. + +## Getting Started -I (@ncw) haven't had much time to work on gpython (I started it in -2013 and have worked on it very sporadically) so someone who wants to -take it in the next direction would be much appreciated. +The [embedding example](https://github.com/go-python/gpython/tree/master/examples/embedding) demonstrates how to +easily embed and invoke gpython from any Go application. -## Limitations and Bugs +Importantly, gpython is able to run multiple interpreter instances simultaneously, +allowing you to embed gpython naturally into your Go application. This makes it +possible to use gpython in a server situation where complete interpreter +independence is an absolute requirement. See this in action in the [multi-context example](https://github.com/go-python/gpython/tree/master/examples/multi-context) + +If you are looking to get involved, a light and easy place to start is adding more convenience functions to [py/util.go](https://github.com/go-python/gpython/tree/master/py/util.go). See [notes.txt](https://github.com/go-python/gpython/blob/master/notes.txt) for bigger ideas. -Lots! -## Similar projects +## Other Projects of Interest * [grumpy](https://github.com/grumpyhome/grumpy) - a python to go transpiler @@ -86,5 +88,5 @@ or on the [Gophers Slack](https://gophers.slack.com/) in the `#go-python` channe ## License This is licensed under the MIT licence, however it contains code which -was ported fairly directly directly from the cpython source code under +was ported fairly directly directly from the CPython source code under the [PSF LICENSE](https://github.com/python/cpython/blob/master/LICENSE). From 2b7e1abd19ecf6792800593483b6b123f9841344 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Wed, 26 Jan 2022 14:10:57 -0600 Subject: [PATCH 72/79] fixed type conversion --- examples/embedding/mylib.module.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/embedding/mylib.module.go b/examples/embedding/mylib.module.go index 9589852b..532bc451 100644 --- a/examples/embedding/mylib.module.go +++ b/examples/embedding/mylib.module.go @@ -105,8 +105,8 @@ func VacationStop_Get(self py.Object, args py.Tuple) (py.Object, error) { stop := self.(*VacationStop) return py.Tuple{ - py.Object(stop.Desc), - py.String(stop.NumNights), + stop.Desc, + stop.NumNights, }, nil } From 3821b4b17a10aa4e7114e504c2e19acb4f826384 Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Fri, 28 Jan 2022 09:40:09 -0600 Subject: [PATCH 73/79] Update README.md Co-authored-by: Sebastien Binet --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 19a6c3b7..8e521a29 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ porting Python to Go might be. It turns out that all those C modules are a significant barrier to making gpython a complete replacement to CPython. -However, to those that want to embed a highly popular and known language +However, to those who want to embed a highly popular and known language into their Go application, gpython could be a great choice over less capable (or lesser known) alternatives. Does the world really need any more interpreted languages? From 170f0d2ba4eb2bbe8b42a6fa026d54b4e6ed68f6 Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Fri, 28 Jan 2022 09:40:18 -0600 Subject: [PATCH 74/79] Update README.md Co-authored-by: Sebastien Binet --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 8e521a29..14d9fe43 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,7 @@ to CPython. However, to those who want to embed a highly popular and known language into their Go application, gpython could be a great choice over less -capable (or lesser known) alternatives. Does the world really need -any more interpreted languages? +capable (or lesser known) alternatives. ## Status From b68b9f64f8ffc2c98e1045757634231ffab4c64b Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Fri, 28 Jan 2022 09:40:49 -0600 Subject: [PATCH 75/79] Update py/util.go Co-authored-by: Sebastien Binet --- py/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/util.go b/py/util.go index 3555b001..d9fbd9ed 100644 --- a/py/util.go +++ b/py/util.go @@ -13,7 +13,7 @@ var ( ErrUnsupportedObjType = errors.New("unsupported obj type") ) -// GetInt is a high-level convenience function that gets the length of the given Object. +// GetLen is a high-level convenience function that returns the length of the given Object. func GetLen(obj Object) (Int, error) { getlen, ok := obj.(I__len__) if !ok { From 759023b0b171ac2c4ba422f6d71073c5a15a10a7 Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Fri, 28 Jan 2022 09:41:06 -0600 Subject: [PATCH 76/79] Update py/util.go Co-authored-by: Sebastien Binet --- py/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/util.go b/py/util.go index d9fbd9ed..a115bd9d 100644 --- a/py/util.go +++ b/py/util.go @@ -60,7 +60,7 @@ func LoadTuple(args Tuple, vars []interface{}) error { return nil } -// LoadAttr gets the named attrib and attempts to store it into the given destination value (based on its type). +// LoadAttr gets the named attribute and attempts to store it into the given destination value (based on its type). func LoadAttr(obj Object, attrName string, dst interface{}) error { attr, err := GetAttrString(obj, attrName) if err != nil { From 0c777160618a55e408c4ecbe7687c0e43f843e4c Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Fri, 28 Jan 2022 09:41:32 -0600 Subject: [PATCH 77/79] Update py/util.go Co-authored-by: Sebastien Binet --- py/util.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/py/util.go b/py/util.go index a115bd9d..60ef8f77 100644 --- a/py/util.go +++ b/py/util.go @@ -86,7 +86,9 @@ func LoadIntsFromList(list Object) ([]int64, error) { } var intList []int64 - if ok && N > 0 { + if N <= 0 { + return nil, nil + } intList = make([]int64, N) var intVal Int From 505755d938931f8e845575e948240496007eaa6d Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Fri, 28 Jan 2022 09:54:40 -0600 Subject: [PATCH 78/79] LoadIntsFromList cleanup --- py/util.go | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/py/util.go b/py/util.go index 60ef8f77..43028bce 100644 --- a/py/util.go +++ b/py/util.go @@ -85,26 +85,24 @@ func LoadIntsFromList(list Object) ([]int64, error) { return nil, nil } - var intList []int64 if N <= 0 { - return nil, nil + return nil, nil } - intList = make([]int64, N) + + intList := make([]int64, N) + for i := Int(0); i < N; i++ { + item, err := getter.M__getitem__(i) + if err != nil { + return nil, err + } var intVal Int - for i := Int(0); i < N; i++ { - item, err := getter.M__getitem__(i) - if err != nil { - return nil, err - } - - intVal, err = GetInt(item) - if err != nil { - return nil, err - } - - intList[i] = int64(intVal) + intVal, err = GetInt(item) + if err != nil { + return nil, err } + + intList[i] = int64(intVal) } return intList, nil From 02d6b3e489ff92db7381e9194e2d8e2ff8e46830 Mon Sep 17 00:00:00 2001 From: Drew O'Meara Date: Fri, 28 Jan 2022 10:01:32 -0600 Subject: [PATCH 79/79] comment edits --- examples/embedding/mylib.module.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/embedding/mylib.module.go b/examples/embedding/mylib.module.go index 532bc451..b510277c 100644 --- a/examples/embedding/mylib.module.go +++ b/examples/embedding/mylib.module.go @@ -11,7 +11,7 @@ import ( "github.com/go-python/gpython/py" ) -// Embedded gpython type delcarations are the bridge between gpython and embedded Go types. +// These gpython py.Object type delcarations are the bridge between gpython and embedded Go types. var ( PyVacationStopType = py.NewType("Stop", "") PyVacationType = py.NewType("Vacation", "") @@ -20,7 +20,8 @@ var ( // init is where you register your embedded module and attach methods to your embedded class types. func init() { - // Attach embedded methods to your embedded python types... + // For each of your embedded python types, attach instance methods. + // When an instance method is invoked, the "self" py.Object is the instance. PyVacationStopType.Dict["Set"] = py.MustNewMethod("Set", VacationStop_Set, 0, "") PyVacationStopType.Dict["Get"] = py.MustNewMethod("Get", VacationStop_Get, 0, "") PyVacationType.Dict["add_stops"] = py.MustNewMethod("Vacation.add_stops", Vacation_add_stops, 0, "") @@ -76,7 +77,7 @@ func VacationStop_new(module py.Object, args py.Tuple) (py.Object, error) { return stop, nil } -// VacationStop_Set is an embedded instance moethod of VacationStop +// VacationStop_Set is an embedded instance method of VacationStop func VacationStop_Set(self py.Object, args py.Tuple) (py.Object, error) { stop := self.(*VacationStop)