introducing py.Context

go-python · sbinet · Feb 7, 2022 · Feb 3, 2022 · Feb 3, 2022 · Feb 3, 2022
commit fd02d904586a0df64f94fd59572e33d37cd4573d
diff --git a/py/frame.go b/py/frame.go
@@ -26,6 +26,7 @@ type TryBlock struct {
 // A python Frame object
 type Frame struct {
 	// Back       *Frame        // previous frame, or nil
+	Context         Context    // host module (state) context
 	Code            *Code      // code segment
 	Builtins        StringDict // builtin symbol table
 	Globals         StringDict // global symbol table
@@ -77,7 +78,7 @@ func (o *Frame) Type() *Type {
 }
 
 // Make a new frame for a code object
-func NewFrame(globals, locals StringDict, code *Code, closure Tuple) *Frame {
+func NewFrame(ctx Context, globals, locals StringDict, code *Code, closure Tuple) *Frame {
 	nlocals := int(code.Nlocals)
 	ncells := len(code.Cellvars)
 	nfrees := len(code.Freevars)
@@ -90,12 +91,13 @@ func NewFrame(globals, locals StringDict, code *Code, closure Tuple) *Frame {
 	cellAndFreeVars := allocation[nlocals:varsize]
 
 	return &Frame{
+		Context:         ctx,
 		Globals:         globals,
 		Locals:          locals,
 		Code:            code,
 		LocalVars:       localVars,
 		CellAndFreeVars: cellAndFreeVars,
-		Builtins:        Builtins.Globals,
+		Builtins:        ctx.Store().Builtins.Globals,
 		Localsplus:      allocation,
 		Stack:           make([]Object, 0, code.Stacksize),
 	}

diff --git a/py/function.go b/py/function.go
@@ -18,6 +18,7 @@ package py
 // A python Function object
 type Function struct {
 	Code        *Code      // A code object, the __code__ attribute
+	Context     Context    // Host VM context
 	Globals     StringDict // A dictionary (other mappings won't do)
 	Defaults    Tuple      // NULL or a tuple
 	KwDefaults  StringDict // NULL or a dict
@@ -26,7 +27,6 @@ type Function struct {
 	Name        string     // The __name__ attribute, a string object
 	Dict        StringDict // The __dict__ attribute, a dict or NULL
 	Weakreflist List       // List of weak references
-	Module      Object     // The __module__ attribute, can be anything
 	Annotations StringDict // Annotations, a dict or NULL
 	Qualname    string     // The qualified name
 }
@@ -56,9 +56,8 @@ func (f *Function) GetDict() StringDict {
 // attribute. qualname should be a unicode object or ""; if "", the
 // __qualname__ attribute is set to the same value as its __name__
 // attribute.
-func NewFunction(code *Code, globals StringDict, qualname string) *Function {
+func NewFunction(ctx Context, code *Code, globals StringDict, qualname string) *Function {
 	var doc Object
-	var module Object = None
 	if len(code.Consts) >= 1 {
 		doc = code.Consts[0]
 		if _, ok := doc.(String); !ok {
@@ -68,29 +67,24 @@ func NewFunction(code *Code, globals StringDict, qualname string) *Function {
 		doc = None
 	}
 
-	// __module__: If module name is in globals, use it. Otherwise, use None.
-	if moduleobj, ok := globals["__name__"]; ok {
-		module = moduleobj
-	}
-
 	if qualname == "" {
 		qualname = code.Name
 	}
 
 	return &Function{
 		Code:     code,
+		Context:  ctx,
 		Qualname: qualname,
 		Globals:  globals,
 		Name:     code.Name,
 		Doc:      doc,
-		Module:   module,
 		Dict:     make(StringDict),
 	}
 }
 
 // Call a function
 func (f *Function) M__call__(args Tuple, kwargs StringDict) (Object, error) {
-	result, err := VmEvalCodeEx(f.Code, f.Globals, NewStringDict(), args, kwargs, f.Defaults, f.KwDefaults, f.Closure)
+	result, err := VmEvalCode(f.Context, f.Code, f.Globals, NewStringDict(), args, kwargs, f.Defaults, f.KwDefaults, f.Closure)
 	if err != nil {
 		return nil, err
 	}

diff --git a/py/py.go b/py/py.go
@@ -24,15 +24,10 @@ type IGoInt64 interface {
 	GoInt64() (int64, error)
 }
 
-// Some well known objects
 var (
 	// Set in vm/eval.go - to avoid circular import
-	VmRun        func(globals, locals StringDict, code *Code, closure Tuple) (res Object, err error)
-	VmRunFrame   func(frame *Frame) (res Object, err error)
-	VmEvalCodeEx func(co *Code, globals, locals StringDict, args []Object, kws StringDict, defs []Object, kwdefs StringDict, closure Tuple) (retval Object, err error)
-
-	// See compile/compile.go - set to avoid circular import
-	Compile func(str, filename, mode string, flags int, dont_inherit bool) (Object, error)
+	VmEvalCode func(ctx Context, code *Code, globals, locals StringDict, args []Object, kws StringDict, defs []Object, kwdefs StringDict, closure Tuple) (retval Object, err error)
+	VmRunFrame func(frame *Frame) (res Object, err error)
 )
 
 // Called to create a new instance of class cls. __new__() is a static method (special-cased so you need not declare it as such) that takes the class of which an instance was requested as its first argument. The remaining arguments are those passed to the object constructor expression (the call to the class). The return value of __new__() should be the new object instance (usually an instance of cls).

diff --git a/py/run.go b/py/run.go
@@ -0,0 +1,179 @@
+// Copyright 2022 The go-python Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package py
+
+type CompileMode string
+
+const (
+	ExecMode   CompileMode = "exec"   // Compile a module
+	EvalMode   CompileMode = "eval"   // Compile an expression
+	SingleMode CompileMode = "single" // Compile a single (interactive) statement
+)
+
+// Context is a gpython environment instance container, providing a high-level mechanism
+// for multiple python interpreters to run concurrently without restriction.
+//
+// Context instances maintain completely independent environments, namely the modules that
+// have been imported and their state.  Modules imported into a Context are instanced
+// from a parent ModuleImpl.  For example, since Contexts each have their
+// own sys module instance, each can set sys.path differently and independently.
+//
+// If you access a Context from multiple groutines, you are responsible that access is not concurrent,
+// with the exception of Close() and Done().
+//
+// See examples/multi-context and examples/embedding.
+type Context interface {
+
+	// Resolves then compiles (if applicable) the given file system pathname into a py.Code ready to be executed.
+	ResolveAndCompile(pathname string, opts CompileOpts) (CompileOut, error)
+
+	// Creates a new py.Module instance and initializes ModuleImpl's code in the new module (if applicable).
+	ModuleInit(impl *ModuleImpl) (*Module, error)
+
+	// RunCode is a lower-level invocation to execute the given py.Code.
+	// Blocks until execution is complete.
+	RunCode(code *Code, globals, locals StringDict, closure Tuple) (result Object, err error)
+
+	// Returns the named module for this context (or an error if not found)
+	GetModule(moduleName string) (*Module, error)
+
+	// Gereric access to this context's modules / state.
+	Store() *ModuleStore
+
+	// Close signals this context is about to go out of scope and any internal resources should be released.
+	// Code execution on a py.Context that has been closed will result in an error.
+	Close() error
+
+	// Done returns a signal that can be used to detect when this Context has fully closed / completed.
+	// If Close() is called while execution in progress, Done() will not signal until execution is complete.
+	Done() <-chan struct{}
+}
+
+// CompileOpts specifies options for high-level compilation.
+type CompileOpts struct {
+	UseSysPaths bool   // If set, sys.path will be used to resolve relative pathnames
+	CurDir      string // If non-empty, this is the path of the current working directory.  If empty, os.Getwd() is used.
+}
+
+// CompileOut the output of high-level compilation -- e.g. ResolveAndCompile()
+type CompileOut struct {
+	SrcPathname string // Resolved pathname the .py file that was compiled (if applicable)
+	PycPathname string // Pathname of the .pyc file read and/or written (if applicable)
+	FileDesc    string // Pathname to be used for a a module's "__file__" attrib
+	Code        *Code  // Read/Output code object ready for execution
+}
+
+// DefaultCoreSysPaths specify default search paths for module sys
+// This can be changed during runtime and plays nice with others using DefaultContextOpts()
+var DefaultCoreSysPaths = []string{
+	".",
+	"lib",
+}
+
+// DefaultAuxSysPaths are secondary default search paths for module sys.
+// This can be changed during runtime and plays nice with others using DefaultContextOpts()
+// They are separated from the default core paths since they the more likley thing you will want to completely replace when using gpython.
+var DefaultAuxSysPaths = []string{
+	"/usr/lib/python3.4",
+	"/usr/local/lib/python3.4/dist-packages",
+	"/usr/lib/python3/dist-packages",
+}
+
+// ContextOpts specifies fundamental environment and input settings for creating a new py.Context
+type ContextOpts struct {
+	SysArgs  []string // sys.argv initializer
+	SysPaths []string // sys.path initializer
+}
+
+var (
+	// DefaultContextOpts should be the default opts created for py.NewContext.
+	// Calling this ensure that you future proof you code for suggested/default settings.
+	DefaultContextOpts = func() ContextOpts {
+		opts := ContextOpts{
+			SysPaths: DefaultCoreSysPaths,
+		}
+		opts.SysPaths = append(opts.SysPaths, DefaultAuxSysPaths...)
+		return opts
+	}
+
+	// NewContext is a high-level call to create a new gpython interpreter context.
+	// See type Context interface.
+	NewContext func(opts ContextOpts) Context
+
+	// Compiles a python buffer into a py.Code object.
+	// Returns a py.Code object or otherwise an error.
+	Compile func(src, srcDesc string, mode CompileMode, flags int, dont_inherit bool) (*Code, error)
+)
+
+// RunFile resolves the given pathname, compiles as needed, executes the code in the given module, and returns the Module to indicate success.
+//
+// See RunCode() for description of inModule.
+func RunFile(ctx Context, pathname string, opts CompileOpts, inModule interface{}) (*Module, error) {
+	out, err := ctx.ResolveAndCompile(pathname, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	return RunCode(ctx, out.Code, out.FileDesc, inModule)
+}
+
+// RunSrc compiles the given python buffer and executes it within the given module and returns the Module to indicate success.
+//
+// See RunCode() for description of inModule.
+func RunSrc(ctx Context, pySrc string, pySrcDesc string, inModule interface{}) (*Module, error) {
+	if pySrcDesc == "" {
+		pySrcDesc = "<run>"
+	}
+	code, err := Compile(pySrc+"\n", pySrcDesc, SingleMode, 0, true)
+	if err != nil {
+		return nil, err
+	}
+
+	return RunCode(ctx, code, pySrcDesc, inModule)
+}
+
+// RunCode executes the given code object within the given module and returns the Module to indicate success.
+// If inModule is a *Module, then the code is run in that module.
+// If inModule is nil, the code is run in a new __main__ module (and the new Module is returned).
+// If inModule is a string, the code is run in a new module with the given name (and the new Module is returned).
+func RunCode(ctx Context, code *Code, codeDesc string, inModule interface{}) (*Module, error) {
+	var (
+		module     *Module
+		moduleName string
+		err        error
+	)
+
+	createNew := false
+	switch mod := inModule.(type) {
+
+	case string:
+		moduleName = mod
+		createNew = true
+	case nil:
+		createNew = true
+	case *Module:
+		_, err = ctx.RunCode(code, mod.Globals, mod.Globals, nil)
+		module = mod
+	default:
+		err = ExceptionNewf(TypeError, "unsupported module type: %v", inModule)
+	}
+
+	if err == nil && createNew {
+		moduleImpl := ModuleImpl{
+			Info: ModuleInfo{
+				Name:     moduleName,
+				FileDesc: codeDesc,
+			},
+			Code: code,
+		}
+		module, err = ctx.ModuleInit(&moduleImpl)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	return module, nil
+}
diff --git a/pytest/pytest.go b/pytest/pytest.go
@@ -11,13 +11,14 @@ import (
 	"strings"
 	"testing"
 
-	_ "github.com/go-python/gpython/builtin"
+	_ "github.com/go-python/gpython/modules"
+
 	"github.com/go-python/gpython/compile"
 	"github.com/go-python/gpython/py"
-	_ "github.com/go-python/gpython/sys"
-	"github.com/go-python/gpython/vm"
 )
 
+var gContext = py.NewContext(py.DefaultContextOpts())
+
 // Compile the program in the file prog to code in the module that is returned
 func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) {
 	f, err := os.Open(prog)
@@ -34,27 +35,32 @@ func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) {
 	if err != nil {
 		t.Fatalf("%s: ReadAll failed: %v", prog, err)
 	}
+	return CompileSrc(t, gContext, string(str), prog)
+}
 
-	obj, err := compile.Compile(string(str), prog, "exec", 0, true)
+func CompileSrc(t testing.TB, ctx py.Context, pySrc string, prog string) (*py.Module, *py.Code) {
+	code, err := compile.Compile(string(pySrc), prog, py.ExecMode, 0, true)
 	if err != nil {
 		t.Fatalf("%s: Compile failed: %v", prog, err)
 	}
 
-	code := obj.(*py.Code)
-	module := py.NewModule("__main__", "", nil, nil)
-	module.Globals["__file__"] = py.String(prog)
+	module, err := ctx.Store().NewModule(ctx, &py.ModuleImpl{
+		Info: py.ModuleInfo{
+			FileDesc: prog,
+		},
+	})
+	if err != nil {
+		t.Fatalf("%s: NewModule failed: %v", prog, err)
+	}
+
 	return module, code
 }
 
 // Run the code in the module
 func run(t testing.TB, module *py.Module, code *py.Code) {
-	_, err := vm.Run(module.Globals, module.Globals, code, nil)
+	_, err := gContext.RunCode(code, module.Globals, module.Globals, nil)
 	if err != nil {
-		if wantErr, ok := module.Globals["err"]; ok {
-			wantErrObj, ok := wantErr.(py.Object)
-			if !ok {
-				t.Fatalf("want err is not py.Object: %#v", wantErr)
-			}
+		if wantErrObj, ok := module.Globals["err"]; ok {
 			gotExc, ok := err.(py.ExceptionInfo)
 			if !ok {
 				t.Fatalf("got err is not ExceptionInfo: %#v", err)