From 30840f2aaf67deca57617ed0c9f942a85536ac2b Mon Sep 17 00:00:00 2001 From: Jens Alfke Date: Mon, 5 Jun 2023 15:52:33 -0700 Subject: [PATCH 1/7] Added js module: new JavaScript API supporting V8 --- go.mod | 4 +- go.sum | 4 + js/js_test.go | 299 ++++++ js/logging.go | 65 ++ js/otto_runner.go | 207 ++++ js/otto_vm.go | 118 +++ js/runner.go | 82 ++ js/service.go | 104 ++ js/service_no_v8.go | 17 + js/services_config.go | 51 + js/string.go | 13 + js/test_utils.go | 36 + js/test_utils_no_v8.go | 13 + js/test_utils_v8.go | 13 + js/underscore-umd-min.js | 6 + js/underscore-umd.js | 2042 ++++++++++++++++++++++++++++++++++++++ js/v8_runner.go | 272 +++++ js/v8_template.go | 200 ++++ js/v8_utils.go | 134 +++ js/v8_vm.go | 247 +++++ js/vm.go | 127 +++ js/vm_otto_only.go | 22 + js/vmpool.go | 229 +++++ js/vmpool_test.go | 219 ++++ 24 files changed, 4523 insertions(+), 1 deletion(-) create mode 100644 js/js_test.go create mode 100644 js/logging.go create mode 100644 js/otto_runner.go create mode 100644 js/otto_vm.go create mode 100644 js/runner.go create mode 100644 js/service.go create mode 100644 js/service_no_v8.go create mode 100644 js/services_config.go create mode 100644 js/string.go create mode 100644 js/test_utils.go create mode 100644 js/test_utils_no_v8.go create mode 100644 js/test_utils_v8.go create mode 100644 js/underscore-umd-min.js create mode 100644 js/underscore-umd.js create mode 100644 js/v8_runner.go create mode 100644 js/v8_template.go create mode 100644 js/v8_utils.go create mode 100644 js/v8_vm.go create mode 100644 js/vm.go create mode 100644 js/vm_otto_only.go create mode 100644 js/vmpool.go create mode 100644 js/vmpool_test.go diff --git a/go.mod b/go.mod index 6784078..d1851ec 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,9 @@ module github.com/couchbase/sg-bucket go 1.19 require ( + github.com/pkg/errors v0.9.1 github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f + github.com/snej/v8go v1.7.3 github.com/stretchr/testify v1.7.1 golang.org/x/text v0.3.7 gopkg.in/couchbase/gocb.v1 v1.6.7 @@ -14,7 +16,6 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.3.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect @@ -24,4 +25,5 @@ require ( gopkg.in/couchbaselabs/jsonx.v1 v1.0.1 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect + rogchap.com/v8go v0.8.0 // indirect ) diff --git a/go.sum b/go.sum index eacf1ad..89d6e35 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f h1:a7clxaGmmqtdNTXyvrp/lVO/Gnkzlhc/+dLs5v965GM= github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f/go.mod h1:/mK7FZ3mFYEn9zvNPhpngTyatyehSwte5bJZ4ehL5Xw= +github.com/snej/v8go v1.7.3 h1:skliM+LRvRdLKwqJK4I+1BCQy4jTZs0u37R6b7aEUaU= +github.com/snej/v8go v1.7.3/go.mod h1:s7IVrqyNoVfwYhndECq3XJ+/y0uq/JUH0/ECsC3k/UQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -42,3 +44,5 @@ gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rogchap.com/v8go v0.8.0 h1:/crDEiga68kOtbIqw3K9Rt9OztYz0LhAPHm2e3wK7Q4= +rogchap.com/v8go v0.8.0/go.mod h1:MxgP3pL2MW4dpme/72QRs8sgNMmM0pRc8DPhcuLWPAs= diff --git a/js/js_test.go b/js/js_test.go new file mode 100644 index 0000000..d3b8822 --- /dev/null +++ b/js/js_test.go @@ -0,0 +1,299 @@ +//go:build cb_sg_v8 + +/* +Copyright 2022-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package js + +import ( + "context" + "encoding/json" + "math" + "math/big" + "strconv" + "testing" + "time" + + "github.com/snej/v8go" + "github.com/stretchr/testify/assert" +) + +func TestSquare(t *testing.T) { + ctx := context.Background() + TestWithVMs(t, func(t *testing.T, vm VM) { + service := NewService(vm, "square", `function(n) {return n * n;}`) + assert.Equal(t, "square", service.Name()) + assert.Equal(t, vm, service.Host()) + + // Test Run: + result, err := service.Run(ctx, 13) + assert.NoError(t, err) + assert.EqualValues(t, 169, result) + + // Test WithRunner: + result, err = service.WithRunner(func(runner Runner) (any, error) { + assert.Nil(t, runner.Context()) + assert.NotNil(t, runner.ContextOrDefault()) + runner.SetContext(ctx) + assert.Equal(t, ctx, runner.Context()) + assert.Equal(t, ctx, runner.ContextOrDefault()) + + return runner.Run(9) + }) + assert.NoError(t, err) + assert.EqualValues(t, 81, result) + }) +} + +func TestSquareV8Args(t *testing.T) { + vm := V8.NewVM() + defer vm.Close() + + service := NewService(vm, "square", `function(n) {return n * n;}`) + result, err := service.WithRunner(func(runner Runner) (any, error) { + v8Runner := runner.(*V8Runner) + result, err := v8Runner.RunWithV8Args(v8Runner.NewInt(9)) + if err != nil { + return nil, err + } + return result.Integer(), nil + }) + assert.NoError(t, err) + assert.EqualValues(t, 81, result) +} + +func TestJSON(t *testing.T) { + ctx := context.Background() + + var pool VMPool + pool.Init(V8, 4) + defer pool.Close() + + service := NewService(&pool, "length", `function(v) {return v.length;}`) + + result, err := service.Run(ctx, []string{"a", "b", "c"}) + if assert.NoError(t, err) { + assert.EqualValues(t, 3, result) + } + + result, err = service.Run(ctx, JSONString(`[1,2,3,4]`)) + if assert.NoError(t, err) { + assert.EqualValues(t, 4, result) + } +} + +func TestCallback(t *testing.T) { + ctx := context.Background() + + vm := V8.NewVM() + defer vm.Close() + + src := `(function() { + return hey(1234, "hey you guys!"); + });` + + var heyParam string + + // A callback function that's callable from JS as hey(num, str) + hey := func(r *V8Runner, this *v8go.Object, args []*v8go.Value) (result any, err error) { + assert.Equal(t, len(args), 2) + assert.Equal(t, int64(1234), args[0].Integer()) + heyParam = args[1].String() + return 5678, nil + } + + service := NewCustomService(vm, "callbacks", func(tmpl *V8BasicTemplate) (V8Template, error) { + err := tmpl.SetScript(src) + tmpl.GlobalCallback("hey", hey) + return tmpl, err + }) + + result, err := service.Run(ctx) + assert.NoError(t, err) + assert.Equal(t, 5678, result) + assert.Equal(t, "hey you guys!", heyParam) +} + +// Test conversion of numbers into/out of JavaScript. +func TestNumbers(t *testing.T) { + ctx := context.Background() + + TestWithVMs(t, func(t *testing.T, vm VM) { + service := NewService(vm, "numbers", `function(n, expectedStr) { + if (typeof(n) != 'number' && typeof(n) != 'bigint') throw "Unexpectedly n is a " + typeof(n); + var str = n.toString(); + console.info("n=",n,"str=",str); + if (str != expectedStr) throw "Got " + str + " instead of " + expectedStr; + return n; + }`) + + t.Run("integers", func(t *testing.T) { + testInt := func(n int64) { + result, err := service.Run(ctx, n, strconv.FormatInt(n, 10)) + if assert.NoError(t, err) { + assert.EqualValues(t, n, result) + } + } + + testInt(-1) + testInt(0) + testInt(1) + testInt(math.MaxInt32) + testInt(math.MinInt32) + testInt(math.MaxInt64) + testInt(math.MinInt64) + testInt(math.MaxInt64 - 1) + testInt(math.MinInt64 + 1) + testInt(JavascriptMaxSafeInt) + testInt(JavascriptMinSafeInt) + testInt(JavascriptMaxSafeInt + 1) + testInt(JavascriptMinSafeInt - 1) + }) + + t.Run("floats", func(t *testing.T) { + testFloat := func(n float64) { + result, err := service.Run(ctx, n, strconv.FormatFloat(n, 'f', -1, 64)) + if assert.NoError(t, err) { + assert.EqualValues(t, n, result) + } + } + + testFloat(-1.0) + testFloat(0.0) + testFloat(0.001) + testFloat(1.0) + testFloat(math.MaxInt32) + testFloat(math.MinInt32) + testFloat(math.MaxInt64) + testFloat(math.MinInt64) + testFloat(float64(JavascriptMaxSafeInt)) + testFloat(float64(JavascriptMinSafeInt)) + testFloat(float64(JavascriptMaxSafeInt + 1)) + testFloat(float64(JavascriptMinSafeInt - 1)) + testFloat(12345678.12345678) + testFloat(22.0 / 7.0) + testFloat(0.1) + }) + + t.Run("json_Number_integer", func(t *testing.T) { + hugeInt := json.Number("123456789012345") + result, err := service.Run(ctx, hugeInt, string(hugeInt)) + if assert.NoError(t, err) { + assert.EqualValues(t, 123456789012345, result) + } + }) + + if vm.Engine().languageVersion >= 11 { // (Otto does not support BigInts) + t.Run("json_Number_huge_integer", func(t *testing.T) { + hugeInt := json.Number("1234567890123456789012345678901234567890") + result, err := service.Run(ctx, hugeInt, string(hugeInt)) + if assert.NoError(t, err) { + ibig := new(big.Int) + ibig, _ = ibig.SetString(string(hugeInt), 10) + assert.EqualValues(t, ibig, result) + } + }) + } + + t.Run("json_Number_float", func(t *testing.T) { + floatStr := json.Number("1234567890.123") + result, err := service.Run(ctx, floatStr, string(floatStr)) + if assert.NoError(t, err) { + assert.EqualValues(t, 1234567890.123, result) + } + }) + }) +} + +// For security purposes, verify that JS APIs to do network or file I/O are not present: +func TestNoIO(t *testing.T) { + ctx := context.Background() + + vm := V8.NewVM() // Otto appears to have no way to refer to the global object... + defer vm.Close() + + service := NewService(vm, "check", `function() { + // Ensure that global fns/classes enabling network or file access are missing: + if (globalThis.fetch !== undefined) throw "fetch exists"; + if (globalThis.XMLHttpRequest !== undefined) throw "XMLHttpRequest exists"; + if (globalThis.File !== undefined) throw "File exists"; + if (globalThis.require !== undefined) throw "require exists"; + // But the following should exist: + if (globalThis.Math === undefined) throw "Math is missing!"; + if (globalThis.String === undefined) throw "String is missing!"; + if (globalThis.Number === undefined) throw "Number is missing!"; + if (globalThis.console === undefined) throw "console is missing!"; + }`) + _, err := service.Run(ctx) + assert.NoError(t, err) +} + +// Verify that ECMAScript modules can't be loaded. (The older `require` is checked in TestNoIO.) +func TestNoModules(t *testing.T) { + ctx := context.Background() + + vm := V8.NewVM() // Otto doesn't support ES modules + defer vm.Close() + + src := `import foo from 'foo'; + (function() { });` + + service := NewCustomService(vm, "check", func(tmpl *V8BasicTemplate) (V8Template, error) { + err := tmpl.SetScript(src) + return tmpl, err + }) + _, err := service.Run(ctx) + assert.ErrorContains(t, err, "Cannot use import statement outside a module") +} + +func TestTimeout(t *testing.T) { + TestWithVMs(t, func(t *testing.T, vm VM) { + ctx := context.Background() + + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + service := NewService(vm, "forever", `function() { while (true) ; }`) + start := time.Now() + _, err := service.Run(ctx) + assert.Less(t, time.Since(start), 4*time.Second) + assert.Equal(t, context.DeadlineExceeded, err) + }) +} + +func TestOutOfMemory(t *testing.T) { + vm := V8.NewVM() + defer vm.Close() + ctx := context.Background() + + service := NewService(vm, "OOM", ` + function() { + let a = ["supercalifragilisticexpialidocious"]; + while (true) { + a = [a, a]; + } + }`) + _, err := service.Run(ctx) + assert.ErrorContains(t, err, "ExecutionTerminated: script execution has been terminated") +} + +func TestStackOverflow(t *testing.T) { + vm := V8.NewVM() + defer vm.Close() + ctx := context.Background() + + service := NewService(vm, "Overflow", ` + function() { + function recurse(n) {console.log("level ", n); return recurse(n + 1) * recurse(n + 2);} + return recurse(0); + }`) + _, err := service.Run(ctx) + assert.ErrorContains(t, err, "Maximum call stack size exceeded") +} diff --git a/js/logging.go b/js/logging.go new file mode 100644 index 0000000..17d86da --- /dev/null +++ b/js/logging.go @@ -0,0 +1,65 @@ +package js + +import ( + "context" + "log" +) + +type LogLevel uint32 + +const ( + // LevelNone disables all logging + LevelNone LogLevel = iota + // LevelError enables only error logging. + LevelError + // LevelWarn enables warn and error logging. + LevelWarn + // LevelInfo enables info, warn, and error logging. + LevelInfo + // LevelDebug enables debug, info, warn, and error logging. + LevelDebug + // LevelTrace enables trace, debug, info, warn, and error logging. + LevelTrace +) + +var ( + logLevelNamesPrint = []string{"JS: [NON] ", "JS: [ERR] ", "JS: [WRN] ", "JS: [INF] ", "JS: [DBG] ", "JS: [TRC] "} +) + +// Set this to configure the log level. +var Logging LogLevel = LevelInfo + +// Set this callback to redirect logging elsewhere. Default value writes to Go `log.Printf` +var LoggingCallback = func(ctx context.Context, level LogLevel, fmt string, args ...any) { + log.Printf(logLevelNamesPrint[level]+fmt, args...) +} + +func logError(ctx context.Context, fmt string, args ...any) { + if Logging >= LevelError { + LoggingCallback(ctx, LevelError, fmt, args...) + } +} + +func warn(ctx context.Context, fmt string, args ...any) { + if Logging >= LevelWarn { + LoggingCallback(ctx, LevelWarn, fmt, args...) + } +} + +func info(ctx context.Context, fmt string, args ...any) { + if Logging >= LevelInfo { + LoggingCallback(ctx, LevelInfo, fmt, args...) + } +} + +func debug(ctx context.Context, fmt string, args ...any) { + if Logging >= LevelDebug { + LoggingCallback(ctx, LevelDebug, fmt, args...) + } +} + +func trace(ctx context.Context, fmt string, args ...any) { + if Logging >= LevelTrace { + LoggingCallback(ctx, LevelTrace, fmt, args...) + } +} diff --git a/js/otto_runner.go b/js/otto_runner.go new file mode 100644 index 0000000..ed52b79 --- /dev/null +++ b/js/otto_runner.go @@ -0,0 +1,207 @@ +/* +Copyright 2022-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package js + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + + "github.com/pkg/errors" + "github.com/robertkrimen/otto" + + _ "github.com/robertkrimen/otto/underscore" +) + +type OttoRunner struct { + baseRunner // "superclass" + otto *otto.Otto // An Otto virtual machine. NOT THREAD SAFE. + fn otto.Value // The compiled function to run +} + +func newOttoRunner(vm *ottoVM, service *Service) (*OttoRunner, error) { + ottoVM := otto.New() + fnobj, err := ottoVM.Object("(" + service.jsFunctionSource + ")") + if err != nil { + return nil, err + } + if fnobj.Class() != "Function" { + return nil, errors.New("JavaScript source does not evaluate to a function") + } + + r := &OttoRunner{ + baseRunner: baseRunner{ + id: service.id, + vm: vm, + }, + otto: ottoVM, + fn: fnobj.Value(), + } + + // Redirect JS logging to the SG console: + err = ottoVM.Set("sg_log", func(call otto.FunctionCall) otto.Value { + ilevel, _ := call.ArgumentList[0].ToInteger() + message, _ := call.ArgumentList[1].ToString() + extra := "" + for _, arg := range call.ArgumentList[2:] { + if arg.IsUndefined() { + break + } + str, _ := arg.ToString() + extra += str + " " + } + + LoggingCallback(r.ContextOrDefault(), LogLevel(ilevel), "%s %s", message, extra) + return otto.UndefinedValue() + }) + if err != nil { + return nil, err + } + _, err = ottoVM.Run(` + console.trace = function(msg,a,b,c,d) {sg_log(5, msg,a,b,c,d);}; + console.debug = function(msg,a,b,c,d) {sg_log(4, msg,a,b,c,d);}; + console.log = function(msg,a,b,c,d) {sg_log(3, msg,a,b,c,d);}; + console.warn = function(msg,a,b,c,d) {sg_log(2, msg,a,b,c,d);}; + console.error = function(msg,a,b,c,d) {sg_log(1, msg,a,b,c,d);};`) + if err != nil { + return nil, err + } + + return r, nil +} + +func (r *OttoRunner) Return() { r.vm.(*ottoVM).returnRunner(r) } + +func (r *OttoRunner) Run(args ...any) (result any, err error) { + // Translate args to Otto values: + jsArgs := make([]any, len(args)) + for i, input := range args { + if jsonStr, ok := input.(JSONString); ok { + if input, err = r.jsonToValue(string(jsonStr)); err != nil { + return nil, err + } + } else { + input, _ = convertJSONNumbers(input) + } + jsArgs[i], err = r.otto.ToValue(input) + if err != nil { + return nil, fmt.Errorf("couldn't convert arg %d, %#v, to JS: %w", i, args[i], err) + } + } + + // If the Context has a timeout, set up a goroutine that will interrupt Otto: + if timeoutChan := r.Context().Done(); timeoutChan != nil { + runnerDoneChan := make(chan bool, 1) + defer func() { + close(runnerDoneChan) + // Catch the panic thrown by the Interrupt fn and make it return an error: + if caught := recover(); caught != nil { + if caught == context.DeadlineExceeded { + err = context.DeadlineExceeded + return + } + panic(caught) + } + }() + + r.otto.Interrupt = make(chan func(), 1) + go func() { + select { + case <-timeoutChan: + r.otto.Interrupt <- func() { + panic(context.DeadlineExceeded) + } + case <-runnerDoneChan: + return + } + }() + } + + // Finally run the function: + resultVal, err := r.fn.Call(r.fn, jsArgs...) + if err != nil { + return nil, err + } + result, _ = resultVal.Export() + return result, nil +} + +func (runner *OttoRunner) jsonToValue(jsonStr string) (any, error) { + if jsonStr == "" { + return otto.NullValue(), nil + } + var parsed any + if err := json.Unmarshal([]byte(jsonStr), &parsed); err != nil { + return nil, fmt.Errorf("unparseable Runner input: %s", jsonStr) + } + return parsed, nil +} + +// Converts json.Number objects to regular Go numeric values. +// Integers that would lose precision are left as json.Number, as are floats that can't be +// converted to float64. (These will appear to be strings in JavaScript.) +// +// The function recurses into JSON arrays and maps; if changes are made, they are copied, +// not modified in place. +// +// Note: This function is not necessary with V8, because v8go already translates json.Numbers +// into JS BigInts. +func convertJSONNumbers(value any) (result any, changed bool) { + switch value := value.(type) { + case json.Number: + if asInt, err := value.Int64(); err == nil { + if asInt <= JavascriptMaxSafeInt && asInt >= JavascriptMinSafeInt { + return asInt, true + } else { + // Integer would lose precision when used in javascript - leave as json.Number + break + } + } else if numErr, _ := err.(*strconv.NumError); numErr.Err == strconv.ErrRange { + // out of range of int64 + break + } else if asFloat, err := value.Float64(); err == nil { + // Can't reliably detect loss of precision in float, due to number of variations in input float format + return asFloat, true + } + case map[string]any: + var copied map[string]any + for k, v := range value { + if newVal, changed := convertJSONNumbers(v); changed { + if copied == nil { + copied = make(map[string]any, len(value)) + for kk, vv := range value { + copied[kk] = vv + } + } + copied[k] = newVal + } + } + if copied != nil { + return copied, true + } + case []any: + var copied []any + for i, v := range value { + if newVal, changed := convertJSONNumbers(v); changed { + if copied == nil { + copied = append(copied, value...) + } + copied[i] = newVal + } + } + if copied != nil { + return copied, true + } + default: + } + return value, false +} diff --git a/js/otto_vm.go b/js/otto_vm.go new file mode 100644 index 0000000..3f4e29c --- /dev/null +++ b/js/otto_vm.go @@ -0,0 +1,118 @@ +/* +Copyright 2022-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package js + +import ( + "fmt" + "time" +) + +type ottoVM struct { + *baseVM + runners []*OttoRunner // Available Runners, indexed by serviceID. nil if in-use + curRunner *OttoRunner // Currently active Runner, if any +} + +const ottoVMName = "Otto" + +// An Engine for instantiating Otto-based VMs and VMPools. +var Otto = &Engine{ + name: ottoVMName, + languageVersion: 5, // https://github.com/robertkrimen/otto#caveat-emptor + factory: func(engine *Engine, services *servicesConfiguration) VM { + return &ottoVM{ + baseVM: &baseVM{engine: engine, services: services}, // "superclass" + runners: []*OttoRunner{}, // Cached reusable Runners + } + }, +} + +func (vm *ottoVM) Close() { + vm.baseVM.close() + if cur := vm.curRunner; cur != nil { + cur.Return() + } + vm.curRunner = nil + vm.runners = nil +} + +// Looks up an already-registered service by name. Returns nil if not found. +func (vm *ottoVM) FindService(name string) *Service { + return vm.services.findServiceNamed(name) +} + +// Must be called when finished using a VM belonging to a VMPool! +// (Harmless no-op when called on a standalone VM.) +func (vm *ottoVM) release() { + if vm.returnToPool != nil { + vm.lastReturned = time.Now() + vm.returnToPool.returnVM(vm) + } +} + +func (vm *ottoVM) hasInitializedService(service *Service) bool { + id := int(service.id) + return id < len(vm.runners) && vm.runners[id] != nil +} + +func (vm *ottoVM) getRunner(service *Service) (Runner, error) { + if vm.closed { + return nil, fmt.Errorf("the js.VM has been closed") + } + if vm.curRunner != nil { + panic("illegal access to v8VM: already has a v8Runner") + } + if !vm.services.hasService(service) { + return nil, fmt.Errorf("unknown js.Service instance passed to VM") + } + if service.v8Init != nil { + return nil, fmt.Errorf("js.Service has custom initialization not supported by Otto") + } + + // Use an existing Runner or create a new one: + var runner *OttoRunner + if int(service.id) < len(vm.runners) { + runner = vm.runners[service.id] + vm.runners[service.id] = nil + } + if runner == nil { + var err error + runner, err = newOttoRunner(vm, service) + if err != nil { + return nil, fmt.Errorf("unexpected error initializing JavaScript service %q: %w", service.Name(), err) + } + for int(service.id) >= len(vm.runners) { + vm.runners = append(vm.runners, nil) + } + } + vm.curRunner = runner + return runner, nil +} + +func (vm *ottoVM) withRunner(service *Service, fn func(Runner) (any, error)) (any, error) { + runner, err := vm.getRunner(service) + if err != nil { + return nil, err + } + defer runner.Return() + return fn(runner) +} + +func (vm *ottoVM) returnRunner(r *OttoRunner) { + r.goContext = nil + if vm.curRunner == r { + vm.curRunner = nil + } else if r.vm != vm { + panic("OttoRunner returned to wrong v8VM!") + } + vm.runners[r.id] = r + vm.release() +} diff --git a/js/runner.go b/js/runner.go new file mode 100644 index 0000000..2b0871f --- /dev/null +++ b/js/runner.go @@ -0,0 +1,82 @@ +/* +Copyright 2022-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package js + +import ( + "context" + "time" +) + +// A Runner represents a Service instantiated in a VM, in its own sandboxed V8 context, +// ready to run the Service's code. +// **NOT thread-safe!** +type Runner interface { + // The JavaScript VM this Runner runs in. + VM() VM + // Returns the Runner back to its VM when you're done with it. + Return() + // Associates a Go `Context` with this Runner. + // If this Context has a deadline, JS calls will abort if it expires. + SetContext(ctx context.Context) + // The associated `Context`, if you've set one; else nil. + Context() context.Context + // The associated `Context`, else the default `context.Background()` instance. + ContextOrDefault() context.Context + // Returns the remaining duration until the Context's deadline, or nil if none. + Timeout() *time.Duration + // Runs the Service's JavaScript function. + // Arguments and the return value are translated from/to Go types for you. + Run(args ...any) (any, error) + // Associates a value with this Runner that you can retrieve later for your own purposes. + SetAssociatedValue(any) + // An optional value you've associated with this Runner; else nil. + AssociatedValue() any +} + +type baseRunner struct { + id serviceID // The service ID in its VM + vm VM // The owning VM object + goContext context.Context // context.Context value for use by Go callbacks + associated any +} + +func (r *baseRunner) VM() VM { return r.vm } +func (r *baseRunner) AssociatedValue() any { return r.associated } +func (r *baseRunner) SetAssociatedValue(obj any) { r.associated = obj } +func (r *baseRunner) SetContext(ctx context.Context) { r.goContext = ctx } +func (r *baseRunner) Context() context.Context { return r.goContext } + +func (r *baseRunner) ContextOrDefault() context.Context { + if r.goContext != nil { + return r.goContext + } else { + return context.TODO() + } +} + +func (r *baseRunner) Timeout() *time.Duration { + if r.goContext != nil { + if deadline, hasDeadline := r.goContext.Deadline(); hasDeadline { + timeout := time.Until(deadline) + return &timeout + } + } + return nil +} + +// The maximum integer that can be represented accurately by a JavaScript number. +// (JavaScript numbers are 64-bit IEEE floats with 53 bits of precision.) +// (https://www.ecma-international.org/ecma-262/5.1/#sec-8.5) +const JavascriptMaxSafeInt = int64(1<<53 - 1) + +// The minimum integer that can be represented accurately by a JavaScript number. +// (JavaScript numbers are 64-bit IEEE floats with 53 bits of precision.) +const JavascriptMinSafeInt = -JavascriptMaxSafeInt diff --git a/js/service.go b/js/service.go new file mode 100644 index 0000000..f5c664e --- /dev/null +++ b/js/service.go @@ -0,0 +1,104 @@ +/* +Copyright 2022-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package js + +import ( + "context" +) + +// A Service represents a JavaScript-based API that runs in a VM or VMPool. +type Service struct { + host ServiceHost + id serviceID + name string + jsFunctionSource string + v8Init TemplateFactory +} + +type serviceID uint32 // internal ID, used as an array index in VM and VMPool. + +// A provider of a JavaScript runtime for Services. VM and VMPool implement this. +type ServiceHost interface { + Engine() *Engine + Close() + FindService(name string) *Service + registerService(*Service) + getRunner(*Service) (Runner, error) + withRunner(*Service, func(Runner) (any, error)) (any, error) +} + +// A factory/initialization function for Services that need to add JS globals or callbacks or +// otherwise extend their runtime environment. They do this by operating on its Template. +// +// The function's parameter is a BasicTemplate that doesn't have a script yet. +// The function MUST call its SetScript method. +// The function may return the Template it was given, or it may instantiate its own struct that +// implements Template (which presumably includes a pointer to the BasicTemplate) and return that. +type TemplateFactory func(base *V8BasicTemplate) (V8Template, error) + +// Creates a new Service in a ServiceHost (a VM or VMPool.) +// The name is primarily for logging; it does not need to be unique. +// The source code should be of the form `function(arg1,arg2…) {…body…; return result;}`. +// If you have a more complex script, like one that defines several functions, use NewCustomService. +func NewService(host ServiceHost, name string, jsFunctionSource string) *Service { + debug(context.Background(), "Creating JavaScript service %q", name) + service := &Service{ + host: host, + name: name, + jsFunctionSource: jsFunctionSource, + } + host.registerService(service) + return service +} + +// Creates a new Service in a ServiceHost (a VM or VMPool.) +// The implementation can extend the Service's JavaScript template environment by defining globals +// and/or callback functions. +func NewCustomService(host ServiceHost, name string, factory TemplateFactory) *Service { + service := NewService(host, name, "") + service.v8Init = factory + return service +} + +// The Service's name, given when it was created. +func (service *Service) Name() string { return service.name } + +// The VM or VMPool that provides the Service's runtime environment. +func (service *Service) Host() ServiceHost { return service.host } + +// Returns a Runner instance that can be used to call the Service's code. +// This may be a new instance, or (if the Service's Template is reuseable) a recycled one. +// You **MUST** call its Return method when you're through with it. +// +// - If the Service's host is a VMPool, this call will block while all the pool's VMs are in use. +// - If the host is a VM, this call will fail if there is another Runner in use belonging to any +// Service hosted by that VM. +func (service *Service) GetRunner() (Runner, error) { + debug(context.Background(), "Running JavaScript service %q", service.name) + return service.host.getRunner(service) +} + +// A convenience wrapper around GetRunner that takes care of returning the Runner. +// It simply returns whatever the callback returns. +func (service *Service) WithRunner(fn func(Runner) (any, error)) (any, error) { + return service.host.withRunner(service, fn) +} + +// A high-level method that runs a service in a VM without your needing to interact with a Runner. +// The arguments can be Go types or V8 Values; any types supported by VM.NewValue. +// The result is converted back to a Go type. +// If the function throws a JavaScript exception it's converted to a Go `error`. +func (service *Service) Run(ctx context.Context, args ...any) (any, error) { + return service.WithRunner(func(runner Runner) (any, error) { + runner.SetContext(ctx) + return runner.Run(args...) + }) +} diff --git a/js/service_no_v8.go b/js/service_no_v8.go new file mode 100644 index 0000000..35d6eec --- /dev/null +++ b/js/service_no_v8.go @@ -0,0 +1,17 @@ +// Copyright 2023-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +//go:build !cb_sg_v8 + +package js + +type V8BasicTemplate struct { +} + +type V8Template struct { +} diff --git a/js/services_config.go b/js/services_config.go new file mode 100644 index 0000000..45e4fdb --- /dev/null +++ b/js/services_config.go @@ -0,0 +1,51 @@ +/* +Copyright 2022-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package js + +import ( + "sync" +) + +// A thread-safe registry of Services. Each is identified by a `serviceID`. +type servicesConfiguration struct { + mutex sync.Mutex + registry []*Service +} + +// Registers a new Service, assigning its `id` field. +func (config *servicesConfiguration) addService(service *Service) { + config.mutex.Lock() + defer config.mutex.Unlock() + + config.registry = append(config.registry, service) + service.id = serviceID(len(config.registry) - 1) +} + +// Checks that the registry contains this Service instance. +func (config *servicesConfiguration) hasService(service *Service) bool { + config.mutex.Lock() + defer config.mutex.Unlock() + + return int(service.id) < len(config.registry) && service == config.registry[service.id] +} + +// Returns the [first] Service with a given name, or nil if not found. +func (config *servicesConfiguration) findServiceNamed(name string) *Service { + config.mutex.Lock() + defer config.mutex.Unlock() + + for _, service := range config.registry { + if service.name == name { + return service + } + } + return nil +} diff --git a/js/string.go b/js/string.go new file mode 100644 index 0000000..b7647f2 --- /dev/null +++ b/js/string.go @@ -0,0 +1,13 @@ +// Copyright 2023-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package js + +// A string type that v8Runner.NewValue will treat specially, by parsing it as JSON and converting +// it to a JavaScript object. +type JSONString string diff --git a/js/test_utils.go b/js/test_utils.go new file mode 100644 index 0000000..a3cc278 --- /dev/null +++ b/js/test_utils.go @@ -0,0 +1,36 @@ +/* +Copyright 2022-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package js + +import "testing" + +// Unit-test utility. Calls the function with each supported type of VM (Otto and V8). +func TestWithVMs(t *testing.T, fn func(t *testing.T, vm VM)) { + for _, engine := range testEngines { + t.Run(engine.String(), func(t *testing.T) { + vm := engine.NewVM() + defer vm.Close() + fn(t, vm) + }) + } +} + +// Unit-test utility. Calls the function with a VMPool of each supported type (Otto and V8). +// The behavior will be basically identical to TestWithVMs unless your test is multi-threaded. +func TestWithVMPools(t *testing.T, maxVMs int, fn func(t *testing.T, pool *VMPool)) { + for _, engine := range testEngines { + t.Run(engine.String(), func(t *testing.T) { + pool := NewVMPool(engine, maxVMs) + defer pool.Close() + fn(t, pool) + }) + } +} diff --git a/js/test_utils_no_v8.go b/js/test_utils_no_v8.go new file mode 100644 index 0000000..3b76cc2 --- /dev/null +++ b/js/test_utils_no_v8.go @@ -0,0 +1,13 @@ +// Copyright 2023-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +//go:build !cb_sg_v8 + +package js + +var testEngines = []*Engine{Otto} diff --git a/js/test_utils_v8.go b/js/test_utils_v8.go new file mode 100644 index 0000000..fade5be --- /dev/null +++ b/js/test_utils_v8.go @@ -0,0 +1,13 @@ +// Copyright 2023-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +//go:build cb_sg_v8 + +package js + +var testEngines = []*Engine{V8, Otto} diff --git a/js/underscore-umd-min.js b/js/underscore-umd-min.js new file mode 100644 index 0000000..b73f16e --- /dev/null +++ b/js/underscore-umd-min.js @@ -0,0 +1,6 @@ +!function(n,r){"object"==typeof exports&&"undefined"!=typeof module?module.exports=r():"function"==typeof define&&define.amd?define("underscore",r):(n="undefined"!=typeof globalThis?globalThis:n||self,function(){var t=n._,e=n._=r();e.noConflict=function(){return n._=t,e}}())}(this,(function(){ +// Underscore.js 1.13.6 +// https://underscorejs.org +// (c) 2009-2022 Jeremy Ashkenas, Julian Gonggrijp, and DocumentCloud and Investigative Reporters & Editors +// Underscore may be freely distributed under the MIT license. +var n="1.13.6",r="object"==typeof self&&self.self===self&&self||"object"==typeof global&&global.global===global&&global||Function("return this")()||{},t=Array.prototype,e=Object.prototype,u="undefined"!=typeof Symbol?Symbol.prototype:null,o=t.push,i=t.slice,a=e.toString,f=e.hasOwnProperty,c="undefined"!=typeof ArrayBuffer,l="undefined"!=typeof DataView,s=Array.isArray,p=Object.keys,v=Object.create,h=c&&ArrayBuffer.isView,y=isNaN,d=isFinite,g=!{toString:null}.propertyIsEnumerable("toString"),b=["valueOf","isPrototypeOf","toString","propertyIsEnumerable","hasOwnProperty","toLocaleString"],m=Math.pow(2,53)-1;function j(n,r){return r=null==r?n.length-1:+r,function(){for(var t=Math.max(arguments.length-r,0),e=Array(t),u=0;u=0&&t<=m}}function J(n){return function(r){return null==r?void 0:r[n]}}var G=J("byteLength"),H=K(G),Q=/\[object ((I|Ui)nt(8|16|32)|Float(32|64)|Uint8Clamped|Big(I|Ui)nt64)Array\]/;var X=c?function(n){return h?h(n)&&!q(n):H(n)&&Q.test(a.call(n))}:C(!1),Y=J("length");function Z(n,r){r=function(n){for(var r={},t=n.length,e=0;e":">",'"':""","'":"'","`":"`"},$n=zn(Ln),Cn=zn(_n(Ln)),Kn=tn.templateSettings={evaluate:/<%([\s\S]+?)%>/g,interpolate:/<%=([\s\S]+?)%>/g,escape:/<%-([\s\S]+?)%>/g},Jn=/(.)^/,Gn={"'":"'","\\":"\\","\r":"r","\n":"n","\u2028":"u2028","\u2029":"u2029"},Hn=/\\|'|\r|\n|\u2028|\u2029/g;function Qn(n){return"\\"+Gn[n]}var Xn=/^\s*(\w|\$)+\s*$/;var Yn=0;function Zn(n,r,t,e,u){if(!(e instanceof r))return n.apply(t,u);var o=Mn(n.prototype),i=n.apply(o,u);return _(i)?i:o}var nr=j((function(n,r){var t=nr.placeholder,e=function(){for(var u=0,o=r.length,i=Array(o),a=0;a1)er(a,r-1,t,e),u=e.length;else for(var f=0,c=a.length;f0&&(t=r.apply(this,arguments)),n<=1&&(r=null),t}}var cr=nr(fr,2);function lr(n,r,t){r=Pn(r,t);for(var e,u=nn(n),o=0,i=u.length;o0?0:u-1;o>=0&&o0?a=o>=0?o:Math.max(o+f,a):f=o>=0?Math.min(o+1,f):o+f+1;else if(t&&o&&f)return e[o=t(e,u)]===u?o:-1;if(u!=u)return(o=r(i.call(e,a,f),$))>=0?o+a:-1;for(o=n>0?a:f-1;o>=0&&o0?0:i-1;for(u||(e=r[o?o[a]:a],a+=n);a>=0&&a=3;return r(n,Rn(t,u,4),e,o)}}var wr=_r(1),Ar=_r(-1);function xr(n,r,t){var e=[];return r=Pn(r,t),mr(n,(function(n,t,u){r(n,t,u)&&e.push(n)})),e}function Sr(n,r,t){r=Pn(r,t);for(var e=!tr(n)&&nn(n),u=(e||n).length,o=0;o=0}var Er=j((function(n,r,t){var e,u;return D(r)?u=r:(r=Bn(r),e=r.slice(0,-1),r=r[r.length-1]),jr(n,(function(n){var o=u;if(!o){if(e&&e.length&&(n=Nn(n,e)),null==n)return;o=n[r]}return null==o?o:o.apply(n,t)}))}));function Br(n,r){return jr(n,Dn(r))}function Nr(n,r,t){var e,u,o=-1/0,i=-1/0;if(null==r||"number"==typeof r&&"object"!=typeof n[0]&&null!=n)for(var a=0,f=(n=tr(n)?n:jn(n)).length;ao&&(o=e);else r=Pn(r,t),mr(n,(function(n,t,e){((u=r(n,t,e))>i||u===-1/0&&o===-1/0)&&(o=n,i=u)}));return o}var Ir=/[^\ud800-\udfff]|[\ud800-\udbff][\udc00-\udfff]|[\ud800-\udfff]/g;function Tr(n){return n?U(n)?i.call(n):S(n)?n.match(Ir):tr(n)?jr(n,Tn):jn(n):[]}function kr(n,r,t){if(null==r||t)return tr(n)||(n=jn(n)),n[Un(n.length-1)];var e=Tr(n),u=Y(e);r=Math.max(Math.min(r,u),0);for(var o=u-1,i=0;i1&&(e=Rn(e,r[1])),r=an(n)):(e=qr,r=er(r,!1,!1),n=Object(n));for(var u=0,o=r.length;u1&&(t=r[1])):(r=jr(er(r,!1,!1),String),e=function(n,t){return!Mr(r,t)}),Ur(n,e,t)}));function zr(n,r,t){return i.call(n,0,Math.max(0,n.length-(null==r||t?1:r)))}function Lr(n,r,t){return null==n||n.length<1?null==r||t?void 0:[]:null==r||t?n[0]:zr(n,n.length-r)}function $r(n,r,t){return i.call(n,null==r||t?1:r)}var Cr=j((function(n,r){return r=er(r,!0,!0),xr(n,(function(n){return!Mr(r,n)}))})),Kr=j((function(n,r){return Cr(n,r)}));function Jr(n,r,t,e){A(r)||(e=t,t=r,r=!1),null!=t&&(t=Pn(t,e));for(var u=[],o=[],i=0,a=Y(n);ir?(e&&(clearTimeout(e),e=null),a=c,i=n.apply(u,o),e||(u=o=null)):e||!1===t.trailing||(e=setTimeout(f,l)),i};return c.cancel=function(){clearTimeout(e),a=0,e=u=o=null},c},debounce:function(n,r,t){var e,u,o,i,a,f=function(){var c=Wn()-u;r>c?e=setTimeout(f,r-c):(e=null,t||(i=n.apply(a,o)),e||(o=a=null))},c=j((function(c){return a=this,o=c,u=Wn(),e||(e=setTimeout(f,r),t&&(i=n.apply(a,o))),i}));return c.cancel=function(){clearTimeout(e),e=o=a=null},c},wrap:function(n,r){return nr(r,n)},negate:ar,compose:function(){var n=arguments,r=n.length-1;return function(){for(var t=r,e=n[r].apply(this,arguments);t--;)e=n[t].call(this,e);return e}},after:function(n,r){return function(){if(--n<1)return r.apply(this,arguments)}},before:fr,once:cr,findKey:lr,findIndex:pr,findLastIndex:vr,sortedIndex:hr,indexOf:dr,lastIndexOf:gr,find:br,detect:br,findWhere:function(n,r){return br(n,kn(r))},each:mr,forEach:mr,map:jr,collect:jr,reduce:wr,foldl:wr,inject:wr,reduceRight:Ar,foldr:Ar,filter:xr,select:xr,reject:function(n,r,t){return xr(n,ar(Pn(r)),t)},every:Sr,all:Sr,some:Or,any:Or,contains:Mr,includes:Mr,include:Mr,invoke:Er,pluck:Br,where:function(n,r){return xr(n,kn(r))},max:Nr,min:function(n,r,t){var e,u,o=1/0,i=1/0;if(null==r||"number"==typeof r&&"object"!=typeof n[0]&&null!=n)for(var a=0,f=(n=tr(n)?n:jn(n)).length;ae||void 0===t)return 1;if(t= 0 && sizeProperty <= MAX_ARRAY_INDEX; + } + } + + // Internal helper to generate a function to obtain property `key` from `obj`. + function shallowProperty(key) { + return function(obj) { + return obj == null ? void 0 : obj[key]; + }; + } + + // Internal helper to obtain the `byteLength` property of an object. + var getByteLength = shallowProperty('byteLength'); + + // Internal helper to determine whether we should spend extensive checks against + // `ArrayBuffer` et al. + var isBufferLike = createSizePropertyCheck(getByteLength); + + // Is a given value a typed array? + var typedArrayPattern = /\[object ((I|Ui)nt(8|16|32)|Float(32|64)|Uint8Clamped|Big(I|Ui)nt64)Array\]/; + function isTypedArray(obj) { + // `ArrayBuffer.isView` is the most future-proof, so use it when available. + // Otherwise, fall back on the above regular expression. + return nativeIsView ? (nativeIsView(obj) && !isDataView$1(obj)) : + isBufferLike(obj) && typedArrayPattern.test(toString.call(obj)); + } + + var isTypedArray$1 = supportsArrayBuffer ? isTypedArray : constant(false); + + // Internal helper to obtain the `length` property of an object. + var getLength = shallowProperty('length'); + + // Internal helper to create a simple lookup structure. + // `collectNonEnumProps` used to depend on `_.contains`, but this led to + // circular imports. `emulatedSet` is a one-off solution that only works for + // arrays of strings. + function emulatedSet(keys) { + var hash = {}; + for (var l = keys.length, i = 0; i < l; ++i) hash[keys[i]] = true; + return { + contains: function(key) { return hash[key] === true; }, + push: function(key) { + hash[key] = true; + return keys.push(key); + } + }; + } + + // Internal helper. Checks `keys` for the presence of keys in IE < 9 that won't + // be iterated by `for key in ...` and thus missed. Extends `keys` in place if + // needed. + function collectNonEnumProps(obj, keys) { + keys = emulatedSet(keys); + var nonEnumIdx = nonEnumerableProps.length; + var constructor = obj.constructor; + var proto = (isFunction$1(constructor) && constructor.prototype) || ObjProto; + + // Constructor is a special case. + var prop = 'constructor'; + if (has$1(obj, prop) && !keys.contains(prop)) keys.push(prop); + + while (nonEnumIdx--) { + prop = nonEnumerableProps[nonEnumIdx]; + if (prop in obj && obj[prop] !== proto[prop] && !keys.contains(prop)) { + keys.push(prop); + } + } + } + + // Retrieve the names of an object's own properties. + // Delegates to **ECMAScript 5**'s native `Object.keys`. + function keys(obj) { + if (!isObject(obj)) return []; + if (nativeKeys) return nativeKeys(obj); + var keys = []; + for (var key in obj) if (has$1(obj, key)) keys.push(key); + // Ahem, IE < 9. + if (hasEnumBug) collectNonEnumProps(obj, keys); + return keys; + } + + // Is a given array, string, or object empty? + // An "empty" object has no enumerable own-properties. + function isEmpty(obj) { + if (obj == null) return true; + // Skip the more expensive `toString`-based type checks if `obj` has no + // `.length`. + var length = getLength(obj); + if (typeof length == 'number' && ( + isArray(obj) || isString(obj) || isArguments$1(obj) + )) return length === 0; + return getLength(keys(obj)) === 0; + } + + // Returns whether an object has a given set of `key:value` pairs. + function isMatch(object, attrs) { + var _keys = keys(attrs), length = _keys.length; + if (object == null) return !length; + var obj = Object(object); + for (var i = 0; i < length; i++) { + var key = _keys[i]; + if (attrs[key] !== obj[key] || !(key in obj)) return false; + } + return true; + } + + // If Underscore is called as a function, it returns a wrapped object that can + // be used OO-style. This wrapper holds altered versions of all functions added + // through `_.mixin`. Wrapped objects may be chained. + function _$1(obj) { + if (obj instanceof _$1) return obj; + if (!(this instanceof _$1)) return new _$1(obj); + this._wrapped = obj; + } + + _$1.VERSION = VERSION; + + // Extracts the result from a wrapped and chained object. + _$1.prototype.value = function() { + return this._wrapped; + }; + + // Provide unwrapping proxies for some methods used in engine operations + // such as arithmetic and JSON stringification. + _$1.prototype.valueOf = _$1.prototype.toJSON = _$1.prototype.value; + + _$1.prototype.toString = function() { + return String(this._wrapped); + }; + + // Internal function to wrap or shallow-copy an ArrayBuffer, + // typed array or DataView to a new view, reusing the buffer. + function toBufferView(bufferSource) { + return new Uint8Array( + bufferSource.buffer || bufferSource, + bufferSource.byteOffset || 0, + getByteLength(bufferSource) + ); + } + + // We use this string twice, so give it a name for minification. + var tagDataView = '[object DataView]'; + + // Internal recursive comparison function for `_.isEqual`. + function eq(a, b, aStack, bStack) { + // Identical objects are equal. `0 === -0`, but they aren't identical. + // See the [Harmony `egal` proposal](https://wiki.ecmascript.org/doku.php?id=harmony:egal). + if (a === b) return a !== 0 || 1 / a === 1 / b; + // `null` or `undefined` only equal to itself (strict comparison). + if (a == null || b == null) return false; + // `NaN`s are equivalent, but non-reflexive. + if (a !== a) return b !== b; + // Exhaust primitive checks + var type = typeof a; + if (type !== 'function' && type !== 'object' && typeof b != 'object') return false; + return deepEq(a, b, aStack, bStack); + } + + // Internal recursive comparison function for `_.isEqual`. + function deepEq(a, b, aStack, bStack) { + // Unwrap any wrapped objects. + if (a instanceof _$1) a = a._wrapped; + if (b instanceof _$1) b = b._wrapped; + // Compare `[[Class]]` names. + var className = toString.call(a); + if (className !== toString.call(b)) return false; + // Work around a bug in IE 10 - Edge 13. + if (hasStringTagBug && className == '[object Object]' && isDataView$1(a)) { + if (!isDataView$1(b)) return false; + className = tagDataView; + } + switch (className) { + // These types are compared by value. + case '[object RegExp]': + // RegExps are coerced to strings for comparison (Note: '' + /a/i === '/a/i') + case '[object String]': + // Primitives and their corresponding object wrappers are equivalent; thus, `"5"` is + // equivalent to `new String("5")`. + return '' + a === '' + b; + case '[object Number]': + // `NaN`s are equivalent, but non-reflexive. + // Object(NaN) is equivalent to NaN. + if (+a !== +a) return +b !== +b; + // An `egal` comparison is performed for other numeric values. + return +a === 0 ? 1 / +a === 1 / b : +a === +b; + case '[object Date]': + case '[object Boolean]': + // Coerce dates and booleans to numeric primitive values. Dates are compared by their + // millisecond representations. Note that invalid dates with millisecond representations + // of `NaN` are not equivalent. + return +a === +b; + case '[object Symbol]': + return SymbolProto.valueOf.call(a) === SymbolProto.valueOf.call(b); + case '[object ArrayBuffer]': + case tagDataView: + // Coerce to typed array so we can fall through. + return deepEq(toBufferView(a), toBufferView(b), aStack, bStack); + } + + var areArrays = className === '[object Array]'; + if (!areArrays && isTypedArray$1(a)) { + var byteLength = getByteLength(a); + if (byteLength !== getByteLength(b)) return false; + if (a.buffer === b.buffer && a.byteOffset === b.byteOffset) return true; + areArrays = true; + } + if (!areArrays) { + if (typeof a != 'object' || typeof b != 'object') return false; + + // Objects with different constructors are not equivalent, but `Object`s or `Array`s + // from different frames are. + var aCtor = a.constructor, bCtor = b.constructor; + if (aCtor !== bCtor && !(isFunction$1(aCtor) && aCtor instanceof aCtor && + isFunction$1(bCtor) && bCtor instanceof bCtor) + && ('constructor' in a && 'constructor' in b)) { + return false; + } + } + // Assume equality for cyclic structures. The algorithm for detecting cyclic + // structures is adapted from ES 5.1 section 15.12.3, abstract operation `JO`. + + // Initializing stack of traversed objects. + // It's done here since we only need them for objects and arrays comparison. + aStack = aStack || []; + bStack = bStack || []; + var length = aStack.length; + while (length--) { + // Linear search. Performance is inversely proportional to the number of + // unique nested structures. + if (aStack[length] === a) return bStack[length] === b; + } + + // Add the first object to the stack of traversed objects. + aStack.push(a); + bStack.push(b); + + // Recursively compare objects and arrays. + if (areArrays) { + // Compare array lengths to determine if a deep comparison is necessary. + length = a.length; + if (length !== b.length) return false; + // Deep compare the contents, ignoring non-numeric properties. + while (length--) { + if (!eq(a[length], b[length], aStack, bStack)) return false; + } + } else { + // Deep compare objects. + var _keys = keys(a), key; + length = _keys.length; + // Ensure that both objects contain the same number of properties before comparing deep equality. + if (keys(b).length !== length) return false; + while (length--) { + // Deep compare each member + key = _keys[length]; + if (!(has$1(b, key) && eq(a[key], b[key], aStack, bStack))) return false; + } + } + // Remove the first object from the stack of traversed objects. + aStack.pop(); + bStack.pop(); + return true; + } + + // Perform a deep comparison to check if two objects are equal. + function isEqual(a, b) { + return eq(a, b); + } + + // Retrieve all the enumerable property names of an object. + function allKeys(obj) { + if (!isObject(obj)) return []; + var keys = []; + for (var key in obj) keys.push(key); + // Ahem, IE < 9. + if (hasEnumBug) collectNonEnumProps(obj, keys); + return keys; + } + + // Since the regular `Object.prototype.toString` type tests don't work for + // some types in IE 11, we use a fingerprinting heuristic instead, based + // on the methods. It's not great, but it's the best we got. + // The fingerprint method lists are defined below. + function ie11fingerprint(methods) { + var length = getLength(methods); + return function(obj) { + if (obj == null) return false; + // `Map`, `WeakMap` and `Set` have no enumerable keys. + var keys = allKeys(obj); + if (getLength(keys)) return false; + for (var i = 0; i < length; i++) { + if (!isFunction$1(obj[methods[i]])) return false; + } + // If we are testing against `WeakMap`, we need to ensure that + // `obj` doesn't have a `forEach` method in order to distinguish + // it from a regular `Map`. + return methods !== weakMapMethods || !isFunction$1(obj[forEachName]); + }; + } + + // In the interest of compact minification, we write + // each string in the fingerprints only once. + var forEachName = 'forEach', + hasName = 'has', + commonInit = ['clear', 'delete'], + mapTail = ['get', hasName, 'set']; + + // `Map`, `WeakMap` and `Set` each have slightly different + // combinations of the above sublists. + var mapMethods = commonInit.concat(forEachName, mapTail), + weakMapMethods = commonInit.concat(mapTail), + setMethods = ['add'].concat(commonInit, forEachName, hasName); + + var isMap = isIE11 ? ie11fingerprint(mapMethods) : tagTester('Map'); + + var isWeakMap = isIE11 ? ie11fingerprint(weakMapMethods) : tagTester('WeakMap'); + + var isSet = isIE11 ? ie11fingerprint(setMethods) : tagTester('Set'); + + var isWeakSet = tagTester('WeakSet'); + + // Retrieve the values of an object's properties. + function values(obj) { + var _keys = keys(obj); + var length = _keys.length; + var values = Array(length); + for (var i = 0; i < length; i++) { + values[i] = obj[_keys[i]]; + } + return values; + } + + // Convert an object into a list of `[key, value]` pairs. + // The opposite of `_.object` with one argument. + function pairs(obj) { + var _keys = keys(obj); + var length = _keys.length; + var pairs = Array(length); + for (var i = 0; i < length; i++) { + pairs[i] = [_keys[i], obj[_keys[i]]]; + } + return pairs; + } + + // Invert the keys and values of an object. The values must be serializable. + function invert(obj) { + var result = {}; + var _keys = keys(obj); + for (var i = 0, length = _keys.length; i < length; i++) { + result[obj[_keys[i]]] = _keys[i]; + } + return result; + } + + // Return a sorted list of the function names available on the object. + function functions(obj) { + var names = []; + for (var key in obj) { + if (isFunction$1(obj[key])) names.push(key); + } + return names.sort(); + } + + // An internal function for creating assigner functions. + function createAssigner(keysFunc, defaults) { + return function(obj) { + var length = arguments.length; + if (defaults) obj = Object(obj); + if (length < 2 || obj == null) return obj; + for (var index = 1; index < length; index++) { + var source = arguments[index], + keys = keysFunc(source), + l = keys.length; + for (var i = 0; i < l; i++) { + var key = keys[i]; + if (!defaults || obj[key] === void 0) obj[key] = source[key]; + } + } + return obj; + }; + } + + // Extend a given object with all the properties in passed-in object(s). + var extend = createAssigner(allKeys); + + // Assigns a given object with all the own properties in the passed-in + // object(s). + // (https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object/assign) + var extendOwn = createAssigner(keys); + + // Fill in a given object with default properties. + var defaults = createAssigner(allKeys, true); + + // Create a naked function reference for surrogate-prototype-swapping. + function ctor() { + return function(){}; + } + + // An internal function for creating a new object that inherits from another. + function baseCreate(prototype) { + if (!isObject(prototype)) return {}; + if (nativeCreate) return nativeCreate(prototype); + var Ctor = ctor(); + Ctor.prototype = prototype; + var result = new Ctor; + Ctor.prototype = null; + return result; + } + + // Creates an object that inherits from the given prototype object. + // If additional properties are provided then they will be added to the + // created object. + function create(prototype, props) { + var result = baseCreate(prototype); + if (props) extendOwn(result, props); + return result; + } + + // Create a (shallow-cloned) duplicate of an object. + function clone(obj) { + if (!isObject(obj)) return obj; + return isArray(obj) ? obj.slice() : extend({}, obj); + } + + // Invokes `interceptor` with the `obj` and then returns `obj`. + // The primary purpose of this method is to "tap into" a method chain, in + // order to perform operations on intermediate results within the chain. + function tap(obj, interceptor) { + interceptor(obj); + return obj; + } + + // Normalize a (deep) property `path` to array. + // Like `_.iteratee`, this function can be customized. + function toPath$1(path) { + return isArray(path) ? path : [path]; + } + _$1.toPath = toPath$1; + + // Internal wrapper for `_.toPath` to enable minification. + // Similar to `cb` for `_.iteratee`. + function toPath(path) { + return _$1.toPath(path); + } + + // Internal function to obtain a nested property in `obj` along `path`. + function deepGet(obj, path) { + var length = path.length; + for (var i = 0; i < length; i++) { + if (obj == null) return void 0; + obj = obj[path[i]]; + } + return length ? obj : void 0; + } + + // Get the value of the (deep) property on `path` from `object`. + // If any property in `path` does not exist or if the value is + // `undefined`, return `defaultValue` instead. + // The `path` is normalized through `_.toPath`. + function get(object, path, defaultValue) { + var value = deepGet(object, toPath(path)); + return isUndefined(value) ? defaultValue : value; + } + + // Shortcut function for checking if an object has a given property directly on + // itself (in other words, not on a prototype). Unlike the internal `has` + // function, this public version can also traverse nested properties. + function has(obj, path) { + path = toPath(path); + var length = path.length; + for (var i = 0; i < length; i++) { + var key = path[i]; + if (!has$1(obj, key)) return false; + obj = obj[key]; + } + return !!length; + } + + // Keep the identity function around for default iteratees. + function identity(value) { + return value; + } + + // Returns a predicate for checking whether an object has a given set of + // `key:value` pairs. + function matcher(attrs) { + attrs = extendOwn({}, attrs); + return function(obj) { + return isMatch(obj, attrs); + }; + } + + // Creates a function that, when passed an object, will traverse that object’s + // properties down the given `path`, specified as an array of keys or indices. + function property(path) { + path = toPath(path); + return function(obj) { + return deepGet(obj, path); + }; + } + + // Internal function that returns an efficient (for current engines) version + // of the passed-in callback, to be repeatedly applied in other Underscore + // functions. + function optimizeCb(func, context, argCount) { + if (context === void 0) return func; + switch (argCount == null ? 3 : argCount) { + case 1: return function(value) { + return func.call(context, value); + }; + // The 2-argument case is omitted because we’re not using it. + case 3: return function(value, index, collection) { + return func.call(context, value, index, collection); + }; + case 4: return function(accumulator, value, index, collection) { + return func.call(context, accumulator, value, index, collection); + }; + } + return function() { + return func.apply(context, arguments); + }; + } + + // An internal function to generate callbacks that can be applied to each + // element in a collection, returning the desired result — either `_.identity`, + // an arbitrary callback, a property matcher, or a property accessor. + function baseIteratee(value, context, argCount) { + if (value == null) return identity; + if (isFunction$1(value)) return optimizeCb(value, context, argCount); + if (isObject(value) && !isArray(value)) return matcher(value); + return property(value); + } + + // External wrapper for our callback generator. Users may customize + // `_.iteratee` if they want additional predicate/iteratee shorthand styles. + // This abstraction hides the internal-only `argCount` argument. + function iteratee(value, context) { + return baseIteratee(value, context, Infinity); + } + _$1.iteratee = iteratee; + + // The function we call internally to generate a callback. It invokes + // `_.iteratee` if overridden, otherwise `baseIteratee`. + function cb(value, context, argCount) { + if (_$1.iteratee !== iteratee) return _$1.iteratee(value, context); + return baseIteratee(value, context, argCount); + } + + // Returns the results of applying the `iteratee` to each element of `obj`. + // In contrast to `_.map` it returns an object. + function mapObject(obj, iteratee, context) { + iteratee = cb(iteratee, context); + var _keys = keys(obj), + length = _keys.length, + results = {}; + for (var index = 0; index < length; index++) { + var currentKey = _keys[index]; + results[currentKey] = iteratee(obj[currentKey], currentKey, obj); + } + return results; + } + + // Predicate-generating function. Often useful outside of Underscore. + function noop(){} + + // Generates a function for a given object that returns a given property. + function propertyOf(obj) { + if (obj == null) return noop; + return function(path) { + return get(obj, path); + }; + } + + // Run a function **n** times. + function times(n, iteratee, context) { + var accum = Array(Math.max(0, n)); + iteratee = optimizeCb(iteratee, context, 1); + for (var i = 0; i < n; i++) accum[i] = iteratee(i); + return accum; + } + + // Return a random integer between `min` and `max` (inclusive). + function random(min, max) { + if (max == null) { + max = min; + min = 0; + } + return min + Math.floor(Math.random() * (max - min + 1)); + } + + // A (possibly faster) way to get the current timestamp as an integer. + var now = Date.now || function() { + return new Date().getTime(); + }; + + // Internal helper to generate functions for escaping and unescaping strings + // to/from HTML interpolation. + function createEscaper(map) { + var escaper = function(match) { + return map[match]; + }; + // Regexes for identifying a key that needs to be escaped. + var source = '(?:' + keys(map).join('|') + ')'; + var testRegexp = RegExp(source); + var replaceRegexp = RegExp(source, 'g'); + return function(string) { + string = string == null ? '' : '' + string; + return testRegexp.test(string) ? string.replace(replaceRegexp, escaper) : string; + }; + } + + // Internal list of HTML entities for escaping. + var escapeMap = { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + "'": ''', + '`': '`' + }; + + // Function for escaping strings to HTML interpolation. + var _escape = createEscaper(escapeMap); + + // Internal list of HTML entities for unescaping. + var unescapeMap = invert(escapeMap); + + // Function for unescaping strings from HTML interpolation. + var _unescape = createEscaper(unescapeMap); + + // By default, Underscore uses ERB-style template delimiters. Change the + // following template settings to use alternative delimiters. + var templateSettings = _$1.templateSettings = { + evaluate: /<%([\s\S]+?)%>/g, + interpolate: /<%=([\s\S]+?)%>/g, + escape: /<%-([\s\S]+?)%>/g + }; + + // When customizing `_.templateSettings`, if you don't want to define an + // interpolation, evaluation or escaping regex, we need one that is + // guaranteed not to match. + var noMatch = /(.)^/; + + // Certain characters need to be escaped so that they can be put into a + // string literal. + var escapes = { + "'": "'", + '\\': '\\', + '\r': 'r', + '\n': 'n', + '\u2028': 'u2028', + '\u2029': 'u2029' + }; + + var escapeRegExp = /\\|'|\r|\n|\u2028|\u2029/g; + + function escapeChar(match) { + return '\\' + escapes[match]; + } + + // In order to prevent third-party code injection through + // `_.templateSettings.variable`, we test it against the following regular + // expression. It is intentionally a bit more liberal than just matching valid + // identifiers, but still prevents possible loopholes through defaults or + // destructuring assignment. + var bareIdentifier = /^\s*(\w|\$)+\s*$/; + + // JavaScript micro-templating, similar to John Resig's implementation. + // Underscore templating handles arbitrary delimiters, preserves whitespace, + // and correctly escapes quotes within interpolated code. + // NB: `oldSettings` only exists for backwards compatibility. + function template(text, settings, oldSettings) { + if (!settings && oldSettings) settings = oldSettings; + settings = defaults({}, settings, _$1.templateSettings); + + // Combine delimiters into one regular expression via alternation. + var matcher = RegExp([ + (settings.escape || noMatch).source, + (settings.interpolate || noMatch).source, + (settings.evaluate || noMatch).source + ].join('|') + '|$', 'g'); + + // Compile the template source, escaping string literals appropriately. + var index = 0; + var source = "__p+='"; + text.replace(matcher, function(match, escape, interpolate, evaluate, offset) { + source += text.slice(index, offset).replace(escapeRegExp, escapeChar); + index = offset + match.length; + + if (escape) { + source += "'+\n((__t=(" + escape + "))==null?'':_.escape(__t))+\n'"; + } else if (interpolate) { + source += "'+\n((__t=(" + interpolate + "))==null?'':__t)+\n'"; + } else if (evaluate) { + source += "';\n" + evaluate + "\n__p+='"; + } + + // Adobe VMs need the match returned to produce the correct offset. + return match; + }); + source += "';\n"; + + var argument = settings.variable; + if (argument) { + // Insure against third-party code injection. (CVE-2021-23358) + if (!bareIdentifier.test(argument)) throw new Error( + 'variable is not a bare identifier: ' + argument + ); + } else { + // If a variable is not specified, place data values in local scope. + source = 'with(obj||{}){\n' + source + '}\n'; + argument = 'obj'; + } + + source = "var __t,__p='',__j=Array.prototype.join," + + "print=function(){__p+=__j.call(arguments,'');};\n" + + source + 'return __p;\n'; + + var render; + try { + render = new Function(argument, '_', source); + } catch (e) { + e.source = source; + throw e; + } + + var template = function(data) { + return render.call(this, data, _$1); + }; + + // Provide the compiled source as a convenience for precompilation. + template.source = 'function(' + argument + '){\n' + source + '}'; + + return template; + } + + // Traverses the children of `obj` along `path`. If a child is a function, it + // is invoked with its parent as context. Returns the value of the final + // child, or `fallback` if any child is undefined. + function result(obj, path, fallback) { + path = toPath(path); + var length = path.length; + if (!length) { + return isFunction$1(fallback) ? fallback.call(obj) : fallback; + } + for (var i = 0; i < length; i++) { + var prop = obj == null ? void 0 : obj[path[i]]; + if (prop === void 0) { + prop = fallback; + i = length; // Ensure we don't continue iterating. + } + obj = isFunction$1(prop) ? prop.call(obj) : prop; + } + return obj; + } + + // Generate a unique integer id (unique within the entire client session). + // Useful for temporary DOM ids. + var idCounter = 0; + function uniqueId(prefix) { + var id = ++idCounter + ''; + return prefix ? prefix + id : id; + } + + // Start chaining a wrapped Underscore object. + function chain(obj) { + var instance = _$1(obj); + instance._chain = true; + return instance; + } + + // Internal function to execute `sourceFunc` bound to `context` with optional + // `args`. Determines whether to execute a function as a constructor or as a + // normal function. + function executeBound(sourceFunc, boundFunc, context, callingContext, args) { + if (!(callingContext instanceof boundFunc)) return sourceFunc.apply(context, args); + var self = baseCreate(sourceFunc.prototype); + var result = sourceFunc.apply(self, args); + if (isObject(result)) return result; + return self; + } + + // Partially apply a function by creating a version that has had some of its + // arguments pre-filled, without changing its dynamic `this` context. `_` acts + // as a placeholder by default, allowing any combination of arguments to be + // pre-filled. Set `_.partial.placeholder` for a custom placeholder argument. + var partial = restArguments(function(func, boundArgs) { + var placeholder = partial.placeholder; + var bound = function() { + var position = 0, length = boundArgs.length; + var args = Array(length); + for (var i = 0; i < length; i++) { + args[i] = boundArgs[i] === placeholder ? arguments[position++] : boundArgs[i]; + } + while (position < arguments.length) args.push(arguments[position++]); + return executeBound(func, bound, this, this, args); + }; + return bound; + }); + + partial.placeholder = _$1; + + // Create a function bound to a given object (assigning `this`, and arguments, + // optionally). + var bind = restArguments(function(func, context, args) { + if (!isFunction$1(func)) throw new TypeError('Bind must be called on a function'); + var bound = restArguments(function(callArgs) { + return executeBound(func, bound, context, this, args.concat(callArgs)); + }); + return bound; + }); + + // Internal helper for collection methods to determine whether a collection + // should be iterated as an array or as an object. + // Related: https://people.mozilla.org/~jorendorff/es6-draft.html#sec-tolength + // Avoids a very nasty iOS 8 JIT bug on ARM-64. #2094 + var isArrayLike = createSizePropertyCheck(getLength); + + // Internal implementation of a recursive `flatten` function. + function flatten$1(input, depth, strict, output) { + output = output || []; + if (!depth && depth !== 0) { + depth = Infinity; + } else if (depth <= 0) { + return output.concat(input); + } + var idx = output.length; + for (var i = 0, length = getLength(input); i < length; i++) { + var value = input[i]; + if (isArrayLike(value) && (isArray(value) || isArguments$1(value))) { + // Flatten current level of array or arguments object. + if (depth > 1) { + flatten$1(value, depth - 1, strict, output); + idx = output.length; + } else { + var j = 0, len = value.length; + while (j < len) output[idx++] = value[j++]; + } + } else if (!strict) { + output[idx++] = value; + } + } + return output; + } + + // Bind a number of an object's methods to that object. Remaining arguments + // are the method names to be bound. Useful for ensuring that all callbacks + // defined on an object belong to it. + var bindAll = restArguments(function(obj, keys) { + keys = flatten$1(keys, false, false); + var index = keys.length; + if (index < 1) throw new Error('bindAll must be passed function names'); + while (index--) { + var key = keys[index]; + obj[key] = bind(obj[key], obj); + } + return obj; + }); + + // Memoize an expensive function by storing its results. + function memoize(func, hasher) { + var memoize = function(key) { + var cache = memoize.cache; + var address = '' + (hasher ? hasher.apply(this, arguments) : key); + if (!has$1(cache, address)) cache[address] = func.apply(this, arguments); + return cache[address]; + }; + memoize.cache = {}; + return memoize; + } + + // Delays a function for the given number of milliseconds, and then calls + // it with the arguments supplied. + var delay = restArguments(function(func, wait, args) { + return setTimeout(function() { + return func.apply(null, args); + }, wait); + }); + + // Defers a function, scheduling it to run after the current call stack has + // cleared. + var defer = partial(delay, _$1, 1); + + // Returns a function, that, when invoked, will only be triggered at most once + // during a given window of time. Normally, the throttled function will run + // as much as it can, without ever going more than once per `wait` duration; + // but if you'd like to disable the execution on the leading edge, pass + // `{leading: false}`. To disable execution on the trailing edge, ditto. + function throttle(func, wait, options) { + var timeout, context, args, result; + var previous = 0; + if (!options) options = {}; + + var later = function() { + previous = options.leading === false ? 0 : now(); + timeout = null; + result = func.apply(context, args); + if (!timeout) context = args = null; + }; + + var throttled = function() { + var _now = now(); + if (!previous && options.leading === false) previous = _now; + var remaining = wait - (_now - previous); + context = this; + args = arguments; + if (remaining <= 0 || remaining > wait) { + if (timeout) { + clearTimeout(timeout); + timeout = null; + } + previous = _now; + result = func.apply(context, args); + if (!timeout) context = args = null; + } else if (!timeout && options.trailing !== false) { + timeout = setTimeout(later, remaining); + } + return result; + }; + + throttled.cancel = function() { + clearTimeout(timeout); + previous = 0; + timeout = context = args = null; + }; + + return throttled; + } + + // When a sequence of calls of the returned function ends, the argument + // function is triggered. The end of a sequence is defined by the `wait` + // parameter. If `immediate` is passed, the argument function will be + // triggered at the beginning of the sequence instead of at the end. + function debounce(func, wait, immediate) { + var timeout, previous, args, result, context; + + var later = function() { + var passed = now() - previous; + if (wait > passed) { + timeout = setTimeout(later, wait - passed); + } else { + timeout = null; + if (!immediate) result = func.apply(context, args); + // This check is needed because `func` can recursively invoke `debounced`. + if (!timeout) args = context = null; + } + }; + + var debounced = restArguments(function(_args) { + context = this; + args = _args; + previous = now(); + if (!timeout) { + timeout = setTimeout(later, wait); + if (immediate) result = func.apply(context, args); + } + return result; + }); + + debounced.cancel = function() { + clearTimeout(timeout); + timeout = args = context = null; + }; + + return debounced; + } + + // Returns the first function passed as an argument to the second, + // allowing you to adjust arguments, run code before and after, and + // conditionally execute the original function. + function wrap(func, wrapper) { + return partial(wrapper, func); + } + + // Returns a negated version of the passed-in predicate. + function negate(predicate) { + return function() { + return !predicate.apply(this, arguments); + }; + } + + // Returns a function that is the composition of a list of functions, each + // consuming the return value of the function that follows. + function compose() { + var args = arguments; + var start = args.length - 1; + return function() { + var i = start; + var result = args[start].apply(this, arguments); + while (i--) result = args[i].call(this, result); + return result; + }; + } + + // Returns a function that will only be executed on and after the Nth call. + function after(times, func) { + return function() { + if (--times < 1) { + return func.apply(this, arguments); + } + }; + } + + // Returns a function that will only be executed up to (but not including) the + // Nth call. + function before(times, func) { + var memo; + return function() { + if (--times > 0) { + memo = func.apply(this, arguments); + } + if (times <= 1) func = null; + return memo; + }; + } + + // Returns a function that will be executed at most one time, no matter how + // often you call it. Useful for lazy initialization. + var once = partial(before, 2); + + // Returns the first key on an object that passes a truth test. + function findKey(obj, predicate, context) { + predicate = cb(predicate, context); + var _keys = keys(obj), key; + for (var i = 0, length = _keys.length; i < length; i++) { + key = _keys[i]; + if (predicate(obj[key], key, obj)) return key; + } + } + + // Internal function to generate `_.findIndex` and `_.findLastIndex`. + function createPredicateIndexFinder(dir) { + return function(array, predicate, context) { + predicate = cb(predicate, context); + var length = getLength(array); + var index = dir > 0 ? 0 : length - 1; + for (; index >= 0 && index < length; index += dir) { + if (predicate(array[index], index, array)) return index; + } + return -1; + }; + } + + // Returns the first index on an array-like that passes a truth test. + var findIndex = createPredicateIndexFinder(1); + + // Returns the last index on an array-like that passes a truth test. + var findLastIndex = createPredicateIndexFinder(-1); + + // Use a comparator function to figure out the smallest index at which + // an object should be inserted so as to maintain order. Uses binary search. + function sortedIndex(array, obj, iteratee, context) { + iteratee = cb(iteratee, context, 1); + var value = iteratee(obj); + var low = 0, high = getLength(array); + while (low < high) { + var mid = Math.floor((low + high) / 2); + if (iteratee(array[mid]) < value) low = mid + 1; else high = mid; + } + return low; + } + + // Internal function to generate the `_.indexOf` and `_.lastIndexOf` functions. + function createIndexFinder(dir, predicateFind, sortedIndex) { + return function(array, item, idx) { + var i = 0, length = getLength(array); + if (typeof idx == 'number') { + if (dir > 0) { + i = idx >= 0 ? idx : Math.max(idx + length, i); + } else { + length = idx >= 0 ? Math.min(idx + 1, length) : idx + length + 1; + } + } else if (sortedIndex && idx && length) { + idx = sortedIndex(array, item); + return array[idx] === item ? idx : -1; + } + if (item !== item) { + idx = predicateFind(slice.call(array, i, length), isNaN$1); + return idx >= 0 ? idx + i : -1; + } + for (idx = dir > 0 ? i : length - 1; idx >= 0 && idx < length; idx += dir) { + if (array[idx] === item) return idx; + } + return -1; + }; + } + + // Return the position of the first occurrence of an item in an array, + // or -1 if the item is not included in the array. + // If the array is large and already in sort order, pass `true` + // for **isSorted** to use binary search. + var indexOf = createIndexFinder(1, findIndex, sortedIndex); + + // Return the position of the last occurrence of an item in an array, + // or -1 if the item is not included in the array. + var lastIndexOf = createIndexFinder(-1, findLastIndex); + + // Return the first value which passes a truth test. + function find(obj, predicate, context) { + var keyFinder = isArrayLike(obj) ? findIndex : findKey; + var key = keyFinder(obj, predicate, context); + if (key !== void 0 && key !== -1) return obj[key]; + } + + // Convenience version of a common use case of `_.find`: getting the first + // object containing specific `key:value` pairs. + function findWhere(obj, attrs) { + return find(obj, matcher(attrs)); + } + + // The cornerstone for collection functions, an `each` + // implementation, aka `forEach`. + // Handles raw objects in addition to array-likes. Treats all + // sparse array-likes as if they were dense. + function each(obj, iteratee, context) { + iteratee = optimizeCb(iteratee, context); + var i, length; + if (isArrayLike(obj)) { + for (i = 0, length = obj.length; i < length; i++) { + iteratee(obj[i], i, obj); + } + } else { + var _keys = keys(obj); + for (i = 0, length = _keys.length; i < length; i++) { + iteratee(obj[_keys[i]], _keys[i], obj); + } + } + return obj; + } + + // Return the results of applying the iteratee to each element. + function map(obj, iteratee, context) { + iteratee = cb(iteratee, context); + var _keys = !isArrayLike(obj) && keys(obj), + length = (_keys || obj).length, + results = Array(length); + for (var index = 0; index < length; index++) { + var currentKey = _keys ? _keys[index] : index; + results[index] = iteratee(obj[currentKey], currentKey, obj); + } + return results; + } + + // Internal helper to create a reducing function, iterating left or right. + function createReduce(dir) { + // Wrap code that reassigns argument variables in a separate function than + // the one that accesses `arguments.length` to avoid a perf hit. (#1991) + var reducer = function(obj, iteratee, memo, initial) { + var _keys = !isArrayLike(obj) && keys(obj), + length = (_keys || obj).length, + index = dir > 0 ? 0 : length - 1; + if (!initial) { + memo = obj[_keys ? _keys[index] : index]; + index += dir; + } + for (; index >= 0 && index < length; index += dir) { + var currentKey = _keys ? _keys[index] : index; + memo = iteratee(memo, obj[currentKey], currentKey, obj); + } + return memo; + }; + + return function(obj, iteratee, memo, context) { + var initial = arguments.length >= 3; + return reducer(obj, optimizeCb(iteratee, context, 4), memo, initial); + }; + } + + // **Reduce** builds up a single result from a list of values, aka `inject`, + // or `foldl`. + var reduce = createReduce(1); + + // The right-associative version of reduce, also known as `foldr`. + var reduceRight = createReduce(-1); + + // Return all the elements that pass a truth test. + function filter(obj, predicate, context) { + var results = []; + predicate = cb(predicate, context); + each(obj, function(value, index, list) { + if (predicate(value, index, list)) results.push(value); + }); + return results; + } + + // Return all the elements for which a truth test fails. + function reject(obj, predicate, context) { + return filter(obj, negate(cb(predicate)), context); + } + + // Determine whether all of the elements pass a truth test. + function every(obj, predicate, context) { + predicate = cb(predicate, context); + var _keys = !isArrayLike(obj) && keys(obj), + length = (_keys || obj).length; + for (var index = 0; index < length; index++) { + var currentKey = _keys ? _keys[index] : index; + if (!predicate(obj[currentKey], currentKey, obj)) return false; + } + return true; + } + + // Determine if at least one element in the object passes a truth test. + function some(obj, predicate, context) { + predicate = cb(predicate, context); + var _keys = !isArrayLike(obj) && keys(obj), + length = (_keys || obj).length; + for (var index = 0; index < length; index++) { + var currentKey = _keys ? _keys[index] : index; + if (predicate(obj[currentKey], currentKey, obj)) return true; + } + return false; + } + + // Determine if the array or object contains a given item (using `===`). + function contains(obj, item, fromIndex, guard) { + if (!isArrayLike(obj)) obj = values(obj); + if (typeof fromIndex != 'number' || guard) fromIndex = 0; + return indexOf(obj, item, fromIndex) >= 0; + } + + // Invoke a method (with arguments) on every item in a collection. + var invoke = restArguments(function(obj, path, args) { + var contextPath, func; + if (isFunction$1(path)) { + func = path; + } else { + path = toPath(path); + contextPath = path.slice(0, -1); + path = path[path.length - 1]; + } + return map(obj, function(context) { + var method = func; + if (!method) { + if (contextPath && contextPath.length) { + context = deepGet(context, contextPath); + } + if (context == null) return void 0; + method = context[path]; + } + return method == null ? method : method.apply(context, args); + }); + }); + + // Convenience version of a common use case of `_.map`: fetching a property. + function pluck(obj, key) { + return map(obj, property(key)); + } + + // Convenience version of a common use case of `_.filter`: selecting only + // objects containing specific `key:value` pairs. + function where(obj, attrs) { + return filter(obj, matcher(attrs)); + } + + // Return the maximum element (or element-based computation). + function max(obj, iteratee, context) { + var result = -Infinity, lastComputed = -Infinity, + value, computed; + if (iteratee == null || (typeof iteratee == 'number' && typeof obj[0] != 'object' && obj != null)) { + obj = isArrayLike(obj) ? obj : values(obj); + for (var i = 0, length = obj.length; i < length; i++) { + value = obj[i]; + if (value != null && value > result) { + result = value; + } + } + } else { + iteratee = cb(iteratee, context); + each(obj, function(v, index, list) { + computed = iteratee(v, index, list); + if (computed > lastComputed || (computed === -Infinity && result === -Infinity)) { + result = v; + lastComputed = computed; + } + }); + } + return result; + } + + // Return the minimum element (or element-based computation). + function min(obj, iteratee, context) { + var result = Infinity, lastComputed = Infinity, + value, computed; + if (iteratee == null || (typeof iteratee == 'number' && typeof obj[0] != 'object' && obj != null)) { + obj = isArrayLike(obj) ? obj : values(obj); + for (var i = 0, length = obj.length; i < length; i++) { + value = obj[i]; + if (value != null && value < result) { + result = value; + } + } + } else { + iteratee = cb(iteratee, context); + each(obj, function(v, index, list) { + computed = iteratee(v, index, list); + if (computed < lastComputed || (computed === Infinity && result === Infinity)) { + result = v; + lastComputed = computed; + } + }); + } + return result; + } + + // Safely create a real, live array from anything iterable. + var reStrSymbol = /[^\ud800-\udfff]|[\ud800-\udbff][\udc00-\udfff]|[\ud800-\udfff]/g; + function toArray(obj) { + if (!obj) return []; + if (isArray(obj)) return slice.call(obj); + if (isString(obj)) { + // Keep surrogate pair characters together. + return obj.match(reStrSymbol); + } + if (isArrayLike(obj)) return map(obj, identity); + return values(obj); + } + + // Sample **n** random values from a collection using the modern version of the + // [Fisher-Yates shuffle](https://en.wikipedia.org/wiki/Fisher–Yates_shuffle). + // If **n** is not specified, returns a single random element. + // The internal `guard` argument allows it to work with `_.map`. + function sample(obj, n, guard) { + if (n == null || guard) { + if (!isArrayLike(obj)) obj = values(obj); + return obj[random(obj.length - 1)]; + } + var sample = toArray(obj); + var length = getLength(sample); + n = Math.max(Math.min(n, length), 0); + var last = length - 1; + for (var index = 0; index < n; index++) { + var rand = random(index, last); + var temp = sample[index]; + sample[index] = sample[rand]; + sample[rand] = temp; + } + return sample.slice(0, n); + } + + // Shuffle a collection. + function shuffle(obj) { + return sample(obj, Infinity); + } + + // Sort the object's values by a criterion produced by an iteratee. + function sortBy(obj, iteratee, context) { + var index = 0; + iteratee = cb(iteratee, context); + return pluck(map(obj, function(value, key, list) { + return { + value: value, + index: index++, + criteria: iteratee(value, key, list) + }; + }).sort(function(left, right) { + var a = left.criteria; + var b = right.criteria; + if (a !== b) { + if (a > b || a === void 0) return 1; + if (a < b || b === void 0) return -1; + } + return left.index - right.index; + }), 'value'); + } + + // An internal function used for aggregate "group by" operations. + function group(behavior, partition) { + return function(obj, iteratee, context) { + var result = partition ? [[], []] : {}; + iteratee = cb(iteratee, context); + each(obj, function(value, index) { + var key = iteratee(value, index, obj); + behavior(result, value, key); + }); + return result; + }; + } + + // Groups the object's values by a criterion. Pass either a string attribute + // to group by, or a function that returns the criterion. + var groupBy = group(function(result, value, key) { + if (has$1(result, key)) result[key].push(value); else result[key] = [value]; + }); + + // Indexes the object's values by a criterion, similar to `_.groupBy`, but for + // when you know that your index values will be unique. + var indexBy = group(function(result, value, key) { + result[key] = value; + }); + + // Counts instances of an object that group by a certain criterion. Pass + // either a string attribute to count by, or a function that returns the + // criterion. + var countBy = group(function(result, value, key) { + if (has$1(result, key)) result[key]++; else result[key] = 1; + }); + + // Split a collection into two arrays: one whose elements all pass the given + // truth test, and one whose elements all do not pass the truth test. + var partition = group(function(result, value, pass) { + result[pass ? 0 : 1].push(value); + }, true); + + // Return the number of elements in a collection. + function size(obj) { + if (obj == null) return 0; + return isArrayLike(obj) ? obj.length : keys(obj).length; + } + + // Internal `_.pick` helper function to determine whether `key` is an enumerable + // property name of `obj`. + function keyInObj(value, key, obj) { + return key in obj; + } + + // Return a copy of the object only containing the allowed properties. + var pick = restArguments(function(obj, keys) { + var result = {}, iteratee = keys[0]; + if (obj == null) return result; + if (isFunction$1(iteratee)) { + if (keys.length > 1) iteratee = optimizeCb(iteratee, keys[1]); + keys = allKeys(obj); + } else { + iteratee = keyInObj; + keys = flatten$1(keys, false, false); + obj = Object(obj); + } + for (var i = 0, length = keys.length; i < length; i++) { + var key = keys[i]; + var value = obj[key]; + if (iteratee(value, key, obj)) result[key] = value; + } + return result; + }); + + // Return a copy of the object without the disallowed properties. + var omit = restArguments(function(obj, keys) { + var iteratee = keys[0], context; + if (isFunction$1(iteratee)) { + iteratee = negate(iteratee); + if (keys.length > 1) context = keys[1]; + } else { + keys = map(flatten$1(keys, false, false), String); + iteratee = function(value, key) { + return !contains(keys, key); + }; + } + return pick(obj, iteratee, context); + }); + + // Returns everything but the last entry of the array. Especially useful on + // the arguments object. Passing **n** will return all the values in + // the array, excluding the last N. + function initial(array, n, guard) { + return slice.call(array, 0, Math.max(0, array.length - (n == null || guard ? 1 : n))); + } + + // Get the first element of an array. Passing **n** will return the first N + // values in the array. The **guard** check allows it to work with `_.map`. + function first(array, n, guard) { + if (array == null || array.length < 1) return n == null || guard ? void 0 : []; + if (n == null || guard) return array[0]; + return initial(array, array.length - n); + } + + // Returns everything but the first entry of the `array`. Especially useful on + // the `arguments` object. Passing an **n** will return the rest N values in the + // `array`. + function rest(array, n, guard) { + return slice.call(array, n == null || guard ? 1 : n); + } + + // Get the last element of an array. Passing **n** will return the last N + // values in the array. + function last(array, n, guard) { + if (array == null || array.length < 1) return n == null || guard ? void 0 : []; + if (n == null || guard) return array[array.length - 1]; + return rest(array, Math.max(0, array.length - n)); + } + + // Trim out all falsy values from an array. + function compact(array) { + return filter(array, Boolean); + } + + // Flatten out an array, either recursively (by default), or up to `depth`. + // Passing `true` or `false` as `depth` means `1` or `Infinity`, respectively. + function flatten(array, depth) { + return flatten$1(array, depth, false); + } + + // Take the difference between one array and a number of other arrays. + // Only the elements present in just the first array will remain. + var difference = restArguments(function(array, rest) { + rest = flatten$1(rest, true, true); + return filter(array, function(value){ + return !contains(rest, value); + }); + }); + + // Return a version of the array that does not contain the specified value(s). + var without = restArguments(function(array, otherArrays) { + return difference(array, otherArrays); + }); + + // Produce a duplicate-free version of the array. If the array has already + // been sorted, you have the option of using a faster algorithm. + // The faster algorithm will not work with an iteratee if the iteratee + // is not a one-to-one function, so providing an iteratee will disable + // the faster algorithm. + function uniq(array, isSorted, iteratee, context) { + if (!isBoolean(isSorted)) { + context = iteratee; + iteratee = isSorted; + isSorted = false; + } + if (iteratee != null) iteratee = cb(iteratee, context); + var result = []; + var seen = []; + for (var i = 0, length = getLength(array); i < length; i++) { + var value = array[i], + computed = iteratee ? iteratee(value, i, array) : value; + if (isSorted && !iteratee) { + if (!i || seen !== computed) result.push(value); + seen = computed; + } else if (iteratee) { + if (!contains(seen, computed)) { + seen.push(computed); + result.push(value); + } + } else if (!contains(result, value)) { + result.push(value); + } + } + return result; + } + + // Produce an array that contains the union: each distinct element from all of + // the passed-in arrays. + var union = restArguments(function(arrays) { + return uniq(flatten$1(arrays, true, true)); + }); + + // Produce an array that contains every item shared between all the + // passed-in arrays. + function intersection(array) { + var result = []; + var argsLength = arguments.length; + for (var i = 0, length = getLength(array); i < length; i++) { + var item = array[i]; + if (contains(result, item)) continue; + var j; + for (j = 1; j < argsLength; j++) { + if (!contains(arguments[j], item)) break; + } + if (j === argsLength) result.push(item); + } + return result; + } + + // Complement of zip. Unzip accepts an array of arrays and groups + // each array's elements on shared indices. + function unzip(array) { + var length = (array && max(array, getLength).length) || 0; + var result = Array(length); + + for (var index = 0; index < length; index++) { + result[index] = pluck(array, index); + } + return result; + } + + // Zip together multiple lists into a single array -- elements that share + // an index go together. + var zip = restArguments(unzip); + + // Converts lists into objects. Pass either a single array of `[key, value]` + // pairs, or two parallel arrays of the same length -- one of keys, and one of + // the corresponding values. Passing by pairs is the reverse of `_.pairs`. + function object(list, values) { + var result = {}; + for (var i = 0, length = getLength(list); i < length; i++) { + if (values) { + result[list[i]] = values[i]; + } else { + result[list[i][0]] = list[i][1]; + } + } + return result; + } + + // Generate an integer Array containing an arithmetic progression. A port of + // the native Python `range()` function. See + // [the Python documentation](https://docs.python.org/library/functions.html#range). + function range(start, stop, step) { + if (stop == null) { + stop = start || 0; + start = 0; + } + if (!step) { + step = stop < start ? -1 : 1; + } + + var length = Math.max(Math.ceil((stop - start) / step), 0); + var range = Array(length); + + for (var idx = 0; idx < length; idx++, start += step) { + range[idx] = start; + } + + return range; + } + + // Chunk a single array into multiple arrays, each containing `count` or fewer + // items. + function chunk(array, count) { + if (count == null || count < 1) return []; + var result = []; + var i = 0, length = array.length; + while (i < length) { + result.push(slice.call(array, i, i += count)); + } + return result; + } + + // Helper function to continue chaining intermediate results. + function chainResult(instance, obj) { + return instance._chain ? _$1(obj).chain() : obj; + } + + // Add your own custom functions to the Underscore object. + function mixin(obj) { + each(functions(obj), function(name) { + var func = _$1[name] = obj[name]; + _$1.prototype[name] = function() { + var args = [this._wrapped]; + push.apply(args, arguments); + return chainResult(this, func.apply(_$1, args)); + }; + }); + return _$1; + } + + // Add all mutator `Array` functions to the wrapper. + each(['pop', 'push', 'reverse', 'shift', 'sort', 'splice', 'unshift'], function(name) { + var method = ArrayProto[name]; + _$1.prototype[name] = function() { + var obj = this._wrapped; + if (obj != null) { + method.apply(obj, arguments); + if ((name === 'shift' || name === 'splice') && obj.length === 0) { + delete obj[0]; + } + } + return chainResult(this, obj); + }; + }); + + // Add all accessor `Array` functions to the wrapper. + each(['concat', 'join', 'slice'], function(name) { + var method = ArrayProto[name]; + _$1.prototype[name] = function() { + var obj = this._wrapped; + if (obj != null) obj = method.apply(obj, arguments); + return chainResult(this, obj); + }; + }); + + // Named Exports + + var allExports = { + __proto__: null, + VERSION: VERSION, + restArguments: restArguments, + isObject: isObject, + isNull: isNull, + isUndefined: isUndefined, + isBoolean: isBoolean, + isElement: isElement, + isString: isString, + isNumber: isNumber, + isDate: isDate, + isRegExp: isRegExp, + isError: isError, + isSymbol: isSymbol, + isArrayBuffer: isArrayBuffer, + isDataView: isDataView$1, + isArray: isArray, + isFunction: isFunction$1, + isArguments: isArguments$1, + isFinite: isFinite$1, + isNaN: isNaN$1, + isTypedArray: isTypedArray$1, + isEmpty: isEmpty, + isMatch: isMatch, + isEqual: isEqual, + isMap: isMap, + isWeakMap: isWeakMap, + isSet: isSet, + isWeakSet: isWeakSet, + keys: keys, + allKeys: allKeys, + values: values, + pairs: pairs, + invert: invert, + functions: functions, + methods: functions, + extend: extend, + extendOwn: extendOwn, + assign: extendOwn, + defaults: defaults, + create: create, + clone: clone, + tap: tap, + get: get, + has: has, + mapObject: mapObject, + identity: identity, + constant: constant, + noop: noop, + toPath: toPath$1, + property: property, + propertyOf: propertyOf, + matcher: matcher, + matches: matcher, + times: times, + random: random, + now: now, + escape: _escape, + unescape: _unescape, + templateSettings: templateSettings, + template: template, + result: result, + uniqueId: uniqueId, + chain: chain, + iteratee: iteratee, + partial: partial, + bind: bind, + bindAll: bindAll, + memoize: memoize, + delay: delay, + defer: defer, + throttle: throttle, + debounce: debounce, + wrap: wrap, + negate: negate, + compose: compose, + after: after, + before: before, + once: once, + findKey: findKey, + findIndex: findIndex, + findLastIndex: findLastIndex, + sortedIndex: sortedIndex, + indexOf: indexOf, + lastIndexOf: lastIndexOf, + find: find, + detect: find, + findWhere: findWhere, + each: each, + forEach: each, + map: map, + collect: map, + reduce: reduce, + foldl: reduce, + inject: reduce, + reduceRight: reduceRight, + foldr: reduceRight, + filter: filter, + select: filter, + reject: reject, + every: every, + all: every, + some: some, + any: some, + contains: contains, + includes: contains, + include: contains, + invoke: invoke, + pluck: pluck, + where: where, + max: max, + min: min, + shuffle: shuffle, + sample: sample, + sortBy: sortBy, + groupBy: groupBy, + indexBy: indexBy, + countBy: countBy, + partition: partition, + toArray: toArray, + size: size, + pick: pick, + omit: omit, + first: first, + head: first, + take: first, + initial: initial, + last: last, + rest: rest, + tail: rest, + drop: rest, + compact: compact, + flatten: flatten, + without: without, + uniq: uniq, + unique: uniq, + union: union, + intersection: intersection, + difference: difference, + unzip: unzip, + transpose: unzip, + zip: zip, + object: object, + range: range, + chunk: chunk, + mixin: mixin, + 'default': _$1 + }; + + // Default Export + + // Add all of the Underscore functions to the wrapper object. + var _ = mixin(allExports); + // Legacy Node.js API. + _._ = _; + + return _; + +}))); +//# sourceMappingURL=underscore-umd.js.map diff --git a/js/v8_runner.go b/js/v8_runner.go new file mode 100644 index 0000000..ebb0cb4 --- /dev/null +++ b/js/v8_runner.go @@ -0,0 +1,272 @@ +//go:build cb_sg_v8 + +/* +Copyright 2022-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package js + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/pkg/errors" + v8 "github.com/snej/v8go" // Docs: https://pkg.go.dev/github.com/snej/v8go +) + +type V8Runner struct { + baseRunner + v8vm *v8VM + template V8Template // The Service template I'm created from + ctx *v8.Context // V8 object managing this execution context + mainFn *v8.Function // The entry-point function (returned by the Service's script) + Client any // You can put whatever you want here, to point back to your state +} + +func newV8Runner(vm *v8VM, template V8Template, id serviceID) (*V8Runner, error) { + // Create a V8 Context and run the setup script in it: + ctx := v8.NewContext(vm.iso, template.Global()) + if _, err := vm.setupScript.Run(ctx); err != nil { + return nil, errors.Wrap(err, "Unexpected error in JavaScript initialization code") + } + + // Now run the service's script, which returns the service's main function: + result, err := template.Script().Run(ctx) + if err != nil { + ctx.Close() + return nil, fmt.Errorf("JavaScript error initializing %s: %w", template.Name(), err) + } + mainFn, err := result.AsFunction() + if err != nil { + ctx.Close() + return nil, fmt.Errorf("%s's script did not return a function: %w", template.Name(), err) + } + + return &V8Runner{ + baseRunner: baseRunner{ + id: id, + vm: vm, + }, + v8vm: vm, + template: template, + ctx: ctx, + mainFn: mainFn, + }, nil +} + +func (r *V8Runner) Template() V8Template { return r.template } + +// Always call this when finished with a v8Runner acquired from Service.GetRunner. +func (r *V8Runner) Return() { + r.v8vm.returnRunner(r) +} + +// Disposes the V8 resources for a runner. +func (r *V8Runner) close() { + if r.vm != nil && r.v8vm.curRunner == r { + r.v8vm.curRunner = nil + } + if r.ctx != nil { + r.ctx.Close() + r.ctx = nil + } + r.vm = nil +} + +//////// CALLING JAVASCRIPT FUNCTIONS + +// Runs the Service's function, returning the result as a V8 Value. +// If the function throws an exception, it's returned as a v8.JSError. +func (r *V8Runner) Run(args ...any) (any, error) { + v8args, err := r.ConvertArgs(args...) + if err == nil { + var result *v8.Value + if result, err = r.RunWithV8Args(v8args...); err == nil { + return ValueToGo(result) + } + } + return nil, err +} + +// Runs the Service's function -- the same as `Run`, but requires that the args already be +// V8 Values or Objects. +// If the function throws an exception, it's returned as a v8.JSError. +func (r *V8Runner) RunWithV8Args(args ...v8.Valuer) (*v8.Value, error) { + return r.Call(r.mainFn, r.mainFn, args...) +} + +// A convenience wrapper around RunWithV8Args() that returns a v8.Object. +func (r *V8Runner) RunAsObject(args ...v8.Valuer) (*v8.Object, error) { + if val, err := r.RunWithV8Args(args...); err != nil { + return nil, err + } else if obj, err := val.AsObject(); err != nil { + return nil, err + } else { + return obj, nil + } +} + +// Calls a v8.Function, presumably one that you defined in the Service's Template. +// Use this instead of the v8.Function.Call method, because it: +// - waits for Promises to resolve, and returns the resolved value or rejection error; +// - stops with a DeadlineExceeded error if the v8Runner's Go Context times out. +func (r *V8Runner) Call(fn *v8.Function, this v8.Valuer, args ...v8.Valuer) (*v8.Value, error) { + return r.ResolvePromise(r.JustCall(fn, this, args...)) +} + +// Calls a V8 function; like v8Runner.Call except it does _not_ resolve Promises. +func (r *V8Runner) JustCall(fn *v8.Function, this v8.Valuer, args ...v8.Valuer) (*v8.Value, error) { + timedOut := false + if timeout := r.Timeout(); timeout != nil { + if *timeout <= 0 { + // Already timed out + return nil, context.DeadlineExceeded + } else { + // Future timeout; start a goroutine that will make the VM's thread terminate: + timer := time.NewTimer(*timeout) + completed := make(chan bool) + defer close(completed) + iso := r.v8vm.iso + go func() { + defer timer.Stop() + + select { + case <-completed: + return + case <-timer.C: + timedOut = true + iso.TerminateExecution() + } + }() + } + } + val, err := fn.Call(this, args...) + if jsErr, ok := err.(*v8.JSError); ok && strings.HasPrefix(jsErr.Message, "ExecutionTerminated:") { + if timedOut { + err = context.DeadlineExceeded + } + } + return val, err +} + +// Postprocesses a V8 Value returned from v8Runner.JustCall. +// - If it's a Promise, it returns its resolved value or error. +// - If the Promise hasn't completed yet, it lets V8 run until it completes. +// - Otherwise it +// Returns a DeadlineExceeded error if the v8Runner's Go Context has timed out, +// or times out while waiting on a Promise. +func (r *V8Runner) ResolvePromise(val *v8.Value, err error) (*v8.Value, error) { + if err != nil || !val.IsPromise() { + return val, err + } + for { + switch p, _ := val.AsPromise(); p.State() { + case v8.Fulfilled: + return p.Result(), nil + case v8.Rejected: + return nil, errors.New(p.Result().DetailString()) + case v8.Pending: + r.ctx.PerformMicrotaskCheckpoint() // run VM to make progress on the promise + deadline, hasDeadline := r.ContextOrDefault().Deadline() + if hasDeadline && time.Now().After(deadline) { + return nil, context.DeadlineExceeded + } + // go round the loop again... + default: + return nil, fmt.Errorf("illegal v8.Promise state %d", p) // impossible + } + } +} + +func (r *V8Runner) WithTemporaryValues(fn func()) { + r.ctx.WithTemporaryValues(fn) +} + +//////// CONVERTING GO VALUES TO JAVASCRIPT: + +// Converts its arguments to an array of V8 values by calling v8Runner.NewValue on each one. +// Useful for calling JS functions. +func (r *V8Runner) ConvertArgs(args ...any) ([]v8.Valuer, error) { + v8args := make([]v8.Valuer, len(args)) + for i, arg := range args { + var err error + if v8args[i], err = r.NewValue(arg); err != nil { + return nil, err + } + } + return v8args, nil +} + +// Converts a Go value to a JavaScript value (*v8.Value). Supported types are: +// - nil (converted to `null`) +// - boolean +// - int, int32, int64, uint32, uint64, float64 +// - *big.Int +// - json.Number +// - string +// - JSONString -- will be parsed by V8's JSON.parse +// - Any JSON Marshalable type -- will be marshaled to JSON then parsed by V8's JSON.parse +func (r *V8Runner) NewValue(val any) (v8Val *v8.Value, err error) { + if val == nil { + return r.NullValue(), nil // v8.NewValue panics if given nil :-p + } + v8Val, err = r.ctx.NewValue(val) + if err != nil { + if jsonStr, ok := val.(JSONString); ok { + if jsonStr != "" { + return r.JSONParse(string(jsonStr)) + } else { + return r.NullValue(), nil + } + } else if jsonData, jsonErr := json.Marshal(val); jsonErr == nil { + return r.JSONParse(string(jsonData)) + } + } + return v8Val, err +} + +// Creates a JavaScript number value. +func (r *V8Runner) NewInt(i int) *v8.Value { return mustSucceed(r.ctx.NewValue(i)) } + +// Creates a JavaScript string value. +func (r *V8Runner) NewString(str string) *v8.Value { return newString(r.v8vm.iso, str) } + +// Marshals a Go value to JSON and returns it as a V8 string. +func (r *V8Runner) NewJSONString(val any) (*v8.Value, error) { return newJSONString(r.ctx, val) } + +// Parses a JSON string to a V8 value, by calling JSON.parse() on it. +func (r *V8Runner) JSONParse(json string) (*v8.Value, error) { return v8.JSONParse(r.ctx, json) } + +// Returns a value representing JavaScript 'undefined'. +func (r *V8Runner) UndefinedValue() *v8.Value { return v8.Undefined(r.v8vm.iso) } + +// Returns a value representing JavaScript 'null'. +func (r *V8Runner) NullValue() *v8.Value { return v8.Null(r.v8vm.iso) } + +//////// CONVERTING JAVASCRIPT VALUES BACK TO GO: + +// Encodes a V8 value as a JSON string. +func (r *V8Runner) JSONStringify(val *v8.Value) (string, error) { return v8.JSONStringify(r.ctx, val) } + +//////// INSTANTIATING TEMPLATES: + +// Creates a V8 Object from a template previously created by BasicTemplate.NewObjectTemplate. +// (Not needed if you added the template as a property of the global object.) +func (r *V8Runner) NewInstance(o *v8.ObjectTemplate) (*v8.Object, error) { + return o.NewInstance(r.ctx) +} + +// Creates a V8 Function from a template previously created by BasicTemplate.NewCallback. +// (Not needed if you added the template as a property of the global object.) +func (r *V8Runner) NewFunctionInstance(f *v8.FunctionTemplate) *v8.Function { + return f.GetFunction(r.ctx) +} diff --git a/js/v8_template.go b/js/v8_template.go new file mode 100644 index 0000000..fc9f6b1 --- /dev/null +++ b/js/v8_template.go @@ -0,0 +1,200 @@ +//go:build cb_sg_v8 + +/* +Copyright 2022-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package js + +import ( + "fmt" + + "github.com/pkg/errors" + v8 "github.com/snej/v8go" +) + +// A V8Template manages a Service's initial JS runtime environment -- its script code, main function +// and global functions & variables -- in the form of V8 "template" objects. +// +// A V8Template belongs to both a Service and a VM. It's created the first time the Service needs to +// create a v8Runner in that VM. +// +// Once the V8Template is initialized, each v8Runner's V8 Context is very cheaply initialized by +// making references to these templates, without needing to compile or run anything. +// +// Many clients -- those that just need to run a single JS function with no callbacks -- can +// ignore this interface entirely. +// +// Very few clients will need to implement this interface; most can just use the BasicTemplate +// struct that implements it, which is passed to NewCustomService's callback. +type V8Template interface { + // The JavaScript global namespace. + // Any named property set on this object by the Set method is exposed as a global + // variable/context/function. + Global() *v8.ObjectTemplate + + // The compiled script. When the Service is instantiated as a v8Runner, _before_ the runner is + // called, this script is run and its return value becomes the v8Runner's function entry point. + Script() *v8.UnboundScript + + // The Service's name. + Name() string + + // If this returns true, a v8Runner instance will be cached by a VM and reused for the next call. + // This is faster, but it means that any changes to the v8Runner's global variables will persist. + Reusable() bool +} + +// Base implementation of Template. +// Manages a Service's initial JS runtime environment -- its script code, main function +// and global functions & variables -- in the form of V8 "template" objects. +// +// "Subclasses" can be created by embedding this in another struct; this is useful if you create +// additional object or function templates and need to store references to them for use by the +// Runner. +type V8BasicTemplate struct { + vm *v8VM + global *v8.ObjectTemplate + script *v8.UnboundScript + name string + oneShot bool +} + +func (t *V8BasicTemplate) Global() *v8.ObjectTemplate { return t.global } +func (t *V8BasicTemplate) Script() *v8.UnboundScript { return t.script } +func (t *V8BasicTemplate) Name() string { return t.name } +func (t *V8BasicTemplate) Reusable() bool { return !t.oneShot } + +// Compiles JavaScript source code that becomes the service's script. +// A TemplateFactory function (the callback passed to NewCustomService) must call this. +// The source code can be any JavaScript, but running it must return a JS function object; +// that is, the last statement of the script must be or return a function expression. +// This statement cannot simply be a function; that's a syntax error. To return a function, +// wrap it in parentheses, e.g. `(function(arg1,arg2…) {…body…; return result;});`, +// or give the function a name and simply return it by name. +func (t *V8BasicTemplate) SetScript(jsSourceCode string) error { + var err error + t.script, err = t.vm.iso.CompileUnboundScript(jsSourceCode, t.name+".js", v8.CompileOptions{}) + return err +} + +// Sets the Template's Reusable property, which defaults to true. +// If your Service's script is considered untrusted, or if it modifies global variables and needs +// them reset on each run, you can set this to false. +func (t *V8BasicTemplate) SetReusable(reuseable bool) { t.oneShot = !reuseable } + +// Creates a JS template object, which will be instantiated as a real object in a v8Runner's context. +func (t *V8BasicTemplate) NewObjectTemplate() *v8.ObjectTemplate { + return v8.NewObjectTemplate(t.vm.iso) +} + +// A Go function that can be registered as a callback from JavaScript. +// For convenience, Service callbacks get passed the v8Runner instance, +// and return Go values -- allowed types are nil, numbers, bool, string, v8.Value, v8.Object. +type TemplateCallback = func(r *V8Runner, this *v8.Object, args []*v8.Value) (result any, err error) + +// Defines a JS global function that calls calls into the given Go function. +func (t *V8BasicTemplate) GlobalCallback(name string, cb TemplateCallback) { + _ = t.global.Set(name, t.NewCallback(cb), v8.ReadOnly) +} + +// Creates a JS function-template object that calls into the given Go function. +func (t *V8BasicTemplate) NewCallback(callback TemplateCallback) *v8.FunctionTemplate { + vm := t.vm + return v8.NewFunctionTemplate(vm.iso, func(info *v8.FunctionCallbackInfo) *v8.Value { + runner := vm.currentRunner(info.Context()) + result, err := callback(runner, info.This(), info.Args()) + if err == nil { + if v8Result, newValErr := runner.NewValue(result); err == nil { + return v8Result + } else { + err = errors.Wrap(newValErr, "Could not convert a callback's result to JavaScript") + } + } + return v8Throw(vm.iso, err) + }) +} + +// Converts a Go value to a JavaScript value (*v8.Value). Supported types are: +// - nil (converted to `null`) +// - boolean +// - integer and float types (32-bit and larger) +// - *big.Int +// - json.Number +// - string +func (t *V8BasicTemplate) NewValue(val any) (*v8.Value, error) { + if val == nil { + return v8.Null(t.vm.iso), nil // v8.NewValue panics if given nil :-p + } + return v8.NewValue(t.vm.iso, val) +} + +// Creates a JS string value. +func (t *V8BasicTemplate) NewString(str string) *v8.Value { + return newString(t.vm.iso, str) +} + +//////// INTERNALS: + +func newV8Template(vm *v8VM, service *Service) (V8Template, error) { + basicTmpl := &V8BasicTemplate{ + name: service.name, + vm: vm, + global: v8.NewObjectTemplate(vm.iso), + } + if err := basicTmpl.defineSgLog(); err != nil { + return nil, err + } + + var tmpl V8Template + var err error + if service.v8Init != nil { + tmpl, err = service.v8Init(basicTmpl) + } else { + err = basicTmpl.SetScript(`(` + service.jsFunctionSource + `)`) + tmpl = basicTmpl + } + + if err != nil { + return nil, fmt.Errorf("failed to initialize JS service %q: %+w", service.name, err) + } else if tmpl == nil { + return nil, fmt.Errorf("js.TemplateFactory %q returned nil", service.name) + } else if tmpl.Script() == nil { + return nil, fmt.Errorf("js.TemplateFactory %q failed to initialize Service's script", service.name) + } else { + return tmpl, nil + } +} + +// Defines a global `sg_log` function that writes to SG's log. +func (service *V8BasicTemplate) defineSgLog() error { + return service.global.Set("sg_log", service.NewCallback(func(r *V8Runner, this *v8.Object, args []*v8.Value) (any, error) { + if len(args) >= 2 { + level := LogLevel(args[0].Integer()) + msg := args[1].String() + extra := "" + for i := 2; i < len(args); i++ { + extra += " " + extra += args[i].DetailString() + } + LoggingCallback(r.ContextOrDefault(), level, "%s %s", msg, extra) + } + return nil, nil + })) +} + +// Sets up the standard console logging functions, delegating to `sg_log`. +const kSetupLoggingJS = ` + console.trace = function(...args) {sg_log(5, ...args);}; + console.debug = function(...args) {sg_log(4, ...args);}; + console.log = function(...args) {sg_log(3, ...args);}; + console.info = console.log; + console.warn = function(...args) {sg_log(2, ...args);}; + console.error = function(...args) {sg_log(1, ...args);}; +` diff --git a/js/v8_utils.go b/js/v8_utils.go new file mode 100644 index 0000000..5d1a0ae --- /dev/null +++ b/js/v8_utils.go @@ -0,0 +1,134 @@ +//go:build cb_sg_v8 + +/* +Copyright 2022-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package js + +import ( + "encoding/json" + "fmt" + "math" + + v8 "github.com/snej/v8go" +) + +// CONVERTING V8 VALUES BACK TO GO: + +// Converts a JS string to a Go string. +func StringToGo(val *v8.Value) (string, bool) { + if val.IsString() { + return val.String(), true + } else { + return "", false + } +} + +// Converts a V8 value back to a Go representation. +// Recognizes JS strings, numbers, booleans. `null` and `undefined` are returned as nil. +// Other JS types are run through `JSON.stringify` and `json.Unmarshal`. +func ValueToGo(val *v8.Value) (any, error) { + switch val.GetType() { + case v8.UndefinedType, v8.NullType: + return nil, nil + case v8.FalseType: + return false, nil + case v8.TrueType: + return true, nil + case v8.NumberType: + return intify(val.Number()), nil + case v8.BigIntType: + big := val.BigInt() + if big.IsInt64() { + return big.Int64(), nil + } + return big, nil + case v8.StringType: + return val.String(), nil + default: + // Otherwise detour through JSON: + if j, err := val.MarshalJSON(); err == nil { + var result any + if json.Unmarshal(j, &result) == nil { + return result, nil + } + } + return nil, fmt.Errorf("couldn't convert JavaScript value `%s`", val.DetailString()) + } +} + +// Converts a float64 to an int or int64 if possible without losing accuracy. +func intify(f float64) any { + if f == math.Floor(f) && f >= float64(JavascriptMinSafeInt) && f < float64(JavascriptMaxSafeInt) { + if i64 := int64(f); i64 >= math.MinInt && i64 <= math.MaxInt { + return int(i64) // Return int if possible + } else { + return i64 // Return int64 if out of range of 32-bit int + } + } else { + return f // Return float64 if not integral or not in range of int64 + } +} + +// Converts a V8 array of strings to Go. Any non-strings in the array are ignored. +func StringArrayToGo(val *v8.Value) (result []string, err error) { + obj, err := val.AsObject() + if err == nil { + for i := uint32(0); obj.HasIdx(i); i++ { + if item, err := obj.GetIdx(i); err == nil && item.IsString() { + result = append(result, item.String()) + } + } + } + return +} + +//////// CONVERTING GO TO V8 VALUES: + +// Converts a Go string into a JS string value. Assumes this cannot fail. +// (AFAIK, v8.NewValue only fails if the input type is invalid, or V8 runs out of memory.) +func newString(i *v8.Isolate, str string) *v8.Value { + return mustSucceed(v8.NewValue(i, str)) +} + +// Marshals a Go value to JSON, and returns the string as a V8 Value. +func newJSONString(ctx *v8.Context, val any) (*v8.Value, error) { + if val == nil { + return v8.Null(ctx.Isolate()), nil + } else if jsonBytes, err := json.Marshal(val); err != nil { + return nil, err + } else { + return ctx.NewValue(string(jsonBytes)) + } +} + +//////// ERROR UTILITIES: + +// Returns an error back to a V8 caller. +// Calls v8.Isolate.ThrowException, with the Go error's string as the message. +func v8Throw(i *v8.Isolate, err error) *v8.Value { + var errStr string + // if httpErr, ok := err.(*base.HTTPError); ok { + // errStr = fmt.Sprintf("[%d] %s", httpErr.Status, httpErr.Message) + // } else { + errStr = err.Error() + // } + return i.ThrowException(newString(i, errStr)) +} + +// Simple utility to wrap a function that returns a value and an error; returns just the value, panicking if there was an error. +// This is kind of equivalent to those 3-prong to 2-prong electric plug adapters... +// Needless to say, it should only be used if you know the error cannot occur, or that if it occurs something is very, very wrong. +func mustSucceed[T any](result T, err error) T { + if err != nil { + panic(fmt.Sprintf(`ASSERTION FAILURE: expected a %T, got error "%v"`, result, err)) + } + return result +} diff --git a/js/v8_vm.go b/js/v8_vm.go new file mode 100644 index 0000000..039577b --- /dev/null +++ b/js/v8_vm.go @@ -0,0 +1,247 @@ +//go:build cb_sg_v8 + +/* +Copyright 2022-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package js + +import ( + _ "embed" + "fmt" + "sync" + "time" + + "github.com/pkg/errors" + v8 "github.com/snej/v8go" +) + +// v8go docs: https://pkg.go.dev/github.com/snej/v8go +// General V8 API docs: https://v8.dev/docs/embed + +type v8VM struct { + *baseVM + iso *v8.Isolate // A V8 virtual machine. NOT THREAD SAFE. + setupScript *v8.UnboundScript // JS code to set up a new v8.Context + templates []V8Template // Already-created Templates, indexed by serviceID + runners []*V8Runner // Available Runners, indexed by serviceID. nil if in-use + curRunner *V8Runner // Currently active v8Runner, if any +} + +// The initial heap size in bytes of a V8 VM (Isolate). +// Changing this variable affects subsequently created V8 VMs. +var V8InitialHeap uint64 = 4 * 1024 * 1024 + +// The maximum heap size in bytes of a V8 VM; if it exceeds this, the script is terminated. +// Changing this variable affects subsequently created V8 VMs. +var V8MaxHeap uint64 = 32 * 1024 * 1024 + +// The maximum size in bytes of V8's JavaScript call stack. +// If you change this, do so before creating any V8 VMs, otherwise it'll have no effect. +// (Warning: When I set this to values >= 800, V8 crashed the process on heap overflow, +// instead of just terminating the script. This was on macOS; YMMV on other platforms.) +var V8StackSizeLimit = 400 * 1024 + +const v8VMName = "V8" + +// Returns the Engine with the given name, else nil. +// Valid names are "V8" and "Otto", which map to the instances `V8` and `Otto`. +func EngineNamed(name string) *Engine { + switch name { + case v8VMName: + return V8 + case ottoVMName: + return Otto + default: + return nil + } +} + +// A VMType for instantiating V8-based VMs and VMPools. +var V8 = &Engine{ + name: v8VMName, + languageVersion: 13, // ES2022. Can't find exact version on the V8 website --Jens 1/2023 + factory: v8VMFactory, +} + +var v8Init sync.Once + +func v8VMFactory(engine *Engine, services *servicesConfiguration) VM { + v8Init.Do(func() { + v8.SetFlags(fmt.Sprintf("--stack_size=%d", V8StackSizeLimit/1024)) + }) + return &v8VM{ + baseVM: &baseVM{engine: engine, services: services}, + iso: v8.NewIsolateWith(V8InitialHeap, V8MaxHeap), // The V8 v8VM + templates: []V8Template{}, // Instantiated Services + runners: []*V8Runner{}, // Cached reusable Runners + } +} + +// Shuts down a v8VM. It's a good idea to call this explicitly when done with it, +// _unless_ it belongs to a VMPool. +func (vm *v8VM) Close() { + vm.baseVM.close() + if cur := vm.curRunner; cur != nil { + cur.Return() + } + for _, runner := range vm.runners { + if runner != nil { + runner.close() + } + } + vm.templates = nil + if vm.iso != nil { + vm.iso.Dispose() + vm.iso = nil + } +} + +// Looks up an already-registered service by name. Returns nil if not found. +func (vm *v8VM) FindService(name string) *Service { + return vm.services.findServiceNamed(name) +} + +//////// INTERNALS: + +// Must be called when finished using a v8VM belonging to a VMPool! +// (Harmless no-op when called on a standalone v8VM.) +func (vm *v8VM) release() { + if vm.returnToPool != nil { + vm.lastReturned = time.Now() + vm.returnToPool.returnVM(vm) + } +} + +// Returns a Template for the given Service. +func (vm *v8VM) getTemplate(service *Service) (V8Template, error) { + var tmpl V8Template + if !vm.services.hasService(service) { + return nil, fmt.Errorf("unknown js.Service instance passed to VM") + } + if int(service.id) < len(vm.templates) { + tmpl = vm.templates[service.id] + } + if tmpl == nil { + if vm.setupScript == nil { + // The setup script points console logging to SG, and loads the Underscore.js library: + var err error + vm.setupScript, err = vm.iso.CompileUnboundScript(kSetupLoggingJS+kUnderscoreJS, "setupScript.js", v8.CompileOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "Couldn't compile setup script") + } + } + + var err error + tmpl, err = newV8Template(vm, service) + if err != nil { + return nil, err + } + + for int(service.id) >= len(vm.templates) { + vm.templates = append(vm.templates, nil) + } + vm.templates[service.id] = tmpl + } + return tmpl, nil +} + +func (vm *v8VM) hasInitializedService(service *Service) bool { + id := int(service.id) + return id < len(vm.templates) && vm.templates[id] != nil +} + +// Produces a v8Runner object that can run the given service. +// Be sure to call Runner.Return when done. +// Since v8VM is single-threaded, calling getRunner when a v8Runner already exists and hasn't been +// returned yet is assumed to be illegal concurrent access; it will trigger a panic. +func (vm *v8VM) getRunner(service *Service) (Runner, error) { + if vm.iso == nil { + return nil, fmt.Errorf("the js.VM has been closed") + } + if vm.curRunner != nil { + panic("illegal access to v8VM: already has a v8Runner") + } + var runner *V8Runner + index := int(service.id) + if index < len(vm.runners) { + runner = vm.runners[index] + vm.runners[index] = nil + } + if runner == nil { + tmpl, err := vm.getTemplate(service) + if tmpl == nil { + return nil, err + } + runner, err = newV8Runner(vm, tmpl, service.id) + if err != nil { + return nil, err + } + } else { + runner.vm = vm + } + vm.curRunner = runner + vm.iso.Lock() + return runner, nil +} + +func (vm *v8VM) withRunner(service *Service, fn func(Runner) (any, error)) (any, error) { + runner, err := vm.getRunner(service) + if err != nil { + vm.release() + return nil, err + } + defer runner.Return() + var result any + runner.(*V8Runner).WithTemporaryValues(func() { + result, err = fn(runner) + }) + return result, err +} + +// Called by v8Runner.Return; either closes its V8 resources or saves it for reuse. +// Also returns the v8VM to its Pool, if it came from one. +func (vm *v8VM) returnRunner(r *V8Runner) { + r.goContext = nil + if vm.curRunner == r { + vm.iso.Unlock() + vm.curRunner = nil + } else if r.vm != vm { + panic("v8Runner returned to wrong v8VM!") + } + if r.template.Reusable() { + for int(r.id) >= len(vm.runners) { + vm.runners = append(vm.runners, nil) + } + vm.runners[r.id] = r + } else { + r.close() + } + vm.release() +} + +// Returns the v8Runner that owns the given V8 Context. +func (vm *v8VM) currentRunner(ctx *v8.Context) *V8Runner { + // IMPORTANT: This is kind of a hack, but we can get away with it because a v8VM has only one + // active v8Runner at a time. If it were to be multiple Runners, we'd need to maintain a map + // from Contexts to Runners. + if vm.curRunner == nil { + panic(fmt.Sprintf("Unknown v8.Context passed to v8VM.currentRunner: %v, expected none", ctx)) + } + if ctx != vm.curRunner.ctx { + panic(fmt.Sprintf("Unknown v8.Context passed to v8VM.currentRunner: %v, expected %v", ctx, vm.curRunner.ctx)) + } + return vm.curRunner +} + +// The Underscore.js utility library, version 1.13.6, downloaded 2022-Nov-23 from +// +// +//go:embed underscore-umd-min.js +var kUnderscoreJS string diff --git a/js/vm.go b/js/vm.go new file mode 100644 index 0000000..de5e7bb --- /dev/null +++ b/js/vm.go @@ -0,0 +1,127 @@ +/* +Copyright 2022-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package js + +import ( + "context" + "time" +) + +//////// ENGINE + +// An opaque object identifying a JavaScript engine (V8 or Otto) +type Engine struct { + name string + languageVersion int + factory func(*Engine, *servicesConfiguration) VM +} + +// The name identifying this engine ("V8" or "Otto") +func (engine *Engine) String() string { return engine.name } + +// The edition number of the ECMAScript spec supported by this Engine. +// For example a value of 6 is 6th Edition, better known as ES2015, aka "Modern JavaScript". +// See https://en.wikipedia.org/wiki/ECMAScript_version_history +func (engine *Engine) LanguageVersion() int { return engine.languageVersion } + +// Language version (ECMAScript edition #) of ES2015 +const ES2015 = 6 + +// Creates a JavaScript virtual machine of the given type. +// This object should be used only on a single goroutine at a time. +func (engine *Engine) NewVM() VM { + return engine.newVM(&servicesConfiguration{}) +} + +func (engine *Engine) newVM(services *servicesConfiguration) VM { + return engine.factory(engine, services) +} + +//////// VM + +// Represents a single-threaded JavaScript virtual machine. +// This doesn't do much on its own; it acts as a ServiceHost for Service and Runner objects. +// +// **Not thread-safe!** A VM instance must be used only on one goroutine at a time. +// A Service whose ServiceHost is a VM can only be used on a single goroutine; any concurrent +// use will trigger a panic in VM.getRunner. +// The VMPool takes care of this, by vending VM instances that are known not to be in use. +type VM interface { + Engine() *Engine + Close() + FindService(name string) *Service + + registerService(*Service) + hasInitializedService(*Service) bool + getRunner(*Service) (Runner, error) + withRunner(*Service, func(Runner) (any, error)) (any, error) + setReturnToPool(*VMPool) + getReturnToPool() *VMPool + getLastReturned() time.Time +} + +// Syntax-checks a string containing a JavaScript function definition +func ValidateJavascriptFunction(vm VM, jsFunc string, minArgs int, maxArgs int) error { + service := vm.FindService("ValidateJavascriptFunction") + if service == nil { + service = NewService(vm, "ValidateJavascriptFunction", ` + function(jsFunc, minArgs, maxArgs) { + var fn = Function('"use strict"; return ' + jsFunc)() + var typ = typeof(fn); + if (typ !== 'function') { + throw "code is not a function, but a " + typ; + } else if (fn.length < minArgs) { + throw "function must have at least " + minArgs + " parameters"; + } else if (fn.length > maxArgs) { + throw "function must have no more than " + maxArgs + " parameters"; + } + } + `) + } + _, err := service.Run(context.Background(), jsFunc, minArgs, maxArgs) + return err +} + +//////// BASEVM + +// A base "class" containing shared properties and methods for use by VM implementations. +type baseVM struct { + engine *Engine + services *servicesConfiguration // Factories for services + returnToPool *VMPool // Pool to return me to, or nil + lastReturned time.Time // Time that v8VM was last returned to its pool + closed bool +} + +func (vm *baseVM) Engine() *Engine { return vm.engine } + +func (vm *baseVM) close() { + if vm.returnToPool != nil { + panic("Don't Close a VM that belongs to a VMPool") + } + vm.services = nil + vm.closed = true +} + +func (vm *baseVM) registerService(service *Service) { + if vm.services == nil { + if vm.closed { + panic("Using an already-closed js.VM") + } else { + panic("You forgot to initialize a js.VM") // Must call NewVM() + } + } + vm.services.addService(service) +} + +func (vm *baseVM) setReturnToPool(pool *VMPool) { vm.returnToPool = pool } +func (vm *baseVM) getReturnToPool() *VMPool { return vm.returnToPool } +func (vm *baseVM) getLastReturned() time.Time { return vm.lastReturned } diff --git a/js/vm_otto_only.go b/js/vm_otto_only.go new file mode 100644 index 0000000..65dd556 --- /dev/null +++ b/js/vm_otto_only.go @@ -0,0 +1,22 @@ +// Copyright 2023-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +//go:build !cb_sg_v8 + +package js + +// Returns the Engine with the given name, else nil. +// Valid names are "V8" and "Otto", which map to the instances `V8` and `Otto`. +func EngineNamed(name string) *Engine { + switch name { + case ottoVMName: + return Otto + default: + return nil + } +} diff --git a/js/vmpool.go b/js/vmpool.go new file mode 100644 index 0000000..0da4dc8 --- /dev/null +++ b/js/vmpool.go @@ -0,0 +1,229 @@ +/* +Copyright 2022-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package js + +import ( + "context" + "fmt" + "sync" + "time" +) + +//////// VM POOL + +// A thread-safe ServiceHost for Services and Runners that owns a set of VMs +// and allocates an available one when a Runner is needed. +type VMPool struct { + maxInUse int // Max number of simultaneously in-use VMs + services *servicesConfiguration // Defines the services (owned VMs also have references) + tickets chan bool // Each item in this channel represents availability of a VM + engine *Engine // Factory function that creates IVMs + mutex sync.Mutex // Must be locked to access fields below + vms_ []VM // LIFO cache of idle VMs, recently used at end + curInUse_ int // Current number of VMs "checked out" +} + +func NewVMPool(typ *Engine, maxVMs int) *VMPool { + pool := new(VMPool) + pool.Init(typ, maxVMs) + return pool +} + +func (pool *VMPool) Init(typ *Engine, maxVMs int) { + pool.maxInUse = maxVMs + pool.services = &servicesConfiguration{} + pool.engine = typ + pool.vms_ = make([]VM, 0, maxVMs) + pool.tickets = make(chan bool, maxVMs) + for i := 0; i < maxVMs; i++ { + pool.tickets <- true + } + info(context.Background(), "js.VMPool: Init, max %d VMs", maxVMs) +} + +func (pool *VMPool) Engine() *Engine { return pool.engine } + +// Tears down a VMPool, freeing up its cached VMs. +// It's a good idea to call this when using V8, as the VMs may be holding onto a lot of external +// memory managed by V8, and this will clean up that memory sooner than Go's GC will. +func (pool *VMPool) Close() { + if inUse := pool.InUseCount(); inUse > 0 { + warn(context.Background(), "A js.VMPool is being closed with %d VMs still in use", inUse) + } + + // First stop all waiting `Get` calls: + close(pool.tickets) + + // Now pull all the VMs out of the pool and close them. + // This isn't necessary, but it frees up memory sooner. + n := pool.closeAll() + info(context.Background(), + "js.VMPool.Close: Closed pool with %d VM(s)", n) +} + +// Returns the number of VMs currently in use. +func (pool *VMPool) InUseCount() int { + pool.mutex.Lock() + defer pool.mutex.Unlock() + return pool.curInUse_ +} + +// Closes all idle VMs cached by this pool. It will reallocate them when it needs to. +func (pool *VMPool) PurgeUnusedVMs() { + n := pool.closeAll() + info(context.Background(), + "js.VMPool.PurgeUnusedVMs: Closed %d idle VM(s)", n) +} + +func (pool *VMPool) FindService(name string) *Service { + return pool.services.findServiceNamed(name) +} + +//////// INTERNALS: + +func (pool *VMPool) registerService(service *Service) { + if pool.services == nil { + panic("You forgot to initialize a VMPool") + } + pool.services.addService(service) +} + +// Produces an idle `VM` that can be used by this goroutine. +// You MUST call returnVM, or VM.release, when done. +func (pool *VMPool) getVM(service *Service) (VM, error) { + // Pull a ticket; this blocks until less than `maxVMs` VMs are in use: + if _, ok := <-pool.tickets; !ok { + return nil, fmt.Errorf("the VMPool has been closed") + } + + // Pop a VM from the channel: + vm, inUse := pool.pop(service) + if vm == nil { + // Nothing in the pool, so create a new VM instance. + vm = pool.engine.newVM(pool.services) + info(context.Background(), + "js.VMPool.getVM: No VMs free; created a new one") + } + + vm.setReturnToPool(pool) + debug(context.Background(), + "js.VMPool.getVM: %d/%d VMs now in use", inUse, pool.maxInUse) + return vm, nil +} + +// Returns a used `VM` back to the pool for reuse; called by VM.Return +func (pool *VMPool) returnVM(vm VM) { + if vm != nil && vm.getReturnToPool() == pool { + vm.setReturnToPool(nil) + + inUse := pool.push(vm) + debug(context.Background(), + "js.VMPool.returnVM: %d/%d VMs now in use", inUse, pool.maxInUse) + + // Return a ticket to the channel: + pool.tickets <- true + } +} + +// Instantiates a Runner for a named Service, in an available VM. +// You MUST call Runner.Return when done -- it will return the associated VM too. +func (pool *VMPool) getRunner(service *Service) (Runner, error) { + if vm, err := pool.getVM(service); err != nil { + return nil, err + } else if runner, err := vm.getRunner(service); err != nil { + pool.returnVM(vm) + return nil, err + } else { + return runner, err + } +} + +func (pool *VMPool) withRunner(service *Service, fn func(Runner) (any, error)) (any, error) { + if vm, err := pool.getVM(service); err != nil { + return nil, err + } else { + return vm.withRunner(service, fn) + // (I don't need to call returnVM: it will return itself when it's done with its + // Runner, because its returnToPool is set.) + } +} + +//////// POOL MANAGEMENT -- The low level stuff that requires a mutex. + +// A VMPool will stop caching a VM that hasn't been used for this long. +const kVMStaleDuration = time.Minute + +// Just gets a VM from the pool, and increments the in-use count. Thread-safe. +func (pool *VMPool) pop(service *Service) (vm VM, inUse int) { + pool.mutex.Lock() + defer pool.mutex.Unlock() + + pool.curInUse_ += 1 + inUse = pool.curInUse_ + + if n := len(pool.vms_); n > 0 { + // Find the most recently-used VM that already has this service: + vms := pool.vms_ + var i int + for i = n - 1; i >= 0; i -= 1 { + if vms[i].hasInitializedService(service) { + break + } + } + if i < 0 { + // If no VM has this service, choose the most recently used one: + i = n - 1 + } + + // Delete this VM from the array before returning it: + vm = vms[i] + copy(vms[i:], vms[i+1:]) + vms = vms[:n-1] + + // If the oldest VM in the pool hasn't been used in a while, get rid of it: + if n > 1 { + oldest := vms[0] + if stale := time.Since(oldest.getLastReturned()); stale > kVMStaleDuration { + vms = vms[1:] + oldest.setReturnToPool(nil) + oldest.Close() + debug(context.Background(), + "js.VMPool.pop: Disposed a stale VM not used in %v", stale) + } + } + pool.vms_ = vms + } + return +} + +// Just adds a VM to the pool, and decrements the in-use count. Thread-safe. +func (pool *VMPool) push(vm VM) (inUse int) { + pool.mutex.Lock() + defer pool.mutex.Unlock() + + pool.vms_ = append(pool.vms_, vm) + pool.curInUse_ -= 1 + return pool.curInUse_ +} + +// Removes all idle VMs from the pool and closes them. (Does not alter the in-use count.) +func (pool *VMPool) closeAll() int { + pool.mutex.Lock() + vms := pool.vms_ + pool.vms_ = nil + pool.mutex.Unlock() + + for _, vm := range vms { + vm.setReturnToPool(nil) + vm.Close() + } + return len(vms) +} diff --git a/js/vmpool_test.go b/js/vmpool_test.go new file mode 100644 index 0000000..25a1b3e --- /dev/null +++ b/js/vmpool_test.go @@ -0,0 +1,219 @@ +//go:build cb_sg_v8 + +/* +Copyright 2022-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package js + +import ( + "context" + "fmt" + "log" + "runtime" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +//////// VALIDATION TESTS + +func TestValidateJavascriptFunction(t *testing.T) { + TestWithVMs(t, func(t *testing.T, vm VM) { + assert.NoError(t, ValidateJavascriptFunction(vm, `function(doc) {return doc.x;}`, 1, 1)) + if vm.Engine().LanguageVersion() >= ES2015 { // Otto does not support new-style function syntax + assert.NoError(t, ValidateJavascriptFunction(vm, `(doc,foo) => {return doc.x;}`, 2, 2)) + } + err := ValidateJavascriptFunction(vm, `function() {return doc.x;}`, 1, 2) + assert.ErrorContains(t, err, "function must have at least 1 parameters") + err = ValidateJavascriptFunction(vm, `function(doc, foo, bar) {return doc.x;}`, 1, 2) + assert.ErrorContains(t, err, "function must have no more than 2 parameters") + err = ValidateJavascriptFunction(vm, `function(doc) {return doc.x;`, 1, 1) + assert.ErrorContains(t, err, "SyntaxError") + err = ValidateJavascriptFunction(vm, `"not a function"`, 1, 1) + assert.ErrorContains(t, err, "code is not a function, but a string") + err = ValidateJavascriptFunction(vm, `17 + 34`, 1, 1) + assert.ErrorContains(t, err, "code is not a function, but a number") + }) +} + +//////// CONCURRENCY TESTS + +const kVMPoolTestScript = `function(n) {return n * n;}` +const kVMPoolTestTimeout = 60 * time.Second +const kVMPoolTestNumTasks = 65536 + +func TestPoolsSequentially(t *testing.T) { + ctx := context.Background() + + if !assertPriorTimeoutAtLeast(t, ctx, kVMPoolTestTimeout) { + return + } + TestWithVMPools(t, 4, func(t *testing.T, pool *VMPool) { + service := NewService(pool, "testy", kVMPoolTestScript) + runSequentially(ctx, kVMPoolTestNumTasks, func(ctx context.Context) bool { + result, err := service.Run(ctx, 13) + return assert.NoError(t, err) && assert.EqualValues(t, result, 169) + }) + }) +} + +func TestPoolsConcurrently(t *testing.T) { + maxProcs := runtime.GOMAXPROCS(0) + log.Printf("FYI, GOMAXPROCS = %d", maxProcs) + if !assert.GreaterOrEqual(t, maxProcs, 2, "Not enough OS threads available") { + return + } + + ctx := context.Background() + + if !assertPriorTimeoutAtLeast(t, ctx, kVMPoolTestTimeout) { + return + } + + TestWithVMPools(t, maxProcs, func(t *testing.T, pool *VMPool) { + numTasks := kVMPoolTestNumTasks + service := NewService(pool, "testy", kVMPoolTestScript) + t.Run("Function", func(t *testing.T) { + testConcurrently(t, ctx, numTasks, maxProcs, func(ctx context.Context) bool { + result, err := service.Run(ctx, 13) + return assert.NoError(t, err) && assert.EqualValues(t, result, 169) + }) + }) + }) +} + +//////// CONCURRENCY BENCHMARKS + +func BenchmarkVMPoolIntsSequentially(b *testing.B) { + ctx := context.Background() + if !assertPriorTimeoutAtLeast(b, ctx, kVMPoolTestTimeout) { + return + } + pool := NewVMPool(V8, 32) + service := NewService(pool, "testy", kVMPoolTestScript) + testFunc := func(ctx context.Context) bool { + result, err := service.Run(ctx, 13) + return assert.NoError(b, err) && assert.EqualValues(b, result, 169) + } + b.ResetTimer() + runSequentially(ctx, b.N, testFunc) + b.StopTimer() + pool.Close() +} + +func BenchmarkVMPoolIntsConcurrently(b *testing.B) { + const kNumThreads = 8 + ctx := context.Background() + pool := NewVMPool(V8, 32) + service := NewService(pool, "testy", kVMPoolTestScript) + testFunc := func(ctx context.Context) bool { + result, err := service.Run(ctx, 13) + return assert.NoError(b, err) && assert.EqualValues(b, result, 169) + } + b.ResetTimer() + runConcurrently(ctx, b.N, kNumThreads, testFunc) + b.StopTimer() + pool.Close() +} + +func BenchmarkVMPoolStringsSequentially(b *testing.B) { + fmt.Printf("-------- N = %d -------\n", b.N) + ctx := context.Background() + pool := NewVMPool(V8, 32) + service := NewService(pool, "testy", kVMPoolTestScript) + testFunc := func(ctx context.Context) bool { + result, err := service.Run(ctx, "This is a test of the js package") + return assert.NoError(b, err) && assert.EqualValues(b, result, "This is a test of the js packageThis is a test of the js package") + } + b.ResetTimer() + runSequentially(ctx, b.N, testFunc) + b.StopTimer() + pool.Close() +} + +func BenchmarkVMPoolStringsConcurrently(b *testing.B) { + const kNumThreads = 8 + ctx := context.Background() + pool := NewVMPool(V8, 32) + service := NewService(pool, "testy", kVMPoolTestScript) + testFunc := func(ctx context.Context) bool { + result, err := service.Run(ctx, "This is a test of the js package") + return assert.NoError(b, err) && assert.EqualValues(b, result, "This is a test of the js packageThis is a test of the js package") + } + b.ResetTimer() + runConcurrently(ctx, b.N, kNumThreads, testFunc) + b.StopTimer() + pool.Close() +} + +//////// SUPPORT FUNCTIONS + +func runSequentially(ctx context.Context, numTasks int, testFunc func(context.Context) bool) time.Duration { + ctx, cancel := context.WithTimeout(ctx, kVMPoolTestTimeout) + defer cancel() + startTime := time.Now() + for i := 0; i < numTasks; i++ { + if !testFunc(ctx) { + break + } + } + return time.Since(startTime) +} + +func runConcurrently(ctx context.Context, numTasks int, numThreads int, testFunc func(context.Context) bool) time.Duration { + var wg sync.WaitGroup + startTime := time.Now() + for i := 0; i < numThreads; i++ { + wg.Add(1) + go func() { + defer wg.Done() + myCtx, cancel := context.WithTimeout(ctx, kVMPoolTestTimeout) + defer cancel() + for j := 0; j < numTasks/numThreads; j++ { + if !testFunc(myCtx) { + break + } + } + }() + } + wg.Wait() + return time.Since(startTime) +} + +// Asserts that running testFunc in 100 concurrent goroutines is no more than 10% slower +// than running it 100 times in succession. A low bar indeed, but can detect some serious +// bottlenecks, or of course deadlocks. +func testConcurrently(t *testing.T, ctx context.Context, numTasks int, numThreads int, testFunc func(context.Context) bool) bool { + // prime the pump: + runSequentially(ctx, 1, testFunc) + + warn(context.TODO(), "---- Starting %d sequential tasks ----", numTasks) + sequentialDuration := runSequentially(ctx, numTasks, testFunc) + warn(context.TODO(), "---- Starting %d concurrent tasks on %d goroutines ----", numTasks, numThreads) + concurrentDuration := runConcurrently(ctx, numTasks, numThreads, testFunc) + warn(context.TODO(), "---- End ----") + + log.Printf("---- %d sequential took %v, concurrent (%d threads) took %v ... speedup is %f", + numTasks, sequentialDuration, numThreads, concurrentDuration, + float64(sequentialDuration)/float64(concurrentDuration)) + if numThreads < 4 { + // In CI there tend to be few threads available, and the machine is usually heavily + // loaded, which makes the timing unpredictable. + return true + } + return assert.LessOrEqual(t, float64(concurrentDuration), 1.1*float64(sequentialDuration)) +} + +func assertPriorTimeoutAtLeast(t testing.TB, ctx context.Context, min time.Duration) bool { + deadline, exists := ctx.Deadline() + return !exists || assert.GreaterOrEqual(t, time.Until(deadline), min) +} From fa73ec2cc4669dd7d79f2284e60b48dc66325509 Mon Sep 17 00:00:00 2001 From: Jens Alfke Date: Tue, 20 Jun 2023 13:58:25 -0700 Subject: [PATCH 2/7] Addressing review comments --- go.mod | 2 +- js/otto_runner.go | 2 +- js/otto_vm.go | 4 ++-- js/v8_runner.go | 13 ++++++++----- js/v8_template.go | 3 +-- js/v8_utils.go | 8 +------- js/v8_vm.go | 5 ++--- js/vmpool_test.go | 1 - 8 files changed, 16 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index d1851ec..303d756 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/couchbase/sg-bucket go 1.19 require ( - github.com/pkg/errors v0.9.1 github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f github.com/snej/v8go v1.7.3 github.com/stretchr/testify v1.7.1 @@ -16,6 +15,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.3.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect diff --git a/js/otto_runner.go b/js/otto_runner.go index ed52b79..2a0e13c 100644 --- a/js/otto_runner.go +++ b/js/otto_runner.go @@ -13,10 +13,10 @@ package js import ( "context" "encoding/json" + "errors" "fmt" "strconv" - "github.com/pkg/errors" "github.com/robertkrimen/otto" _ "github.com/robertkrimen/otto/underscore" diff --git a/js/otto_vm.go b/js/otto_vm.go index 3f4e29c..0d5df15 100644 --- a/js/otto_vm.go +++ b/js/otto_vm.go @@ -68,10 +68,10 @@ func (vm *ottoVM) getRunner(service *Service) (Runner, error) { return nil, fmt.Errorf("the js.VM has been closed") } if vm.curRunner != nil { - panic("illegal access to v8VM: already has a v8Runner") + panic("illegal access to ottoVM: already has a ottoRunner") } if !vm.services.hasService(service) { - return nil, fmt.Errorf("unknown js.Service instance passed to VM") + return nil, fmt.Errorf("unknown js.Service instance passed to VM: %v", service) } if service.v8Init != nil { return nil, fmt.Errorf("js.Service has custom initialization not supported by Otto") diff --git a/js/v8_runner.go b/js/v8_runner.go index ebb0cb4..fec9736 100644 --- a/js/v8_runner.go +++ b/js/v8_runner.go @@ -15,11 +15,11 @@ package js import ( "context" "encoding/json" + "errors" "fmt" "strings" "time" - "github.com/pkg/errors" v8 "github.com/snej/v8go" // Docs: https://pkg.go.dev/github.com/snej/v8go ) @@ -32,22 +32,25 @@ type V8Runner struct { Client any // You can put whatever you want here, to point back to your state } -func newV8Runner(vm *v8VM, template V8Template, id serviceID) (*V8Runner, error) { +func newV8Runner(vm *v8VM, template V8Template, id serviceID) (runner *V8Runner, err error) { // Create a V8 Context and run the setup script in it: ctx := v8.NewContext(vm.iso, template.Global()) + defer func() { + if err != nil { + ctx.Close() + } + }() if _, err := vm.setupScript.Run(ctx); err != nil { - return nil, errors.Wrap(err, "Unexpected error in JavaScript initialization code") + return nil, fmt.Errorf("Unexpected error in JavaScript initialization code: %w", err) } // Now run the service's script, which returns the service's main function: result, err := template.Script().Run(ctx) if err != nil { - ctx.Close() return nil, fmt.Errorf("JavaScript error initializing %s: %w", template.Name(), err) } mainFn, err := result.AsFunction() if err != nil { - ctx.Close() return nil, fmt.Errorf("%s's script did not return a function: %w", template.Name(), err) } diff --git a/js/v8_template.go b/js/v8_template.go index fc9f6b1..b0b5b93 100644 --- a/js/v8_template.go +++ b/js/v8_template.go @@ -15,7 +15,6 @@ package js import ( "fmt" - "github.com/pkg/errors" v8 "github.com/snej/v8go" ) @@ -114,7 +113,7 @@ func (t *V8BasicTemplate) NewCallback(callback TemplateCallback) *v8.FunctionTem if v8Result, newValErr := runner.NewValue(result); err == nil { return v8Result } else { - err = errors.Wrap(newValErr, "Could not convert a callback's result to JavaScript") + err = fmt.Errorf("Could not convert a callback's result to JavaScript: %w", newValErr) } } return v8Throw(vm.iso, err) diff --git a/js/v8_utils.go b/js/v8_utils.go index 5d1a0ae..57ad6a3 100644 --- a/js/v8_utils.go +++ b/js/v8_utils.go @@ -114,13 +114,7 @@ func newJSONString(ctx *v8.Context, val any) (*v8.Value, error) { // Returns an error back to a V8 caller. // Calls v8.Isolate.ThrowException, with the Go error's string as the message. func v8Throw(i *v8.Isolate, err error) *v8.Value { - var errStr string - // if httpErr, ok := err.(*base.HTTPError); ok { - // errStr = fmt.Sprintf("[%d] %s", httpErr.Status, httpErr.Message) - // } else { - errStr = err.Error() - // } - return i.ThrowException(newString(i, errStr)) + return i.ThrowException(newString(i, err.Error())) } // Simple utility to wrap a function that returns a value and an error; returns just the value, panicking if there was an error. diff --git a/js/v8_vm.go b/js/v8_vm.go index 039577b..d22ab41 100644 --- a/js/v8_vm.go +++ b/js/v8_vm.go @@ -18,7 +18,6 @@ import ( "sync" "time" - "github.com/pkg/errors" v8 "github.com/snej/v8go" ) @@ -123,7 +122,7 @@ func (vm *v8VM) release() { func (vm *v8VM) getTemplate(service *Service) (V8Template, error) { var tmpl V8Template if !vm.services.hasService(service) { - return nil, fmt.Errorf("unknown js.Service instance passed to VM") + return nil, fmt.Errorf("unknown js.Service instance passed to VM: %v", service) } if int(service.id) < len(vm.templates) { tmpl = vm.templates[service.id] @@ -134,7 +133,7 @@ func (vm *v8VM) getTemplate(service *Service) (V8Template, error) { var err error vm.setupScript, err = vm.iso.CompileUnboundScript(kSetupLoggingJS+kUnderscoreJS, "setupScript.js", v8.CompileOptions{}) if err != nil { - return nil, errors.Wrapf(err, "Couldn't compile setup script") + return nil, fmt.Errorf("Couldn't compile setup script: %w", err) } } diff --git a/js/vmpool_test.go b/js/vmpool_test.go index 25a1b3e..e8a04dc 100644 --- a/js/vmpool_test.go +++ b/js/vmpool_test.go @@ -68,7 +68,6 @@ func TestPoolsSequentially(t *testing.T) { func TestPoolsConcurrently(t *testing.T) { maxProcs := runtime.GOMAXPROCS(0) - log.Printf("FYI, GOMAXPROCS = %d", maxProcs) if !assert.GreaterOrEqual(t, maxProcs, 2, "Not enough OS threads available") { return } From 88ea0817fe4ad722ac5886bef7c767e825c9e3da Mon Sep 17 00:00:00 2001 From: Jens Alfke Date: Tue, 20 Jun 2023 14:43:52 -0700 Subject: [PATCH 3/7] Added license text --- js/logging.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/js/logging.go b/js/logging.go index 17d86da..d35a945 100644 --- a/js/logging.go +++ b/js/logging.go @@ -1,3 +1,13 @@ +/* +Copyright 2023-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + package js import ( From c77093455120f7a13df8573a8f8a7be061fd94b0 Mon Sep 17 00:00:00 2001 From: Jens Alfke Date: Tue, 20 Jun 2023 16:35:02 -0700 Subject: [PATCH 4/7] js module: Improved use of Contexts - VM and VMPool now have an associated Context, which must be given when creating one. (VMs created by VMPool inherit its Context.) - VM and VMPool now have a Context() method. - Runner.Context() now returns the VM's Context by default, but calling Runner.SetContext() overrides it until the Runner is returned. - Removed Runner.ContextOrDefault(); use Context() instead. - The `ctx` parameter to Service.Run() may be nil, in which case the default Context (the VM's) is used. - Tests use a new testCtx() function that's similar to the one in SG.base. It creates a Context derived from context.TODO that cancels when the test exits. Some renaming to avoid confusion: - Renamed v8Runner.ctx to v8ctx. - Renamed baseRunner.goContext to ctx. - Renamed other variables of type *v8.Context to v8ctx. --- js/js_test.go | 52 ++++++++++++++++++++--------------------------- js/otto_runner.go | 2 +- js/otto_vm.go | 9 ++++---- js/runner.go | 25 +++++++++-------------- js/service.go | 12 +++++++---- js/test_utils.go | 24 +++++++++++++++++----- js/v8_runner.go | 40 +++++++++++++++++++----------------- js/v8_template.go | 2 +- js/v8_utils.go | 6 +++--- js/v8_vm.go | 15 +++++++------- js/vm.go | 18 ++++++++++------ js/vmpool.go | 35 +++++++++++++++---------------- js/vmpool_test.go | 26 ++++++++++++------------ 13 files changed, 139 insertions(+), 127 deletions(-) diff --git a/js/js_test.go b/js/js_test.go index d3b8822..70bf772 100644 --- a/js/js_test.go +++ b/js/js_test.go @@ -26,25 +26,20 @@ import ( ) func TestSquare(t *testing.T) { - ctx := context.Background() TestWithVMs(t, func(t *testing.T, vm VM) { + assert.NotNil(t, vm.Context()) service := NewService(vm, "square", `function(n) {return n * n;}`) assert.Equal(t, "square", service.Name()) assert.Equal(t, vm, service.Host()) // Test Run: - result, err := service.Run(ctx, 13) + result, err := service.Run(vm.Context(), 13) assert.NoError(t, err) assert.EqualValues(t, 169, result) // Test WithRunner: result, err = service.WithRunner(func(runner Runner) (any, error) { - assert.Nil(t, runner.Context()) - assert.NotNil(t, runner.ContextOrDefault()) - runner.SetContext(ctx) - assert.Equal(t, ctx, runner.Context()) - assert.Equal(t, ctx, runner.ContextOrDefault()) - + assert.Equal(t, vm.Context(), runner.Context()) return runner.Run(9) }) assert.NoError(t, err) @@ -53,7 +48,7 @@ func TestSquare(t *testing.T) { } func TestSquareV8Args(t *testing.T) { - vm := V8.NewVM() + vm := V8.NewVM(testCtx(t)) defer vm.Close() service := NewService(vm, "square", `function(n) {return n * n;}`) @@ -70,10 +65,10 @@ func TestSquareV8Args(t *testing.T) { } func TestJSON(t *testing.T) { - ctx := context.Background() + ctx := testCtx(t) var pool VMPool - pool.Init(V8, 4) + pool.Init(ctx, V8, 4) defer pool.Close() service := NewService(&pool, "length", `function(v) {return v.length;}`) @@ -90,9 +85,9 @@ func TestJSON(t *testing.T) { } func TestCallback(t *testing.T) { - ctx := context.Background() + ctx := testCtx(t) - vm := V8.NewVM() + vm := V8.NewVM(ctx) defer vm.Close() src := `(function() { @@ -123,8 +118,6 @@ func TestCallback(t *testing.T) { // Test conversion of numbers into/out of JavaScript. func TestNumbers(t *testing.T) { - ctx := context.Background() - TestWithVMs(t, func(t *testing.T, vm VM) { service := NewService(vm, "numbers", `function(n, expectedStr) { if (typeof(n) != 'number' && typeof(n) != 'bigint') throw "Unexpectedly n is a " + typeof(n); @@ -136,7 +129,7 @@ func TestNumbers(t *testing.T) { t.Run("integers", func(t *testing.T) { testInt := func(n int64) { - result, err := service.Run(ctx, n, strconv.FormatInt(n, 10)) + result, err := service.Run(nil, n, strconv.FormatInt(n, 10)) if assert.NoError(t, err) { assert.EqualValues(t, n, result) } @@ -159,7 +152,7 @@ func TestNumbers(t *testing.T) { t.Run("floats", func(t *testing.T) { testFloat := func(n float64) { - result, err := service.Run(ctx, n, strconv.FormatFloat(n, 'f', -1, 64)) + result, err := service.Run(nil, n, strconv.FormatFloat(n, 'f', -1, 64)) if assert.NoError(t, err) { assert.EqualValues(t, n, result) } @@ -184,7 +177,7 @@ func TestNumbers(t *testing.T) { t.Run("json_Number_integer", func(t *testing.T) { hugeInt := json.Number("123456789012345") - result, err := service.Run(ctx, hugeInt, string(hugeInt)) + result, err := service.Run(nil, hugeInt, string(hugeInt)) if assert.NoError(t, err) { assert.EqualValues(t, 123456789012345, result) } @@ -193,7 +186,7 @@ func TestNumbers(t *testing.T) { if vm.Engine().languageVersion >= 11 { // (Otto does not support BigInts) t.Run("json_Number_huge_integer", func(t *testing.T) { hugeInt := json.Number("1234567890123456789012345678901234567890") - result, err := service.Run(ctx, hugeInt, string(hugeInt)) + result, err := service.Run(nil, hugeInt, string(hugeInt)) if assert.NoError(t, err) { ibig := new(big.Int) ibig, _ = ibig.SetString(string(hugeInt), 10) @@ -204,7 +197,7 @@ func TestNumbers(t *testing.T) { t.Run("json_Number_float", func(t *testing.T) { floatStr := json.Number("1234567890.123") - result, err := service.Run(ctx, floatStr, string(floatStr)) + result, err := service.Run(nil, floatStr, string(floatStr)) if assert.NoError(t, err) { assert.EqualValues(t, 1234567890.123, result) } @@ -214,9 +207,9 @@ func TestNumbers(t *testing.T) { // For security purposes, verify that JS APIs to do network or file I/O are not present: func TestNoIO(t *testing.T) { - ctx := context.Background() + ctx := testCtx(t) - vm := V8.NewVM() // Otto appears to have no way to refer to the global object... + vm := V8.NewVM(ctx) // Otto appears to have no way to refer to the global object... defer vm.Close() service := NewService(vm, "check", `function() { @@ -237,9 +230,8 @@ func TestNoIO(t *testing.T) { // Verify that ECMAScript modules can't be loaded. (The older `require` is checked in TestNoIO.) func TestNoModules(t *testing.T) { - ctx := context.Background() - - vm := V8.NewVM() // Otto doesn't support ES modules + ctx := testCtx(t) + vm := V8.NewVM(ctx) // Otto doesn't support ES modules defer vm.Close() src := `import foo from 'foo'; @@ -255,7 +247,7 @@ func TestNoModules(t *testing.T) { func TestTimeout(t *testing.T) { TestWithVMs(t, func(t *testing.T, vm VM) { - ctx := context.Background() + ctx := vm.Context() ctx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() @@ -269,9 +261,9 @@ func TestTimeout(t *testing.T) { } func TestOutOfMemory(t *testing.T) { - vm := V8.NewVM() + ctx := testCtx(t) + vm := V8.NewVM(ctx) defer vm.Close() - ctx := context.Background() service := NewService(vm, "OOM", ` function() { @@ -285,9 +277,9 @@ func TestOutOfMemory(t *testing.T) { } func TestStackOverflow(t *testing.T) { - vm := V8.NewVM() + ctx := testCtx(t) + vm := V8.NewVM(ctx) defer vm.Close() - ctx := context.Background() service := NewService(vm, "Overflow", ` function() { diff --git a/js/otto_runner.go b/js/otto_runner.go index 2a0e13c..feb7fb2 100644 --- a/js/otto_runner.go +++ b/js/otto_runner.go @@ -60,7 +60,7 @@ func newOttoRunner(vm *ottoVM, service *Service) (*OttoRunner, error) { extra += str + " " } - LoggingCallback(r.ContextOrDefault(), LogLevel(ilevel), "%s %s", message, extra) + LoggingCallback(r.Context(), LogLevel(ilevel), "%s %s", message, extra) return otto.UndefinedValue() }) if err != nil { diff --git a/js/otto_vm.go b/js/otto_vm.go index 0d5df15..201a1b4 100644 --- a/js/otto_vm.go +++ b/js/otto_vm.go @@ -11,6 +11,7 @@ licenses/APL2.txt. package js import ( + "context" "fmt" "time" ) @@ -27,10 +28,10 @@ const ottoVMName = "Otto" var Otto = &Engine{ name: ottoVMName, languageVersion: 5, // https://github.com/robertkrimen/otto#caveat-emptor - factory: func(engine *Engine, services *servicesConfiguration) VM { + factory: func(ctx context.Context, engine *Engine, services *servicesConfiguration) VM { return &ottoVM{ - baseVM: &baseVM{engine: engine, services: services}, // "superclass" - runners: []*OttoRunner{}, // Cached reusable Runners + baseVM: &baseVM{ctx: ctx, engine: engine, services: services}, // "superclass" + runners: []*OttoRunner{}, // Cached reusable Runners } }, } @@ -107,7 +108,7 @@ func (vm *ottoVM) withRunner(service *Service, fn func(Runner) (any, error)) (an } func (vm *ottoVM) returnRunner(r *OttoRunner) { - r.goContext = nil + r.ctx = nil // clear any override if vm.curRunner == r { vm.curRunner = nil } else if r.vm != vm { diff --git a/js/runner.go b/js/runner.go index 2b0871f..059b0f5 100644 --- a/js/runner.go +++ b/js/runner.go @@ -26,10 +26,8 @@ type Runner interface { // Associates a Go `Context` with this Runner. // If this Context has a deadline, JS calls will abort if it expires. SetContext(ctx context.Context) - // The associated `Context`, if you've set one; else nil. + // The associated `Context`. Defaults to the VM's Context. Context() context.Context - // The associated `Context`, else the default `context.Background()` instance. - ContextOrDefault() context.Context // Returns the remaining duration until the Context's deadline, or nil if none. Timeout() *time.Duration // Runs the Service's JavaScript function. @@ -44,30 +42,27 @@ type Runner interface { type baseRunner struct { id serviceID // The service ID in its VM vm VM // The owning VM object - goContext context.Context // context.Context value for use by Go callbacks + ctx context.Context // Overrides vm.Context() if non-nil associated any } func (r *baseRunner) VM() VM { return r.vm } func (r *baseRunner) AssociatedValue() any { return r.associated } func (r *baseRunner) SetAssociatedValue(obj any) { r.associated = obj } -func (r *baseRunner) SetContext(ctx context.Context) { r.goContext = ctx } -func (r *baseRunner) Context() context.Context { return r.goContext } +func (r *baseRunner) SetContext(ctx context.Context) { r.ctx = ctx } -func (r *baseRunner) ContextOrDefault() context.Context { - if r.goContext != nil { - return r.goContext +func (r *baseRunner) Context() context.Context { + if r.ctx != nil { + return r.ctx } else { - return context.TODO() + return r.vm.Context() } } func (r *baseRunner) Timeout() *time.Duration { - if r.goContext != nil { - if deadline, hasDeadline := r.goContext.Deadline(); hasDeadline { - timeout := time.Until(deadline) - return &timeout - } + if deadline, hasDeadline := r.Context().Deadline(); hasDeadline { + timeout := time.Until(deadline) + return &timeout } return nil } diff --git a/js/service.go b/js/service.go index f5c664e..a7b3163 100644 --- a/js/service.go +++ b/js/service.go @@ -28,6 +28,7 @@ type serviceID uint32 // internal ID, used as an array index in VM and VMPool. // A provider of a JavaScript runtime for Services. VM and VMPool implement this. type ServiceHost interface { Engine() *Engine + Context() context.Context Close() FindService(name string) *Service registerService(*Service) @@ -49,7 +50,7 @@ type TemplateFactory func(base *V8BasicTemplate) (V8Template, error) // The source code should be of the form `function(arg1,arg2…) {…body…; return result;}`. // If you have a more complex script, like one that defines several functions, use NewCustomService. func NewService(host ServiceHost, name string, jsFunctionSource string) *Service { - debug(context.Background(), "Creating JavaScript service %q", name) + debug(host.Context(), "Creating JavaScript service %q", name) service := &Service{ host: host, name: name, @@ -82,7 +83,7 @@ func (service *Service) Host() ServiceHost { return service.host } // - If the host is a VM, this call will fail if there is another Runner in use belonging to any // Service hosted by that VM. func (service *Service) GetRunner() (Runner, error) { - debug(context.Background(), "Running JavaScript service %q", service.name) + debug(service.host.Context(), "Running JavaScript service %q", service.name) return service.host.getRunner(service) } @@ -93,12 +94,15 @@ func (service *Service) WithRunner(fn func(Runner) (any, error)) (any, error) { } // A high-level method that runs a service in a VM without your needing to interact with a Runner. -// The arguments can be Go types or V8 Values; any types supported by VM.NewValue. +// The `ctx` parameter may be nil, to use the VM's Context. +// The arguments can be Go types or V8 Values: any types supported by VM.NewValue. // The result is converted back to a Go type. // If the function throws a JavaScript exception it's converted to a Go `error`. func (service *Service) Run(ctx context.Context, args ...any) (any, error) { return service.WithRunner(func(runner Runner) (any, error) { - runner.SetContext(ctx) + if ctx != nil { + runner.SetContext(ctx) + } return runner.Run(args...) }) } diff --git a/js/test_utils.go b/js/test_utils.go index a3cc278..545d6c2 100644 --- a/js/test_utils.go +++ b/js/test_utils.go @@ -10,15 +10,27 @@ licenses/APL2.txt. package js -import "testing" +import ( + "context" + "testing" +) + +// testCtx creates a context for the given test which is also cancelled once the test has completed. +func testCtx(t testing.TB) context.Context { + ctx, cancelCtx := context.WithCancel(context.TODO()) + t.Cleanup(cancelCtx) + return ctx +} // Unit-test utility. Calls the function with each supported type of VM (Otto and V8). func TestWithVMs(t *testing.T, fn func(t *testing.T, vm VM)) { for _, engine := range testEngines { t.Run(engine.String(), func(t *testing.T) { - vm := engine.NewVM() - defer vm.Close() + ctx, cancelCtx := context.WithCancel(context.TODO()) + vm := engine.NewVM(ctx) fn(t, vm) + vm.Close() + cancelCtx() }) } } @@ -28,9 +40,11 @@ func TestWithVMs(t *testing.T, fn func(t *testing.T, vm VM)) { func TestWithVMPools(t *testing.T, maxVMs int, fn func(t *testing.T, pool *VMPool)) { for _, engine := range testEngines { t.Run(engine.String(), func(t *testing.T) { - pool := NewVMPool(engine, maxVMs) - defer pool.Close() + ctx, cancelCtx := context.WithCancel(context.TODO()) + pool := NewVMPool(ctx, engine, maxVMs) fn(t, pool) + pool.Close() + cancelCtx() }) } } diff --git a/js/v8_runner.go b/js/v8_runner.go index fec9736..f4bd98f 100644 --- a/js/v8_runner.go +++ b/js/v8_runner.go @@ -27,25 +27,25 @@ type V8Runner struct { baseRunner v8vm *v8VM template V8Template // The Service template I'm created from - ctx *v8.Context // V8 object managing this execution context + v8ctx *v8.Context // V8 object managing this execution context mainFn *v8.Function // The entry-point function (returned by the Service's script) Client any // You can put whatever you want here, to point back to your state } func newV8Runner(vm *v8VM, template V8Template, id serviceID) (runner *V8Runner, err error) { // Create a V8 Context and run the setup script in it: - ctx := v8.NewContext(vm.iso, template.Global()) + v8ctx := v8.NewContext(vm.iso, template.Global()) defer func() { if err != nil { - ctx.Close() + v8ctx.Close() } }() - if _, err := vm.setupScript.Run(ctx); err != nil { + if _, err := vm.setupScript.Run(v8ctx); err != nil { return nil, fmt.Errorf("Unexpected error in JavaScript initialization code: %w", err) } // Now run the service's script, which returns the service's main function: - result, err := template.Script().Run(ctx) + result, err := template.Script().Run(v8ctx) if err != nil { return nil, fmt.Errorf("JavaScript error initializing %s: %w", template.Name(), err) } @@ -61,7 +61,7 @@ func newV8Runner(vm *v8VM, template V8Template, id serviceID) (runner *V8Runner, }, v8vm: vm, template: template, - ctx: ctx, + v8ctx: v8ctx, mainFn: mainFn, }, nil } @@ -78,9 +78,9 @@ func (r *V8Runner) close() { if r.vm != nil && r.v8vm.curRunner == r { r.v8vm.curRunner = nil } - if r.ctx != nil { - r.ctx.Close() - r.ctx = nil + if r.v8ctx != nil { + r.v8ctx.Close() + r.v8ctx = nil } r.vm = nil } @@ -178,8 +178,8 @@ func (r *V8Runner) ResolvePromise(val *v8.Value, err error) (*v8.Value, error) { case v8.Rejected: return nil, errors.New(p.Result().DetailString()) case v8.Pending: - r.ctx.PerformMicrotaskCheckpoint() // run VM to make progress on the promise - deadline, hasDeadline := r.ContextOrDefault().Deadline() + r.v8ctx.PerformMicrotaskCheckpoint() // run VM to make progress on the promise + deadline, hasDeadline := r.Context().Deadline() if hasDeadline && time.Now().After(deadline) { return nil, context.DeadlineExceeded } @@ -191,7 +191,7 @@ func (r *V8Runner) ResolvePromise(val *v8.Value, err error) (*v8.Value, error) { } func (r *V8Runner) WithTemporaryValues(fn func()) { - r.ctx.WithTemporaryValues(fn) + r.v8ctx.WithTemporaryValues(fn) } //////// CONVERTING GO VALUES TO JAVASCRIPT: @@ -222,7 +222,7 @@ func (r *V8Runner) NewValue(val any) (v8Val *v8.Value, err error) { if val == nil { return r.NullValue(), nil // v8.NewValue panics if given nil :-p } - v8Val, err = r.ctx.NewValue(val) + v8Val, err = r.v8ctx.NewValue(val) if err != nil { if jsonStr, ok := val.(JSONString); ok { if jsonStr != "" { @@ -238,16 +238,16 @@ func (r *V8Runner) NewValue(val any) (v8Val *v8.Value, err error) { } // Creates a JavaScript number value. -func (r *V8Runner) NewInt(i int) *v8.Value { return mustSucceed(r.ctx.NewValue(i)) } +func (r *V8Runner) NewInt(i int) *v8.Value { return mustSucceed(r.v8ctx.NewValue(i)) } // Creates a JavaScript string value. func (r *V8Runner) NewString(str string) *v8.Value { return newString(r.v8vm.iso, str) } // Marshals a Go value to JSON and returns it as a V8 string. -func (r *V8Runner) NewJSONString(val any) (*v8.Value, error) { return newJSONString(r.ctx, val) } +func (r *V8Runner) NewJSONString(val any) (*v8.Value, error) { return newJSONString(r.v8ctx, val) } // Parses a JSON string to a V8 value, by calling JSON.parse() on it. -func (r *V8Runner) JSONParse(json string) (*v8.Value, error) { return v8.JSONParse(r.ctx, json) } +func (r *V8Runner) JSONParse(json string) (*v8.Value, error) { return v8.JSONParse(r.v8ctx, json) } // Returns a value representing JavaScript 'undefined'. func (r *V8Runner) UndefinedValue() *v8.Value { return v8.Undefined(r.v8vm.iso) } @@ -258,18 +258,20 @@ func (r *V8Runner) NullValue() *v8.Value { return v8.Null(r.v8vm.iso) } //////// CONVERTING JAVASCRIPT VALUES BACK TO GO: // Encodes a V8 value as a JSON string. -func (r *V8Runner) JSONStringify(val *v8.Value) (string, error) { return v8.JSONStringify(r.ctx, val) } +func (r *V8Runner) JSONStringify(val *v8.Value) (string, error) { + return v8.JSONStringify(r.v8ctx, val) +} //////// INSTANTIATING TEMPLATES: // Creates a V8 Object from a template previously created by BasicTemplate.NewObjectTemplate. // (Not needed if you added the template as a property of the global object.) func (r *V8Runner) NewInstance(o *v8.ObjectTemplate) (*v8.Object, error) { - return o.NewInstance(r.ctx) + return o.NewInstance(r.v8ctx) } // Creates a V8 Function from a template previously created by BasicTemplate.NewCallback. // (Not needed if you added the template as a property of the global object.) func (r *V8Runner) NewFunctionInstance(f *v8.FunctionTemplate) *v8.Function { - return f.GetFunction(r.ctx) + return f.GetFunction(r.v8ctx) } diff --git a/js/v8_template.go b/js/v8_template.go index b0b5b93..902b10a 100644 --- a/js/v8_template.go +++ b/js/v8_template.go @@ -182,7 +182,7 @@ func (service *V8BasicTemplate) defineSgLog() error { extra += " " extra += args[i].DetailString() } - LoggingCallback(r.ContextOrDefault(), level, "%s %s", msg, extra) + LoggingCallback(r.Context(), level, "%s %s", msg, extra) } return nil, nil })) diff --git a/js/v8_utils.go b/js/v8_utils.go index 57ad6a3..7d830aa 100644 --- a/js/v8_utils.go +++ b/js/v8_utils.go @@ -99,13 +99,13 @@ func newString(i *v8.Isolate, str string) *v8.Value { } // Marshals a Go value to JSON, and returns the string as a V8 Value. -func newJSONString(ctx *v8.Context, val any) (*v8.Value, error) { +func newJSONString(v8ctx *v8.Context, val any) (*v8.Value, error) { if val == nil { - return v8.Null(ctx.Isolate()), nil + return v8.Null(v8ctx.Isolate()), nil } else if jsonBytes, err := json.Marshal(val); err != nil { return nil, err } else { - return ctx.NewValue(string(jsonBytes)) + return v8ctx.NewValue(string(jsonBytes)) } } diff --git a/js/v8_vm.go b/js/v8_vm.go index d22ab41..90ff74d 100644 --- a/js/v8_vm.go +++ b/js/v8_vm.go @@ -13,6 +13,7 @@ licenses/APL2.txt. package js import ( + "context" _ "embed" "fmt" "sync" @@ -71,12 +72,12 @@ var V8 = &Engine{ var v8Init sync.Once -func v8VMFactory(engine *Engine, services *servicesConfiguration) VM { +func v8VMFactory(ctx context.Context, engine *Engine, services *servicesConfiguration) VM { v8Init.Do(func() { v8.SetFlags(fmt.Sprintf("--stack_size=%d", V8StackSizeLimit/1024)) }) return &v8VM{ - baseVM: &baseVM{engine: engine, services: services}, + baseVM: &baseVM{ctx: ctx, engine: engine, services: services}, iso: v8.NewIsolateWith(V8InitialHeap, V8MaxHeap), // The V8 v8VM templates: []V8Template{}, // Instantiated Services runners: []*V8Runner{}, // Cached reusable Runners @@ -207,7 +208,7 @@ func (vm *v8VM) withRunner(service *Service, fn func(Runner) (any, error)) (any, // Called by v8Runner.Return; either closes its V8 resources or saves it for reuse. // Also returns the v8VM to its Pool, if it came from one. func (vm *v8VM) returnRunner(r *V8Runner) { - r.goContext = nil + r.ctx = nil // clear any override if vm.curRunner == r { vm.iso.Unlock() vm.curRunner = nil @@ -226,15 +227,15 @@ func (vm *v8VM) returnRunner(r *V8Runner) { } // Returns the v8Runner that owns the given V8 Context. -func (vm *v8VM) currentRunner(ctx *v8.Context) *V8Runner { +func (vm *v8VM) currentRunner(v8ctx *v8.Context) *V8Runner { // IMPORTANT: This is kind of a hack, but we can get away with it because a v8VM has only one // active v8Runner at a time. If it were to be multiple Runners, we'd need to maintain a map // from Contexts to Runners. if vm.curRunner == nil { - panic(fmt.Sprintf("Unknown v8.Context passed to v8VM.currentRunner: %v, expected none", ctx)) + panic(fmt.Sprintf("Unknown v8.Context passed to v8VM.currentRunner: %v, expected none", v8ctx)) } - if ctx != vm.curRunner.ctx { - panic(fmt.Sprintf("Unknown v8.Context passed to v8VM.currentRunner: %v, expected %v", ctx, vm.curRunner.ctx)) + if v8ctx != vm.curRunner.v8ctx { + panic(fmt.Sprintf("Unknown v8.Context passed to v8VM.currentRunner: %v, expected %v", v8ctx, vm.curRunner.v8ctx)) } return vm.curRunner } diff --git a/js/vm.go b/js/vm.go index de5e7bb..a5b61e8 100644 --- a/js/vm.go +++ b/js/vm.go @@ -21,7 +21,7 @@ import ( type Engine struct { name string languageVersion int - factory func(*Engine, *servicesConfiguration) VM + factory func(context.Context, *Engine, *servicesConfiguration) VM } // The name identifying this engine ("V8" or "Otto") @@ -37,12 +37,12 @@ const ES2015 = 6 // Creates a JavaScript virtual machine of the given type. // This object should be used only on a single goroutine at a time. -func (engine *Engine) NewVM() VM { - return engine.newVM(&servicesConfiguration{}) +func (engine *Engine) NewVM(ctx context.Context) VM { + return engine.newVM(ctx, &servicesConfiguration{}) } -func (engine *Engine) newVM(services *servicesConfiguration) VM { - return engine.factory(engine, services) +func (engine *Engine) newVM(ctx context.Context, services *servicesConfiguration) VM { + return engine.factory(ctx, engine, services) } //////// VM @@ -56,6 +56,7 @@ func (engine *Engine) newVM(services *servicesConfiguration) VM { // The VMPool takes care of this, by vending VM instances that are known not to be in use. type VM interface { Engine() *Engine + Context() context.Context Close() FindService(name string) *Service @@ -86,7 +87,7 @@ func ValidateJavascriptFunction(vm VM, jsFunc string, minArgs int, maxArgs int) } `) } - _, err := service.Run(context.Background(), jsFunc, minArgs, maxArgs) + _, err := service.Run(vm.Context(), jsFunc, minArgs, maxArgs) return err } @@ -94,6 +95,7 @@ func ValidateJavascriptFunction(vm VM, jsFunc string, minArgs int, maxArgs int) // A base "class" containing shared properties and methods for use by VM implementations. type baseVM struct { + ctx context.Context engine *Engine services *servicesConfiguration // Factories for services returnToPool *VMPool // Pool to return me to, or nil @@ -103,6 +105,10 @@ type baseVM struct { func (vm *baseVM) Engine() *Engine { return vm.engine } +func (vm *baseVM) Context() context.Context { + return vm.ctx +} + func (vm *baseVM) close() { if vm.returnToPool != nil { panic("Don't Close a VM that belongs to a VMPool") diff --git a/js/vmpool.go b/js/vmpool.go index 0da4dc8..cc6f01e 100644 --- a/js/vmpool.go +++ b/js/vmpool.go @@ -22,6 +22,7 @@ import ( // A thread-safe ServiceHost for Services and Runners that owns a set of VMs // and allocates an available one when a Runner is needed. type VMPool struct { + ctx context.Context // Caller-settable Context maxInUse int // Max number of simultaneously in-use VMs services *servicesConfiguration // Defines the services (owned VMs also have references) tickets chan bool // Each item in this channel represents availability of a VM @@ -31,13 +32,14 @@ type VMPool struct { curInUse_ int // Current number of VMs "checked out" } -func NewVMPool(typ *Engine, maxVMs int) *VMPool { +func NewVMPool(ctx context.Context, typ *Engine, maxVMs int) *VMPool { pool := new(VMPool) - pool.Init(typ, maxVMs) + pool.Init(ctx, typ, maxVMs) return pool } -func (pool *VMPool) Init(typ *Engine, maxVMs int) { +func (pool *VMPool) Init(ctx context.Context, typ *Engine, maxVMs int) { + pool.ctx = ctx pool.maxInUse = maxVMs pool.services = &servicesConfiguration{} pool.engine = typ @@ -46,17 +48,18 @@ func (pool *VMPool) Init(typ *Engine, maxVMs int) { for i := 0; i < maxVMs; i++ { pool.tickets <- true } - info(context.Background(), "js.VMPool: Init, max %d VMs", maxVMs) + info(pool.ctx, "js.VMPool: Init, max %d VMs", maxVMs) } -func (pool *VMPool) Engine() *Engine { return pool.engine } +func (pool *VMPool) Context() context.Context { return pool.ctx } +func (pool *VMPool) Engine() *Engine { return pool.engine } // Tears down a VMPool, freeing up its cached VMs. // It's a good idea to call this when using V8, as the VMs may be holding onto a lot of external // memory managed by V8, and this will clean up that memory sooner than Go's GC will. func (pool *VMPool) Close() { if inUse := pool.InUseCount(); inUse > 0 { - warn(context.Background(), "A js.VMPool is being closed with %d VMs still in use", inUse) + warn(pool.ctx, "A js.VMPool is being closed with %d VMs still in use", inUse) } // First stop all waiting `Get` calls: @@ -65,8 +68,7 @@ func (pool *VMPool) Close() { // Now pull all the VMs out of the pool and close them. // This isn't necessary, but it frees up memory sooner. n := pool.closeAll() - info(context.Background(), - "js.VMPool.Close: Closed pool with %d VM(s)", n) + info(pool.ctx, "js.VMPool.Close: Closed pool with %d VM(s)", n) } // Returns the number of VMs currently in use. @@ -79,8 +81,7 @@ func (pool *VMPool) InUseCount() int { // Closes all idle VMs cached by this pool. It will reallocate them when it needs to. func (pool *VMPool) PurgeUnusedVMs() { n := pool.closeAll() - info(context.Background(), - "js.VMPool.PurgeUnusedVMs: Closed %d idle VM(s)", n) + info(pool.ctx, "js.VMPool.PurgeUnusedVMs: Closed %d idle VM(s)", n) } func (pool *VMPool) FindService(name string) *Service { @@ -108,14 +109,12 @@ func (pool *VMPool) getVM(service *Service) (VM, error) { vm, inUse := pool.pop(service) if vm == nil { // Nothing in the pool, so create a new VM instance. - vm = pool.engine.newVM(pool.services) - info(context.Background(), - "js.VMPool.getVM: No VMs free; created a new one") + vm = pool.engine.newVM(pool.ctx, pool.services) + info(pool.ctx, "js.VMPool.getVM: No VMs free; created a new one") } vm.setReturnToPool(pool) - debug(context.Background(), - "js.VMPool.getVM: %d/%d VMs now in use", inUse, pool.maxInUse) + debug(pool.ctx, "js.VMPool.getVM: %d/%d VMs now in use", inUse, pool.maxInUse) return vm, nil } @@ -125,8 +124,7 @@ func (pool *VMPool) returnVM(vm VM) { vm.setReturnToPool(nil) inUse := pool.push(vm) - debug(context.Background(), - "js.VMPool.returnVM: %d/%d VMs now in use", inUse, pool.maxInUse) + debug(pool.ctx, "js.VMPool.returnVM: %d/%d VMs now in use", inUse, pool.maxInUse) // Return a ticket to the channel: pool.tickets <- true @@ -195,8 +193,7 @@ func (pool *VMPool) pop(service *Service) (vm VM, inUse int) { vms = vms[1:] oldest.setReturnToPool(nil) oldest.Close() - debug(context.Background(), - "js.VMPool.pop: Disposed a stale VM not used in %v", stale) + debug(pool.ctx, "js.VMPool.pop: Disposed a stale VM not used in %v", stale) } } pool.vms_ = vms diff --git a/js/vmpool_test.go b/js/vmpool_test.go index e8a04dc..99fc0cd 100644 --- a/js/vmpool_test.go +++ b/js/vmpool_test.go @@ -52,7 +52,7 @@ const kVMPoolTestTimeout = 60 * time.Second const kVMPoolTestNumTasks = 65536 func TestPoolsSequentially(t *testing.T) { - ctx := context.Background() + ctx := testCtx(t) if !assertPriorTimeoutAtLeast(t, ctx, kVMPoolTestTimeout) { return @@ -72,7 +72,7 @@ func TestPoolsConcurrently(t *testing.T) { return } - ctx := context.Background() + ctx := testCtx(t) if !assertPriorTimeoutAtLeast(t, ctx, kVMPoolTestTimeout) { return @@ -93,11 +93,11 @@ func TestPoolsConcurrently(t *testing.T) { //////// CONCURRENCY BENCHMARKS func BenchmarkVMPoolIntsSequentially(b *testing.B) { - ctx := context.Background() + ctx := testCtx(b) if !assertPriorTimeoutAtLeast(b, ctx, kVMPoolTestTimeout) { return } - pool := NewVMPool(V8, 32) + pool := NewVMPool(ctx, V8, 32) service := NewService(pool, "testy", kVMPoolTestScript) testFunc := func(ctx context.Context) bool { result, err := service.Run(ctx, 13) @@ -111,8 +111,8 @@ func BenchmarkVMPoolIntsSequentially(b *testing.B) { func BenchmarkVMPoolIntsConcurrently(b *testing.B) { const kNumThreads = 8 - ctx := context.Background() - pool := NewVMPool(V8, 32) + ctx := testCtx(b) + pool := NewVMPool(ctx, V8, 32) service := NewService(pool, "testy", kVMPoolTestScript) testFunc := func(ctx context.Context) bool { result, err := service.Run(ctx, 13) @@ -126,8 +126,8 @@ func BenchmarkVMPoolIntsConcurrently(b *testing.B) { func BenchmarkVMPoolStringsSequentially(b *testing.B) { fmt.Printf("-------- N = %d -------\n", b.N) - ctx := context.Background() - pool := NewVMPool(V8, 32) + ctx := testCtx(b) + pool := NewVMPool(ctx, V8, 32) service := NewService(pool, "testy", kVMPoolTestScript) testFunc := func(ctx context.Context) bool { result, err := service.Run(ctx, "This is a test of the js package") @@ -141,8 +141,8 @@ func BenchmarkVMPoolStringsSequentially(b *testing.B) { func BenchmarkVMPoolStringsConcurrently(b *testing.B) { const kNumThreads = 8 - ctx := context.Background() - pool := NewVMPool(V8, 32) + ctx := testCtx(b) + pool := NewVMPool(ctx, V8, 32) service := NewService(pool, "testy", kVMPoolTestScript) testFunc := func(ctx context.Context) bool { result, err := service.Run(ctx, "This is a test of the js package") @@ -195,11 +195,11 @@ func testConcurrently(t *testing.T, ctx context.Context, numTasks int, numThread // prime the pump: runSequentially(ctx, 1, testFunc) - warn(context.TODO(), "---- Starting %d sequential tasks ----", numTasks) + warn(ctx, "---- Starting %d sequential tasks ----", numTasks) sequentialDuration := runSequentially(ctx, numTasks, testFunc) - warn(context.TODO(), "---- Starting %d concurrent tasks on %d goroutines ----", numTasks, numThreads) + warn(ctx, "---- Starting %d concurrent tasks on %d goroutines ----", numTasks, numThreads) concurrentDuration := runConcurrently(ctx, numTasks, numThreads, testFunc) - warn(context.TODO(), "---- End ----") + warn(ctx, "---- End ----") log.Printf("---- %d sequential took %v, concurrent (%d threads) took %v ... speedup is %f", numTasks, sequentialDuration, numThreads, concurrentDuration, From 5ee386619ff4b3d56763eade02f97d3506435ac6 Mon Sep 17 00:00:00 2001 From: Jens Alfke Date: Wed, 21 Jun 2023 16:26:23 -0700 Subject: [PATCH 5/7] Removed panics - panics in illegal situations replaced by logError() and returning, with an error if possible. - V8Runner.NewInt() and NewString() now return an error because the V8 call returns an error, although I believe it can only fail in the unlikely case that V8 runs out of memory. --- js/js_test.go | 5 ++++- js/otto_runner.go | 2 +- js/otto_vm.go | 5 +++-- js/v8_runner.go | 4 ++-- js/v8_template.go | 2 +- js/v8_utils.go | 25 ++++++++++++------------- js/v8_vm.go | 8 ++++---- js/vm.go | 9 +++++---- js/vmpool.go | 3 ++- 9 files changed, 34 insertions(+), 29 deletions(-) diff --git a/js/js_test.go b/js/js_test.go index 70bf772..b73da0f 100644 --- a/js/js_test.go +++ b/js/js_test.go @@ -23,6 +23,7 @@ import ( "github.com/snej/v8go" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSquare(t *testing.T) { @@ -54,7 +55,9 @@ func TestSquareV8Args(t *testing.T) { service := NewService(vm, "square", `function(n) {return n * n;}`) result, err := service.WithRunner(func(runner Runner) (any, error) { v8Runner := runner.(*V8Runner) - result, err := v8Runner.RunWithV8Args(v8Runner.NewInt(9)) + nine, err := v8Runner.NewInt(9) + require.NoError(t, err) + result, err := v8Runner.RunWithV8Args(nine) if err != nil { return nil, err } diff --git a/js/otto_runner.go b/js/otto_runner.go index feb7fb2..bc3c126 100644 --- a/js/otto_runner.go +++ b/js/otto_runner.go @@ -118,7 +118,7 @@ func (r *OttoRunner) Run(args ...any) (result any, err error) { select { case <-timeoutChan: r.otto.Interrupt <- func() { - panic(context.DeadlineExceeded) + panic(context.DeadlineExceeded) // This will be caught in the defer block above } case <-runnerDoneChan: return diff --git a/js/otto_vm.go b/js/otto_vm.go index 201a1b4..00df829 100644 --- a/js/otto_vm.go +++ b/js/otto_vm.go @@ -69,7 +69,7 @@ func (vm *ottoVM) getRunner(service *Service) (Runner, error) { return nil, fmt.Errorf("the js.VM has been closed") } if vm.curRunner != nil { - panic("illegal access to ottoVM: already has a ottoRunner") + return nil, fmt.Errorf("illegal access to ottoVM: already has a ottoRunner") } if !vm.services.hasService(service) { return nil, fmt.Errorf("unknown js.Service instance passed to VM: %v", service) @@ -112,7 +112,8 @@ func (vm *ottoVM) returnRunner(r *OttoRunner) { if vm.curRunner == r { vm.curRunner = nil } else if r.vm != vm { - panic("OttoRunner returned to wrong v8VM!") + logError(vm.Context(), "OttoRunner returned to wrong v8VM!") + return } vm.runners[r.id] = r vm.release() diff --git a/js/v8_runner.go b/js/v8_runner.go index f4bd98f..4e42027 100644 --- a/js/v8_runner.go +++ b/js/v8_runner.go @@ -238,10 +238,10 @@ func (r *V8Runner) NewValue(val any) (v8Val *v8.Value, err error) { } // Creates a JavaScript number value. -func (r *V8Runner) NewInt(i int) *v8.Value { return mustSucceed(r.v8ctx.NewValue(i)) } +func (r *V8Runner) NewInt(i int) (*v8.Value, error) { return r.v8ctx.NewValue(i) } // Creates a JavaScript string value. -func (r *V8Runner) NewString(str string) *v8.Value { return newString(r.v8vm.iso, str) } +func (r *V8Runner) NewString(str string) (*v8.Value, error) { return newString(r.v8vm.iso, str) } // Marshals a Go value to JSON and returns it as a V8 string. func (r *V8Runner) NewJSONString(val any) (*v8.Value, error) { return newJSONString(r.v8ctx, val) } diff --git a/js/v8_template.go b/js/v8_template.go index 902b10a..e8ed068 100644 --- a/js/v8_template.go +++ b/js/v8_template.go @@ -135,7 +135,7 @@ func (t *V8BasicTemplate) NewValue(val any) (*v8.Value, error) { } // Creates a JS string value. -func (t *V8BasicTemplate) NewString(str string) *v8.Value { +func (t *V8BasicTemplate) NewString(str string) (*v8.Value, error) { return newString(t.vm.iso, str) } diff --git a/js/v8_utils.go b/js/v8_utils.go index 7d830aa..e30139a 100644 --- a/js/v8_utils.go +++ b/js/v8_utils.go @@ -13,6 +13,7 @@ licenses/APL2.txt. package js import ( + "context" "encoding/json" "fmt" "math" @@ -92,10 +93,14 @@ func StringArrayToGo(val *v8.Value) (result []string, err error) { //////// CONVERTING GO TO V8 VALUES: -// Converts a Go string into a JS string value. Assumes this cannot fail. +// Converts a Go string into a JS string value. // (AFAIK, v8.NewValue only fails if the input type is invalid, or V8 runs out of memory.) -func newString(i *v8.Isolate, str string) *v8.Value { - return mustSucceed(v8.NewValue(i, str)) +func newString(i *v8.Isolate, str string) (*v8.Value, error) { + value, err := v8.NewValue(i, str) + if err != nil { + logError(context.TODO(), "v8.NewValue(%q) failed: %v", str, err) + } + return value, err } // Marshals a Go value to JSON, and returns the string as a V8 Value. @@ -114,15 +119,9 @@ func newJSONString(v8ctx *v8.Context, val any) (*v8.Value, error) { // Returns an error back to a V8 caller. // Calls v8.Isolate.ThrowException, with the Go error's string as the message. func v8Throw(i *v8.Isolate, err error) *v8.Value { - return i.ThrowException(newString(i, err.Error())) -} - -// Simple utility to wrap a function that returns a value and an error; returns just the value, panicking if there was an error. -// This is kind of equivalent to those 3-prong to 2-prong electric plug adapters... -// Needless to say, it should only be used if you know the error cannot occur, or that if it occurs something is very, very wrong. -func mustSucceed[T any](result T, err error) T { - if err != nil { - panic(fmt.Sprintf(`ASSERTION FAILURE: expected a %T, got error "%v"`, result, err)) + message, _ := newString(i, err.Error()) + if message == nil { + message = v8.Null(i) } - return result + return i.ThrowException(message) } diff --git a/js/v8_vm.go b/js/v8_vm.go index 90ff74d..24c21d3 100644 --- a/js/v8_vm.go +++ b/js/v8_vm.go @@ -160,13 +160,13 @@ func (vm *v8VM) hasInitializedService(service *Service) bool { // Produces a v8Runner object that can run the given service. // Be sure to call Runner.Return when done. // Since v8VM is single-threaded, calling getRunner when a v8Runner already exists and hasn't been -// returned yet is assumed to be illegal concurrent access; it will trigger a panic. +// returned yet is assumed to be illegal concurrent access; it will return an error. func (vm *v8VM) getRunner(service *Service) (Runner, error) { if vm.iso == nil { return nil, fmt.Errorf("the js.VM has been closed") } if vm.curRunner != nil { - panic("illegal access to v8VM: already has a v8Runner") + return nil, fmt.Errorf("illegal access to v8VM: already has a v8Runner") } var runner *V8Runner index := int(service.id) @@ -213,7 +213,7 @@ func (vm *v8VM) returnRunner(r *V8Runner) { vm.iso.Unlock() vm.curRunner = nil } else if r.vm != vm { - panic("v8Runner returned to wrong v8VM!") + logError(vm.Context(), "v8Runner returned to wrong v8VM!") } if r.template.Reusable() { for int(r.id) >= len(vm.runners) { @@ -232,7 +232,7 @@ func (vm *v8VM) currentRunner(v8ctx *v8.Context) *V8Runner { // active v8Runner at a time. If it were to be multiple Runners, we'd need to maintain a map // from Contexts to Runners. if vm.curRunner == nil { - panic(fmt.Sprintf("Unknown v8.Context passed to v8VM.currentRunner: %v, expected none", v8ctx)) + logError(vm.Context(), "Unknown v8.Context passed to v8VM.currentRunner: %v, expected none", v8ctx) } if v8ctx != vm.curRunner.v8ctx { panic(fmt.Sprintf("Unknown v8.Context passed to v8VM.currentRunner: %v, expected %v", v8ctx, vm.curRunner.v8ctx)) diff --git a/js/vm.go b/js/vm.go index a5b61e8..56e5798 100644 --- a/js/vm.go +++ b/js/vm.go @@ -52,7 +52,7 @@ func (engine *Engine) newVM(ctx context.Context, services *servicesConfiguration // // **Not thread-safe!** A VM instance must be used only on one goroutine at a time. // A Service whose ServiceHost is a VM can only be used on a single goroutine; any concurrent -// use will trigger a panic in VM.getRunner. +// use will fail with an error. // The VMPool takes care of this, by vending VM instances that are known not to be in use. type VM interface { Engine() *Engine @@ -111,7 +111,8 @@ func (vm *baseVM) Context() context.Context { func (vm *baseVM) close() { if vm.returnToPool != nil { - panic("Don't Close a VM that belongs to a VMPool") + logError(vm.Context(), "Illegal attempt to close a %T that belongs to a VMPool", vm) + return } vm.services = nil vm.closed = true @@ -120,9 +121,9 @@ func (vm *baseVM) close() { func (vm *baseVM) registerService(service *Service) { if vm.services == nil { if vm.closed { - panic("Using an already-closed js.VM") + logError(vm.Context(), "Can't register a js.Service: the VM is already closed") } else { - panic("You forgot to initialize a js.VM") // Must call NewVM() + logError(vm.Context(), "Can't register a Service: the js.VM is uninitialized") // Must call NewVM() } } vm.services.addService(service) diff --git a/js/vmpool.go b/js/vmpool.go index cc6f01e..b5da16f 100644 --- a/js/vmpool.go +++ b/js/vmpool.go @@ -92,7 +92,8 @@ func (pool *VMPool) FindService(name string) *Service { func (pool *VMPool) registerService(service *Service) { if pool.services == nil { - panic("You forgot to initialize a VMPool") + logError(pool.Context(), "Can't register a Service: the js.VMPool is uninitialized") + return } pool.services.addService(service) } From 3779b871a912e20f470741cd5d5af41cfa4e6c0f Mon Sep 17 00:00:00 2001 From: Jens Alfke Date: Tue, 11 Jul 2023 14:18:57 -0700 Subject: [PATCH 6/7] v8go package moved from snej to couchbasedeps --- go.mod | 3 +-- go.sum | 6 ++---- js/js_test.go | 2 +- js/v8_runner.go | 2 +- js/v8_template.go | 2 +- js/v8_utils.go | 2 +- js/v8_vm.go | 4 ++-- 7 files changed, 9 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 303d756..b5a60a4 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/couchbase/sg-bucket go 1.19 require ( + github.com/couchbasedeps/v8go v1.7.5-0.20230711212500-516f3127f59a github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f - github.com/snej/v8go v1.7.3 github.com/stretchr/testify v1.7.1 golang.org/x/text v0.3.7 gopkg.in/couchbase/gocb.v1 v1.6.7 @@ -25,5 +25,4 @@ require ( gopkg.in/couchbaselabs/jsonx.v1 v1.0.1 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect - rogchap.com/v8go v0.8.0 // indirect ) diff --git a/go.sum b/go.sum index 89d6e35..3ec86b3 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/couchbasedeps/v8go v1.7.5-0.20230711212500-516f3127f59a h1:eAnRt+HIzIEN9wOjtIdVcWxKl87gWwRMqUErcmu92Eo= +github.com/couchbasedeps/v8go v1.7.5-0.20230711212500-516f3127f59a/go.mod h1:idYCHzuZs5u+Yv9mYwhOBheFP82+7arI2Y8xVeU2Ttk= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -14,8 +16,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f h1:a7clxaGmmqtdNTXyvrp/lVO/Gnkzlhc/+dLs5v965GM= github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f/go.mod h1:/mK7FZ3mFYEn9zvNPhpngTyatyehSwte5bJZ4ehL5Xw= -github.com/snej/v8go v1.7.3 h1:skliM+LRvRdLKwqJK4I+1BCQy4jTZs0u37R6b7aEUaU= -github.com/snej/v8go v1.7.3/go.mod h1:s7IVrqyNoVfwYhndECq3XJ+/y0uq/JUH0/ECsC3k/UQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -44,5 +44,3 @@ gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -rogchap.com/v8go v0.8.0 h1:/crDEiga68kOtbIqw3K9Rt9OztYz0LhAPHm2e3wK7Q4= -rogchap.com/v8go v0.8.0/go.mod h1:MxgP3pL2MW4dpme/72QRs8sgNMmM0pRc8DPhcuLWPAs= diff --git a/js/js_test.go b/js/js_test.go index b73da0f..d5d458d 100644 --- a/js/js_test.go +++ b/js/js_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/snej/v8go" + "github.com/couchbasedeps/v8go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/js/v8_runner.go b/js/v8_runner.go index 4e42027..33860a7 100644 --- a/js/v8_runner.go +++ b/js/v8_runner.go @@ -20,7 +20,7 @@ import ( "strings" "time" - v8 "github.com/snej/v8go" // Docs: https://pkg.go.dev/github.com/snej/v8go + v8 "github.com/couchbasedeps/v8go" // Docs: https://pkg.go.dev/github.com/couchbasedeps/v8go ) type V8Runner struct { diff --git a/js/v8_template.go b/js/v8_template.go index e8ed068..37069dc 100644 --- a/js/v8_template.go +++ b/js/v8_template.go @@ -15,7 +15,7 @@ package js import ( "fmt" - v8 "github.com/snej/v8go" + v8 "github.com/couchbasedeps/v8go" ) // A V8Template manages a Service's initial JS runtime environment -- its script code, main function diff --git a/js/v8_utils.go b/js/v8_utils.go index e30139a..eefe6ea 100644 --- a/js/v8_utils.go +++ b/js/v8_utils.go @@ -18,7 +18,7 @@ import ( "fmt" "math" - v8 "github.com/snej/v8go" + v8 "github.com/couchbasedeps/v8go" ) // CONVERTING V8 VALUES BACK TO GO: diff --git a/js/v8_vm.go b/js/v8_vm.go index 24c21d3..4306bf8 100644 --- a/js/v8_vm.go +++ b/js/v8_vm.go @@ -19,10 +19,10 @@ import ( "sync" "time" - v8 "github.com/snej/v8go" + v8 "github.com/couchbasedeps/v8go" ) -// v8go docs: https://pkg.go.dev/github.com/snej/v8go +// v8go docs: https://pkg.go.dev/github.com/couchbasedeps/v8go // General V8 API docs: https://v8.dev/docs/embed type v8VM struct { From 7e6d70bdf9bd8dc4cbdf4d4d774fc8a31ae5c3c9 Mon Sep 17 00:00:00 2001 From: Tor Colvin Date: Thu, 24 Aug 2023 17:56:27 -0400 Subject: [PATCH 7/7] Add v8 CI (#95) * add v8 tests * Ignore underscore files * use quotes --- .github/workflows/ci.yml | 45 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6cee98..6d739fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: go-version: 1.20.3 - run: go install github.com/google/addlicense@latest - uses: actions/checkout@v3 - - run: addlicense -check -f licenses/addlicense.tmpl . + - run: addlicense -check -ignore 'js/underscore*.js' -f licenses/addlicense.tmpl . test: runs-on: ${{ matrix.os }} @@ -57,6 +57,31 @@ jobs: uses: guyarb/golang-test-annotations@v0.6.0 with: test-results: test.json + test-v8: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-latest] + env: + GOPRIVATE: github.com/couchbaselabs + steps: + - name: Setup Go Faster + uses: WillAbides/setup-go-faster@v1.8.0 + with: + go-version: 1.20.3 + - uses: actions/checkout@v2 + - name: Build + run: go build -v -tags cb_sg_v8 "./..." + - name: Run Tests + run: go test -tags cb_sg_v8 -timeout=30m -count=1 -json -v "./..." | tee test.json | jq -s -jr 'sort_by(.Package,.Time) | .[].Output | select (. != null )' + shell: bash + - name: Annotate Failures + if: always() + uses: guyarb/golang-test-annotations@v0.6.0 + with: + test-results: test.json + golangci: name: lint @@ -88,3 +113,21 @@ jobs: uses: guyarb/golang-test-annotations@v0.6.0 with: test-results: test.json + + test-v8-race: + runs-on: ubuntu-latest + env: + GOPRIVATE: github.com/couchbaselabs + steps: + - uses: WillAbides/setup-go-faster@v1.8.0 + with: + go-version: 1.20.3 + - uses: actions/checkout@v3 + - name: Run Tests + run: go test -tags cb_sg_v8 -race -timeout=30m -count=1 -json -v "./..." | tee test.json | jq -s -jr 'sort_by(.Package,.Time) | .[].Output | select (. != null )' + shell: bash + - name: Annotate Failures + if: always() + uses: guyarb/golang-test-annotations@v0.6.0 + with: + test-results: test.json