From 9a2df76c65b37b4d0a3a8a51ef3795994e58cd1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Wed, 25 Jan 2023 13:15:43 +0100 Subject: [PATCH 01/45] Towards 1.13.1. --- .../org/scalajs/ir/ScalaJSVersions.scala | 2 +- project/BinaryIncompatibilities.scala | 49 ------------------- project/Build.scala | 2 +- 3 files changed, 2 insertions(+), 51 deletions(-) diff --git a/ir/shared/src/main/scala/org/scalajs/ir/ScalaJSVersions.scala b/ir/shared/src/main/scala/org/scalajs/ir/ScalaJSVersions.scala index 89ca31d304..e442a70641 100644 --- a/ir/shared/src/main/scala/org/scalajs/ir/ScalaJSVersions.scala +++ b/ir/shared/src/main/scala/org/scalajs/ir/ScalaJSVersions.scala @@ -17,7 +17,7 @@ import java.util.concurrent.ConcurrentHashMap import scala.util.matching.Regex object ScalaJSVersions extends VersionChecks( - current = "1.13.0", + current = "1.13.1-SNAPSHOT", binaryEmitted = "1.13" ) diff --git a/project/BinaryIncompatibilities.scala b/project/BinaryIncompatibilities.scala index 653d0c0200..4713fe6bf8 100644 --- a/project/BinaryIncompatibilities.scala +++ b/project/BinaryIncompatibilities.scala @@ -5,21 +5,12 @@ import com.typesafe.tools.mima.core.ProblemFilters._ object BinaryIncompatibilities { val IR = Seq( - // Breaking, but in minor verison, so OK. - exclude[Problem]("org.scalajs.ir.*"), ) val Linker = Seq( - // Breaking, but in minor version, so OK. - exclude[Problem]("org.scalajs.linker.standard.*"), ) val LinkerInterface = Seq( - // Breaking, but in minor version, so OK. - exclude[Problem]("org.scalajs.linker.interface.unstable.*"), - - // private, not an issue - ProblemFilters.exclude[DirectMissingMethodProblem]("org.scalajs.linker.interface.Semantics.this"), ) val SbtPlugin = Seq( @@ -28,47 +19,7 @@ object BinaryIncompatibilities { val TestAdapter = Seq( ) - private val JSTupleUnapplyExclusion: ProblemFilter = { - /* !!! Very delicate - * - * We changed the result type of `js.TupleN.unapply` from `Option` to - * `Some`, to make them irrefutable from Scala 3's point of view. This - * breaks binary compat, so we added a `protected` overload with the old - * binary signature. - * - * Unfortunately, those do not get a *static forwarder* in the class file, - * and hence MiMa still complains about them. Although the error message is - * clearly about "static method"s, the *filter* to apply is - * indistinguishable between the instance and static methods! - * - * Therefore, we implement here our own filter that only matches the - * *static* `unapply` method. - * - * Note that even though MiMa reports potential issues with static methods, - * these are ghost proplems. They do not exist in the .sjsir files to begin - * with, because the companion trait is a JS trait. We only generate static - * forwarders in Scala classes and traits. So filtering out the static - * method incompatibilities is legit. - */ - - val JSTupleUnapplyFullNameRegex = raw"""scala\.scalajs\.js\.Tuple\d+\.unapply""".r - - { (problem: Problem) => - val isStaticJSTupleUnapply = problem match { - case problem: IncompatibleResultTypeProblem => - problem.ref.isStatic && (problem.ref.fullName match { - case JSTupleUnapplyFullNameRegex() => true - case _ => false - }) - case _ => - false - } - !isStaticJSTupleUnapply // true to keep; false to filter out the problem - } - } - val Library = Seq( - JSTupleUnapplyExclusion, ) val TestInterface = Seq( diff --git a/project/Build.scala b/project/Build.scala index 44a901737f..28b6f4d382 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -254,7 +254,7 @@ object Build { val previousVersions = List("1.0.0", "1.0.1", "1.1.0", "1.1.1", "1.2.0", "1.3.0", "1.3.1", "1.4.0", "1.5.0", "1.5.1", "1.6.0", "1.7.0", "1.7.1", - "1.8.0", "1.9.0", "1.10.0", "1.10.1", "1.11.0", "1.12.0") + "1.8.0", "1.9.0", "1.10.0", "1.10.1", "1.11.0", "1.12.0", "1.13.0") val previousVersion = previousVersions.last val previousBinaryCrossVersion = CrossVersion.binaryWith("sjs1_", "") From 3cef9d095172b2b5b8189684991d55fa16878875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Fri, 27 Jan 2023 15:48:06 +0100 Subject: [PATCH 02/45] Fix #4801: Rebase the super JS type as seen from the this type in JS super call. When doing a super call to a method of a path-dependent JS super class, the `superClass.tpe_*` is only valid as seen from the super class' thisType. We need to rebase it with `asSeenFrom` to be in the context of the current class' thisType. --- .../scalajs/nscplugin/ExplicitLocalJS.scala | 36 +++++++++++++++++-- .../jsinterop/NestedJSClassTest.scala | 22 +++++++++--- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/compiler/src/main/scala/org/scalajs/nscplugin/ExplicitLocalJS.scala b/compiler/src/main/scala/org/scalajs/nscplugin/ExplicitLocalJS.scala index 194ec08d8a..42eff98571 100644 --- a/compiler/src/main/scala/org/scalajs/nscplugin/ExplicitLocalJS.scala +++ b/compiler/src/main/scala/org/scalajs/nscplugin/ExplicitLocalJS.scala @@ -317,7 +317,7 @@ abstract class ExplicitLocalJS[G <: Global with Singleton](val global: G) case Apply(fun @ Select(sup: Super, _), _) if !fun.symbol.isConstructor && isInnerOrLocalJSClass(sup.symbol.superClass) => - wrapWithContextualJSClassValue(sup.symbol.superClass.tpe_*) { + wrapWithContextualSuperJSClassValue(sup.symbol.superClass) { super.transform(tree) } @@ -325,7 +325,7 @@ abstract class ExplicitLocalJS[G <: Global with Singleton](val global: G) case Apply(TypeApply(fun @ Select(sup: Super, _), _), _) if !fun.symbol.isConstructor && isInnerOrLocalJSClass(sup.symbol.superClass) => - wrapWithContextualJSClassValue(sup.symbol.superClass.tpe_*) { + wrapWithContextualSuperJSClassValue(sup.symbol.superClass) { super.transform(tree) } @@ -394,6 +394,38 @@ abstract class ExplicitLocalJS[G <: Global with Singleton](val global: G) } } + /** Wraps with the contextual super JS class value for super calls. */ + private def wrapWithContextualSuperJSClassValue(superClass: Symbol)( + tree: Tree): Tree = { + /* #4801 We need to interpret the superClass type as seen from the + * current class' thisType. + * + * For example, in the test NestedJSClassTest.extendInnerJSClassInClass, + * the original `superClass.tpe_*` is + * + * OuterNativeClass_Issue4402.this.InnerClass + * + * because `InnerClass` is path-dependent. However, the path + * `OuterNativeClass.this` is only valid within `OuterNativeClass` + * itself. In the context of the current local class `Subclass`, this + * path must be replaced by the actual path `outer.`. This is precisely + * the role of `asSeenFrom`. We tell it to replace any `superClass.this` + * by `currentClass.this`, and it also transitively replaces paths for + * outer classes of `superClass`, matching them with the corresponding + * outer paths of `currentClass.thisType` if necessary. The result for + * that test case is + * + * outer.InnerClass + */ + val jsClassTypeInSuperClass = superClass.tpe_* + val jsClassTypeAsSeenFromThis = + jsClassTypeInSuperClass.asSeenFrom(currentClass.thisType, superClass) + + wrapWithContextualJSClassValue(jsClassTypeAsSeenFromThis) { + tree + } + } + private def wrapWithContextualJSClassValue(jsClassType: Type)( tree: Tree): Tree = { wrapWithContextualJSClassValue(genJSConstructorOf(tree, jsClassType)) { diff --git a/test-suite/js/src/test/scala/org/scalajs/testsuite/jsinterop/NestedJSClassTest.scala b/test-suite/js/src/test/scala/org/scalajs/testsuite/jsinterop/NestedJSClassTest.scala index 055d7bec6a..32034ec660 100644 --- a/test-suite/js/src/test/scala/org/scalajs/testsuite/jsinterop/NestedJSClassTest.scala +++ b/test-suite/js/src/test/scala/org/scalajs/testsuite/jsinterop/NestedJSClassTest.scala @@ -649,31 +649,39 @@ class NestedJSClassTest { } @Test - def extendInnerJSClassInClass_Issue4402(): Unit = { + def extendInnerJSClassInClass_Issue4402_Issue4801(): Unit = { val msg = "hello world" val outer = js.Dynamic.literal( InnerClass = js.constructorOf[DynamicInnerClass_Issue4402] ).asInstanceOf[OuterNativeClass_Issue4402] - class Subclass(arg: String) extends outer.InnerClass(arg) + class Subclass(arg: String) extends outer.InnerClass(arg) { + override def methodSuper_Issue4801(x: Int): String = + super.methodSuper_Issue4801(x) + " overridden" + } val obj = new Subclass(msg) assertEquals(msg, obj.message) + assertEquals(msg + "3 overridden", obj.methodSuper_Issue4801(3)) } @Test - def extendInnerJSClassInTrait_Issue4402(): Unit = { + def extendInnerJSClassInTrait_Issue4402_Issue4801(): Unit = { val msg = "hello world" val outer = js.Dynamic.literal( InnerClass = js.constructorOf[DynamicInnerClass_Issue4402] ).asInstanceOf[OuterNativeTrait_Issue4402] - class Subclass(arg: String) extends outer.InnerClass(arg) + class Subclass(arg: String) extends outer.InnerClass(arg) { + override def methodSuper_Issue4801(x: Int): String = + super.methodSuper_Issue4801(x) + " overridden" + } val obj = new Subclass(msg) assertEquals(msg, obj.message) + assertEquals(msg + "3 overridden", obj.methodSuper_Issue4801(3)) } } @@ -900,6 +908,8 @@ object NestedJSClassTest { class DynamicInnerClass_Issue4402(arg: String) extends js.Object { val message: String = arg + + def methodSuper_Issue4801(x: Int): String = arg + x } @js.native @@ -908,6 +918,8 @@ object NestedJSClassTest { @js.native class InnerClass(arg: String) extends js.Object { def message: String = js.native + + def methodSuper_Issue4801(x: Int): String = js.native } } @@ -916,6 +928,8 @@ object NestedJSClassTest { @js.native class InnerClass(arg: String) extends js.Object { def message: String = js.native + + def methodSuper_Issue4801(x: Int): String = js.native } } } From 055a18f58f0e38d8062d4dcdcb4f67d49971b24c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Feb 2023 17:00:06 +0000 Subject: [PATCH 03/45] Bump jszip from 3.7.0 to 3.8.0 Bumps [jszip](https://github.com/Stuk/jszip) from 3.7.0 to 3.8.0. - [Release notes](https://github.com/Stuk/jszip/releases) - [Changelog](https://github.com/Stuk/jszip/blob/main/CHANGES.md) - [Commits](https://github.com/Stuk/jszip/compare/v3.7.0...v3.8.0) --- updated-dependencies: - dependency-name: jszip dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0331fea1e7..3ea25312c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -354,7 +354,7 @@ "immediate": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", - "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", "dev": true }, "inherits": { @@ -378,7 +378,7 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "dev": true }, "isstream": { @@ -458,9 +458,9 @@ } }, "jszip": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.7.0.tgz", - "integrity": "sha512-Y2OlFIzrDOPWUnpU0LORIcDn2xN7rC9yKffFM/7pGhQuhO+SUhfm2trkJ/S5amjFvem0Y+1EALz/MEPkvHXVNw==", + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.8.0.tgz", + "integrity": "sha512-cnpQrXvFSLdsR9KR5/x7zdf6c3m8IhZfZzSblFEHSqBaVwD2nvJ4CuCKLyvKvwBgZm08CgfSoiTBQLm5WW9hGw==", "dev": true, "requires": { "lie": "~3.3.0", @@ -735,7 +735,7 @@ "set-immediate-shim": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", - "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=", + "integrity": "sha512-Li5AOqrZWCVA2n5kryzEmqai6bKSIvpz5oUJHPVj6+dsbD3X1ixtsY5tEnsaNpH3pFAHmG8eIHUrtEtohrg+UQ==", "dev": true }, "source-map": { @@ -862,7 +862,7 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "dev": true }, "uuid": { diff --git a/package.json b/package.json index 77e056fa88..3eb761adef 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "private": true, "devDependencies": { "source-map-support": "0.5.19", - "jszip": "3.7.0", + "jszip": "3.8.0", "jsdom": "16.5.0", "node-static": "0.7.11" } From 5a3b7cd1321c6c73352fdec990a4ac75a42750f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Fri, 17 Feb 2023 17:20:37 +0100 Subject: [PATCH 04/45] Reachability: Avoid dispatching the same method on the same interface. When we find a regular call to some (virtual) method `C.m`, we have to reach the implementations of `m` in all the instantiated subclasses of `C`. We do it immediately for the already known subclasses, and maintain a log for subclasses that will be found to be instantiated later. Previously, we did this dispatch for every call site of `C.m`. If M methods call `C.m` and `C` has N instantiated subclasses, this would result in M*N resolutions. This was not strictly unnecessary work, since the `from` argument is different in each case. However, this makes the analysis non-linear, which is bad. One might argue that M*N never gets arbitrarily big for any given `C.m`, so this is not *so* bad. But there is at least one case where M and N typically each grow linearly with the size of the codebase: the `apply` method of `FunctionN` traits. Therefore, this commit changes the approach to make it linear, without losing `from` information. We introduce one indirection in the from-chain: a `FromDispatch(C, m)` represents a dispatch from a virtual call of `C.m`. The first time we call `C.m`, we actually do the resolutions and use `FromDispatch(C, m)` as `from`. We also store the previous `from` in the `dispatchCalledFrom` of `C.m`. For subsequent calls, we enrich `dispatchCalledFrom` but we do not redo the resolutions. The indirection with `FromDispatch` allows to recover all the callers of an actual method, without requiring O(M*N) amount of information. The generated .js files are unchanged. --- On the test suite, this reduces the time spent in "Compute reachability" by approximately 17% (from around 5.2s to around 4.3s on my machine). --- For an unlinkable call to `Await.result`, a typical log output is now: Referring to non-existent class java.util.concurrent.locks.AbstractQueuedSynchronizer called from scala.concurrent.impl.Promise$CompletionLatch called from scala.concurrent.impl.Promise$DefaultPromise.tryAwait(scala.concurrent.duration.Duration)boolean called from scala.concurrent.impl.Promise$DefaultPromise.ready(scala.concurrent.duration.Duration,scala.concurrent.CanAwait)scala.concurrent.impl.Promise$DefaultPromise called from scala.concurrent.impl.Promise$DefaultPromise.result(scala.concurrent.duration.Duration,scala.concurrent.CanAwait)java.lang.Object dispatched from scala.concurrent.Awaitable.result(scala.concurrent.duration.Duration,scala.concurrent.CanAwait)java.lang.Object called from private scala.concurrent.Await$.$anonfun$result$1(scala.concurrent.Awaitable,scala.concurrent.duration.Duration)java.lang.Object called from scala.concurrent.Await$.result(scala.concurrent.Awaitable,scala.concurrent.duration.Duration)java.lang.Object called from helloworld.HelloWorld$.main([java.lang.String)void called from static helloworld.HelloWorld.main([java.lang.String)void called from core module module initializers --- .../scalajs/linker/analyzer/Analysis.scala | 22 ++++++- .../scalajs/linker/analyzer/Analyzer.scala | 60 +++++++++++++------ .../org/scalajs/linker/AnalyzerTest.scala | 8 ++- 3 files changed, 67 insertions(+), 23 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analysis.scala b/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analysis.scala index 82239836ef..116fdb62d3 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analysis.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analysis.scala @@ -78,6 +78,7 @@ object Analysis { def linkedFrom: scala.collection.Seq[From] def instantiatedFrom: scala.collection.Seq[From] + def dispatchCalledFrom: scala.collection.Map[MethodName, scala.collection.Seq[From]] def methodInfos( namespace: MemberNamespace): scala.collection.Map[MethodName, MethodInfo] @@ -208,6 +209,7 @@ object Analysis { sealed trait From final case class FromMethod(methodInfo: MethodInfo) extends From + final case class FromDispatch(classInfo: ClassInfo, methodName: MethodName) extends From final case class FromClass(classInfo: ClassInfo) extends From final case class FromCore(moduleName: String) extends From case object FromExports extends From @@ -303,6 +305,15 @@ object Analysis { @tailrec def loopTrace(optFrom: Option[From], verb: String = "called"): Unit = { + def sameMethod(methodInfo: MethodInfo, fromDispatch: FromDispatch): Boolean = { + methodInfo.owner == fromDispatch.classInfo && + methodInfo.namespace == MemberNamespace.Public && + methodInfo.methodName == fromDispatch.methodName + } + + def followDispatch(fromDispatch: FromDispatch): Option[From] = + fromDispatch.classInfo.dispatchCalledFrom.get(fromDispatch.methodName).flatMap(_.lastOption) + optFrom match { case None => log(level, s"$verb from ... er ... nowhere!? (this is a bug in dce)") @@ -312,8 +323,17 @@ object Analysis { log(level, s"$verb from ${methodInfo.fullDisplayName}") if (onlyOnce(level, methodInfo)) { involvedClasses ++= methodInfo.instantiatedSubclasses - loopTrace(methodInfo.calledFrom.lastOption) + methodInfo.calledFrom.lastOption match { + case Some(fromDispatch: FromDispatch) if sameMethod(methodInfo, fromDispatch) => + // avoid logging "dispatch from C.m" just after "called from C.m" + loopTrace(followDispatch(fromDispatch)) + case nextFrom => + loopTrace(nextFrom) + } } + case from @ FromDispatch(classInfo, methodName) => + log(level, s"dispatched from ${classInfo.displayName}.${methodName.displayName}") + loopTrace(followDispatch(from)) case FromClass(classInfo) => log(level, s"$verb from ${classInfo.displayName}") loopTrace(classInfo.linkedFrom.lastOption) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala b/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala index 0339d72ee9..4d0aec44d4 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala @@ -579,11 +579,13 @@ private final class Analyzer(config: CommonPhaseConfig, var instantiatedFrom: List[From] = Nil + val dispatchCalledFrom: mutable.Map[MethodName, List[From]] = mutable.Map.empty + /** List of all instantiated (Scala) subclasses of this Scala class/trait. * For JS types, this always remains empty. */ var instantiatedSubclasses: List[ClassInfo] = Nil - var methodsCalledLog: List[(MethodName, From)] = Nil + var methodsCalledLog: List[MethodName] = Nil private val nsMethodInfos = { val nsMethodInfos = Array.fill(MemberNamespace.Count) { @@ -947,18 +949,25 @@ private final class Analyzer(config: CommonPhaseConfig, if (isScalaClass) { accessData() + /* First mark the ancestors as subclassInstantiated() and fetch the + * methodsCalledLog, for all ancestors. Only then perform the + * resolved calls for all the logs. This order is important because, + * during the resolved calls, new methods could be called and added + * to the log; they will already see the new subclasses so we should + * *not* see them in the logs, lest we perform some work twice. + */ + val allMethodsCalledLogs = for (ancestor <- ancestors) yield { ancestor.subclassInstantiated() ancestor.instantiatedSubclasses ::= this - ancestor.methodsCalledLog + ancestor -> ancestor.methodsCalledLog } for { - log <- allMethodsCalledLogs - logEntry <- log + (ancestor, ancestorLog) <- allMethodsCalledLogs + methodName <- ancestorLog } { - val methodName = logEntry._1 - implicit val from = logEntry._2 + implicit val from = FromDispatch(ancestor, methodName) callMethodResolved(methodName) } } else { @@ -1029,19 +1038,32 @@ private final class Analyzer(config: CommonPhaseConfig, * detected, and those need to see the updated log, since the loop in * this method won't see them. */ - methodsCalledLog ::= ((methodName, from)) - val subclasses = instantiatedSubclasses - for (subclass <- subclasses) - subclass.callMethodResolved(methodName) - - if (checkAbstractReachability) { - /* Also lookup the method as abstract from this class, to make sure it - * is *declared* on this type. We do this after the concrete lookup to - * avoid work, since a concretely reachable method is already marked as - * abstractly reachable. - */ - if (!methodName.isReflectiveProxy) - lookupAbstractMethod(methodName).reachAbstract() + + dispatchCalledFrom.get(methodName) match { + case Some(froms) => + // Already called before; add the new from + dispatchCalledFrom.update(methodName, from :: froms) + + case None => + // New call + dispatchCalledFrom.update(methodName, from :: Nil) + + val fromDispatch = FromDispatch(this, methodName) + + methodsCalledLog ::= methodName + val subclasses = instantiatedSubclasses + for (subclass <- subclasses) + subclass.callMethodResolved(methodName)(fromDispatch) + + if (checkAbstractReachability) { + /* Also lookup the method as abstract from this class, to make sure it + * is *declared* on this type. We do this after the concrete lookup to + * avoid work, since a concretely reachable method is already marked as + * abstractly reachable. + */ + if (!methodName.isReflectiveProxy) + lookupAbstractMethod(methodName).reachAbstract()(fromDispatch) + } } } diff --git a/linker/shared/src/test/scala/org/scalajs/linker/AnalyzerTest.scala b/linker/shared/src/test/scala/org/scalajs/linker/AnalyzerTest.scala index 01b834bdde..5074537510 100644 --- a/linker/shared/src/test/scala/org/scalajs/linker/AnalyzerTest.scala +++ b/linker/shared/src/test/scala/org/scalajs/linker/AnalyzerTest.scala @@ -246,6 +246,8 @@ class AnalyzerTest { @Test def missingMethod(): AsyncResult = await { + val fooMethodName = m("foo", Nil, V) + val classDefs = Seq( classDef("A", superClass = Some(ObjectClass), methods = List(trivialCtor("A"))) @@ -253,10 +255,10 @@ class AnalyzerTest { val analysis = computeAnalysis(classDefs, reqsFactory.instantiateClass("A", NoArgConstructorName) ++ - reqsFactory.callMethod("A", m("foo", Nil, V))) + reqsFactory.callMethod("A", fooMethodName)) assertContainsError("MissingMethod(A.foo;V)", analysis) { - case MissingMethod(MethInfo("A", "foo;V"), `fromUnitTest`) => true + case MissingMethod(MethInfo("A", "foo;V"), FromDispatch(ClsInfo("A"), `fooMethodName`)) => true } } @@ -279,7 +281,7 @@ class AnalyzerTest { reqsFactory.callMethod("A", fooMethodName)) assertContainsError("MissingMethod(A.foo;I)", analysis) { - case MissingMethod(MethInfo("A", "foo;I"), `fromUnitTest`) => true + case MissingMethod(MethInfo("A", "foo;I"), FromDispatch(ClsInfo("A"), `fooMethodName`)) => true } } From 4f66611c1b33d6d214204fbb2793c8977fa6cde6 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 23 Feb 2023 12:27:25 -0800 Subject: [PATCH 05/45] Use `js.native` instead of `???` for default args --- library/src/main/scala/scala/scalajs/js/JSON.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/library/src/main/scala/scala/scalajs/js/JSON.scala b/library/src/main/scala/scala/scalajs/js/JSON.scala index 6a463c3a4e..b8d4bcf62f 100644 --- a/library/src/main/scala/scala/scalajs/js/JSON.scala +++ b/library/src/main/scala/scala/scalajs/js/JSON.scala @@ -39,7 +39,7 @@ object JSON extends js.Object { * MDN */ def parse(text: String, - reviver: js.Function2[js.Any, js.Any, js.Any] = ???): js.Dynamic = js.native + reviver: js.Function2[js.Any, js.Any, js.Any] = js.native): js.Dynamic = js.native // scalastyle:off line.size.limit /** @@ -85,8 +85,8 @@ object JSON extends js.Object { */ // scalastyle:on line.size.limit def stringify(value: js.Any, - replacer: js.Function2[String, js.Any, js.Any] = ???, - space: Int | String = ???): String = js.native + replacer: js.Function2[String, js.Any, js.Any] = js.native, + space: Int | String = js.native): String = js.native def stringify(value: js.Any, replacer: js.Array[Any]): String = js.native def stringify(value: js.Any, replacer: js.Array[Any], space: Int | String): String = js.native From 9b26af084b8d345e7b6bfad63944dcaf4ad8aa5e Mon Sep 17 00:00:00 2001 From: Tobias Schlatter Date: Sun, 19 Feb 2023 12:54:00 +0100 Subject: [PATCH 06/45] ClassDefChecker: Allow Null and Nothing as array receiver type Should have always been allowed, it just never surfaced. --- .../scalajs/linker/checker/ClassDefChecker.scala | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/checker/ClassDefChecker.scala b/linker/shared/src/main/scala/org/scalajs/linker/checker/ClassDefChecker.scala index fb5ebc1a61..f6b115af48 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/checker/ClassDefChecker.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/checker/ClassDefChecker.scala @@ -644,13 +644,11 @@ private final class ClassDefChecker(classDef: ClassDef, reporter: ErrorReporter) checkTrees(elems, env) case ArrayLength(array) => - if (!array.tpe.isInstanceOf[ArrayType]) - reportError(i"Array type expected but ${array.tpe} found") + checkArrayReceiverType(array.tpe) checkTree(array, env) case ArraySelect(array, index) => - if (!array.tpe.isInstanceOf[ArrayType]) - reportError(i"Array type expected but ${array.tpe} found") + checkArrayReceiverType(array.tpe) checkTree(array, env) checkTree(index, env) @@ -841,6 +839,13 @@ private final class ClassDefChecker(classDef: ClassDef, reporter: ErrorReporter) } } + private def checkArrayReceiverType(tpe: Type)( + implicit ctx: ErrorContext): Unit = tpe match { + case tpe: ArrayType => checkArrayType(tpe) + case NullType | NothingType => // ok + case _ => reportError(i"Array type expected but $tpe found") + } + private def checkArrayType(tpe: ArrayType)( implicit ctx: ErrorContext): Unit = { checkArrayTypeRef(tpe.arrayTypeRef) From 3f9e46ba2c5f7effea4c2a9e4ee3515f2578a054 Mon Sep 17 00:00:00 2001 From: Tobias Schlatter Date: Sun, 19 Feb 2023 13:18:21 +0100 Subject: [PATCH 07/45] Run ClassDefChecker after the Optimizer --- .../linker/checker/ClassDefChecker.scala | 36 ++++++++++++++----- .../scalajs/linker/frontend/IRLoader.scala | 3 +- .../linker/frontend/LinkerFrontendImpl.scala | 3 +- .../org/scalajs/linker/frontend/Refiner.scala | 35 ++++++++++++------ .../frontend/optimizer/OptimizerCore.scala | 25 +++++++------ .../linker/checker/ClassDefCheckerTest.scala | 21 +++++++++-- 6 files changed, 89 insertions(+), 34 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/checker/ClassDefChecker.scala b/linker/shared/src/main/scala/org/scalajs/linker/checker/ClassDefChecker.scala index f6b115af48..99c170db34 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/checker/ClassDefChecker.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/checker/ClassDefChecker.scala @@ -26,7 +26,8 @@ import org.scalajs.logging._ import org.scalajs.linker.checker.ErrorReporter._ /** Checker for the validity of the IR. */ -private final class ClassDefChecker(classDef: ClassDef, reporter: ErrorReporter) { +private final class ClassDefChecker(classDef: ClassDef, + allowReflectiveProxies: Boolean, allowTransients: Boolean, reporter: ErrorReporter) { import ClassDefChecker._ import reporter.reportError @@ -462,8 +463,12 @@ private final class ClassDefChecker(classDef: ClassDef, reporter: ErrorReporter) private def checkMethodNameNamespace(name: MethodName, namespace: MemberNamespace)( implicit ctx: ErrorContext): Unit = { if (name.isReflectiveProxy) { - // Only allowed after the analyzer. - reportError("illegal reflective proxy") + if (allowReflectiveProxies) { + if (namespace != MemberNamespace.Public) + reportError("reflective profixes are only allowed in the public namespace") + } else { + reportError("illegal reflective proxy") + } } if (name.isConstructor != (namespace == MemberNamespace.Constructor)) @@ -652,8 +657,13 @@ private final class ClassDefChecker(classDef: ClassDef, reporter: ErrorReporter) checkTree(array, env) checkTree(index, env) - case _:RecordSelect | _:RecordValue => - reportError("invalid tree") + case RecordSelect(record, _) => + checkAllowTransients() + checkTree(record, env) + + case RecordValue(_, elems) => + checkAllowTransients() + checkTrees(elems, env) case IsInstanceOf(expr, testType) => checkTree(expr, env) @@ -818,13 +828,21 @@ private final class ClassDefChecker(classDef: ClassDef, reporter: ErrorReporter) case CreateJSClass(className, captureValues) => checkTrees(captureValues, env) - case _:Transient => - reportError("invalid tree") + case Transient(transient) => + checkAllowTransients() + transient.traverse(new Traversers.Traverser { + override def traverse(tree: Tree): Unit = checkTree(tree, env) + }) } newEnv } + private def checkAllowTransients()(implicit ctx: ErrorContext): Unit = { + if (!allowTransients) + reportError("invalid transient tree") + } + private def checkIsAsInstanceTargetType(tpe: Type)( implicit ctx: ErrorContext): Unit = { tpe match { @@ -879,9 +897,9 @@ object ClassDefChecker { * * @return Count of IR checking errors (0 in case of success) */ - def check(classDef: ClassDef, logger: Logger): Int = { + def check(classDef: ClassDef, allowReflectiveProxies: Boolean, allowTransients: Boolean, logger: Logger): Int = { val reporter = new LoggerErrorReporter(logger) - new ClassDefChecker(classDef, reporter).checkClassDef() + new ClassDefChecker(classDef, allowReflectiveProxies, allowTransients, reporter).checkClassDef() reporter.errorCount } diff --git a/linker/shared/src/main/scala/org/scalajs/linker/frontend/IRLoader.scala b/linker/shared/src/main/scala/org/scalajs/linker/frontend/IRLoader.scala index 9643bec90a..536a71b6f5 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/frontend/IRLoader.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/frontend/IRLoader.scala @@ -125,7 +125,8 @@ private final class ClassDefAndInfoCache { version = newVersion cacheUpdate = irFile.tree.map { tree => if (checkIR) { - val errorCount = ClassDefChecker.check(tree, logger) + val errorCount = ClassDefChecker.check(tree, + allowReflectiveProxies = false, allowTransients = false, logger) if (errorCount != 0) { throw new LinkingException( s"There were $errorCount ClassDef checking errors.") diff --git a/linker/shared/src/main/scala/org/scalajs/linker/frontend/LinkerFrontendImpl.scala b/linker/shared/src/main/scala/org/scalajs/linker/frontend/LinkerFrontendImpl.scala index 9498836186..11d10064a9 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/frontend/LinkerFrontendImpl.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/frontend/LinkerFrontendImpl.scala @@ -43,7 +43,8 @@ final class LinkerFrontendImpl private (config: LinkerFrontendImpl.Config) private[this] val optOptimizer: Option[IncOptimizer] = LinkerFrontendImplPlatform.createOptimizer(config) - private[this] val refiner: Refiner = new Refiner(config.commonConfig) + private[this] val refiner: Refiner = + new Refiner(config.commonConfig, config.checkIR) private[this] val splitter: ModuleSplitter = config.moduleSplitStyle match { case ModuleSplitStyle.FewestModules => ModuleSplitter.fewestModules() diff --git a/linker/shared/src/main/scala/org/scalajs/linker/frontend/Refiner.scala b/linker/shared/src/main/scala/org/scalajs/linker/frontend/Refiner.scala index 06cfe85df4..729c329093 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/frontend/Refiner.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/frontend/Refiner.scala @@ -23,6 +23,7 @@ import org.scalajs.ir.Trees._ import org.scalajs.logging._ import org.scalajs.linker._ +import org.scalajs.linker.checker.ClassDefChecker import org.scalajs.linker.interface.ModuleInitializer import org.scalajs.linker.standard._ import org.scalajs.linker.standard.ModuleSet.ModuleID @@ -30,10 +31,10 @@ import org.scalajs.linker.analyzer._ import org.scalajs.linker.CollectionsCompat.MutableMapCompatOps /** Does a dead code elimination pass on a [[LinkingUnit]]. */ -final class Refiner(config: CommonPhaseConfig) { +final class Refiner(config: CommonPhaseConfig, checkIR: Boolean) { import Refiner._ - private val inputProvider = new InputProvider + private val inputProvider = new InputProvider(checkIR) def refine(classDefs: Seq[(ClassDef, Version)], moduleInitializers: List[ModuleInitializer], @@ -41,7 +42,7 @@ final class Refiner(config: CommonPhaseConfig) { implicit ec: ExecutionContext): Future[LinkingUnit] = { val linkedClassesByName = classDefs.map(c => c._1.className -> c._1).toMap - inputProvider.update(linkedClassesByName) + inputProvider.update(linkedClassesByName, logger) val analysis = logger.timeFuture("Refiner: Compute reachability") { analyze(moduleInitializers, symbolRequirements, logger) @@ -98,11 +99,13 @@ final class Refiner(config: CommonPhaseConfig) { } private object Refiner { - private class InputProvider extends Analyzer.InputProvider { + private class InputProvider(checkIR: Boolean) extends Analyzer.InputProvider { private var classesByName: Map[ClassName, ClassDef] = _ + private var logger: Logger = _ private val cache = mutable.Map.empty[ClassName, ClassInfoCache] - def update(classesByName: Map[ClassName, ClassDef]): Unit = { + def update(classesByName: Map[ClassName, ClassDef], logger: Logger): Unit = { + this.logger = logger this.classesByName = classesByName } @@ -113,12 +116,12 @@ private object Refiner { } def loadInfo(className: ClassName)(implicit ec: ExecutionContext): Option[Future[Infos.ClassInfo]] = - getCache(className).map(_.loadInfo(classesByName(className))) + getCache(className).map(_.loadInfo(classesByName(className), logger)) private def getCache(className: ClassName): Option[ClassInfoCache] = { cache.get(className).orElse { if (classesByName.contains(className)) { - val fileCache = new ClassInfoCache + val fileCache = new ClassInfoCache(checkIR) cache += className -> fileCache Some(fileCache) } else { @@ -133,22 +136,32 @@ private object Refiner { } } - private class ClassInfoCache { + private class ClassInfoCache(checkIR: Boolean) { private var cacheUsed: Boolean = false private val methodsInfoCaches = MethodDefsInfosCache() private val jsConstructorInfoCache = new JSConstructorDefInfoCache() private val exportedMembersInfoCaches = JSMethodPropDefsInfosCache() private var info: Infos.ClassInfo = _ - def loadInfo(classDef: ClassDef)(implicit ec: ExecutionContext): Future[Infos.ClassInfo] = Future { - update(classDef) + def loadInfo(classDef: ClassDef, logger: Logger)(implicit ec: ExecutionContext): Future[Infos.ClassInfo] = Future { + update(classDef, logger) info } - private def update(classDef: ClassDef): Unit = synchronized { + private def update(classDef: ClassDef, logger: Logger): Unit = synchronized { if (!cacheUsed) { cacheUsed = true + if (checkIR) { + val errorCount = ClassDefChecker.check(classDef, + allowReflectiveProxies = true, allowTransients = true, logger) + if (errorCount != 0) { + throw new AssertionError( + s"There were $errorCount ClassDef checking errors after optimizing. " + + "Please report this as a bug.") + } + } + val builder = new Infos.ClassInfoBuilder(classDef.className, classDef.kind, classDef.superClass.map(_.name), classDef.interfaces.map(_.name), classDef.jsNativeLoadSpec) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/frontend/optimizer/OptimizerCore.scala b/linker/shared/src/main/scala/org/scalajs/linker/frontend/optimizer/OptimizerCore.scala index 264ac04ee1..740c0d840f 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/frontend/optimizer/OptimizerCore.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/frontend/optimizer/OptimizerCore.scala @@ -731,12 +731,12 @@ private[optimizer] abstract class OptimizerCore( val replacement = ReplaceWithVarRef(newName, newSimpleState(Unused), None) val localDef = LocalDef(tcaptureValue.tpe, mutable, replacement) val localIdent = LocalIdent(newName)(ident.pos) - val newParamDef = ParamDef(localIdent, newOriginalName, ptpe, mutable)(paramDef.pos) + val newParamDef = ParamDef(localIdent, newOriginalName, tcaptureValue.tpe.base, mutable)(paramDef.pos) /* Note that the binding will never create a fresh name for a * ReplaceWithVarRef. So this will not put our name alignment at risk. */ - val valueBinding = Binding.temp(paramName, ptpe, mutable, tcaptureValue) + val valueBinding = Binding.temp(paramName, tcaptureValue) captureParamLocalDefs += paramName -> localDef newCaptureParamDefsAndRepls += newParamDef -> replacement @@ -5220,13 +5220,7 @@ private[optimizer] object OptimizerCore { } } - def newReplacement(implicit pos: Position): Tree = - newReplacementInternal(replacement) - - @tailrec - private def newReplacementInternal(replacement: LocalDefReplacement)( - implicit pos: Position): Tree = replacement match { - + def newReplacement(implicit pos: Position): Tree = this.replacement match { case ReplaceWithVarRef(name, used, _) => used.value = Used VarRef(LocalIdent(name))(tpe.base) @@ -5247,7 +5241,18 @@ private[optimizer] object OptimizerCore { This()(tpe.base) case ReplaceWithOtherLocalDef(localDef) => - newReplacementInternal(localDef.replacement) + /* A previous version would push down the `tpe` of this `LocalDef` to + * use for the replacement. While that creates trees with narrower types, + * it also creates inconsistent trees: + * - This() not typed as the enclosing class. + * - VarRef not typed as the corresponding VarDef / ParamDef. + * + * Type based optimizations happen (mainly) in the optimizer so + * consistent downstream types are more important than narrower types; + * notably because it allows us to run the ClassDefChecker after the + * optimizer. + */ + localDef.newReplacement case ReplaceWithConstant(value) => value diff --git a/linker/shared/src/test/scala/org/scalajs/linker/checker/ClassDefCheckerTest.scala b/linker/shared/src/test/scala/org/scalajs/linker/checker/ClassDefCheckerTest.scala index 7337cc82fe..6b5f82190d 100644 --- a/linker/shared/src/test/scala/org/scalajs/linker/checker/ClassDefCheckerTest.scala +++ b/linker/shared/src/test/scala/org/scalajs/linker/checker/ClassDefCheckerTest.scala @@ -158,6 +158,22 @@ class ClassDefCheckerTest { "Abstract methods may only be in the public namespace") } + @Test + def publicReflectiveProxy(): Unit = { + val babarMethodName = MethodName.reflectiveProxy("babar", Nil) + + assertError( + classDef("A", superClass = Some(ObjectClass), + methods = List( + MethodDef(EMF.withNamespace(MemberNamespace.PublicStatic), + babarMethodName, NON, Nil, AnyType, Some(int(1)))(EOH, UNV) + ) + ), + "reflective profixes are only allowed in the public namespace", + allowReflectiveProxies = true + ) + } + @Test def noDuplicateVarDef(): Unit = { val body = Block( @@ -257,7 +273,8 @@ class ClassDefCheckerTest { } private object ClassDefCheckerTest { - private def assertError(clazz: ClassDef, expectMsg: String) = { + private def assertError(clazz: ClassDef, expectMsg: String, + allowReflectiveProxies: Boolean = false, allowTransients: Boolean = false) = { var seen = false val reporter = new ErrorReporter { def reportError(msg: String)(implicit ctx: ErrorReporter.ErrorContext) = { @@ -267,7 +284,7 @@ private object ClassDefCheckerTest { } } - new ClassDefChecker(clazz, reporter).checkClassDef() + new ClassDefChecker(clazz, allowReflectiveProxies, allowTransients, reporter).checkClassDef() assertTrue("no errors reported", seen) } } From 7d944cd5b1fee879fe6947945b637fc834716e29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Tue, 21 Feb 2023 14:58:41 +0100 Subject: [PATCH 08/45] Optimized representation of Infos. Previously, the `ReachabilityInfo` of a method was a collection of fields of the form `Map[ClassName, List[*Name]]` and of the form `List[ClassName]`. When processing these, we could find the same `ClassName` several times, in several of the fields. Every time, we had to call `lookupClass`, which has map lookups and potential asynchronous boundaries. In this commit, we turn the data structure around so that we have, for each `ClassName`, one entry with fields that are `List[*Name]`s and (conceptually) `Boolean`s. This way, `followReachabilityInfo` calls `lookupClass` only once for any given `ClassName`. In addition, we pack all the boolean flags into a unique flag set. This allows to avoid individually checking them when they are all false, which is often the case. These changes bring about a 10-17% speedup for the initial reachability analysis of the test suite, and about a 25% speedup for the analysis during the Refiner. The speedup for the initial phase is computed based on the improved reflective proxy resolution of #4811. --- .../scalajs/linker/analyzer/Analyzer.scala | 203 +++++++-------- .../org/scalajs/linker/analyzer/Infos.scala | 245 +++++++++++------- 2 files changed, 248 insertions(+), 200 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala b/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala index 4d0aec44d4..308295b3de 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala @@ -35,7 +35,7 @@ import org.scalajs.linker.standard._ import org.scalajs.linker.standard.ModuleSet.ModuleID import Analysis._ -import Infos.{NamespacedMethodName, ReachabilityInfo} +import Infos.{NamespacedMethodName, ReachabilityInfo, ReachabilityInfoInClass} private final class Analyzer(config: CommonPhaseConfig, moduleInitializers: Seq[ModuleInitializer], @@ -1267,139 +1267,120 @@ private final class Analyzer(config: CommonPhaseConfig, staticDependencies += info.className } - for (moduleName <- data.accessedModules) { - lookupClass(moduleName) { module => - module.accessModule() - addInstanceDependency(module) - } - } + for (dataInClass <- data.byClass) { + lookupClass(dataInClass.className) { clazz => + val className = dataInClass.className - for (className <- data.instantiatedClasses) { - lookupClass(className) { clazz => - clazz.instantiated() - addInstanceDependency(clazz) - } - } + val flags = dataInClass.flags + if (flags != 0) { + if ((flags & ReachabilityInfoInClass.FlagModuleAccessed) != 0) { + clazz.accessModule() + addInstanceDependency(clazz) + } - for (className <- data.usedInstanceTests) { - staticDependencies += className - lookupClass(className)(_.useInstanceTests()) - } + if ((flags & ReachabilityInfoInClass.FlagInstantiated) != 0) { + clazz.instantiated() + addInstanceDependency(clazz) + } - for (className <- data.accessedClassData) { - staticDependencies += className - lookupClass(className)(_.accessData()) - } + if ((flags & ReachabilityInfoInClass.FlagInstanceTestsUsed) != 0) { + staticDependencies += className + clazz.useInstanceTests() + } - if (data.accessedClassClass) { - /* java.lang.Class is only ever instantiated in the CoreJSLib. - * Therefore, make java.lang.Object depend on it instead of the caller itself. - */ - objectClassInfo.staticDependencies += ClassClass - lookupClass(ClassClass) { clazz => - clazz.instantiated() - clazz.callMethodStatically(MemberNamespace.Constructor, ObjectArgConstructorName) - } - } + if ((flags & ReachabilityInfoInClass.FlagClassDataAccessed) != 0) { + staticDependencies += className + clazz.accessData() + } - for (className <- data.referencedClasses) { - /* No need to add to staticDependencies: The classes will not be - * referenced in the final JS code. - */ - lookupClass(className)(_ => ()) - } + if ((flags & ReachabilityInfoInClass.FlagStaticallyReferenced) != 0) { + staticDependencies += className + } + } - for (className <- data.staticallyReferencedClasses) { - staticDependencies += className - lookupClass(className)(_ => ()) - } + /* Since many of the lists below are likely to be empty, we always + * test `!list.isEmpty` before calling `foreach` or any other + * processing, avoiding closure allocations. + */ - /* `for` loops on maps are written with `while` loops to help the JIT - * compiler to inline and stack allocate tuples created by the iterators - */ + if (!dataInClass.fieldsRead.isEmpty) { + clazz.readFields(dataInClass.fieldsRead) + } - val fieldsReadIterator = data.fieldsRead.iterator - while (fieldsReadIterator.hasNext) { - val (className, fields) = fieldsReadIterator.next() - lookupClass(className)(_.readFields(fields)) - } + if (!dataInClass.fieldsWritten.isEmpty) { + clazz.writeFields(dataInClass.fieldsWritten) + } - val fieldsWrittenIterator = data.fieldsWritten.iterator - while (fieldsWrittenIterator.hasNext) { - val (className, fields) = fieldsWrittenIterator.next() - lookupClass(className)(_.writeFields(fields)) - } + if (!dataInClass.staticFieldsRead.isEmpty) { + staticDependencies += className + clazz.staticFieldsRead ++= dataInClass.staticFieldsRead + } - val staticFieldsReadIterator = data.staticFieldsRead.iterator - while (staticFieldsReadIterator.hasNext) { - val (className, fields) = staticFieldsReadIterator.next() - staticDependencies += className - lookupClass(className)(_.staticFieldsRead ++= fields) - } + if (!dataInClass.staticFieldsWritten.isEmpty) { + staticDependencies += className + clazz.staticFieldsWritten ++= dataInClass.staticFieldsWritten + } - val staticFieldsWrittenIterator = data.staticFieldsWritten.iterator - while (staticFieldsWrittenIterator.hasNext) { - val (className, fields) = staticFieldsWrittenIterator.next() - staticDependencies += className - lookupClass(className)(_.staticFieldsWritten ++= fields) - } + if (!dataInClass.methodsCalled.isEmpty) { + // Do not add to staticDependencies: We call these on the object. + for (methodName <- dataInClass.methodsCalled) + clazz.callMethod(methodName) + } - val methodsCalledIterator = data.methodsCalled.iterator - while (methodsCalledIterator.hasNext) { - val (className, methods) = methodsCalledIterator.next() - // Do not add to staticDependencies: We call these on the object. - lookupClass(className) { classInfo => - for (methodName <- methods) - classInfo.callMethod(methodName) - } - } + if (!dataInClass.methodsCalledStatically.isEmpty) { + staticDependencies += className + for (methodName <- dataInClass.methodsCalledStatically) + clazz.callMethodStatically(methodName) + } - val methodsCalledStaticallyIterator = data.methodsCalledStatically.iterator - while (methodsCalledStaticallyIterator.hasNext) { - val (className, methods) = methodsCalledStaticallyIterator.next() - staticDependencies += className - lookupClass(className) { classInfo => - for (methodName <- methods) - classInfo.callMethodStatically(methodName) + if (!dataInClass.methodsCalledDynamicImport.isEmpty) { + if (isNoModule) { + _errors += DynamicImportWithoutModuleSupport(from) + } else { + dynamicDependencies += className + // In terms of reachability, a dynamic import call is just a static call. + for (methodName <- dataInClass.methodsCalledDynamicImport) + clazz.callMethodStatically(methodName) + } + } + + if (!dataInClass.jsNativeMembersUsed.isEmpty) { + for (member <- dataInClass.jsNativeMembersUsed) + clazz.useJSNativeMember(member) + .foreach(addLoadSpec(externalDependencies, _)) + } } } - if (isNoModule) { - if (data.methodsCalledDynamicImport.nonEmpty) - _errors += DynamicImportWithoutModuleSupport(from) - } else { - val methodsCalledDynamicImportIterator = data.methodsCalledDynamicImport.iterator - while (methodsCalledDynamicImportIterator.hasNext) { - val (className, methods) = methodsCalledDynamicImportIterator.next() - dynamicDependencies += className - lookupClass(className) { classInfo => - // In terms of reachability, a dynamic import call is just a static call. - for (methodName <- methods) - classInfo.callMethodStatically(methodName) + val globalFlags = data.globalFlags + + if (globalFlags != 0) { + if ((globalFlags & ReachabilityInfo.FlagAccessedClassClass) != 0) { + /* java.lang.Class is only ever instantiated in the CoreJSLib. + * Therefore, make java.lang.Object depend on it instead of the caller itself. + */ + objectClassInfo.staticDependencies += ClassClass + lookupClass(ClassClass) { clazz => + clazz.instantiated() + clazz.callMethodStatically(MemberNamespace.Constructor, ObjectArgConstructorName) } } - } - val jsNativeMembersUsedIterator = data.jsNativeMembersUsed.iterator - while (jsNativeMembersUsedIterator.hasNext) { - val (className, members) = jsNativeMembersUsedIterator.next() - lookupClass(className) { classInfo => - for (member <- members) - classInfo.useJSNativeMember(member) - .foreach(addLoadSpec(externalDependencies, _)) + if ((globalFlags & ReachabilityInfo.FlagAccessedNewTarget) != 0 && + config.coreSpec.esFeatures.esVersion < ESVersion.ES2015) { + _errors += NewTargetWithoutES2015Support(from) } - } - if (data.accessedNewTarget && config.coreSpec.esFeatures.esVersion < ESVersion.ES2015) { - _errors += NewTargetWithoutES2015Support(from) - } + if ((globalFlags & ReachabilityInfo.FlagAccessedImportMeta) != 0 && + config.coreSpec.moduleKind != ModuleKind.ESModule) { + _errors += ImportMetaWithoutESModule(from) + } - if (data.accessedImportMeta && config.coreSpec.moduleKind != ModuleKind.ESModule) { - _errors += ImportMetaWithoutESModule(from) + if ((globalFlags & ReachabilityInfo.FlagUsedExponentOperator) != 0 && + config.coreSpec.esFeatures.esVersion < ESVersion.ES2016) { + _errors += ExponentOperatorWithoutES2016Support(from) + } } - - if (data.usedExponentOperator && config.coreSpec.esFeatures.esVersion < ESVersion.ES2016) - _errors += ExponentOperatorWithoutES2016Support(from) } @tailrec diff --git a/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Infos.scala b/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Infos.scala index 37367c4091..d89b4cb1bb 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Infos.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Infos.scala @@ -93,35 +93,45 @@ object Infos { ) final class ReachabilityInfo private[Infos] ( - val fieldsRead: Map[ClassName, List[FieldName]], - val fieldsWritten: Map[ClassName, List[FieldName]], - val staticFieldsRead: Map[ClassName, List[FieldName]], - val staticFieldsWritten: Map[ClassName, List[FieldName]], - val methodsCalled: Map[ClassName, List[MethodName]], - val methodsCalledStatically: Map[ClassName, List[NamespacedMethodName]], - val methodsCalledDynamicImport: Map[ClassName, List[NamespacedMethodName]], - val jsNativeMembersUsed: Map[ClassName, List[MethodName]], - /** For a Scala class, it is instantiated with a `New`; for a JS class, - * its constructor is accessed with a `JSLoadConstructor`. - */ - val instantiatedClasses: List[ClassName], - val accessedModules: List[ClassName], - val usedInstanceTests: List[ClassName], - val accessedClassData: List[ClassName], - val referencedClasses: List[ClassName], - val staticallyReferencedClasses: List[ClassName], - val accessedClassClass: Boolean, - val accessedNewTarget: Boolean, - val accessedImportMeta: Boolean, - val usedExponentOperator: Boolean + val byClass: List[ReachabilityInfoInClass], + val globalFlags: ReachabilityInfo.Flags ) object ReachabilityInfo { - val Empty: ReachabilityInfo = { - new ReachabilityInfo(Map.empty, Map.empty, Map.empty, Map.empty, - Map.empty, Map.empty, Map.empty, Map.empty, Nil, Nil, Nil, Nil, Nil, Nil, - false, false, false, false) - } + type Flags = Int + + final val FlagAccessedClassClass = 1 << 0 + final val FlagAccessedNewTarget = 1 << 1 + final val FlagAccessedImportMeta = 1 << 2 + final val FlagUsedExponentOperator = 1 << 3 + } + + /** Things from a given class that are reached by one method. */ + final class ReachabilityInfoInClass private[Infos] ( + val className: ClassName, + val fieldsRead: List[FieldName], + val fieldsWritten: List[FieldName], + val staticFieldsRead: List[FieldName], + val staticFieldsWritten: List[FieldName], + val methodsCalled: List[MethodName], + val methodsCalledStatically: List[NamespacedMethodName], + val methodsCalledDynamicImport: List[NamespacedMethodName], + val jsNativeMembersUsed: List[MethodName], + val flags: ReachabilityInfoInClass.Flags + ) + + object ReachabilityInfoInClass { + type Flags = Int + + /** For a Scala class, it is instantiated with a `New`; for a JS class, + * its constructor is accessed with a `JSLoadConstructor`. + */ + final val FlagInstantiated = 1 << 0 + + final val FlagModuleAccessed = 1 << 1 + final val FlagInstanceTestsUsed = 1 << 2 + final val FlagClassDataAccessed = 1 << 3 + final val FlagStaticallyReferenced = 1 << 4 } final class ClassInfoBuilder( @@ -178,42 +188,29 @@ object Infos { } final class ReachabilityInfoBuilder { - private val fieldsRead = mutable.Map.empty[ClassName, mutable.Set[FieldName]] - private val fieldsWritten = mutable.Map.empty[ClassName, mutable.Set[FieldName]] - private val staticFieldsRead = mutable.Map.empty[ClassName, mutable.Set[FieldName]] - private val staticFieldsWritten = mutable.Map.empty[ClassName, mutable.Set[FieldName]] - private val methodsCalled = mutable.Map.empty[ClassName, mutable.Set[MethodName]] - private val methodsCalledStatically = mutable.Map.empty[ClassName, mutable.Set[NamespacedMethodName]] - private val methodsCalledDynamicImport = mutable.Map.empty[ClassName, mutable.Set[NamespacedMethodName]] - private val jsNativeMembersUsed = mutable.Map.empty[ClassName, mutable.Set[MethodName]] - private val instantiatedClasses = mutable.Set.empty[ClassName] - private val accessedModules = mutable.Set.empty[ClassName] - private val usedInstanceTests = mutable.Set.empty[ClassName] - private val accessedClassData = mutable.Set.empty[ClassName] - private val referencedClasses = mutable.Set.empty[ClassName] - private val staticallyReferencedClasses = mutable.Set.empty[ClassName] - private var accessedClassClass = false - private var accessedNewTarget = false - private var accessedImportMeta = false - private var usedExponentOperator = false + private val byClass = mutable.Map.empty[ClassName, ReachabilityInfoInClassBuilder] + private var flags: ReachabilityInfo.Flags = 0 + + private def forClass(cls: ClassName): ReachabilityInfoInClassBuilder = + byClass.getOrElseUpdate(cls, new ReachabilityInfoInClassBuilder(cls)) def addFieldRead(cls: ClassName, field: FieldName): this.type = { - fieldsRead.getOrElseUpdate(cls, mutable.Set.empty) += field + forClass(cls).addFieldRead(field) this } def addFieldWritten(cls: ClassName, field: FieldName): this.type = { - fieldsWritten.getOrElseUpdate(cls, mutable.Set.empty) += field + forClass(cls).addFieldWritten(field) this } def addStaticFieldRead(cls: ClassName, field: FieldName): this.type = { - staticFieldsRead.getOrElseUpdate(cls, mutable.Set.empty) += field + forClass(cls).addStaticFieldRead(field) this } def addStaticFieldWritten(cls: ClassName, field: FieldName): this.type = { - staticFieldsWritten.getOrElseUpdate(cls, mutable.Set.empty) += field + forClass(cls).addStaticFieldWritten(field) this } @@ -254,39 +251,40 @@ object Infos { } def addMethodCalled(cls: ClassName, method: MethodName): this.type = { - methodsCalled.getOrElseUpdate(cls, mutable.Set.empty) += method + forClass(cls).addMethodCalled(method) this } def addMethodCalledStatically(cls: ClassName, method: NamespacedMethodName): this.type = { - methodsCalledStatically.getOrElseUpdate(cls, mutable.Set.empty) += method + forClass(cls).addMethodCalledStatically(method) this } def addMethodCalledDynamicImport(cls: ClassName, method: NamespacedMethodName): this.type = { - methodsCalledDynamicImport.getOrElseUpdate(cls, mutable.Set.empty) += method + forClass(cls).addMethodCalledDynamicImport(method) this } def addJSNativeMemberUsed(cls: ClassName, member: MethodName): this.type = { - jsNativeMembersUsed.getOrElseUpdate(cls, mutable.Set.empty) += member + forClass(cls).addJSNativeMemberUsed(member) this } def addInstantiatedClass(cls: ClassName): this.type = { - instantiatedClasses += cls + forClass(cls).setInstantiated() this } def addInstantiatedClass(cls: ClassName, ctor: MethodName): this.type = { - addInstantiatedClass(cls).addMethodCalledStatically(cls, + forClass(cls).setInstantiated().addMethodCalledStatically( NamespacedMethodName(MemberNamespace.Constructor, ctor)) + this } def addAccessedModule(cls: ClassName): this.type = { - accessedModules += cls + forClass(cls).setModuleAccessed() this } @@ -302,7 +300,7 @@ object Infos { } def addUsedInstanceTest(cls: ClassName): this.type = { - usedInstanceTests += cls + forClass(cls).setInstanceTestsUsed() this } @@ -318,7 +316,7 @@ object Infos { } def addAccessedClassData(cls: ClassName): this.type = { - accessedClassData += cls + forClass(cls).setClassDataAccessed() this } @@ -334,12 +332,16 @@ object Infos { } def addReferencedClass(cls: ClassName): this.type = { - referencedClasses += cls + /* We only need the class to appear in `byClass` so that the Analyzer + * knows to perform `lookupClass` for it. But then nothing further needs + * to happen. + */ + forClass(cls) this } def addStaticallyReferencedClass(cls: ClassName): this.type = { - staticallyReferencedClasses += cls + forClass(cls).setStaticallyReferenced() this } @@ -354,51 +356,116 @@ object Infos { this } - def addAccessedClassClass(): this.type = { - accessedClassClass = true + private def setFlag(flag: ReachabilityInfo.Flags): this.type = { + flags |= flag + this + } + + def addAccessedClassClass(): this.type = + setFlag(ReachabilityInfo.FlagAccessedClassClass) + + def addAccessNewTarget(): this.type = + setFlag(ReachabilityInfo.FlagAccessedNewTarget) + + def addAccessImportMeta(): this.type = + setFlag(ReachabilityInfo.FlagAccessedImportMeta) + + def addUsedExponentOperator(): this.type = + setFlag(ReachabilityInfo.FlagUsedExponentOperator) + + def result(): ReachabilityInfo = + new ReachabilityInfo(byClass.valuesIterator.map(_.result()).toList, flags) + } + + final class ReachabilityInfoInClassBuilder(val className: ClassName) { + private val fieldsRead = mutable.Set.empty[FieldName] + private val fieldsWritten = mutable.Set.empty[FieldName] + private val staticFieldsRead = mutable.Set.empty[FieldName] + private val staticFieldsWritten = mutable.Set.empty[FieldName] + private val methodsCalled = mutable.Set.empty[MethodName] + private val methodsCalledStatically = mutable.Set.empty[NamespacedMethodName] + private val methodsCalledDynamicImport = mutable.Set.empty[NamespacedMethodName] + private val jsNativeMembersUsed = mutable.Set.empty[MethodName] + private var flags: ReachabilityInfoInClass.Flags = 0 + + def addFieldRead(field: FieldName): this.type = { + fieldsRead += field + this + } + + def addFieldWritten(field: FieldName): this.type = { + fieldsWritten += field this } - def addAccessNewTarget(): this.type = { - accessedNewTarget = true + def addStaticFieldRead(field: FieldName): this.type = { + staticFieldsRead += field this } - def addAccessImportMeta(): this.type = { - accessedImportMeta = true + def addStaticFieldWritten(field: FieldName): this.type = { + staticFieldsWritten += field this } - def addUsedExponentOperator(): this.type = { - usedExponentOperator = true + def addMethodCalled(method: MethodName): this.type = { + methodsCalled += method this } - def result(): ReachabilityInfo = { - def toMapOfLists[A, B](m: mutable.Map[A, mutable.Set[B]]): Map[A, List[B]] = - m.map(kv => kv._1 -> kv._2.toList).toMap + def addMethodCalledStatically(method: NamespacedMethodName): this.type = { + methodsCalledStatically += method + this + } - new ReachabilityInfo( - fieldsRead = toMapOfLists(fieldsRead), - fieldsWritten = toMapOfLists(fieldsWritten), - staticFieldsRead = toMapOfLists(staticFieldsRead), - staticFieldsWritten = toMapOfLists(staticFieldsWritten), - methodsCalled = toMapOfLists(methodsCalled), - methodsCalledStatically = toMapOfLists(methodsCalledStatically), - methodsCalledDynamicImport = toMapOfLists(methodsCalledDynamicImport), - jsNativeMembersUsed = toMapOfLists(jsNativeMembersUsed), - instantiatedClasses = instantiatedClasses.toList, - accessedModules = accessedModules.toList, - usedInstanceTests = usedInstanceTests.toList, - accessedClassData = accessedClassData.toList, - referencedClasses = referencedClasses.toList, - staticallyReferencedClasses = staticallyReferencedClasses.toList, - accessedClassClass = accessedClassClass, - accessedNewTarget = accessedNewTarget, - accessedImportMeta = accessedImportMeta, - usedExponentOperator = usedExponentOperator + def addMethodCalledDynamicImport(method: NamespacedMethodName): this.type = { + methodsCalledDynamicImport += method + this + } + + def addJSNativeMemberUsed(member: MethodName): this.type = { + jsNativeMembersUsed += member + this + } + + private def setFlag(flag: ReachabilityInfoInClass.Flags): this.type = { + flags |= flag + this + } + + def setInstantiated(): this.type = + setFlag(ReachabilityInfoInClass.FlagInstantiated) + + def setModuleAccessed(): this.type = + setFlag(ReachabilityInfoInClass.FlagModuleAccessed) + + def setInstanceTestsUsed(): this.type = + setFlag(ReachabilityInfoInClass.FlagInstanceTestsUsed) + + def setClassDataAccessed(): this.type = + setFlag(ReachabilityInfoInClass.FlagClassDataAccessed) + + def setStaticallyReferenced(): this.type = + setFlag(ReachabilityInfoInClass.FlagStaticallyReferenced) + + def result(): ReachabilityInfoInClass = { + new ReachabilityInfoInClass( + className, + fieldsRead = toLikelyEmptyList(fieldsRead), + fieldsWritten = toLikelyEmptyList(fieldsWritten), + staticFieldsRead = toLikelyEmptyList(staticFieldsRead), + staticFieldsWritten = toLikelyEmptyList(staticFieldsWritten), + methodsCalled = toLikelyEmptyList(methodsCalled), + methodsCalledStatically = toLikelyEmptyList(methodsCalledStatically), + methodsCalledDynamicImport = toLikelyEmptyList(methodsCalledDynamicImport), + jsNativeMembersUsed = toLikelyEmptyList(jsNativeMembersUsed), + flags = flags ) } + + private def toLikelyEmptyList[A](set: mutable.Set[A]): List[A] = + if (set.isEmpty) Nil + else set.toList } /** Generates the [[ClassInfo]] of a From 572262c82e92c81f8786f4cb764d6cd50dd05bf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Thu, 23 Feb 2023 13:53:44 +0100 Subject: [PATCH 09/45] Give a meaningful toString() to `ir.Version`, for debugging purposes. --- ir/shared/src/main/scala/org/scalajs/ir/Version.scala | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ir/shared/src/main/scala/org/scalajs/ir/Version.scala b/ir/shared/src/main/scala/org/scalajs/ir/Version.scala index 6531b8c2a4..0ff29715a5 100644 --- a/ir/shared/src/main/scala/org/scalajs/ir/Version.scala +++ b/ir/shared/src/main/scala/org/scalajs/ir/Version.scala @@ -48,6 +48,17 @@ final class Version private (private val v: Array[Byte]) extends AnyVal { @inline private def isVersioned: Boolean = v != null + + // For debugging purposes + override def toString(): String = { + if (v == null) { + "Unversioned" + } else { + val typeByte = v(0) + val otherBytesStr = v.iterator.drop(1).map(b => "%02x".format(b & 0xff)).mkString + s"Version($typeByte, $otherBytesStr)" + } + } } object Version { From 68cac64342f4ba3bc405ef43986c45f8c3b2d5e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Sat, 25 Feb 2023 18:27:44 +0100 Subject: [PATCH 10/45] Call the testing hook `generatedJSAST` once for every `ClassDef`. Instead of once per compilation unit with the list of all generated `ClassDef`s. This requires a bit more work on the testing infrastructure side, but gives more freedom to `GenJSCode` to call it when and how it wants to. --- .../org/scalajs/nscplugin/GenJSCode.scala | 7 ++-- .../org/scalajs/nscplugin/ScalaJSPlugin.scala | 8 ++-- .../nscplugin/test/util/JSASTTest.scala | 38 ++++++++++++++----- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/compiler/src/main/scala/org/scalajs/nscplugin/GenJSCode.scala b/compiler/src/main/scala/org/scalajs/nscplugin/GenJSCode.scala index 019ab2cc24..3ea9be91ed 100644 --- a/compiler/src/main/scala/org/scalajs/nscplugin/GenJSCode.scala +++ b/compiler/src/main/scala/org/scalajs/nscplugin/GenJSCode.scala @@ -67,8 +67,8 @@ abstract class GenJSCode[G <: Global with Singleton](val global: G) val phaseName: String = "jscode" override val description: String = "generate JavaScript code from ASTs" - /** testing: this will be called when ASTs are generated */ - def generatedJSAST(clDefs: List[js.ClassDef]): Unit + /** testing: this will be called for each generated `ClassDef`. */ + def generatedJSAST(clDef: js.ClassDef): Unit /** Implicit conversion from nsc Position to ir.Position. */ implicit def pos2irPos(pos: Position): ir.Position = { @@ -423,9 +423,8 @@ abstract class GenJSCode[G <: Global with Singleton](val global: G) regularClasses ::: staticForwarderClasses } - generatedJSAST(clDefs) - for (tree <- clDefs) { + generatedJSAST(tree) genIRFile(cunit, tree) } } catch { diff --git a/compiler/src/main/scala/org/scalajs/nscplugin/ScalaJSPlugin.scala b/compiler/src/main/scala/org/scalajs/nscplugin/ScalaJSPlugin.scala index e7f68f6554..5273a84b4d 100644 --- a/compiler/src/main/scala/org/scalajs/nscplugin/ScalaJSPlugin.scala +++ b/compiler/src/main/scala/org/scalajs/nscplugin/ScalaJSPlugin.scala @@ -40,8 +40,8 @@ class ScalaJSPlugin(val global: Global) extends NscPlugin { } } - /** Called when the JS ASTs are generated. Override for testing */ - def generatedJSAST(clDefs: List[Trees.ClassDef]): Unit = {} + /** Called for each generated `ClassDef`. Override for testing. */ + def generatedJSAST(clDef: Trees.ClassDef): Unit = {} /** A trick to avoid early initializers while still enforcing that `global` * is initialized early. @@ -98,8 +98,8 @@ class ScalaJSPlugin(val global: Global) extends NscPlugin { override val runsAfter = List("mixin") override val runsBefore = List("delambdafy", "cleanup", "terminal") - def generatedJSAST(clDefs: List[Trees.ClassDef]): Unit = - ScalaJSPlugin.this.generatedJSAST(clDefs) + def generatedJSAST(clDef: Trees.ClassDef): Unit = + ScalaJSPlugin.this.generatedJSAST(clDef) } override def init(options: List[String], error: String => Unit): Boolean = { diff --git a/compiler/src/test/scala/org/scalajs/nscplugin/test/util/JSASTTest.scala b/compiler/src/test/scala/org/scalajs/nscplugin/test/util/JSASTTest.scala index 76b7be968a..d7f163f573 100644 --- a/compiler/src/test/scala/org/scalajs/nscplugin/test/util/JSASTTest.scala +++ b/compiler/src/test/scala/org/scalajs/nscplugin/test/util/JSASTTest.scala @@ -17,6 +17,7 @@ import language.implicitConversions import scala.tools.nsc._ import scala.reflect.internal.util.SourceFile +import scala.collection.mutable import scala.util.control.ControlThrowable import org.junit.Assert._ @@ -27,8 +28,6 @@ import ir.{Trees => js} abstract class JSASTTest extends DirectTest { - private var lastAST: JSAST = _ - class JSAST(val clDefs: List[js.ClassDef]) { type Pat = PartialFunction[js.IRNode, Unit] @@ -148,26 +147,45 @@ abstract class JSASTTest extends DirectTest { implicit def string2ast(str: String): JSAST = stringAST(str) + private var generatedClassDefs: Option[mutable.ListBuffer[js.ClassDef]] = None + + private def captureGeneratedClassDefs(body: => Unit): JSAST = { + if (generatedClassDefs.isDefined) + throw new IllegalStateException(s"Nested or concurrent calls to captureGeneratedClassDefs") + + val buffer = new mutable.ListBuffer[js.ClassDef] + generatedClassDefs = Some(buffer) + try { + body + new JSAST(buffer.toList) + } finally { + generatedClassDefs = None + } + } + override def newScalaJSPlugin(global: Global): ScalaJSPlugin = { new ScalaJSPlugin(global) { - override def generatedJSAST(cld: List[js.ClassDef]): Unit = { - lastAST = new JSAST(cld) + override def generatedJSAST(cld: js.ClassDef): Unit = { + for (buffer <- generatedClassDefs) + buffer += cld } } } def stringAST(code: String): JSAST = stringAST(defaultGlobal)(code) def stringAST(global: Global)(code: String): JSAST = { - if (!compileString(global)(code)) - throw new IllegalArgumentException("snippet did not compile") - lastAST + captureGeneratedClassDefs { + if (!compileString(global)(code)) + throw new IllegalArgumentException("snippet did not compile") + } } def sourceAST(source: SourceFile): JSAST = sourceAST(defaultGlobal)(source) def sourceAST(global: Global)(source: SourceFile): JSAST = { - if (!compileSources(global)(source)) - throw new IllegalArgumentException("snippet did not compile") - lastAST + captureGeneratedClassDefs { + if (!compileSources(global)(source)) + throw new IllegalArgumentException("snippet did not compile") + } } } From 87c7df068cd05ff9b977ca8eba0937347a940a22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Thu, 23 Feb 2023 13:52:05 +0100 Subject: [PATCH 11/45] Hash all js.ClassDefs generated by the compiler. Some `js.ClassDef`s did not go through `Hashers.hashClassDef`, notably LMF-generated classes and static forwarder classes. This caused their content to be reoptimized and re-emitted on every run. We now hash *all* class defs in one centralized way, so that this problem does not appear in the future anymore. --- .../org/scalajs/nscplugin/GenJSCode.scala | 93 +++++++++---------- 1 file changed, 44 insertions(+), 49 deletions(-) diff --git a/compiler/src/main/scala/org/scalajs/nscplugin/GenJSCode.scala b/compiler/src/main/scala/org/scalajs/nscplugin/GenJSCode.scala index 3ea9be91ed..903549ba91 100644 --- a/compiler/src/main/scala/org/scalajs/nscplugin/GenJSCode.scala +++ b/compiler/src/main/scala/org/scalajs/nscplugin/GenJSCode.scala @@ -236,7 +236,7 @@ abstract class GenJSCode[G <: Global with Singleton](val global: G) // Global class generation state ------------------------------------------- private val lazilyGeneratedAnonClasses = mutable.Map.empty[Symbol, ClassDef] - private val generatedClasses = ListBuffer.empty[js.ClassDef] + private val generatedClasses = ListBuffer.empty[(js.ClassDef, Position)] private val generatedStaticForwarderClasses = ListBuffer.empty[(Symbol, js.ClassDef)] private def consumeLazilyGeneratedAnonClass(sym: Symbol): ClassDef = { @@ -338,44 +338,25 @@ abstract class GenJSCode[G <: Global with Singleton](val global: G) fieldsMutatedInCurrentClass := mutable.Set.empty, generatedSAMWrapperCount := new VarBox(0) ) { - try { - val tree = if (isJSType(sym)) { - if (!sym.isTraitOrInterface && isNonNativeJSClass(sym) && - !isJSFunctionDef(sym)) { - genNonNativeJSClass(cd) - } else { - genJSClassData(cd) - } - } else if (sym.isTraitOrInterface) { - genInterface(cd) + val tree = if (isJSType(sym)) { + if (!sym.isTraitOrInterface && isNonNativeJSClass(sym) && + !isJSFunctionDef(sym)) { + genNonNativeJSClass(cd) } else { - genClass(cd) + genJSClassData(cd) } - - generatedClasses += tree - } catch { - case e: ir.InvalidIRException => - e.tree match { - case ir.Trees.Transient(UndefinedParam) => - reporter.error(sym.pos, - "Found a dangling UndefinedParam at " + - s"${e.tree.pos}. This is likely due to a bad " + - "interaction between a macro or a compiler plugin " + - "and the Scala.js compiler plugin. If you hit " + - "this, please let us know.") - - case _ => - reporter.error(sym.pos, - "The Scala.js compiler generated invalid IR for " + - "this class. Please report this as a bug. IR: " + - e.tree) - } + } else if (sym.isTraitOrInterface) { + genInterface(cd) + } else { + genClass(cd) } + + generatedClasses += tree -> sym.pos } } } - val clDefs = if (generatedStaticForwarderClasses.isEmpty) { + val clDefs: List[(js.ClassDef, Position)] = if (generatedStaticForwarderClasses.isEmpty) { /* Fast path, applicable under -Xno-forwarders, as well as when all * the `object`s of a compilation unit have a companion class. */ @@ -402,7 +383,7 @@ abstract class GenJSCode[G <: Global with Singleton](val global: G) classDef.name.name.nameString.toLowerCase(java.util.Locale.ENGLISH) val generatedCaseInsensitiveNames = - regularClasses.map(caseInsensitiveNameOf).toSet + regularClasses.map(pair => caseInsensitiveNameOf(pair._1)).toSet val staticForwarderClasses = generatedStaticForwarderClasses.toList .withFilter { case (site, classDef) => if (!generatedCaseInsensitiveNames.contains(caseInsensitiveNameOf(classDef))) { @@ -418,14 +399,34 @@ abstract class GenJSCode[G <: Global with Singleton](val global: G) false } } - .map(_._2) + .map(pair => (pair._2, pair._1.pos)) regularClasses ::: staticForwarderClasses } - for (tree <- clDefs) { - generatedJSAST(tree) - genIRFile(cunit, tree) + for ((classDef, pos) <- clDefs) { + try { + val hashedClassDef = Hashers.hashClassDef(classDef) + generatedJSAST(hashedClassDef) + genIRFile(cunit, hashedClassDef) + } catch { + case e: ir.InvalidIRException => + e.tree match { + case ir.Trees.Transient(UndefinedParam) => + reporter.error(pos, + "Found a dangling UndefinedParam at " + + s"${e.tree.pos}. This is likely due to a bad " + + "interaction between a macro or a compiler plugin " + + "and the Scala.js compiler plugin. If you hit " + + "this, please let us know.") + + case _ => + reporter.error(pos, + "The Scala.js compiler generated invalid IR for " + + "this class. Please report this as a bug. IR: " + + e.tree) + } + } } } catch { // Handle exceptions in exactly the same way as the JVM backend @@ -607,7 +608,7 @@ abstract class GenJSCode[G <: Global with Singleton](val global: G) else if (isHijacked) ClassKind.HijackedClass else ClassKind.Class - val classDefinition = js.ClassDef( + js.ClassDef( classIdent, originalName, kind, @@ -623,8 +624,6 @@ abstract class GenJSCode[G <: Global with Singleton](val global: G) jsNativeMembers, topLevelExportDefs)( optimizerHints) - - Hashers.hashClassDef(classDefinition) } /** Gen the IR ClassDef for a non-native JS class. */ @@ -755,7 +754,7 @@ abstract class GenJSCode[G <: Global with Singleton](val global: G) if (isStaticModule(sym)) ClassKind.JSModuleClass else ClassKind.JSClass - val classDefinition = js.ClassDef( + js.ClassDef( classIdent, originalNameOfClass(sym), kind, @@ -771,8 +770,6 @@ abstract class GenJSCode[G <: Global with Singleton](val global: G) jsNativeMembers = Nil, topLevelExports)( OptimizerHints.empty) - - Hashers.hashClassDef(classDefinition) } /** Generate an instance of an anonymous (non-lambda) JS class inline @@ -825,7 +822,7 @@ abstract class GenJSCode[G <: Global with Singleton](val global: G) origJsClass.optimizerHints) } - generatedClasses += newClassDef + generatedClasses += newClassDef -> pos // Construct inline class definition @@ -1035,12 +1032,10 @@ abstract class GenJSCode[G <: Global with Singleton](val global: G) if (!isCandidateForForwarders(sym)) generatedMethods else generatedMethods ::: genStaticForwardersForClassOrInterface(generatedMethods, sym) - val classDef = js.ClassDef(classIdent, originalNameOfClass(sym), ClassKind.Interface, + js.ClassDef(classIdent, originalNameOfClass(sym), ClassKind.Interface, None, None, interfaces, None, None, fields = Nil, methods = allMemberDefs, None, Nil, Nil, Nil)( OptimizerHints.empty) - - Hashers.hashClassDef(classDef) } private lazy val jsTypeInterfacesBlacklist: Set[Symbol] = @@ -3107,7 +3102,7 @@ abstract class GenJSCode[G <: Global with Singleton](val global: G) val classDef = consumeLazilyGeneratedAnonClass(clsSym) tryGenAnonFunctionClass(classDef, args.map(genExpr)).getOrElse { // Cannot optimize anonymous function class. Generate full class. - generatedClasses += nestedGenerateClass(clsSym)(genClass(classDef)) + generatedClasses += nestedGenerateClass(clsSym)(genClass(classDef)) -> clsSym.pos genNew(clsSym, ctor, genActualArgs(ctor, args)) } } else if (isJSType(clsSym)) { @@ -6294,7 +6289,7 @@ abstract class GenJSCode[G <: Global with Singleton](val global: G) Nil)( js.OptimizerHints.empty.withInline(true)) - generatedClasses += classDef + generatedClasses += classDef -> pos className } From f647921d5baaf1c5cfb1b5b43bfce37257cee790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Wed, 22 Feb 2023 11:28:19 +0100 Subject: [PATCH 12/45] Refactoring: Move the call to `createReflProxy` inside `findReflectiveTarget`. Along with the call to `workQueue.enqueue` that protects it. --- .../org/scalajs/linker/analyzer/Analyzer.scala | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala b/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala index 308295b3de..0aa792d9e7 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala @@ -743,18 +743,13 @@ private final class Analyzer(config: CommonPhaseConfig, tryLookupMethod(proxyName).foreach(onSuccess) } else { publicMethodInfos.get(proxyName).fold { - workQueue.enqueue(findReflectiveTarget(proxyName)) { maybeTarget => - maybeTarget.foreach { reflectiveTarget => - val proxy = createReflProxy(proxyName, reflectiveTarget.methodName) - onSuccess(proxy) - } - } + findReflectiveTarget(proxyName)(onSuccess) } (onSuccess) } } private def findReflectiveTarget(proxyName: MethodName)( - implicit from: From): Future[Option[MethodInfo]] = { + onSuccess: MethodInfo => Unit)(implicit from: From): Unit = { /* The lookup for a target method in this code implements the * algorithm defining `java.lang.Class.getMethod`. This mimics how * reflective calls are implemented on the JVM, at link time. @@ -776,10 +771,17 @@ private final class Analyzer(config: CommonPhaseConfig, val candidates = superClassesThenAncestors.map(_.findProxyMatch(proxyName)) - locally { + val targetFuture = locally { implicit val iec = ec Future.sequence(candidates).map(_.collectFirst { case Some(m) => m }) } + + workQueue.enqueue(targetFuture) { maybeTarget => + maybeTarget.foreach { reflectiveTarget => + val proxy = createReflProxy(proxyName, reflectiveTarget.methodName) + onSuccess(proxy) + } + } } private def findProxyMatch(proxyName: MethodName)( From bfd81ef7c10f2311f1a2c102b9eb0c5bdab60dbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Mon, 20 Feb 2023 14:27:35 +0100 Subject: [PATCH 13/45] Reachability analysis: optimize resolution of reflective proxies. There are several major changes, contributing together to a 20x speedup (!) of the reachability analysis time on our test suite. The speedup may not be representative of more standard codebases, which we expect not to have so many reflective calls in the first place. --- 1. Compute ancestors for reflective lookup once per class. We extract the computation of all the ancestors, in the correct lookup order, in a private `lazy val` of type `List[ClassInfo]`. --- 2. Compute all the proxy target candidates for a class at once. Iterating over all the public methods every time we need the candidates for a proxy is expensive. In the worst case, it can lead to a O(NxM) factor where N is the number of reflective calls, and M the number of methods in a class. We now build a map of all the possible candidates for a class the first time one proxy is requested. Building the map is in theory O(M), and then each lookup becomes O(1). This turns the overall worst-case complexity from O(NxM) to O(N+M). --- 3. Fast, synchronous path for 0 and 1 proxy candidates. When there is 0 or 1 proxy candidates, we can avoid all the complicated machinery to compute the most specific one. This is significant because that machinery involves Future computations. We even push the call to `workQueue.enqueue` in the case with more than one candidate. This avoids most uses of `workQueue.enqueue` for reflective proxy reasons. --- .../scalajs/linker/analyzer/Analyzer.scala | 85 ++++++++++++++----- 1 file changed, 66 insertions(+), 19 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala b/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala index 0aa792d9e7..ac64acd3e1 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/analyzer/Analyzer.scala @@ -765,32 +765,79 @@ private final class Analyzer(config: CommonPhaseConfig, * if the IR retained the information that a method is protected. */ - val superClasses = - Iterator.iterate(this)(_.superClass.orNull).takeWhile(_ ne null) - val superClassesThenAncestors = superClasses ++ ancestors.iterator + @tailrec + def findFirstNonEmptyCandidates(ancestors: List[ClassInfo]): List[MethodInfo] = { + ancestors match { + case ancestor :: nextAncestors => + val candidates = ancestor.findProxyCandidates(proxyName) + if (candidates.isEmpty) + findFirstNonEmptyCandidates(nextAncestors) + else + candidates + case Nil => + Nil + } + } - val candidates = superClassesThenAncestors.map(_.findProxyMatch(proxyName)) + val candidates = findFirstNonEmptyCandidates(ancestorsInReflectiveTargetOrder) - val targetFuture = locally { - implicit val iec = ec - Future.sequence(candidates).map(_.collectFirst { case Some(m) => m }) - } + candidates match { + case Nil => + () - workQueue.enqueue(targetFuture) { maybeTarget => - maybeTarget.foreach { reflectiveTarget => - val proxy = createReflProxy(proxyName, reflectiveTarget.methodName) + case onlyCandidate :: Nil => + // Fast path that does not require workQueue.enqueue + val proxy = createReflProxy(proxyName, onlyCandidate.methodName) onSuccess(proxy) + + case _ => + val targetFuture = computeMostSpecificProxyMatch(candidates) + workQueue.enqueue(targetFuture) { reflectiveTarget => + val proxy = createReflProxy(proxyName, reflectiveTarget.methodName) + onSuccess(proxy) + } + } + } + + private lazy val ancestorsInReflectiveTargetOrder: List[ClassInfo] = { + val b = new mutable.ListBuffer[ClassInfo] + + @tailrec + def addSuperClasses(superClass: ClassInfo): Unit = { + b += superClass + superClass.superClass match { + case Some(next) => addSuperClasses(next) + case None => () + } + } + addSuperClasses(this) + + b.prependToList(ancestors.filter(_.isInterface)) + } + + private def findProxyCandidates(proxyName: MethodName): List[MethodInfo] = + proxyCandidates.getOrElse(proxyName, Nil) + + private lazy val proxyCandidates = { + val result = mutable.Map.empty[MethodName, List[MethodInfo]] + val iter = publicMethodInfos.valuesIterator + while (iter.hasNext) { + val m = iter.next() + val include = { + // TODO In theory we should filter out protected methods + !m.isReflectiveProxy && !m.isDefaultBridge && !m.isAbstract + } + if (include) { + val proxyName = MethodName.reflectiveProxy(m.methodName.simpleName, m.methodName.paramTypeRefs) + val prev = result.getOrElse(proxyName, Nil) + result.update(proxyName, m :: prev) } } + result } - private def findProxyMatch(proxyName: MethodName)( - implicit from: From): Future[Option[MethodInfo]] = { - val candidates = publicMethodInfos.valuesIterator.filter { m => - // TODO In theory we should filter out protected methods - !m.isReflectiveProxy && !m.isDefaultBridge && !m.isAbstract && - reflProxyMatches(m.methodName, proxyName) - }.toSeq + private def computeMostSpecificProxyMatch(candidates: List[MethodInfo])( + implicit from: From): Future[MethodInfo] = { /* From the JavaDoc of java.lang.Class.getMethod: * @@ -823,7 +870,7 @@ private final class Analyzer(config: CommonPhaseConfig, * the implementation of reflective calls. This is bug-compatible with * Scala/JVM. */ - targets.headOption + targets.head } } } From 577dad060aa8ad4b922f1fc05b0fc84eea4d00cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Wed, 22 Feb 2023 18:29:21 +0100 Subject: [PATCH 14/45] Emitter: Add a cache for the full class. Not so much because it takes time to generate, but to preserve the `eq` of the generated trees, which will be important in the for the printed tree cache in `BasicLinkerBackend` in the upcoming commits. --- .../linker/backend/emitter/Emitter.scala | 88 +++++++++++++++++-- 1 file changed, 80 insertions(+), 8 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala index b535edd5de..a4ef4e0eff 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala @@ -503,14 +503,21 @@ final class Emitter(config: Emitter.Config) { classEmitter.genExportedMember(linkedClass, useESClass, member)(moduleContext, memberCache)) } - val fullClass = for { - ctor <- ctorWithGlobals - memberMethods <- WithGlobals.list(memberMethodsWithGlobals) - exportedMembers <- WithGlobals.list(exportedMembersWithGlobals) - clazz <- classEmitter.buildClass(linkedClass, useESClass, ctor, - memberMethods, exportedMembers)(moduleContext, classCache) - } yield { - clazz + val fullClass = { + val fullClassCache = classCache.getFullClassCache() + + fullClassCache.getOrElseUpdate(useESClass, ctorWithGlobals, + memberMethodsWithGlobals, exportedMembersWithGlobals, { + for { + ctor <- ctorWithGlobals + memberMethods <- WithGlobals.list(memberMethodsWithGlobals) + exportedMembers <- WithGlobals.list(exportedMembersWithGlobals) + clazz <- classEmitter.buildClass(linkedClass, useESClass, ctor, + memberMethods, exportedMembers)(moduleContext, fullClassCache) + } yield { + clazz + } + }) } addToMain(fullClass) @@ -593,6 +600,8 @@ final class Emitter(config: Emitter.Config) { private[this] val _exportedMembersCache = mutable.Map.empty[Int, MethodCache[js.Tree]] + private[this] var _fullClassCache: Option[FullClassCache] = None + override def invalidate(): Unit = { /* Do not invalidate contained methods, as they have their own * invalidation logic. @@ -607,6 +616,7 @@ final class Emitter(config: Emitter.Config) { _methodCaches.foreach(_.valuesIterator.foreach(_.startRun())) _memberMethodCache.valuesIterator.foreach(_.startRun()) _constructorCache.foreach(_.startRun()) + _fullClassCache.foreach(_.startRun()) } def getCache(version: Version): DesugaredClassCache = { @@ -644,6 +654,14 @@ final class Emitter(config: Emitter.Config) { def getExportedMemberCache(idx: Int): MethodCache[js.Tree] = _exportedMembersCache.getOrElseUpdate(idx, new MethodCache) + def getFullClassCache(): FullClassCache = { + _fullClassCache.getOrElse { + val cache = new FullClassCache + _fullClassCache = Some(cache) + cache + } + } + def cleanAfterRun(): Boolean = { _methodCaches.foreach(_.filterInPlace((_, c) => c.cleanAfterRun())) _memberMethodCache.filterInPlace((_, c) => c.cleanAfterRun()) @@ -653,6 +671,9 @@ final class Emitter(config: Emitter.Config) { _exportedMembersCache.filterInPlace((_, c) => c.cleanAfterRun()) + if (_fullClassCache.exists(!_.cleanAfterRun())) + _fullClassCache = None + if (!_cacheUsed) invalidate() @@ -695,6 +716,57 @@ final class Emitter(config: Emitter.Config) { } } + private class FullClassCache extends knowledgeGuardian.KnowledgeAccessor { + private[this] var _tree: WithGlobals[js.Tree] = null + private[this] var _lastUseESClass: Boolean = false + private[this] var _lastCtor: WithGlobals[js.Tree] = null + private[this] var _lastMemberMethods: List[WithGlobals[js.MethodDef]] = null + private[this] var _lastExportedMembers: List[WithGlobals[js.Tree]] = null + private[this] var _cacheUsed = false + + override def invalidate(): Unit = { + super.invalidate() + _tree = null + _lastCtor = null + _lastMemberMethods = null + _lastExportedMembers = null + } + + def startRun(): Unit = _cacheUsed = false + + def getOrElseUpdate(useESClass: Boolean, ctor: WithGlobals[js.Tree], + memberMethods: List[WithGlobals[js.MethodDef]], exportedMembers: List[WithGlobals[js.Tree]], + compute: => WithGlobals[js.Tree]): WithGlobals[js.Tree] = { + + @tailrec + def allSame[A <: AnyRef](xs: List[A], ys: List[A]): Boolean = { + xs.isEmpty == ys.isEmpty && { + xs.isEmpty || + ((xs.head eq ys.head) && allSame(xs.tail, ys.tail)) + } + } + + if (_tree == null || (_lastCtor ne ctor) || !allSame(_lastMemberMethods, memberMethods) || + !allSame(_lastExportedMembers, exportedMembers)) { + invalidate() + _tree = compute + _lastCtor = ctor + _lastMemberMethods = memberMethods + _lastExportedMembers = exportedMembers + } + + _cacheUsed = true + _tree + } + + def cleanAfterRun(): Boolean = { + if (!_cacheUsed) + invalidate() + + _cacheUsed + } + } + private class CoreJSLibCache extends knowledgeGuardian.KnowledgeAccessor { private[this] var _lastModuleContext: ModuleContext = _ private[this] var _lib: WithGlobals[CoreJSLib.Lib] = _ From 48e12fd568f7edc5b602c039491e971f43f8c209 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Wed, 22 Feb 2023 16:52:07 +0100 Subject: [PATCH 15/45] Introduce SourceMapWriter.FragmentBuilder. This class has the same interface as `SourceMapWriter` (they share a common parent `SourceMapWriter.Builder`) but instead of directly writing to a source map file, it builds a `Fragment`. A `Fragment` can later be inserted in another `SourceMapWriter.Builder`, as long as the latter is at the start of a line. It is independent of other content in the source map, so it can be reused several times. Fragments will be used to incrementally build source maps when only some trees change. --- .../linker/backend/javascript/Printers.scala | 2 +- .../backend/javascript/SourceMapWriter.scala | 225 ++++++++++++------ 2 files changed, 148 insertions(+), 79 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Printers.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Printers.scala index f5935f4f92..13f668ac18 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Printers.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Printers.scala @@ -726,7 +726,7 @@ object Printers { } class JSTreePrinterWithSourceMap(_out: Writer, - sourceMap: SourceMapWriter) extends JSTreePrinter(_out) { + sourceMap: SourceMapWriter.Builder) extends JSTreePrinter(_out) { private var column = 0 diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala index 98ba129d6c..aa5d8fa038 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala @@ -18,14 +18,14 @@ import java.nio.ByteBuffer import java.nio.charset.StandardCharsets import java.{util => ju} -import scala.collection.mutable.{ ListBuffer, HashMap, Stack, StringBuilder } +import scala.collection.mutable.{ ArrayBuffer, ListBuffer, HashMap, Stack, StringBuilder } import org.scalajs.ir import org.scalajs.ir.OriginalName import org.scalajs.ir.Position import org.scalajs.ir.Position._ -private object SourceMapWriter { +object SourceMapWriter { private val Base64Map = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + @@ -73,10 +73,143 @@ private object SourceMapWriter { nameStack = ju.Arrays.copyOf(nameStack, newSize) } } + + private sealed abstract class FragmentElement + + private object FragmentElement { + case object NewLine extends FragmentElement + + // name is nullable + final case class Segment(columnInGenerated: Int, pos: Position, name: String) + extends FragmentElement + } + + final class Fragment private[SourceMapWriter] ( + private[SourceMapWriter] val elements: Array[FragmentElement]) + + object Fragment { + val Empty: Fragment = new Fragment(new Array(0)) + } + + sealed abstract class Builder { + // Strings are nullable in this stack + private val nodePosStack = new SourceMapWriter.NodePosStack + nodePosStack.push(NoPosition, null) + + private var pendingColumnInGenerated: Int = -1 + private var pendingPos: Position = NoPosition + private var pendingIsIdent: Boolean = false + // pendingName string is nullable + private var pendingName: String = null + + final def nextLine(): Unit = { + writePendingSegment() + doWriteNewLine() + pendingColumnInGenerated = -1 + pendingPos = nodePosStack.topPos + pendingName = nodePosStack.topName + } + + final def startNode(column: Int, originalPos: Position): Unit = { + nodePosStack.push(originalPos, null) + startSegment(column, originalPos, isIdent = false, null) + } + + final def startIdentNode(column: Int, originalPos: Position, + optOriginalName: OriginalName): Unit = { + // TODO The then branch allocates a String; we should avoid that at some point + val originalName = + if (optOriginalName.isDefined) optOriginalName.get.toString() + else null + nodePosStack.push(originalPos, originalName) + startSegment(column, originalPos, isIdent = true, originalName) + } + + final def endNode(column: Int): Unit = { + nodePosStack.pop() + startSegment(column, nodePosStack.topPos, isIdent = false, + nodePosStack.topName) + } + + final def insertFragment(fragment: Fragment): Unit = { + require(pendingColumnInGenerated < 0, s"Cannot add fragment when in the middle of a line") + + val elements = fragment.elements + val len = elements.length + var i = 0 + while (i != len) { + elements(i) match { + case FragmentElement.Segment(columnInGenerated, pos, name) => + doWriteSegment(columnInGenerated, pos, name) + case FragmentElement.NewLine => + doWriteNewLine() + } + i += 1 + } + } + + final def complete(): Unit = { + writePendingSegment() + doComplete() + } + + private def startSegment(startColumn: Int, originalPos: Position, + isIdent: Boolean, originalName: String): Unit = { + // scalastyle:off return + + // There is no point in outputting a segment with the same information + if ((originalPos == pendingPos) && (isIdent == pendingIsIdent) && + (originalName == pendingName)) { + return + } + + // Write pending segment if it covers a non-empty range + if (startColumn != pendingColumnInGenerated) + writePendingSegment() + + // New pending + pendingColumnInGenerated = startColumn + pendingPos = originalPos + pendingIsIdent = isIdent + pendingName = originalName + + // scalastyle:on return + } + + private def writePendingSegment(): Unit = { + if (pendingColumnInGenerated >= 0) + doWriteSegment(pendingColumnInGenerated, pendingPos, pendingName) + } + + protected def doWriteNewLine(): Unit + + protected def doWriteSegment(columnInGenerated: Int, pos: Position, name: String): Unit + + protected def doComplete(): Unit + } + + final class FragmentBuilder extends Builder { + private val elements = new ArrayBuffer[FragmentElement] + + protected def doWriteNewLine(): Unit = + elements += FragmentElement.NewLine + + protected def doWriteSegment(columnInGenerated: Int, pos: Position, name: String): Unit = + elements += FragmentElement.Segment(columnInGenerated, pos, name) + + protected def doComplete(): Unit = { + if (elements.nonEmpty && elements.last != FragmentElement.NewLine) + throw new IllegalStateException("Trying to complete a fragment in the middle of a line") + } + + def result(): Fragment = + new Fragment(elements.toArray) + } } final class SourceMapWriter(out: Writer, jsFileName: String, - relativizeBaseURI: Option[URI]) { + relativizeBaseURI: Option[URI]) + extends SourceMapWriter.Builder { import SourceMapWriter._ @@ -86,10 +219,6 @@ final class SourceMapWriter(out: Writer, jsFileName: String, private val names = new ListBuffer[String] private val _nameToIndex = new HashMap[String, Int] - // Strings are nullable in this stack - private val nodePosStack = new SourceMapWriter.NodePosStack - nodePosStack.push(NoPosition, null) - private var lineCountInGenerated = 0 private var lastColumnInGenerated = 0 private var firstSegmentOfLine = true @@ -99,12 +228,6 @@ final class SourceMapWriter(out: Writer, jsFileName: String, private var lastColumn: Int = 0 private var lastNameIndex: Int = 0 - private var pendingColumnInGenerated: Int = -1 - private var pendingPos: Position = NoPosition - private var pendingIsIdent: Boolean = false - // pendingName string is nullable - private var pendingName: String = null - writeHeader() private def sourceToIndex(source: SourceFile): Int = { @@ -136,84 +259,32 @@ final class SourceMapWriter(out: Writer, jsFileName: String, out.write(",\n\"mappings\": \"") } - def nextLine(): Unit = { - writePendingSegment() + protected def doWriteNewLine(): Unit = { out.write(';') lineCountInGenerated += 1 lastColumnInGenerated = 0 firstSegmentOfLine = true - pendingColumnInGenerated = -1 - pendingPos = nodePosStack.topPos - pendingName = nodePosStack.topName - } - - def startNode(column: Int, originalPos: Position): Unit = { - nodePosStack.push(originalPos, null) - startSegment(column, originalPos, isIdent = false, null) - } - - def startIdentNode(column: Int, originalPos: Position, - optOriginalName: OriginalName): Unit = { - // TODO The then branch allocates a String; we should avoid that at some point - val originalName = - if (optOriginalName.isDefined) optOriginalName.get.toString() - else null - nodePosStack.push(originalPos, originalName) - startSegment(column, originalPos, isIdent = true, originalName) } - def endNode(column: Int): Unit = { - nodePosStack.pop() - startSegment(column, nodePosStack.topPos, isIdent = false, - nodePosStack.topName) - } - - private def startSegment(startColumn: Int, originalPos: Position, - isIdent: Boolean, originalName: String): Unit = { - // scalastyle:off return - - // There is no point in outputting a segment with the same information - if ((originalPos == pendingPos) && (isIdent == pendingIsIdent) && - (originalName == pendingName)) { - return - } - - // Write pending segment if it covers a non-empty range - if (startColumn != pendingColumnInGenerated) - writePendingSegment() - - // New pending - pendingColumnInGenerated = startColumn - pendingPos = originalPos - pendingIsIdent = isIdent - pendingName = originalName - - // scalastyle:on return - } - - private def writePendingSegment(): Unit = { + protected def doWriteSegment(columnInGenerated: Int, pos: Position, name: String): Unit = { // scalastyle:off return - if (pendingColumnInGenerated < 0) - return - // Segments of a line are separated by ',' if (firstSegmentOfLine) firstSegmentOfLine = false else out.write(',') // Generated column field - writeBase64VLQ(pendingColumnInGenerated-lastColumnInGenerated) - lastColumnInGenerated = pendingColumnInGenerated + writeBase64VLQ(columnInGenerated-lastColumnInGenerated) + lastColumnInGenerated = columnInGenerated // If the position is NoPosition, stop here - val pendingPos1 = pendingPos - if (pendingPos1.isEmpty) + if (pos.isEmpty) return // Extract relevant properties of pendingPos - val source = pendingPos1.source - val line = pendingPos1.line - val column = pendingPos1.column + val source = pos.source + val line = pos.line + val column = pos.column // Source index field if (source eq lastSource) { // highly likely @@ -234,8 +305,8 @@ final class SourceMapWriter(out: Writer, jsFileName: String, lastColumn = column // Name field - if (pendingName != null) { - val nameIndex = nameToIndex(pendingName) + if (name != null) { + val nameIndex = nameToIndex(name) writeBase64VLQ(nameIndex-lastNameIndex) lastNameIndex = nameIndex } @@ -243,9 +314,7 @@ final class SourceMapWriter(out: Writer, jsFileName: String, // scalastyle:on return } - def complete(): Unit = { - writePendingSegment() - + protected def doComplete(): Unit = { var restSources = sources.result() out.write("\",\n\"sources\": [") while (restSources.nonEmpty) { From 95948290f7e14e39730760024ced99ea5565f707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Sat, 25 Feb 2023 20:41:36 +0100 Subject: [PATCH 16/45] Make the list of top-level statements first-class in the back-end. Previously, they were bundled in a `js.Block`. This was already problematic in the GCC back-end, where they had to be taken apart. In the basic back-end, explicitly separating them will also allow to cache the output of individual top-level trees. --- .../closure/ClosureAstTransformer.scala | 25 +++---------- .../closure/ClosureLinkerBackend.scala | 4 +-- .../linker/backend/BasicLinkerBackend.scala | 8 +++-- .../linker/backend/emitter/Emitter.scala | 36 ++++++++++++------- 4 files changed, 35 insertions(+), 38 deletions(-) diff --git a/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureAstTransformer.scala b/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureAstTransformer.scala index 2fa5248ec4..fa759a4858 100644 --- a/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureAstTransformer.scala +++ b/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureAstTransformer.scala @@ -30,10 +30,10 @@ import java.lang.{Double => JDouble} import java.net.URI private[closure] object ClosureAstTransformer { - def transformScript(tree: Tree, featureSet: FeatureSet, + def transformScript(topLevelTrees: List[Tree], featureSet: FeatureSet, relativizeBaseURI: Option[URI]): Node = { val transformer = new ClosureAstTransformer(featureSet, relativizeBaseURI) - transformer.transformScript(tree) + transformer.transformScript(topLevelTrees) } } @@ -41,27 +41,10 @@ private class ClosureAstTransformer(featureSet: FeatureSet, relativizeBaseURI: Option[URI]) { private val dummySourceName = new java.net.URI("virtualfile:scala.js-ir") - def transformScript(tree: Tree): Node = { - /* Top-level `js.Block`s must be explicitly flattened here. - * Our `js.Block`s do not have the same semantics as GCC's `BLOCK`s: GCC's - * impose strict scoping for `let`s, `const`s and `class`es, while ours are - * only a means of putting together several statements in one `js.Tree` - * (in fact, they automatically flatten themselves out upon construction). - */ + def transformScript(topLevelTrees: List[Tree]): Node = { val script = setNodePosition(new Node(Token.SCRIPT), NoPosition) - - tree match { - case Block(stats) => - transformBlockStats(stats)(NoPosition).foreach(script.addChildToBack(_)) - - case Skip() => - - case tree => - script.addChildToBack(transformStat(tree)(NoPosition)) - } - + transformBlockStats(topLevelTrees)(NoPosition).foreach(script.addChildToBack(_)) script.putProp(Node.FEATURE_SET, featureSet) - script } diff --git a/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala b/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala index 276cec4f71..df931c2c45 100644 --- a/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala +++ b/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala @@ -127,8 +127,8 @@ final class ClosureLinkerBackend(config: LinkerBackendImpl.Config) } } - private def buildChunk(tree: js.Tree): JSChunk = { - val root = ClosureAstTransformer.transformScript(tree, + private def buildChunk(topLevelTrees: List[js.Tree]): JSChunk = { + val root = ClosureAstTransformer.transformScript(topLevelTrees, languageMode.toFeatureSet(), config.relativizeSourceMapBase) val chunk = new JSChunk("Scala.js") diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala index 1973120ce0..2a7223b9f5 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala @@ -63,7 +63,10 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config) val printer = new Printers.JSTreePrinter(jsFileWriter) jsFileWriter.write(emitterResult.header) jsFileWriter.write("'use strict';\n") - printer.printTopLevelTree(emitterResult.body(moduleID)) + + for (topLevelTree <- emitterResult.body(moduleID)) + printer.printTopLevelTree(topLevelTree) + jsFileWriter.write(emitterResult.footer) } @@ -84,7 +87,8 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config) jsFileWriter.write("'use strict';\n") smWriter.nextLine() - printer.printTopLevelTree(emitterResult.body(moduleID)) + for (topLevelTree <- emitterResult.body(moduleID)) + printer.printTopLevelTree(topLevelTree) jsFileWriter.write(emitterResult.footer) jsFileWriter.write("//# sourceMappingURL=" + sourceMapURI + "\n") diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala index a4ef4e0eff..31a059454c 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala @@ -106,7 +106,7 @@ final class Emitter(config: Emitter.Config) { } private def emitInternal(moduleSet: ModuleSet, - logger: Logger): WithGlobals[Map[ModuleID, js.Tree]] = { + logger: Logger): WithGlobals[Map[ModuleID, List[js.Tree]]] = { // Reset caching stats. statsClassesReused = 0 statsClassesInvalidated = 0 @@ -147,7 +147,7 @@ final class Emitter(config: Emitter.Config) { */ @tailrec private def emitAvoidGlobalClash(moduleSet: ModuleSet, - logger: Logger, secondAttempt: Boolean): WithGlobals[Map[ModuleID, js.Tree]] = { + logger: Logger, secondAttempt: Boolean): WithGlobals[Map[ModuleID, List[js.Tree]]] = { val result = emitOnce(moduleSet, logger) val mentionedDangerousGlobalRefs = @@ -172,7 +172,7 @@ final class Emitter(config: Emitter.Config) { } private def emitOnce(moduleSet: ModuleSet, - logger: Logger): WithGlobals[Map[ModuleID, js.Tree]] = { + logger: Logger): WithGlobals[Map[ModuleID, List[js.Tree]]] = { // Genreate classes first so we can measure time separately. val generatedClasses = logger.time("Emitter: Generate Classes") { moduleSet.modules.map { module => @@ -217,7 +217,7 @@ final class Emitter(config: Emitter.Config) { * requires consistency between the Analyzer and the Emitter. As such, * it is crucial that we verify it. */ - val defTrees = js.Block( + val defTreesIterator: Iterator[js.Tree] = ( /* The definitions of the CoreJSLib that come before the definition * of `j.l.Object`. They depend on nothing else. */ @@ -267,19 +267,29 @@ final class Emitter(config: Emitter.Config) { extractWithGlobals(classEmitter.genModuleInitializer(initializer)( moduleContext, uncachedKnowledge)) } - )(Position.NoPosition) + ) + + /* Flatten all the top-level js.Block's, because we temporarily use + * them to gather several top-level trees under a single `js.Tree`. + * TODO We should improve this in the future. + */ + val defTrees: List[js.Tree] = defTreesIterator.flatMap { + case js.Block(stats) => stats + case js.Skip() => Nil + case stat => stat :: Nil + }.toList - assert(!defTrees.isInstanceOf[js.Skip], { + assert(!defTrees.isEmpty, { val classNames = module.classDefs.map(_.fullName).mkString(", ") s"Module ${module.id} is empty. Classes in this module: $classNames" }) - val allTrees = js.Block( - /* Module imports, which depend on nothing. - * All classes potentially depend on them. - */ - extractWithGlobals(genModuleImports(module)) :+ defTrees - )(Position.NoPosition) + /* Module imports, which depend on nothing. + * All classes potentially depend on them. + */ + val moduleImports = extractWithGlobals(genModuleImports(module)) + + val allTrees = moduleImports ::: defTrees classIter.foreach { genClass => trackedGlobalRefs = unionPreserveEmpty(trackedGlobalRefs, genClass.trackedGlobalRefs) @@ -790,7 +800,7 @@ object Emitter { /** Result of an emitter run. */ final class Result private[Emitter]( val header: String, - val body: Map[ModuleID, js.Tree], + val body: Map[ModuleID, List[js.Tree]], val footer: String, val topLevelVarDecls: List[String], val globalRefs: Set[String] From 24541d69cb0bc73441cf8ffcd87862826c4100e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Wed, 22 Feb 2023 17:46:53 +0100 Subject: [PATCH 17/45] Keep a cache of the JS code and source map fragment of top-level trees. In `BasicLinkerBackend`, we now cache the JavaScript source code produced by each top-level tree, as well as the corresponding source map `Fragment`. Trees are identified with their reference identity. The caches are stored per-`ModuleID`, since * several modules can be written in parallel, and * the emitter throws caches away across `ModuleID`s anyway. This makes the tree-to-source step of our linker truly incremental. The source maps building is incremental up to a `Fragment`. We cannot avoid a full pass for actually emitting the content, because the `name` table and `source` table are inherently global and cannot be incrementalized. These changes bring about a 2x speedup to the "BasicBackend: Write result" step of the linker. This can be improved later by: * caching more trees (some "rare, small" trees such as top-level exports, static initialization calls and some module-related trees are not cached in the `Emitter`), and * caching the UTF-8-encoded bytes instead of the yet-to-be-encoded Strings. --- .../linker/backend/BasicLinkerBackend.scala | 153 +++++++++++++++++- .../linker/BasicLinkerBackendTest.scala | 85 ++++++++++ .../linker/testutils/CapturingLogger.scala | 13 +- 3 files changed, 241 insertions(+), 10 deletions(-) create mode 100644 linker/shared/src/test/scala/org/scalajs/linker/BasicLinkerBackendTest.scala diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala index 2a7223b9f5..52427d1a0d 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala @@ -24,7 +24,7 @@ import org.scalajs.linker.standard._ import org.scalajs.linker.standard.ModuleSet.ModuleID import org.scalajs.linker.backend.emitter.Emitter -import org.scalajs.linker.backend.javascript.{Printers, SourceMapWriter} +import org.scalajs.linker.backend.javascript.{Printers, SourceMapWriter, Trees => js} /** The basic backend for the Scala.js linker. * @@ -33,6 +33,8 @@ import org.scalajs.linker.backend.javascript.{Printers, SourceMapWriter} final class BasicLinkerBackend(config: LinkerBackendImpl.Config) extends LinkerBackendImpl(config) { + import BasicLinkerBackend._ + private[this] val emitter = { val emitterConfig = Emitter.Config(config.commonConfig.coreSpec) .withJSHeader(config.jsHeader) @@ -43,6 +45,8 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config) val symbolRequirements: SymbolRequirement = emitter.symbolRequirements + private val printedModuleSetCache = new PrintedModuleSetCache(config.sourceMap) + override def injectedIRFiles: Seq[IRFile] = emitter.injectedIRFiles /** Emit the given [[standard.ModuleSet ModuleSet]] to the target output. @@ -60,26 +64,29 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config) val writer = new OutputWriter(output, config) { protected def writeModule(moduleID: ModuleID, jsFileWriter: Writer): Unit = { - val printer = new Printers.JSTreePrinter(jsFileWriter) + val printedModuleCache = printedModuleSetCache.getModuleCache(moduleID) + jsFileWriter.write(emitterResult.header) jsFileWriter.write("'use strict';\n") - for (topLevelTree <- emitterResult.body(moduleID)) - printer.printTopLevelTree(topLevelTree) + for (topLevelTree <- emitterResult.body(moduleID)) { + val printedTree = printedModuleCache.getPrintedTree(topLevelTree) + jsFileWriter.write(printedTree.jsCode) + } jsFileWriter.write(emitterResult.footer) } protected def writeModule(moduleID: ModuleID, jsFileWriter: Writer, sourceMapWriter: Writer): Unit = { + val printedModuleCache = printedModuleSetCache.getModuleCache(moduleID) + val jsFileURI = OutputPatternsImpl.jsFileURI(config.outputPatterns, moduleID.id) val sourceMapURI = OutputPatternsImpl.sourceMapURI(config.outputPatterns, moduleID.id) val smWriter = new SourceMapWriter(sourceMapWriter, jsFileURI, config.relativizeSourceMapBase) - val printer = new Printers.JSTreePrinterWithSourceMap(jsFileWriter, smWriter) - jsFileWriter.write(emitterResult.header) for (_ <- 0 until emitterResult.header.count(_ == '\n')) smWriter.nextLine() @@ -87,8 +94,11 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config) jsFileWriter.write("'use strict';\n") smWriter.nextLine() - for (topLevelTree <- emitterResult.body(moduleID)) - printer.printTopLevelTree(topLevelTree) + for (topLevelTree <- emitterResult.body(moduleID)) { + val printedTree = printedModuleCache.getPrintedTree(topLevelTree) + jsFileWriter.write(printedTree.jsCode) + smWriter.insertFragment(printedTree.sourceMapFragment) + } jsFileWriter.write(emitterResult.footer) jsFileWriter.write("//# sourceMappingURL=" + sourceMapURI + "\n") @@ -97,8 +107,135 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config) } } + printedModuleSetCache.startRun() + logger.timeFuture("BasicBackend: Write result") { writer.write(moduleSet) + }.andThen { case _ => + printedModuleSetCache.cleanAfterRun() + printedModuleSetCache.logStats(logger) + } + } +} + +private object BasicLinkerBackend { + private final class PrintedModuleSetCache(withSourceMaps: Boolean) { + private val modules = new java.util.concurrent.ConcurrentHashMap[ModuleID, PrintedModuleCache] + + private var totalTopLevelTrees = 0 + private var recomputedTopLevelTrees = 0 + + def startRun(): Unit = { + totalTopLevelTrees = 0 + recomputedTopLevelTrees = 0 + } + + def getModuleCache(moduleID: ModuleID): PrintedModuleCache = { + val result = modules.computeIfAbsent(moduleID, { _ => + if (withSourceMaps) new PrintedModuleCacheWithSourceMaps + else new PrintedModuleCache + }) + + result.startRun() + result + } + + def cleanAfterRun(): Unit = { + val iter = modules.entrySet().iterator() + while (iter.hasNext()) { + val moduleCache = iter.next().getValue() + if (moduleCache.cleanAfterRun()) { + totalTopLevelTrees += moduleCache.getTotalTopLevelTrees + recomputedTopLevelTrees += moduleCache.getRecomputedTopLevelTrees + } else { + iter.remove() + } + } + } + + def logStats(logger: Logger): Unit = { + /* This message is extracted in BasicLinkerBackendTest to assert that we + * do not invalidate anything in a no-op second run. + */ + logger.debug( + s"BasicBackend: total top-level trees: $totalTopLevelTrees; re-computed: $recomputedTopLevelTrees") + } + } + + /* TODO Instead of caching the String, we should cache the already + * UTF-8-encoded Byte array. + */ + private final class PrintedTree(val jsCode: String, val sourceMapFragment: SourceMapWriter.Fragment) { + var cachedUsed: Boolean = false + } + + private sealed class PrintedModuleCache { + private var cacheUsed = false + private val cache = new java.util.IdentityHashMap[js.Tree, PrintedTree] + + private var totalTopLevelTrees = 0 + private var recomputedTopLevelTrees = 0 + + def startRun(): Unit = { + cacheUsed = true + totalTopLevelTrees = 0 + recomputedTopLevelTrees = 0 + } + + def getPrintedTree(tree: js.Tree): PrintedTree = { + totalTopLevelTrees += 1 + + val result = cache.computeIfAbsent(tree, { (tree: js.Tree) => + recomputedTopLevelTrees += 1 + computePrintedTree(tree) + }) + + result.cachedUsed = true + result + } + + protected def computePrintedTree(tree: js.Tree): PrintedTree = { + val jsCodeWriter = new java.io.StringWriter() + val printer = new Printers.JSTreePrinter(jsCodeWriter) + + printer.printTopLevelTree(tree) + + new PrintedTree(jsCodeWriter.toString(), SourceMapWriter.Fragment.Empty) + } + + def cleanAfterRun(): Boolean = { + if (cacheUsed) { + cacheUsed = false + + val iter = cache.entrySet().iterator() + while (iter.hasNext()) { + val printedTree = iter.next().getValue() + if (printedTree.cachedUsed) + printedTree.cachedUsed = false + else + iter.remove() + } + + true + } else { + false + } + } + + def getTotalTopLevelTrees: Int = totalTopLevelTrees + def getRecomputedTopLevelTrees: Int = recomputedTopLevelTrees + } + + private final class PrintedModuleCacheWithSourceMaps extends PrintedModuleCache { + override protected def computePrintedTree(tree: js.Tree): PrintedTree = { + val jsCodeWriter = new java.io.StringWriter() + val smFragmentBuilder = new SourceMapWriter.FragmentBuilder() + val printer = new Printers.JSTreePrinterWithSourceMap(jsCodeWriter, smFragmentBuilder) + + printer.printTopLevelTree(tree) + smFragmentBuilder.complete() + + new PrintedTree(jsCodeWriter.toString(), smFragmentBuilder.result()) } } } diff --git a/linker/shared/src/test/scala/org/scalajs/linker/BasicLinkerBackendTest.scala b/linker/shared/src/test/scala/org/scalajs/linker/BasicLinkerBackendTest.scala new file mode 100644 index 0000000000..ef3c552141 --- /dev/null +++ b/linker/shared/src/test/scala/org/scalajs/linker/BasicLinkerBackendTest.scala @@ -0,0 +1,85 @@ +/* + * Scala.js (https://www.scala-js.org/) + * + * Copyright EPFL. + * + * Licensed under Apache License 2.0 + * (https://www.apache.org/licenses/LICENSE-2.0). + * + * See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + */ + +package org.scalajs.linker + +import java.nio.charset.StandardCharsets + +import scala.concurrent.{ExecutionContext, Future} + +import org.junit.Test +import org.junit.Assert._ + +import org.scalajs.ir.Trees._ + +import org.scalajs.junit.async._ + +import org.scalajs.linker.interface._ +import org.scalajs.linker.standard._ +import org.scalajs.linker.testutils._ +import org.scalajs.linker.testutils.TestIRBuilder._ + +import org.scalajs.logging._ + +class BasicLinkerBackendTest { + import scala.concurrent.ExecutionContext.Implicits.global + + private val BackendInvalidatedTopLevelTreesStatsMessage = + raw"""BasicBackend: total top-level trees: (\d+); re-computed: (\d+)""".r + + /** Makes sure that linking a "substantial" program (using `println`) twice + * does not invalidate any top-level tree in the second run. + */ + @Test + def linkNoSecondAttemptInEmitter(): AsyncResult = await { + val classDefs = List( + mainTestClassDef(systemOutPrintln(str("Hello world!"))) + ) + + val logger1 = new CapturingLogger + val logger2 = new CapturingLogger + + val config = StandardConfig().withCheckIR(true) + val linker = StandardImpl.linker(config) + val classDefsFiles = classDefs.map(MemClassDefIRFile(_)) + + val initializers = MainTestModuleInitializers + val outputDir = MemOutputDirectory() + + for { + javalib <- TestIRRepo.javalib + allIRFiles = javalib ++ classDefsFiles + _ <- linker.link(allIRFiles, initializers, outputDir, logger1) + _ <- linker.link(allIRFiles, initializers, outputDir, logger2) + } yield { + val lines1 = logger1.allLogLines + val Seq(total1, recomputed1) = + lines1.assertContainsMatch(BackendInvalidatedTopLevelTreesStatsMessage).map(_.toInt) + + val lines2 = logger2.allLogLines + val Seq(total2, recomputed2) = + lines2.assertContainsMatch(BackendInvalidatedTopLevelTreesStatsMessage).map(_.toInt) + + // At the time of writing this test, total1 reports 382 trees + assertTrue( + s"Not enough total top-level trees (got $total1); extraction must have gone wrong", + total1 > 300) + + assertEquals("First run must invalidate every top-level tree", total1, recomputed1) + assertEquals("Second run must have the same total as first run", total1, total2) + + assertEquals( + "Second run must not invalidate any top-level tree beside the module initializer", + 1, recomputed2) + } + } +} diff --git a/linker/shared/src/test/scala/org/scalajs/linker/testutils/CapturingLogger.scala b/linker/shared/src/test/scala/org/scalajs/linker/testutils/CapturingLogger.scala index 37fe8c1338..d8a8744022 100644 --- a/linker/shared/src/test/scala/org/scalajs/linker/testutils/CapturingLogger.scala +++ b/linker/shared/src/test/scala/org/scalajs/linker/testutils/CapturingLogger.scala @@ -13,6 +13,7 @@ package org.scalajs.linker.testutils import scala.collection.mutable +import scala.util.matching.Regex import org.scalajs.logging._ @@ -21,7 +22,7 @@ import org.junit.Assert._ final class CapturingLogger extends Logger { import CapturingLogger._ - val lines = mutable.ListBuffer.empty[LogLine] + private val lines = mutable.ListBuffer.empty[LogLine] def log(level: Level, message: => String): Unit = lines += new LogLine(level, message) @@ -33,7 +34,7 @@ final class CapturingLogger extends Logger { } object CapturingLogger { - final class LogLine(val level: Level, val message: String) { + final case class LogLine(val level: Level, val message: String) { def contains(messagePart: String): Boolean = message.contains(messagePart) @@ -78,6 +79,14 @@ object CapturingLogger { def assertContainsError(messagePart: String): Unit = assertContains(Level.Error, messagePart) + def assertContainsMatch(messageRegex: Regex): Seq[String] = { + lines.collectFirst { + case LogLine(_, messageRegex(captures @ _*)) => captures + }.getOrElse { + throw new AssertionError(s"expected a log line matching '$messageRegex', but got \n${this}") + } + } + override def toString(): String = lines.mkString(" ", "\n ", "") } From c4c5e2adad3f240b5e5955e9a60d7b6451623ce1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Tue, 28 Feb 2023 17:48:35 +0100 Subject: [PATCH 18/45] Directly write bytes in the back-end instead of using a UTF-8 encoder. Profiling showed that a significant amount of time was spent in the UTF-8 encoders. Since the `SourceMapWriter` and `Printers` only produce ASCII characters anyway, we now directly write them as `Byte`s in a byte array. The class `ByteArrayWriter` provides an interface to write ASCII-as-bytes. We still use UTF-8 encoders for the header and footer of `.js` files. This brings about a 3x speedup to the back-end in the incremental case, and about 30% speedup in the cold start case. --- .../src/main/scala/org/scalajs/ir/Utils.scala | 8 - .../closure/ClosureLinkerBackend.scala | 23 ++- .../linker/backend/BasicLinkerBackend.scala | 37 ++-- .../scalajs/linker/backend/OutputWriter.scala | 33 +--- .../backend/javascript/ByteArrayWriter.scala | 174 ++++++++++++++++++ .../linker/backend/javascript/Printers.scala | 43 ++++- .../backend/javascript/SourceMapWriter.scala | 40 ++-- .../linker/backend/javascript/Trees.scala | 6 +- .../linker/backend/javascript/Utils.scala | 78 -------- 9 files changed, 275 insertions(+), 167 deletions(-) create mode 100644 linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala delete mode 100644 linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Utils.scala diff --git a/ir/shared/src/main/scala/org/scalajs/ir/Utils.scala b/ir/shared/src/main/scala/org/scalajs/ir/Utils.scala index 807ecd56bd..3e54a88091 100644 --- a/ir/shared/src/main/scala/org/scalajs/ir/Utils.scala +++ b/ir/shared/src/main/scala/org/scalajs/ir/Utils.scala @@ -14,10 +14,6 @@ package org.scalajs.ir private[ir] object Utils { - /* !!! BEGIN CODE VERY SIMILAR TO linker/.../javascript/Utils.scala and - * js-envs/.../JSUtils.scala - */ - private final val EscapeJSChars = "\\b\\t\\n\\v\\f\\r\\\"\\\\" private[ir] def printEscapeJS(str: String, out: java.io.Writer): Unit = { @@ -64,10 +60,6 @@ private[ir] object Utils { } } - /* !!! END CODE VERY SIMILAR TO linker/.../javascript/Utils.scala and - * js-envs/.../JSUtils.scala - */ - /** A ByteArrayOutput stream that allows to jump back to a given * position and complete some bytes. Methods must be called in the * following order only: diff --git a/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala b/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala index df931c2c45..641973d527 100644 --- a/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala +++ b/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala @@ -15,6 +15,7 @@ package org.scalajs.linker.backend.closure import scala.concurrent._ import java.io.Writer +import java.nio.charset.StandardCharsets import java.util.{Arrays, HashSet} import com.google.javascript.jscomp.{ @@ -30,7 +31,7 @@ import org.scalajs.linker.interface._ import org.scalajs.linker.interface.unstable.OutputPatternsImpl import org.scalajs.linker.backend._ import org.scalajs.linker.backend.emitter.Emitter -import org.scalajs.linker.backend.javascript.{Trees => js} +import org.scalajs.linker.backend.javascript.{ByteArrayWriter, Trees => js} import org.scalajs.linker.standard._ import org.scalajs.linker.standard.ModuleSet.ModuleID @@ -207,21 +208,27 @@ final class ClosureLinkerBackend(config: LinkerBackendImpl.Config) writer.write(footer) } - protected def writeModule(moduleID: ModuleID, jsFileWriter: Writer): Unit = { - writeCode(jsFileWriter) + protected def writeModule(moduleID: ModuleID, jsFileWriter: ByteArrayWriter): Unit = { + val jsFileStrWriter = new java.io.OutputStreamWriter(jsFileWriter, StandardCharsets.UTF_8) + writeCode(jsFileStrWriter) + jsFileStrWriter.flush() } - protected def writeModule(moduleID: ModuleID, jsFileWriter: Writer, - sourceMapWriter: Writer): Unit = { + protected def writeModule(moduleID: ModuleID, jsFileWriter: ByteArrayWriter, + sourceMapWriter: ByteArrayWriter): Unit = { val jsFileURI = OutputPatternsImpl.jsFileURI(config.outputPatterns, moduleID.id) val sourceMapURI = OutputPatternsImpl.sourceMapURI(config.outputPatterns, moduleID.id) - writeCode(jsFileWriter) - jsFileWriter.write("//# sourceMappingURL=" + sourceMapURI + "\n") + val jsFileStrWriter = new java.io.OutputStreamWriter(jsFileWriter, StandardCharsets.UTF_8) + writeCode(jsFileStrWriter) + jsFileStrWriter.write("//# sourceMappingURL=" + sourceMapURI + "\n") + jsFileStrWriter.flush() + val sourceMapStrWriter = new java.io.OutputStreamWriter(sourceMapWriter, StandardCharsets.UTF_8) val sourceMap = gccResult.get._2 sourceMap.setWrapperPrefix(header) - sourceMap.appendTo(sourceMapWriter, jsFileURI) + sourceMap.appendTo(sourceMapStrWriter, jsFileURI) + sourceMapStrWriter.flush() } } diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala index 52427d1a0d..19fb7a52cf 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala @@ -14,7 +14,7 @@ package org.scalajs.linker.backend import scala.concurrent._ -import java.io.Writer +import java.nio.charset.StandardCharsets import org.scalajs.logging.Logger @@ -24,7 +24,7 @@ import org.scalajs.linker.standard._ import org.scalajs.linker.standard.ModuleSet.ModuleID import org.scalajs.linker.backend.emitter.Emitter -import org.scalajs.linker.backend.javascript.{Printers, SourceMapWriter, Trees => js} +import org.scalajs.linker.backend.javascript.{ByteArrayWriter, Printers, SourceMapWriter, Trees => js} /** The basic backend for the Scala.js linker. * @@ -63,22 +63,22 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config) } val writer = new OutputWriter(output, config) { - protected def writeModule(moduleID: ModuleID, jsFileWriter: Writer): Unit = { + protected def writeModule(moduleID: ModuleID, jsFileWriter: ByteArrayWriter): Unit = { val printedModuleCache = printedModuleSetCache.getModuleCache(moduleID) - jsFileWriter.write(emitterResult.header) - jsFileWriter.write("'use strict';\n") + jsFileWriter.write(emitterResult.header.getBytes(StandardCharsets.UTF_8)) + jsFileWriter.writeASCIIString("'use strict';\n") for (topLevelTree <- emitterResult.body(moduleID)) { val printedTree = printedModuleCache.getPrintedTree(topLevelTree) jsFileWriter.write(printedTree.jsCode) } - jsFileWriter.write(emitterResult.footer) + jsFileWriter.write(emitterResult.footer.getBytes(StandardCharsets.UTF_8)) } - protected def writeModule(moduleID: ModuleID, jsFileWriter: Writer, - sourceMapWriter: Writer): Unit = { + protected def writeModule(moduleID: ModuleID, jsFileWriter: ByteArrayWriter, + sourceMapWriter: ByteArrayWriter): Unit = { val printedModuleCache = printedModuleSetCache.getModuleCache(moduleID) val jsFileURI = OutputPatternsImpl.jsFileURI(config.outputPatterns, moduleID.id) @@ -87,11 +87,11 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config) val smWriter = new SourceMapWriter(sourceMapWriter, jsFileURI, config.relativizeSourceMapBase) - jsFileWriter.write(emitterResult.header) + jsFileWriter.write(emitterResult.header.getBytes(StandardCharsets.UTF_8)) for (_ <- 0 until emitterResult.header.count(_ == '\n')) smWriter.nextLine() - jsFileWriter.write("'use strict';\n") + jsFileWriter.writeASCIIString("'use strict';\n") smWriter.nextLine() for (topLevelTree <- emitterResult.body(moduleID)) { @@ -100,8 +100,8 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config) smWriter.insertFragment(printedTree.sourceMapFragment) } - jsFileWriter.write(emitterResult.footer) - jsFileWriter.write("//# sourceMappingURL=" + sourceMapURI + "\n") + jsFileWriter.write(emitterResult.footer.getBytes(StandardCharsets.UTF_8)) + jsFileWriter.write(("//# sourceMappingURL=" + sourceMapURI + "\n").getBytes(StandardCharsets.UTF_8)) smWriter.complete() } @@ -162,10 +162,7 @@ private object BasicLinkerBackend { } } - /* TODO Instead of caching the String, we should cache the already - * UTF-8-encoded Byte array. - */ - private final class PrintedTree(val jsCode: String, val sourceMapFragment: SourceMapWriter.Fragment) { + private final class PrintedTree(val jsCode: Array[Byte], val sourceMapFragment: SourceMapWriter.Fragment) { var cachedUsed: Boolean = false } @@ -195,12 +192,12 @@ private object BasicLinkerBackend { } protected def computePrintedTree(tree: js.Tree): PrintedTree = { - val jsCodeWriter = new java.io.StringWriter() + val jsCodeWriter = new ByteArrayWriter() val printer = new Printers.JSTreePrinter(jsCodeWriter) printer.printTopLevelTree(tree) - new PrintedTree(jsCodeWriter.toString(), SourceMapWriter.Fragment.Empty) + new PrintedTree(jsCodeWriter.toByteArray(), SourceMapWriter.Fragment.Empty) } def cleanAfterRun(): Boolean = { @@ -228,14 +225,14 @@ private object BasicLinkerBackend { private final class PrintedModuleCacheWithSourceMaps extends PrintedModuleCache { override protected def computePrintedTree(tree: js.Tree): PrintedTree = { - val jsCodeWriter = new java.io.StringWriter() + val jsCodeWriter = new ByteArrayWriter() val smFragmentBuilder = new SourceMapWriter.FragmentBuilder() val printer = new Printers.JSTreePrinterWithSourceMap(jsCodeWriter, smFragmentBuilder) printer.printTopLevelTree(tree) smFragmentBuilder.complete() - new PrintedTree(jsCodeWriter.toString(), smFragmentBuilder.result()) + new PrintedTree(jsCodeWriter.toByteArray(), smFragmentBuilder.result()) } } } diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/OutputWriter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/OutputWriter.scala index 5a1b2c9ddb..1a54878c86 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/OutputWriter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/OutputWriter.scala @@ -16,24 +16,24 @@ import scala.concurrent._ import java.io._ import java.nio.ByteBuffer -import java.nio.charset.StandardCharsets import org.scalajs.linker.interface.{OutputDirectory, Report} import org.scalajs.linker.interface.unstable.{OutputDirectoryImpl, OutputPatternsImpl, ReportImpl} import org.scalajs.linker.standard.{ModuleSet, IOThrottler} import org.scalajs.linker.standard.ModuleSet.ModuleID +import org.scalajs.linker.backend.javascript.ByteArrayWriter + private[backend] abstract class OutputWriter(output: OutputDirectory, config: LinkerBackendImpl.Config) { - import OutputWriter.ByteArrayWriter private val outputImpl = OutputDirectoryImpl.fromOutputDirectory(output) private val moduleKind = config.commonConfig.coreSpec.moduleKind - protected def writeModule(moduleID: ModuleID, jsFileWriter: Writer): Unit + protected def writeModule(moduleID: ModuleID, jsFileWriter: ByteArrayWriter): Unit - protected def writeModule(moduleID: ModuleID, jsFileWriter: Writer, - sourceMapWriter: Writer): Unit + protected def writeModule(moduleID: ModuleID, jsFileWriter: ByteArrayWriter, + sourceMapWriter: ByteArrayWriter): Unit def write(moduleSet: ModuleSet)(implicit ec: ExecutionContext): Future[Report] = { val ioThrottler = new IOThrottler(config.maxConcurrentWrites) @@ -71,10 +71,10 @@ private[backend] abstract class OutputWriter(output: OutputDirectory, val codeWriter = new ByteArrayWriter val smWriter = new ByteArrayWriter - writeModule(moduleID, codeWriter.writer, smWriter.writer) + writeModule(moduleID, codeWriter, smWriter) - val code = codeWriter.result() - val sourceMap = smWriter.result() + val code = codeWriter.toByteBuffer() + val sourceMap = smWriter.toByteBuffer() for { _ <- outputImpl.writeFull(jsFileName, code) @@ -85,9 +85,9 @@ private[backend] abstract class OutputWriter(output: OutputDirectory, } else { val codeWriter = new ByteArrayWriter - writeModule(moduleID, codeWriter.writer) + writeModule(moduleID, codeWriter) - val code = codeWriter.result() + val code = codeWriter.toByteBuffer() for { _ <- outputImpl.writeFull(jsFileName, code) @@ -97,16 +97,3 @@ private[backend] abstract class OutputWriter(output: OutputDirectory, } } } - -private object OutputWriter { - private class ByteArrayWriter { - private val byteStream = new ByteArrayOutputStream - - val writer: Writer = new OutputStreamWriter(byteStream, StandardCharsets.UTF_8) - - def result(): ByteBuffer = { - writer.close() - ByteBuffer.wrap(byteStream.toByteArray()) - } - } -} diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala new file mode 100644 index 0000000000..f23db88f32 --- /dev/null +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala @@ -0,0 +1,174 @@ +/* + * Scala.js (https://www.scala-js.org/) + * + * Copyright EPFL. + * + * Licensed under Apache License 2.0 + * (https://www.apache.org/licenses/LICENSE-2.0). + * + * See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + */ + +package org.scalajs.linker.backend.javascript + +import java.io.OutputStream +import java.nio.ByteBuffer + +/** Like a `java.io.ByteArrayOutputStream` but with more control. */ +private[backend] final class ByteArrayWriter extends OutputStream { + private var buffer: Array[Byte] = new Array[Byte](1024) + private var size: Int = 0 + + private def ensureCapacity(capacity: Int): Unit = { + if (buffer.length < capacity) + buffer = java.util.Arrays.copyOf(buffer, powerOfTwoAtLeast(capacity)) + } + + private def powerOfTwoAtLeast(capacity: Int): Int = + java.lang.Integer.highestOneBit(capacity - 1) << 1 + + private def grow(): Unit = + buffer = java.util.Arrays.copyOf(buffer, buffer.length * 2) + + def write(b: Int): Unit = { + if (size == buffer.length) + grow() + buffer(size) = b.toByte + size += 1 + } + + override def write(bs: Array[Byte]): Unit = + write(bs, 0, bs.length) + + override def write(bs: Array[Byte], start: Int, len: Int): Unit = { + val newSize = size + len + ensureCapacity(newSize) + System.arraycopy(bs, start, buffer, size, len) + size = newSize + } + + def writeASCIIString(str: String): Unit = { + val len = str.length() + val oldSize = size + val newSize = oldSize + len + ensureCapacity(newSize) + + val buffer = this.buffer // local copy -- after ensureCapacity! + var i = 0 + while (i != len) { + buffer(oldSize + i) = str.charAt(i).toByte + i += 1 + } + + size = newSize + } + + /** Writes an ASCII-escaped JavaScript string to the buffer. + * + * @return + * the number of ASCII chars (i.e., bytes) that were written + */ + def writeASCIIEscapedJSString(str: String): Int = { + // scalastyle:off return + + /* Note that Java and JavaScript happen to use the same encoding for + * Unicode, namely UTF-16, which means that 1 char from Java always equals + * 1 char in JavaScript. + */ + + // First, a fast path for cases where we do not need to escape anything. + + val oldSize = size + val len = str.length() + ensureCapacity(oldSize + len) + + val buffer = this.buffer // local copy -- after ensureCapacity! + var i = 0 + while (i != len) { + val c = str.charAt(i).toInt + + if (c >= 32 && c <= 126 && c != '\"' && c != '\\') { + buffer(oldSize + i) = c.toByte + i += 1 + } else { + return writeASCIIEscapedJSStringSlowPath(str, i) + } + } + + size = oldSize + len + len // number of bytes written + + // scalastyle:on return + } + + /** Slow path when we encounter at least one char needing an escape. + * + * When calling this method, the first `start` chars of `str` have already + * been written in the buffer from offset `size` onwards, and there are + * still at least `str.length() - start` bytes available in the buffer. + * + * @return + * the number of ASCII chars (i.e., bytes) that were written in total, + * including the first `start` bytes. + */ + private def writeASCIIEscapedJSStringSlowPath(str: String, start: Int): Int = { + val oldSize = size + + var offset = oldSize + start + val len = str.length() + var i = start + + // Loop invariant: there is at least `len - i` bytes available in the buffer + while (i != len) { + val c = str.charAt(i).toInt + i += 1 + + if (c >= 32 && c <= 126 && c != '\"' && c != '\\') { + buffer(offset) = c.toByte + offset += 1 + } else { + // Grow if needed: at most 6 bytes for the escape + room to maintain the invariant + ensureCapacity(offset + 6 + (len - i)) + + buffer(offset) = '\\' + + if (8 <= c && c < 14) { + buffer(offset + 1) = ByteArrayWriter.EscapeJSBytes(c) + offset += 2 + } else if (c == '\"') { + buffer(offset + 1) = '\"' + offset += 2 + } else if (c == '\\') { + buffer(offset + 1) = '\\' + offset += 2 + } else { + def hexDigit(x: Int): Byte = + if (x < 10) (x + '0').toByte else (x + ('a' - 10)).toByte + + buffer(offset + 1) = 'u' + buffer(offset + 2) = hexDigit(c >> 12) + buffer(offset + 3) = hexDigit((c >> 8) & 0x0f) + buffer(offset + 4) = hexDigit((c >> 4) & 0x0f) + buffer(offset + 5) = hexDigit(c & 0x0f) + + offset += 6 + } + } + } + + size = offset + offset - oldSize // number of bytes written in total + } + + def toByteBuffer(): ByteBuffer = + ByteBuffer.wrap(buffer, 0, size).asReadOnlyBuffer() + + def toByteArray(): Array[Byte] = + java.util.Arrays.copyOf(buffer, size) +} + +private object ByteArrayWriter { + private final val EscapeJSBytes: Array[Byte] = + "01234567btnvfr".toArray.map(_.toByte) // offsets 0 to 7 are unused +} diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Printers.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Printers.scala index 13f668ac18..2df5acc9f9 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Printers.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Printers.scala @@ -17,12 +17,9 @@ import scala.annotation.switch // Unimport default print and println to avoid invoking them by mistake import scala.Predef.{print => _, println => _, _} -import java.io.Writer - import org.scalajs.ir import ir.Position import ir.Position.NoPosition -import ir.Printers.IndentationManager import Trees._ @@ -32,8 +29,37 @@ import Trees._ * hotspots in this object. */ object Printers { + private val ReusableIndentArray = Array.fill(128)(' '.toByte) + + class JSTreePrinter(protected val out: ByteArrayWriter) { + private final val IndentStep = 2 + + private var indentMargin = 0 + private var indentArray = ReusableIndentArray + + private def indent(): Unit = indentMargin += IndentStep + private def undent(): Unit = indentMargin -= IndentStep - class JSTreePrinter(protected val out: Writer) extends IndentationManager { + protected final def getIndentMargin(): Int = indentMargin + + protected def println(): Unit = { + out.write('\n') + val indentArray = this.indentArray + val indentMargin = this.indentMargin + val bigEnoughIndentArray = + if (indentMargin <= indentArray.length) indentArray + else growIndentArray() + out.write(bigEnoughIndentArray, 0, indentMargin) + } + + private def growIndentArray(): Array[Byte] = { + val oldIndentArray = indentArray + val oldLen = oldIndentArray.length + val newIndentArray = java.util.Arrays.copyOf(oldIndentArray, oldLen * 2) + System.arraycopy(oldIndentArray, 0, newIndentArray, oldLen, oldLen) + indentArray = newIndentArray + newIndentArray + } def printTopLevelTree(tree: Tree): Unit = { tree match { @@ -700,7 +726,7 @@ object Printers { } protected def printEscapeJS(s: String): Unit = - Utils.printEscapeJS(s, out) + out.writeASCIIEscapedJSString(s) protected def print(ident: Ident): Unit = printEscapeJS(ident.name) @@ -718,14 +744,15 @@ object Printers { protected def print(exportName: ExportName): Unit = printEscapeJS(exportName.name) + /** Prints an ASCII string -- use for syntax strings, not for user strings. */ protected def print(s: String): Unit = - out.write(s) + out.writeASCIIString(s) protected def print(c: Int): Unit = out.write(c) } - class JSTreePrinterWithSourceMap(_out: Writer, + class JSTreePrinterWithSourceMap(_out: ByteArrayWriter, sourceMap: SourceMapWriter.Builder) extends JSTreePrinter(_out) { private var column = 0 @@ -742,7 +769,7 @@ object Printers { } override protected def printEscapeJS(s: String): Unit = - column += Utils.printEscapeJS(s, out) + column += out.writeASCIIEscapedJSString(s) override protected def print(ident: Ident): Unit = { if (ident.pos.isDefined) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala index aa5d8fa038..ff779a802f 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala @@ -38,12 +38,6 @@ object SourceMapWriter { private final val VLQBaseMask = VLQBase - 1 private final val VLQContinuationBit = VLQBase - private def printJSONString(s: String, out: Writer) = { - out.write('\"') - Utils.printEscapeJS(s, out) - out.write('\"') - } - private final class NodePosStack { private var topIndex: Int = -1 private var posStack: Array[Position] = new Array(128) @@ -207,7 +201,7 @@ object SourceMapWriter { } } -final class SourceMapWriter(out: Writer, jsFileName: String, +final class SourceMapWriter(out: ByteArrayWriter, jsFileName: String, relativizeBaseURI: Option[URI]) extends SourceMapWriter.Builder { @@ -252,11 +246,17 @@ final class SourceMapWriter(out: Writer, jsFileName: String, } } + private def writeJSONString(s: String): Unit = { + out.write('\"') + out.writeASCIIEscapedJSString(s) + out.write('\"') + } + private def writeHeader(): Unit = { - out.write("{\n\"version\": 3") - out.write(",\n\"file\": ") - printJSONString(jsFileName, out) - out.write(",\n\"mappings\": \"") + out.writeASCIIString("{\n\"version\": 3") + out.writeASCIIString(",\n\"file\": ") + writeJSONString(jsFileName) + out.writeASCIIString(",\n\"mappings\": \"") } protected def doWriteNewLine(): Unit = { @@ -316,25 +316,25 @@ final class SourceMapWriter(out: Writer, jsFileName: String, protected def doComplete(): Unit = { var restSources = sources.result() - out.write("\",\n\"sources\": [") + out.writeASCIIString("\",\n\"sources\": [") while (restSources.nonEmpty) { - printJSONString(restSources.head, out) + writeJSONString(restSources.head) restSources = restSources.tail if (restSources.nonEmpty) - out.write(", ") + out.writeASCIIString(", ") } var restNames = names.result() - out.write("],\n\"names\": [") + out.writeASCIIString("],\n\"names\": [") while (restNames.nonEmpty) { - printJSONString(restNames.head, out) + writeJSONString(restNames.head) restNames = restNames.tail if (restNames.nonEmpty) - out.write(", ") + out.writeASCIIString(", ") } - out.write("],\n\"lineCount\": ") - out.write(lineCountInGenerated.toString) - out.write("\n}\n") + out.writeASCIIString("],\n\"lineCount\": ") + out.writeASCIIString(lineCountInGenerated.toString) + out.writeASCIIString("\n}\n") } /** Write the Base 64 VLQ of an integer to the mappings diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Trees.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Trees.scala index dfc28ccbd0..27da8e50c1 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Trees.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Trees.scala @@ -14,6 +14,8 @@ package org.scalajs.linker.backend.javascript import scala.annotation.switch +import java.nio.charset.StandardCharsets + import org.scalajs.ir import org.scalajs.ir.{OriginalName, Position} import org.scalajs.ir.OriginalName.NoOriginalName @@ -30,10 +32,10 @@ object Trees { val pos: Position def show: String = { - val writer = new java.io.StringWriter + val writer = new ByteArrayWriter() val printer = new Printers.JSTreePrinter(writer) printer.printTree(this, isStat = true) - writer.toString() + new String(writer.toByteArray(), StandardCharsets.US_ASCII) } } diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Utils.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Utils.scala deleted file mode 100644 index c492110b5d..0000000000 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/Utils.scala +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Scala.js (https://www.scala-js.org/) - * - * Copyright EPFL. - * - * Licensed under Apache License 2.0 - * (https://www.apache.org/licenses/LICENSE-2.0). - * - * See the NOTICE file distributed with this work for - * additional information regarding copyright ownership. - */ - -package org.scalajs.linker.backend.javascript - -private[javascript] object Utils { - - /* !!! BEGIN CODE VERY SIMILAR TO ir/.../Utils.scala and - * js-envs/.../JSUtils.scala - */ - - private final val EscapeJSChars = "\\b\\t\\n\\v\\f\\r\\\"\\\\" - - def printEscapeJS(str: String, out: java.io.Writer): Int = { - /* Note that Java and JavaScript happen to use the same encoding for - * Unicode, namely UTF-16, which means that 1 char from Java always equals - * 1 char in JavaScript. */ - val end = str.length() - var i = 0 - var writtenChars = 0 - /* Loop prints all consecutive ASCII printable characters starting - * from current i and one non ASCII printable character (if it exists). - * The new i is set at the end of the appended characters. - */ - while (i != end) { - val start = i - var c: Int = str.charAt(i) - // Find all consecutive ASCII printable characters from `start` - while (i != end && c >= 32 && c <= 126 && c != 34 && c != 92) { - i += 1 - if (i != end) - c = str.charAt(i) - } - // Print ASCII printable characters from `start` - if (start != i) { - out.write(str, start, i - start) - writtenChars += i - } - - // Print next non ASCII printable character - if (i != end) { - def escapeJSEncoded(c: Int): Unit = { - if (7 < c && c < 14) { - val i = 2 * (c - 8) - out.write(EscapeJSChars, i, 2) - writtenChars += 2 - } else if (c == 34) { - out.write(EscapeJSChars, 12, 2) - writtenChars += 2 - } else if (c == 92) { - out.write(EscapeJSChars, 14, 2) - writtenChars += 2 - } else { - out.write("\\u%04x".format(c)) - writtenChars += 6 - } - } - escapeJSEncoded(c) - i += 1 - } - } - writtenChars - } - - /* !!! END CODE VERY SIMILAR TO ir/.../Utils.scala and - * js-envs/.../JSUtils.scala - */ - -} From d716dbab2ee2a7753f114b509b17293fd1bad628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Fri, 24 Feb 2023 11:54:03 +0100 Subject: [PATCH 19/45] Preallocate the final buffers with 110% the size of the previous run. This means that in the common case, we will only need 0 or 1 `grow()` of the buffer. This speeds up the back-end by about 10% in the incremental case. --- .../linker/backend/BasicLinkerBackend.scala | 24 +++++++++++++++++++ .../backend/javascript/ByteArrayWriter.scala | 5 ++++ 2 files changed, 29 insertions(+) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala index 19fb7a52cf..cdfe141c62 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala @@ -66,6 +66,8 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config) protected def writeModule(moduleID: ModuleID, jsFileWriter: ByteArrayWriter): Unit = { val printedModuleCache = printedModuleSetCache.getModuleCache(moduleID) + jsFileWriter.sizeHint(sizeHintFor(printedModuleCache.getPreviousFinalJSFileSize())) + jsFileWriter.write(emitterResult.header.getBytes(StandardCharsets.UTF_8)) jsFileWriter.writeASCIIString("'use strict';\n") @@ -75,12 +77,17 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config) } jsFileWriter.write(emitterResult.footer.getBytes(StandardCharsets.UTF_8)) + + printedModuleCache.recordFinalSizes(jsFileWriter.currentSize, 0) } protected def writeModule(moduleID: ModuleID, jsFileWriter: ByteArrayWriter, sourceMapWriter: ByteArrayWriter): Unit = { val printedModuleCache = printedModuleSetCache.getModuleCache(moduleID) + jsFileWriter.sizeHint(sizeHintFor(printedModuleCache.getPreviousFinalJSFileSize())) + sourceMapWriter.sizeHint(sizeHintFor(printedModuleCache.getPreviousFinalSourceMapSize())) + val jsFileURI = OutputPatternsImpl.jsFileURI(config.outputPatterns, moduleID.id) val sourceMapURI = OutputPatternsImpl.sourceMapURI(config.outputPatterns, moduleID.id) @@ -104,7 +111,12 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config) jsFileWriter.write(("//# sourceMappingURL=" + sourceMapURI + "\n").getBytes(StandardCharsets.UTF_8)) smWriter.complete() + + printedModuleCache.recordFinalSizes(jsFileWriter.currentSize, sourceMapWriter.currentSize) } + + private def sizeHintFor(previousSize: Int): Int = + previousSize + (previousSize / 10) } printedModuleSetCache.startRun() @@ -170,6 +182,9 @@ private object BasicLinkerBackend { private var cacheUsed = false private val cache = new java.util.IdentityHashMap[js.Tree, PrintedTree] + private var previousFinalJSFileSize: Int = 0 + private var previousFinalSourceMapSize: Int = 0 + private var totalTopLevelTrees = 0 private var recomputedTopLevelTrees = 0 @@ -179,6 +194,15 @@ private object BasicLinkerBackend { recomputedTopLevelTrees = 0 } + def getPreviousFinalJSFileSize(): Int = previousFinalJSFileSize + + def getPreviousFinalSourceMapSize(): Int = previousFinalSourceMapSize + + def recordFinalSizes(finalJSFileSize: Int, finalSourceMapSize: Int): Unit = { + previousFinalJSFileSize = finalJSFileSize + previousFinalSourceMapSize = finalSourceMapSize + } + def getPrintedTree(tree: js.Tree): PrintedTree = { totalTopLevelTrees += 1 diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala index f23db88f32..83adbd32c6 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala @@ -20,6 +20,11 @@ private[backend] final class ByteArrayWriter extends OutputStream { private var buffer: Array[Byte] = new Array[Byte](1024) private var size: Int = 0 + def currentSize: Int = size + + def sizeHint(capacity: Int): Unit = + ensureCapacity(capacity) + private def ensureCapacity(capacity: Int): Unit = { if (buffer.length < capacity) buffer = java.util.Arrays.copyOf(buffer, powerOfTwoAtLeast(capacity)) From b008d682fce0ec38ba8e6decf016e264a6e53b53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Fri, 3 Mar 2023 18:50:50 +0100 Subject: [PATCH 20/45] IR checker: avoid calling the i"" interpolator on the success path. Also avoid two useless calls to `lookupClass` which were around. This appears to be speed up the IR checker by 2x. --- .../scalajs/linker/checker/IRChecker.scala | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/checker/IRChecker.scala b/linker/shared/src/main/scala/org/scalajs/linker/checker/IRChecker.scala index 2801ec93d4..2e3678a6d4 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/checker/IRChecker.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/checker/IRChecker.scala @@ -198,15 +198,14 @@ private final class IRChecker(unit: LinkingUnit, reporter: ErrorReporter) { private def typecheck(tree: Tree, env: Env): Unit = { implicit val ctx = ErrorContext(tree) - def checkApplyGeneric(methodName: MethodName, methodFullName: String, + def checkApplyGeneric(receiverTypeForError: Any, methodName: MethodName, args: List[Tree], tpe: Type, isStatic: Boolean): Unit = { val (methodParams, resultType) = inferMethodType(methodName, isStatic) for ((actual, formal) <- args zip methodParams) { typecheckExpect(actual, env, formal) } if (tpe != resultType) - reportError(i"Call to $methodFullName of type $resultType "+ - i"typed as ${tree.tpe}") + reportError(i"Call to $receiverTypeForError.$methodName of type $resultType typed as ${tree.tpe}") } tree match { @@ -313,8 +312,7 @@ private final class IRChecker(unit: LinkingUnit, reporter: ErrorReporter) { val clazz = lookupClass(className) if (clazz.kind != ClassKind.Class) reportError(i"new $className which is not a class") - checkApplyGeneric(ctor.name, i"$className.$ctor", args, NoType, - isStatic = false) + checkApplyGeneric(className, ctor.name, args, NoType, isStatic = false) case LoadModule(className) => val clazz = lookupClass(className) @@ -404,8 +402,7 @@ private final class IRChecker(unit: LinkingUnit, reporter: ErrorReporter) { true } if (fullCheck) { - checkApplyGeneric(method, i"${receiver.tpe}.$method", args, tree.tpe, - isStatic = false) + checkApplyGeneric(receiver.tpe, method, args, tree.tpe, isStatic = false) } else { for (arg <- args) typecheckExpr(arg, env) @@ -413,24 +410,19 @@ private final class IRChecker(unit: LinkingUnit, reporter: ErrorReporter) { case ApplyStatically(_, receiver, className, MethodIdent(method), args) => typecheckExpect(receiver, env, ClassType(className)) - checkApplyGeneric(method, i"$className.$method", args, tree.tpe, - isStatic = false) + checkApplyGeneric(className, method, args, tree.tpe, isStatic = false) case ApplyStatic(_, className, MethodIdent(method), args) => - val clazz = lookupClass(className) - checkApplyGeneric(method, i"$className.$method", args, tree.tpe, - isStatic = true) + checkApplyGeneric(className, method, args, tree.tpe, isStatic = true) case ApplyDynamicImport(_, className, MethodIdent(method), args) => - val clazz = lookupClass(className) - val methodFullName = i"$className.$method" - - checkApplyGeneric(method, methodFullName, args, AnyType, isStatic = true) + checkApplyGeneric(className, method, args, AnyType, isStatic = true) val resultType = method.resultTypeRef if (resultType != ClassRef(ObjectClass)) { - reportError(i"illegal dynamic import call to $methodFullName with " + - i"non-object result type: $resultType") + reportError( + i"illegal dynamic import call to $className.$method " + + i"with non-object result type: $resultType") } case UnaryOp(op, lhs) => From 1268d31a8027bd252be0a1456f0bb88887d3fc49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Thu, 23 Feb 2023 11:42:58 +0100 Subject: [PATCH 21/45] Cache the static initialization trees. So that the `BasicLinkerBackend` can identify them as unchanged. --- .../linker/backend/emitter/ClassEmitter.scala | 22 +++++++++---------- .../linker/backend/emitter/Emitter.scala | 9 ++++++-- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/ClassEmitter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/ClassEmitter.scala index 07d5044001..a12406a32d 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/ClassEmitter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/ClassEmitter.scala @@ -480,22 +480,22 @@ private[emitter] final class ClassEmitter(sjsGen: SJSGen) { WithGlobals.list(statsWithGlobals) } + /** Does the class need static initialization generated by `genStaticInitialization`? */ + def needStaticInitialization(tree: LinkedClass): Boolean = { + tree.methods.exists { m => + m.flags.namespace == MemberNamespace.StaticConstructor && + m.methodName.isStaticInitializer + } + } + /** Generates the static initializer invocation of a class. */ def genStaticInitialization(tree: LinkedClass)( implicit moduleContext: ModuleContext, globalKnowledge: GlobalKnowledge): List[js.Tree] = { implicit val pos = tree.pos - val hasStaticInit = tree.methods.exists { m => - m.flags.namespace == MemberNamespace.StaticConstructor && - m.methodName.isStaticInitializer - } - if (hasStaticInit) { - val field = globalVar("sct", (tree.className, StaticInitializerName), - StaticInitializerOriginalName) - js.Apply(field, Nil) :: Nil - } else { - Nil - } + val field = globalVar("sct", (tree.className, StaticInitializerName), + StaticInitializerOriginalName) + js.Apply(field, Nil) :: Nil } /** Generates the class initializer invocation of a class. */ diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala index 31a059454c..c12b6df045 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala @@ -578,8 +578,12 @@ final class Emitter(config: Emitter.Config) { // Static initialization - val staticInitialization = - classEmitter.genStaticInitialization(linkedClass)(moduleContext, uncachedKnowledge) + val staticInitialization = if (classEmitter.needStaticInitialization(linkedClass)) { + classTreeCache.staticInitialization.getOrElseUpdate( + classEmitter.genStaticInitialization(linkedClass)(moduleContext, classCache)) + } else { + Nil + } // Build the result @@ -877,6 +881,7 @@ object Emitter { val typeData = new OneTimeCache[WithGlobals[js.Tree]] val setTypeData = new OneTimeCache[js.Tree] val moduleAccessor = new OneTimeCache[WithGlobals[js.Tree]] + val staticInitialization = new OneTimeCache[List[js.Tree]] val staticFields = new OneTimeCache[WithGlobals[List[js.Tree]]] } From 414ffbd79d6c1ef6374ff2e2ac5e32d43098a7cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Fri, 24 Feb 2023 16:30:15 +0100 Subject: [PATCH 22/45] Cache the module import trees. So that the `BasicLinkerBackend` can identify them as unchanged. --- .../linker/backend/emitter/Emitter.scala | 44 +++++++++++++++++-- .../linker/backend/emitter/WithGlobals.scala | 2 + 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala index c12b6df045..70c464cb0f 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala @@ -55,6 +55,8 @@ final class Emitter(config: Emitter.Config) { val coreJSLibCache: CoreJSLibCache = new CoreJSLibCache + val moduleCaches: mutable.Map[ModuleID, ModuleCache] = mutable.Map.empty + val classCaches: mutable.Map[ClassID, ClassCache] = mutable.Map.empty } @@ -135,6 +137,7 @@ final class Emitter(config: Emitter.Config) { s"invalidated: $statsMethodsInvalidated") // Inform caches about run completion. + state.moduleCaches.filterInPlace((_, c) => c.cleanAfterRun()) classCaches.filterInPlace((_, c) => c.cleanAfterRun()) } } @@ -191,9 +194,16 @@ final class Emitter(config: Emitter.Config) { val moduleTrees = logger.time("Emitter: Write trees") { moduleSet.modules.map { module => val moduleContext = ModuleContext.fromModule(module) + val moduleCache = state.moduleCaches.getOrElseUpdate(module.id, new ModuleCache) val moduleClasses = generatedClasses(module.id) + val moduleImports = extractWithGlobals { + moduleCache.getOrComputeImports(module.externalDependencies, module.internalDependencies) { + genModuleImports(module) + } + } + val topLevelExports = extractWithGlobals { // We do not cache top level exports since typically there are few. classEmitter.genTopLevelExports(module.topLevelExports)( @@ -279,16 +289,15 @@ final class Emitter(config: Emitter.Config) { case stat => stat :: Nil }.toList + // Make sure that there is at least one non-import definition. assert(!defTrees.isEmpty, { val classNames = module.classDefs.map(_.fullName).mkString(", ") s"Module ${module.id} is empty. Classes in this module: $classNames" }) - /* Module imports, which depend on nothing. + /* Add module imports, which depend on nothing, at the front. * All classes potentially depend on them. */ - val moduleImports = extractWithGlobals(genModuleImports(module)) - val allTrees = moduleImports ::: defTrees classIter.foreach { genClass => @@ -319,7 +328,7 @@ final class Emitter(config: Emitter.Config) { moduleKind match { case ModuleKind.NoModule => - WithGlobals(Nil) + WithGlobals.nil case ModuleKind.ESModule => val imports = importParts.map { case (ident, moduleName) => @@ -598,6 +607,33 @@ final class Emitter(config: Emitter.Config) { // Caching + private final class ModuleCache { + private[this] var _cacheUsed: Boolean = false + + private[this] var _importsCache: WithGlobals[List[js.Tree]] = WithGlobals.nil + private[this] var _lastExternalDependencies: Set[String] = Set.empty + private[this] var _lastInternalDependencies: Set[ModuleID] = Set.empty + + def getOrComputeImports(externalDependencies: Set[String], internalDependencies: Set[ModuleID])( + compute: => WithGlobals[List[js.Tree]]): WithGlobals[List[js.Tree]] = { + + _cacheUsed = true + + if (externalDependencies != _lastExternalDependencies || internalDependencies != _lastInternalDependencies) { + _importsCache = compute + _lastExternalDependencies = externalDependencies + _lastInternalDependencies = internalDependencies + } + _importsCache + } + + def cleanAfterRun(): Boolean = { + val result = _cacheUsed + _cacheUsed = false + result + } + } + private final class ClassCache extends knowledgeGuardian.KnowledgeAccessor { private[this] var _cache: DesugaredClassCache = null private[this] var _lastVersion: Version = Version.Unversioned diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/WithGlobals.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/WithGlobals.scala index b2d4556121..65b10a9ed1 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/WithGlobals.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/WithGlobals.scala @@ -91,6 +91,8 @@ private[emitter] object WithGlobals { def apply[A](value: A): WithGlobals[A] = new WithGlobals(value, Set.empty) + val nil: WithGlobals[Nil.type] = WithGlobals(Nil) + def list[A](xs: List[WithGlobals[A]]): WithGlobals[List[A]] = { /* This could be a cascade of flatMap's, but the following should be more * efficient. From 5de8e6f85059b3453ccc12ba140d4a5fd56f2058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Thu, 2 Mar 2023 17:11:15 +0100 Subject: [PATCH 23/45] Cache the module initializer trees. So that the `BasicLinkerBackend` can identify them as unchanged. --- .../linker/backend/emitter/Emitter.scala | 48 ++++++++++++++++--- .../linker/BasicLinkerBackendTest.scala | 7 +-- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala index 70c464cb0f..2081541f6a 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala @@ -210,6 +210,16 @@ final class Emitter(config: Emitter.Config) { moduleContext, uncachedKnowledge) } + val moduleInitializers = extractWithGlobals { + val initializers = module.initializers.toList + moduleCache.getOrComputeInitializers(initializers) { + WithGlobals.list(initializers.map { initializer => + classEmitter.genModuleInitializer(initializer)( + moduleContext, moduleCache) + }) + } + } + val coreJSLib = if (module.isRoot) Some(extractWithGlobals(state.coreJSLibCache.build(moduleContext))) else None @@ -270,13 +280,10 @@ final class Emitter(config: Emitter.Config) { * causing JS static initializers to run. Those also must not observe * a non-initialized state of other static fields. */ - topLevelExports ++ + topLevelExports.iterator ++ /* Module initializers, which by spec run at the end. */ - module.initializers.iterator.map { initializer => - extractWithGlobals(classEmitter.genModuleInitializer(initializer)( - moduleContext, uncachedKnowledge)) - } + moduleInitializers.iterator ) /* Flatten all the top-level js.Block's, because we temporarily use @@ -607,13 +614,30 @@ final class Emitter(config: Emitter.Config) { // Caching - private final class ModuleCache { + private final class ModuleCache extends knowledgeGuardian.KnowledgeAccessor { private[this] var _cacheUsed: Boolean = false private[this] var _importsCache: WithGlobals[List[js.Tree]] = WithGlobals.nil private[this] var _lastExternalDependencies: Set[String] = Set.empty private[this] var _lastInternalDependencies: Set[ModuleID] = Set.empty + private[this] var _initializersCache: WithGlobals[List[js.Tree]] = WithGlobals.nil + private[this] var _lastInitializers: List[ModuleInitializer.Initializer] = Nil + + override def invalidate(): Unit = { + super.invalidate() + + /* In order to keep reasoning as local as possible, we also invalidate + * the imports cache, although imports do not use any global knowledge. + */ + _importsCache = WithGlobals.nil + _lastExternalDependencies = Set.empty + _lastInternalDependencies = Set.empty + + _initializersCache = WithGlobals.nil + _lastInitializers = Nil + } + def getOrComputeImports(externalDependencies: Set[String], internalDependencies: Set[ModuleID])( compute: => WithGlobals[List[js.Tree]]): WithGlobals[List[js.Tree]] = { @@ -627,6 +651,18 @@ final class Emitter(config: Emitter.Config) { _importsCache } + def getOrComputeInitializers(initializers: List[ModuleInitializer.Initializer])( + compute: => WithGlobals[List[js.Tree]]): WithGlobals[List[js.Tree]] = { + + _cacheUsed = true + + if (initializers != _lastInitializers) { + _initializersCache = compute + _lastInitializers = initializers + } + _initializersCache + } + def cleanAfterRun(): Boolean = { val result = _cacheUsed _cacheUsed = false diff --git a/linker/shared/src/test/scala/org/scalajs/linker/BasicLinkerBackendTest.scala b/linker/shared/src/test/scala/org/scalajs/linker/BasicLinkerBackendTest.scala index ef3c552141..aa43ba52c1 100644 --- a/linker/shared/src/test/scala/org/scalajs/linker/BasicLinkerBackendTest.scala +++ b/linker/shared/src/test/scala/org/scalajs/linker/BasicLinkerBackendTest.scala @@ -40,7 +40,7 @@ class BasicLinkerBackendTest { * does not invalidate any top-level tree in the second run. */ @Test - def linkNoSecondAttemptInEmitter(): AsyncResult = await { + def noInvalidatedTopLevelTreeInSecondRun(): AsyncResult = await { val classDefs = List( mainTestClassDef(systemOutPrintln(str("Hello world!"))) ) @@ -76,10 +76,7 @@ class BasicLinkerBackendTest { assertEquals("First run must invalidate every top-level tree", total1, recomputed1) assertEquals("Second run must have the same total as first run", total1, total2) - - assertEquals( - "Second run must not invalidate any top-level tree beside the module initializer", - 1, recomputed2) + assertEquals("Second run must not invalidate any top-level tree", 0, recomputed2) } } } From e99a25c74df2d55e0386dcbd7bf4a64b3e38f577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Fri, 3 Mar 2023 10:25:52 +0100 Subject: [PATCH 24/45] Cache the top-level export trees. So that the `BasicLinkerBackend` can identify them as unchanged. --- .../linker/backend/emitter/Emitter.scala | 55 ++++++++++++++++++- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala index 2081541f6a..82a84fd72f 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/Emitter.scala @@ -205,9 +205,13 @@ final class Emitter(config: Emitter.Config) { } val topLevelExports = extractWithGlobals { - // We do not cache top level exports since typically there are few. - classEmitter.genTopLevelExports(module.topLevelExports)( - moduleContext, uncachedKnowledge) + /* We cache top level exports all together, rather than individually, + * since typically there are few. + */ + moduleCache.getOrComputeTopLevelExports(module.topLevelExports) { + classEmitter.genTopLevelExports(module.topLevelExports)( + moduleContext, moduleCache) + } } val moduleInitializers = extractWithGlobals { @@ -621,6 +625,9 @@ final class Emitter(config: Emitter.Config) { private[this] var _lastExternalDependencies: Set[String] = Set.empty private[this] var _lastInternalDependencies: Set[ModuleID] = Set.empty + private[this] var _topLevelExportsCache: WithGlobals[List[js.Tree]] = WithGlobals.nil + private[this] var _lastTopLevelExports: List[LinkedTopLevelExport] = Nil + private[this] var _initializersCache: WithGlobals[List[js.Tree]] = WithGlobals.nil private[this] var _lastInitializers: List[ModuleInitializer.Initializer] = Nil @@ -634,6 +641,9 @@ final class Emitter(config: Emitter.Config) { _lastExternalDependencies = Set.empty _lastInternalDependencies = Set.empty + _topLevelExportsCache = WithGlobals.nil + _lastTopLevelExports = Nil + _initializersCache = WithGlobals.nil _lastInitializers = Nil } @@ -651,6 +661,45 @@ final class Emitter(config: Emitter.Config) { _importsCache } + def getOrComputeTopLevelExports(topLevelExports: List[LinkedTopLevelExport])( + compute: => WithGlobals[List[js.Tree]]): WithGlobals[List[js.Tree]] = { + + _cacheUsed = true + + if (!sameTopLevelExports(topLevelExports, _lastTopLevelExports)) { + _topLevelExportsCache = compute + _lastTopLevelExports = topLevelExports + } + _topLevelExportsCache + } + + private def sameTopLevelExports(tles1: List[LinkedTopLevelExport], tles2: List[LinkedTopLevelExport]): Boolean = { + import org.scalajs.ir.Trees._ + + /* Because of how/when we use this method, we already know that all the + * `tles1` and `tles2` have the same `moduleID` (namely the ID of the + * module represented by this `ModuleCache`). Therefore, we do not + * compare that field. + */ + + tles1.corresponds(tles2) { (tle1, tle2) => + tle1.tree.pos == tle2.tree.pos && tle1.owningClass == tle2.owningClass && { + (tle1.tree, tle2.tree) match { + case (TopLevelJSClassExportDef(_, exportName1), TopLevelJSClassExportDef(_, exportName2)) => + exportName1 == exportName2 + case (TopLevelModuleExportDef(_, exportName1), TopLevelModuleExportDef(_, exportName2)) => + exportName1 == exportName2 + case (TopLevelMethodExportDef(_, methodDef1), TopLevelMethodExportDef(_, methodDef2)) => + methodDef1.version.sameVersion(methodDef2.version) + case (TopLevelFieldExportDef(_, exportName1, field1), TopLevelFieldExportDef(_, exportName2, field2)) => + exportName1 == exportName2 && field1.name == field2.name && field1.pos == field2.pos + case _ => + false + } + } + } + } + def getOrComputeInitializers(initializers: List[ModuleInitializer.Initializer])( compute: => WithGlobals[List[js.Tree]]): WithGlobals[List[js.Tree]] = { From 7c640c7de7c4be40c7dc10d2bc05a81e45de3077 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 9 Mar 2023 05:38:02 +0000 Subject: [PATCH 25/45] Add update+accumulate methods to `AtomicReference` --- .../concurrent/atomic/AtomicReference.scala | 27 +++++++++++++++++++ .../util/concurrent/atomic/AtomicTest.scala | 16 +++++++++++ 2 files changed, 43 insertions(+) diff --git a/javalib/src/main/scala/java/util/concurrent/atomic/AtomicReference.scala b/javalib/src/main/scala/java/util/concurrent/atomic/AtomicReference.scala index 9a025f4cd5..b3cb37ddfe 100644 --- a/javalib/src/main/scala/java/util/concurrent/atomic/AtomicReference.scala +++ b/javalib/src/main/scala/java/util/concurrent/atomic/AtomicReference.scala @@ -12,6 +12,9 @@ package java.util.concurrent.atomic +import java.util.function.BinaryOperator +import java.util.function.UnaryOperator + class AtomicReference[T <: AnyRef]( private[this] var value: T) extends Serializable { @@ -41,6 +44,30 @@ class AtomicReference[T <: AnyRef]( old } + final def getAndUpdate(updateFunction: UnaryOperator[T]): T = { + val old = value + value = updateFunction.apply(old) + old + } + + final def updateAndGet(updateFunction: UnaryOperator[T]): T = { + val old = value + value = updateFunction.apply(old) + value + } + + final def getAndAccumulate(x: T, accumulatorFunction: BinaryOperator[T]): T = { + val old = value + value = accumulatorFunction.apply(old, x) + old + } + + final def accumulateAndGet(x: T, accumulatorFunction: BinaryOperator[T]): T = { + val old = value + value = accumulatorFunction.apply(old, x) + value + } + override def toString(): String = String.valueOf(value) } diff --git a/test-suite/shared/src/test/scala/org/scalajs/testsuite/javalib/util/concurrent/atomic/AtomicTest.scala b/test-suite/shared/src/test/scala/org/scalajs/testsuite/javalib/util/concurrent/atomic/AtomicTest.scala index 8bd3a2d8a6..9ba37185d1 100644 --- a/test-suite/shared/src/test/scala/org/scalajs/testsuite/javalib/util/concurrent/atomic/AtomicTest.scala +++ b/test-suite/shared/src/test/scala/org/scalajs/testsuite/javalib/util/concurrent/atomic/AtomicTest.scala @@ -107,6 +107,22 @@ class AtomicTest { assertFalse(atomic.compareAndSet(thing1bis, thing2)) assertSame(thing1, atomic.getAndSet(thing2)) assertSame(thing2, atomic.get()) + + atomic.set(thing1) + assertSame(thing1, atomic.getAndUpdate(f => Foo(f.i * 2))) + assertEquals(thing2, atomic.get()) + + atomic.set(thing1) + assertEquals(thing2, atomic.updateAndGet(f => Foo(f.i * 2))) + assertEquals(thing2, atomic.get()) + + atomic.set(thing1) + assertSame(thing1, atomic.getAndAccumulate(thing2, (x, y) => Foo(x.i - y.i))) + assertEquals(Foo(-5), atomic.get()) + + atomic.set(thing1) + assertEquals(Foo(-5), atomic.accumulateAndGet(thing2, (x, y) => Foo(x.i - y.i))) + assertEquals(Foo(-5), atomic.get()) } @Test def atomicReferenceArrayTest(): Unit = { From 5a1f8a621a37a553e7f08278f7ce1624ba8d461f Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 9 Mar 2023 05:51:36 +0000 Subject: [PATCH 26/45] Add update+accumulate methods to `AtomicInteger` --- .../concurrent/atomic/AtomicInteger.scala | 27 +++++++++++++++++++ .../util/concurrent/atomic/AtomicTest.scala | 16 +++++++++++ 2 files changed, 43 insertions(+) diff --git a/javalib/src/main/scala/java/util/concurrent/atomic/AtomicInteger.scala b/javalib/src/main/scala/java/util/concurrent/atomic/AtomicInteger.scala index 0622ba8e10..9aaca49c6b 100644 --- a/javalib/src/main/scala/java/util/concurrent/atomic/AtomicInteger.scala +++ b/javalib/src/main/scala/java/util/concurrent/atomic/AtomicInteger.scala @@ -12,6 +12,9 @@ package java.util.concurrent.atomic +import java.util.function.IntBinaryOperator +import java.util.function.IntUnaryOperator + class AtomicInteger(private[this] var value: Int) extends Number with Serializable { @@ -65,6 +68,30 @@ class AtomicInteger(private[this] var value: Int) newValue } + final def getAndUpdate(updateFunction: IntUnaryOperator): Int = { + val old = value + value = updateFunction.applyAsInt(old) + old + } + + final def updateAndGet(updateFunction: IntUnaryOperator): Int = { + val old = value + value = updateFunction.applyAsInt(old) + value + } + + final def getAndAccumulate(x: Int, accumulatorFunction: IntBinaryOperator): Int = { + val old = value + value = accumulatorFunction.applyAsInt(old, x) + old + } + + final def accumulateAndGet(x: Int, accumulatorFunction: IntBinaryOperator): Int = { + val old = value + value = accumulatorFunction.applyAsInt(old, x) + value + } + override def toString(): String = value.toString() diff --git a/test-suite/shared/src/test/scala/org/scalajs/testsuite/javalib/util/concurrent/atomic/AtomicTest.scala b/test-suite/shared/src/test/scala/org/scalajs/testsuite/javalib/util/concurrent/atomic/AtomicTest.scala index 9ba37185d1..f87e09fbd0 100644 --- a/test-suite/shared/src/test/scala/org/scalajs/testsuite/javalib/util/concurrent/atomic/AtomicTest.scala +++ b/test-suite/shared/src/test/scala/org/scalajs/testsuite/javalib/util/concurrent/atomic/AtomicTest.scala @@ -69,6 +69,22 @@ class AtomicTest { assertEquals(10, atomic.get()) assertTrue(atomic.compareAndSet(10, 20)) assertEquals(20, atomic.get()) + + atomic.set(10) + assertEquals(10, atomic.getAndUpdate(_ * 2)) + assertEquals(20, atomic.get()) + + atomic.set(10) + assertEquals(20, atomic.updateAndGet(_ * 2)) + assertEquals(20, atomic.get()) + + atomic.set(10) + assertEquals(10, atomic.getAndAccumulate(20, (x, y) => x - y)) + assertEquals(-10, atomic.get()) + + atomic.set(10) + assertEquals(-10, atomic.accumulateAndGet(20, (x, y) => x - y)) + assertEquals(-10, atomic.get()) } @Test def atomicBooleanTest(): Unit = { From 35864bd9fd2f5ea099132968ffbac2571d026341 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 9 Mar 2023 05:55:40 +0000 Subject: [PATCH 27/45] Add update+accumulate methods to `AtomicLong` --- .../util/concurrent/atomic/AtomicLong.scala | 27 +++++++++++++++++++ .../util/concurrent/atomic/AtomicTest.scala | 16 +++++++++++ 2 files changed, 43 insertions(+) diff --git a/javalib/src/main/scala/java/util/concurrent/atomic/AtomicLong.scala b/javalib/src/main/scala/java/util/concurrent/atomic/AtomicLong.scala index 01c4c0b98b..d58dcb4d26 100644 --- a/javalib/src/main/scala/java/util/concurrent/atomic/AtomicLong.scala +++ b/javalib/src/main/scala/java/util/concurrent/atomic/AtomicLong.scala @@ -12,6 +12,9 @@ package java.util.concurrent.atomic +import java.util.function.LongBinaryOperator +import java.util.function.LongUnaryOperator + class AtomicLong(private[this] var value: Long) extends Number with Serializable { def this() = this(0L) @@ -63,6 +66,30 @@ class AtomicLong(private[this] var value: Long) extends Number with Serializable newValue } + final def getAndUpdate(updateFunction: LongUnaryOperator): Long = { + val old = value + value = updateFunction.applyAsLong(old) + old + } + + final def updateAndGet(updateFunction: LongUnaryOperator): Long = { + val old = value + value = updateFunction.applyAsLong(old) + value + } + + final def getAndAccumulate(x: Long, accumulatorFunction: LongBinaryOperator): Long = { + val old = value + value = accumulatorFunction.applyAsLong(old, x) + old + } + + final def accumulateAndGet(x: Long, accumulatorFunction: LongBinaryOperator): Long = { + val old = value + value = accumulatorFunction.applyAsLong(old, x) + value + } + override def toString(): String = value.toString() diff --git a/test-suite/shared/src/test/scala/org/scalajs/testsuite/javalib/util/concurrent/atomic/AtomicTest.scala b/test-suite/shared/src/test/scala/org/scalajs/testsuite/javalib/util/concurrent/atomic/AtomicTest.scala index f87e09fbd0..596fbe9cef 100644 --- a/test-suite/shared/src/test/scala/org/scalajs/testsuite/javalib/util/concurrent/atomic/AtomicTest.scala +++ b/test-suite/shared/src/test/scala/org/scalajs/testsuite/javalib/util/concurrent/atomic/AtomicTest.scala @@ -42,6 +42,22 @@ class AtomicTest { assertEquals(10L, atomic.get()) assertTrue(atomic.compareAndSet(10, 20)) assertEquals(20L, atomic.get()) + + atomic.set(10L) + assertEquals(10L, atomic.getAndUpdate(_ * 2L)) + assertEquals(20L, atomic.get()) + + atomic.set(10L) + assertEquals(20L, atomic.updateAndGet(_ * 2L)) + assertEquals(20L, atomic.get()) + + atomic.set(10L) + assertEquals(10L, atomic.getAndAccumulate(20L, (x, y) => x - y)) + assertEquals(-10L, atomic.get()) + + atomic.set(10L) + assertEquals(-10L, atomic.accumulateAndGet(20L, (x, y) => x - y)) + assertEquals(-10L, atomic.get()) } @Test def atomicIntegerTest(): Unit = { From cebb6466691ce822cb25ba1940751e11f3fb42d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Thu, 9 Mar 2023 20:12:10 +0100 Subject: [PATCH 28/45] In test-html.js, use express instead of node-static. We used node-static, but it has not been updated in 4 years, and is starting to accumulate security advisories. express is a bit bigger in scope than we need, but it is much better maintained. --- package-lock.json | 2394 ++++++++++++++++++++++++++++++++++++++++-- package.json | 6 +- scripts/test-html.js | 9 +- 3 files changed, 2312 insertions(+), 97 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3ea25312c1..2fc99952f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,1783 @@ { + "name": "scalajs", + "lockfileVersion": 2, "requires": true, - "lockfileVersion": 1, + "packages": { + "": { + "devDependencies": { + "express": "4.18.2", + "jsdom": "16.5.0", + "jszip": "3.8.0", + "source-map-support": "0.5.19" + } + }, + "node_modules/abab": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz", + "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==", + "dev": true + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dev": true, + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "8.8.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", + "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-globals": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz", + "integrity": "sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg==", + "dev": true, + "dependencies": { + "acorn": "^7.1.1", + "acorn-walk": "^7.1.1" + } + }, + "node_modules/acorn-globals/node_modules/acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz", + "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "dev": true + }, + "node_modules/asn1": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "dev": true, + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, + "node_modules/assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==", + "dev": true, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true + }, + "node_modules/aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/aws4": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz", + "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==", + "dev": true + }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "dev": true, + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, + "node_modules/body-parser": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", + "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", + "dev": true, + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.1", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dev": true, + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/browser-process-hrtime": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", + "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==", + "dev": true + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==", + "dev": true + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dev": true, + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-disposition/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "dev": true + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true + }, + "node_modules/cssom": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", + "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==", + "dev": true + }, + "node_modules/cssstyle": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz", + "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==", + "dev": true, + "dependencies": { + "cssom": "~0.3.6" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cssstyle/node_modules/cssom": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", + "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==", + "dev": true + }, + "node_modules/dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==", + "dev": true, + "dependencies": { + "assert-plus": "^1.0.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/data-urls": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", + "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==", + "dev": true, + "dependencies": { + "abab": "^2.0.3", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^8.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/decimal.js": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz", + "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==", + "dev": true + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "dev": true, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/domexception": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", + "integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==", + "dev": true, + "dependencies": { + "webidl-conversions": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/domexception/node_modules/webidl-conversions": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", + "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==", + "dev": true, + "dependencies": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "dev": true + }, + "node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "dev": true + }, + "node_modules/escodegen": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.0.0.tgz", + "integrity": "sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw==", + "dev": true, + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.18.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", + "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "dev": true, + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.1", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.5.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.2.0", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/express/node_modules/qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dev": true, + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/express/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true + }, + "node_modules/extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==", + "dev": true, + "engines": [ + "node >=0.6.0" + ] + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true + }, + "node_modules/finalhandler": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "dev": true, + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 0.12" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "node_modules/get-intrinsic": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz", + "integrity": "sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==", + "dev": true, + "dependencies": { + "assert-plus": "^1.0.0" + } + }, + "node_modules/har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/har-validator": { + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", + "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", + "deprecated": "this library is no longer supported", + "dev": true, + "dependencies": { + "ajv": "^6.12.3", + "har-schema": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/html-encoding-sniffer": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", + "integrity": "sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==", + "dev": true, + "dependencies": { + "whatwg-encoding": "^1.0.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dev": true, + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==", + "dev": true, + "dependencies": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + }, + "engines": { + "node": ">=0.8", + "npm": ">=1.3.7" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "dev": true + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "dev": true, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "dev": true + }, + "node_modules/is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==", + "dev": true + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, + "node_modules/isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==", + "dev": true + }, + "node_modules/jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==", + "dev": true + }, + "node_modules/jsdom": { + "version": "16.5.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.5.0.tgz", + "integrity": "sha512-QxZH0nmDTnTTVI0YDm4RUlaUPl5dcyn62G5TMDNfMmTW+J1u1v9gCR8WR+WZ6UghAa7nKJjDOFaI00eMMWvJFQ==", + "dev": true, + "dependencies": { + "abab": "^2.0.5", + "acorn": "^8.0.5", + "acorn-globals": "^6.0.0", + "cssom": "^0.4.4", + "cssstyle": "^2.3.0", + "data-urls": "^2.0.0", + "decimal.js": "^10.2.1", + "domexception": "^2.0.1", + "escodegen": "^2.0.0", + "html-encoding-sniffer": "^2.0.1", + "is-potential-custom-element-name": "^1.0.0", + "nwsapi": "^2.2.0", + "parse5": "6.0.1", + "request": "^2.88.2", + "request-promise-native": "^1.0.9", + "saxes": "^5.0.1", + "symbol-tree": "^3.2.4", + "tough-cookie": "^4.0.0", + "w3c-hr-time": "^1.0.2", + "w3c-xmlserializer": "^2.0.0", + "webidl-conversions": "^6.1.0", + "whatwg-encoding": "^1.0.5", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^8.0.0", + "ws": "^7.4.4", + "xml-name-validator": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "canvas": "^2.5.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "dev": true + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "dev": true + }, + "node_modules/jsprim": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.2.tgz", + "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==", + "dev": true, + "dependencies": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.4.0", + "verror": "1.10.0" + }, + "engines": { + "node": ">=0.6.0" + } + }, + "node_modules/jszip": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.8.0.tgz", + "integrity": "sha512-cnpQrXvFSLdsR9KR5/x7zdf6c3m8IhZfZzSblFEHSqBaVwD2nvJ4CuCKLyvKvwBgZm08CgfSoiTBQLm5WW9hGw==", + "dev": true, + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "set-immediate-shim": "~1.0.1" + } + }, + "node_modules/levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", + "dev": true, + "dependencies": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dev": true, + "dependencies": { + "immediate": "~3.0.5" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==", + "dev": true + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/nwsapi": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.2.tgz", + "integrity": "sha512-90yv+6538zuvUMnN+zCr8LuV6bPFdq50304114vJYJ8RDyK8D5O9Phpbd6SZWgI7PwzmmfN1upeOJlvybDSgCw==", + "dev": true + }, + "node_modules/oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/object-inspect": { + "version": "1.12.3", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", + "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dev": true, + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/optionator": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "dev": true, + "dependencies": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.6", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "word-wrap": "~1.2.3" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "dev": true + }, + "node_modules/parse5": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", + "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", + "dev": true + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==", + "dev": true + }, + "node_modules/performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==", + "dev": true + }, + "node_modules/prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dev": true, + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/psl": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", + "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==", + "dev": true + }, + "node_modules/punycode": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/qs": { + "version": "6.5.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.3.tgz", + "integrity": "sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==", + "dev": true, + "engines": { + "node": ">=0.6" + } + }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "dev": true + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "dev": true, + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/request": { + "version": "2.88.2", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "deprecated": "request has been deprecated, see https://github.com/request/request/issues/3142", + "dev": true, + "dependencies": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.3", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.5.0", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/request-promise-core": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.4.tgz", + "integrity": "sha512-TTbAfBBRdWD7aNNOoVOBH4pN/KigV6LyapYNNlAPA8JwbovRti1E88m3sYAwsLi5ryhPKsE9APwnjFTgdUjTpw==", + "dev": true, + "dependencies": { + "lodash": "^4.17.19" + }, + "engines": { + "node": ">=0.10.0" + }, + "peerDependencies": { + "request": "^2.34" + } + }, + "node_modules/request-promise-native": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.9.tgz", + "integrity": "sha512-wcW+sIUiWnKgNY0dqCpOZkUbF/I+YPi+f09JZIDa39Ec+q82CpSYniDp+ISgTTbKmnpJWASeJBPZmoxH84wt3g==", + "deprecated": "request-promise-native has been deprecated because it extends the now deprecated request package, see https://github.com/request/request/issues/3142", + "dev": true, + "dependencies": { + "request-promise-core": "1.1.4", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" + }, + "engines": { + "node": ">=0.12.0" + }, + "peerDependencies": { + "request": "^2.34" + } + }, + "node_modules/request-promise-native/node_modules/tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dev": true, + "dependencies": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/request/node_modules/tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dev": true, + "dependencies": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, + "node_modules/saxes": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", + "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", + "dev": true, + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "dev": true, + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/serve-static": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "dev": true, + "dependencies": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.18.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/set-immediate-shim": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", + "integrity": "sha512-Li5AOqrZWCVA2n5kryzEmqai6bKSIvpz5oUJHPVj6+dsbD3X1ixtsY5tEnsaNpH3pFAHmG8eIHUrtEtohrg+UQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "dev": true + }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "dev": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/sshpk": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", + "integrity": "sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ==", + "dev": true, + "dependencies": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + }, + "bin": { + "sshpk-conv": "bin/sshpk-conv", + "sshpk-sign": "bin/sshpk-sign", + "sshpk-verify": "bin/sshpk-verify" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/stealthy-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", + "integrity": "sha512-ZnWpYnYugiOVEY5GkcuJK1io5V8QmNYChG62gSit9pQVGErXtrKuPC55ITaVSukmMta5qpMU7vqLt2Lnni4f/g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "dev": true + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "dev": true, + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tough-cookie": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.2.tgz", + "integrity": "sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ==", + "dev": true, + "dependencies": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tr46": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz", + "integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==", + "dev": true, + "dependencies": { + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dev": true, + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", + "dev": true + }, + "node_modules/type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", + "dev": true, + "dependencies": { + "prelude-ls": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dev": true, + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", + "dev": true, + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dev": true, + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "dev": true, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "dev": true, + "bin": { + "uuid": "bin/uuid" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==", + "dev": true, + "engines": [ + "node >=0.6.0" + ], + "dependencies": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "node_modules/verror/node_modules/core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==", + "dev": true + }, + "node_modules/w3c-hr-time": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz", + "integrity": "sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ==", + "deprecated": "Use your platform's native performance.now() and performance.timeOrigin.", + "dev": true, + "dependencies": { + "browser-process-hrtime": "^1.0.0" + } + }, + "node_modules/w3c-xmlserializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz", + "integrity": "sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA==", + "dev": true, + "dependencies": { + "xml-name-validator": "^3.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/webidl-conversions": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", + "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==", + "dev": true, + "engines": { + "node": ">=10.4" + } + }, + "node_modules/whatwg-encoding": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", + "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "dev": true, + "dependencies": { + "iconv-lite": "0.4.24" + } + }, + "node_modules/whatwg-mimetype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", + "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==", + "dev": true + }, + "node_modules/whatwg-url": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.7.0.tgz", + "integrity": "sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg==", + "dev": true, + "dependencies": { + "lodash": "^4.7.0", + "tr46": "^2.1.0", + "webidl-conversions": "^6.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "dev": true, + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xml-name-validator": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", + "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==", + "dev": true + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "dev": true + } + }, "dependencies": { "abab": { "version": "2.0.6", @@ -8,10 +1785,20 @@ "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==", "dev": true }, + "accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dev": true, + "requires": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + } + }, "acorn": { - "version": "8.7.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.7.1.tgz", - "integrity": "sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A==", + "version": "8.8.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", + "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==", "dev": true }, "acorn-globals": { @@ -50,6 +1837,12 @@ "uri-js": "^4.2.2" } }, + "array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "dev": true + }, "asn1": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", @@ -78,9 +1871,9 @@ "dev": true }, "aws4": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", - "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==", + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz", + "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==", "dev": true }, "bcrypt-pbkdf": { @@ -92,6 +1885,37 @@ "tweetnacl": "^0.14.3" } }, + "body-parser": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", + "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", + "dev": true, + "requires": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.1", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "dependencies": { + "qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dev": true, + "requires": { + "side-channel": "^1.0.4" + } + } + } + }, "browser-process-hrtime": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", @@ -99,23 +1923,33 @@ "dev": true }, "buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true + }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==", "dev": true }, - "colors": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", - "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==", - "dev": true - }, "combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -125,10 +1959,45 @@ "delayed-stream": "~1.0.0" } }, + "content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dev": true, + "requires": { + "safe-buffer": "5.2.1" + }, + "dependencies": { + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + } + } + }, + "content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "dev": true + }, + "cookie": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "dev": true + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "dev": true + }, "core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", "dev": true }, "cssom": { @@ -174,10 +2043,19 @@ "whatwg-url": "^8.0.0" } }, + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, "decimal.js": { - "version": "10.3.1", - "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz", - "integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ==", + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz", + "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==", "dev": true }, "deep-is": { @@ -192,6 +2070,18 @@ "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", "dev": true }, + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true + }, + "destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "dev": true + }, "domexception": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", @@ -219,6 +2109,24 @@ "safer-buffer": "^2.1.0" } }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "dev": true + }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "dev": true + }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "dev": true + }, "escodegen": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.0.0.tgz", @@ -250,6 +2158,68 @@ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "dev": true + }, + "express": { + "version": "4.18.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", + "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "dev": true, + "requires": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.1", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.5.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.2.0", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "dependencies": { + "qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dev": true, + "requires": { + "side-channel": "^1.0.4" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + } + } + }, "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -280,6 +2250,21 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "finalhandler": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "dev": true, + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + } + }, "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", @@ -297,6 +2282,35 @@ "mime-types": "^2.1.12" } }, + "forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "dev": true + }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "dev": true + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "get-intrinsic": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz", + "integrity": "sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + } + }, "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -322,6 +2336,21 @@ "har-schema": "^2.0.0" } }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true + }, "html-encoding-sniffer": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", @@ -331,6 +2360,19 @@ "whatwg-encoding": "^1.0.5" } }, + "http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dev": true, + "requires": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + } + }, "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -363,6 +2405,12 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true }, + "ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "dev": true + }, "is-potential-custom-element-name": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", @@ -494,6 +2542,24 @@ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dev": true }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "dev": true + }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==", + "dev": true + }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "dev": true + }, "mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", @@ -515,27 +2581,22 @@ "mime-db": "1.52.0" } }, - "minimist": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", - "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=", + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "dev": true }, - "node-static": { - "version": "0.7.11", - "resolved": "https://registry.npmjs.org/node-static/-/node-static-0.7.11.tgz", - "integrity": "sha512-zfWC/gICcqb74D9ndyvxZWaI1jzcoHmf4UTHWQchBNuNMxdBLJMDiUgZ1tjGLEIe/BMhj2DxKD8HOuc2062pDQ==", - "dev": true, - "requires": { - "colors": ">=0.6.0", - "mime": "^1.2.9", - "optimist": ">=0.3.4" - } + "negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "dev": true }, "nwsapi": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.1.tgz", - "integrity": "sha512-JYOWTeFoS0Z93587vRJgASD5Ut11fYl5NyihP3KrYBvMe1FRRs6RN7m20SA/16GM4P6hTnZjT+UmDOt38UeXNg==", + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.2.tgz", + "integrity": "sha512-90yv+6538zuvUMnN+zCr8LuV6bPFdq50304114vJYJ8RDyK8D5O9Phpbd6SZWgI7PwzmmfN1upeOJlvybDSgCw==", "dev": true }, "oauth-sign": { @@ -544,14 +2605,19 @@ "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", "dev": true }, - "optimist": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", - "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", + "object-inspect": { + "version": "1.12.3", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", + "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", + "dev": true + }, + "on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", "dev": true, "requires": { - "minimist": "~0.0.1", - "wordwrap": "~0.0.2" + "ee-first": "1.1.1" } }, "optionator": { @@ -580,6 +2646,18 @@ "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", "dev": true }, + "parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "dev": true + }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==", + "dev": true + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -598,16 +2676,26 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "dev": true }, + "proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dev": true, + "requires": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + } + }, "psl": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", - "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", + "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==", "dev": true }, "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", "dev": true }, "qs": { @@ -616,10 +2704,34 @@ "integrity": "sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==", "dev": true }, + "querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "dev": true + }, + "range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true + }, + "raw-body": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "dev": true, + "requires": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, "readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", "dev": true, "requires": { "core-util-is": "~1.0.0", @@ -629,14 +2741,6 @@ "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" - }, - "dependencies": { - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true - } } }, "request": { @@ -711,10 +2815,16 @@ } } }, + "requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true + }, "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "dev": true }, "safer-buffer": { @@ -732,12 +2842,70 @@ "xmlchars": "^2.2.0" } }, + "send": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "dev": true, + "requires": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "dependencies": { + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + } + } + }, + "serve-static": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "dev": true, + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.18.0" + } + }, "set-immediate-shim": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", "integrity": "sha512-Li5AOqrZWCVA2n5kryzEmqai6bKSIvpz5oUJHPVj6+dsbD3X1ixtsY5tEnsaNpH3pFAHmG8eIHUrtEtohrg+UQ==", "dev": true }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "dev": true + }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dev": true, + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -771,6 +2939,12 @@ "tweetnacl": "~0.14.0" } }, + "statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "dev": true + }, "stealthy-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", @@ -784,14 +2958,6 @@ "dev": true, "requires": { "safe-buffer": "~5.1.0" - }, - "dependencies": { - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true - } } }, "symbol-tree": { @@ -800,15 +2966,22 @@ "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", "dev": true }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "dev": true + }, "tough-cookie": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz", - "integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==", + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.2.tgz", + "integrity": "sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ==", "dev": true, "requires": { "psl": "^1.1.33", "punycode": "^2.1.1", - "universalify": "^0.1.2" + "universalify": "^0.2.0", + "url-parse": "^1.5.3" } }, "tr46": { @@ -844,10 +3017,26 @@ "prelude-ls": "~1.1.2" } }, + "type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dev": true, + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } + }, "universalify": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", - "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", + "dev": true + }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", "dev": true }, "uri-js": { @@ -859,18 +3048,40 @@ "punycode": "^2.1.0" } }, + "url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dev": true, + "requires": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "dev": true }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "dev": true + }, "uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", "dev": true }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "dev": true + }, "verror": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", @@ -880,6 +3091,14 @@ "assert-plus": "^1.0.0", "core-util-is": "1.0.2", "extsprintf": "^1.2.0" + }, + "dependencies": { + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==", + "dev": true + } } }, "w3c-hr-time": { @@ -938,17 +3157,12 @@ "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", "dev": true }, - "wordwrap": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", - "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", - "dev": true - }, "ws": { - "version": "7.5.8", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.8.tgz", - "integrity": "sha512-ri1Id1WinAX5Jqn9HejiGb8crfRio0Qgu8+MtL36rlTA6RLsMdWt1Az/19A2Qij6uSHUMphEFaTKa4WG+UNHNw==", - "dev": true + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "dev": true, + "requires": {} }, "xml-name-validator": { "version": "3.0.0", diff --git a/package.json b/package.json index 3eb761adef..f03449f559 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,9 @@ { "private": true, "devDependencies": { - "source-map-support": "0.5.19", - "jszip": "3.8.0", + "express": "4.18.2", "jsdom": "16.5.0", - "node-static": "0.7.11" + "jszip": "3.8.0", + "source-map-support": "0.5.19" } } diff --git a/scripts/test-html.js b/scripts/test-html.js index ba772e3900..4033d32a1e 100644 --- a/scripts/test-html.js +++ b/scripts/test-html.js @@ -1,7 +1,7 @@ +const express = require("express"); +const http = require("http"); const process = require("process"); const { JSDOM } = require("jsdom"); -const http = require("http"); -const static = require('node-static'); const servingDirectory = process.argv[2]; const requestPath = process.argv[3]; @@ -60,8 +60,9 @@ function waitComplete(dom) { } function serveDirectory(dir) { - const fileServer = new static.Server(dir); - const server = http.createServer((req, res) => fileServer.serve(req, res)); + const app = express(); + app.use(express.static(dir)); + const server = http.createServer(app); return new Promise((res, rej) => { server.listen(() => res(server)); From f1b7a1cb36da7a65c5d0a09e5cb0682ef57a31dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Fri, 3 Mar 2023 14:46:06 +0100 Subject: [PATCH 29/45] Source map writer: Move the `webURI` construction to `doComplete`. This is equivalent, but lets us analyze the performance of `doWriteSegment` in a more fine-grained way. --- .../linker/backend/javascript/SourceMapWriter.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala index ff779a802f..5ee8667e70 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala @@ -207,7 +207,7 @@ final class SourceMapWriter(out: ByteArrayWriter, jsFileName: String, import SourceMapWriter._ - private val sources = new ListBuffer[String] + private val sources = new ListBuffer[SourceFile] private val _srcToIndex = new HashMap[SourceFile, Int] private val names = new ListBuffer[String] @@ -230,7 +230,7 @@ final class SourceMapWriter(out: ByteArrayWriter, jsFileName: String, } else { val index = sources.size _srcToIndex.put(source, index) - sources += SourceFileUtil.webURI(relativizeBaseURI, source) + sources += source index } } @@ -315,10 +315,11 @@ final class SourceMapWriter(out: ByteArrayWriter, jsFileName: String, } protected def doComplete(): Unit = { + val relativizeBaseURI = this.relativizeBaseURI // local copy var restSources = sources.result() out.writeASCIIString("\",\n\"sources\": [") while (restSources.nonEmpty) { - writeJSONString(restSources.head) + writeJSONString(SourceFileUtil.webURI(relativizeBaseURI, restSources.head)) restSources = restSources.tail if (restSources.nonEmpty) out.writeASCIIString(", ") From 49000d4494807449f81ff665b3e8024b1904f227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Fri, 3 Mar 2023 16:15:05 +0100 Subject: [PATCH 30/45] SourceMapWriter: use an `Array[Byte]` for the `Base64Map`. Using `String.charAt` is now a useless indirection, although it does not appear to make a real difference. --- .../linker/backend/javascript/SourceMapWriter.scala | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala index 5ee8667e70..99ffb9c9c1 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala @@ -26,10 +26,13 @@ import org.scalajs.ir.Position import org.scalajs.ir.Position._ object SourceMapWriter { - private val Base64Map = - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + - "abcdefghijklmnopqrstuvwxyz" + - "0123456789+/" + private val Base64Map: Array[Byte] = { + ( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + + "abcdefghijklmnopqrstuvwxyz" + + "0123456789+/" + ).toArray.map(_.toByte) + } // Some constants for writeBase64VLQ // Each base-64 digit covers 6 bits, but 1 is used for the continuation @@ -380,7 +383,7 @@ final class SourceMapWriter(out: ByteArrayWriter, jsFileName: String, value = value >>> VLQBaseShift if (value != 0) digit |= VLQContinuationBit - out.write(Base64Map.charAt(digit)) + out.write(Base64Map(digit)) } while (value != 0) } writeBase64VLQSlowPath(value) From 1ff3ba8eea56add4e2404c475d49b20b03ea2d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Fri, 3 Mar 2023 14:56:24 +0100 Subject: [PATCH 31/45] SourceMapWriter: Optimize the lookups in the name and source tables. We use `java.util.HashMap`s instead of Scala `HashMap`s. This avoids the overhead of Scala's `==`, and allows to combine `contains`+`apply` as a single `get` without incurring any boxing. This change brings about a 20% speedup to the generation of source maps in `BasicLinkerBackend`. --- .../backend/javascript/SourceMapWriter.scala | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala index 99ffb9c9c1..0082f1115c 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala @@ -18,7 +18,7 @@ import java.nio.ByteBuffer import java.nio.charset.StandardCharsets import java.{util => ju} -import scala.collection.mutable.{ ArrayBuffer, ListBuffer, HashMap, Stack, StringBuilder } +import scala.collection.mutable.{ArrayBuffer, ListBuffer} import org.scalajs.ir import org.scalajs.ir.OriginalName @@ -211,10 +211,10 @@ final class SourceMapWriter(out: ByteArrayWriter, jsFileName: String, import SourceMapWriter._ private val sources = new ListBuffer[SourceFile] - private val _srcToIndex = new HashMap[SourceFile, Int] + private val _srcToIndex = new ju.HashMap[SourceFile, Integer] private val names = new ListBuffer[String] - private val _nameToIndex = new HashMap[String, Int] + private val _nameToIndex = new ju.HashMap[String, Integer] private var lineCountInGenerated = 0 private var lastColumnInGenerated = 0 @@ -228,8 +228,9 @@ final class SourceMapWriter(out: ByteArrayWriter, jsFileName: String, writeHeader() private def sourceToIndex(source: SourceFile): Int = { - if (_srcToIndex.contains(source)) { - _srcToIndex(source) + val existing = _srcToIndex.get(source) + if (existing != null) { + existing.intValue() } else { val index = sources.size _srcToIndex.put(source, index) @@ -239,8 +240,9 @@ final class SourceMapWriter(out: ByteArrayWriter, jsFileName: String, } private def nameToIndex(name: String): Int = { - if (_nameToIndex.contains(name)) { - _nameToIndex(name) + val existing = _nameToIndex.get(name) + if (existing != null) { + existing.intValue() } else { val index = names.size _nameToIndex.put(name, index) From dad81443ff099efd5e5384695a9454caf1534574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Thu, 23 Feb 2023 14:15:00 +0100 Subject: [PATCH 32/45] Store Position.isEmpty as a field. In some cases, that method significantly showed up in the profiles of `SourceMapWriter.insertSegment`. We can afford to take a bit more time at the construction of `Position` instances, because they are cached. Even though this does not seem to result in a measurable difference in execution time, the less work `insertSegment` has to do, the better. --- ir/shared/src/main/scala/org/scalajs/ir/Position.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ir/shared/src/main/scala/org/scalajs/ir/Position.scala b/ir/shared/src/main/scala/org/scalajs/ir/Position.scala index c2b60fb598..3406627ee2 100644 --- a/ir/shared/src/main/scala/org/scalajs/ir/Position.scala +++ b/ir/shared/src/main/scala/org/scalajs/ir/Position.scala @@ -20,9 +20,7 @@ final case class Position( /** Zero-based column number. */ column: Int ) { - def show: String = s"$line:$column" - - def isEmpty: Boolean = { + private val _isEmpty: Boolean = { def isEmptySlowPath(): Boolean = { source.getScheme == null && source.getRawAuthority == null && source.getRawQuery == null && source.getRawFragment == null @@ -30,6 +28,10 @@ final case class Position( source.getRawPath == "" && isEmptySlowPath() } + def show: String = s"$line:$column" + + def isEmpty: Boolean = _isEmpty + def isDefined: Boolean = !isEmpty def orElse(that: => Position): Position = if (isDefined) this else that From b34c2bcfd7cebad59bbdf737d94a484e017a4638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Tue, 7 Mar 2023 20:25:57 +0100 Subject: [PATCH 33/45] Micro-optimize `writeBase64VLQ`. Based on profiles and measures, using more fast paths and branchless algorithms. We also remove the extra `return`. It used to be necessary to generate better bytecode, but since Scala 2.12.16 and 2.13.9, the backend generates the same good bytecode without the `return`. These optimizations bring somewhere between 5% and 10% speedup to source map generation in the incremental case. --- .../backend/javascript/SourceMapWriter.scala | 91 ++++++++++++++----- 1 file changed, 68 insertions(+), 23 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala index 0082f1115c..77c9ef9cbb 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala @@ -26,13 +26,8 @@ import org.scalajs.ir.Position import org.scalajs.ir.Position._ object SourceMapWriter { - private val Base64Map: Array[Byte] = { - ( - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + - "abcdefghijklmnopqrstuvwxyz" + - "0123456789+/" - ).toArray.map(_.toByte) - } + private val Base64UpperMap: Array[Byte] = + "ghijklmnopqrstuvwxyz0123456789+/".toArray.map(_.toByte) // Some constants for writeBase64VLQ // Each base-64 digit covers 6 bits, but 1 is used for the continuation @@ -343,13 +338,14 @@ final class SourceMapWriter(out: ByteArrayWriter, jsFileName: String, out.writeASCIIString("\n}\n") } - /** Write the Base 64 VLQ of an integer to the mappings - * Inspired by the implementation in Closure Compiler: - * http://code.google.com/p/closure-compiler/source/browse/src/com/google/debugging/sourcemap/Base64VLQ.java + /** Write the Base 64 VLQ of an integer to the mappings. + * + * !!! This method is surprisingly performance-sensitive. In an incremental + * run of the linker, it takes half of the time of the `BasicLinkerBackend` + * and systematically shows up on performance profiles. If you change it, + * profile it and measure performance of source map generation. */ private def writeBase64VLQ(value0: Int): Unit = { - // scalastyle:off return - /* The sign is encoded in the least significant bit, while the * absolute value is shifted one bit to the left. * So in theory the "definition" of `value` is: @@ -374,24 +370,73 @@ final class SourceMapWriter(out: ByteArrayWriter, jsFileName: String, val signExtended = value0 >> 31 val value = (((value0 ^ signExtended) - signExtended) << 1) | (signExtended & 1) - // Write as many base-64 digits as necessary to encode value - if (value < 26) { - return out.write('A' + value) + /* Now that we have a non-negative `value`, we encode it in base64 by + * blocks of 5 bits. Each base64 digit stores 6 bits, but the most + * significant one is used as a continuation bit (1 to continue, 0 to + * indicate the last block). The payload is stored in little endian, with + * the least significant blocks first. + * + * We could use a unique lookup table for the 64 base64 digits. However, + * since in every path we either always pick in the lower half (for the + * last byte) or the upper half (for continuation bytes), we use two + * distinct functions, and omit the implicit `| VLQContinuationBit` in the + * upper half. + * + * The upper half, in `continuationByte`, actually uses a lookup table. + * + * The lower half, in `lastByte`, uses a branchless, memory access-free + * algorithm. The logical way to write it would be + * if (v < 26) v + 'A' else (v - 26) + 'a' + * Because 'a' == 'A' + 32, this is equivalent to + * if (v < 26) v + 'A' else v - 26 + 'A' + 32 + * Factoring out v + 'A' and adding constants, we get + * v + 'A' + (if (v < 26) 0 else 6) + * We rewrite the condition as the following branchless algorithm: + * ((25 - v) >> 31) & 6 + * It is equivalent because: + * * (25 - v) is < 0 iff v >= 26 + * * i.e., its sign bit is 1 iff v >= 26 + * * (25 - v) >> 31 is all-1's if v >= 26, and all-0's if v < 26 + * * ((25 - v) >> 31) & 6 is 6 if v >= 26, and 0 if v < 26 + * This gives us the algorithm used in `lastByte`: + * v + 'A' + (((25 - v) >> 31) & 6) + * + * Compared to the lookup table, this seems to exhibit a 5-10% speedup for + * the source map generation. + */ + + // Precondition: 0 <= v < 32, i.e., (v & 31) == v + def continuationByte(v: Int): Byte = + Base64UpperMap(v) + + // Precondition: 0 <= v < 32, i.e., (v & 31) == v + def lastByte(v: Int): Int = + v + 'A' + (((25 - v) >> 31) & 6) + + // Write as many base-64 digits as necessary to encode `value` + if ((value & ~31) == 0) { + // fast path for value < 32 -- store as a single byte (about 7/8 of the time for the test suite) + out.write(lastByte(value)) + } else if ((value & ~1023) == 0) { + // fast path for 32 <= value < 1024 -- store as two bytes (about 1/8 of the time for the test suite) + out.write(continuationByte(value & VLQBaseMask)) + out.write(lastByte(value >>> 5)) } else { + // slow path for 1024 <= value -- store as 3 bytes or more (a negligible fraction of the time) def writeBase64VLQSlowPath(value0: Int): Unit = { var value = value0 - do { - var digit = value & VLQBaseMask + var digit = 0 + while ({ + digit = value & VLQBaseMask value = value >>> VLQBaseShift - if (value != 0) - digit |= VLQContinuationBit - out.write(Base64Map(digit)) - } while (value != 0) + value != 0 + }) { + out.write(continuationByte(digit)) + } + out.write(lastByte(digit)) } writeBase64VLQSlowPath(value) } - - // scalastyle:on return } private def writeBase64VLQ0(): Unit = From 68e68d09003a59ecac6051d5c36a177c9bad53f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Tue, 7 Mar 2023 20:46:16 +0100 Subject: [PATCH 34/45] Micro-optimize `doWriteSegment` with direct writes to the array. Previously, we had to repeatedly call `out.write` byte-by-byte, which required a capacity check for each byte. We now replace it by a single capacity check per `doWriteSegment`. We do that with a pair of unsafe methods on `ByteArrayWriter`, which give direct access to the underlying array. `unsafeStartDirectWrite` takes the maximum amount of bytes that will be written, ensures that the buffer is large enough, and returns it. `unsafeEndDirectWrite` closes the unsafe "portion" by setting the new size of relevant bytes in the array. This comes as a last resort to make `doWriteSegment` stand out less in profiles of the backend. This change brings about a 5-10% speedup to source map generation. --- .../backend/javascript/ByteArrayWriter.scala | 8 +++ .../backend/javascript/SourceMapWriter.scala | 64 +++++++++++++------ 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala index 83adbd32c6..e3cd37f6f2 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala @@ -166,6 +166,14 @@ private[backend] final class ByteArrayWriter extends OutputStream { offset - oldSize // number of bytes written in total } + def unsafeStartDirectWrite(maxBytes: Int): Array[Byte] = { + ensureCapacity(size + maxBytes) + buffer + } + + def unsafeEndDirectWrite(newSize: Int): Unit = + size = newSize + def toByteBuffer(): ByteBuffer = ByteBuffer.wrap(buffer, 0, size).asReadOnlyBuffer() diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala index 77c9ef9cbb..17b8380891 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/SourceMapWriter.scala @@ -269,17 +269,30 @@ final class SourceMapWriter(out: ByteArrayWriter, jsFileName: String, protected def doWriteSegment(columnInGenerated: Int, pos: Position, name: String): Unit = { // scalastyle:off return + /* This method is incredibly performance-sensitive, so we resort to + * "unsafe" direct access to the underlying array of `out`. + */ + val MaxSegmentLength = 1 + 5 * 7 // ',' + max 5 base64VLQ of max 7 bytes each + val buffer = out.unsafeStartDirectWrite(maxBytes = MaxSegmentLength) + var offset = out.currentSize + // Segments of a line are separated by ',' - if (firstSegmentOfLine) firstSegmentOfLine = false - else out.write(',') + if (firstSegmentOfLine) { + firstSegmentOfLine = false + } else { + buffer(offset) = ',' + offset += 1 + } // Generated column field - writeBase64VLQ(columnInGenerated-lastColumnInGenerated) + offset = writeBase64VLQ(buffer, offset, columnInGenerated-lastColumnInGenerated) lastColumnInGenerated = columnInGenerated // If the position is NoPosition, stop here - if (pos.isEmpty) + if (pos.isEmpty) { + out.unsafeEndDirectWrite(offset) return + } // Extract relevant properties of pendingPos val source = pos.source @@ -288,29 +301,32 @@ final class SourceMapWriter(out: ByteArrayWriter, jsFileName: String, // Source index field if (source eq lastSource) { // highly likely - writeBase64VLQ0() + buffer(offset) = 'A' // 0 in Base64VLQ + offset += 1 } else { val sourceIndex = sourceToIndex(source) - writeBase64VLQ(sourceIndex-lastSourceIndex) + offset = writeBase64VLQ(buffer, offset, sourceIndex-lastSourceIndex) lastSource = source lastSourceIndex = sourceIndex } // Line field - writeBase64VLQ(line - lastLine) + offset = writeBase64VLQ(buffer, offset, line - lastLine) lastLine = line // Column field - writeBase64VLQ(column - lastColumn) + offset = writeBase64VLQ(buffer, offset, column - lastColumn) lastColumn = column // Name field if (name != null) { val nameIndex = nameToIndex(name) - writeBase64VLQ(nameIndex-lastNameIndex) + offset = writeBase64VLQ(buffer, offset, nameIndex-lastNameIndex) lastNameIndex = nameIndex } + out.unsafeEndDirectWrite(offset) + // scalastyle:on return } @@ -344,8 +360,12 @@ final class SourceMapWriter(out: ByteArrayWriter, jsFileName: String, * run of the linker, it takes half of the time of the `BasicLinkerBackend` * and systematically shows up on performance profiles. If you change it, * profile it and measure performance of source map generation. + * + * @return + * the offset past the written bytes in the `buffer`, i.e., `offset + x` + * where `x` is the amount of bytes written */ - private def writeBase64VLQ(value0: Int): Unit = { + private def writeBase64VLQ(buffer: Array[Byte], offset: Int, value0: Int): Int = { /* The sign is encoded in the least significant bit, while the * absolute value is shifted one bit to the left. * So in theory the "definition" of `value` is: @@ -410,20 +430,23 @@ final class SourceMapWriter(out: ByteArrayWriter, jsFileName: String, Base64UpperMap(v) // Precondition: 0 <= v < 32, i.e., (v & 31) == v - def lastByte(v: Int): Int = - v + 'A' + (((25 - v) >> 31) & 6) + def lastByte(v: Int): Byte = + (v + 'A' + (((25 - v) >> 31) & 6)).toByte // Write as many base-64 digits as necessary to encode `value` if ((value & ~31) == 0) { // fast path for value < 32 -- store as a single byte (about 7/8 of the time for the test suite) - out.write(lastByte(value)) + buffer(offset) = lastByte(value) + offset + 1 } else if ((value & ~1023) == 0) { // fast path for 32 <= value < 1024 -- store as two bytes (about 1/8 of the time for the test suite) - out.write(continuationByte(value & VLQBaseMask)) - out.write(lastByte(value >>> 5)) + buffer(offset) = continuationByte(value & VLQBaseMask) + buffer(offset + 1) = lastByte(value >>> 5) + offset + 2 } else { // slow path for 1024 <= value -- store as 3 bytes or more (a negligible fraction of the time) - def writeBase64VLQSlowPath(value0: Int): Unit = { + def writeBase64VLQSlowPath(value0: Int): Int = { + var offset1 = offset var value = value0 var digit = 0 while ({ @@ -431,14 +454,13 @@ final class SourceMapWriter(out: ByteArrayWriter, jsFileName: String, value = value >>> VLQBaseShift value != 0 }) { - out.write(continuationByte(digit)) + buffer(offset1) = continuationByte(digit) + offset1 += 1 } - out.write(lastByte(digit)) + buffer(offset1) = lastByte(digit) + offset1 + 1 } writeBase64VLQSlowPath(value) } } - - private def writeBase64VLQ0(): Unit = - out.write('A') } From 14b8c237b9934de2a0231f1f15f5658c83ff64ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Fri, 3 Mar 2023 21:24:54 +0100 Subject: [PATCH 35/45] Backend: Avoid doing any I/O for modules that have not changed. Previously, we had to read the file to check whether the bytes would be the same. Now, we can tell that a module has not changed since the previous incremental run without opening the file, avoiding any useless I/O. We cache the list of `js.Tree`s emitted for each module, in addition to the map from those `js.Tree`s to their printed output. When re-emitting a module, we test whether the list of new JS trees is the same, and if it is, we entirely bypass the rewriting of modules. This has no real effect on single-module outputs. However, for multi-module outputs, the perfomance improvement is massive: about a 50x (!) speedup for the back-end when changing only one source file, resulting in two output modules being re-emitted. --- .../unstable/OutputDirectoryImpl.scala | 23 +++ .../scalajs/linker/PathOutputDirectory.scala | 8 + .../closure/ClosureLinkerBackend.scala | 18 +- .../linker/backend/BasicLinkerBackend.scala | 172 ++++++++++++------ .../scalajs/linker/backend/OutputWriter.scala | 67 +++---- .../backend/javascript/ByteArrayWriter.scala | 11 +- .../linker/BasicLinkerBackendTest.scala | 105 +++++++---- 7 files changed, 274 insertions(+), 130 deletions(-) diff --git a/linker-interface/shared/src/main/scala/org/scalajs/linker/interface/unstable/OutputDirectoryImpl.scala b/linker-interface/shared/src/main/scala/org/scalajs/linker/interface/unstable/OutputDirectoryImpl.scala index c8c98c24f9..3091391e4c 100644 --- a/linker-interface/shared/src/main/scala/org/scalajs/linker/interface/unstable/OutputDirectoryImpl.scala +++ b/linker-interface/shared/src/main/scala/org/scalajs/linker/interface/unstable/OutputDirectoryImpl.scala @@ -26,10 +26,33 @@ abstract class OutputDirectoryImpl extends OutputDirectory { * Writing should only result in a file write if the contents of the file * actually changed. Further, if the underlying filesystem allows it, the * file should be written atomically. + * + * Calling this method is equivalent to calling + * `writeFull(name, buf, skipContentCheck = false)`. */ def writeFull(name: String, buf: ByteBuffer)( implicit ec: ExecutionContext): Future[Unit] + /** Writes to the given file. + * + * - If `skipContentCheck` is `false`, writing should only result in a file + * write if the contents of the file actually changed. + * - If it is `true`, the implementation is encouraged not to check for the + * file contents, and always write it; however, this not mandatory for + * backward compatibility reasons. + * + * If the underlying filesystem allows it, the file should be written + * atomically. + * + * The default implementation of this method calls `writeFull` without the + * `skipContentCheck`, which is suboptimal. Therefore, it is encouraged to + * override it. + */ + def writeFull(name: String, buf: ByteBuffer, skipContentCheck: Boolean)( + implicit ec: ExecutionContext): Future[Unit] = { + writeFull(name, buf) + } + /** Fully read the given file into a new ByteBuffer. */ def readFull(name: String)( implicit ec: ExecutionContext): Future[ByteBuffer] diff --git a/linker/jvm/src/main/scala/org/scalajs/linker/PathOutputDirectory.scala b/linker/jvm/src/main/scala/org/scalajs/linker/PathOutputDirectory.scala index d77f94c72b..22ee601909 100644 --- a/linker/jvm/src/main/scala/org/scalajs/linker/PathOutputDirectory.scala +++ b/linker/jvm/src/main/scala/org/scalajs/linker/PathOutputDirectory.scala @@ -43,6 +43,14 @@ object PathOutputDirectory { } } + override def writeFull(name: String, buf: ByteBuffer, skipContentCheck: Boolean)( + implicit ec: ExecutionContext): Future[Unit] = { + if (skipContentCheck) + writeAtomic(name, buf) + else + writeFull(name, buf) + } + def readFull(name: String)(implicit ec: ExecutionContext): Future[ByteBuffer] = withChannel(getPath(name), StandardOpenOption.READ)(readFromChannel(_)) diff --git a/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala b/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala index 641973d527..003e873773 100644 --- a/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala +++ b/linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala @@ -14,7 +14,8 @@ package org.scalajs.linker.backend.closure import scala.concurrent._ -import java.io.Writer +import java.io.{ByteArrayOutputStream, Writer} +import java.nio.ByteBuffer import java.nio.charset.StandardCharsets import java.util.{Arrays, HashSet} @@ -31,7 +32,7 @@ import org.scalajs.linker.interface._ import org.scalajs.linker.interface.unstable.OutputPatternsImpl import org.scalajs.linker.backend._ import org.scalajs.linker.backend.emitter.Emitter -import org.scalajs.linker.backend.javascript.{ByteArrayWriter, Trees => js} +import org.scalajs.linker.backend.javascript.{Trees => js} import org.scalajs.linker.standard._ import org.scalajs.linker.standard.ModuleSet.ModuleID @@ -200,7 +201,7 @@ final class ClosureLinkerBackend(config: LinkerBackendImpl.Config) * We call `.get` in the write methods to fail if we get a called anyways. */ - val writer = new OutputWriter(output, config) { + val writer = new OutputWriter(output, config, skipContentCheck = false) { private def writeCode(writer: Writer): Unit = { val code = gccResult.get._1 writer.write(header) @@ -208,27 +209,32 @@ final class ClosureLinkerBackend(config: LinkerBackendImpl.Config) writer.write(footer) } - protected def writeModule(moduleID: ModuleID, jsFileWriter: ByteArrayWriter): Unit = { + protected def writeModuleWithoutSourceMap(moduleID: ModuleID, force: Boolean): Option[ByteBuffer] = { + val jsFileWriter = new ByteArrayOutputStream() val jsFileStrWriter = new java.io.OutputStreamWriter(jsFileWriter, StandardCharsets.UTF_8) writeCode(jsFileStrWriter) jsFileStrWriter.flush() + Some(ByteBuffer.wrap(jsFileWriter.toByteArray())) } - protected def writeModule(moduleID: ModuleID, jsFileWriter: ByteArrayWriter, - sourceMapWriter: ByteArrayWriter): Unit = { + protected def writeModuleWithSourceMap(moduleID: ModuleID, force: Boolean): Option[(ByteBuffer, ByteBuffer)] = { val jsFileURI = OutputPatternsImpl.jsFileURI(config.outputPatterns, moduleID.id) val sourceMapURI = OutputPatternsImpl.sourceMapURI(config.outputPatterns, moduleID.id) + val jsFileWriter = new ByteArrayOutputStream() val jsFileStrWriter = new java.io.OutputStreamWriter(jsFileWriter, StandardCharsets.UTF_8) writeCode(jsFileStrWriter) jsFileStrWriter.write("//# sourceMappingURL=" + sourceMapURI + "\n") jsFileStrWriter.flush() + val sourceMapWriter = new ByteArrayOutputStream() val sourceMapStrWriter = new java.io.OutputStreamWriter(sourceMapWriter, StandardCharsets.UTF_8) val sourceMap = gccResult.get._2 sourceMap.setWrapperPrefix(header) sourceMap.appendTo(sourceMapStrWriter, jsFileURI) sourceMapStrWriter.flush() + + Some((ByteBuffer.wrap(jsFileWriter.toByteArray()), ByteBuffer.wrap(sourceMapWriter.toByteArray()))) } } diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala index cdfe141c62..a1238e7433 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala @@ -14,6 +14,7 @@ package org.scalajs.linker.backend import scala.concurrent._ +import java.nio.ByteBuffer import java.nio.charset.StandardCharsets import org.scalajs.logging.Logger @@ -45,6 +46,8 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config) val symbolRequirements: SymbolRequirement = emitter.symbolRequirements + private var isFirstRun: Boolean = true + private val printedModuleSetCache = new PrintedModuleSetCache(config.sourceMap) override def injectedIRFiles: Seq[IRFile] = emitter.injectedIRFiles @@ -62,65 +65,82 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config) emitter.emit(moduleSet, logger) } - val writer = new OutputWriter(output, config) { - protected def writeModule(moduleID: ModuleID, jsFileWriter: ByteArrayWriter): Unit = { - val printedModuleCache = printedModuleSetCache.getModuleCache(moduleID) + val skipContentCheck = !isFirstRun + isFirstRun = false - jsFileWriter.sizeHint(sizeHintFor(printedModuleCache.getPreviousFinalJSFileSize())) + printedModuleSetCache.startRun(moduleSet) + val allChanged = + printedModuleSetCache.updateGlobal(emitterResult.header, emitterResult.footer) - jsFileWriter.write(emitterResult.header.getBytes(StandardCharsets.UTF_8)) - jsFileWriter.writeASCIIString("'use strict';\n") + val writer = new OutputWriter(output, config, skipContentCheck) { + protected def writeModuleWithoutSourceMap(moduleID: ModuleID, force: Boolean): Option[ByteBuffer] = { + val cache = printedModuleSetCache.getModuleCache(moduleID) + val changed = cache.update(emitterResult.body(moduleID)) - for (topLevelTree <- emitterResult.body(moduleID)) { - val printedTree = printedModuleCache.getPrintedTree(topLevelTree) - jsFileWriter.write(printedTree.jsCode) - } + if (force || changed || allChanged) { + printedModuleSetCache.incRewrittenModules() + + val jsFileWriter = new ByteArrayWriter(sizeHintFor(cache.getPreviousFinalJSFileSize())) + + jsFileWriter.write(printedModuleSetCache.headerBytes) + jsFileWriter.writeASCIIString("'use strict';\n") + + for (printedTree <- cache.printedTrees) + jsFileWriter.write(printedTree.jsCode) - jsFileWriter.write(emitterResult.footer.getBytes(StandardCharsets.UTF_8)) + jsFileWriter.write(printedModuleSetCache.footerBytes) - printedModuleCache.recordFinalSizes(jsFileWriter.currentSize, 0) + cache.recordFinalSizes(jsFileWriter.currentSize, 0) + Some(jsFileWriter.toByteBuffer()) + } else { + None + } } - protected def writeModule(moduleID: ModuleID, jsFileWriter: ByteArrayWriter, - sourceMapWriter: ByteArrayWriter): Unit = { - val printedModuleCache = printedModuleSetCache.getModuleCache(moduleID) + protected def writeModuleWithSourceMap(moduleID: ModuleID, force: Boolean): Option[(ByteBuffer, ByteBuffer)] = { + val cache = printedModuleSetCache.getModuleCache(moduleID) + val changed = cache.update(emitterResult.body(moduleID)) - jsFileWriter.sizeHint(sizeHintFor(printedModuleCache.getPreviousFinalJSFileSize())) - sourceMapWriter.sizeHint(sizeHintFor(printedModuleCache.getPreviousFinalSourceMapSize())) + if (force || changed || allChanged) { + printedModuleSetCache.incRewrittenModules() - val jsFileURI = OutputPatternsImpl.jsFileURI(config.outputPatterns, moduleID.id) - val sourceMapURI = OutputPatternsImpl.sourceMapURI(config.outputPatterns, moduleID.id) + val jsFileWriter = new ByteArrayWriter(sizeHintFor(cache.getPreviousFinalJSFileSize())) + val sourceMapWriter = new ByteArrayWriter(sizeHintFor(cache.getPreviousFinalSourceMapSize())) - val smWriter = new SourceMapWriter(sourceMapWriter, jsFileURI, - config.relativizeSourceMapBase) + val jsFileURI = OutputPatternsImpl.jsFileURI(config.outputPatterns, moduleID.id) + val sourceMapURI = OutputPatternsImpl.sourceMapURI(config.outputPatterns, moduleID.id) - jsFileWriter.write(emitterResult.header.getBytes(StandardCharsets.UTF_8)) - for (_ <- 0 until emitterResult.header.count(_ == '\n')) - smWriter.nextLine() + val smWriter = new SourceMapWriter(sourceMapWriter, jsFileURI, + config.relativizeSourceMapBase) - jsFileWriter.writeASCIIString("'use strict';\n") - smWriter.nextLine() + jsFileWriter.write(printedModuleSetCache.headerBytes) + for (_ <- 0 until printedModuleSetCache.headerNewLineCount) + smWriter.nextLine() - for (topLevelTree <- emitterResult.body(moduleID)) { - val printedTree = printedModuleCache.getPrintedTree(topLevelTree) - jsFileWriter.write(printedTree.jsCode) - smWriter.insertFragment(printedTree.sourceMapFragment) - } + jsFileWriter.writeASCIIString("'use strict';\n") + smWriter.nextLine() + + for (printedTree <- cache.printedTrees) { + jsFileWriter.write(printedTree.jsCode) + smWriter.insertFragment(printedTree.sourceMapFragment) + } - jsFileWriter.write(emitterResult.footer.getBytes(StandardCharsets.UTF_8)) - jsFileWriter.write(("//# sourceMappingURL=" + sourceMapURI + "\n").getBytes(StandardCharsets.UTF_8)) + jsFileWriter.write(printedModuleSetCache.footerBytes) + jsFileWriter.write(("//# sourceMappingURL=" + sourceMapURI + "\n").getBytes(StandardCharsets.UTF_8)) - smWriter.complete() + smWriter.complete() - printedModuleCache.recordFinalSizes(jsFileWriter.currentSize, sourceMapWriter.currentSize) + cache.recordFinalSizes(jsFileWriter.currentSize, sourceMapWriter.currentSize) + Some((jsFileWriter.toByteBuffer(), sourceMapWriter.toByteBuffer())) + } else { + None + } } private def sizeHintFor(previousSize: Int): Int = previousSize + (previousSize / 10) } - printedModuleSetCache.startRun() - logger.timeFuture("BasicBackend: Write result") { writer.write(moduleSet) }.andThen { case _ => @@ -132,16 +152,46 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config) private object BasicLinkerBackend { private final class PrintedModuleSetCache(withSourceMaps: Boolean) { + private var lastHeader: String = null + private var lastFooter: String = null + + private var _headerBytesCache: Array[Byte] = null + private var _footerBytesCache: Array[Byte] = null + private var _headerNewLineCountCache: Int = 0 + private val modules = new java.util.concurrent.ConcurrentHashMap[ModuleID, PrintedModuleCache] + private var totalModules = 0 + private val rewrittenModules = new java.util.concurrent.atomic.AtomicInteger(0) + private var totalTopLevelTrees = 0 private var recomputedTopLevelTrees = 0 - def startRun(): Unit = { + def startRun(moduleSet: ModuleSet): Unit = { + totalModules = moduleSet.modules.size + rewrittenModules.set(0) + totalTopLevelTrees = 0 recomputedTopLevelTrees = 0 } + def updateGlobal(header: String, footer: String): Boolean = { + if (header == lastHeader && footer == lastFooter) { + false + } else { + _headerBytesCache = header.getBytes(StandardCharsets.UTF_8) + _footerBytesCache = footer.getBytes(StandardCharsets.UTF_8) + _headerNewLineCountCache = _headerBytesCache.count(_ == '\n') + lastHeader = header + lastFooter = footer + true + } + } + + def headerBytes: Array[Byte] = _headerBytesCache + def footerBytes: Array[Byte] = _footerBytesCache + def headerNewLineCount: Int = _headerNewLineCountCache + def getModuleCache(moduleID: ModuleID): PrintedModuleCache = { val result = modules.computeIfAbsent(moduleID, { _ => if (withSourceMaps) new PrintedModuleCacheWithSourceMaps @@ -152,6 +202,9 @@ private object BasicLinkerBackend { result } + def incRewrittenModules(): Unit = + rewrittenModules.incrementAndGet() + def cleanAfterRun(): Unit = { val iter = modules.entrySet().iterator() while (iter.hasNext()) { @@ -166,11 +219,13 @@ private object BasicLinkerBackend { } def logStats(logger: Logger): Unit = { - /* This message is extracted in BasicLinkerBackendTest to assert that we - * do not invalidate anything in a no-op second run. + /* These messages are extracted in BasicLinkerBackendTest to assert that + * we do not invalidate anything in a no-op second run. */ logger.debug( s"BasicBackend: total top-level trees: $totalTopLevelTrees; re-computed: $recomputedTopLevelTrees") + logger.debug( + s"BasicBackend: total modules: $totalModules; re-written: ${rewrittenModules.get()}") } } @@ -180,17 +235,18 @@ private object BasicLinkerBackend { private sealed class PrintedModuleCache { private var cacheUsed = false + private var changed = false + private var lastJSTrees: List[js.Tree] = Nil + private var printedTreesCache: List[PrintedTree] = Nil private val cache = new java.util.IdentityHashMap[js.Tree, PrintedTree] private var previousFinalJSFileSize: Int = 0 private var previousFinalSourceMapSize: Int = 0 - private var totalTopLevelTrees = 0 private var recomputedTopLevelTrees = 0 def startRun(): Unit = { cacheUsed = true - totalTopLevelTrees = 0 recomputedTopLevelTrees = 0 } @@ -203,9 +259,17 @@ private object BasicLinkerBackend { previousFinalSourceMapSize = finalSourceMapSize } - def getPrintedTree(tree: js.Tree): PrintedTree = { - totalTopLevelTrees += 1 + def update(newJSTrees: List[js.Tree]): Boolean = { + val changed = !newJSTrees.corresponds(lastJSTrees)(_ eq _) + this.changed = changed + if (changed) { + printedTreesCache = newJSTrees.map(getOrComputePrintedTree(_)) + lastJSTrees = newJSTrees + } + changed + } + private def getOrComputePrintedTree(tree: js.Tree): PrintedTree = { val result = cache.computeIfAbsent(tree, { (tree: js.Tree) => recomputedTopLevelTrees += 1 computePrintedTree(tree) @@ -224,17 +288,21 @@ private object BasicLinkerBackend { new PrintedTree(jsCodeWriter.toByteArray(), SourceMapWriter.Fragment.Empty) } + def printedTrees: List[PrintedTree] = printedTreesCache + def cleanAfterRun(): Boolean = { if (cacheUsed) { cacheUsed = false - val iter = cache.entrySet().iterator() - while (iter.hasNext()) { - val printedTree = iter.next().getValue() - if (printedTree.cachedUsed) - printedTree.cachedUsed = false - else - iter.remove() + if (changed) { + val iter = cache.entrySet().iterator() + while (iter.hasNext()) { + val printedTree = iter.next().getValue() + if (printedTree.cachedUsed) + printedTree.cachedUsed = false + else + iter.remove() + } } true @@ -243,7 +311,7 @@ private object BasicLinkerBackend { } } - def getTotalTopLevelTrees: Int = totalTopLevelTrees + def getTotalTopLevelTrees: Int = lastJSTrees.size def getRecomputedTopLevelTrees: Int = recomputedTopLevelTrees } diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/OutputWriter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/OutputWriter.scala index 1a54878c86..49113d8e41 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/OutputWriter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/OutputWriter.scala @@ -25,26 +25,26 @@ import org.scalajs.linker.standard.ModuleSet.ModuleID import org.scalajs.linker.backend.javascript.ByteArrayWriter private[backend] abstract class OutputWriter(output: OutputDirectory, - config: LinkerBackendImpl.Config) { + config: LinkerBackendImpl.Config, skipContentCheck: Boolean) { private val outputImpl = OutputDirectoryImpl.fromOutputDirectory(output) private val moduleKind = config.commonConfig.coreSpec.moduleKind - protected def writeModule(moduleID: ModuleID, jsFileWriter: ByteArrayWriter): Unit + protected def writeModuleWithoutSourceMap(moduleID: ModuleID, force: Boolean): Option[ByteBuffer] - protected def writeModule(moduleID: ModuleID, jsFileWriter: ByteArrayWriter, - sourceMapWriter: ByteArrayWriter): Unit + protected def writeModuleWithSourceMap(moduleID: ModuleID, force: Boolean): Option[(ByteBuffer, ByteBuffer)] def write(moduleSet: ModuleSet)(implicit ec: ExecutionContext): Future[Report] = { val ioThrottler = new IOThrottler(config.maxConcurrentWrites) - def filesToRemove(seen: Iterable[String], reports: List[Report.Module]): Set[String] = - seen.toSet -- reports.flatMap(r => r.jsFileName :: r.sourceMapName.toList) + def filesToRemove(seen: Set[String], reports: List[Report.Module]): Set[String] = + seen -- reports.flatMap(r => r.jsFileName :: r.sourceMapName.toList) for { - currentFiles <- outputImpl.listFiles() + currentFilesList <- outputImpl.listFiles() + currentFiles = currentFilesList.toSet reports <- Future.traverse(moduleSet.modules) { m => - ioThrottler.throttle(writeModule(m.id)) + ioThrottler.throttle(writeModule(m.id, currentFiles)) } _ <- Future.traverse(filesToRemove(currentFiles, reports)) { f => ioThrottler.throttle(outputImpl.delete(f)) @@ -61,38 +61,39 @@ private[backend] abstract class OutputWriter(output: OutputDirectory, } } - private def writeModule(moduleID: ModuleID)( + private def writeModule(moduleID: ModuleID, existingFiles: Set[String])( implicit ec: ExecutionContext): Future[Report.Module] = { val jsFileName = OutputPatternsImpl.jsFile(config.outputPatterns, moduleID.id) if (config.sourceMap) { val sourceMapFileName = OutputPatternsImpl.sourceMapFile(config.outputPatterns, moduleID.id) - - val codeWriter = new ByteArrayWriter - val smWriter = new ByteArrayWriter - - writeModule(moduleID, codeWriter, smWriter) - - val code = codeWriter.toByteBuffer() - val sourceMap = smWriter.toByteBuffer() - - for { - _ <- outputImpl.writeFull(jsFileName, code) - _ <- outputImpl.writeFull(sourceMapFileName, sourceMap) - } yield { - new ReportImpl.ModuleImpl(moduleID.id, jsFileName, Some(sourceMapFileName), moduleKind) + val report = new ReportImpl.ModuleImpl(moduleID.id, jsFileName, Some(sourceMapFileName), moduleKind) + val force = !existingFiles.contains(jsFileName) || !existingFiles.contains(sourceMapFileName) + + writeModuleWithSourceMap(moduleID, force) match { + case Some((code, sourceMap)) => + for { + _ <- outputImpl.writeFull(jsFileName, code, skipContentCheck) + _ <- outputImpl.writeFull(sourceMapFileName, sourceMap, skipContentCheck) + } yield { + report + } + case None => + Future.successful(report) } } else { - val codeWriter = new ByteArrayWriter - - writeModule(moduleID, codeWriter) - - val code = codeWriter.toByteBuffer() - - for { - _ <- outputImpl.writeFull(jsFileName, code) - } yield { - new ReportImpl.ModuleImpl(moduleID.id, jsFileName, None, moduleKind) + val report = new ReportImpl.ModuleImpl(moduleID.id, jsFileName, None, moduleKind) + val force = !existingFiles.contains(jsFileName) + + writeModuleWithoutSourceMap(moduleID, force) match { + case Some(code) => + for { + _ <- outputImpl.writeFull(jsFileName, code, skipContentCheck) + } yield { + report + } + case None => + Future.successful(report) } } } diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala index 83adbd32c6..d5e563c866 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/javascript/ByteArrayWriter.scala @@ -16,14 +16,15 @@ import java.io.OutputStream import java.nio.ByteBuffer /** Like a `java.io.ByteArrayOutputStream` but with more control. */ -private[backend] final class ByteArrayWriter extends OutputStream { - private var buffer: Array[Byte] = new Array[Byte](1024) +private[backend] final class ByteArrayWriter(originalCapacity: Int) extends OutputStream { + private var buffer: Array[Byte] = + new Array[Byte](powerOfTwoAtLeast(Math.max(originalCapacity, 1024))) + private var size: Int = 0 - def currentSize: Int = size + def this() = this(0) - def sizeHint(capacity: Int): Unit = - ensureCapacity(capacity) + def currentSize: Int = size private def ensureCapacity(capacity: Int): Unit = { if (buffer.length < capacity) diff --git a/linker/shared/src/test/scala/org/scalajs/linker/BasicLinkerBackendTest.scala b/linker/shared/src/test/scala/org/scalajs/linker/BasicLinkerBackendTest.scala index aa43ba52c1..1ce36b9153 100644 --- a/linker/shared/src/test/scala/org/scalajs/linker/BasicLinkerBackendTest.scala +++ b/linker/shared/src/test/scala/org/scalajs/linker/BasicLinkerBackendTest.scala @@ -36,47 +36,84 @@ class BasicLinkerBackendTest { private val BackendInvalidatedTopLevelTreesStatsMessage = raw"""BasicBackend: total top-level trees: (\d+); re-computed: (\d+)""".r + private val BackendInvalidatedModulesStatsMessage = + raw"""BasicBackend: total modules: (\d+); re-written: (\d+)""".r + /** Makes sure that linking a "substantial" program (using `println`) twice - * does not invalidate any top-level tree in the second run. + * does not invalidate any top-level tree nor module in the second run. */ @Test - def noInvalidatedTopLevelTreeInSecondRun(): AsyncResult = await { + def noInvalidatedTopLevelTreeOrModuleInSecondRun(): AsyncResult = await { + import ModuleSplitStyle._ + val classDefs = List( mainTestClassDef(systemOutPrintln(str("Hello world!"))) ) - val logger1 = new CapturingLogger - val logger2 = new CapturingLogger - - val config = StandardConfig().withCheckIR(true) - val linker = StandardImpl.linker(config) - val classDefsFiles = classDefs.map(MemClassDefIRFile(_)) - - val initializers = MainTestModuleInitializers - val outputDir = MemOutputDirectory() - - for { - javalib <- TestIRRepo.javalib - allIRFiles = javalib ++ classDefsFiles - _ <- linker.link(allIRFiles, initializers, outputDir, logger1) - _ <- linker.link(allIRFiles, initializers, outputDir, logger2) - } yield { - val lines1 = logger1.allLogLines - val Seq(total1, recomputed1) = - lines1.assertContainsMatch(BackendInvalidatedTopLevelTreesStatsMessage).map(_.toInt) - - val lines2 = logger2.allLogLines - val Seq(total2, recomputed2) = - lines2.assertContainsMatch(BackendInvalidatedTopLevelTreesStatsMessage).map(_.toInt) - - // At the time of writing this test, total1 reports 382 trees - assertTrue( - s"Not enough total top-level trees (got $total1); extraction must have gone wrong", - total1 > 300) - - assertEquals("First run must invalidate every top-level tree", total1, recomputed1) - assertEquals("Second run must have the same total as first run", total1, total2) - assertEquals("Second run must not invalidate any top-level tree", 0, recomputed2) + val results = for (splitStyle <- List(FewestModules, SmallestModules)) yield { + val logger1 = new CapturingLogger + val logger2 = new CapturingLogger + + val config = StandardConfig() + .withCheckIR(true) + .withModuleKind(ModuleKind.ESModule) + .withModuleSplitStyle(splitStyle) + + val linker = StandardImpl.linker(config) + val classDefsFiles = classDefs.map(MemClassDefIRFile(_)) + + val initializers = MainTestModuleInitializers + val outputDir = MemOutputDirectory() + + for { + javalib <- TestIRRepo.javalib + allIRFiles = javalib ++ classDefsFiles + _ <- linker.link(allIRFiles, initializers, outputDir, logger1) + _ <- linker.link(allIRFiles, initializers, outputDir, logger2) + } yield { + val lines1 = logger1.allLogLines + val lines2 = logger2.allLogLines + + // Top-level trees + + val Seq(totalTrees1, recomputedTrees1) = + lines1.assertContainsMatch(BackendInvalidatedTopLevelTreesStatsMessage).map(_.toInt) + + val Seq(totalTrees2, recomputedTrees2) = + lines2.assertContainsMatch(BackendInvalidatedTopLevelTreesStatsMessage).map(_.toInt) + + // At the time of writing this test, totalTrees1 reports 382 trees + assertTrue( + s"Not enough total top-level trees (got $totalTrees1); extraction must have gone wrong", + totalTrees1 > 300) + + assertEquals("First run must invalidate every top-level tree", totalTrees1, recomputedTrees1) + assertEquals("Second run must have the same total top-level trees as first run", totalTrees1, totalTrees2) + assertEquals("Second run must not invalidate any top-level tree", 0, recomputedTrees2) + + // Modules + + val Seq(totalModules1, rewrittenModules1) = + lines1.assertContainsMatch(BackendInvalidatedModulesStatsMessage).map(_.toInt) + + val Seq(totalModules2, rewrittenModules2) = + lines2.assertContainsMatch(BackendInvalidatedModulesStatsMessage).map(_.toInt) + + if (splitStyle == FewestModules) { + assertEquals("Expected exactly one module with FewestModules", 1, totalModules1) + } else { + // At the time of writing this test, totalModules1 reports 9 modules + assertTrue( + s"Not enough total modules (got $totalModules1); extraction must have gone wrong", + totalModules1 > 5) + } + + assertEquals("First run must invalidate every module", totalModules1, rewrittenModules1) + assertEquals("Second run must have the same total modules as first run", totalModules1, totalModules2) + assertEquals("Second run must not invalidate any module", 0, rewrittenModules2) + } } + + Future.sequence(results) } } From 6af1ae828698824c5aa8e998db210c54af25b6ec Mon Sep 17 00:00:00 2001 From: Tobias Schlatter Date: Sun, 19 Mar 2023 14:04:28 +0100 Subject: [PATCH 36/45] Add javalib and javalibintf to local publish instructions Forgotten in efe177b0405ff0f2ada3ef8036d6984e9e49d704 / ba8ba87e3e68b72157de311acc5d5a96610da15b --- DEVELOPING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVELOPING.md b/DEVELOPING.md index bf409ce03e..8c1961abc7 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -170,7 +170,7 @@ To publish your changes locally to be used in a separate project, use the following incantations. `SCALA_VERSION` refers to the Scala version used by the separate project. - > ;ir2_12/publishLocal;linkerInterface2_12/publishLocal;linker2_12/publishLocal;testAdapter2_12/publishLocal;sbtPlugin/publishLocal + > ;ir2_12/publishLocal;linkerInterface2_12/publishLocal;linker2_12/publishLocal;testAdapter2_12/publishLocal;sbtPlugin/publishLocal;javalib/publishLocal;javalibintf/publishLocal > ++SCALA_VERSION > ;compiler2_12/publishLocal;library2_12/publishLocal;testInterface2_12/publishLocal;testBridge2_12/publishLocal;jUnitRuntime2_12/publishLocal;jUnitPlugin2_12/publishLocal From f23cc32c85a15282cfeda7f16615e5e32c616f8f Mon Sep 17 00:00:00 2001 From: Tobias Schlatter Date: Sun, 19 Mar 2023 17:08:31 +0100 Subject: [PATCH 37/45] Use default Scala minor version in local publish for non-compiler This is consistent with publish.sh. --- DEVELOPING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DEVELOPING.md b/DEVELOPING.md index 8c1961abc7..1664d8a3c4 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -171,7 +171,7 @@ following incantations. `SCALA_VERSION` refers to the Scala version used by the separate project. > ;ir2_12/publishLocal;linkerInterface2_12/publishLocal;linker2_12/publishLocal;testAdapter2_12/publishLocal;sbtPlugin/publishLocal;javalib/publishLocal;javalibintf/publishLocal - > ++SCALA_VERSION - > ;compiler2_12/publishLocal;library2_12/publishLocal;testInterface2_12/publishLocal;testBridge2_12/publishLocal;jUnitRuntime2_12/publishLocal;jUnitPlugin2_12/publishLocal + > ;library2_12/publishLocal;testInterface2_12/publishLocal;testBridge2_12/publishLocal;jUnitRuntime2_12/publishLocal;jUnitPlugin2_12/publishLocal + > ++SCALA_VERSION compiler2_12/publishLocal -If using a non-2.12.x version for the Scala version, the `2_12` suffixes must be adapted in the last command (not in the first command). +If using a non-2.12.x version for the Scala version, the `2_12` suffixes must be adapted in the second and third command (not in the first command). From a7bf01684e07cfa89efa7ac119e364209630d75d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Sat, 25 Mar 2023 12:22:53 +0100 Subject: [PATCH 38/45] Rename file `MemIRFile.scala` to `MemIRFileImpl.scala`. Because the only class it defines is called `MemIRFileImpl`. --- .../linker/standard/{MemIRFile.scala => MemIRFileImpl.scala} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename linker/shared/src/main/scala/org/scalajs/linker/standard/{MemIRFile.scala => MemIRFileImpl.scala} (100%) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/standard/MemIRFile.scala b/linker/shared/src/main/scala/org/scalajs/linker/standard/MemIRFileImpl.scala similarity index 100% rename from linker/shared/src/main/scala/org/scalajs/linker/standard/MemIRFile.scala rename to linker/shared/src/main/scala/org/scalajs/linker/standard/MemIRFileImpl.scala From b6d70c144d58832dc8ad73bf222044323713f029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Sat, 25 Mar 2023 12:32:15 +0100 Subject: [PATCH 39/45] Introduce a new `linker.standard.MemClassDefIRFileImpl`. It is similar to `MemIRFileImpl`, but directly contains a `ClassDef` instead of a serialized byte array. --- .../standard/MemClassDefIRFileImpl.scala | 36 +++++++++++++++++++ .../linker/testutils/MemClassDefIRFile.scala | 19 +++------- 2 files changed, 41 insertions(+), 14 deletions(-) create mode 100644 linker/shared/src/main/scala/org/scalajs/linker/standard/MemClassDefIRFileImpl.scala diff --git a/linker/shared/src/main/scala/org/scalajs/linker/standard/MemClassDefIRFileImpl.scala b/linker/shared/src/main/scala/org/scalajs/linker/standard/MemClassDefIRFileImpl.scala new file mode 100644 index 0000000000..78bb665e06 --- /dev/null +++ b/linker/shared/src/main/scala/org/scalajs/linker/standard/MemClassDefIRFileImpl.scala @@ -0,0 +1,36 @@ +/* + * Scala.js (https://www.scala-js.org/) + * + * Copyright EPFL. + * + * Licensed under Apache License 2.0 + * (https://www.apache.org/licenses/LICENSE-2.0). + * + * See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + */ + +package org.scalajs.linker.standard + +import scala.concurrent._ + +import org.scalajs.ir.EntryPointsInfo +import org.scalajs.ir.Trees.ClassDef +import org.scalajs.ir.Version + +import org.scalajs.linker.interface.unstable.IRFileImpl + +/** A simple in-memory virtual Scala.js IR file with a ClassDef. */ +final class MemClassDefIRFileImpl( + path: String, + version: Version, + classDef: ClassDef +) extends IRFileImpl(path, version) { + private val _entryPointsInfo = EntryPointsInfo.forClassDef(classDef) + + def entryPointsInfo(implicit ec: ExecutionContext): Future[EntryPointsInfo] = + Future.successful(_entryPointsInfo) + + def tree(implicit ec: ExecutionContext): Future[ClassDef] = + Future.successful(classDef) +} diff --git a/linker/shared/src/test/scala/org/scalajs/linker/testutils/MemClassDefIRFile.scala b/linker/shared/src/test/scala/org/scalajs/linker/testutils/MemClassDefIRFile.scala index 48d1c13907..e7f279c819 100644 --- a/linker/shared/src/test/scala/org/scalajs/linker/testutils/MemClassDefIRFile.scala +++ b/linker/shared/src/test/scala/org/scalajs/linker/testutils/MemClassDefIRFile.scala @@ -14,27 +14,18 @@ package org.scalajs.linker.testutils import scala.concurrent._ -import org.scalajs.ir.EntryPointsInfo import org.scalajs.ir.Trees.ClassDef import org.scalajs.ir.Version import org.scalajs.linker.interface.IRFile -import org.scalajs.linker.interface.unstable.IRFileImpl - -private final class MemClassDefIRFile(classDef: ClassDef, version: Version) - extends IRFileImpl("mem://" + classDef.name.name + ".sjsir", version) { - - def tree(implicit ec: ExecutionContext): Future[ClassDef] = - Future(classDef) - - def entryPointsInfo(implicit ec: ExecutionContext): Future[EntryPointsInfo] = - tree.map(EntryPointsInfo.forClassDef) -} +import org.scalajs.linker.standard.MemClassDefIRFileImpl object MemClassDefIRFile { def apply(classDef: ClassDef): IRFile = apply(classDef, Version.Unversioned) - def apply(classDef: ClassDef, version: Version): IRFile = - new MemClassDefIRFile(classDef, version) + def apply(classDef: ClassDef, version: Version): IRFile = { + val path = "mem://" + classDef.name.name.nameString + ".sjsir" + new MemClassDefIRFileImpl(path, version, classDef) + } } From ed7ae892b884a3d4babe2164d67d7be37e5d6c32 Mon Sep 17 00:00:00 2001 From: Tobias Schlatter Date: Sun, 12 Mar 2023 13:33:37 +0100 Subject: [PATCH 40/45] Eagerly deserialize linker private lib Otherwise, deserialization might happen multiple times on subsequent linker runs. --- .../backend/emitter/PrivateLibHolder.scala | 16 ++++++++++------ .../backend/emitter/PrivateLibHolder.scala | 15 ++++++++------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/linker/js/src/main/scala/org/scalajs/linker/backend/emitter/PrivateLibHolder.scala b/linker/js/src/main/scala/org/scalajs/linker/backend/emitter/PrivateLibHolder.scala index 84b5b93d13..893d040212 100644 --- a/linker/js/src/main/scala/org/scalajs/linker/backend/emitter/PrivateLibHolder.scala +++ b/linker/js/src/main/scala/org/scalajs/linker/backend/emitter/PrivateLibHolder.scala @@ -12,19 +12,23 @@ package org.scalajs.linker.backend.emitter +import java.nio.ByteBuffer +import java.util.Base64 + import org.scalajs.ir import org.scalajs.linker.interface.IRFile -import org.scalajs.linker.standard.MemIRFileImpl +import org.scalajs.linker.standard.MemClassDefIRFileImpl object PrivateLibHolder { + private val stableVersion = ir.Version.fromInt(0) // never changes + val files: Seq[IRFile] = { for ((name, contentBase64) <- PrivateLibData.pathsAndContents) yield { - new MemIRFileImpl( - path = "org/scalajs/linker/runtime/" + name, - version = ir.Version.fromInt(0), // never changes - content = java.util.Base64.getDecoder().decode(contentBase64) - ) + val path = "org/scalajs/linker/runtime/" + name + val content = Base64.getDecoder().decode(contentBase64) + val tree = ir.Serializers.deserialize(ByteBuffer.wrap(content)) + new MemClassDefIRFileImpl(path, stableVersion, tree) } } } diff --git a/linker/jvm/src/main/scala/org/scalajs/linker/backend/emitter/PrivateLibHolder.scala b/linker/jvm/src/main/scala/org/scalajs/linker/backend/emitter/PrivateLibHolder.scala index 96b78cbd89..44801549e7 100644 --- a/linker/jvm/src/main/scala/org/scalajs/linker/backend/emitter/PrivateLibHolder.scala +++ b/linker/jvm/src/main/scala/org/scalajs/linker/backend/emitter/PrivateLibHolder.scala @@ -12,14 +12,17 @@ package org.scalajs.linker.backend.emitter -import java.io._ +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer import org.scalajs.ir import org.scalajs.linker.interface.IRFile -import org.scalajs.linker.standard.MemIRFileImpl +import org.scalajs.linker.standard.MemClassDefIRFileImpl object PrivateLibHolder { + private val stableVersion = ir.Version.fromInt(0) // never changes + private val sjsirPaths = Seq( "org/scalajs/linker/runtime/RuntimeLong.sjsir", "org/scalajs/linker/runtime/RuntimeLong$.sjsir", @@ -30,11 +33,9 @@ object PrivateLibHolder { val files: Seq[IRFile] = { for (path <- sjsirPaths) yield { val name = path.substring(path.lastIndexOf('/') + 1) - new MemIRFileImpl( - path = path, - version = ir.Version.fromInt(0), // never changes - content = readResource(name) - ) + val content = readResource(name) + val tree = ir.Serializers.deserialize(ByteBuffer.wrap(content)) + new MemClassDefIRFileImpl(path, stableVersion, tree) } } From df376ba8989c2de60121cf11abcd0fd093f3899a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Mon, 3 Apr 2023 15:09:32 +0200 Subject: [PATCH 41/45] Fix #4841: Tolerate multiple callbacks from AsynchronousFileChannel. Apparently, on JDK 17 on Windows, `AsynchronousFileChannel` can call `CompletionHandler.completed` and/or `failed` more than once, although the documentation does not suggest that it is valid behavior. We now tolerate these situations by using `Promise.trySuccess` and `tryFailure` instead of `success` and `failure`. --- .../scalajs/linker/PathOutputDirectory.scala | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/linker/jvm/src/main/scala/org/scalajs/linker/PathOutputDirectory.scala b/linker/jvm/src/main/scala/org/scalajs/linker/PathOutputDirectory.scala index 22ee601909..f3412c3c40 100644 --- a/linker/jvm/src/main/scala/org/scalajs/linker/PathOutputDirectory.scala +++ b/linker/jvm/src/main/scala/org/scalajs/linker/PathOutputDirectory.scala @@ -31,6 +31,13 @@ object PathOutputDirectory { new Impl(directory) } + /* #4841 In the `CompletionHandler`s of our `AsynchronousFileChannel`s, we + * use `promise.trySuccess` and `tryFailure` instead of `success` and + * `failure`. This avoids `IllegalStateException` if there are double calls + * to `CompletionHandler.{completed,failed}`. It should not happen, but we + * observed it to happen on Windows anyway. + */ + private final class Impl(directory: Path) extends OutputDirectoryImpl { def writeFull(name: String, buf: ByteBuffer)(implicit ec: ExecutionContext): Future[Unit] = { val file = getPath(name) @@ -126,11 +133,11 @@ object PathOutputDirectory { if (buf.hasRemaining()) writeLoop() else - promise.success(()) + promise.trySuccess(()) } def failed(exc: Throwable, unit: Unit): Unit = - promise.failure(exc) + promise.tryFailure(exc) } writeLoop() @@ -154,14 +161,14 @@ object PathOutputDirectory { def completed(read: Integer, unit: Unit): Unit = { if (read == -1 || !buf.hasRemaining()) { buf.flip() - promise.success(buf) + promise.trySuccess(buf) } else { readLoop() } } def failed(exc: Throwable, unit: Unit): Unit = - promise.failure(exc) + promise.tryFailure(exc) } readLoop() @@ -229,7 +236,7 @@ object PathOutputDirectory { /* We have checked the file size beforehand. So if we get here, * there's no diff. */ - promise.success(false) + promise.trySuccess(false) } else { pos += read @@ -239,7 +246,7 @@ object PathOutputDirectory { tmpCmpBuf.limit(read) if (readBuf != tmpCmpBuf) { - promise.success(true) + promise.trySuccess(true) } else { cmpBuf.position(cmpBuf.position() + read) readNext() @@ -248,7 +255,7 @@ object PathOutputDirectory { } def failed(exc: Throwable, unit: Unit): Unit = - promise.failure(exc) + promise.tryFailure(exc) } readNext() From 968cfe5c362c5a4e3fb8732f5a25a58cc0346dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Wed, 5 Apr 2023 16:43:06 +0200 Subject: [PATCH 42/45] Fix #4831: Correctly compare fieldDefs in KnowledgeGuardian. Previously, we compared the essentially compared the nodes for identity, which causes spurious invalidations in some cases. We cannot completely compare them structurally, because the name of `JSFieldDef`s can be arbitrary `Tree`s, which we cannot efficiently compare. Therefore, we use an elaborate combination of the version of the class, a flag for whether there is any JS field, and the list of non-JS field names. See the comment for the new method `computeFieldDefsVersion` for details. --- .../main/scala/org/scalajs/ir/Version.scala | 10 ++++++ .../backend/emitter/KnowledgeGuardian.scala | 34 +++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/ir/shared/src/main/scala/org/scalajs/ir/Version.scala b/ir/shared/src/main/scala/org/scalajs/ir/Version.scala index 0ff29715a5..f30be5f7ee 100644 --- a/ir/shared/src/main/scala/org/scalajs/ir/Version.scala +++ b/ir/shared/src/main/scala/org/scalajs/ir/Version.scala @@ -108,6 +108,16 @@ object Version { new Version(buf.array()) } + /** Create a non-hash version from the given [[UTF8String]]. + * + * Strictly equivalent to (but potentially more efficient): + * {{{ + * fromBytes(Array.tabulate(utf8String.length)(utf8String(_))) + * }}} + */ + def fromUTF8String(utf8String: UTF8String): Version = + make(Type.Ephemeral, utf8String.bytes) + /** Create a combined, non-hash version from the given bytes. * * Returns [[Unversioned]] if at least one of versions is [[Unversioned]]. diff --git a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/KnowledgeGuardian.scala b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/KnowledgeGuardian.scala index 18bfea85b3..21ba3c0640 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/KnowledgeGuardian.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/backend/emitter/KnowledgeGuardian.scala @@ -18,6 +18,7 @@ import org.scalajs.ir.ClassKind import org.scalajs.ir.Names._ import org.scalajs.ir.Trees._ import org.scalajs.ir.Types.Type +import org.scalajs.ir.Version import org.scalajs.linker.interface.ModuleKind import org.scalajs.linker.standard._ @@ -241,6 +242,7 @@ private[emitter] final class KnowledgeGuardian(config: Emitter.Config) { private var jsNativeLoadSpec = computeJSNativeLoadSpec(initClass) private var jsNativeMemberLoadSpecs = computeJSNativeMemberLoadSpecs(initClass) private var superClass = computeSuperClass(initClass) + private var fieldDefsVersion = computeFieldDefsVersion(initClass) private var fieldDefs = computeFieldDefs(initClass) private var staticFieldMirrors = initStaticFieldMirrors private var module = initModule @@ -309,9 +311,10 @@ private[emitter] final class KnowledgeGuardian(config: Emitter.Config) { invalidateAskers(superClassAskers) } - val newFieldDefs = computeFieldDefs(linkedClass) - if (newFieldDefs != fieldDefs) { - fieldDefs = newFieldDefs + val newFieldDefsVersion = computeFieldDefsVersion(linkedClass) + if (!newFieldDefsVersion.sameVersion(fieldDefsVersion)) { + fieldDefsVersion = newFieldDefsVersion + fieldDefs = computeFieldDefs(linkedClass) invalidateAskers(fieldDefsAskers) } @@ -353,6 +356,31 @@ private[emitter] final class KnowledgeGuardian(config: Emitter.Config) { private def computeSuperClass(linkedClass: LinkedClass): ClassName = linkedClass.superClass.fold[ClassName](null.asInstanceOf[ClassName])(_.name) + /** Computes the version of the fields of a `LinkedClass`. + * + * The version is composed of + * + * - the `version` of the `LinkedClass` itself, which will change every + * time the definition of a field changes, + * - a boolean indicating whether there is at least one `JSFieldDef`, + * which will change every time the reachability analysis of the + * `JSFieldDef`s changes (because we either keep all or none of + * them), and + * - the list of names of the `FieldDef`s, which will change every time + * the reachability analysis of the `FieldDef`s changes. + * + * We do not try to use the names of `JSFieldDef`s because they are + * `Tree`s, which are not efficiently comparable nor versionable here. + */ + private def computeFieldDefsVersion(linkedClass: LinkedClass): Version = { + val hasAnyJSField = linkedClass.fields.exists(_.isInstanceOf[JSFieldDef]) + val hasAnyJSFieldVersion = Version.fromInt(if (hasAnyJSField) 1 else 0) + val scalaFieldNamesVersion = linkedClass.fields.collect { + case FieldDef(_, FieldIdent(name), _, _) => Version.fromUTF8String(name.encoded) + } + Version.combine((linkedClass.version :: hasAnyJSFieldVersion :: scalaFieldNamesVersion): _*) + } + private def computeFieldDefs(linkedClass: LinkedClass): List[AnyFieldDef] = linkedClass.fields From 8e83161d724f73c03f6b600e3bf4afade3925e67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Fri, 7 Apr 2023 15:32:10 +0200 Subject: [PATCH 43/45] Upgrade to sbt-header 5.9.0. sbt-header was the only dependency for which we relied on the old Bintray/JFrog repositories. The latest version is published on Maven Central, like the rest of our dependencies. --- project/build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.sbt b/project/build.sbt index 86b3eaa21f..240ffb135c 100644 --- a/project/build.sbt +++ b/project/build.sbt @@ -1,4 +1,4 @@ -addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.0.0") +addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.9.0") addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.8.1") From 68f9cb6e27c87b31b1e2800726d1ea79f8deb638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Wed, 5 Apr 2023 14:24:50 +0200 Subject: [PATCH 44/45] Fix #4835: Treat excluded classes for Tagger as special hops. Previously, we more or less abused the concept of dynamic dependencies to handle them. This was based on the observation that we "must" split modules at those boundaries. See https://github.com/scala-js/scala-js/issues/4327#issuecomment-1059984564 However, as #4835 highlights, that observation ignored one aspect of dynamic dependencies: that when A--dyn-->B, A--static-->C and B--static-->C, by the time we load `B`, we know that `C` has already been loaded, and it is therefore OK to put A and C in the same module but not B. For the excluded classes for `SmallModulesFor`, we cannot make that assumption. On the contrary, relying on it causes a circular dependency between the A-C module and the B module. Instead, we now handle hops into excluded classes, from non-excluded classes, as special hops that need to be tracked. In the eventual tagging scheme of a class, we include the number of hops into excluded classes that are done to reach it from a root. This commit also fixes #4833. Co-authored-by: Tobias Schlatter --- .../frontend/modulesplitter/Tagger.scala | 150 +++++++++++++++--- .../linker/SmallModulesForSplittingTest.scala | 140 ++++++++++++++++ 2 files changed, 272 insertions(+), 18 deletions(-) diff --git a/linker/shared/src/main/scala/org/scalajs/linker/frontend/modulesplitter/Tagger.scala b/linker/shared/src/main/scala/org/scalajs/linker/frontend/modulesplitter/Tagger.scala index 9212fa9f13..f09088be26 100644 --- a/linker/shared/src/main/scala/org/scalajs/linker/frontend/modulesplitter/Tagger.scala +++ b/linker/shared/src/main/scala/org/scalajs/linker/frontend/modulesplitter/Tagger.scala @@ -18,6 +18,7 @@ import scala.collection.immutable import scala.collection.mutable import java.nio.charset.StandardCharsets +import java.nio.ByteBuffer import org.scalajs.ir.Names.ClassName import org.scalajs.ir.SHA1 @@ -67,6 +68,94 @@ import org.scalajs.linker.standard.ModuleSet.ModuleID * * The (transitive) dependencies of the class are nevertheless taken into * account and tagged as appropriate. + * In particular, to avoid cycles and excessive splitting alike (see #4835), + * we need to introduce an additonal tagging mechanism. + * + * To illustrate the problem, take the following dependency graph as an example + * + * a -> b -> c + * + * where B is excluded. Naively, we would want to group a and c together into a'. + * However, this would lead to a circular dependency between a' and c. + * + * Nevertheless, in the absence of b, or if b is not an excluded class, we'd + * want to perform the grouping to avoid unnecessary splitting. + * + * We achieve this by tracking an additional tag, representing the maximum + * number of hops from an excluded class (aka fine) to a non-excluded class + * (aka coarse) class for any path from an entrypoint to the given class. + * + * We then only permit grouping coarse classes with the same tag. This avoids + * the creation of cycles. + * + * The following is a proof that this strategy avoids cycles. + * + * Given + * + * G = (V, E), acyclic, V = F ∪ C, F ∩ C = ∅ + * the original dependency graph, + * F: set of fine classes, + * C: set of coarse classes + * + * t : V → ℕ (the maxExcludedHopCount tag) + * ∀ (v1, v2) ∈ E : t(v1) ≤ t(v2) + * ∀ (f, c) ∈ E : f ∈ F, c ∈ C ⇒ t(f) < t(c) + * + * Define + * + * G' = (V', E'), V' = F ∪ C' (the new grouped graph) + * + * C' = { n ∈ ℕ | ∃ c ∈ C : t(c) = n } + * + * E' = { (f1, f2) ∈ E | f1, f2 ∈ F } ∪ + * { (f, n) | f ∈ F, ∃ c ∈ C : (f, c) ∈ E : t(c) = n } ∪ + * { (n, f) | f ∈ F, ∃ c ∈ C : (c, f) ∈ E : t(c) = n } ∪ + * { (n, m) | n ≠ m, ∃ c1, c2 ∈ C : (c1, c2) ∈ E : t(c1) = n, t(c2) = m } + * + * t' : V' → ℕ: + * + * t'(f) = t(f) (if f ∈ F) + * t'(n) = n (if n ∈ C') + * + * Lemma 1 (unproven) + * + * ∀ (v1, v2) ∈ E' : t'(v1) ≤ t'(v2) + * + * Lemma 2 (unproven) + * + * ∀ (f, n) ∈ E' : f ∈ F, n ∈ C' : t'(f) < t'(n) + * + * Lemma 3 + * + * ∀ (n, m) ∈ E' : n,m ∈ C' ⇒ t'(n) < t'(m) + * + * Follows from Lemma 1 and (n, m) ∈ E' ⇒ n ≠ m (by definition). + * + * Theorem + * + * G' is acyclic + * + * Proof by contradiction. + * + * Assume ∃ p = x1, ..., xn (x1 = xn, n > 1, xi ∈ V) + * + * ∃ xi ∈ C' by contradiction: ∀ xi ∈ F ⇒ p is a cycle in G + * + * ∃ xi ∈ F by contradiction: ∀ xi ∈ C' ⇒ + * t'(xi) increases strictly monotonically (by Lemma 3), + * but x1 = xn ⇒ t'(x1) = t'(xn) + * + * Therefore, + * + * ∃ (xi, xj) ∈ p : xi ∈ F, xj ∈ C' + * + * Therefore (by Lemma 1) + * + * t'(x1) ≤ ... ≤ t'(xi) < t'(xj) ≤ ... ≤ t'(xn) ⇒ t'(x1) < t'(xn) + * + * But x1 = xn ⇒ t'(x1) = t'(xn), which is a contradiction. + * + * Therefore, G' is acyclic. */ private class Tagger(infos: ModuleAnalyzer.DependencyInfo, excludedClasses: scala.collection.Set[ClassName] = Set.empty) { @@ -84,40 +173,47 @@ private class Tagger(infos: ModuleAnalyzer.DependencyInfo, } } - private def tag(className: ClassName, pathRoot: ModuleID, pathSteps: List[ClassName]): Unit = { + private def tag(className: ClassName, pathRoot: ModuleID, pathSteps: List[ClassName], + excludedHopCount: Int, fromExcluded: Boolean): Unit = { + val isExcluded = excludedClasses.contains(className) + + val newExcludedHopCount = + if (fromExcluded && !isExcluded) excludedHopCount + 1 // hop from fine to coarse + else excludedHopCount + val updated = allPaths .getOrElseUpdate(className, new Paths) - .put(pathRoot, pathSteps) + .put(pathRoot, pathSteps, newExcludedHopCount) if (updated) { val classInfo = infos.classDependencies(className) classInfo .staticDependencies - .foreach(staticEdge(_, pathRoot, pathSteps)) + .foreach(staticEdge(_, pathRoot, pathSteps, newExcludedHopCount, fromExcluded = isExcluded)) classInfo .dynamicDependencies - .foreach(dynamicEdge(_, pathRoot, pathSteps)) + .foreach(dynamicEdge(_, pathRoot, pathSteps, newExcludedHopCount, fromExcluded = isExcluded)) } } - private def staticEdge(className: ClassName, pathRoot: ModuleID, pathSteps: List[ClassName]): Unit = { - if (excludedClasses.contains(className)) - // Force a "dynamic edge" to the external module. - dynamicEdge(className, pathRoot, pathSteps) - else - tag(className, pathRoot, pathSteps) + private def staticEdge(className: ClassName, pathRoot: ModuleID, pathSteps: List[ClassName], + excludedHopCount: Int, fromExcluded: Boolean): Unit = { + tag(className, pathRoot, pathSteps, excludedHopCount, fromExcluded) } - private def dynamicEdge(className: ClassName, pathRoot: ModuleID, pathSteps: List[ClassName]): Unit = - tag(className, pathRoot, pathSteps :+ className) + private def dynamicEdge(className: ClassName, pathRoot: ModuleID, pathSteps: List[ClassName], + excludedHopCount: Int, fromExcluded: Boolean): Unit = { + tag(className, pathRoot, pathSteps :+ className, excludedHopCount, fromExcluded) + } private def tagEntryPoints(): Unit = { for { (moduleID, deps) <- infos.publicModuleDependencies className <- deps } { - staticEdge(className, moduleID, Nil) + staticEdge(className, pathRoot = moduleID, pathSteps = Nil, + excludedHopCount = 0, fromExcluded = false) } } } @@ -131,25 +227,32 @@ private object Tagger { * - All non-empty, mutually prefix-free paths of dynamic import hops. */ private final class Paths { + private var maxExcludedHopCount = 0 private val direct = mutable.Set.empty[ModuleID] private val dynamic = mutable.Map.empty[ModuleID, DynamicPaths] - def put(pathRoot: ModuleID, pathSteps: List[ClassName]): Boolean = { - if (pathSteps.isEmpty) { + def put(pathRoot: ModuleID, pathSteps: List[ClassName], excludedHopCount: Int): Boolean = { + val hopCountsChanged = excludedHopCount > maxExcludedHopCount + + if (hopCountsChanged) + maxExcludedHopCount = excludedHopCount + + val stepsChanged = if (pathSteps.isEmpty) { direct.add(pathRoot) } else { dynamic .getOrElseUpdate(pathRoot, new DynamicPaths) .put(pathSteps) } + hopCountsChanged || stepsChanged } def moduleID(internalModuleIDPrefix: String): ModuleID = { - if (direct.size == 1 && dynamic.isEmpty) { + if (direct.size == 1 && dynamic.isEmpty && maxExcludedHopCount == 0) { /* Class is only used by a single public module. Put it there. * - * Note that we must not do this if there are any dynamic modules - * requiring this class. Otherwise, the dynamically loaded module + * Note that we must not do this if there are any dynamic or excluded + * modules requiring this class. Otherwise, the dynamically loaded module * will try to import the public module (but importing public modules is * forbidden). */ @@ -161,6 +264,10 @@ private object Tagger { */ val digestBuilder = new SHA1.DigestBuilder + // Excluded hop counts (exclude 0 for fast path in FewestModules mode) + if (maxExcludedHopCount > 0) + digestBuilder.update(intToBytes(maxExcludedHopCount)) + // Public modules using this. for (id <- direct.toList.sortBy(_.id)) digestBuilder.update(id.id.getBytes(StandardCharsets.UTF_8)) @@ -184,6 +291,13 @@ private object Tagger { } } + private def intToBytes(x: Int): Array[Byte] = { + val result = new Array[Byte](4) + val buf = ByteBuffer.wrap(result) + buf.putInt(x) + result + } + private def dynamicEnds: immutable.SortedSet[ClassName] = { val builder = immutable.SortedSet.newBuilder[ClassName] /* We ignore paths that originate in a module that imports this class diff --git a/linker/shared/src/test/scala/org/scalajs/linker/SmallModulesForSplittingTest.scala b/linker/shared/src/test/scala/org/scalajs/linker/SmallModulesForSplittingTest.scala index 2a8df79153..44390e4039 100644 --- a/linker/shared/src/test/scala/org/scalajs/linker/SmallModulesForSplittingTest.scala +++ b/linker/shared/src/test/scala/org/scalajs/linker/SmallModulesForSplittingTest.scala @@ -12,6 +12,7 @@ package org.scalajs.linker +import scala.collection.mutable import scala.concurrent._ import org.junit.Test @@ -25,6 +26,7 @@ import org.scalajs.ir.Types._ import org.scalajs.junit.async._ import org.scalajs.linker.interface._ +import org.scalajs.linker.standard.ModuleSet import org.scalajs.linker.testutils.LinkingUtils._ import org.scalajs.linker.testutils.TestIRBuilder._ @@ -92,4 +94,142 @@ class SmallModulesForSplittingTest { assertEquals(5, moduleSet.modules.size) } } + + @Test + def noCircularDepsThroughFineGrainedClasses_Issue4835(): AsyncResult = await { + /* Test a particular shape of dependencies that used to produce modules + * with cyclic dependencies. Because of that, it used to fail when creating + * the ModuleSet with + * "requirement failed: Must have exactly one root module". + * With that `require` statement disabled in the `ModuleSet` constructor, + * it used to then fail in `checkNoCyclicDependencies`. + */ + + val SMF = EMF.withNamespace(MemberNamespace.PublicStatic) + + def methodHolder(name: ClassName, methodName: String, body: Tree): ClassDef = { + classDef(name, + kind = ClassKind.Interface, + methods = List( + MethodDef(SMF, m(methodName, Nil, I), NON, Nil, IntType, Some(body))( + EOH.withNoinline(true), UNV) + )) + } + + def call(name: ClassName, methodName: String): Tree = + ApplyStatic(EAF, name, m(methodName, Nil, I), Nil)(IntType) + + val EntryPointsClass = ClassName("lib.EntryPoints") + val entryPointsClassDef = classDef( + EntryPointsClass, + superClass = Some(ObjectClass), + methods = List( + trivialCtor(EntryPointsClass) + ), + topLevelExportDefs = List( + TopLevelMethodExportDef("moda", + JSMethodDef(SMF, str("expa"), Nil, None, call("lib.A", "baz"))(EOH, UNV)), + TopLevelMethodExportDef("modb", + JSMethodDef(SMF, str("expb"), Nil, None, call("lib.A", "baz"))(EOH, UNV)) + ) + ) + + val classDefs = Seq( + entryPointsClassDef, + methodHolder("lib.A", "baz", BinaryOp(BinaryOp.Int_+, call("app.C", "foo"), call("lib.B", "bar"))), + methodHolder("lib.B", "bar", int(1)), + methodHolder("app.C", "foo", BinaryOp(BinaryOp.Int_+, call("lib.B", "bar"), int(1))) + ) + + val linkerConfig = StandardConfig() + .withModuleKind(ModuleKind.ESModule) + .withModuleSplitStyle(ModuleSplitStyle.SmallModulesFor(List("app"))) + .withSourceMap(false) + + for { + moduleSet <- linkToModuleSet(classDefs, Nil, config = linkerConfig) + } yield { + checkNoCyclicDependencies(moduleSet) + } + } + + @Test + def noCircularDepsThroughFineGrainedClasses2_Issue4835(): AsyncResult = await { + /* Another situation with potential circular dependencies, which was + * imagined while fixing #4835. + */ + + val SMF = EMF.withNamespace(MemberNamespace.PublicStatic) + + def methodHolder(name: ClassName, methodName: String, body: Tree): ClassDef = { + classDef(name, + kind = ClassKind.Interface, + methods = List( + MethodDef(SMF, m(methodName, Nil, I), NON, Nil, IntType, Some(body))( + EOH.withNoinline(true), UNV) + )) + } + + def call(name: ClassName, methodName: String): Tree = + ApplyStatic(EAF, name, m(methodName, Nil, I), Nil)(IntType) + + val EntryPointsClass = ClassName("entry.EntryPoints") + val entryPointsClassDef = classDef( + EntryPointsClass, + superClass = Some(ObjectClass), + methods = List( + trivialCtor(EntryPointsClass) + ), + topLevelExportDefs = List( + TopLevelMethodExportDef("moda", + JSMethodDef(SMF, str("expa"), Nil, None, call("app.A", "baz"))(EOH, UNV)), + TopLevelMethodExportDef("modb", + JSMethodDef(SMF, str("expb"), Nil, None, call("app.A", "baz"))(EOH, UNV)) + ) + ) + + val classDefs = Seq( + entryPointsClassDef, + methodHolder("app.A", "baz", call("lib.B", "bar")), + methodHolder("lib.B", "bar", call("app.C", "foo")), + methodHolder("app.C", "foo", call("lib.D", "bar")), + methodHolder("lib.D", "bar", int(1)) + ) + + val linkerConfig = StandardConfig() + .withModuleKind(ModuleKind.ESModule) + .withModuleSplitStyle(ModuleSplitStyle.SmallModulesFor(List("app"))) + .withSourceMap(false) + + for { + moduleSet <- linkToModuleSet(classDefs, Nil, config = linkerConfig) + } yield { + checkNoCyclicDependencies(moduleSet) + } + } + + private def checkNoCyclicDependencies(moduleSet: ModuleSet): Unit = { + val processedModuleIDs = mutable.Set.empty[ModuleSet.ModuleID] + var remainingModules = moduleSet.modules + + /* At each step of the loop, find all the modules in `remainingModules` for + * which all `internalDependencies` already belong to `processedModuleIDs`. + * Remove them from `remainingModules` and add them to `processedModuleIDs` + * instead. + * If no such module can be found, it means that there is a cycle within + * the remaining ones. + * When `remainingModules` is empty, we have shown that there is no cycle. + */ + while (remainingModules.nonEmpty) { + val (newRoots, nextRemaining) = remainingModules.partition { m => + m.internalDependencies.forall(processedModuleIDs.contains(_)) + } + if (newRoots.isEmpty) + fail("Found cycle in modules: " + remainingModules.map(_.id).mkString(", ")) + + for (root <- newRoots) + processedModuleIDs += root.id + remainingModules = nextRemaining + } + } } From b143936b09ffa7695f1d2135dee098729fb75aa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Doeraene?= Date: Sun, 9 Apr 2023 23:05:51 +0200 Subject: [PATCH 45/45] Version 1.13.1. --- ir/shared/src/main/scala/org/scalajs/ir/ScalaJSVersions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ir/shared/src/main/scala/org/scalajs/ir/ScalaJSVersions.scala b/ir/shared/src/main/scala/org/scalajs/ir/ScalaJSVersions.scala index e442a70641..42c0038528 100644 --- a/ir/shared/src/main/scala/org/scalajs/ir/ScalaJSVersions.scala +++ b/ir/shared/src/main/scala/org/scalajs/ir/ScalaJSVersions.scala @@ -17,7 +17,7 @@ import java.util.concurrent.ConcurrentHashMap import scala.util.matching.Regex object ScalaJSVersions extends VersionChecks( - current = "1.13.1-SNAPSHOT", + current = "1.13.1", binaryEmitted = "1.13" )