Skip to content

DO NOT MERGE: On disk cache #4963

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

DO NOT MERGE: On disk cache #4963

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fac3b52
Remove `force` parameter from OutputWriter methods
gzm0 Mar 10, 2024
788b7b5
Move module changed check to OutputWriter
gzm0 Mar 10, 2024
143a24d
No nested post transforms
gzm0 Mar 10, 2024
639d93b
wip
gzm0 Mar 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,15 +216,17 @@ final class ClosureLinkerBackend(config: LinkerBackendImpl.Config)
writer.write(footer)
}

protected def writeModuleWithoutSourceMap(moduleID: ModuleID, force: Boolean): Option[ByteBuffer] = {
protected def moduleChanged(moduleID: ModuleID): Boolean = true // no incremental support

protected def writeModuleWithoutSourceMap(moduleID: ModuleID, prevFile: Option[ByteBuffer]): ByteBuffer = {
val jsFileWriter = new ByteArrayOutputStream()
val jsFileStrWriter = new java.io.OutputStreamWriter(jsFileWriter, StandardCharsets.UTF_8)
writeCode(jsFileStrWriter)
jsFileStrWriter.flush()
Some(ByteBuffer.wrap(jsFileWriter.toByteArray()))
ByteBuffer.wrap(jsFileWriter.toByteArray())
}

protected def writeModuleWithSourceMap(moduleID: ModuleID, force: Boolean): Option[(ByteBuffer, ByteBuffer)] = {
protected def writeModuleWithSourceMap(moduleID: ModuleID, prevFile: Option[ByteBuffer]): (ByteBuffer, ByteBuffer) = {
val jsFileURI = OutputPatternsImpl.jsFileURI(config.outputPatterns, moduleID.id)
val sourceMapURI = OutputPatternsImpl.sourceMapURI(config.outputPatterns, moduleID.id)

Expand All @@ -241,7 +243,7 @@ final class ClosureLinkerBackend(config: LinkerBackendImpl.Config)
sourceMap.appendTo(sourceMapStrWriter, jsFileURI)
sourceMapStrWriter.flush()

Some((ByteBuffer.wrap(jsFileWriter.toByteArray()), ByteBuffer.wrap(sourceMapWriter.toByteArray())))
(ByteBuffer.wrap(jsFileWriter.toByteArray()), ByteBuffer.wrap(sourceMapWriter.toByteArray()))
}
}

Expand Down
146 changes: 47 additions & 99 deletions linker/shared/src/main/scala/org/scalajs/linker/backend/BasicLinkerBackend.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,19 +41,19 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config)
private[this] var totalModules = 0
private[this] val rewrittenModules = new AtomicInteger(0)

private[this] val bodyPrinter: BodyPrinter = {
if (config.minify) IdentityPostTransformerBasedBodyPrinter
else if (config.sourceMap) PrintedTreeWithSourceMapBodyPrinter
else PrintedTreeWithoutSourceMapBodyPrinter
private[this] val postTransformer: Emitter.PostTransformer = {
if (config.minify) Emitter.PostTransformer.Identity
else if (config.sourceMap) PostTransformerWithSourceMap
else PostTransformerWithoutSourceMap
}

private[this] val emitter: Emitter[bodyPrinter.TreeType] = {
private[this] val emitter: Emitter = {
val emitterConfig = Emitter.Config(config.commonConfig.coreSpec)
.withJSHeader(config.jsHeader)
.withInternalModulePattern(m => OutputPatternsImpl.moduleName(config.outputPatterns, m.id))
.withMinify(config.minify)

new Emitter(emitterConfig, bodyPrinter.postTransformer)
new Emitter(emitterConfig, postTransformer)
}

val symbolRequirements: SymbolRequirement = emitter.symbolRequirements
Expand Down Expand Up @@ -90,64 +90,63 @@ final class BasicLinkerBackend(config: LinkerBackendImpl.Config)
val allChanged = allChanged0 || config.minify

val writer = new OutputWriter(output, config, skipContentCheck) {
protected def writeModuleWithoutSourceMap(moduleID: ModuleID, force: Boolean): Option[ByteBuffer] = {
protected def moduleChanged(moduleID: ModuleID): Boolean =
allChanged || emitterResult.body(moduleID)._2

protected def writeModuleWithoutSourceMap(moduleID: ModuleID, prevFile: Option[ByteBuffer]): ByteBuffer = {
val cache = printedModuleSetCache.getModuleCache(moduleID)
val (trees, changed) = emitterResult.body(moduleID)
val (trees, _) = emitterResult.body(moduleID)

if (force || changed || allChanged) {
rewrittenModules.incrementAndGet()
rewrittenModules.incrementAndGet()

val jsFileWriter = new ByteArrayWriter(sizeHintFor(cache.getPreviousFinalJSFileSize()))
val jsFileWriter = new ByteArrayWriter(sizeHintFor(cache.getPreviousFinalJSFileSize()))

jsFileWriter.write(printedModuleSetCache.headerBytes)
jsFileWriter.writeASCIIString("'use strict';\n")
jsFileWriter.write(printedModuleSetCache.headerBytes)
jsFileWriter.writeASCIIString("'use strict';\n")

bodyPrinter.printWithoutSourceMap(trees, jsFileWriter)
val printer = new Printers.JSTreePrinter(prevFile, jsFileWriter)
for (tree <- trees)
printer.printStat(tree)

jsFileWriter.write(printedModuleSetCache.footerBytes)
jsFileWriter.write(printedModuleSetCache.footerBytes)

cache.recordFinalSizes(jsFileWriter.currentSize, 0)
Some(jsFileWriter.toByteBuffer())
} else {
None
}
cache.recordFinalSizes(jsFileWriter.currentSize, 0)
jsFileWriter.toByteBuffer()
}

protected def writeModuleWithSourceMap(moduleID: ModuleID, force: Boolean): Option[(ByteBuffer, ByteBuffer)] = {
protected def writeModuleWithSourceMap(moduleID: ModuleID, prevFile: Option[ByteBuffer]): (ByteBuffer, ByteBuffer) = {
val cache = printedModuleSetCache.getModuleCache(moduleID)
val (trees, changed) = emitterResult.body(moduleID)
val (trees, _) = emitterResult.body(moduleID)

if (force || changed || allChanged) {
rewrittenModules.incrementAndGet()
rewrittenModules.incrementAndGet()

val jsFileWriter = new ByteArrayWriter(sizeHintFor(cache.getPreviousFinalJSFileSize()))
val sourceMapWriter = new ByteArrayWriter(sizeHintFor(cache.getPreviousFinalSourceMapSize()))
val jsFileWriter = new ByteArrayWriter(sizeHintFor(cache.getPreviousFinalJSFileSize()))
val sourceMapWriter = new ByteArrayWriter(sizeHintFor(cache.getPreviousFinalSourceMapSize()))

val jsFileURI = OutputPatternsImpl.jsFileURI(config.outputPatterns, moduleID.id)
val sourceMapURI = OutputPatternsImpl.sourceMapURI(config.outputPatterns, moduleID.id)
val jsFileURI = OutputPatternsImpl.jsFileURI(config.outputPatterns, moduleID.id)
val sourceMapURI = OutputPatternsImpl.sourceMapURI(config.outputPatterns, moduleID.id)

val smWriter = new SourceMapWriter(sourceMapWriter, jsFileURI,
config.relativizeSourceMapBase)
val smWriter = new SourceMapWriter(sourceMapWriter, jsFileURI,
config.relativizeSourceMapBase)

jsFileWriter.write(printedModuleSetCache.headerBytes)
for (_ <- 0 until printedModuleSetCache.headerNewLineCount)
smWriter.nextLine()

jsFileWriter.writeASCIIString("'use strict';\n")
jsFileWriter.write(printedModuleSetCache.headerBytes)
for (_ <- 0 until printedModuleSetCache.headerNewLineCount)
smWriter.nextLine()

bodyPrinter.printWithSourceMap(trees, jsFileWriter, smWriter)
jsFileWriter.writeASCIIString("'use strict';\n")
smWriter.nextLine()

jsFileWriter.write(printedModuleSetCache.footerBytes)
jsFileWriter.write(("//# sourceMappingURL=" + sourceMapURI + "\n").getBytes(StandardCharsets.UTF_8))
val printer = new Printers.JSTreePrinterWithSourceMap(prevFile, jsFileWriter, smWriter, initIndent = 0)
for (tree <- trees)
printer.printStat(tree)

smWriter.complete()
jsFileWriter.write(printedModuleSetCache.footerBytes)
jsFileWriter.write(("//# sourceMappingURL=" + sourceMapURI + "\n").getBytes(StandardCharsets.UTF_8))

cache.recordFinalSizes(jsFileWriter.currentSize, sourceMapWriter.currentSize)
Some((jsFileWriter.toByteBuffer(), sourceMapWriter.toByteBuffer()))
} else {
None
}
smWriter.complete()

cache.recordFinalSizes(jsFileWriter.currentSize, sourceMapWriter.currentSize)
(jsFileWriter.toByteBuffer(), sourceMapWriter.toByteBuffer())
}

private def sizeHintFor(previousSize: Int): Int =
Expand Down Expand Up @@ -240,64 +239,13 @@ private object BasicLinkerBackend {
}
}

private abstract class BodyPrinter {
type TreeType >: Null <: js.Tree

val postTransformer: Emitter.PostTransformer[TreeType]

def printWithoutSourceMap(trees: List[TreeType], jsFileWriter: ByteArrayWriter): Unit
def printWithSourceMap(trees: List[TreeType], jsFileWriter: ByteArrayWriter, smWriter: SourceMapWriter): Unit
}

private object IdentityPostTransformerBasedBodyPrinter extends BodyPrinter {
type TreeType = js.Tree

val postTransformer: Emitter.PostTransformer[TreeType] = Emitter.PostTransformer.Identity

def printWithoutSourceMap(trees: List[TreeType], jsFileWriter: ByteArrayWriter): Unit = {
val printer = new Printers.JSTreePrinter(jsFileWriter)
for (tree <- trees)
printer.printStat(tree)
}

def printWithSourceMap(trees: List[TreeType], jsFileWriter: ByteArrayWriter, smWriter: SourceMapWriter): Unit = {
val printer = new Printers.JSTreePrinterWithSourceMap(jsFileWriter, smWriter, initIndent = 0)
for (tree <- trees)
printer.printStat(tree)
}
}

private abstract class PrintedTreeBasedBodyPrinter(
val postTransformer: Emitter.PostTransformer[js.PrintedTree]
) extends BodyPrinter {
type TreeType = js.PrintedTree

def printWithoutSourceMap(trees: List[TreeType], jsFileWriter: ByteArrayWriter): Unit = {
for (tree <- trees)
jsFileWriter.write(tree.jsCode)
}

def printWithSourceMap(trees: List[TreeType], jsFileWriter: ByteArrayWriter, smWriter: SourceMapWriter): Unit = {
for (tree <- trees) {
jsFileWriter.write(tree.jsCode)
smWriter.insertFragment(tree.sourceMapFragment)
}
}
}

private object PrintedTreeWithoutSourceMapBodyPrinter
extends PrintedTreeBasedBodyPrinter(PostTransformerWithoutSourceMap)

private object PrintedTreeWithSourceMapBodyPrinter
extends PrintedTreeBasedBodyPrinter(PostTransformerWithSourceMap)

private object PostTransformerWithoutSourceMap extends Emitter.PostTransformer[js.PrintedTree] {
private object PostTransformerWithoutSourceMap extends Emitter.PostTransformer {
def transformStats(trees: List[js.Tree], indent: Int): List[js.PrintedTree] = {
if (trees.isEmpty) {
Nil // Fast path
} else {
val jsCodeWriter = new ByteArrayWriter()
val printer = new Printers.JSTreePrinter(jsCodeWriter, indent)
val printer = new Printers.JSTreePrinter(None, jsCodeWriter, indent)

trees.foreach(printer.printStat(_))

Expand All @@ -306,14 +254,14 @@ private object BasicLinkerBackend {
}
}

private object PostTransformerWithSourceMap extends Emitter.PostTransformer[js.PrintedTree] {
private object PostTransformerWithSourceMap extends Emitter.PostTransformer {
def transformStats(trees: List[js.Tree], indent: Int): List[js.PrintedTree] = {
if (trees.isEmpty) {
Nil // Fast path
} else {
val jsCodeWriter = new ByteArrayWriter()
val smFragmentBuilder = new SourceMapWriter.FragmentBuilder()
val printer = new Printers.JSTreePrinterWithSourceMap(jsCodeWriter, smFragmentBuilder, indent)
val printer = new Printers.JSTreePrinterWithSourceMap(None, jsCodeWriter, smFragmentBuilder, indent)

trees.foreach(printer.printStat(_))
smFragmentBuilder.complete()
Expand Down
58 changes: 33 additions & 25 deletions linker/shared/src/main/scala/org/scalajs/linker/backend/OutputWriter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,11 @@ private[backend] abstract class OutputWriter(output: OutputDirectory,
private val outputImpl = OutputDirectoryImpl.fromOutputDirectory(output)
private val moduleKind = config.commonConfig.coreSpec.moduleKind

protected def writeModuleWithoutSourceMap(moduleID: ModuleID, force: Boolean): Option[ByteBuffer]
protected def moduleChanged(moduleID: ModuleID): Boolean

protected def writeModuleWithSourceMap(moduleID: ModuleID, force: Boolean): Option[(ByteBuffer, ByteBuffer)]
protected def writeModuleWithoutSourceMap(moduleID: ModuleID, prevFile: Option[ByteBuffer]): ByteBuffer

protected def writeModuleWithSourceMap(moduleID: ModuleID, prevFile: Option[ByteBuffer]): (ByteBuffer, ByteBuffer)

def write(moduleSet: ModuleSet)(implicit ec: ExecutionContext): Future[Report] = {
val ioThrottler = new IOThrottler(config.maxConcurrentWrites)
Expand Down Expand Up @@ -65,35 +67,41 @@ private[backend] abstract class OutputWriter(output: OutputDirectory,
implicit ec: ExecutionContext): Future[Report.Module] = {
val jsFileName = OutputPatternsImpl.jsFile(config.outputPatterns, moduleID.id)

val prevFileFuture =
if (existingFiles.contains(jsFileName)) outputImpl.readFull(jsFileName).map(Some(_))
else Future.successful(None)

if (config.sourceMap) {
val sourceMapFileName = OutputPatternsImpl.sourceMapFile(config.outputPatterns, moduleID.id)
val report = new ReportImpl.ModuleImpl(moduleID.id, jsFileName, Some(sourceMapFileName), moduleKind)
val force = !existingFiles.contains(jsFileName) || !existingFiles.contains(sourceMapFileName)

writeModuleWithSourceMap(moduleID, force) match {
case Some((code, sourceMap)) =>
for {
_ <- outputImpl.writeFull(jsFileName, code, skipContentCheck)
_ <- outputImpl.writeFull(sourceMapFileName, sourceMap, skipContentCheck)
} yield {
report
}
case None =>
Future.successful(report)

if (moduleChanged(moduleID)) {
for {
prevFile <- prevFileFuture
(code, sourceMap) = writeModuleWithSourceMap(moduleID, prevFile)

// TODO: We should not read the file again, but use the existing buffer to compare if a write is required.
_ <- outputImpl.writeFull(jsFileName, code, skipContentCheck)
_ <- outputImpl.writeFull(sourceMapFileName, sourceMap, skipContentCheck)
} yield {
report
}
} else {
Future.successful(report)
}
} else {
val report = new ReportImpl.ModuleImpl(moduleID.id, jsFileName, None, moduleKind)
val force = !existingFiles.contains(jsFileName)

writeModuleWithoutSourceMap(moduleID, force) match {
case Some(code) =>
for {
_ <- outputImpl.writeFull(jsFileName, code, skipContentCheck)
} yield {
report
}
case None =>
Future.successful(report)

if (moduleChanged(moduleID)) {
for {
prevFile <- prevFileFuture
code = writeModuleWithoutSourceMap(moduleID, prevFile)
_ <- outputImpl.writeFull(jsFileName, code, skipContentCheck)
} yield {
report
}
} else {
Future.successful(report)
}
}
}
Expand Down
Loading