From f711be2e9b79dded4635d3451d01995f3c546811 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 1 May 2025 20:19:07 +0100 Subject: [PATCH 1/2] feat: bulk insert cache entries to KV --- .changeset/wet-monkeys-look.md | 5 +++ packages/cloudflare/src/cli/args.ts | 5 +++ .../cloudflare/src/cli/commands/deploy.ts | 3 +- .../src/cli/commands/populate-cache.ts | 42 ++++++++++++------- .../cloudflare/src/cli/commands/preview.ts | 3 +- .../cloudflare/src/cli/commands/upload.ts | 3 +- 6 files changed, 43 insertions(+), 18 deletions(-) create mode 100644 .changeset/wet-monkeys-look.md diff --git a/.changeset/wet-monkeys-look.md b/.changeset/wet-monkeys-look.md new file mode 100644 index 00000000..eda465fb --- /dev/null +++ b/.changeset/wet-monkeys-look.md @@ -0,0 +1,5 @@ +--- +"@opennextjs/cloudflare": patch +--- + +feat: bulk insert cache entries to KV diff --git a/packages/cloudflare/src/cli/args.ts b/packages/cloudflare/src/cli/args.ts index 2a741a4a..292e3f7e 100644 --- a/packages/cloudflare/src/cli/args.ts +++ b/packages/cloudflare/src/cli/args.ts @@ -15,11 +15,13 @@ export type Arguments = ( | { command: "preview" | "deploy" | "upload"; passthroughArgs: string[]; + cacheChunkSize?: number; } | { command: "populateCache"; target: WranglerTarget; environment?: string; + cacheChunkSize?: number; } ) & { outputDir?: string }; @@ -30,6 +32,7 @@ export function getArgs(): Arguments { output: { type: "string", short: "o" }, noMinify: { type: "boolean", default: false }, skipWranglerConfigCheck: { type: "boolean", default: false }, + cacheChunkSize: { type: "string" }, }, allowPositionals: true, }); @@ -58,6 +61,7 @@ export function getArgs(): Arguments { command: positionals[0], outputDir, passthroughArgs, + ...(values.cacheChunkSize && { cacheChunkSize: Number(values.cacheChunkSize) }), }; case "populateCache": if (!isWranglerTarget(positionals[1])) { @@ -68,6 +72,7 @@ export function getArgs(): Arguments { outputDir, target: positionals[1], environment: getWranglerEnvironmentFlag(passthroughArgs), + ...(values.cacheChunkSize && { cacheChunkSize: Number(values.cacheChunkSize) }), }; default: throw new Error( diff --git a/packages/cloudflare/src/cli/commands/deploy.ts b/packages/cloudflare/src/cli/commands/deploy.ts index d5487b53..342ca89a 100644 --- a/packages/cloudflare/src/cli/commands/deploy.ts +++ b/packages/cloudflare/src/cli/commands/deploy.ts @@ -7,11 +7,12 @@ import { populateCache } from "./populate-cache.js"; export async function deploy( options: BuildOptions, config: OpenNextConfig, - deployOptions: { passthroughArgs: string[] } + deployOptions: { passthroughArgs: string[]; cacheChunkSize?: number } ) { await populateCache(options, config, { target: "remote", environment: getWranglerEnvironmentFlag(deployOptions.passthroughArgs), + cacheChunkSize: deployOptions.cacheChunkSize, }); runWrangler(options, ["deploy", ...deployOptions.passthroughArgs], { logging: "all" }); diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index 42f1704d..39a45d3d 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -1,4 +1,4 @@ -import { cpSync, existsSync } from "node:fs"; +import { cpSync, existsSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import path from "node:path"; import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; @@ -133,7 +133,7 @@ async function populateR2IncrementalCache( async function populateKVIncrementalCache( options: BuildOptions, - populateCacheOptions: { target: WranglerTarget; environment?: string } + populateCacheOptions: { target: WranglerTarget; environment?: string; cacheChunkSize?: number } ) { logger.info("\nPopulating KV incremental cache..."); @@ -147,24 +147,36 @@ async function populateKVIncrementalCache( const assets = getCacheAssets(options); - for (const { fullPath, key, buildId, isFetch } of tqdm(assets)) { - const cacheKey = computeCacheKey(key, { - prefix: proxy.env[KV_CACHE_PREFIX_ENV_NAME], - buildId, - cacheType: isFetch ? "fetch" : "cache", - }); + const chunkSize = populateCacheOptions.cacheChunkSize ?? 25; + const totalChunks = Math.ceil(assets.length / chunkSize); + + logger.info(`Inserting ${assets.length} assets to KV in chunks of ${chunkSize}`); + + for (const i of tqdm(Array.from({ length: totalChunks }, (_, i) => i))) { + const chunkPath = path.join(options.outputDir, "cloudflare", `cache-chunk-${i}.json`); + + const kvMapping = assets + .slice(i * chunkSize, (i + 1) * chunkSize) + .map(({ fullPath, key, buildId, isFetch }) => ({ + key: computeCacheKey(key, { + prefix: proxy.env[KV_CACHE_PREFIX_ENV_NAME], + buildId, + cacheType: isFetch ? "fetch" : "cache", + }), + value: readFileSync(fullPath, "utf8"), + })); + + writeFileSync(chunkPath, JSON.stringify(kvMapping)); runWrangler( options, - [ - "kv key put", - JSON.stringify(cacheKey), - `--binding ${JSON.stringify(KV_CACHE_BINDING_NAME)}`, - `--path ${JSON.stringify(fullPath)}`, - ], + ["kv bulk put", JSON.stringify(chunkPath), `--binding ${JSON.stringify(KV_CACHE_BINDING_NAME)}`], { ...populateCacheOptions, logging: "error" } ); + + rmSync(chunkPath); } + logger.info(`Successfully populated cache with ${assets.length} assets`); } @@ -209,7 +221,7 @@ function populateStaticAssetsIncrementalCache(options: BuildOptions) { export async function populateCache( options: BuildOptions, config: OpenNextConfig, - populateCacheOptions: { target: WranglerTarget; environment?: string } + populateCacheOptions: { target: WranglerTarget; environment?: string; cacheChunkSize?: number } ) { const { incrementalCache, tagCache } = config.default.override ?? {}; diff --git a/packages/cloudflare/src/cli/commands/preview.ts b/packages/cloudflare/src/cli/commands/preview.ts index 9f8e5baa..a9555bc6 100644 --- a/packages/cloudflare/src/cli/commands/preview.ts +++ b/packages/cloudflare/src/cli/commands/preview.ts @@ -7,11 +7,12 @@ import { populateCache } from "./populate-cache.js"; export async function preview( options: BuildOptions, config: OpenNextConfig, - previewOptions: { passthroughArgs: string[] } + previewOptions: { passthroughArgs: string[]; cacheChunkSize?: number } ) { await populateCache(options, config, { target: "local", environment: getWranglerEnvironmentFlag(previewOptions.passthroughArgs), + cacheChunkSize: previewOptions.cacheChunkSize, }); runWrangler(options, ["dev", ...previewOptions.passthroughArgs], { logging: "all" }); diff --git a/packages/cloudflare/src/cli/commands/upload.ts b/packages/cloudflare/src/cli/commands/upload.ts index ad9be4e4..9d35eb8f 100644 --- a/packages/cloudflare/src/cli/commands/upload.ts +++ b/packages/cloudflare/src/cli/commands/upload.ts @@ -7,11 +7,12 @@ import { populateCache } from "./populate-cache.js"; export async function upload( options: BuildOptions, config: OpenNextConfig, - uploadOptions: { passthroughArgs: string[] } + uploadOptions: { passthroughArgs: string[]; cacheChunkSize?: number } ) { await populateCache(options, config, { target: "remote", environment: getWranglerEnvironmentFlag(uploadOptions.passthroughArgs), + cacheChunkSize: uploadOptions.cacheChunkSize, }); runWrangler(options, ["versions upload", ...uploadOptions.passthroughArgs], { logging: "all" }); From f6627f01a865b744399159789dfed1d7241ecf0d Mon Sep 17 00:00:00 2001 From: James Anderson Date: Fri, 2 May 2025 10:22:16 +0100 Subject: [PATCH 2/2] Update packages/cloudflare/src/cli/commands/populate-cache.ts --- packages/cloudflare/src/cli/commands/populate-cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index 39a45d3d..2a0c89e2 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -147,7 +147,7 @@ async function populateKVIncrementalCache( const assets = getCacheAssets(options); - const chunkSize = populateCacheOptions.cacheChunkSize ?? 25; + const chunkSize = Math.max(1, populateCacheOptions.cacheChunkSize ?? 25); const totalChunks = Math.ceil(assets.length / chunkSize); logger.info(`Inserting ${assets.length} assets to KV in chunks of ${chunkSize}`);