Skip to content

Commit 7eb43b0

Browse files
authored
Merge pull request #3031 from github/cklin/overlay-upload-limit
Overlay: add database upload size limit
2 parents eef4c44 + eeeb083 commit 7eb43b0

File tree

3 files changed

+66
-1
lines changed

3 files changed

+66
-1
lines changed

lib/overlay-database-utils.js

Lines changed: 28 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

lib/overlay-database-utils.js.map

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/overlay-database-utils.ts

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,23 @@ export enum OverlayDatabaseMode {
1818

1919
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3";
2020

21+
/**
22+
* The maximum (uncompressed) size of the overlay base database that we will
23+
* upload. Actions Cache has an overall capacity of 10 GB, and the Actions Cache
24+
* client library uses zstd compression.
25+
*
26+
* Ideally we would apply a size limit to the compressed overlay-base database,
27+
* but we cannot do so because compression is handled transparently by the
28+
* Actions Cache client library. Instead we place a limit on the uncompressed
29+
* size of the overlay-base database.
30+
*
31+
* Assuming 2.5:1 compression ratio, the 6 GB limit on uncompressed data would
32+
* translate to a limit of around 2.4 GB after compression.
33+
*/
34+
const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6000;
35+
const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES =
36+
OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1_000_000;
37+
2138
/**
2239
* Writes a JSON file containing Git OIDs for all tracked files (represented
2340
* by path relative to the source root) under the source root. The file is
@@ -212,6 +229,26 @@ export async function uploadOverlayBaseDatabaseToCache(
212229
});
213230

214231
const dbLocation = config.dbLocation;
232+
233+
const databaseSizeBytes = await tryGetFolderBytes(dbLocation, logger);
234+
if (databaseSizeBytes === undefined) {
235+
logger.warning(
236+
"Failed to determine database size. " +
237+
"Skip uploading overlay-base database to cache.",
238+
);
239+
return false;
240+
}
241+
242+
if (databaseSizeBytes > OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES) {
243+
const databaseSizeMB = Math.round(databaseSizeBytes / 1_000_000);
244+
logger.warning(
245+
`Database size (${databaseSizeMB} MB) ` +
246+
`exceeds maximum upload size (${OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB} MB). ` +
247+
"Skip uploading overlay-base database to cache.",
248+
);
249+
return false;
250+
}
251+
215252
const codeQlVersion = (await codeql.getVersion()).version;
216253
const checkoutPath = getRequiredInput("checkout_path");
217254
const cacheKey = await generateCacheKey(config, codeQlVersion, checkoutPath);

0 commit comments

Comments
 (0)