Skip to content

Commit 2c1d6bf

Browse files
deepak1556deepak1556
committed
[v8] Allow for 4GB TypedArrays
https://chromium-review.googlesource.com/c/v8/v8/+/2249668
1 parent 207d8a3 commit 2c1d6bf

File tree

3 files changed

+253
-0
lines changed

3 files changed

+253
-0
lines changed

‎patches/node/.patches

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,5 @@ chore_sethostcleanupfinalizationgroupcallback_has_been_removed_from.patch
3838
fix_remove_bad_semicolon_outside_fn.patch
3939
fix_comment_out_incompatible_crypto_modules.patch
4040
test_account_for_non-node_basename.patch
41+
lib_src_switch_buffer_kmaxlength_to_size_t.patch
42+
update_tests_after_increasing_typed_array_size.patch

‎patches/node/lib_src_switch_buffer_kmaxlength_to_size_t.patch

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
2+
From: Ben Noordhuis <info@bnoordhuis.nl>
3+
Date: Sat, 18 Jan 2020 10:55:31 +0100
4+
Subject: lib,src: switch Buffer::kMaxLength to size_t
5+
MIME-Version: 1.0
6+
Content-Type: text/plain; charset=UTF-8
7+
Content-Transfer-Encoding: 8bit
8+
9+
Change the type of `Buffer::kMaxLength` to size_t because upcoming
10+
changes in V8 will allow typed arrays > 2 GB on 64 bits platforms.
11+
12+
Not all platforms handle file reads and writes > 2 GB though so keep
13+
enforcing the 2 GB typed array limit for I/O operations.
14+
15+
Fixes: https://github.com/nodejs/node/issues/31399
16+
Refs: https://github.com/libuv/libuv/pull/1501
17+
18+
PR-URL: https://github.com/nodejs/node/pull/31406
19+
Reviewed-By: Richard Lau <riclau@uk.ibm.com>
20+
Reviewed-By: David Carlier <devnexen@gmail.com>
21+
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>
22+
Reviewed-By: Luigi Pinca <luigipinca@gmail.com>
23+
Reviewed-By: Anna Henningsen <anna@addaleax.net>
24+
Reviewed-By: Rich Trott <rtrott@gmail.com>
25+
Reviewed-By: Tobias Nießen <tniessen@tnie.de>
26+
Reviewed-By: Shelley Vohr <codebytere@gmail.com>
27+
28+
diff --git a/lib/fs.js b/lib/fs.js
29+
index 4a19e3b6033711d8c77d1ac9ea7e0f2cd9742ce9..59a41fe62c68c05ab09ad280e4e9ecdb2ca23349 100644
30+
--- a/lib/fs.js
31+
+++ b/lib/fs.js
32+
@@ -24,6 +24,10 @@
33+
34+
'use strict';
35+
36+
+// Most platforms don't allow reads or writes >= 2 GB.
37+
+// See https://github.com/libuv/libuv/pull/1501.
38+
+const kIoMaxLength = 2 ** 31 - 1;
39+
+
40+
const {
41+
Map,
42+
MathMax,
43+
@@ -52,7 +56,7 @@ const {
44+
const pathModule = require('path');
45+
const { isArrayBufferView } = require('internal/util/types');
46+
const binding = internalBinding('fs');
47+
-const { Buffer, kMaxLength } = require('buffer');
48+
+const { Buffer } = require('buffer');
49+
const {
50+
codes: {
51+
ERR_FS_FILE_TOO_LARGE,
52+
@@ -274,7 +278,7 @@ function readFileAfterStat(err, stats) {
53+
54+
const size = context.size = isFileType(stats, S_IFREG) ? stats[8] : 0;
55+
56+
- if (size > kMaxLength) {
57+
+ if (size > kIoMaxLength) {
58+
err = new ERR_FS_FILE_TOO_LARGE(size);
59+
return context.close(err);
60+
}
61+
@@ -331,7 +335,7 @@ function tryCreateBuffer(size, fd, isUserFd) {
62+
let threw = true;
63+
let buffer;
64+
try {
65+
- if (size > kMaxLength) {
66+
+ if (size > kIoMaxLength) {
67+
throw new ERR_FS_FILE_TOO_LARGE(size);
68+
}
69+
buffer = Buffer.allocUnsafe(size);
70+
diff --git a/lib/internal/errors.js b/lib/internal/errors.js
71+
index 1e987cefb156df8e7a494bcb80547ae8e0ea649f..567d82c7bd12a7233481d80042d331afd7471674 100644
72+
--- a/lib/internal/errors.js
73+
+++ b/lib/internal/errors.js
74+
@@ -822,9 +822,7 @@ E('ERR_FALSY_VALUE_REJECTION', function(reason) {
75+
this.reason = reason;
76+
return 'Promise was rejected with falsy value';
77+
}, Error);
78+
-E('ERR_FS_FILE_TOO_LARGE', 'File size (%s) is greater than possible Buffer: ' +
79+
- `${kMaxLength} bytes`,
80+
- RangeError);
81+
+E('ERR_FS_FILE_TOO_LARGE', 'File size (%s) is greater than 2 GB', RangeError);
82+
E('ERR_FS_INVALID_SYMLINK_TYPE',
83+
'Symlink type must be one of "dir", "file", or "junction". Received "%s"',
84+
Error); // Switch to TypeError. The current implementation does not seem right
85+
diff --git a/lib/internal/fs/promises.js b/lib/internal/fs/promises.js
86+
index d653724474f314cd1c6bebe0a2d9285439d54928..98335fdc1027409a2f17ae50fba378f5d78b2cab 100644
87+
--- a/lib/internal/fs/promises.js
88+
+++ b/lib/internal/fs/promises.js
89+
@@ -1,5 +1,9 @@
90+
'use strict';
91+
92+
+// Most platforms don't allow reads or writes >= 2 GB.
93+
+// See https://github.com/libuv/libuv/pull/1501.
94+
+const kIoMaxLength = 2 ** 31 - 1;
95+
+
96+
const {
97+
MathMax,
98+
MathMin,
99+
@@ -15,7 +19,7 @@ const {
100+
S_IFREG
101+
} = internalBinding('constants').fs;
102+
const binding = internalBinding('fs');
103+
-const { Buffer, kMaxLength } = require('buffer');
104+
+const { Buffer } = require('buffer');
105+
const {
106+
ERR_FS_FILE_TOO_LARGE,
107+
ERR_INVALID_ARG_TYPE,
108+
@@ -166,7 +170,7 @@ async function readFileHandle(filehandle, options) {
109+
size = 0;
110+
}
111+
112+
- if (size > kMaxLength)
113+
+ if (size > kIoMaxLength)
114+
throw new ERR_FS_FILE_TOO_LARGE(size);
115+
116+
const chunks = [];
117+
diff --git a/src/node_buffer.cc b/src/node_buffer.cc
118+
index 59baa45413d500272d0e293ab06bfe4d24e5e0cb..4d1951b740240bff231b7f4c855beb5b73d076af 100644
119+
--- a/src/node_buffer.cc
120+
+++ b/src/node_buffer.cc
121+
@@ -62,6 +62,7 @@ using v8::Local;
122+
using v8::Maybe;
123+
using v8::MaybeLocal;
124+
using v8::Nothing;
125+
+using v8::Number;
126+
using v8::Object;
127+
using v8::String;
128+
using v8::Uint32;
129+
@@ -1161,7 +1162,7 @@ void Initialize(Local<Object> target,
130+
131+
target->Set(env->context(),
132+
FIXED_ONE_BYTE_STRING(env->isolate(), "kMaxLength"),
133+
- Integer::NewFromUnsigned(env->isolate(), kMaxLength)).Check();
134+
+ Number::New(env->isolate(), kMaxLength)).Check();
135+
136+
target->Set(env->context(),
137+
FIXED_ONE_BYTE_STRING(env->isolate(), "kStringMaxLength"),
138+
diff --git a/src/node_buffer.h b/src/node_buffer.h
139+
index 11010017ce0df8367b1992bd9df57117ff50454d..606a6f5caa3b11b6d2a9068ed2fd65800530a5eb 100644
140+
--- a/src/node_buffer.h
141+
+++ b/src/node_buffer.h
142+
@@ -29,7 +29,7 @@ namespace node {
143+
144+
namespace Buffer {
145+
146+
-static const unsigned int kMaxLength = v8::TypedArray::kMaxLength;
147+
+static const size_t kMaxLength = v8::TypedArray::kMaxLength;
148+
149+
typedef void (*FreeCallback)(char* data, void* hint);
150+
151+
diff --git a/test/parallel/test-fs-util-validateoffsetlengthwrite.js b/test/parallel/test-fs-util-validateoffsetlengthwrite.js
152+
index be6d8acea77efa5adc82a6bcaaa192167b510fb0..e2c583749d041d76da630bbbf6b46ac490076c56 100644
153+
--- a/test/parallel/test-fs-util-validateoffsetlengthwrite.js
154+
+++ b/test/parallel/test-fs-util-validateoffsetlengthwrite.js
155+
@@ -4,7 +4,10 @@
156+
require('../common');
157+
const assert = require('assert');
158+
const { validateOffsetLengthWrite } = require('internal/fs/utils');
159+
-const { kMaxLength } = require('buffer');
160+
+
161+
+// Most platforms don't allow reads or writes >= 2 GB.
162+
+// See https://github.com/libuv/libuv/pull/1501.
163+
+const kIoMaxLength = 2 ** 31 - 1;
164+
165+
// RangeError when offset > byteLength
166+
{
167+
@@ -22,27 +25,27 @@ const { kMaxLength } = require('buffer');
168+
);
169+
}
170+
171+
-// RangeError when byteLength > kMaxLength, and length > kMaxLength - offset .
172+
+// RangeError when byteLength > kIoMaxLength, and length > kIoMaxLength - offset .
173+
{
174+
- const offset = kMaxLength;
175+
+ const offset = kIoMaxLength;
176+
const length = 100;
177+
- const byteLength = kMaxLength + 1;
178+
+ const byteLength = kIoMaxLength + 1;
179+
assert.throws(
180+
() => validateOffsetLengthWrite(offset, length, byteLength),
181+
{
182+
code: 'ERR_OUT_OF_RANGE',
183+
name: 'RangeError',
184+
message: 'The value of "length" is out of range. ' +
185+
- `It must be <= ${kMaxLength - offset}. Received ${length}`
186+
+ `It must be <= ${kIoMaxLength - offset}. Received ${length}`
187+
}
188+
);
189+
}
190+
191+
-// RangeError when byteLength < kMaxLength, and length > byteLength - offset .
192+
+// RangeError when byteLength < kIoMaxLength, and length > byteLength - offset.
193+
{
194+
- const offset = kMaxLength - 150;
195+
+ const offset = kIoMaxLength - 150;
196+
const length = 200;
197+
- const byteLength = kMaxLength - 100;
198+
+ const byteLength = kIoMaxLength - 100;
199+
assert.throws(
200+
() => validateOffsetLengthWrite(offset, length, byteLength),
201+
{

‎patches/node/update_tests_after_increasing_typed_array_size.patch

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
2+
From: Kim-Anh Tran <kimanh@chromium.org>
3+
Date: Fri, 19 Jun 2020 11:29:45 +0200
4+
Subject: Update tests after increasing typed array size
5+
6+
Refs https://github.com/v8/node/pull/114
7+
8+
diff --git a/test/parallel/test-buffer-alloc.js b/test/parallel/test-buffer-alloc.js
9+
index 070a3803802e56ec7736791559ec7d7204625754..9f626c0683ae74ae984496c77d0b73c9b4c7b872 100644
10+
--- a/test/parallel/test-buffer-alloc.js
11+
+++ b/test/parallel/test-buffer-alloc.js
12+
@@ -8,8 +8,8 @@ const SlowBuffer = require('buffer').SlowBuffer;
13+
// Verify the maximum Uint8Array size. There is no concrete limit by spec. The
14+
// internal limits should be updated if this fails.
15+
assert.throws(
16+
- () => new Uint8Array(2 ** 32),
17+
- { message: 'Invalid typed array length: 4294967296' }
18+
+ () => new Uint8Array(2 ** 32 + 1),
19+
+ { message: 'Invalid typed array length: 4294967297' }
20+
);
21+
22+
const b = Buffer.allocUnsafe(1024);
23+
diff --git a/test/parallel/test-buffer-over-max-length.js b/test/parallel/test-buffer-over-max-length.js
24+
index b04f89aae0344c008dff4c737aac96f53d6f7e17..ddbd93ddb088deb53304cb06d8fcdc59214dbc31 100644
25+
--- a/test/parallel/test-buffer-over-max-length.js
26+
+++ b/test/parallel/test-buffer-over-max-length.js
27+
@@ -12,11 +12,11 @@ const bufferMaxSizeMsg = {
28+
message: /^The value "[^"]*" is invalid for option "size"$/
29+
};
30+
31+
-assert.throws(() => Buffer((-1 >>> 0) + 1), bufferMaxSizeMsg);
32+
-assert.throws(() => SlowBuffer((-1 >>> 0) + 1), bufferMaxSizeMsg);
33+
-assert.throws(() => Buffer.alloc((-1 >>> 0) + 1), bufferMaxSizeMsg);
34+
-assert.throws(() => Buffer.allocUnsafe((-1 >>> 0) + 1), bufferMaxSizeMsg);
35+
-assert.throws(() => Buffer.allocUnsafeSlow((-1 >>> 0) + 1), bufferMaxSizeMsg);
36+
+assert.throws(() => Buffer((-1 >>> 0) + 2), bufferMaxSizeMsg);
37+
+assert.throws(() => SlowBuffer((-1 >>> 0) + 2), bufferMaxSizeMsg);
38+
+assert.throws(() => Buffer.alloc((-1 >>> 0) + 2), bufferMaxSizeMsg);
39+
+assert.throws(() => Buffer.allocUnsafe((-1 >>> 0) + 2), bufferMaxSizeMsg);
40+
+assert.throws(() => Buffer.allocUnsafeSlow((-1 >>> 0) + 2), bufferMaxSizeMsg);
41+
42+
assert.throws(() => Buffer(kMaxLength + 1), bufferMaxSizeMsg);
43+
assert.throws(() => SlowBuffer(kMaxLength + 1), bufferMaxSizeMsg);
44+
@@ -25,5 +25,5 @@ assert.throws(() => Buffer.allocUnsafe(kMaxLength + 1), bufferMaxSizeMsg);
45+
assert.throws(() => Buffer.allocUnsafeSlow(kMaxLength + 1), bufferMaxSizeMsg);
46+
47+
// issue GH-4331
48+
-assert.throws(() => Buffer.allocUnsafe(0x100000000), bufferMaxSizeMsg);
49+
+assert.throws(() => Buffer.allocUnsafe(0x100000001), bufferMaxSizeMsg);
50+
assert.throws(() => Buffer.allocUnsafe(0xFFFFFFFFF), bufferMaxSizeMsg);

0 commit comments

Comments
 (0)