From ff5c84d4af5fc35a67485e5f54ca675196aafb09 Mon Sep 17 00:00:00 2001 From: alonre24 Date: Sun, 12 Nov 2023 10:17:42 +0200 Subject: [PATCH 001/112] upload snapshots for integration branches as well [MOD-6102] (#1132) --- .circleci/config.yml | 20 ++++++++++++-------- sbin/upload-artifacts | 10 +++++++++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9aeb0877a..a7964af65 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -330,13 +330,15 @@ jobs: - early-returns - build-steps - test-steps + - persist-artifacts - run: name: Upload artifacts to S3 command: | - if [[ -n $CIRCLE_BRANCH && "<>" == "yes" ]]; then + if [[ -n $CIRCLE_BRANCH ]]; then make upload-artifacts SHOW=1 + else + make upload-release SHOW=1 fi - - persist-artifacts build-macos-m1: macos: @@ -350,13 +352,15 @@ jobs: - early-returns - build-steps - test-steps + - persist-artifacts - run: name: Upload artifacts to S3 command: | - if [[ -n $CIRCLE_BRANCH && "<>" == "yes" ]]; then + if [[ -n $CIRCLE_BRANCH ]]; then make upload-artifacts SHOW=1 VERBOSE=1 + else + make upload-release SHOW=1 fi - - persist-artifacts coverage: docker: @@ -410,10 +414,10 @@ jobs: command: | mkdir -p bin ln -s ~/workspace/artifacts bin/artifacts - if [[ -n $CIRCLE_TAG && "<>" != 1 ]]; then + if [[ -n $CIRCLE_TAG ]]; then make upload-release SHOW=1 else - make upload-release SHOW=1 STAGING=1 + make upload-artifacts SHOW=1 fi release-qa-tests: @@ -527,11 +531,11 @@ workflows: parameters: platform: [jammy, focal, bionic] - build-macos-x64: - <<: *on-version-tags + <<: *on-integ-and-version-tags context: common - build-macos-m1: context: common - <<: *on-version-tags + <<: *on-integ-and-version-tags - coverage: <<: *always - sanitize: diff --git a/sbin/upload-artifacts b/sbin/upload-artifacts index a682c884c..d303d1045 100755 --- a/sbin/upload-artifacts +++ b/sbin/upload-artifacts @@ -34,6 +34,7 @@ fi ARCH=$($READIES/bin/platform --arch) [[ $ARCH == x64 ]] && ARCH="x86_64" +[[ $ARCH == arm64v8 ]] && ARCH="aarch64" OS=$($READIES/bin/platform --os) [[ $OS == linux ]] && OS="Linux" @@ -51,7 +52,14 @@ OS=$($READIES/bin/platform --os) [[ $OSNICK == rocky8 ]] && OSNICK=rhel8 [[ $OSNICK == rocky9 ]] && OSNICK=rhel9 -[[ $OSNICK == bigsur ]] && OSNICK=catalina +if [[ $OS == macos ]]; then + # as we don't build on macOS for every platform, we converge to a least common denominator + if [[ $ARCH == x86_64 ]]; then + OSNICK=catalina # to be aligned with the rest of the modules in redis stack + else + [[ $OSNICK == ventura ]] && OSNICK=monterey + fi +fi PLATFORM="$OS-$OSNICK-$ARCH" From c9dc9fbc668af340f4b8cb1360c2bb58327cdbf7 Mon Sep 17 00:00:00 2001 From: alon Date: Sun, 12 Nov 2023 17:00:14 +0200 Subject: [PATCH 002/112] test to see if uploading to staging works --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a7964af65..1fca023c6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -519,23 +519,23 @@ workflows: name: build <<: *not-on-integ-branch - build-platforms: - <<: *on-integ-and-version-tags + <<: *always context: common matrix: parameters: platform: [jammy, focal, bionic, centos7, rocky8, rocky9, bullseye, amzn2] - build-arm-platforms: - <<: *on-integ-and-version-tags + <<: *always context: common matrix: parameters: platform: [jammy, focal, bionic] - build-macos-x64: - <<: *on-integ-and-version-tags + <<: *always context: common - build-macos-m1: context: common - <<: *on-integ-and-version-tags + <<: *always - coverage: <<: *always - sanitize: @@ -546,7 +546,7 @@ workflows: san-type: [address] - upload-artifacts: name: upload-artifacts-to-staging-lab - <<: *on-integ-branch + <<: *always staging-lab: "1" context: common requires: From 2253633cc709c753f516d018e5a0578700bddc06 Mon Sep 17 00:00:00 2001 From: alon Date: Sun, 12 Nov 2023 17:38:36 +0200 Subject: [PATCH 003/112] persist artifacts form snapshots dir --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1fca023c6..12452be22 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -132,6 +132,7 @@ commands: - artifacts/*.zip - artifacts/*.tgz - artifacts/*.tar + - artifacts/snapshots/* build-steps: parameters: From c14ddbdc87596a432913336ba4d36b96a2631b01 Mon Sep 17 00:00:00 2001 From: alon Date: Sun, 12 Nov 2023 18:35:43 +0200 Subject: [PATCH 004/112] persist artifacts form snapshots dir --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 12452be22..94cdbdcc7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -520,23 +520,23 @@ workflows: name: build <<: *not-on-integ-branch - build-platforms: - <<: *always + <<: *on-integ-and-version-tags context: common matrix: parameters: platform: [jammy, focal, bionic, centos7, rocky8, rocky9, bullseye, amzn2] - build-arm-platforms: - <<: *always + <<: *on-integ-and-version-tags context: common matrix: parameters: platform: [jammy, focal, bionic] - build-macos-x64: - <<: *always + <<: *on-integ-and-version-tags context: common - build-macos-m1: context: common - <<: *always + <<: *on-integ-and-version-tags - coverage: <<: *always - sanitize: @@ -547,7 +547,7 @@ workflows: san-type: [address] - upload-artifacts: name: upload-artifacts-to-staging-lab - <<: *always + <<: *on-integ-branch staging-lab: "1" context: common requires: From bf0f8d596f6e96e395e37937ac9c4e39276a980c Mon Sep 17 00:00:00 2001 From: nafraf Date: Thu, 23 Nov 2023 09:13:56 -0500 Subject: [PATCH 005/112] Mariner2: Fix upload dir (#1142) * Mariner2: Fix upload dir * Fix Upload artifacts --- .github/workflows/mariner2.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mariner2.yml b/.github/workflows/mariner2.yml index d93c40353..ecec78652 100644 --- a/.github/workflows/mariner2.yml +++ b/.github/workflows/mariner2.yml @@ -52,7 +52,8 @@ jobs: aws-region: "us-east-1" - name: Upload artifacts to S3 - staging run: | - make upload-artifacts SHOW=1 STAGING=1 VERBOSE=1 + make upload-artifacts SHOW=1 VERBOSE=1 + make upload-release SHOW=1 STAGING=1 VERBOSE=1 - name: Upload artifacts to S3 - release # todo: trigger this manually instead if: ${{ github.ref != 'refs/heads/master' }} - run: make upload-artifacts SHOW=1 VERBOSE=1 + run: make upload-release SHOW=1 VERBOSE=1 From 1e0042be0e455497f9a1c5a168051c0bfe42024b Mon Sep 17 00:00:00 2001 From: Lior Kogan Date: Mon, 27 Nov 2023 22:09:49 +0200 Subject: [PATCH 006/112] Update json.mget.md --- docs/commands/json.mget.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/commands/json.mget.md b/docs/commands/json.mget.md index 9f2fb5682..b266b3cbd 100644 --- a/docs/commands/json.mget.md +++ b/docs/commands/json.mget.md @@ -13,7 +13,7 @@ is key to parse. Returns `null` for nonexistent keys.
path -is JSONPath to specify. Default is root `$`. Returns `null` for nonexistent paths. +is JSONPath to specify. Returns `null` for nonexistent paths.
From c4282caa710f16aa3b0ec257fda94b38f7ca5136 Mon Sep 17 00:00:00 2001 From: efeldblum <109016390+ephraimfeldblum@users.noreply.github.com> Date: Wed, 29 Nov 2023 17:46:52 +0200 Subject: [PATCH 007/112] MOD-6156: Revert change to update default path to '$' (#1143) --- redis_json/src/c_api.rs | 2 +- redis_json/src/commands.rs | 57 +++++++++++++------------------------- tests/pytest/test_resp3.py | 24 ++++++++++------ 3 files changed, 37 insertions(+), 46 deletions(-) diff --git a/redis_json/src/c_api.rs b/redis_json/src/c_api.rs index 8689b9063..14b2599f5 100644 --- a/redis_json/src/c_api.rs +++ b/redis_json/src/c_api.rs @@ -526,7 +526,7 @@ macro_rules! redis_json_module_export_shared_api { #[no_mangle] pub extern "C" fn JSONAPI_pathFree(json_path: *mut c_void) { - unsafe { Box::from_raw(json_path.cast::()) }; + unsafe { drop(Box::from_raw(json_path.cast::())) }; } #[no_mangle] diff --git a/redis_json/src/commands.rs b/redis_json/src/commands.rs index aedd397b0..51f5f167f 100644 --- a/redis_json/src/commands.rs +++ b/redis_json/src/commands.rs @@ -27,8 +27,8 @@ use itertools::FoldWhile::{Continue, Done}; use itertools::{EitherOrBoth, Itertools}; use serde::{Serialize, Serializer}; -const JSON_ROOT_PATH: &str = "$"; const JSON_ROOT_PATH_LEGACY: &str = "."; +const JSON_ROOT_PATH: &str = "$"; const CMD_ARG_NOESCAPE: &str = "NOESCAPE"; const CMD_ARG_INDENT: &str = "INDENT"; const CMD_ARG_NEWLINE: &str = "NEWLINE"; @@ -86,12 +86,8 @@ fn is_resp3(ctx: &Context) -> bool { } /// Returns the deault path for the given RESP version -fn default_path(ctx: &Context) -> &str { - if is_resp3(ctx) { - JSON_ROOT_PATH - } else { - JSON_ROOT_PATH_LEGACY - } +const fn default_path() -> &'static str { + JSON_ROOT_PATH_LEGACY } /// @@ -146,7 +142,7 @@ pub fn json_get(manager: M, ctx: &Context, args: Vec) - // path is optional -> no path found we use legacy root "." if paths.is_empty() { - paths.push(Path::new(default_path(ctx))); + paths.push(Path::new(default_path())); } let key = manager.open_key_read(ctx, &key)?; @@ -563,7 +559,7 @@ pub fn json_del(manager: M, ctx: &Context, args: Vec) - let key = args.next_arg()?; let path = match args.next() { - None => Path::new(default_path(ctx)), + None => Path::new(default_path()), Some(s) => Path::new(s.try_as_str()?), }; @@ -613,8 +609,6 @@ pub fn json_mget(manager: M, ctx: &Context, args: Vec) return Err(RedisError::WrongArity); } - let is_legacy = path.is_legacy(); - let results: Result, RedisError> = keys .iter() .map(|key| { @@ -624,16 +618,12 @@ pub fn json_mget(manager: M, ctx: &Context, args: Vec) json_key.get_value().map_or(Ok(RedisValue::Null), |value| { value.map_or(Ok(RedisValue::Null), |doc| { let key_value = KeyValue::new(doc); - if format_options.is_resp3_reply() { - Ok(key_value.to_resp3_path(&path, &format_options)) + let res = if !path.is_legacy() { + key_value.to_string_multi(path.get_path(), &format_options) } else { - let res = if !is_legacy { - key_value.to_string_multi(path.get_path(), &format_options) - } else { - key_value.to_string_single(path.get_path(), &format_options) - }; - Ok(res.map_or(RedisValue::Null, |v| v.into())) - } + key_value.to_string_single(path.get_path(), &format_options) + }; + Ok(res.map_or(RedisValue::Null, |v| v.into())) }) }) }) @@ -649,7 +639,7 @@ pub fn json_mget(manager: M, ctx: &Context, args: Vec) pub fn json_type(manager: M, ctx: &Context, args: Vec) -> RedisResult { let mut args = args.into_iter().skip(1); let key = args.next_arg()?; - let path = Path::new(args.next_str().unwrap_or(default_path(ctx))); + let path = Path::new(args.next_str().unwrap_or(default_path())); let key = manager.open_key_read(ctx, &key)?; @@ -956,7 +946,7 @@ pub fn json_str_append( path = Path::new(path_or_json); json = val.try_as_str()?; } else { - path = Path::new(default_path(ctx)); + path = Path::new(default_path()); json = path_or_json; } @@ -1035,7 +1025,7 @@ where pub fn json_str_len(manager: M, ctx: &Context, args: Vec) -> RedisResult { let mut args = args.into_iter().skip(1); let key = args.next_arg()?; - let path = Path::new(args.next_str().unwrap_or(default_path(ctx))); + let path = Path::new(args.next_str().unwrap_or(default_path())); let key = manager.open_key_read(ctx, &key)?; @@ -1332,7 +1322,7 @@ where pub fn json_arr_len(manager: M, ctx: &Context, args: Vec) -> RedisResult { let mut args = args.into_iter().skip(1); let key = args.next_arg()?; - let path = Path::new(args.next_str().unwrap_or(default_path(ctx))); + let path = Path::new(args.next_str().unwrap_or(default_path())); let is_legacy = path.is_legacy(); let key = manager.open_key_read(ctx, &key)?; let root = match key.get_value()? { @@ -1417,14 +1407,7 @@ pub fn json_arr_pop(manager: M, ctx: &Context, args: Vec { - if format_options.is_resp3_reply() { - (Path::new(JSON_ROOT_PATH), i64::MAX) - } else { - // Legacy behavior for backward compatibility - (Path::new(JSON_ROOT_PATH_LEGACY), i64::MAX) - } - } + None => (Path::new(default_path()), i64::MAX), Some(s) => { let path = Path::new(s.try_as_str()?); let index = args.next_i64().unwrap_or(-1); @@ -1602,7 +1585,7 @@ where pub fn json_obj_keys(manager: M, ctx: &Context, args: Vec) -> RedisResult { let mut args = args.into_iter().skip(1); let key = args.next_arg()?; - let path = Path::new(args.next_str().unwrap_or(default_path(ctx))); + let path = Path::new(args.next_str().unwrap_or(default_path())); let mut key = manager.open_key_read(ctx, &key)?; if path.is_legacy() { @@ -1661,7 +1644,7 @@ pub fn json_obj_len(manager: M, ctx: &Context, args: Vec(manager: M, ctx: &Context, args: Vec) )?; let paths = if paths.is_empty() { - vec![Path::new(JSON_ROOT_PATH)] + vec![Path::new(default_path())] } else { paths }; @@ -1767,7 +1750,7 @@ pub fn json_debug(manager: M, ctx: &Context, args: Vec) match args.next_str()?.to_uppercase().as_str() { "MEMORY" => { let key = args.next_arg()?; - let path = Path::new(args.next_str().unwrap_or(default_path(ctx))); + let path = Path::new(args.next_str().unwrap_or(default_path())); let key = manager.open_key_read(ctx, &key)?; if path.is_legacy() { @@ -1811,7 +1794,7 @@ pub fn json_resp(manager: M, ctx: &Context, args: Vec) let key = args.next_arg()?; let path = match args.next() { - None => Path::new(default_path(ctx)), + None => Path::new(default_path()), Some(s) => Path::new(s.try_as_str()?), }; diff --git a/tests/pytest/test_resp3.py b/tests/pytest/test_resp3.py index 4c6a360fa..a27bf3cf2 100644 --- a/tests/pytest/test_resp3.py +++ b/tests/pytest/test_resp3.py @@ -321,20 +321,28 @@ def test_resp_default_path(self): # Test JSON.X commands on object type when default path is used r.assertTrue(r.execute_command('JSON.SET', 'test_resp3', '$', '{"a":[{"b":2},{"g":[1,2]},3]}')) r.assertEqual(r.execute_command('JSON.GET', 'test_resp3', 'FORMAT', 'EXPAND'), [[{"a":[{"b":2},{"g":[1,2]},3]}]]) - r.assertEqual(json.loads(r.execute_command('JSON.GET', 'test_resp3')), [{"a":[{"b":2},{"g":[1,2]},3]}]) - r.assertEqual(r.execute_command('JSON.OBJKEYS', 'test_resp3'), [['a']]) - r.assertEqual(r.execute_command('JSON.OBJLEN', 'test_resp3'), [1]) - r.assertEqual(r.execute_command('JSON.TYPE', 'test_resp3'), [['object']]) - r.assertEqual(r.execute_command('JSON.DEBUG', 'MEMORY', 'test_resp3'), [507]) + r.assertEqual(json.loads(r.execute_command('JSON.GET', 'test_resp3')), {"a":[{"b":2},{"g":[1,2]},3]}) + r.assertEqual(r.execute_command('JSON.OBJKEYS', 'test_resp3'), ['a']) + r.assertEqual(r.execute_command('JSON.OBJLEN', 'test_resp3'), 1) + r.assertEqual(r.execute_command('JSON.TYPE', 'test_resp3'), ['object']) + r.assertEqual(r.execute_command('JSON.DEBUG', 'MEMORY', 'test_resp3'), 507) r.assertEqual(r.execute_command('JSON.DEL', 'test_resp3'), 1) # Test JSON.strX commands on object type when default path is used - r.assertTrue(r.execute_command('JSON.SET', 'test_resp3_str', '$', '"test_resp3_str"')) - r.assertEqual(r.execute_command('JSON.STRLEN', 'test_resp3_str'), [14]) + string = 'test_resp3_str' + length = len(string) + r.assertTrue(r.execute_command('JSON.SET', 'test_resp3_str', '$', fr'"{string}"')) + r.assertEqual(r.execute_command('JSON.STRLEN', 'test_resp3_str'), length) + string = '_append' + length = length + len(string) + r.assertTrue(r.execute_command('JSON.STRAPPEND', 'test_resp3_str', fr'"{string}"')) + r.assertEqual(r.execute_command('JSON.STRLEN', 'test_resp3_str'), length) # Test JSON.arrX commands on object type when default path is used r.assertTrue(r.execute_command('JSON.SET', 'test_resp3_arr', '$', '[true, 1, "dud"]')) - r.assertEqual(r.execute_command('JSON.ARRLEN', 'test_resp3_arr'), [3]) + r.assertEqual(r.execute_command('JSON.ARRLEN', 'test_resp3_arr'), 3) + r.assertEqual(r.executeCommand('JSON.ARRPOP', 'test_resp3_arr'), '"dud"') + r.assertEqual(r.execute_command('JSON.ARRLEN', 'test_resp3_arr'), 2) def test_fail_with_resp2(): r = Env(protocol=2) From d6e7623af88d69935eb439082ee8df1b9235445a Mon Sep 17 00:00:00 2001 From: filipe oliveira Date: Fri, 8 Dec 2023 11:31:53 +0000 Subject: [PATCH 008/112] Enabled SPOT Instance deployments on CI benchmarks (#1141) --- tests/benchmarks/defaults.yml | 6 ++++++ tests/benchmarks/json_arrappend_geojson.yml | 4 +--- ...t_ResultSet.totalResultsAvailable_jsonsl-yahoo2_json.yml | 4 +--- tests/benchmarks/json_get_[0]_jsonsl-1.yml | 4 +--- tests/benchmarks/json_get_[7]_jsonsl-1.yml | 4 +--- tests/benchmarks/json_get_[8].zero_jsonsl-1.yml | 4 +--- ..._[web-app].servlet[0][servlet-name]_json-parser-0000.yml | 4 +--- .../json_get_[web-app].servlet[0]_json-parser-0000.yml | 4 +--- .../json_get_[web-app].servlet_json-parser-0000.yml | 4 +--- .../benchmarks/json_get_array_of_docs[1]_pass_100_json.yml | 4 +--- .../json_get_array_of_docs[1]sclr_pass_100_json.yml | 4 +--- tests/benchmarks/json_get_array_of_docs_pass_100_json.yml | 4 +--- tests/benchmarks/json_get_fulldoc_json-parser-0000.yml | 4 +--- tests/benchmarks/json_get_fulldoc_jsonsl-1.yml | 4 +--- tests/benchmarks/json_get_fulldoc_jsonsl-yahoo2_json.yml | 4 +--- tests/benchmarks/json_get_fulldoc_jsonsl-yelp_json.yml | 4 +--- tests/benchmarks/json_get_fulldoc_pass_100_json.yml | 4 +--- tests/benchmarks/json_get_key_empty.yml | 4 +--- tests/benchmarks/json_get_message.code_jsonsl-yelp_json.yml | 4 +--- tests/benchmarks/json_get_sclr_pass_100_json.yml | 4 +--- tests/benchmarks/json_get_sub_doc.sclr_pass_100_json.yml | 4 +--- tests/benchmarks/json_get_sub_doc_pass_100_json.yml | 4 +--- tests/benchmarks/json_numincrby_num_1.yml | 4 +--- tests/benchmarks/json_nummultby_num_2.yml | 4 +--- .../json_recursive_descent_with_filter_uid_issue674.yml | 4 +--- ...ResultSet.totalResultsAvailable_1_jsonsl-yahoo2_json.yml | 4 +--- tests/benchmarks/json_set_[0]foo_jsonsl-1.yml | 4 +--- ...b-app].servlet[0][servlet-name]_bar_json-parser-0000.yml | 4 +--- .../json_set_fulldoc_api_replies_q1_google_autocomplete.yml | 4 +--- .../json_set_fulldoc_api_replies_q2_gmaps_areatraffic.yml | 4 +--- .../json_set_fulldoc_api_replies_q3_gmaps_passiveassist.yml | 4 +--- .../json_set_fulldoc_api_replies_q4_gmaps_assist.yml | 4 +--- .../json_set_fulldoc_api_replies_q5_gmaps_place.yml | 4 +--- tests/benchmarks/json_set_fulldoc_pass-json-parser-0000.yml | 4 +--- tests/benchmarks/json_set_fulldoc_pass_100_json.yml | 4 +--- tests/benchmarks/json_set_fulldoc_yahoo2.yml | 4 +--- tests/benchmarks/json_set_key_empty.yml | 4 +--- .../benchmarks/json_set_message.code_1_jsonsl-yelp_json.yml | 4 +--- tests/benchmarks/json_set_num_0.yml | 4 +--- tests/benchmarks/json_set_sclr_1_pass_100_json.yml | 4 +--- tests/benchmarks/json_set_sclr_pass_100_json.yml | 4 +--- tests/benchmarks/json_vs_hashes_hset_key_simple.yml | 4 +--- tests/benchmarks/json_vs_hashes_json.set_key_simple.yml | 4 +--- 43 files changed, 48 insertions(+), 126 deletions(-) diff --git a/tests/benchmarks/defaults.yml b/tests/benchmarks/defaults.yml index a2d2bbd19..b52de40f2 100644 --- a/tests/benchmarks/defaults.yml +++ b/tests/benchmarks/defaults.yml @@ -1,4 +1,10 @@ version: 0.2 + +remote: + - type: oss-standalone + - setup: redisearch-m5d + - spot_instance: oss-redisearch-m5-spot-instances + exporter: redistimeseries: break_by: diff --git a/tests/benchmarks/json_arrappend_geojson.yml b/tests/benchmarks/json_arrappend_geojson.yml index 45e38832f..0c05ca729 100644 --- a/tests/benchmarks/json_arrappend_geojson.yml +++ b/tests/benchmarks/json_arrappend_geojson.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_arrappend_geojson" description: "JSON.ARRAPPEND path:sonde:foo $.properties.coordinateProperties || https://github.com/RedisJSON/RedisJSON/issues/295" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/redisjson-gh295-dump.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_ResultSet.totalResultsAvailable_jsonsl-yahoo2_json.yml b/tests/benchmarks/json_get_ResultSet.totalResultsAvailable_jsonsl-yahoo2_json.yml index 1619c36bc..c3c1bd29f 100644 --- a/tests/benchmarks/json_get_ResultSet.totalResultsAvailable_jsonsl-yahoo2_json.yml +++ b/tests/benchmarks/json_get_ResultSet.totalResultsAvailable_jsonsl-yahoo2_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_ResultSet.totalResultsAvailable_jsonsl-yahoo2_json" description: "JSON.GET jsonsl-yahoo2 $.ResultSet.totalResultsAvailable || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_[0]_jsonsl-1.yml b/tests/benchmarks/json_get_[0]_jsonsl-1.yml index b6081f50e..715f6ea5d 100644 --- a/tests/benchmarks/json_get_[0]_jsonsl-1.yml +++ b/tests/benchmarks/json_get_[0]_jsonsl-1.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_[0]_jsonsl-1" description: "JSON.GET jsonsl-1 $.[0] || {jsonsl-1.json size: 1.4 KB} https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_[7]_jsonsl-1.yml b/tests/benchmarks/json_get_[7]_jsonsl-1.yml index 7e89ca405..68da37b6f 100644 --- a/tests/benchmarks/json_get_[7]_jsonsl-1.yml +++ b/tests/benchmarks/json_get_[7]_jsonsl-1.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_[7]_jsonsl-1" description: "JSON.GET jsonsl-1 $.[7] || {jsonsl-1.json size: 1.4 KB} https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_[8].zero_jsonsl-1.yml b/tests/benchmarks/json_get_[8].zero_jsonsl-1.yml index 4045f38a2..e6d6632c2 100644 --- a/tests/benchmarks/json_get_[8].zero_jsonsl-1.yml +++ b/tests/benchmarks/json_get_[8].zero_jsonsl-1.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_[8].zero_jsonsl-1" description: "JSON.GET jsonsl-1 $.[8].0 || {jsonsl-1.json size: 1.4 KB} https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_[web-app].servlet[0][servlet-name]_json-parser-0000.yml b/tests/benchmarks/json_get_[web-app].servlet[0][servlet-name]_json-parser-0000.yml index bedd2bea2..0c5f55b0d 100644 --- a/tests/benchmarks/json_get_[web-app].servlet[0][servlet-name]_json-parser-0000.yml +++ b/tests/benchmarks/json_get_[web-app].servlet[0][servlet-name]_json-parser-0000.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_[web-app].servlet[0][servlet-name]_json-parser-0000" description: "JSON.GET json-parser-0000 $[web-app].servlet[0][servlet-name] {json-parser-0000.json size: 3.5K} || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_[web-app].servlet[0]_json-parser-0000.yml b/tests/benchmarks/json_get_[web-app].servlet[0]_json-parser-0000.yml index e60d63562..c46139bfe 100644 --- a/tests/benchmarks/json_get_[web-app].servlet[0]_json-parser-0000.yml +++ b/tests/benchmarks/json_get_[web-app].servlet[0]_json-parser-0000.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_[web-app].servlet[0]_json-parser-0000" description: "JSON.GET json-parser-0000 $[web-app].servlet[0] {json-parser-0000.json size: 3.5K} || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_[web-app].servlet_json-parser-0000.yml b/tests/benchmarks/json_get_[web-app].servlet_json-parser-0000.yml index 568f2805c..dbf78c0e0 100644 --- a/tests/benchmarks/json_get_[web-app].servlet_json-parser-0000.yml +++ b/tests/benchmarks/json_get_[web-app].servlet_json-parser-0000.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_[web-app].servlet_json-parser-0000" description: "JSON.GET json-parser-0000 $[web-app].servlet {json-parser-0000.json size: 3.5K} || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_array_of_docs[1]_pass_100_json.yml b/tests/benchmarks/json_get_array_of_docs[1]_pass_100_json.yml index f78e06d52..6fb2ce91f 100644 --- a/tests/benchmarks/json_get_array_of_docs[1]_pass_100_json.yml +++ b/tests/benchmarks/json_get_array_of_docs[1]_pass_100_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_array_of_docs[1]_pass_100_json" description: "JSON.GET pass-100 $.array_of_docs[1] || {Full document: pass-100.json https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_array_of_docs[1]sclr_pass_100_json.yml b/tests/benchmarks/json_get_array_of_docs[1]sclr_pass_100_json.yml index 3a124b6e2..77b926a83 100644 --- a/tests/benchmarks/json_get_array_of_docs[1]sclr_pass_100_json.yml +++ b/tests/benchmarks/json_get_array_of_docs[1]sclr_pass_100_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_array_of_docs[1]sclr_pass_100_json" description: "JSON.GET pass-100 $.array_of_docs[1] || {Full document: pass-100.json https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_array_of_docs_pass_100_json.yml b/tests/benchmarks/json_get_array_of_docs_pass_100_json.yml index 4c1149586..d312b1dea 100644 --- a/tests/benchmarks/json_get_array_of_docs_pass_100_json.yml +++ b/tests/benchmarks/json_get_array_of_docs_pass_100_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_array_of_docs_pass_100_json" description: "JSON.GET pass-100 $.array_of_docs || {Full document: pass-100.json https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_fulldoc_json-parser-0000.yml b/tests/benchmarks/json_get_fulldoc_json-parser-0000.yml index 01df63a78..a1f1284bb 100644 --- a/tests/benchmarks/json_get_fulldoc_json-parser-0000.yml +++ b/tests/benchmarks/json_get_fulldoc_json-parser-0000.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_fulldoc_json-parser-0000" description: "JSON.GET json-parser-0000 $ {json-parser-0000.json size: 3.5K} || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_fulldoc_jsonsl-1.yml b/tests/benchmarks/json_get_fulldoc_jsonsl-1.yml index 2268ec034..c55660a26 100644 --- a/tests/benchmarks/json_get_fulldoc_jsonsl-1.yml +++ b/tests/benchmarks/json_get_fulldoc_jsonsl-1.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_fulldoc_jsonsl-1" description: "JSON.GET jsonsl-1 $ || {jsonsl-1.json size: 1.4 KB} https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_fulldoc_jsonsl-yahoo2_json.yml b/tests/benchmarks/json_get_fulldoc_jsonsl-yahoo2_json.yml index da0820a53..f022b5f18 100644 --- a/tests/benchmarks/json_get_fulldoc_jsonsl-yahoo2_json.yml +++ b/tests/benchmarks/json_get_fulldoc_jsonsl-yahoo2_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_fulldoc_jsonsl-yahoo2_json" description: "JSON.GET jsonsl-yahoo2 $ || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_fulldoc_jsonsl-yelp_json.yml b/tests/benchmarks/json_get_fulldoc_jsonsl-yelp_json.yml index 08bfbe190..8d952ae4e 100644 --- a/tests/benchmarks/json_get_fulldoc_jsonsl-yelp_json.yml +++ b/tests/benchmarks/json_get_fulldoc_jsonsl-yelp_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_fulldoc_jsonsl-yelp_json" description: "JSON.GET jsonsl-yelp $ || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_fulldoc_pass_100_json.yml b/tests/benchmarks/json_get_fulldoc_pass_100_json.yml index 684b68b93..1634ae493 100644 --- a/tests/benchmarks/json_get_fulldoc_pass_100_json.yml +++ b/tests/benchmarks/json_get_fulldoc_pass_100_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_fulldoc_pass_100_json" description: "JSON.GET pass-100 $ || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_key_empty.yml b/tests/benchmarks/json_get_key_empty.yml index d54637617..4d0995ca5 100644 --- a/tests/benchmarks/json_get_key_empty.yml +++ b/tests/benchmarks/json_get_key_empty.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_key_empty" description: "JSON.GET key $ || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_message.code_jsonsl-yelp_json.yml b/tests/benchmarks/json_get_message.code_jsonsl-yelp_json.yml index dfb581a64..527601c4a 100644 --- a/tests/benchmarks/json_get_message.code_jsonsl-yelp_json.yml +++ b/tests/benchmarks/json_get_message.code_jsonsl-yelp_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_message.code_jsonsl-yelp_json" description: "JSON.GET jsonsl-yelp $.message.code || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_sclr_pass_100_json.yml b/tests/benchmarks/json_get_sclr_pass_100_json.yml index 97aec8e37..e5c1b398c 100644 --- a/tests/benchmarks/json_get_sclr_pass_100_json.yml +++ b/tests/benchmarks/json_get_sclr_pass_100_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_sclr_pass_100_json" description: "JSON.GET pass-100 $.sclr || {pass-100.json size: 380 B} https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_sub_doc.sclr_pass_100_json.yml b/tests/benchmarks/json_get_sub_doc.sclr_pass_100_json.yml index 93212c8b5..5fa298308 100644 --- a/tests/benchmarks/json_get_sub_doc.sclr_pass_100_json.yml +++ b/tests/benchmarks/json_get_sub_doc.sclr_pass_100_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_sub_doc.sclr_pass_100_json" description: "JSON.GET pass-100 $.sub_doc.sclr || {Full document: pass-100.json 380B} https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_get_sub_doc_pass_100_json.yml b/tests/benchmarks/json_get_sub_doc_pass_100_json.yml index 3b0f35eec..2131ba662 100644 --- a/tests/benchmarks/json_get_sub_doc_pass_100_json.yml +++ b/tests/benchmarks/json_get_sub_doc_pass_100_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_get_sub_doc_pass_100_json" description: "JSON.GET pass-100 $.sub_doc || {Full document: pass-100.json} https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_numincrby_num_1.yml b/tests/benchmarks/json_numincrby_num_1.yml index 8a8baf6f9..942190acd 100644 --- a/tests/benchmarks/json_numincrby_num_1.yml +++ b/tests/benchmarks/json_numincrby_num_1.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_numincrby_num_1" description: "JSON.NUMINCRBY num $ 1 || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_nummultby_num_2.yml b/tests/benchmarks/json_nummultby_num_2.yml index c8f8e1d59..37a635b46 100644 --- a/tests/benchmarks/json_nummultby_num_2.yml +++ b/tests/benchmarks/json_nummultby_num_2.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_nummultby_num_2" description: "JSON.NUMMULTBY num $ 2 || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_recursive_descent_with_filter_uid_issue674.yml b/tests/benchmarks/json_recursive_descent_with_filter_uid_issue674.yml index 263939b26..765a4c7ee 100644 --- a/tests/benchmarks/json_recursive_descent_with_filter_uid_issue674.yml +++ b/tests/benchmarks/json_recursive_descent_with_filter_uid_issue674.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_recursive_descent_with_filter_uid_issue674" description: "JSON.GET test '$..[?(@.uid==1198)].MD5ModelUID' || {issue674.json size: 618 KB} https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.doc_674.rdb" clientconfig: diff --git a/tests/benchmarks/json_set_ResultSet.totalResultsAvailable_1_jsonsl-yahoo2_json.yml b/tests/benchmarks/json_set_ResultSet.totalResultsAvailable_1_jsonsl-yahoo2_json.yml index 74a7a879a..bc3011ebc 100644 --- a/tests/benchmarks/json_set_ResultSet.totalResultsAvailable_1_jsonsl-yahoo2_json.yml +++ b/tests/benchmarks/json_set_ResultSet.totalResultsAvailable_1_jsonsl-yahoo2_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_set_ResultSet.totalResultsAvailable_1_jsonsl-yahoo2_json" description: "JSON.SET jsonsl-yahoo2 $.ResultSet.totalResultsAvailable 1 || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_set_[0]foo_jsonsl-1.yml b/tests/benchmarks/json_set_[0]foo_jsonsl-1.yml index 3fdbc7b98..9966d8600 100644 --- a/tests/benchmarks/json_set_[0]foo_jsonsl-1.yml +++ b/tests/benchmarks/json_set_[0]foo_jsonsl-1.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_set_[0]foo_jsonsl-1" description: "JSON.SET jsonsl-1 $.[0] foo || {jsonsl-1.json size: 1.4 KB} https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_set_[web-app].servlet[0][servlet-name]_bar_json-parser-0000.yml b/tests/benchmarks/json_set_[web-app].servlet[0][servlet-name]_bar_json-parser-0000.yml index 2492bc99e..fd4497c4d 100644 --- a/tests/benchmarks/json_set_[web-app].servlet[0][servlet-name]_bar_json-parser-0000.yml +++ b/tests/benchmarks/json_set_[web-app].servlet[0][servlet-name]_bar_json-parser-0000.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_set_[web-app].servlet[0][servlet-name]_bar_json-parser-0000" description: "JSON.SET json-parser-0000 $.[web-app].servlet[0][servlet-name] [bar] {json-parser-0000.json size: 3.5K} || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_set_fulldoc_api_replies_q1_google_autocomplete.yml b/tests/benchmarks/json_set_fulldoc_api_replies_q1_google_autocomplete.yml index 6de9acf22..085d6001a 100644 --- a/tests/benchmarks/json_set_fulldoc_api_replies_q1_google_autocomplete.yml +++ b/tests/benchmarks/json_set_fulldoc_api_replies_q1_google_autocomplete.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_set_fulldoc_api_replies_q1_google_autocomplete" description: "JSON.SET of full docs using as data the API replies of common services on the internet" -remote: - - type: oss-standalone - - setup: redisearch-m5d + clientconfig: - tool: redis-benchmark - min-tool-version: "6.2.0" diff --git a/tests/benchmarks/json_set_fulldoc_api_replies_q2_gmaps_areatraffic.yml b/tests/benchmarks/json_set_fulldoc_api_replies_q2_gmaps_areatraffic.yml index 29c168e53..6d12234c4 100644 --- a/tests/benchmarks/json_set_fulldoc_api_replies_q2_gmaps_areatraffic.yml +++ b/tests/benchmarks/json_set_fulldoc_api_replies_q2_gmaps_areatraffic.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_set_fulldoc_api_replies_q2_gmaps_areatraffic" description: "JSON.SET of full docs using as data the API replies of common services on the internet" -remote: - - type: oss-standalone - - setup: redisearch-m5d + clientconfig: - tool: redis-benchmark - min-tool-version: "6.2.0" diff --git a/tests/benchmarks/json_set_fulldoc_api_replies_q3_gmaps_passiveassist.yml b/tests/benchmarks/json_set_fulldoc_api_replies_q3_gmaps_passiveassist.yml index 833bee450..825855f17 100644 --- a/tests/benchmarks/json_set_fulldoc_api_replies_q3_gmaps_passiveassist.yml +++ b/tests/benchmarks/json_set_fulldoc_api_replies_q3_gmaps_passiveassist.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_set_fulldoc_api_replies_q3_gmaps_passiveassist" description: "JSON.SET of full docs using as data the API replies of common services on the internet" -remote: - - type: oss-standalone - - setup: redisearch-m5d + clientconfig: - tool: redis-benchmark - min-tool-version: "6.2.0" diff --git a/tests/benchmarks/json_set_fulldoc_api_replies_q4_gmaps_assist.yml b/tests/benchmarks/json_set_fulldoc_api_replies_q4_gmaps_assist.yml index 1bfe2e078..80c9d966c 100644 --- a/tests/benchmarks/json_set_fulldoc_api_replies_q4_gmaps_assist.yml +++ b/tests/benchmarks/json_set_fulldoc_api_replies_q4_gmaps_assist.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_set_fulldoc_api_replies_q4_gmaps_assist" description: "JSON.SET of full docs using as data the API replies of common services on the internet" -remote: - - type: oss-standalone - - setup: redisearch-m5d + clientconfig: - tool: redis-benchmark - min-tool-version: "6.2.0" diff --git a/tests/benchmarks/json_set_fulldoc_api_replies_q5_gmaps_place.yml b/tests/benchmarks/json_set_fulldoc_api_replies_q5_gmaps_place.yml index 7b5f41251..836390c2f 100644 --- a/tests/benchmarks/json_set_fulldoc_api_replies_q5_gmaps_place.yml +++ b/tests/benchmarks/json_set_fulldoc_api_replies_q5_gmaps_place.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_set_fulldoc_api_replies_q5_gmaps_place" description: "JSON.SET of full docs using as data the API replies of common services on the internet" -remote: - - type: oss-standalone - - setup: redisearch-m5d + clientconfig: - tool: redis-benchmark - min-tool-version: "6.2.0" diff --git a/tests/benchmarks/json_set_fulldoc_pass-json-parser-0000.yml b/tests/benchmarks/json_set_fulldoc_pass-json-parser-0000.yml index 625848572..9815c9180 100644 --- a/tests/benchmarks/json_set_fulldoc_pass-json-parser-0000.yml +++ b/tests/benchmarks/json_set_fulldoc_pass-json-parser-0000.yml @@ -1,8 +1,6 @@ version: 0.2 name: "json_set_fulldoc_pass-json-parser-0000" -remote: - - type: oss-standalone - - setup: redisearch-m5d + clientconfig: - tool: redis-benchmark - min-tool-version: "6.2.0" diff --git a/tests/benchmarks/json_set_fulldoc_pass_100_json.yml b/tests/benchmarks/json_set_fulldoc_pass_100_json.yml index e6d17d5cc..709bce4ed 100644 --- a/tests/benchmarks/json_set_fulldoc_pass_100_json.yml +++ b/tests/benchmarks/json_set_fulldoc_pass_100_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_set_fulldoc_pass_100_json" description: "JSON.SET pass-100 $ {pass-100.json size: 380 B} || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + clientconfig: - tool: redis-benchmark - min-tool-version: "6.2.0" diff --git a/tests/benchmarks/json_set_fulldoc_yahoo2.yml b/tests/benchmarks/json_set_fulldoc_yahoo2.yml index 3548a4ac0..031986bc6 100644 --- a/tests/benchmarks/json_set_fulldoc_yahoo2.yml +++ b/tests/benchmarks/json_set_fulldoc_yahoo2.yml @@ -1,8 +1,6 @@ version: 0.2 name: "json_set_fulldoc_yahoo2" -remote: - - type: oss-standalone - - setup: redisearch-m5d + clientconfig: - tool: redis-benchmark - min-tool-version: "6.2.0" diff --git a/tests/benchmarks/json_set_key_empty.yml b/tests/benchmarks/json_set_key_empty.yml index fa284f6dd..b9753e592 100644 --- a/tests/benchmarks/json_set_key_empty.yml +++ b/tests/benchmarks/json_set_key_empty.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_set_key_empty" description: "JSON.SET key $ {} || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_set_message.code_1_jsonsl-yelp_json.yml b/tests/benchmarks/json_set_message.code_1_jsonsl-yelp_json.yml index f0366cabb..3d94e6b7f 100644 --- a/tests/benchmarks/json_set_message.code_1_jsonsl-yelp_json.yml +++ b/tests/benchmarks/json_set_message.code_1_jsonsl-yelp_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_set_message.code_1_jsonsl-yelp_json" description: "JSON.SET jsonsl-yelp $.message.code 1 || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_set_num_0.yml b/tests/benchmarks/json_set_num_0.yml index 79078063e..0da905cde 100644 --- a/tests/benchmarks/json_set_num_0.yml +++ b/tests/benchmarks/json_set_num_0.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_set_num_0" description: "JSON.SET num $ 0 || https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_set_sclr_1_pass_100_json.yml b/tests/benchmarks/json_set_sclr_1_pass_100_json.yml index 258bc69cb..690b33b28 100644 --- a/tests/benchmarks/json_set_sclr_1_pass_100_json.yml +++ b/tests/benchmarks/json_set_sclr_1_pass_100_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_set_sclr_1_pass_100_json" description: "JSON.SET pass-100 $.sclr 1 || {pass-100.json size: 380 B} https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_set_sclr_pass_100_json.yml b/tests/benchmarks/json_set_sclr_pass_100_json.yml index 69ae03cc3..ad1291b0f 100644 --- a/tests/benchmarks/json_set_sclr_pass_100_json.yml +++ b/tests/benchmarks/json_set_sclr_pass_100_json.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_set_sclr_pass_100_json" description: "JSON.SET pass-100 $.sclr 1 || {pass-100.json size: 380 B} https://oss.redislabs.com/redisjson/performance/" -remote: - - type: oss-standalone - - setup: redisearch-m5d + dbconfig: - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisjson/performance.docs/performance.docs.rdb" clientconfig: diff --git a/tests/benchmarks/json_vs_hashes_hset_key_simple.yml b/tests/benchmarks/json_vs_hashes_hset_key_simple.yml index 76cdcbdf9..b1406d5c6 100644 --- a/tests/benchmarks/json_vs_hashes_hset_key_simple.yml +++ b/tests/benchmarks/json_vs_hashes_hset_key_simple.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_vs_hashes_hset_key_simple" description: 'HSET key_hash field1 value1 field2 value2 || Use-case to compare against JSON.SET key_json $ {"field1":"value1","field2":"value2"}' -remote: - - type: oss-standalone - - setup: redisearch-m5d + clientconfig: - tool: redis-benchmark - min-tool-version: "6.2.0" diff --git a/tests/benchmarks/json_vs_hashes_json.set_key_simple.yml b/tests/benchmarks/json_vs_hashes_json.set_key_simple.yml index f62c0425a..03e97e15d 100644 --- a/tests/benchmarks/json_vs_hashes_json.set_key_simple.yml +++ b/tests/benchmarks/json_vs_hashes_json.set_key_simple.yml @@ -1,9 +1,7 @@ version: 0.2 name: "json_vs_hashes_json.set_key_simple" description: 'JSON.SET key_json $ {"field1":"value1","field2":"value2"} || Use-case to compare against HSET key_hash field1 value1 field2 value2' -remote: - - type: oss-standalone - - setup: redisearch-m5d + clientconfig: - tool: redis-benchmark - min-tool-version: "6.2.0" From 46851b241917f2288b977f6e9d6507a7bee0fc54 Mon Sep 17 00:00:00 2001 From: Lior Kogan Date: Mon, 11 Dec 2023 13:49:38 +0200 Subject: [PATCH 009/112] json.merge.md - fix example --- docs/commands/json.merge.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/commands/json.merge.md b/docs/commands/json.merge.md index bb8f4c4ca..3c86b50fc 100644 --- a/docs/commands/json.merge.md +++ b/docs/commands/json.merge.md @@ -70,7 +70,7 @@ redis> JSON.GET doc $ {{< highlight bash >}} redis> JSON.SET doc $ '{"a":2}' OK -redis> JSON.MERGE doc $.a 'null' +redis> JSON.MERGE doc $ '{"a":null}' OK redis> JSON.GET doc $ "[{}]" From b76a38a72fabde899a58b88c25d2d475ca8de6bd Mon Sep 17 00:00:00 2001 From: Victor Polevoy Date: Tue, 12 Dec 2023 11:29:09 +0100 Subject: [PATCH 010/112] Update the dependencies. --- Cargo.lock | 632 +++++++++++++++++-------------- Cargo.toml | 3 +- redis_json/src/ivalue_manager.rs | 20 +- 3 files changed, 372 insertions(+), 283 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc4ed007c..fdaa252c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,39 +19,26 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ "cfg-if", + "getrandom", "once_cell", "version_check", + "zerocopy", ] [[package]] name = "aho-corasick" -version = "1.0.4" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -73,13 +60,19 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "bindgen" version = "0.66.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "cexpr", "clang-sys", "lazy_static", @@ -92,7 +85,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.29", + "syn 2.0.40", "which", ] @@ -104,9 +97,21 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] [[package]] name = "block-buffer" @@ -119,33 +124,30 @@ dependencies = [ [[package]] name = "bson" -version = "0.14.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c177ed0122f24ce5e0f05bf9b29e79f3ac1a359bc504e0e14c3b34896c71c00" +checksum = "61570f4de0cc9c03b481c96057b3ae7c6ff7b5b35da8b0832c44f0131987a718" dependencies = [ - "byteorder", - "chrono", + "ahash", + "base64", + "bitvec", "hex", - "libc", - "linked-hash-map", - "md5", + "indexmap 1.9.3", + "js-sys", + "once_cell", "rand", "serde", + "serde_bytes", "serde_json", "time", + "uuid", ] [[package]] name = "bumpalo" -version = "3.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" - -[[package]] -name = "byteorder" -version = "1.4.3" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "cc" @@ -171,21 +173,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "chrono" -version = "0.4.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", - "num-traits 0.2.16", - "time", - "wasm-bindgen", - "winapi", -] - [[package]] name = "clang-sys" version = "1.6.1" @@ -197,17 +184,11 @@ dependencies = [ "libloading", ] -[[package]] -name = "core-foundation-sys" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" - [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ "libc", ] @@ -224,17 +205,23 @@ dependencies = [ [[package]] name = "dashmap" -version = "5.5.1" +version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd72493923899c6f10c641bdbdeddc7183d6396641d99c1a0d1597f37f92e28" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "lock_api", "once_cell", "parking_lot_core", ] +[[package]] +name = "deranged" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" + [[package]] name = "digest" version = "0.10.7" @@ -264,9 +251,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" dependencies = [ "humantime", "is-terminal", @@ -283,24 +270,19 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.2" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ - "errno-dragonfly", "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] -name = "errno-dragonfly" -version = "0.1.2" +name = "funty" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "generic-array" @@ -314,20 +296,20 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.16" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "libc", - "wasi 0.9.0+wasi-snapshot-preview1", + "wasi", ] [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "glob" @@ -335,6 +317,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.13.2" @@ -346,9 +334,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "heck" @@ -358,44 +346,30 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "hex" -version = "0.3.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] -name = "humantime" -version = "2.1.0" +name = "home" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "iana-time-zone" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows", + "windows-sys 0.48.0", ] [[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" +name = "humantime" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "ijson" @@ -411,12 +385,22 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.3", ] [[package]] @@ -427,7 +411,7 @@ checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -441,15 +425,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" dependencies = [ "wasm-bindgen", ] @@ -483,9 +467,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "libloading" @@ -497,43 +481,37 @@ dependencies = [ "winapi", ] -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - [[package]] name = "linkme" -version = "0.3.15" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f948366ad5bb46b5514ba7a7a80643726eef08b06632592699676748c8bc33b" +checksum = "b1e6b0bb9ca88d3c5ae88240beb9683821f903b824ee8381ef9ab4e8522fbfa9" dependencies = [ "linkme-impl", ] [[package]] name = "linkme-impl" -version = "0.3.15" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc28438cad73dcc90ff3466fc329a9252b1b8ba668eb0d5668ba97088cf4eef0" +checksum = "b3b3f61e557a617ec6ba36c79431e1f3b5e100d67cfbdb61ed6ef384298af016" dependencies = [ "proc-macro2", "quote 1.0.33", - "syn 2.0.29", + "syn 2.0.40", ] [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -545,17 +523,11 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -[[package]] -name = "md5" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e6bcd6433cff03a4bfc3d9834d504467db1f1cf6d0ea765d37d330249ed629d" - [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" @@ -583,16 +555,15 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" dependencies = [ "bitflags 1.3.2", "cfg-if", "libc", "memoffset", "pin-utils", - "static_assertions", ] [[package]] @@ -611,44 +582,44 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" dependencies = [ - "num-traits 0.2.16", + "num-traits 0.2.17", ] [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", ] [[package]] name = "object" -version = "0.32.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -659,19 +630,20 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "pest" -version = "2.7.2" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a" +checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" dependencies = [ + "memchr", "thiserror", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.7.2" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "666d00490d4ac815001da55838c500eafb0320019bbaa44444137c48b443a853" +checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" dependencies = [ "pest", "pest_generator", @@ -679,22 +651,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.2" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ca01446f50dbda87c1786af8770d535423fa8a53aec03b8f4e3d7eb10e0929" +checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote 1.0.33", - "syn 2.0.29", + "syn 2.0.40", ] [[package]] name = "pest_meta" -version = "2.7.2" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56af0a30af74d0445c0bf6d9d051c979b516a1a5af790d251daee76005420a48" +checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" dependencies = [ "once_cell", "pest", @@ -715,19 +687,19 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.12" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" +checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.29", + "syn 2.0.40", ] [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" dependencies = [ "unicode-ident", ] @@ -747,24 +719,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" -version = "0.7.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ - "getrandom", "libc", "rand_chacha", "rand_core", - "rand_hc", ] [[package]] name = "rand_chacha" -version = "0.2.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", "rand_core", @@ -772,22 +748,13 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.5.1" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom", ] -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core", -] - [[package]] name = "redis-module" version = "2.0.7" @@ -796,7 +763,7 @@ checksum = "9d3b95d9fe5b8681bacea6532bbc7632590ae4499cecc8e019685b515fddda71" dependencies = [ "backtrace", "bindgen", - "bitflags 2.4.0", + "bitflags 2.4.1", "cc", "cfg-if", "enum-primitive-derive", @@ -804,7 +771,7 @@ dependencies = [ "linkme", "log", "nix", - "num-traits 0.2.16", + "num-traits 0.2.17", "redis-module-macros-internals", "regex", "serde", @@ -856,18 +823,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.9.3" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", @@ -877,9 +844,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.6" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", @@ -888,9 +855,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "rustc-demangle" @@ -906,15 +873,15 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.8" +version = "0.38.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -925,9 +892,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "scopeguard" @@ -944,6 +911,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.171" @@ -952,16 +928,16 @@ checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ "proc-macro2", "quote 1.0.33", - "syn 2.0.29", + "syn 2.0.40", ] [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ - "indexmap", + "indexmap 2.1.0", "itoa", "ryu", "serde", @@ -981,9 +957,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -992,21 +968,15 @@ dependencies = [ [[package]] name = "shlex" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" [[package]] name = "smallvec" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" - -[[package]] -name = "static_assertions" -version = "1.1.0" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" [[package]] name = "strum_macros" @@ -1045,9 +1015,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.29" +version = "2.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" +checksum = "13fa70a4ee923979ffb522cacce59d34421ebdea5625e1073c4326ef9d2dd42e" dependencies = [ "proc-macro2", "quote 1.0.33", @@ -1063,51 +1033,74 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "termcolor" -version = "1.2.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" dependencies = [ "winapi-util", ] [[package]] name = "thiserror" -version = "1.0.47" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.47" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote 1.0.33", - "syn 2.0.29", + "syn 2.0.40", ] [[package]] name = "time" -version = "0.1.45" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +checksum = "a79d09ac6b08c1ab3906a2f7cc2e81a0e27c7ae89c63812df75e52bef0751e07" dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", + "deranged", + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + +[[package]] +name = "time-macros" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75c65469ed6b3a4809d987a41eb1dc918e9bc1d92211cbad7ae82931846f7451" +dependencies = [ + "time-core", ] [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ucd-trie" @@ -1117,9 +1110,9 @@ checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-xid" @@ -1128,28 +1121,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" [[package]] -name = "version_check" -version = "0.9.4" +name = "uuid" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +dependencies = [ + "getrandom", + "serde", +] [[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" +name = "version_check" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" +version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1157,24 +1154,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote 1.0.33", - "syn 2.0.29", + "syn 2.0.40", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" dependencies = [ "quote 1.0.33", "wasm-bindgen-macro-support", @@ -1182,32 +1179,33 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote 1.0.33", - "syn 2.0.29", + "syn 2.0.40", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" [[package]] name = "which" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix", ] [[package]] @@ -1228,9 +1226,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -1242,21 +1240,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" +name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", ] [[package]] name = "windows-sys" -version = "0.48.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.0", ] [[package]] @@ -1265,13 +1263,28 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", ] [[package]] @@ -1280,38 +1293,109 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "zerocopy" +version = "0.7.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "306dca4455518f1f31635ec308b6b3e4eb1b11758cefafc782827d0aa7acb5c7" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be912bf68235a88fbefd1b73415cb218405958d1655b2ece9035a19920bdf6ba" +dependencies = [ + "proc-macro2", + "quote 1.0.33", + "syn 2.0.40", +] diff --git a/Cargo.toml b/Cargo.toml index 2d0aef012..98db3a0a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [workspace] +resolver = "2" members = [ "json_path", @@ -14,7 +15,7 @@ serde_json = { version="1.0", features = ["unbounded_depth"]} # this project will not be able to compromise it. serde = { version = "=1.0.171", features = ["derive"] } serde_derive = "=1.0.171" -bson = "0.14" +bson = "2" [workspace.package] edition = "2021" diff --git a/redis_json/src/ivalue_manager.rs b/redis_json/src/ivalue_manager.rs index dca54911e..2ff2ef4ec 100644 --- a/redis_json/src/ivalue_manager.rs +++ b/redis_json/src/ivalue_manager.rs @@ -10,7 +10,7 @@ use crate::manager::{Manager, ReadHolder, WriteHolder}; use crate::redisjson::normalize_arr_start_index; use crate::Format; use crate::REDIS_JSON_TYPE; -use bson::decode_document; +use bson::{from_document, Document}; use ijson::object::Entry; use ijson::{DestructuredMut, INumber, IObject, IString, IValue, ValueType}; use redis_module::key::{verify_type, KeyFlags, RedisKey, RedisKeyWritable}; @@ -618,9 +618,13 @@ impl<'a> Manager for RedisIValueJsonKeyManager<'a> { } IValue::deserialize(&mut deserializer).map_err(|e| e.into()) } - Format::BSON => decode_document(&mut Cursor::new(val.as_bytes())).map_or_else( + Format::BSON => from_document( + Document::from_reader(&mut Cursor::new(val.as_bytes())) + .map_err(|e| e.to_string())?, + ) + .map_or_else( |e| Err(e.to_string().into()), - |docs| { + |docs: Document| { let v = if docs.is_empty() { IValue::NULL } else { @@ -718,11 +722,11 @@ mod tests { phantom: PhantomData, }; let json = r#"{ - "a": 100.12, - "b": "foo", - "c": true, - "d": 126, - "e": -112, + "a": 100.12, + "b": "foo", + "c": true, + "d": 126, + "e": -112, "f": 7388608, "g": -6388608, "h": 9388608, From d513fbd428d92a08cc7b63b4f72d43db4ac35b20 Mon Sep 17 00:00:00 2001 From: Binbin Date: Fri, 15 Dec 2023 17:49:41 +0800 Subject: [PATCH 011/112] Fix the error example that cannot be executed in json.merge The example execution will return an error: ``` redis> JSON.SET doc $ '{"f1": {"a":1}, "f2":{"a":2}}' OK redis> JSON.GET doc "{\"f1\":{\"a\":1},\"f2\":{\"a\":2}}" redis> JSON.MERGE doc $ '{"f1": 'null', "f2":{"a":3, "b":4}, "f3":'[2,4,6]'}' Invalid argument(s) redis> JSON.MERGE doc $ '{"f1": null, "f2":{"a":3, "b":4}, "f3":[2,4,6]}' OK redis> JSON.GET doc"{\"f2\":{\"a\":3,\"b\":4},\"f3\":[2,4,6]}" ``` --- docs/commands/json.merge.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/commands/json.merge.md b/docs/commands/json.merge.md index 3c86b50fc..a83d65555 100644 --- a/docs/commands/json.merge.md +++ b/docs/commands/json.merge.md @@ -101,7 +101,7 @@ redis> JSON.SET doc $ '{"f1": {"a":1}, "f2":{"a":2}}' OK redis> JSON.GET doc "{\"f1\":{\"a\":1},\"f2\":{\"a\":2}}" -redis> JSON.MERGE doc $ '{"f1": 'null', "f2":{"a":3, "b":4}, "f3":'[2,4,6]'}' +redis> JSON.MERGE doc $ '{"f1": null, "f2":{"a":3, "b":4}, "f3":[2,4,6]}' OK redis> JSON.GET doc "{\"f2\":{\"a\":3,\"b\":4},\"f3\":[2,4,6]}" From 8222b61d50d1d9dd4df6a937c6f17a8d576aab6d Mon Sep 17 00:00:00 2001 From: Omer Shadmi <76992134+oshadmi@users.noreply.github.com> Date: Fri, 15 Dec 2023 18:23:39 +0200 Subject: [PATCH 012/112] MOD-6283: Rollback to use RLTest=0.7.5 (#1158) Use new RLTest but with no-progress --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3875159d0..0a0a1a149 100644 --- a/Makefile +++ b/Makefile @@ -191,7 +191,7 @@ run: test: cargo_test pytest pytest: - $(SHOW)MODULE=$(abspath $(TARGET)) $(realpath ./tests/pytest/tests.sh) + $(SHOW)MODULE=$(abspath $(TARGET)) RLTEST_ARGS='--no-progress' $(realpath ./tests/pytest/tests.sh) cargo_test: $(SHOW)cargo $(CARGO_TOOLCHAIN) test --all From eb0f0fc8cd3e263737a17d0b2610d1aae68790e6 Mon Sep 17 00:00:00 2001 From: Binbin Date: Tue, 19 Dec 2023 11:10:48 +0800 Subject: [PATCH 013/112] Update JSON.TYPE dummy response is empty array The command forgot to record the corresponding response. --- docs/commands/json.type.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/commands/json.type.md b/docs/commands/json.type.md index 94a0939db..3d5a93a9b 100644 --- a/docs/commands/json.type.md +++ b/docs/commands/json.type.md @@ -33,6 +33,7 @@ redis> JSON.TYPE doc $..a 1) "integer" 2) "boolean" redis> JSON.TYPE doc $..dummy +(empty array) {{< / highlight >}} ## See also From d703abec556cf6719ca16ffb79ce432b47a95bd8 Mon Sep 17 00:00:00 2001 From: Binbin Date: Tue, 19 Dec 2023 17:22:23 +0800 Subject: [PATCH 014/112] Fix execution examples in JSON MSET doc In fact, executing the example directly will return this error: ``` redis> flushall OK redis> JSON.MSET doc1 $ '{"a":2}' doc2 $.f.a '3' doc3 $ '{"f1": {"a":1}, "f2":{"a":2}}' (error) ERR new objects must be created at the root ``` Add some JSON.SET in front of the example to avoid error and fix the incorrect responses. --- docs/commands/json.mset.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/commands/json.mset.md b/docs/commands/json.mset.md index 9584e4bcb..23532d6eb 100644 --- a/docs/commands/json.mset.md +++ b/docs/commands/json.mset.md @@ -37,14 +37,20 @@ For more information about replies, see [Redis serialization protocol specificat Add a new values in multiple keys {{< highlight bash >}} +redis> JSON.SET doc1 $ '{"a":1}' +OK +redis> JSON.SET doc2 $ '{"f":{"a":2}}' +OK +redis> JSON.SET doc3 $ '{"f1": {"a":0}, "f2":{"a":0}}' +OK redis> JSON.MSET doc1 $ '{"a":2}' doc2 $.f.a '3' doc3 $ '{"f1": {"a":1}, "f2":{"a":2}}' OK redis> JSON.GET doc1 $ "[{\"a\":2}]" redis> JSON.GET doc2 $ -"[{\"f\":{\"a\":3]" +"[{\"f\":{\"a\":3}}]" redis> JSON.GET doc3 -"{\"f1\":{\"a\":3},\"f2\":{\"a\":3}}" +"{\"f1\":{\"a\":1},\"f2\":{\"a\":2}}" {{< / highlight >}} From aa086e7170262a359c03cc74fc0e0e5b1139677e Mon Sep 17 00:00:00 2001 From: Binbin Date: Tue, 19 Dec 2023 19:25:22 +0800 Subject: [PATCH 015/112] Combine the three JSON.SET commands into a single JSON.MSET command --- docs/commands/json.mset.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/commands/json.mset.md b/docs/commands/json.mset.md index 23532d6eb..c8ee61775 100644 --- a/docs/commands/json.mset.md +++ b/docs/commands/json.mset.md @@ -37,13 +37,9 @@ For more information about replies, see [Redis serialization protocol specificat Add a new values in multiple keys {{< highlight bash >}} -redis> JSON.SET doc1 $ '{"a":1}' +redis> JSON.MSET doc1 $ '{"a":1}' doc2 $ '{"f":{"a":2}}' doc3 $ '{"f1":{"a":0},"f2":{"a":0}}' OK -redis> JSON.SET doc2 $ '{"f":{"a":2}}' -OK -redis> JSON.SET doc3 $ '{"f1": {"a":0}, "f2":{"a":0}}' -OK -redis> JSON.MSET doc1 $ '{"a":2}' doc2 $.f.a '3' doc3 $ '{"f1": {"a":1}, "f2":{"a":2}}' +redis> JSON.MSET doc1 $ '{"a":2}' doc2 $.f.a '3' doc3 $ '{"f1":{"a":1},"f2":{"a":2}}' OK redis> JSON.GET doc1 $ "[{\"a\":2}]" From ba7e892610f7589c4b79f5c5b21435b6b309f1ed Mon Sep 17 00:00:00 2001 From: Victor Polevoy Date: Tue, 19 Dec 2023 10:50:27 +0100 Subject: [PATCH 016/112] Update dependencies --- Cargo.lock | 62 +++++++++++++++++++++---------------------- redis_json/Cargo.toml | 8 +++--- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fdaa252c6..d15987127 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,7 +85,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.40", + "syn 2.0.41", "which", ] @@ -124,9 +124,9 @@ dependencies = [ [[package]] name = "bson" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61570f4de0cc9c03b481c96057b3ae7c6ff7b5b35da8b0832c44f0131987a718" +checksum = "88c18b51216e1f74b9d769cead6ace2f82b965b807e3d73330aabe9faec31c84" dependencies = [ "ahash", "base64", @@ -358,11 +358,11 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "home" -version = "0.5.5" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -416,9 +416,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.10.5" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" dependencies = [ "either", ] @@ -483,22 +483,22 @@ dependencies = [ [[package]] name = "linkme" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e6b0bb9ca88d3c5ae88240beb9683821f903b824ee8381ef9ab4e8522fbfa9" +checksum = "73cd2fa5f00af00e5ed9ea726c496bf0e58cb7c54bf9f14b7e0f80b5d14a3578" dependencies = [ "linkme-impl", ] [[package]] name = "linkme-impl" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3b3f61e557a617ec6ba36c79431e1f3b5e100d67cfbdb61ed6ef384298af016" +checksum = "5b43a34f4fbf8b3e0e163af8764916780c7c6fac8422183590f877a67036b85e" dependencies = [ "proc-macro2", "quote 1.0.33", - "syn 2.0.40", + "syn 2.0.41", ] [[package]] @@ -659,7 +659,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote 1.0.33", - "syn 2.0.40", + "syn 2.0.41", ] [[package]] @@ -692,7 +692,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.40", + "syn 2.0.41", ] [[package]] @@ -807,7 +807,7 @@ dependencies = [ name = "redis_json" version = "99.99.99" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.1", "bson", "env_logger", "ijson", @@ -928,7 +928,7 @@ checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ "proc-macro2", "quote 1.0.33", - "syn 2.0.40", + "syn 2.0.41", ] [[package]] @@ -1015,9 +1015,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.40" +version = "2.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13fa70a4ee923979ffb522cacce59d34421ebdea5625e1073c4326ef9d2dd42e" +checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" dependencies = [ "proc-macro2", "quote 1.0.33", @@ -1050,22 +1050,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" dependencies = [ "proc-macro2", "quote 1.0.33", - "syn 2.0.40", + "syn 2.0.41", ] [[package]] @@ -1163,7 +1163,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote 1.0.33", - "syn 2.0.40", + "syn 2.0.41", "wasm-bindgen-shared", ] @@ -1185,7 +1185,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote 1.0.33", - "syn 2.0.40", + "syn 2.0.41", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1382,20 +1382,20 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.30" +version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "306dca4455518f1f31635ec308b6b3e4eb1b11758cefafc782827d0aa7acb5c7" +checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.30" +version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be912bf68235a88fbefd1b73415cb218405958d1655b2ece9035a19920bdf6ba" +checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ "proc-macro2", "quote 1.0.33", - "syn 2.0.40", + "syn 2.0.41", ] diff --git a/redis_json/Cargo.toml b/redis_json/Cargo.toml index 0f5b4ddc4..c7aa36998 100644 --- a/redis_json/Cargo.toml +++ b/redis_json/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Guy Korland ", "Meir Shpilraien Date: Thu, 21 Dec 2023 10:40:42 +0100 Subject: [PATCH 017/112] Run on ubuntu latest --- .github/workflows/freebsd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml index f7f39c72b..a264f534d 100644 --- a/.github/workflows/freebsd.yml +++ b/.github/workflows/freebsd.yml @@ -9,7 +9,7 @@ on: jobs: build: - runs-on: macos-12 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From dacb41bdbf07bf8fc1381f87bc21333bd4e964e0 Mon Sep 17 00:00:00 2001 From: Victor Polevoy Date: Thu, 21 Dec 2023 10:44:12 +0100 Subject: [PATCH 018/112] Update the FreeBSD action version --- .github/workflows/freebsd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml index a264f534d..cd45eecd6 100644 --- a/.github/workflows/freebsd.yml +++ b/.github/workflows/freebsd.yml @@ -14,12 +14,12 @@ jobs: steps: - uses: actions/checkout@v3 - name: build - uses: vmactions/freebsd-vm@v0.3.0 + uses: vmactions/freebsd-vm@v1 with: usesh: true sync: rsync copyback: false - prepare: pkg install -y bash curl lang/rust devel/llvm-devel + #prepare: pkg install -y bash curl lang/rust devel/llvm-devel run: | cargo build --all --all-targets --verbose curl https://sh.rustup.rs -sSf > rustup.sh From fc4f81fe1530cf8d1b8171578cd6916d1d131adb Mon Sep 17 00:00:00 2001 From: Victor Polevoy Date: Thu, 21 Dec 2023 10:49:11 +0100 Subject: [PATCH 019/112] Install CURL --- .github/workflows/freebsd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml index cd45eecd6..99ec6078a 100644 --- a/.github/workflows/freebsd.yml +++ b/.github/workflows/freebsd.yml @@ -19,7 +19,7 @@ jobs: usesh: true sync: rsync copyback: false - #prepare: pkg install -y bash curl lang/rust devel/llvm-devel + prepare: pkg install -y bash curl lang/rust devel/llvm-devel run: | cargo build --all --all-targets --verbose curl https://sh.rustup.rs -sSf > rustup.sh From 6cf7bca8d19764ac8797de7e22543c5133512c11 Mon Sep 17 00:00:00 2001 From: Binbin Date: Fri, 22 Dec 2023 14:50:50 +0800 Subject: [PATCH 020/112] Update prompt to be consistent with other command examples --- docs/commands/json.arrindex.md | 2 +- docs/commands/json.arrinsert.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/commands/json.arrindex.md b/docs/commands/json.arrindex.md index 5a9a054c6..77840ff16 100644 --- a/docs/commands/json.arrindex.md +++ b/docs/commands/json.arrindex.md @@ -68,7 +68,7 @@ redis> JSON.ARRAPPEND item:1 $.colors '"blue"' Return the new length of the `colors` array. {{< highlight bash >}} -JSON.GET item:1 +redis> JSON.GET item:1 "{\"name\":\"Noise-cancelling Bluetooth headphones\",\"description\":\"Wireless Bluetooth headphones with noise-cancelling technology\",\"connection\":{\"wireless\":true,\"type\":\"Bluetooth\"},\"price\":99.98,\"stock\":25,\"colors\":[\"black\",\"silver\",\"blue\"]}" {{< / highlight >}} diff --git a/docs/commands/json.arrinsert.md b/docs/commands/json.arrinsert.md index 1e89cdf68..04d91f4f7 100644 --- a/docs/commands/json.arrinsert.md +++ b/docs/commands/json.arrinsert.md @@ -57,7 +57,7 @@ redis> JSON.ARRAPPEND item:1 $.colors '"blue"' Return the new length of the `colors` array. {{< highlight bash >}} -JSON.GET item:1 +redis> JSON.GET item:1 "{\"name\":\"Noise-cancelling Bluetooth headphones\",\"description\":\"Wireless Bluetooth headphones with noise-cancelling technology\",\"connection\":{\"wireless\":true,\"type\":\"Bluetooth\"},\"price\":99.98,\"stock\":25,\"colors\":[\"black\",\"silver\",\"blue\"]}" {{< / highlight >}} From fd6255485b16be3851759196f11b11b5651e662e Mon Sep 17 00:00:00 2001 From: Lior Kogan Date: Thu, 11 Jan 2024 15:18:52 +0200 Subject: [PATCH 021/112] Create SECURITY.md --- SECURITY.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..ec125ed77 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,42 @@ +# Security Policy + +## Supported Versions + +RedisJSON is generally backwards compatible with very few exceptions, so we +recommend users to always use the latest version to experience stability, +performance and security. + +We generally backport security issues to a single previous major version, +unless this is not possible or feasible with a reasonable effort. + +| Version | Supported | +| ------- | ------------------ | +| 2.6 | :white_check_mark: | +| 2.4 | :white_check_mark: | +| < 2.4 | :x: | + +## Reporting a Vulnerability + +If you believe you've discovered a serious vulnerability, please contact the +Redis core team at redis@redis.io. We will evaluate your report and if +necessary issue a fix and an advisory. If the issue was previously undisclosed, +we'll also mention your name in the credits. + +## Responsible Disclosure + +In some cases, we may apply a responsible disclosure process to reported or +otherwise discovered vulnerabilities. We will usually do that for a critical +vulnerability, and only if we have a good reason to believe information about +it is not yet public. + +This process involves providing an early notification about the vulnerability, +its impact and mitigations to a short list of vendors under a time-limited +embargo on public disclosure. + +Vendors on the list are individuals or organizations that maintain Redis +distributions or provide Redis as a service, who have third party users who +will benefit from the vendor's ability to prepare for a new version or deploy a +fix early. + +If you believe you should be on the list, please contact us and we will +consider your request based on the above criteria. From a5bafc4f6bae93edd82cf9bc05accca621568607 Mon Sep 17 00:00:00 2001 From: Lior Kogan Date: Thu, 11 Jan 2024 15:20:09 +0200 Subject: [PATCH 022/112] Update wordlist.txt --- .github/wordlist.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/wordlist.txt b/.github/wordlist.txt index 1cdd9984d..585d9b14c 100644 --- a/.github/wordlist.txt +++ b/.github/wordlist.txt @@ -108,3 +108,5 @@ uint vachhanihpavan www zstd +mitigations +backport From 9c5d82fbaa17cc121fecd5cba61f505b931e22d2 Mon Sep 17 00:00:00 2001 From: nafraf Date: Wed, 17 Jan 2024 12:35:26 -0500 Subject: [PATCH 023/112] Update readis dep: fix paella.wget() (#1173) --- deps/readies | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/readies b/deps/readies index eef12bc28..a1d34e41d 160000 --- a/deps/readies +++ b/deps/readies @@ -1 +1 @@ -Subproject commit eef12bc28fcb62c00b5550d705aa7b544f3111ca +Subproject commit a1d34e41d644bf5c32ec41d609a61996eb031301 From 865033ee710fa15b12acc6928096d49cbf82a7dd Mon Sep 17 00:00:00 2001 From: meiravgri <109056284+meiravgri@users.noreply.github.com> Date: Thu, 8 Feb 2024 08:42:54 +0200 Subject: [PATCH 024/112] Update ahash version https://github.com/tkaitchuck/aHash/issues/200 https://users.rust-lang.org/t/error-e0635-unknown-feature-stdsimd/106445 --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index d15987127..e7b66f5cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,7 +19,7 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ From e7b5f31407ed03d75c50ad89fd321cadffe5e8e6 Mon Sep 17 00:00:00 2001 From: GuyAv46 Date: Thu, 8 Feb 2024 10:45:56 +0200 Subject: [PATCH 025/112] fix Cargo.lock file --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index e7b66f5cb..ab73ad354 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,7 +21,7 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" name = "ahash" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if", "getrandom", From 1c60fe588334f3969a374e74b31b27fe79c64131 Mon Sep 17 00:00:00 2001 From: Victor Polevoy Date: Thu, 29 Feb 2024 18:32:32 +0100 Subject: [PATCH 026/112] Add test to verify the unicode characters parse fine. There were reports that Redis json modules couldn't handle the unicode characters in strings. This commit adds two tests to make sure those are handled fine for RedisJSON. Also adds a single thread mutex for the ijson unit tests --- Cargo.lock | 347 +++++++++++++++---------------- redis_json/src/ivalue_manager.rs | 32 +++ tests/pytest/test.py | 77 ++++--- tests/pytest/test_multi.py | 15 +- 4 files changed, 247 insertions(+), 224 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab73ad354..b05cb5fd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.7" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +checksum = "8b79b82693f705137f8fb9b37871d99e4f9a7df12b917eed79c3d3954830a60b" dependencies = [ "cfg-if", "getrandom", @@ -72,7 +72,7 @@ version = "0.66.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cexpr", "clang-sys", "lazy_static", @@ -81,11 +81,11 @@ dependencies = [ "peeking_take_while", "prettyplease", "proc-macro2", - "quote 1.0.33", + "quote 1.0.35", "regex", "rustc-hash", "shlex", - "syn 2.0.41", + "syn 2.0.52", "which", ] @@ -97,9 +97,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "bitvec" @@ -124,15 +124,15 @@ dependencies = [ [[package]] name = "bson" -version = "2.8.1" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88c18b51216e1f74b9d769cead6ace2f82b965b807e3d73330aabe9faec31c84" +checksum = "ce21468c1c9c154a85696bb25c20582511438edb6ad67f846ba1378ffdd80222" dependencies = [ "ahash", "base64", "bitvec", "hex", - "indexmap 1.9.3", + "indexmap", "js-sys", "once_cell", "rand", @@ -145,18 +145,15 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" [[package]] name = "cc" -version = "1.0.83" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" -dependencies = [ - "libc", -] +checksum = "02f341c093d19155a6e41631ce5971aac4e9a868262212153124c15fa22d1cdc" [[package]] name = "cexpr" @@ -175,9 +172,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clang-sys" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" dependencies = [ "glob", "libc", @@ -186,9 +183,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -218,9 +215,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" [[package]] name = "digest" @@ -234,9 +231,9 @@ dependencies = [ [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "enum-primitive-derive" @@ -251,9 +248,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ "humantime", "is-terminal", @@ -296,9 +293,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "libc", @@ -317,12 +314,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.13.2" @@ -346,9 +337,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hex" @@ -385,19 +376,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.3" +version = "2.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - -[[package]] -name = "indexmap" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "967d6dd42f16dbf0eb8040cb9e477933562684d3918f7d253f2ff9087fb3e7a3" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -405,20 +386,20 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ "hermit-abi", - "rustix", - "windows-sys 0.48.0", + "libc", + "windows-sys 0.52.0", ] [[package]] name = "itertools" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -431,9 +412,9 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" dependencies = [ "wasm-bindgen", ] @@ -467,45 +448,45 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.151" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libloading" -version = "0.7.4" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" dependencies = [ "cfg-if", - "winapi", + "windows-sys 0.48.0", ] [[package]] name = "linkme" -version = "0.3.19" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73cd2fa5f00af00e5ed9ea726c496bf0e58cb7c54bf9f14b7e0f80b5d14a3578" +checksum = "1a78816ac097580aa7fd9d2e9cc7395dda34367c07267a8657516d4ad5e2e3d3" dependencies = [ "linkme-impl", ] [[package]] name = "linkme-impl" -version = "0.3.19" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b43a34f4fbf8b3e0e163af8764916780c7c6fac8422183590f877a67036b85e" +checksum = "ee9023a564f8bf7fe3da285a50c3e70de0df3e2bf277ff7c4e76d66008ef93b0" dependencies = [ "proc-macro2", - "quote 1.0.33", - "syn 2.0.41", + "quote 1.0.35", + "syn 2.0.52", ] [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" @@ -519,15 +500,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memoffset" @@ -546,9 +527,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] @@ -582,23 +563,23 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" dependencies = [ - "num-traits 0.2.17", + "num-traits 0.2.18", ] [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", ] [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -630,9 +611,9 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "pest" -version = "2.7.5" +version = "2.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" +checksum = "219c0dcc30b6a27553f9cc242972b67f75b60eb0db71f0b5462f38b058c41546" dependencies = [ "memchr", "thiserror", @@ -641,9 +622,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.5" +version = "2.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" +checksum = "22e1288dbd7786462961e69bfd4df7848c1e37e8b74303dbdab82c3a9cdd2809" dependencies = [ "pest", "pest_generator", @@ -651,22 +632,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.5" +version = "2.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" +checksum = "1381c29a877c6d34b8c176e734f35d7f7f5b3adaefe940cb4d1bb7af94678e2e" dependencies = [ "pest", "pest_meta", "proc-macro2", - "quote 1.0.33", - "syn 2.0.41", + "quote 1.0.35", + "syn 2.0.52", ] [[package]] name = "pest_meta" -version = "2.7.5" +version = "2.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" +checksum = "d0934d6907f148c22a3acbda520c7eed243ad7487a30f51f6ce52b58b7077a8a" dependencies = [ "once_cell", "pest", @@ -687,19 +668,19 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "syn 2.0.41", + "syn 2.0.52", ] [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -712,9 +693,9 @@ checksum = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -763,7 +744,7 @@ checksum = "9d3b95d9fe5b8681bacea6532bbc7632590ae4499cecc8e019685b515fddda71" dependencies = [ "backtrace", "bindgen", - "bitflags 2.4.1", + "bitflags 2.4.2", "cc", "cfg-if", "enum-primitive-derive", @@ -771,7 +752,7 @@ dependencies = [ "linkme", "log", "nix", - "num-traits 0.2.17", + "num-traits 0.2.18", "redis-module-macros-internals", "regex", "serde", @@ -785,7 +766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9c92438cbeaaba0f99e2944ceeb2e34fc8fa7955576296570575df046b81197" dependencies = [ "proc-macro2", - "quote 1.0.33", + "quote 1.0.35", "serde", "serde_syn", "syn 1.0.109", @@ -799,7 +780,7 @@ checksum = "bd1ba5724bf8bfd0c9d6f1169e45255957175b7d81944b325909024a159bc74c" dependencies = [ "lazy_static", "proc-macro2", - "quote 1.0.33", + "quote 1.0.35", "syn 1.0.109", ] @@ -807,7 +788,7 @@ dependencies = [ name = "redis_json" version = "99.99.99" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "bson", "env_logger", "ijson", @@ -832,9 +813,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.2" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", @@ -844,9 +825,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", @@ -873,11 +854,11 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", @@ -892,9 +873,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "scopeguard" @@ -913,9 +894,9 @@ dependencies = [ [[package]] name = "serde_bytes" -version = "0.11.12" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" +checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" dependencies = [ "serde", ] @@ -927,17 +908,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ "proc-macro2", - "quote 1.0.33", - "syn 2.0.41", + "quote 1.0.35", + "syn 2.0.52", ] [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "cb0652c533506ad7a2e353cce269330d6afd8bdfb6d75e0ace5b35aacbd7b9e9" dependencies = [ - "indexmap 2.1.0", + "indexmap", "itoa", "ryu", "serde", @@ -968,15 +949,15 @@ dependencies = [ [[package]] name = "shlex" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "strum_macros" @@ -986,7 +967,7 @@ checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ "heck", "proc-macro2", - "quote 1.0.33", + "quote 1.0.35", "rustversion", "syn 1.0.109", ] @@ -1009,18 +990,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", - "quote 1.0.33", + "quote 1.0.35", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.41" +version = "2.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" +checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" dependencies = [ "proc-macro2", - "quote 1.0.33", + "quote 1.0.35", "unicode-ident", ] @@ -1041,31 +1022,31 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "termcolor" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] [[package]] name = "thiserror" -version = "1.0.51" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.51" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", - "quote 1.0.33", - "syn 2.0.41", + "quote 1.0.35", + "syn 2.0.52", ] [[package]] @@ -1122,9 +1103,9 @@ checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" [[package]] name = "uuid" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ "getrandom", "serde", @@ -1144,9 +1125,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1154,47 +1135,47 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", - "quote 1.0.33", - "syn 2.0.41", + "quote 1.0.35", + "syn 2.0.52", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" dependencies = [ - "quote 1.0.33", + "quote 1.0.35", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", - "quote 1.0.33", - "syn 2.0.41", + "quote 1.0.35", + "syn 2.0.52", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" [[package]] name = "which" @@ -1254,7 +1235,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.4", ] [[package]] @@ -1274,17 +1255,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] [[package]] @@ -1295,9 +1276,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" [[package]] name = "windows_aarch64_msvc" @@ -1307,9 +1288,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" [[package]] name = "windows_i686_gnu" @@ -1319,9 +1300,9 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" [[package]] name = "windows_i686_msvc" @@ -1331,9 +1312,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" [[package]] name = "windows_x86_64_gnu" @@ -1343,9 +1324,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" [[package]] name = "windows_x86_64_gnullvm" @@ -1355,9 +1336,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" [[package]] name = "windows_x86_64_msvc" @@ -1367,9 +1348,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" [[package]] name = "wyz" @@ -1382,20 +1363,20 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.31" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.31" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", - "quote 1.0.33", - "syn 2.0.41", + "quote 1.0.35", + "syn 2.0.52", ] diff --git a/redis_json/src/ivalue_manager.rs b/redis_json/src/ivalue_manager.rs index 2ff2ef4ec..36f483be7 100644 --- a/redis_json/src/ivalue_manager.rs +++ b/redis_json/src/ivalue_manager.rs @@ -716,8 +716,12 @@ impl<'a> Manager for RedisIValueJsonKeyManager<'a> { mod tests { use super::*; + static single_thread_test_mutex: std::sync::Mutex<()> = std::sync::Mutex::new(()); + #[test] fn test_get_memory() { + let _guard = single_thread_test_mutex.lock(); + let manager = RedisIValueJsonKeyManager { phantom: PhantomData, }; @@ -740,4 +744,32 @@ mod tests { let res = manager.get_memory(&value).unwrap(); assert_eq!(res, 903); } + + /// Tests the deserialiser of IValue for a string with unicode + /// characters, to ensure that the deserialiser can handle + /// unicode characters well. + #[test] + fn test_unicode_characters() { + let _guard = single_thread_test_mutex.lock(); + + let json = r#""\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0""#; + let value: IValue = serde_json::from_str(json).expect("IValue parses fine."); + assert_eq!( + value.as_string().unwrap(), + "\u{a0}\u{a0}\u{a0}\u{a0}\u{a0}\u{a0}\u{a0}" + ); + + let json = r#"{"\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0":"\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0"}"#; + let value: IValue = serde_json::from_str(json).expect("IValue parses fine."); + assert_eq!( + value + .as_object() + .unwrap() + .get("\u{a0}\u{a0}\u{a0}\u{a0}\u{a0}\u{a0}\u{a0}") + .unwrap() + .as_string() + .unwrap(), + "\u{a0}\u{a0}\u{a0}\u{a0}\u{a0}\u{a0}\u{a0}" + ); + } } diff --git a/tests/pytest/test.py b/tests/pytest/test.py index 86f4bd1d2..16ebc1a7f 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -136,8 +136,8 @@ def testSetRootWithJSONValuesShouldSucceed(env): r.assertEqual(v, s) def testSetAddNewImmediateChild(env): - - r = env + + r = env r.assertOk(r.execute_command('JSON.SET', 'test', '$', json.dumps(docs))) # Make sure children are initially missing r.assertEqual(r.execute_command('JSON.GET', 'test', '$.basic.dict.new_child_1'), '[]') @@ -183,7 +183,7 @@ def testSetGetWholeBasicDocumentShouldBeEqual(env): def testSetBehaviorModifyingSubcommands(env): """Test JSON.SET's NX and XX subcommands""" r = env - + # test against the root r.assertIsNone(r.execute_command('JSON.SET', 'test', '.', '{}', 'XX')) r.assertOk(r.execute_command('JSON.SET', 'test', '.', '{}', 'NX')) @@ -239,7 +239,7 @@ def testSetGetWithSpecialKey(env): "$a": "$a", "$a[": "$a[" } - + # Set doc using individual keys using legacy syntax (with implicit `$` root) r.assertOk(r.execute_command('JSON.SET', 'x', '$', '{"$":"$"}')) r.assertOk(r.execute_command('JSON.SET', 'x', 'a', '"a"')) @@ -254,7 +254,7 @@ def testSetGetWithSpecialKey(env): # Get key "$" r.assertEqual(json.loads(r.execute_command('JSON.GET', 'x', '$.$')), ["$"]) # dot notation r.assertEqual(json.loads(r.execute_command('JSON.GET', 'x', '$["$"]')), ["$"]) # bracket notation - r.assertEqual(json.loads(r.execute_command('JSON.GET', 'x', '$')), [doc]) + r.assertEqual(json.loads(r.execute_command('JSON.GET', 'x', '$')), [doc]) # Get key "a" r.assertEqual(json.loads(r.execute_command('JSON.GET', 'x', '$.a')), ["a"]) # dot notation r.assertEqual(json.loads(r.execute_command('JSON.GET', 'x', '$["a"]')), ["a"]) # bracket notation @@ -265,7 +265,7 @@ def testSetGetWithSpecialKey(env): r.assertEqual(json.loads(r.execute_command('JSON.GET', 'x', '$a')), "$a") # legacy # Get key "$a[" r.assertEqual(json.loads(r.execute_command('JSON.GET', 'x', '$["$a["]')), ["$a["]) # bracket notation (cannot use dot notation) - + def testSetWithPathErrors(env): r = env @@ -348,7 +348,7 @@ def testGetFormatting(env): r.assertEqual(res, f.format(newline=newline, space=space, indent=indent)) def testBackwardRDB(env): - env.skipOnCluster() + env.skipOnCluster() if env.useAof: env.skip() dbFileName = env.cmd('config', 'get', 'dbfilename')[1] @@ -425,7 +425,7 @@ def testToggleCommand(env): # Test Toggeling Empty Path r.assertOk(r.execute_command('JSON.SET', 'test', '.', '{"foo":"bar"}')) r.expect('JSON.TOGGLE', 'test', '.bar').raiseError() - + # Test Toggeling Non Boolean r.assertOk(r.execute_command('JSON.SET', 'test', '.', '{"foo":"bar"}')) r.expect('JSON.TOGGLE','test','.foo').raiseError() @@ -591,7 +591,7 @@ def testClear(env): r.expect('JSON.CLEAR', 'test', '$.b..a').equal(0) r.expect('JSON.GET', 'test', '$').equal('[{"a":[1,2],"b":{"c":"d"}}]') - # Key doesn't exist + # Key doesn't exist r.expect('JSON.CLEAR', 'not_test_key', '$').raiseError() def testClearScalar(env): @@ -613,7 +613,7 @@ def testClearScalar(env): r.assertEqual(r.execute_command('JSON.CLEAR', 'test', '$.*'), 2) res = r.execute_command('JSON.GET', 'test', '$.*') r.assertEqual(json.loads(res), ['string value', None, True, 0, 0]) - + # Do not clear already cleared values r.assertEqual(r.execute_command('JSON.CLEAR', 'test', '$.*'), 0) @@ -730,9 +730,9 @@ def testArrInsertCommand(env): r.assertEqual(r.execute_command('JSON.GET', 'test', jpath), "[3,2,1,4]") r.assertEqual(r.execute_command('JSON.ARRINSERT', 'test', jpath, 1, '5'), 5) - r.assertEqual(r.execute_command('JSON.ARRINSERT', 'test', jpath, -2, '6'), 6) + r.assertEqual(r.execute_command('JSON.ARRINSERT', 'test', jpath, -2, '6'), 6) r.assertEqual(r.execute_command('JSON.GET', 'test', jpath), "[3,5,2,6,1,4]") - + r.assertEqual(r.execute_command('JSON.ARRINSERT', 'test', jpath, -3, '7', '{"A":"Z"}', '9'), 9) r.assertEqual(r.execute_command('JSON.GET', 'test', jpath), '[3,5,2,7,{"A":"Z"},9,6,1,4]') @@ -925,14 +925,14 @@ def testNumIncrCommand(env): def testNumCommandOverflow(env): """Test JSON.NUMINCRBY and JSON.NUMMULTBY commands overflow """ r = env - + # test overflow on root r.assertOk(r.execute_command('JSON.SET', 'big_num', '.', '1.6350000000001313e+308')) r.expect('JSON.NUMINCRBY', 'big_num', '.', '1.6350000000001313e+308').raiseError() r.expect('JSON.NUMMULTBY', 'big_num', '.', '2').raiseError() # (value remains) r.assertEqual(r.execute_command('JSON.GET', 'big_num', '.'), '1.6350000000001313e308') - + # test overflow on nested object value r.assertOk(r.execute_command('JSON.SET', 'nested_obj_big_num', '$', '{"l1":{"l2_a":1.6350000000001313e+308,"l2_b":2}}')) r.expect('JSON.NUMINCRBY', 'nested_obj_big_num', '$.l1.l2_a', '1.6350000000001313e+308').raiseError() @@ -1081,7 +1081,7 @@ def testMSET(env): def testMSET_Partial(env): - # Make sure MSET doesn't stop on the first update that can't be updated + # Make sure MSET doesn't stop on the first update that can't be updated env.expect("JSON.SET", "a{s}", '$', '{"x": {"y":[10,20], "z":[30,40]}}').ok() env.expect("JSON.SET", "b{s}", '$', '{"x": 60}').ok() env.expect("JSON.MSET", "a{s}", '$.x', '{}', "a{s}", '$.x.z[1]', '50', 'b{s}', '$.x', '70').ok() @@ -1121,13 +1121,13 @@ def testCrashInParserMOD2099(env): r = env r.assertOk(r.execute_command('JSON.SET', 'test', '$', '{"a":{"x":{"i":10}}, "b":{"x":{"i":20, "j":5}}}')) - + res = r.execute_command('JSON.GET', 'test', '$..x[?(@>10)]') r.assertEqual(res, '[20]') - + res = r.execute_command('JSON.GET', 'test', '$..x[?($>10)]') r.assertEqual(res, '[]') - + def testInfoEverything(env): @@ -1138,12 +1138,12 @@ def testInfoEverything(env): def testCopyCommand(env): """Test COPY command and make sure behavior of json keys is similar to hash keys""" - env.skipOnCluster() + env.skipOnCluster() env.skipOnVersionSmaller('6.2') r = env - + values = {"foo": "bar", "fu": "wunderbar"} - + ### Copy json to a new key (from json1 to json2) r.assertOk(r.execute_command('JSON.SET', 'json1', '$', json.dumps(values))) r.assertTrue(r.execute_command('COPY', 'json1', 'json2')) @@ -1160,16 +1160,16 @@ def testCopyCommand(env): # Check new values r.assertEqual(r.execute_command('HGETALL', 'hash1'), values) r.assertEqual(r.execute_command('HGETALL', 'hash2'), values) - + new_values = {"ganz": "neue"} - + ### Copy hash to an existing key - hash_values = list(reduce(lambda acc, v: acc + v, new_values.items())) + hash_values = list(reduce(lambda acc, v: acc + v, new_values.items())) r.assertEqual(r.execute_command('HSET', 'hash3', *hash_values), int(len(hash_values) / 2)) # Do not overwrite without REPLACE (from hash to hash) r.assertFalse(r.execute_command('COPY', 'hash3', 'hash2')) # Do not overwrite without REPLACE (from hash to json) - r.assertFalse(r.execute_command('COPY', 'hash3', 'json2')) + r.assertFalse(r.execute_command('COPY', 'hash3', 'json2')) # Overwrite with REPLACE (from hash to hash) r.assertTrue(r.execute_command('COPY', 'hash3', 'hash2', 'REPLACE')) # Overwrite with REPLACE (from hash to json) @@ -1268,13 +1268,13 @@ def testFilter(env): "pat_not_str5": [".*foo"], } r.expect('JSON.SET', 'doc', '$', json.dumps(doc)).ok() - + # regex match using a static regex pattern r.expect('JSON.GET', 'doc', '$.arr[?(@ =~ ".*foo")]').equal('["kafoosh","foolish","ffool"]') - + # regex match using a field r.expect('JSON.GET', 'doc', '$.arr[?(@ =~ $.pat_regex)]').equal('["kafoosh","foolish","ffool"]') - + # regex case-insensitive match using a field (notice the `.*` before the filter) r.expect('JSON.GET', 'doc', '$.arr.*[?(@ =~ $.pat_plain)]').equal('["foo","FoO"]') @@ -1295,7 +1295,7 @@ def testFilter(env): # plain string match r.expect('JSON.GET', 'doc', '$.arr[?(@ == $.pat_plain)]').equal('["(?i)^[f][o][o]$"]') - + def testFilterExpression(env): # Test JSONPath filter with 3 or more operands r = env @@ -1337,14 +1337,14 @@ def testMerge(env): # Test merge error - invalid JSON r.expect('JSON.MERGE', 'test_merge', '$.a', '{"b":{"h":"i" "bye"}}').error().contains("expected") - # Test with none existing key with path $.a + # Test with none existing key with path $.a r.expect('JSON.MERGE', 'test_merge_new', '$.a', '{"a":"i"}').raiseError() # Test with none existing key -> create key r.assertOk(r.execute_command('JSON.MERGE', 'test_merge_new', '$', '{"h":"i"}')) r.expect('JSON.GET', 'test_merge_new').equal('{"h":"i"}') - - + + def testMergeArray(env): # Test JSON.MERGE with arrays r = env @@ -1414,12 +1414,23 @@ def testRDBUnboundedDepth(env): # concat the string_126 at the end of itself json_value = nest_object(3, 5, "__deep_leaf", 420) r.expect('JSON.SET', 'doc', '$..__leaf', json_value).ok() - + # RDB dump and restore the key 'doc' and check that the key is still valid dump = env.execute_command('dump', 'doc', **{NEVER_DECODE: []}) r.expect('RESTORE', 'doc1', 0, dump).ok() r.expect('JSON.GET', 'doc1', '$..__leaf..__deep_leaf').equal('[420]') +def testUnicodeCharacters(env): + # Test unicode strings parsing and processing. + + r = env + + r.assertOk(r.execute_command('JSON.SET', 'test', '$', '{"\u00a0": "⍢∪⇰"}')) + r.expect('JSON.GET', 'test', '$').equal('[{"\u00a0":"⍢∪⇰"}]') + + r.assertOk(r.execute_command('JSON.SET', 'test', '$', '{"\u00a0": { "name": "\u00a0\u00a0" } }')) + r.expect('JSON.GET', 'test', '$').equal('[{"\u00a0":{"name":"\u00a0\u00a0"}}]') + # class CacheTestCase(BaseReJSONTest): # @property # def module_args(env): diff --git a/tests/pytest/test_multi.py b/tests/pytest/test_multi.py index 8512d1571..fd1a0b5b1 100644 --- a/tests/pytest/test_multi.py +++ b/tests/pytest/test_multi.py @@ -228,7 +228,6 @@ def testSetAndGetCommands(env): r.assertEqual(res, '"inizio"') - def testMGetCommand(env): """Test REJSON.MGET command""" r = env @@ -271,7 +270,7 @@ def testMGetCommand(env): res = r.execute_command('JSON.MGET', '{doc}:1', '{doc}:2', '..a') r.assertEqual(res, [json.dumps(json.loads(res1)[0]), json.dumps(json.loads(res2)[0])]) - # Test wrong key + # Test wrong key res = r.execute_command('JSON.MGET', '{doc}:1', '{doc}:wrong_key_type', '{doc}:2', '..a') r.assertEqual(res, [json.dumps(json.loads(res1)[0]), None, json.dumps(json.loads(res2)[0])]) @@ -821,7 +820,7 @@ def testToggleCommand(env): # Test MEMORY USAGE key # """ # r = env -# jdata, jtypes = load_types_data('a') +# jdata, jtypes = load_types_data('a') # r.assertOk(r.execute_command('JSON.SET', 'doc1', '$', json.dumps(jdata))) # res = r.execute_command('MEMORY', 'USAGE', 'doc1') # r.assertEqual(res, 211) @@ -1072,7 +1071,7 @@ def testErrorMessage(env): json.arrpop doc1 .bzzz 1 (error) ERR key 'bzzz' does not exist at level 0 in path json.arrpop doc1zzz .b 1 - (error) WRONGTYPE Operation against a key holding the wrong kind of value + (error) WRONGTYPE Operation against a key holding the wrong kind of value """ # ARRINDEX @@ -1269,11 +1268,11 @@ def testErrorMessage(env): """ Legacy 1.0.8: json.set doc1 . '{"a":[0, 1, 2, 3, 4, 5], "b":{"x":100}}' OK - json.strlen doc1 .b + json.strlen doc1 .b (error) ERR wrong type of path value - expected string but found object - json.strlen doc1 .bzzz + json.strlen doc1 .bzzz (error) ERR key 'bzzz' does not exist at level 0 in path - json.strlen doc1zzz .b + json.strlen doc1zzz .b (nil) """ @@ -1345,7 +1344,7 @@ def testErrorMessage(env): def testFilterDup_issue667(env): """Test issue #667 """ r = env - + r.assertOk(r.execute_command('JSON.SET', 'test', '$', From 44f872b7b72eff479f681515aeb21a37ab66dedf Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Wed, 6 Mar 2024 16:45:18 +0200 Subject: [PATCH 027/112] promote u64 to f64 (cherry picked from commit c3452731b02ee77c9fe33442a0942c70ccf7ace9) --- .gitignore | 5 ++- json_path/src/json_node.rs | 42 ++++++++++++------------ json_path/src/select_value.rs | 13 ++++---- redis_json/src/ivalue_manager.rs | 20 +++++++----- redis_json/src/key_value.rs | 10 +++++- tests/pytest/test.py | 56 ++++++++++++++++++++++++++++++++ 6 files changed, 107 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 426d97c80..afc3b3a87 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,7 @@ wordlist.dic -venv/ \ No newline at end of file +# RLTest +config.txt + +venv/ diff --git a/json_path/src/json_node.rs b/json_path/src/json_node.rs index 6df0d7dec..85eff372b 100644 --- a/json_path/src/json_node.rs +++ b/json_path/src/json_node.rs @@ -17,15 +17,9 @@ impl SelectValue for Value { Self::Null => SelectValueType::Null, Self::Array(_) => SelectValueType::Array, Self::Object(_) => SelectValueType::Object, - Self::Number(n) => { - if n.is_i64() || n.is_u64() { - SelectValueType::Long - } else if n.is_f64() { - SelectValueType::Double - } else { - panic!("bad type for Number value"); - } - } + Self::Number(n) if n.is_i64() => SelectValueType::Long, + Self::Number(n) if n.is_f64() | n.is_u64() => SelectValueType::Double, + _ => panic!("bad type for Number value"), } } @@ -92,6 +86,13 @@ impl SelectValue for Value { matches!(self, Self::Array(_)) } + fn is_double(&self) -> Option { + match self { + Self::Number(num) => Some(num.is_f64()), + _ => None, + } + } + fn get_str(&self) -> String { match self { Self::String(s) => s.to_string(), @@ -160,7 +161,7 @@ impl SelectValue for IValue { ValueType::Object => SelectValueType::Object, ValueType::Number => { let num = self.as_number().unwrap(); - if num.has_decimal_point() { + if num.has_decimal_point() | num.to_i64().is_none() { SelectValueType::Double } else { SelectValueType::Long @@ -218,6 +219,10 @@ impl SelectValue for IValue { self.is_array() } + fn is_double(&self) -> Option { + Some(self.as_number()?.has_decimal_point()) + } + fn get_str(&self) -> String { self.as_string().expect("not a string").to_string() } @@ -231,20 +236,13 @@ impl SelectValue for IValue { } fn get_long(&self) -> i64 { - let n = self.as_number().expect("not a number"); - if n.has_decimal_point() { - panic!("not a long"); - } else { - n.to_i64().unwrap() - } + self.as_number() + .expect("not a number") + .to_i64() + .expect("not a long") } fn get_double(&self) -> f64 { - let n = self.as_number().expect("not a number"); - if n.has_decimal_point() { - n.to_f64().unwrap() - } else { - panic!("not a double"); - } + self.as_number().expect("not a number").to_f64_lossy() } } diff --git a/json_path/src/select_value.rs b/json_path/src/select_value.rs index 3410a6650..243c029dd 100644 --- a/json_path/src/select_value.rs +++ b/json_path/src/select_value.rs @@ -1,9 +1,9 @@ -/* - * Copyright Redis Ltd. 2016 - present - * Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or - * the Server Side Public License v1 (SSPLv1). - */ - +/* + * Copyright Redis Ltd. 2016 - present + * Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + * the Server Side Public License v1 (SSPLv1). + */ + use serde::Serialize; use std::fmt::Debug; @@ -29,6 +29,7 @@ pub trait SelectValue: Debug + Eq + PartialEq + Default + Clone + Serialize { fn get_key<'a>(&'a self, key: &str) -> Option<&'a Self>; fn get_index(&self, index: usize) -> Option<&Self>; fn is_array(&self) -> bool; + fn is_double(&self) -> Option; fn get_str(&self) -> String; fn as_str(&self) -> &str; diff --git a/redis_json/src/ivalue_manager.rs b/redis_json/src/ivalue_manager.rs index 36f483be7..27daa74f0 100644 --- a/redis_json/src/ivalue_manager.rs +++ b/redis_json/src/ivalue_manager.rs @@ -17,6 +17,7 @@ use redis_module::key::{verify_type, KeyFlags, RedisKey, RedisKeyWritable}; use redis_module::raw::{RedisModuleKey, Status}; use redis_module::rediserror::RedisError; use redis_module::{Context, NotifyEvent, RedisResult, RedisString}; +use json_path::select_value::{SelectValue, SelectValueType}; use serde::{Deserialize, Serialize}; use serde_json::Number; use std::io::Cursor; @@ -213,15 +214,16 @@ impl<'a> IValueKeyHolderWrite<'a> { if let serde_json::Value::Number(in_value) = in_value { let mut res = None; self.do_op(&path, |v| { - let num_res = match ( - v.as_number().unwrap().has_decimal_point(), - in_value.as_i64(), - ) { - (false, Some(num2)) => Ok(((op1_fun)(v.to_i64().unwrap(), num2)).into()), + let num_res = match (v.get_type(), in_value.as_i64()) { + (SelectValueType::Long, Some(num2)) => { + let num1 = v.get_long(); + let res = op1_fun(num1, num2); + Ok(res.into()) + } _ => { - let num1 = v.to_f64().unwrap(); + let num1 = v.get_double(); let num2 = in_value.as_f64().unwrap(); - INumber::try_from((op2_fun)(num1, num2)) + INumber::try_from(op2_fun(num1, num2)) .map_err(|_| RedisError::Str("result is not a number")) } }; @@ -374,11 +376,11 @@ impl<'a> WriteHolder for IValueKeyHolderWrite<'a> { } fn incr_by(&mut self, path: Vec, num: &str) -> Result { - self.do_num_op(path, num, |i1, i2| i1 + i2, |f1, f2| f1 + f2) + self.do_num_op(path, num, i64::wrapping_add, |f1, f2| f1 + f2) } fn mult_by(&mut self, path: Vec, num: &str) -> Result { - self.do_num_op(path, num, |i1, i2| i1 * i2, |f1, f2| f1 * f2) + self.do_num_op(path, num, i64::wrapping_mul, |f1, f2| f1 * f2) } fn pow_by(&mut self, path: Vec, num: &str) -> Result { diff --git a/redis_json/src/key_value.rs b/redis_json/src/key_value.rs index c78b3e33f..4c05ac434 100644 --- a/redis_json/src/key_value.rs +++ b/redis_json/src/key_value.rs @@ -380,7 +380,15 @@ impl<'a, V: SelectValue + 'a> KeyValue<'a, V> { SelectValueType::Null => "null", SelectValueType::Bool => "boolean", SelectValueType::Long => "integer", - SelectValueType::Double => "number", + // For dealing with u64 values over i64::MAX, get_type() replies + // that they are SelectValueType::Double to prevent panics from + // incorrect casts. However when querying the type of such a value, + // any response other than 'integer' is a breaking change + SelectValueType::Double => match value.is_double() { + Some(true) => "number", + Some(false) => "integer", + _ => unreachable!(), + }, SelectValueType::String => "string", SelectValueType::Array => "array", SelectValueType::Object => "object", diff --git a/tests/pytest/test.py b/tests/pytest/test.py index 16ebc1a7f..6f7546d34 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -1431,6 +1431,62 @@ def testUnicodeCharacters(env): r.assertOk(r.execute_command('JSON.SET', 'test', '$', '{"\u00a0": { "name": "\u00a0\u00a0" } }')) r.expect('JSON.GET', 'test', '$').equal('[{"\u00a0":{"name":"\u00a0\u00a0"}}]') +def test_promote_u64_to_f64(env): + r = env + i64max = 2 ** 63 - 1 + + # i64 + i64 behaves normally + r.expect('JSON.SET', 'num', '$', 0).ok() + r.expect('JSON.TYPE', 'num', '$').equal(['integer']) + res = r.execute_command('JSON.GET', 'num', '$') + val = json.loads(res)[0] + r.assertEqual(val, 0) + res = r.execute_command('JSON.NUMINCRBY', 'num', '$', i64max) + val = json.loads(res)[0] + r.assertEqual(val, i64max) # i64 + i64 no overflow + r.assertNotEqual(val, float(i64max)) # i64max is not representable as f64 + r.expect('JSON.TYPE', 'num', '$').equal(['integer']) # no promotion + res = r.execute_command('JSON.NUMINCRBY', 'num', '$', 1) + val = json.loads(res)[0] + r.assertEqual(val, -(i64max + 1)) # i64 + i64 overflow wraps. as prior, not breaking + r.assertNotEqual(val, i64max + 1) # i64 + i64 is not promoted to u64 + r.assertNotEqual(val, float(i64max) + float(1)) # i64 + i64 is not promoted to f64 + r.expect('JSON.TYPE', 'num', '$').equal(['integer']) # no promotion + + # i64 + u64 used to have inconsistent behavior + r.expect('JSON.SET', 'num', '$', 0).ok() + res = r.execute_command('JSON.NUMINCRBY', 'num', '$', i64max + 2) + val = json.loads(res)[0] + r.assertNotEqual(val, -(i64max + 1) + 1) # i64 + u64 is not i64 + r.assertNotEqual(val, i64max + 2) # i64 + u64 is not u64 + r.assertEqual(val, float(i64max + 2)) # i64 + u64 promotes to f64. as prior, not breaking + r.expect('JSON.TYPE', 'num', '$').equal(['number']) # promoted + + # u64 + i64 used to crash + r.expect('JSON.SET', 'num', '$', i64max + 1).ok() + r.expect('JSON.TYPE', 'num', '$').equal(['integer']) # as prior, not breaking + res = r.execute_command('JSON.GET', 'num', '$') + val = json.loads(res)[0] + r.assertNotEqual(val, -(i64max + 1)) # not i64 + r.assertEqual(val, i64max + 1) # as prior, not breaking + res = r.execute_command('JSON.NUMINCRBY', 'num', '$', 1) + val = json.loads(res)[0] + r.assertNotEqual(val, -(i64max + 1) + 1) # u64 + i64 is not i64 + r.assertNotEqual(val, i64max + 2) # u64 + i64 is not u64 + r.assertEqual(val, float(i64max + 2)) # u64 + i64 promotes to f64. used to crash + r.expect('JSON.TYPE', 'num', '$').equal(['number']) # promoted + + # u64 + u64 used to have inconsistent behavior + r.expect('JSON.SET', 'num', '$', i64max + 1).ok() + r.expect('JSON.CLEAR', 'num', '$').equal(1) # clear u64 used to crash + r.expect('JSON.SET', 'num', '$', i64max + 1).ok() + res = r.execute_command('JSON.NUMINCRBY', 'num', '$', i64max + 2) + val = json.loads(res)[0] + r.assertNotEqual(val, -(i64max + 1) + i64max + 2) # u64 + u64 is not i64 + r.assertNotEqual(val, 2) # u64 + u64 is not u64 + r.assertEqual(val, float(2 * i64max + 3)) # u64 + u64 promotes to f64. as prior, not breaking + r.expect('JSON.TYPE', 'num', '$').equal(['number']) # promoted + # class CacheTestCase(BaseReJSONTest): # @property # def module_args(env): From 9c648b9fdbf0510727f45f7d0fceac6dc39070a4 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Wed, 6 Mar 2024 16:50:46 +0200 Subject: [PATCH 028/112] fmt (cherry picked from commit 1de5d77403dbeec488bb126db5ee68b32229b82a) --- redis_json/src/ivalue_manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis_json/src/ivalue_manager.rs b/redis_json/src/ivalue_manager.rs index 27daa74f0..16c89a2b4 100644 --- a/redis_json/src/ivalue_manager.rs +++ b/redis_json/src/ivalue_manager.rs @@ -13,11 +13,11 @@ use crate::REDIS_JSON_TYPE; use bson::{from_document, Document}; use ijson::object::Entry; use ijson::{DestructuredMut, INumber, IObject, IString, IValue, ValueType}; +use json_path::select_value::{SelectValue, SelectValueType}; use redis_module::key::{verify_type, KeyFlags, RedisKey, RedisKeyWritable}; use redis_module::raw::{RedisModuleKey, Status}; use redis_module::rediserror::RedisError; use redis_module::{Context, NotifyEvent, RedisResult, RedisString}; -use json_path::select_value::{SelectValue, SelectValueType}; use serde::{Deserialize, Serialize}; use serde_json::Number; use std::io::Cursor; From dbfe6a671dcf5672dcb4b6c2e94e9b702e916b72 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Wed, 13 Mar 2024 13:11:04 +0200 Subject: [PATCH 029/112] fix break from 1992 (cherry picked from commit 27e72cc9cd59eb9e4ab8494b0ab8c9ea8e0e50be) --- json_path/src/json_node.rs | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/json_path/src/json_node.rs b/json_path/src/json_node.rs index 85eff372b..3fcf860b1 100644 --- a/json_path/src/json_node.rs +++ b/json_path/src/json_node.rs @@ -18,7 +18,7 @@ impl SelectValue for Value { Self::Array(_) => SelectValueType::Array, Self::Object(_) => SelectValueType::Object, Self::Number(n) if n.is_i64() => SelectValueType::Long, - Self::Number(n) if n.is_f64() | n.is_u64() => SelectValueType::Double, + Self::Number(n) if n.is_f64() || n.is_u64() => SelectValueType::Double, _ => panic!("bad type for Number value"), } } @@ -122,31 +122,16 @@ impl SelectValue for Value { fn get_long(&self) -> i64 { match self { - Self::Number(n) => { - if let Some(n) = n.as_i64() { - n - } else { - panic!("not a long"); - } - } - _ => { - panic!("not a long"); - } + Self::Number(n) if n.is_i64() => n.as_i64().unwrap(), + _ => panic!("not a long"), } } fn get_double(&self) -> f64 { match self { - Self::Number(n) => { - if n.is_f64() { - n.as_f64().unwrap() - } else { - panic!("not a double"); - } - } - _ => { - panic!("not a double"); - } + Self::Number(n) if n.is_f64() => n.as_f64().unwrap(), + Self::Number(n) if n.is_u64() => n.as_u64().unwrap() as _, + _ => panic!("not a double"), } } } From 805180debb4a39cd8dac9c31e4694e5f7c709c0a Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Wed, 13 Mar 2024 13:14:58 +0200 Subject: [PATCH 030/112] bool to bit (cherry picked from commit ac6b3506be72df2711266d9d624a2ed3af038185) --- json_path/src/json_node.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json_path/src/json_node.rs b/json_path/src/json_node.rs index 3fcf860b1..22193de56 100644 --- a/json_path/src/json_node.rs +++ b/json_path/src/json_node.rs @@ -18,7 +18,7 @@ impl SelectValue for Value { Self::Array(_) => SelectValueType::Array, Self::Object(_) => SelectValueType::Object, Self::Number(n) if n.is_i64() => SelectValueType::Long, - Self::Number(n) if n.is_f64() || n.is_u64() => SelectValueType::Double, + Self::Number(n) if n.is_f64() | n.is_u64() => SelectValueType::Double, _ => panic!("bad type for Number value"), } } From a5c2683260ff199e8aa7fda99d5990942c6b40ad Mon Sep 17 00:00:00 2001 From: Shockingly Good Date: Wed, 27 Mar 2024 13:13:13 +0100 Subject: [PATCH 031/112] Fix the tests.sh script. If the status file does not exist and we are creating it, don't fail with an error if the file doesn't exist; it obviously won't exist until we create it. This change creates the file in case it didn't exist, resulting in the proper continuation of the script and a proper exit code. --- tests/pytest/tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/tests.sh b/tests/pytest/tests.sh index 9f5485feb..811f3dbda 100755 --- a/tests/pytest/tests.sh +++ b/tests/pytest/tests.sh @@ -665,6 +665,7 @@ fi if [[ -n $STATFILE ]]; then mkdir -p "$(dirname "$STATFILE")" + touch $STATFILE if [[ -f $STATFILE ]]; then (( E |= $(cat $STATFILE || echo 1) )) || true fi From 8b556bcd135a9fd9207920611f08520c1b42d548 Mon Sep 17 00:00:00 2001 From: josh rotenberg Date: Mon, 15 Apr 2024 16:00:23 -0700 Subject: [PATCH 032/112] fix a couple links in the README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2b78f2817..3d0bd8d11 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ RedisJSON is a [Redis](https://redis.io/) module that implements [ECMA-404 The J * [JSONPath](https://goessner.net/articles/JsonPath/) syntax for selecting elements inside documents * Documents are stored as binary data in a tree structure, allowing fast access to sub-elements * Typed atomic operations for all JSON values types -* Secondary index support when combined with [RediSearch](https://redisearch.io) +* Secondary index support when combined with [RediSearch](https://redis.io/docs/latest/develop/interact/search-and-query/) ## Quick start @@ -31,7 +31,7 @@ docker run -p 6379:6379 --name redis-stack redis/redis-stack:latest ## Documentation -Read the docs at +Read the docs at ### How do I Redis? From 9c3a389bcf7c8a44fb4eddc198c7642757c07dc7 Mon Sep 17 00:00:00 2001 From: YaacovHazan Date: Sat, 18 May 2024 22:42:13 +0300 Subject: [PATCH 033/112] add compatible_redis_version in ramp.yml --- ramp.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ramp.yml b/ramp.yml index 5fb95ae68..fbf7f6040 100644 --- a/ramp.yml +++ b/ramp.yml @@ -8,6 +8,7 @@ license: Redis Source Available License 2.0 (RSALv2) or the Server Side Public L command_line_args: "" min_redis_version: "7.1" min_redis_pack_version: "7.2" +compatible_redis_version: "7.2" capabilities: - types - no_multi_key From e9da8e384362601c4e66acdfc89c5b8a802f82c4 Mon Sep 17 00:00:00 2001 From: Victor Polevoy Date: Tue, 21 May 2024 15:32:24 +0200 Subject: [PATCH 034/112] Fix the pytest/tests.sh script --- Cargo.lock | 427 +++++++++++++++++------------------------- tests/pytest/tests.sh | 24 +-- 2 files changed, 185 insertions(+), 266 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b05cb5fd1..06f500d27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b79b82693f705137f8fb9b37871d99e4f9a7df12b917eed79c3d3954830a60b" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "getrandom", @@ -32,24 +32,24 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] [[package]] name = "autocfg" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" dependencies = [ "addr2line", "cc", @@ -72,7 +72,7 @@ version = "0.66.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "cexpr", "clang-sys", "lazy_static", @@ -81,11 +81,11 @@ dependencies = [ "peeking_take_while", "prettyplease", "proc-macro2", - "quote 1.0.35", + "quote 1.0.36", "regex", "rustc-hash", "shlex", - "syn 2.0.52", + "syn 2.0.65", "which", ] @@ -97,9 +97,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "bitvec" @@ -124,9 +124,9 @@ dependencies = [ [[package]] name = "bson" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce21468c1c9c154a85696bb25c20582511438edb6ad67f846ba1378ffdd80222" +checksum = "4d43b38e074cc0de2957f10947e376a1d88b9c4dbab340b590800cc1b2e066b2" dependencies = [ "ahash", "base64", @@ -145,15 +145,15 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.15.3" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "cc" -version = "1.0.88" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02f341c093d19155a6e41631ce5971aac4e9a868262212153124c15fa22d1cdc" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" [[package]] name = "cexpr" @@ -207,7 +207,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core", @@ -231,9 +231,9 @@ dependencies = [ [[package]] name = "either" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" [[package]] name = "enum-primitive-derive" @@ -267,12 +267,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -293,9 +293,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -325,9 +325,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "heck" @@ -353,7 +353,7 @@ version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -376,12 +376,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.4" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "967d6dd42f16dbf0eb8040cb9e477933562684d3918f7d253f2ff9087fb3e7a3" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -392,7 +392,7 @@ checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -406,15 +406,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -448,51 +448,51 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libloading" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" +checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" dependencies = [ "cfg-if", - "windows-sys 0.48.0", + "windows-targets", ] [[package]] name = "linkme" -version = "0.3.23" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a78816ac097580aa7fd9d2e9cc7395dda34367c07267a8657516d4ad5e2e3d3" +checksum = "833222afbfe72868ac8f9770c91a33673f0d5fefc37c9dbe94aa3548b571623f" dependencies = [ "linkme-impl", ] [[package]] name = "linkme-impl" -version = "0.3.23" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee9023a564f8bf7fe3da285a50c3e70de0df3e2bf277ff7c4e76d66008ef93b0" +checksum = "39f0dea92dbea3271557cc2e1848723967bba81f722f95026860974ec9283f08" dependencies = [ "proc-macro2", - "quote 1.0.35", - "syn 2.0.52", + "quote 1.0.36", + "syn 2.0.65", ] [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -506,9 +506,9 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "memoffset" @@ -527,9 +527,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" dependencies = [ "adler", ] @@ -563,14 +563,14 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" dependencies = [ - "num-traits 0.2.18", + "num-traits 0.2.19", ] [[package]] name = "num-traits" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] @@ -592,15 +592,15 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.48.5", + "windows-targets", ] [[package]] @@ -611,9 +611,9 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "pest" -version = "2.7.7" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c0dcc30b6a27553f9cc242972b67f75b60eb0db71f0b5462f38b058c41546" +checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" dependencies = [ "memchr", "thiserror", @@ -622,9 +622,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.7" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e1288dbd7786462961e69bfd4df7848c1e37e8b74303dbdab82c3a9cdd2809" +checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459" dependencies = [ "pest", "pest_generator", @@ -632,22 +632,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.7" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1381c29a877c6d34b8c176e734f35d7f7f5b3adaefe940cb4d1bb7af94678e2e" +checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" dependencies = [ "pest", "pest_meta", "proc-macro2", - "quote 1.0.35", - "syn 2.0.52", + "quote 1.0.36", + "syn 2.0.65", ] [[package]] name = "pest_meta" -version = "2.7.7" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0934d6907f148c22a3acbda520c7eed243ad7487a30f51f6ce52b58b7077a8a" +checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" dependencies = [ "once_cell", "pest", @@ -668,19 +668,19 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.16" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.52", + "syn 2.0.65", ] [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "0b33eb56c327dec362a9e55b3ad14f9d2f0904fb5a5b03b513ab5465399e9f43" dependencies = [ "unicode-ident", ] @@ -693,9 +693,9 @@ checksum = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -744,7 +744,7 @@ checksum = "9d3b95d9fe5b8681bacea6532bbc7632590ae4499cecc8e019685b515fddda71" dependencies = [ "backtrace", "bindgen", - "bitflags 2.4.2", + "bitflags 2.5.0", "cc", "cfg-if", "enum-primitive-derive", @@ -752,7 +752,7 @@ dependencies = [ "linkme", "log", "nix", - "num-traits 0.2.18", + "num-traits 0.2.19", "redis-module-macros-internals", "regex", "serde", @@ -766,7 +766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9c92438cbeaaba0f99e2944ceeb2e34fc8fa7955576296570575df046b81197" dependencies = [ "proc-macro2", - "quote 1.0.35", + "quote 1.0.36", "serde", "serde_syn", "syn 1.0.109", @@ -780,7 +780,7 @@ checksum = "bd1ba5724bf8bfd0c9d6f1169e45255957175b7d81944b325909024a159bc74c" dependencies = [ "lazy_static", "proc-macro2", - "quote 1.0.35", + "quote 1.0.36", "syn 1.0.109", ] @@ -788,7 +788,7 @@ dependencies = [ name = "redis_json" version = "99.99.99" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "bson", "env_logger", "ijson", @@ -804,18 +804,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", ] [[package]] name = "regex" -version = "1.10.3" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", @@ -825,9 +825,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", @@ -836,15 +836,15 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" @@ -854,28 +854,28 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.31" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "ryu" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "scopeguard" @@ -908,8 +908,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ "proc-macro2", - "quote 1.0.35", - "syn 2.0.52", + "quote 1.0.36", + "syn 2.0.65", ] [[package]] @@ -955,9 +955,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "smallvec" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "strum_macros" @@ -967,7 +967,7 @@ checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ "heck", "proc-macro2", - "quote 1.0.35", + "quote 1.0.36", "rustversion", "syn 1.0.109", ] @@ -990,18 +990,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", - "quote 1.0.35", + "quote 1.0.36", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.52" +version = "2.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" +checksum = "d2863d96a84c6439701d7a38f9de935ec562c8832cc55d1dde0f513b52fad106" dependencies = [ "proc-macro2", - "quote 1.0.35", + "quote 1.0.36", "unicode-ident", ] @@ -1031,22 +1031,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.57" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.57" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", - "quote 1.0.35", - "syn 2.0.52", + "quote 1.0.36", + "syn 2.0.65", ] [[package]] @@ -1103,9 +1103,9 @@ checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" [[package]] name = "uuid" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ "getrandom", "serde", @@ -1125,9 +1125,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1135,47 +1135,47 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", - "quote 1.0.35", - "syn 2.0.52", + "quote 1.0.36", + "syn 2.0.65", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ - "quote 1.0.35", + "quote 1.0.36", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", - "quote 1.0.35", - "syn 2.0.52", + "quote 1.0.36", + "syn 2.0.65", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "which" @@ -1189,44 +1189,13 @@ dependencies = [ "rustix", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" dependencies = [ - "winapi", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", + "windows-sys", ] [[package]] @@ -1235,122 +1204,72 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.4", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows-targets", ] [[package]] name = "windows-targets" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.4", - "windows_aarch64_msvc 0.52.4", - "windows_i686_gnu 0.52.4", - "windows_i686_msvc 0.52.4", - "windows_x86_64_gnu 0.52.4", - "windows_x86_64_gnullvm 0.52.4", - "windows_x86_64_msvc 0.52.4", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" -version = "0.52.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" [[package]] -name = "windows_i686_msvc" -version = "0.48.5" +name = "windows_i686_gnullvm" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" -version = "0.52.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "wyz" @@ -1363,20 +1282,20 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.32" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", - "quote 1.0.35", - "syn 2.0.52", + "quote 1.0.36", + "syn 2.0.65", ] diff --git a/tests/pytest/tests.sh b/tests/pytest/tests.sh index 811f3dbda..f204da86f 100755 --- a/tests/pytest/tests.sh +++ b/tests/pytest/tests.sh @@ -23,9 +23,9 @@ cd $HERE help() { cat <<-'END' Run Python tests using RLTest - + [ARGVARS...] tests.sh [--help|help] [] - + Argument variables: MODULE=path Path to redisjson.so MODARGS=args RediSearch module arguments @@ -57,7 +57,7 @@ help() { COV=1 Run with coverage analysis VG=1 Run with Valgrind VG_LEAKS=0 Do not detect leaks - SAN=type Use LLVM sanitizer (type=address|memory|leak|thread) + SAN=type Use LLVM sanitizer (type=address|memory|leak|thread) BB=1 Enable Python debugger (break using BB() in tests) GDB=1 Enable interactive gdb debugging (in single-test mode) @@ -87,7 +87,7 @@ help() { END } -#---------------------------------------------------------------------------------------------- +#---------------------------------------------------------------------------------------------- traps() { local func="$1" @@ -120,7 +120,7 @@ stop() { traps 'stop' SIGINT -#---------------------------------------------------------------------------------------------- +#---------------------------------------------------------------------------------------------- setup_rltest() { if [[ $RLTEST == view ]]; then @@ -143,7 +143,7 @@ setup_rltest() { echo "PYTHONPATH=$PYTHONPATH" fi fi - + if [[ $RLTEST_VERBOSE == 1 ]]; then RLTEST_ARGS+=" -v" fi @@ -178,7 +178,7 @@ setup_clang_sanitizer() { # for RLTest export SANITIZER="$SAN" export SHORT_READ_BYTES_DELTA=512 - + # --no-output-catch --exit-on-failure --check-exitcode RLTEST_SAN_ARGS="--sanitizer $SAN" @@ -402,7 +402,7 @@ run_tests() { fi [[ $RLEC == 1 ]] && export RLEC_CLUSTER=1 - + local E=0 if [[ $NOP != 1 ]]; then { $OP python3 -m RLTest @$rltest_config; (( E |= $? )); } || true @@ -416,7 +416,7 @@ run_tests() { echo "killing external redis-server: $XREDIS_PID" kill -TERM $XREDIS_PID fi - + if [[ -n $GITHUB_ACTIONS ]]; then echo "::endgroup::" fi @@ -552,7 +552,7 @@ if [[ $LIST == 1 ]]; then RLTEST_ARGS+=" --collect-only" fi -#---------------------------------------------------------------------------------------------- +#---------------------------------------------------------------------------------------------- if [[ $QUICK == 1 ]]; then GEN=${GEN:-1} @@ -665,9 +665,9 @@ fi if [[ -n $STATFILE ]]; then mkdir -p "$(dirname "$STATFILE")" - touch $STATFILE if [[ -f $STATFILE ]]; then - (( E |= $(cat $STATFILE || echo 1) )) || true + VALUE=$(cat $STATFILE 2>/dev/null || echo 1) + (( E |= VALUE )) || true fi echo $E > $STATFILE fi From 3d505c6010f4a8a81ff8f9737ff2b72a96667994 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Sun, 23 Jun 2024 13:05:32 +0300 Subject: [PATCH 035/112] update s3 bucket --- tests/pytest/test_short_read.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/test_short_read.py b/tests/pytest/test_short_read.py index 832c43d12..fbc9b8919 100644 --- a/tests/pytest/test_short_read.py +++ b/tests/pytest/test_short_read.py @@ -20,7 +20,7 @@ Defaults.no_log = True CREATE_INDICES_TARGET_DIR = '/tmp/test' -BASE_RDBS_URL = 'https://s3.amazonaws.com/redismodules/redisearch-oss/rdbs/' +BASE_RDBS_URL = 'https://dev.cto.redis.s3.amazonaws.com/RediSearch/rdbs/' SHORT_READ_BYTES_DELTA = int(os.getenv('SHORT_READ_BYTES_DELTA', '1')) OS = os.getenv('OS') From 7879b9065af3ae4217b71343ebd32a8a3ae4b612 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Sun, 23 Jun 2024 13:32:38 +0300 Subject: [PATCH 036/112] certificate --- tests/pytest/test_short_read.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/test_short_read.py b/tests/pytest/test_short_read.py index fbc9b8919..53bb96915 100644 --- a/tests/pytest/test_short_read.py +++ b/tests/pytest/test_short_read.py @@ -49,7 +49,7 @@ def downloadFiles(target_dir): if not os.path.exists(path_dir): os.makedirs(path_dir) if not os.path.exists(path): - subprocess.call(['wget', '-q', BASE_RDBS_URL + f, '-O', path]) + subprocess.call(['wget', '--no-check-certificate', '-q', BASE_RDBS_URL + f, '-O', path]) _, ext = os.path.splitext(f) if ext == '.zip': if not unzip(path, path_dir): From 6d6bb5706346eca326f3e905db4e4104472022e5 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Sun, 23 Jun 2024 19:03:11 +0000 Subject: [PATCH 037/112] 1. Write to AOF outside the iterator 2. Added notify_keyspace_event API to manager --- redis_json/src/commands.rs | 9 ++++--- redis_json/src/ivalue_manager.rs | 8 ++++++ redis_json/src/manager.rs | 1 + tests/pytest/test.py | 43 ++++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 3 deletions(-) diff --git a/redis_json/src/commands.rs b/redis_json/src/commands.rs index 51f5f167f..ea22ef221 100644 --- a/redis_json/src/commands.rs +++ b/redis_json/src/commands.rs @@ -344,7 +344,7 @@ pub fn json_mset(manager: M, ctx: &Context, args: Vec) actions.push((redis_key, update_info, value)); } - actions + let res = actions .into_iter() .fold(REDIS_OK, |res, (mut redis_key, update_info, value)| { let updated = if let Some(update_info) = update_info { @@ -354,10 +354,13 @@ pub fn json_mset(manager: M, ctx: &Context, args: Vec) redis_key.set_value(Vec::new(), value)? }; if updated { - redis_key.apply_changes(ctx, "json.mset")? + redis_key.notify_keyspace_event(ctx, "json.mset")? } res - }) + }); + + ctx.replicate_verbatim(); + res } fn apply_updates( diff --git a/redis_json/src/ivalue_manager.rs b/redis_json/src/ivalue_manager.rs index 16c89a2b4..6ece0558f 100644 --- a/redis_json/src/ivalue_manager.rs +++ b/redis_json/src/ivalue_manager.rs @@ -290,6 +290,14 @@ impl<'a> WriteHolder for IValueKeyHolderWrite<'a> { } } + fn notify_keyspace_event(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError> { + if ctx.notify_keyspace_event(NotifyEvent::MODULE, command, &self.key_name) != Status::Ok { + Err(RedisError::Str("failed notify key space event")) + } else { + Ok(()) + } + } + fn delete(&mut self) -> Result<(), RedisError> { self.key.delete()?; Ok(()) diff --git a/redis_json/src/manager.rs b/redis_json/src/manager.rs index 743fb83ca..d1ac65038 100644 --- a/redis_json/src/manager.rs +++ b/redis_json/src/manager.rs @@ -64,6 +64,7 @@ pub trait WriteHolder { fn arr_trim(&mut self, path: Vec, start: i64, stop: i64) -> Result; fn clear(&mut self, path: Vec) -> Result; fn apply_changes(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError>; + fn notify_keyspace_event(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError>; } pub trait Manager { diff --git a/tests/pytest/test.py b/tests/pytest/test.py index 6f7546d34..d9026b3a0 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -6,6 +6,7 @@ import os import redis import json +import time from RLTest import Env from includes import * from redis.client import NEVER_DECODE @@ -1487,6 +1488,48 @@ def test_promote_u64_to_f64(env): r.assertEqual(val, float(2 * i64max + 3)) # u64 + u64 promotes to f64. as prior, not breaking r.expect('JSON.TYPE', 'num', '$').equal(['number']) # promoted + +def test_mset_replication_in_aof(env): + if not env.useAof: + env.skip() + with env.getClusterConnectionIfNeeded() as r: + r.execute_command('config', 'set', 'notify-keyspace-events', 'KEA') + + pubsub = r.pubsub() + pubsub.psubscribe('__key*') + + time.sleep(1) + env.assertEqual('psubscribe', pubsub.get_message(timeout=1)['type']) + + command = [b'JSON.MSET'] + data = 'a' * 100 + num_params = 5 + for i in range(0, num_params): + key = f'k:{i}' + value = json.dumps({"data": data}) + command.append(key.encode()) + command.append(b'$') + command.append(value.encode()) + env.expect(*command).ok() + + for i in range(0, num_params): + msg = pubsub.get_message(timeout=1) + env.assertEqual(msg['data'], 'json.mset') + env.assertEqual(msg['type'], 'pmessage') + msg = pubsub.get_message(timeout=1) + env.assertEqual(msg['data'], f'k:{i}') + env.assertEqual(msg['type'], 'pmessage') + + # verify JSON.MSET is appearing in the AOF once only + role = 'master' + aof_fn = env.envRunner._getFileName(role, '.aof.1.incr.aof') + with open(f'{env.logDir}/appendonlydir/{aof_fn}', 'r') as fd: + aof_content = [l for l in fd.readlines() if 'JSON.MSET' in l] + assert(len(aof_content) == 1) + + + + # class CacheTestCase(BaseReJSONTest): # @property # def module_args(env): From fbb9288f1e45734e0172921a13ffe7e4042377c1 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Mon, 24 Jun 2024 12:11:51 +0000 Subject: [PATCH 038/112] skip on cluster mode and enforce useAof --- tests/pytest/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/test.py b/tests/pytest/test.py index d9026b3a0..0b18b2df4 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -1490,8 +1490,8 @@ def test_promote_u64_to_f64(env): def test_mset_replication_in_aof(env): - if not env.useAof: - env.skip() + env.skipOnCluster() + env = Env(useAof=True) with env.getClusterConnectionIfNeeded() as r: r.execute_command('config', 'set', 'notify-keyspace-events', 'KEA') From eb99f24fef8b6896c20ecbd330b43c0243fc312a Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Tue, 25 Jun 2024 11:08:24 +0000 Subject: [PATCH 039/112] call to replicate_verbatim in seperate to apply_changes. the apply change API call only to notify --- redis_json/src/commands.rs | 23 +++++++++++++++++++++++ redis_json/src/ivalue_manager.rs | 7 +------ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/redis_json/src/commands.rs b/redis_json/src/commands.rs index ea22ef221..f773d6ac6 100644 --- a/redis_json/src/commands.rs +++ b/redis_json/src/commands.rs @@ -193,6 +193,7 @@ pub fn json_set(manager: M, ctx: &Context, args: Vec) - if op != SetOptions::NotExists { redis_key.set_value(Vec::new(), val)?; redis_key.apply_changes(ctx, "json.set")?; + ctx.replicate_verbatim(); REDIS_OK } else { Ok(RedisValue::Null) @@ -203,6 +204,7 @@ pub fn json_set(manager: M, ctx: &Context, args: Vec) - let updated = apply_updates::(&mut redis_key, val, update_info); if updated { redis_key.apply_changes(ctx, "json.set")?; + ctx.replicate_verbatim(); REDIS_OK } else { Ok(RedisValue::Null) @@ -217,6 +219,7 @@ pub fn json_set(manager: M, ctx: &Context, args: Vec) - if path.get_path() == JSON_ROOT_PATH { redis_key.set_value(Vec::new(), val)?; redis_key.apply_changes(ctx, "json.set")?; + ctx.replicate_verbatim(); REDIS_OK } else { Err(RedisError::Str( @@ -258,6 +261,7 @@ pub fn json_merge(manager: M, ctx: &Context, args: Vec) if path.get_path() == JSON_ROOT_PATH { redis_key.merge_value(Vec::new(), val)?; redis_key.apply_changes(ctx, "json.merge")?; + ctx.replicate_verbatim(); REDIS_OK } else { let mut update_info = @@ -283,6 +287,7 @@ pub fn json_merge(manager: M, ctx: &Context, args: Vec) } if res { redis_key.apply_changes(ctx, "json.merge")?; + ctx.replicate_verbatim(); REDIS_OK } else { Ok(RedisValue::Null) @@ -297,6 +302,7 @@ pub fn json_merge(manager: M, ctx: &Context, args: Vec) // Nothing to merge with it's a new doc redis_key.set_value(Vec::new(), val)?; redis_key.apply_changes(ctx, "json.merge")?; + ctx.replicate_verbatim(); REDIS_OK } else { Err(RedisError::Str( @@ -585,6 +591,7 @@ pub fn json_del(manager: M, ctx: &Context, args: Vec) - }; if res > 0 { redis_key.apply_changes(ctx, "json.del")?; + ctx.replicate_verbatim(); } res } @@ -781,6 +788,7 @@ where } if need_notify { redis_key.apply_changes(ctx, cmd)?; + ctx.replicate_verbatim(); } Ok(res) } @@ -812,6 +820,7 @@ where }); } redis_key.apply_changes(ctx, cmd)?; + ctx.replicate_verbatim(); Ok(res.unwrap().to_string().into()) } else { Err(RedisError::String( @@ -898,6 +907,7 @@ where } if need_notify { redis_key.apply_changes(ctx, "json.toggle")?; + ctx.replicate_verbatim(); } Ok(res.into()) } @@ -920,6 +930,7 @@ where res = redis_key.bool_toggle(p)?; } redis_key.apply_changes(ctx, "json.toggle")?; + ctx.replicate_verbatim(); Ok(res.to_string().into()) } else { Err(RedisError::String( @@ -990,6 +1001,7 @@ where } if need_notify { redis_key.apply_changes(ctx, "json.strappend")?; + ctx.replicate_verbatim(); } Ok(res.into()) } @@ -1014,6 +1026,7 @@ where res = Some(redis_key.str_append(p, json.to_string())?); } redis_key.apply_changes(ctx, "json.strappend")?; + ctx.replicate_verbatim(); Ok(res.unwrap().into()) } else { Err(RedisError::String( @@ -1120,6 +1133,7 @@ where } else if paths.len() == 1 { let res = redis_key.arr_append(paths.pop().unwrap(), args)?; redis_key.apply_changes(ctx, "json.arrappend")?; + ctx.replicate_verbatim(); Ok(res.into()) } else { let mut res = 0; @@ -1127,6 +1141,7 @@ where res = redis_key.arr_append(p, args.clone())?; } redis_key.apply_changes(ctx, "json.arrappend")?; + ctx.replicate_verbatim(); Ok(res.into()) } } @@ -1158,6 +1173,7 @@ where } if need_notify { redis_key.apply_changes(ctx, "json.arrappend")?; + ctx.replicate_verbatim(); } Ok(res.into()) } @@ -1286,6 +1302,7 @@ where if need_notify { redis_key.apply_changes(ctx, "json.arrinsert")?; + ctx.replicate_verbatim(); } Ok(res.into()) } @@ -1315,6 +1332,7 @@ where res = Some(redis_key.arr_insert(p, &args, index)?); } redis_key.apply_changes(ctx, "json.arrinsert")?; + ctx.replicate_verbatim(); Ok(res.unwrap().into()) } } @@ -1467,6 +1485,7 @@ where } if need_notify { redis_key.apply_changes(ctx, "json.arrpop")?; + ctx.replicate_verbatim(); } Ok(res.into()) } @@ -1494,6 +1513,7 @@ where })?); } redis_key.apply_changes(ctx, "json.arrpop")?; + ctx.replicate_verbatim(); res } else { Err(RedisError::String( @@ -1549,6 +1569,7 @@ where } if need_notify { redis_key.apply_changes(ctx, "json.arrtrim")?; + ctx.replicate_verbatim(); } Ok(res.into()) } @@ -1578,6 +1599,7 @@ where res = Some(redis_key.arr_trim(p, start, stop)?); } redis_key.apply_changes(ctx, "json.arrtrim")?; + ctx.replicate_verbatim(); Ok(res.unwrap().into()) } } @@ -1737,6 +1759,7 @@ pub fn json_clear(manager: M, ctx: &Context, args: Vec) } if cleared > 0 { redis_key.apply_changes(ctx, "json.clear")?; + ctx.replicate_verbatim(); } Ok(cleared.into()) } diff --git a/redis_json/src/ivalue_manager.rs b/redis_json/src/ivalue_manager.rs index 6ece0558f..a30ae2179 100644 --- a/redis_json/src/ivalue_manager.rs +++ b/redis_json/src/ivalue_manager.rs @@ -282,12 +282,7 @@ impl<'a> IValueKeyHolderWrite<'a> { impl<'a> WriteHolder for IValueKeyHolderWrite<'a> { fn apply_changes(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError> { - if ctx.notify_keyspace_event(NotifyEvent::MODULE, command, &self.key_name) != Status::Ok { - Err(RedisError::Str("failed notify key space event")) - } else { - ctx.replicate_verbatim(); - Ok(()) - } + self.notify_keyspace_event(ctx, command) } fn notify_keyspace_event(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError> { From 79965429a4c14d3986c913d6fa26292c410f9eef Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Tue, 25 Jun 2024 14:27:51 +0300 Subject: [PATCH 040/112] update deps of deps --- Cargo.lock | 158 +++++++++++++++++-------------- Cargo.toml | 10 +- redis_json/src/formatter.rs | 2 +- redis_json/src/ivalue_manager.rs | 6 +- 4 files changed, 95 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 06f500d27..2a9df331c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -47,9 +47,9 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", @@ -72,7 +72,7 @@ version = "0.66.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cexpr", "clang-sys", "lazy_static", @@ -85,7 +85,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.65", + "syn 2.0.68", "which", ] @@ -97,9 +97,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "bitvec" @@ -124,9 +124,9 @@ dependencies = [ [[package]] name = "bson" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d43b38e074cc0de2957f10947e376a1d88b9c4dbab340b590800cc1b2e066b2" +checksum = "d8a88e82b9106923b5c4d6edfca9e7db958d4e98a478ec115022e81b9b38e2c8" dependencies = [ "ahash", "base64", @@ -151,9 +151,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "cc" -version = "1.0.98" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" +checksum = "c891175c3fb232128f48de6590095e59198bbeb8620c310be349bfc3afd12c7b" [[package]] name = "cexpr" @@ -172,9 +172,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clang-sys" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", @@ -218,6 +218,9 @@ name = "deranged" version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] [[package]] name = "digest" @@ -304,9 +307,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "glob" @@ -436,9 +439,9 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lazycell" @@ -454,9 +457,9 @@ checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libloading" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" dependencies = [ "cfg-if", "windows-targets", @@ -464,22 +467,22 @@ dependencies = [ [[package]] name = "linkme" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "833222afbfe72868ac8f9770c91a33673f0d5fefc37c9dbe94aa3548b571623f" +checksum = "ccb76662d78edc9f9bf56360d6919bdacc8b7761227727e5082f128eeb90bbf5" dependencies = [ "linkme-impl", ] [[package]] name = "linkme-impl" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39f0dea92dbea3271557cc2e1848723967bba81f722f95026860974ec9283f08" +checksum = "f8dccda732e04fa3baf2e17cf835bfe2601c7c2edafd64417c627dabae3a8cda" dependencies = [ "proc-macro2", "quote 1.0.36", - "syn 2.0.65", + "syn 2.0.68", ] [[package]] @@ -506,9 +509,9 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memoffset" @@ -527,9 +530,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" dependencies = [ "adler", ] @@ -557,6 +560,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-traits" version = "0.1.43" @@ -577,9 +586,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" dependencies = [ "memchr", ] @@ -640,7 +649,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote 1.0.36", - "syn 2.0.65", + "syn 2.0.68", ] [[package]] @@ -660,6 +669,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -673,14 +688,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.65", + "syn 2.0.68", ] [[package]] name = "proc-macro2" -version = "1.0.83" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b33eb56c327dec362a9e55b3ad14f9d2f0904fb5a5b03b513ab5465399e9f43" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -744,7 +759,7 @@ checksum = "9d3b95d9fe5b8681bacea6532bbc7632590ae4499cecc8e019685b515fddda71" dependencies = [ "backtrace", "bindgen", - "bitflags 2.5.0", + "bitflags 2.6.0", "cc", "cfg-if", "enum-primitive-derive", @@ -788,7 +803,7 @@ dependencies = [ name = "redis_json" version = "99.99.99" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "bson", "env_logger", "ijson", @@ -804,18 +819,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] name = "regex" -version = "1.10.4" +version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", @@ -825,9 +840,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", @@ -836,9 +851,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "rustc-demangle" @@ -858,7 +873,7 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", @@ -885,38 +900,38 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.171" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" +checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.171" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote 1.0.36", - "syn 2.0.65", + "syn 2.0.68", ] [[package]] name = "serde_json" -version = "1.0.109" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0652c533506ad7a2e353cce269330d6afd8bdfb6d75e0ace5b35aacbd7b9e9" +checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4" dependencies = [ "indexmap", "itoa", @@ -996,9 +1011,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.65" +version = "2.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2863d96a84c6439701d7a38f9de935ec562c8832cc55d1dde0f513b52fad106" +checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" dependencies = [ "proc-macro2", "quote 1.0.36", @@ -1046,17 +1061,19 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote 1.0.36", - "syn 2.0.65", + "syn 2.0.68", ] [[package]] name = "time" -version = "0.3.26" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a79d09ac6b08c1ab3906a2f7cc2e81a0e27c7ae89c63812df75e52bef0751e07" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", + "num-conv", + "powerfmt", "serde", "time-core", "time-macros", @@ -1064,16 +1081,17 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.12" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75c65469ed6b3a4809d987a41eb1dc918e9bc1d92211cbad7ae82931846f7451" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] @@ -1103,9 +1121,9 @@ checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" [[package]] name = "uuid" -version = "1.8.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" dependencies = [ "getrandom", "serde", @@ -1144,7 +1162,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote 1.0.36", - "syn 2.0.65", + "syn 2.0.68", "wasm-bindgen-shared", ] @@ -1166,7 +1184,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote 1.0.36", - "syn 2.0.65", + "syn 2.0.68", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1297,5 +1315,5 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote 1.0.36", - "syn 2.0.65", + "syn 2.0.68", ] diff --git a/Cargo.toml b/Cargo.toml index 98db3a0a5..6a98cf077 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,13 +8,9 @@ members = [ [workspace.dependencies] ijson = { git="https://github.com/RedisJSON/ijson", rev="e0119ac74f6c4ee918718ee122c3948b74ebeba8", default_features=false} -serde_json = { version="1.0", features = ["unbounded_depth"]} -# DO NOT CHANGE to avoid security issues. This exact version -# specification is required for the second level dependencies -# to use exactly this version too, so that the dependencies of -# this project will not be able to compromise it. -serde = { version = "=1.0.171", features = ["derive"] } -serde_derive = "=1.0.171" +serde_json = { version="1", features = ["unbounded_depth"]} +serde = { version = "1", features = ["derive"] } +serde_derive = "1" bson = "2" [workspace.package] diff --git a/redis_json/src/formatter.rs b/redis_json/src/formatter.rs index a26ca7908..f30b60e92 100644 --- a/redis_json/src/formatter.rs +++ b/redis_json/src/formatter.rs @@ -35,7 +35,7 @@ DEALINGS IN THE SOFTWARE. use serde_json::ser::Formatter; use std::io; -pub use crate::redisjson::{Format, ReplyFormat}; +pub use crate::redisjson::ReplyFormat; pub struct ReplyFormatOptions<'a> { pub format: ReplyFormat, diff --git a/redis_json/src/ivalue_manager.rs b/redis_json/src/ivalue_manager.rs index 16c89a2b4..938b981a2 100644 --- a/redis_json/src/ivalue_manager.rs +++ b/redis_json/src/ivalue_manager.rs @@ -718,11 +718,11 @@ impl<'a> Manager for RedisIValueJsonKeyManager<'a> { mod tests { use super::*; - static single_thread_test_mutex: std::sync::Mutex<()> = std::sync::Mutex::new(()); + static SINGLE_THREAD_TEST_MUTEX: std::sync::Mutex<()> = std::sync::Mutex::new(()); #[test] fn test_get_memory() { - let _guard = single_thread_test_mutex.lock(); + let _guard = SINGLE_THREAD_TEST_MUTEX.lock(); let manager = RedisIValueJsonKeyManager { phantom: PhantomData, @@ -752,7 +752,7 @@ mod tests { /// unicode characters well. #[test] fn test_unicode_characters() { - let _guard = single_thread_test_mutex.lock(); + let _guard = SINGLE_THREAD_TEST_MUTEX.lock(); let json = r#""\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0""#; let value: IValue = serde_json::from_str(json).expect("IValue parses fine."); From 1879306be91add3804b4f4d8b0862de70bbc7377 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Tue, 25 Jun 2024 17:24:59 +0300 Subject: [PATCH 041/112] apt update --- tests/pytest/tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/tests.sh b/tests/pytest/tests.sh index f204da86f..d4fc3f9a8 100755 --- a/tests/pytest/tests.sh +++ b/tests/pytest/tests.sh @@ -186,6 +186,7 @@ setup_clang_sanitizer() { REDIS_SERVER=${REDIS_SERVER:-redis-server-asan-$SAN_REDIS_SUFFIX} if ! command -v $REDIS_SERVER > /dev/null; then echo "Building Redis $SAN_REDIS_VER for clang-asan ..." + runn sudo apt -qq update -y runn $READIES/bin/getredis --force -v $SAN_REDIS_VER --own-openssl --no-run \ --suffix asan-${SAN_REDIS_SUFFIX} --clang-asan --clang-san-blacklist $ignorelist fi From 3f8669bc1798e084c539bd0f15b185cf54ece2f8 Mon Sep 17 00:00:00 2001 From: Victor Polevoy Date: Wed, 26 Jun 2024 15:29:19 +0200 Subject: [PATCH 042/112] Update the dependencies --- Cargo.lock | 128 ++++++++++++++++++++++++++++-------------- json_path/Cargo.toml | 2 +- redis_json/Cargo.toml | 4 +- 3 files changed, 88 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2a9df331c..c0eb2969b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,6 +39,55 @@ dependencies = [ "memchr", ] +[[package]] +name = "anstream" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" + +[[package]] +name = "anstyle-parse" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +dependencies = [ + "anstyle", + "windows-sys", +] + [[package]] name = "autocfg" version = "1.3.0" @@ -151,9 +200,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "cc" -version = "1.0.100" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c891175c3fb232128f48de6590095e59198bbeb8620c310be349bfc3afd12c7b" +checksum = "ac367972e516d45567c7eafc73d24e1c193dcf200a8d94e9db7b3d38b349572d" [[package]] name = "cexpr" @@ -181,6 +230,12 @@ dependencies = [ "libloading", ] +[[package]] +name = "colorchoice" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" + [[package]] name = "cpufeatures" version = "0.2.12" @@ -234,9 +289,9 @@ dependencies = [ [[package]] name = "either" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "enum-primitive-derive" @@ -249,17 +304,27 @@ dependencies = [ "syn 0.11.11", ] +[[package]] +name = "env_filter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +dependencies = [ + "log", + "regex", +] + [[package]] name = "env_logger" -version = "0.10.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" +checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" dependencies = [ + "anstream", + "anstyle", + "env_filter", "humantime", - "is-terminal", "log", - "regex", - "termcolor", ] [[package]] @@ -338,12 +403,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - [[package]] name = "hex" version = "0.4.3" @@ -388,21 +447,16 @@ dependencies = [ ] [[package]] -name = "is-terminal" -version = "0.4.12" +name = "is_terminal_polyfill" +version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys", -] +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" [[package]] name = "itertools" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" dependencies = [ "either", ] @@ -1035,15 +1089,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "thiserror" version = "1.0.61" @@ -1119,6 +1164,12 @@ version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.9.1" @@ -1207,15 +1258,6 @@ dependencies = [ "rustix", ] -[[package]] -name = "winapi-util" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" -dependencies = [ - "windows-sys", -] - [[package]] name = "windows-sys" version = "0.52.0" diff --git a/json_path/Cargo.toml b/json_path/Cargo.toml index 41558f32e..b409d46c3 100644 --- a/json_path/Cargo.toml +++ b/json_path/Cargo.toml @@ -16,7 +16,7 @@ log = "0.4" regex = "1" [dev-dependencies] -env_logger = "0.10.0" +env_logger = "0.11" [[bin]] name = "jsonpath" diff --git a/redis_json/Cargo.toml b/redis_json/Cargo.toml index c7aa36998..4065b356c 100644 --- a/redis_json/Cargo.toml +++ b/redis_json/Cargo.toml @@ -16,7 +16,7 @@ crate-type = ["cdylib", "rlib"] name = "rejson" [dev-dependencies] -env_logger = "0.10" +env_logger = "0.11" [dependencies] bitflags = "2" @@ -27,7 +27,7 @@ serde.workspace = true libc = "0.2" redis-module ={ version = "^2.0.7", default-features = false, features = ["min-redis-compatibility-version-7-2"] } redis-module-macros = "^2.0.7" -itertools = "0.12" +itertools = "0.13" json_path = {path="../json_path"} linkme = "0.3" From 6a17501957ce969c6f480becd9049d05d696e26e Mon Sep 17 00:00:00 2001 From: Victor Polevoy Date: Wed, 26 Jun 2024 15:29:46 +0200 Subject: [PATCH 043/112] Fix the asan build --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0a0a1a149..d3b458057 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,7 @@ ifeq ($(DEBUG),1) else NIGHTLY=1 CARGO_FLAGS += -Zbuild-std - RUST_FLAGS += -Zsanitizer=$(SAN) + RUST_FLAGS += -Zsanitizer=$(SAN) -C link-args=-znostart-stop-gc ifeq ($(SAN),memory) RUST_FLAGS += -Zsanitizer-memory-track-origins endif From 33a4dfd2f91165d4a8887d5de4c7cec888beeeb5 Mon Sep 17 00:00:00 2001 From: Victor Polevoy Date: Tue, 2 Jul 2024 12:30:29 +0200 Subject: [PATCH 044/112] Install clang-18 for coverage --- .circleci/config.yml | 1 + Makefile | 7 +++ sbin/install_clang.sh | 18 +++++++ sbin/update_clang_alternatives.sh | 88 +++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+) create mode 100755 sbin/install_clang.sh create mode 100755 sbin/update_clang_alternatives.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 94cdbdcc7..d9f3aa609 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -374,6 +374,7 @@ jobs: name: Build & Test shell: /bin/bash -l -eo pipefail command: | + make clang-install make coverage SHOW=1 make upload-cov SHOW=1 no_output_timeout: 30m diff --git a/Makefile b/Makefile index d3b458057..bd04658c2 100644 --- a/Makefile +++ b/Makefile @@ -234,6 +234,13 @@ upload-artifacts: #---------------------------------------------------------------------------------------------- +clang-install: + ./sbin/install_clang.sh + +.PHONY: clang-install + +#---------------------------------------------------------------------------------------------- + COV_EXCLUDE_DIRS += bin deps tests COV_EXCLUDE.llvm += $(foreach D,$(COV_EXCLUDE_DIRS),'$(realpath $(ROOT))/$(D)/*') diff --git a/sbin/install_clang.sh b/sbin/install_clang.sh new file mode 100755 index 000000000..53fa323de --- /dev/null +++ b/sbin/install_clang.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env sh + +export CWD=$(dirname `which "${0}"`) +export CLANG_VERSION=18 +export DEBIAN_FRONTEND=noninteractive + +wget https://apt.llvm.org/llvm.sh -O llvm.sh + +chmod u+x llvm.sh + +# expected to fail: +./llvm.sh $CLANG_VERSION + +apt-get install python3-lldb-18 --yes --force-yes + +./llvm.sh $CLANG_VERSION + +$CWD/update_clang_alternatives.sh $CLANG_VERSION 1 diff --git a/sbin/update_clang_alternatives.sh b/sbin/update_clang_alternatives.sh new file mode 100755 index 000000000..ec140e224 --- /dev/null +++ b/sbin/update_clang_alternatives.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +# --slave /usr/bin/$1 $1 /usr/bin/$1-\${version} \\ + +function register_clang_version { + local version=$1 + local priority=$2 + + update-alternatives \ + --verbose \ + --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${version} ${priority} \ + --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-${version} \ + --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-${version} \ + --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-${version} \ + --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-${version} \ + --slave /usr/bin/llvm-cat llvm-cat /usr/bin/llvm-cat-${version} \ + --slave /usr/bin/llvm-cfi-verify llvm-cfi-verify /usr/bin/llvm-cfi-verify-${version} \ + --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-${version} \ + --slave /usr/bin/llvm-cvtres llvm-cvtres /usr/bin/llvm-cvtres-${version} \ + --slave /usr/bin/llvm-cxxdump llvm-cxxdump /usr/bin/llvm-cxxdump-${version} \ + --slave /usr/bin/llvm-cxxfilt llvm-cxxfilt /usr/bin/llvm-cxxfilt-${version} \ + --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-${version} \ + --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-${version} \ + --slave /usr/bin/llvm-dlltool llvm-dlltool /usr/bin/llvm-dlltool-${version} \ + --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-${version} \ + --slave /usr/bin/llvm-dwp llvm-dwp /usr/bin/llvm-dwp-${version} \ + --slave /usr/bin/llvm-exegesis llvm-exegesis /usr/bin/llvm-exegesis-${version} \ + --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-${version} \ + --slave /usr/bin/llvm-lib llvm-lib /usr/bin/llvm-lib-${version} \ + --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-${version} \ + --slave /usr/bin/llvm-lto llvm-lto /usr/bin/llvm-lto-${version} \ + --slave /usr/bin/llvm-lto2 llvm-lto2 /usr/bin/llvm-lto2-${version} \ + --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-${version} \ + --slave /usr/bin/llvm-mca llvm-mca /usr/bin/llvm-mca-${version} \ + --slave /usr/bin/llvm-modextract llvm-modextract /usr/bin/llvm-modextract-${version} \ + --slave /usr/bin/llvm-mt llvm-mt /usr/bin/llvm-mt-${version} \ + --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-${version} \ + --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-${version} \ + --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-${version} \ + --slave /usr/bin/llvm-opt-report llvm-opt-report /usr/bin/llvm-opt-report-${version} \ + --slave /usr/bin/llvm-pdbutil llvm-pdbutil /usr/bin/llvm-pdbutil-${version} \ + --slave /usr/bin/llvm-PerfectShuffle llvm-PerfectShuffle /usr/bin/llvm-PerfectShuffle-${version} \ + --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-${version} \ + --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-${version} \ + --slave /usr/bin/llvm-rc llvm-rc /usr/bin/llvm-rc-${version} \ + --slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-${version} \ + --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-${version} \ + --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-${version} \ + --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-${version} \ + --slave /usr/bin/llvm-split llvm-split /usr/bin/llvm-split-${version} \ + --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-${version} \ + --slave /usr/bin/llvm-strings llvm-strings /usr/bin/llvm-strings-${version} \ + --slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-${version} \ + --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-${version} \ + --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-${version} \ + --slave /usr/bin/llvm-undname llvm-undname /usr/bin/llvm-undname-${version} \ + --slave /usr/bin/llvm-xray llvm-xray /usr/bin/llvm-xray-${version} + + + update-alternatives \ + --verbose \ + --install /usr/bin/clang clang /usr/bin/clang-${version} ${priority} \ + --slave /usr/bin/clang++ clang++ /usr/bin/clang++-${version} \ + --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-${version} \ + --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-${version} \ + --slave /usr/bin/clang-cl clang-cl /usr/bin/clang-cl-${version} \ + --slave /usr/bin/clangd clangd /usr/bin/clangd-${version} \ + --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-${version} \ + --slave /usr/bin/clang-check clang-check /usr/bin/clang-check-${version} \ + --slave /usr/bin/clang-query clang-query /usr/bin/clang-query-${version} \ + --slave /usr/bin/asan_symbolize asan_symbolize /usr/bin/asan_symbolize-${version} \ + --slave /usr/bin/bugpoint bugpoint /usr/bin/bugpoint-${version} \ + --slave /usr/bin/dsymutil dsymutil /usr/bin/dsymutil-${version} \ + --slave /usr/bin/lld lld /usr/bin/lld-${version} \ + --slave /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-${version} \ + --slave /usr/bin/lld-link lld-link /usr/bin/lld-link-${version} \ + --slave /usr/bin/llc llc /usr/bin/llc-${version} \ + --slave /usr/bin/lli lli /usr/bin/lli-${version} \ + --slave /usr/bin/obj2yaml obj2yaml /usr/bin/obj2yaml-${version} \ + --slave /usr/bin/opt opt /usr/bin/opt-${version} \ + --slave /usr/bin/sanstats sanstats /usr/bin/sanstats-${version} \ + --slave /usr/bin/verify-uselistorder verify-uselistorder /usr/bin/verify-uselistorder-${version} \ + --slave /usr/bin/wasm-ld wasm-ld /usr/bin/wasm-ld-${version} \ + --slave /usr/bin/yaml2obj yaml2obj /usr/bin/yaml2obj-${version} + +} + +register_clang_version $1 $2 From a1b7c9046a5958054adbe02fbb20ba6397dd912e Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Tue, 2 Jul 2024 11:58:48 +0000 Subject: [PATCH 045/112] create new apply_changes API in the manager to call him while we want to seperate the replication from notify --- redis_json/src/commands.rs | 25 +------------------------ redis_json/src/ivalue_manager.rs | 8 ++++++++ redis_json/src/manager.rs | 4 ++++ 3 files changed, 13 insertions(+), 24 deletions(-) diff --git a/redis_json/src/commands.rs b/redis_json/src/commands.rs index f773d6ac6..c9328a9c5 100644 --- a/redis_json/src/commands.rs +++ b/redis_json/src/commands.rs @@ -193,7 +193,6 @@ pub fn json_set(manager: M, ctx: &Context, args: Vec) - if op != SetOptions::NotExists { redis_key.set_value(Vec::new(), val)?; redis_key.apply_changes(ctx, "json.set")?; - ctx.replicate_verbatim(); REDIS_OK } else { Ok(RedisValue::Null) @@ -204,7 +203,6 @@ pub fn json_set(manager: M, ctx: &Context, args: Vec) - let updated = apply_updates::(&mut redis_key, val, update_info); if updated { redis_key.apply_changes(ctx, "json.set")?; - ctx.replicate_verbatim(); REDIS_OK } else { Ok(RedisValue::Null) @@ -219,7 +217,6 @@ pub fn json_set(manager: M, ctx: &Context, args: Vec) - if path.get_path() == JSON_ROOT_PATH { redis_key.set_value(Vec::new(), val)?; redis_key.apply_changes(ctx, "json.set")?; - ctx.replicate_verbatim(); REDIS_OK } else { Err(RedisError::Str( @@ -261,7 +258,6 @@ pub fn json_merge(manager: M, ctx: &Context, args: Vec) if path.get_path() == JSON_ROOT_PATH { redis_key.merge_value(Vec::new(), val)?; redis_key.apply_changes(ctx, "json.merge")?; - ctx.replicate_verbatim(); REDIS_OK } else { let mut update_info = @@ -287,7 +283,6 @@ pub fn json_merge(manager: M, ctx: &Context, args: Vec) } if res { redis_key.apply_changes(ctx, "json.merge")?; - ctx.replicate_verbatim(); REDIS_OK } else { Ok(RedisValue::Null) @@ -302,7 +297,6 @@ pub fn json_merge(manager: M, ctx: &Context, args: Vec) // Nothing to merge with it's a new doc redis_key.set_value(Vec::new(), val)?; redis_key.apply_changes(ctx, "json.merge")?; - ctx.replicate_verbatim(); REDIS_OK } else { Err(RedisError::Str( @@ -365,7 +359,7 @@ pub fn json_mset(manager: M, ctx: &Context, args: Vec) res }); - ctx.replicate_verbatim(); + manager.apply_changes(ctx); res } @@ -591,7 +585,6 @@ pub fn json_del(manager: M, ctx: &Context, args: Vec) - }; if res > 0 { redis_key.apply_changes(ctx, "json.del")?; - ctx.replicate_verbatim(); } res } @@ -788,7 +781,6 @@ where } if need_notify { redis_key.apply_changes(ctx, cmd)?; - ctx.replicate_verbatim(); } Ok(res) } @@ -820,7 +812,6 @@ where }); } redis_key.apply_changes(ctx, cmd)?; - ctx.replicate_verbatim(); Ok(res.unwrap().to_string().into()) } else { Err(RedisError::String( @@ -907,7 +898,6 @@ where } if need_notify { redis_key.apply_changes(ctx, "json.toggle")?; - ctx.replicate_verbatim(); } Ok(res.into()) } @@ -930,7 +920,6 @@ where res = redis_key.bool_toggle(p)?; } redis_key.apply_changes(ctx, "json.toggle")?; - ctx.replicate_verbatim(); Ok(res.to_string().into()) } else { Err(RedisError::String( @@ -1001,7 +990,6 @@ where } if need_notify { redis_key.apply_changes(ctx, "json.strappend")?; - ctx.replicate_verbatim(); } Ok(res.into()) } @@ -1026,7 +1014,6 @@ where res = Some(redis_key.str_append(p, json.to_string())?); } redis_key.apply_changes(ctx, "json.strappend")?; - ctx.replicate_verbatim(); Ok(res.unwrap().into()) } else { Err(RedisError::String( @@ -1133,7 +1120,6 @@ where } else if paths.len() == 1 { let res = redis_key.arr_append(paths.pop().unwrap(), args)?; redis_key.apply_changes(ctx, "json.arrappend")?; - ctx.replicate_verbatim(); Ok(res.into()) } else { let mut res = 0; @@ -1141,7 +1127,6 @@ where res = redis_key.arr_append(p, args.clone())?; } redis_key.apply_changes(ctx, "json.arrappend")?; - ctx.replicate_verbatim(); Ok(res.into()) } } @@ -1173,7 +1158,6 @@ where } if need_notify { redis_key.apply_changes(ctx, "json.arrappend")?; - ctx.replicate_verbatim(); } Ok(res.into()) } @@ -1302,7 +1286,6 @@ where if need_notify { redis_key.apply_changes(ctx, "json.arrinsert")?; - ctx.replicate_verbatim(); } Ok(res.into()) } @@ -1332,7 +1315,6 @@ where res = Some(redis_key.arr_insert(p, &args, index)?); } redis_key.apply_changes(ctx, "json.arrinsert")?; - ctx.replicate_verbatim(); Ok(res.unwrap().into()) } } @@ -1485,7 +1467,6 @@ where } if need_notify { redis_key.apply_changes(ctx, "json.arrpop")?; - ctx.replicate_verbatim(); } Ok(res.into()) } @@ -1513,7 +1494,6 @@ where })?); } redis_key.apply_changes(ctx, "json.arrpop")?; - ctx.replicate_verbatim(); res } else { Err(RedisError::String( @@ -1569,7 +1549,6 @@ where } if need_notify { redis_key.apply_changes(ctx, "json.arrtrim")?; - ctx.replicate_verbatim(); } Ok(res.into()) } @@ -1599,7 +1578,6 @@ where res = Some(redis_key.arr_trim(p, start, stop)?); } redis_key.apply_changes(ctx, "json.arrtrim")?; - ctx.replicate_verbatim(); Ok(res.unwrap().into()) } } @@ -1759,7 +1737,6 @@ pub fn json_clear(manager: M, ctx: &Context, args: Vec) } if cleared > 0 { redis_key.apply_changes(ctx, "json.clear")?; - ctx.replicate_verbatim(); } Ok(cleared.into()) } diff --git a/redis_json/src/ivalue_manager.rs b/redis_json/src/ivalue_manager.rs index a30ae2179..de1ab20c8 100644 --- a/redis_json/src/ivalue_manager.rs +++ b/redis_json/src/ivalue_manager.rs @@ -283,6 +283,7 @@ impl<'a> IValueKeyHolderWrite<'a> { impl<'a> WriteHolder for IValueKeyHolderWrite<'a> { fn apply_changes(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError> { self.notify_keyspace_event(ctx, command) + .inspect(|_| ctx.replicate_verbatim()) } fn notify_keyspace_event(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError> { @@ -613,6 +614,13 @@ impl<'a> Manager for RedisIValueJsonKeyManager<'a> { val: None, }) } + /** + * This function is used to apply changes to the slave and AOF. + * It is called after the command is executed. + */ + fn apply_changes(&self, ctx: &Context) { + ctx.replicate_verbatim(); + } fn from_str(&self, val: &str, format: Format, limit_depth: bool) -> Result { match format { diff --git a/redis_json/src/manager.rs b/redis_json/src/manager.rs index d1ac65038..989bb5a58 100644 --- a/redis_json/src/manager.rs +++ b/redis_json/src/manager.rs @@ -93,6 +93,10 @@ pub trait Manager { ctx: &Context, key: RedisString, ) -> Result; + fn apply_changes( + &self, + ctx: &Context + ); #[allow(clippy::wrong_self_convention)] fn from_str(&self, val: &str, format: Format, limit_depth: bool) -> Result; fn get_memory(&self, v: &Self::V) -> Result; From 355de5723035ca363dba7c9b47f6b382564529c1 Mon Sep 17 00:00:00 2001 From: Victor Polevoy Date: Tue, 2 Jul 2024 15:38:39 +0200 Subject: [PATCH 046/112] Fix coverage --- redis_json/src/c_api.rs | 75 ++++++++++++++++++++++++++--------------- redis_json/src/lib.rs | 1 - 2 files changed, 48 insertions(+), 28 deletions(-) diff --git a/redis_json/src/c_api.rs b/redis_json/src/c_api.rs index 14b2599f5..b4d399aed 100644 --- a/redis_json/src/c_api.rs +++ b/redis_json/src/c_api.rs @@ -22,9 +22,6 @@ use redis_module::{key::KeyFlags, Context, RedisString, Status}; use crate::manager::{Manager, ReadHolder}; -// extern crate readies_wd40; -// use crate::readies_wd40::{BB, _BB, getenv}; - // // structs // @@ -50,7 +47,7 @@ struct ResultsIterator<'a, V: SelectValue> { pub static mut LLAPI_CTX: Option<*mut rawmod::RedisModuleCtx> = None; #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub fn create_rmstring( +pub(crate) fn create_rmstring( ctx: *mut rawmod::RedisModuleCtx, from_str: &str, str: *mut *mut rawmod::RedisModuleString, @@ -64,7 +61,7 @@ pub fn create_rmstring( Status::Err as c_int } -pub fn json_api_open_key_internal( +pub(crate) fn json_api_open_key_internal( manager: M, ctx: *mut rawmod::RedisModuleCtx, key: RedisString, @@ -78,7 +75,7 @@ pub fn json_api_open_key_internal( null() } -pub fn json_api_open_key_with_flags_internal( +pub(crate) fn json_api_open_key_with_flags_internal( manager: M, ctx: *mut rawmod::RedisModuleCtx, key: RedisString, @@ -93,7 +90,11 @@ pub fn json_api_open_key_with_flags_internal( null() } -pub fn json_api_get_at(_: M, json: *const c_void, index: size_t) -> *const c_void { +pub(crate) fn json_api_get_at( + _: M, + json: *const c_void, + index: size_t, +) -> *const c_void { let json = unsafe { &*(json.cast::()) }; match json.get_type() { SelectValueType::Array => json @@ -104,7 +105,11 @@ pub fn json_api_get_at(_: M, json: *const c_void, index: size_t) -> } #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub fn json_api_get_len(_: M, json: *const c_void, count: *mut libc::size_t) -> c_int { +pub(crate) fn json_api_get_len( + _: M, + json: *const c_void, + count: *mut libc::size_t, +) -> c_int { let json = unsafe { &*(json.cast::()) }; let len = match json.get_type() { SelectValueType::String => Some(json.get_str().len()), @@ -120,11 +125,11 @@ pub fn json_api_get_len(_: M, json: *const c_void, count: *mut libc: } } -pub fn json_api_get_type(_: M, json: *const c_void) -> c_int { +pub(crate) fn json_api_get_type(_: M, json: *const c_void) -> c_int { json_api_get_type_internal(unsafe { &*(json.cast::()) }) as c_int } -pub fn json_api_get_string( +pub(crate) fn json_api_get_string( _: M, json: *const c_void, str: *mut *const c_char, @@ -141,7 +146,7 @@ pub fn json_api_get_string( } } -pub fn json_api_get_json( +pub(crate) fn json_api_get_json( _: M, json: *const c_void, ctx: *mut rawmod::RedisModuleCtx, @@ -152,7 +157,7 @@ pub fn json_api_get_json( create_rmstring(ctx, &res, str) } -pub fn json_api_get_json_from_iter( +pub(crate) fn json_api_get_json_from_iter( _: M, iter: *mut c_void, ctx: *mut rawmod::RedisModuleCtx, @@ -169,7 +174,11 @@ pub fn json_api_get_json_from_iter( } #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub fn json_api_get_int(_: M, json: *const c_void, val: *mut c_longlong) -> c_int { +pub(crate) fn json_api_get_int( + _: M, + json: *const c_void, + val: *mut c_longlong, +) -> c_int { let json = unsafe { &*(json.cast::()) }; match json.get_type() { SelectValueType::Long => { @@ -181,7 +190,11 @@ pub fn json_api_get_int(_: M, json: *const c_void, val: *mut c_longl } #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub fn json_api_get_double(_: M, json: *const c_void, val: *mut c_double) -> c_int { +pub(crate) fn json_api_get_double( + _: M, + json: *const c_void, + val: *mut c_double, +) -> c_int { let json = unsafe { &*(json.cast::()) }; match json.get_type() { SelectValueType::Double => { @@ -197,7 +210,11 @@ pub fn json_api_get_double(_: M, json: *const c_void, val: *mut c_do } #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub fn json_api_get_boolean(_: M, json: *const c_void, val: *mut c_int) -> c_int { +pub(crate) fn json_api_get_boolean( + _: M, + json: *const c_void, + val: *mut c_int, +) -> c_int { let json = unsafe { &*(json.cast::()) }; match json.get_type() { SelectValueType::Bool => { @@ -211,7 +228,7 @@ pub fn json_api_get_boolean(_: M, json: *const c_void, val: *mut c_i //--------------------------------------------------------------------------------------------- #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub fn set_string(from_str: &str, str: *mut *const c_char, len: *mut size_t) -> c_int { +pub(crate) fn set_string(from_str: &str, str: *mut *const c_char, len: *mut size_t) -> c_int { if !str.is_null() { unsafe { *str = from_str.as_ptr().cast::(); @@ -234,7 +251,7 @@ fn json_api_get_type_internal(v: &V) -> JSONType { } } -pub fn json_api_next(_: M, iter: *mut c_void) -> *const c_void { +pub(crate) fn json_api_next(_: M, iter: *mut c_void) -> *const c_void { let iter = unsafe { &mut *(iter.cast::>()) }; if iter.pos >= iter.results.len() { null_mut() @@ -245,24 +262,28 @@ pub fn json_api_next(_: M, iter: *mut c_void) -> *const c_void { } } -pub fn json_api_len(_: M, iter: *const c_void) -> size_t { +pub(crate) fn json_api_len(_: M, iter: *const c_void) -> size_t { let iter = unsafe { &*(iter.cast::>()) }; iter.results.len() as size_t } -pub fn json_api_free_iter(_: M, iter: *mut c_void) { +pub(crate) fn json_api_free_iter(_: M, iter: *mut c_void) { unsafe { drop(Box::from_raw(iter.cast::>())); } } -pub fn json_api_reset_iter(_: M, iter: *mut c_void) { +pub(crate) fn json_api_reset_iter(_: M, iter: *mut c_void) { let iter = unsafe { &mut *(iter.cast::>()) }; iter.pos = 0; } #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub fn json_api_get(_: M, val: *const c_void, path: *const c_char) -> *const c_void { +pub(crate) fn json_api_get( + _: M, + val: *const c_void, + path: *const c_char, +) -> *const c_void { let v = unsafe { &*(val.cast::()) }; let path = unsafe { CStr::from_ptr(path).to_str().unwrap() }; let query = match compile(path) { @@ -278,11 +299,11 @@ pub fn json_api_get(_: M, val: *const c_void, path: *const c_char) - .cast::() } -pub fn json_api_is_json(m: M, key: *mut rawmod::RedisModuleKey) -> c_int { +pub(crate) fn json_api_is_json(m: M, key: *mut rawmod::RedisModuleKey) -> c_int { m.is_json(key).map_or(0, |res| res as c_int) } -pub fn json_api_get_key_value(_: M, val: *const c_void) -> *const c_void { +pub(crate) fn json_api_get_key_value(_: M, val: *const c_void) -> *const c_void { let json = unsafe { &*(val.cast::()) }; match json.get_type() { SelectValueType::Object => Box::into_raw(Box::new(json.items().unwrap())).cast::(), @@ -290,7 +311,7 @@ pub fn json_api_get_key_value(_: M, val: *const c_void) -> *const c_ } } -pub fn json_api_next_key_value<'a, M: Manager>( +pub(crate) fn json_api_next_key_value<'a, M: Manager>( _: M, iter: *mut c_void, str: *mut *mut rawmod::RedisModuleString, @@ -307,7 +328,7 @@ where } } -pub fn json_api_free_key_values_iter<'a, M: Manager>(_: M, iter: *mut c_void) +pub(crate) fn json_api_free_key_values_iter<'a, M: Manager>(_: M, iter: *mut c_void) where M::V: 'a, { @@ -317,7 +338,7 @@ where } } -pub fn get_llapi_ctx() -> Context { +pub(crate) fn get_llapi_ctx() -> Context { Context::new(unsafe { LLAPI_CTX.unwrap() }) } @@ -518,7 +539,7 @@ macro_rules! redis_json_module_export_shared_api { match json_path::compile(path) { Ok(q) => Box::into_raw(Box::new(q)).cast::(), Err(e) => { - create_rmstring(ctx, &format!("{}", e), err_msg); + crate::c_api::create_rmstring(ctx, &format!("{}", e), err_msg); std::ptr::null() } } diff --git a/redis_json/src/lib.rs b/redis_json/src/lib.rs index 1570e0537..adbf1502a 100644 --- a/redis_json/src/lib.rs +++ b/redis_json/src/lib.rs @@ -128,7 +128,6 @@ macro_rules! redis_json_module_create {( }; use libc::size_t; use std::collections::HashMap; - use $crate::c_api::create_rmstring; macro_rules! json_command { ($cmd:ident) => { From 9dfdb7aed838ad5561aebebe8f506e85c78c1e81 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Wed, 3 Jul 2024 08:57:10 +0000 Subject: [PATCH 047/112] lint --- redis_json/src/ivalue_manager.rs | 6 +++--- redis_json/src/manager.rs | 5 +---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/redis_json/src/ivalue_manager.rs b/redis_json/src/ivalue_manager.rs index 145be0816..bd1424283 100644 --- a/redis_json/src/ivalue_manager.rs +++ b/redis_json/src/ivalue_manager.rs @@ -615,9 +615,9 @@ impl<'a> Manager for RedisIValueJsonKeyManager<'a> { }) } /** - * This function is used to apply changes to the slave and AOF. - * It is called after the command is executed. - */ + * This function is used to apply changes to the slave and AOF. + * It is called after the command is executed. + */ fn apply_changes(&self, ctx: &Context) { ctx.replicate_verbatim(); } diff --git a/redis_json/src/manager.rs b/redis_json/src/manager.rs index 989bb5a58..30bf52d52 100644 --- a/redis_json/src/manager.rs +++ b/redis_json/src/manager.rs @@ -93,10 +93,7 @@ pub trait Manager { ctx: &Context, key: RedisString, ) -> Result; - fn apply_changes( - &self, - ctx: &Context - ); + fn apply_changes(&self, ctx: &Context); #[allow(clippy::wrong_self_convention)] fn from_str(&self, val: &str, format: Format, limit_depth: bool) -> Result; fn get_memory(&self, v: &Self::V) -> Result; From b635bbd2e5f3db7b7147d7a611b93948fc13660c Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Wed, 3 Jul 2024 10:50:02 +0000 Subject: [PATCH 048/112] fix --- redis_json/src/commands.rs | 113 ++++++++++++++++++++----------- redis_json/src/ivalue_manager.rs | 5 -- redis_json/src/manager.rs | 1 - 3 files changed, 75 insertions(+), 44 deletions(-) diff --git a/redis_json/src/commands.rs b/redis_json/src/commands.rs index c9328a9c5..20a86f8f7 100644 --- a/redis_json/src/commands.rs +++ b/redis_json/src/commands.rs @@ -192,7 +192,8 @@ pub fn json_set(manager: M, ctx: &Context, args: Vec) - if path.get_path() == JSON_ROOT_PATH { if op != SetOptions::NotExists { redis_key.set_value(Vec::new(), val)?; - redis_key.apply_changes(ctx, "json.set")?; + redis_key.notify_keyspace_event(ctx, "json.set")?; + manager.apply_changes(ctx); REDIS_OK } else { Ok(RedisValue::Null) @@ -202,7 +203,8 @@ pub fn json_set(manager: M, ctx: &Context, args: Vec) - if !update_info.is_empty() { let updated = apply_updates::(&mut redis_key, val, update_info); if updated { - redis_key.apply_changes(ctx, "json.set")?; + redis_key.notify_keyspace_event(ctx, "json.set")?; + manager.apply_changes(ctx); REDIS_OK } else { Ok(RedisValue::Null) @@ -216,7 +218,8 @@ pub fn json_set(manager: M, ctx: &Context, args: Vec) - (None, _) => { if path.get_path() == JSON_ROOT_PATH { redis_key.set_value(Vec::new(), val)?; - redis_key.apply_changes(ctx, "json.set")?; + redis_key.notify_keyspace_event(ctx, "json.set")?; + manager.apply_changes(ctx); REDIS_OK } else { Err(RedisError::Str( @@ -257,7 +260,8 @@ pub fn json_merge(manager: M, ctx: &Context, args: Vec) Some(doc) => { if path.get_path() == JSON_ROOT_PATH { redis_key.merge_value(Vec::new(), val)?; - redis_key.apply_changes(ctx, "json.merge")?; + redis_key.notify_keyspace_event(ctx, "json.merge")?; + manager.apply_changes(ctx); REDIS_OK } else { let mut update_info = @@ -282,7 +286,8 @@ pub fn json_merge(manager: M, ctx: &Context, args: Vec) } } if res { - redis_key.apply_changes(ctx, "json.merge")?; + redis_key.notify_keyspace_event(ctx, "json.merge")?; + manager.apply_changes(ctx); REDIS_OK } else { Ok(RedisValue::Null) @@ -296,7 +301,8 @@ pub fn json_merge(manager: M, ctx: &Context, args: Vec) if path.get_path() == JSON_ROOT_PATH { // Nothing to merge with it's a new doc redis_key.set_value(Vec::new(), val)?; - redis_key.apply_changes(ctx, "json.merge")?; + redis_key.notify_keyspace_event(ctx, "json.merge")?; + manager.apply_changes(ctx); REDIS_OK } else { Err(RedisError::Str( @@ -584,7 +590,8 @@ pub fn json_del(manager: M, ctx: &Context, args: Vec) - changed }; if res > 0 { - redis_key.apply_changes(ctx, "json.del")?; + redis_key.notify_keyspace_event(ctx, "json.del")?; + manager.apply_changes(ctx); } res } @@ -718,7 +725,7 @@ where // check context flags to see if RESP3 is enabled if is_resp3(ctx) { - let res = json_num_op_impl::(&mut redis_key, ctx, path.get_path(), number, op, cmd)? + let res = json_num_op_impl::(manager, &mut redis_key, ctx, path.get_path(), number, op, cmd)? .drain(..) .map(|v| { v.map_or(RedisValue::Null, |v| { @@ -733,9 +740,9 @@ where .into(); Ok(res) } else if path.is_legacy() { - json_num_op_legacy::(&mut redis_key, ctx, path.get_path(), number, op, cmd) + json_num_op_legacy::(manager, &mut redis_key, ctx, path.get_path(), number, op, cmd) } else { - let results = json_num_op_impl::(&mut redis_key, ctx, path.get_path(), number, op, cmd)?; + let results = json_num_op_impl::(manager, &mut redis_key, ctx, path.get_path(), number, op, cmd)?; // Convert to RESP2 format return as one JSON array let values = to_json_value::(results, Value::Null); @@ -744,6 +751,7 @@ where } fn json_num_op_impl( + manager: M, redis_key: &mut M::WriteHolder, ctx: &Context, path: &str, @@ -780,12 +788,14 @@ where }); } if need_notify { - redis_key.apply_changes(ctx, cmd)?; + redis_key.notify_keyspace_event(ctx, cmd)?; + manager.apply_changes(ctx); } Ok(res) } fn json_num_op_legacy( + manager: M, redis_key: &mut M::WriteHolder, ctx: &Context, path: &str, @@ -811,7 +821,8 @@ where NumOp::Pow => redis_key.pow_by(p, number)?, }); } - redis_key.apply_changes(ctx, cmd)?; + redis_key.notify_keyspace_event(ctx, cmd)?; + manager.apply_changes(ctx); Ok(res.unwrap().to_string().into()) } else { Err(RedisError::String( @@ -867,13 +878,14 @@ pub fn json_bool_toggle( let mut redis_key = manager.open_key_write(ctx, key)?; if path.is_legacy() { - json_bool_toggle_legacy::(&mut redis_key, ctx, path.get_path()) + json_bool_toggle_legacy::(manager, &mut redis_key, ctx, path.get_path()) } else { - json_bool_toggle_impl::(&mut redis_key, ctx, path.get_path()) + json_bool_toggle_impl::(manager, &mut redis_key, ctx, path.get_path()) } } fn json_bool_toggle_impl( + manager: M, redis_key: &mut M::WriteHolder, ctx: &Context, path: &str, @@ -897,12 +909,14 @@ where }); } if need_notify { - redis_key.apply_changes(ctx, "json.toggle")?; + redis_key.notify_keyspace_event(ctx, "json.toggle")?; + manager.apply_changes(ctx); } Ok(res.into()) } fn json_bool_toggle_legacy( + manager: M, redis_key: &mut M::WriteHolder, ctx: &Context, path: &str, @@ -919,7 +933,8 @@ where for p in paths { res = redis_key.bool_toggle(p)?; } - redis_key.apply_changes(ctx, "json.toggle")?; + redis_key.notify_keyspace_event(ctx, "json.toggle")?; + manager.apply_changes(ctx); Ok(res.to_string().into()) } else { Err(RedisError::String( @@ -956,13 +971,14 @@ pub fn json_str_append( let mut redis_key = manager.open_key_write(ctx, key)?; if path.is_legacy() { - json_str_append_legacy::(&mut redis_key, ctx, path.get_path(), json) + json_str_append_legacy::(manager, &mut redis_key, ctx, path.get_path(), json) } else { - json_str_append_impl::(&mut redis_key, ctx, path.get_path(), json) + json_str_append_impl::(manager, &mut redis_key, ctx, path.get_path(), json) } } fn json_str_append_impl( + manager: M, redis_key: &mut M::WriteHolder, ctx: &Context, path: &str, @@ -989,12 +1005,14 @@ where }); } if need_notify { - redis_key.apply_changes(ctx, "json.strappend")?; + redis_key.notify_keyspace_event(ctx, "json.strappend")?; + manager.apply_changes(ctx); } Ok(res.into()) } fn json_str_append_legacy( + manager: M, redis_key: &mut M::WriteHolder, ctx: &Context, path: &str, @@ -1013,7 +1031,8 @@ where for p in paths { res = Some(redis_key.str_append(p, json.to_string())?); } - redis_key.apply_changes(ctx, "json.strappend")?; + redis_key.notify_keyspace_event(ctx, "json.strappend")?; + manager.apply_changes(ctx); Ok(res.unwrap().into()) } else { Err(RedisError::String( @@ -1092,13 +1111,14 @@ pub fn json_arr_append( let mut redis_key = manager.open_key_write(ctx, key)?; if path.is_legacy() { - json_arr_append_legacy::(&mut redis_key, ctx, &path, args) + json_arr_append_legacy::(manager, &mut redis_key, ctx, &path, args) } else { - json_arr_append_impl::(&mut redis_key, ctx, path.get_path(), args) + json_arr_append_impl::(manager, &mut redis_key, ctx, path.get_path(), args) } } fn json_arr_append_legacy( + manager: M, redis_key: &mut M::WriteHolder, ctx: &Context, path: &Path, @@ -1119,19 +1139,22 @@ where )) } else if paths.len() == 1 { let res = redis_key.arr_append(paths.pop().unwrap(), args)?; - redis_key.apply_changes(ctx, "json.arrappend")?; + redis_key.notify_keyspace_event(ctx, "json.arrappend")?; + manager.apply_changes(ctx); Ok(res.into()) } else { let mut res = 0; for p in paths { res = redis_key.arr_append(p, args.clone())?; } - redis_key.apply_changes(ctx, "json.arrappend")?; + redis_key.notify_keyspace_event(ctx, "json.arrappend")?; + manager.apply_changes(ctx); Ok(res.into()) } } fn json_arr_append_impl( + manager: M, redis_key: &mut M::WriteHolder, ctx: &Context, path: &str, @@ -1157,7 +1180,8 @@ where }); } if need_notify { - redis_key.apply_changes(ctx, "json.arrappend")?; + redis_key.notify_keyspace_event(ctx, "json.arrappend")?; + manager.apply_changes(ctx); } Ok(res.into()) } @@ -1250,13 +1274,14 @@ pub fn json_arr_insert( )?; let mut redis_key = manager.open_key_write(ctx, key)?; if path.is_legacy() { - json_arr_insert_legacy::(&mut redis_key, ctx, path.get_path(), index, args) + json_arr_insert_legacy::(manager, &mut redis_key, ctx, path.get_path(), index, args) } else { - json_arr_insert_impl::(&mut redis_key, ctx, path.get_path(), index, args) + json_arr_insert_impl::(manager, &mut redis_key, ctx, path.get_path(), index, args) } } fn json_arr_insert_impl( + manager: M, redis_key: &mut M::WriteHolder, ctx: &Context, path: &str, @@ -1285,12 +1310,14 @@ where } if need_notify { - redis_key.apply_changes(ctx, "json.arrinsert")?; + redis_key.notify_keyspace_event(ctx, "json.arrinsert")?; + manager.apply_changes(ctx); } Ok(res.into()) } fn json_arr_insert_legacy( + manager: M, redis_key: &mut M::WriteHolder, ctx: &Context, path: &str, @@ -1314,7 +1341,8 @@ where for p in paths { res = Some(redis_key.arr_insert(p, &args, index)?); } - redis_key.apply_changes(ctx, "json.arrinsert")?; + redis_key.notify_keyspace_event(ctx, "json.arrinsert")?; + manager.apply_changes(ctx); Ok(res.unwrap().into()) } } @@ -1427,13 +1455,14 @@ pub fn json_arr_pop(manager: M, ctx: &Context, args: Vec(&mut redis_key, ctx, path.get_path(), index) + json_arr_pop_legacy::(manager, &mut redis_key, ctx, path.get_path(), index) } else { - json_arr_pop_impl::(&mut redis_key, ctx, path.get_path(), index, &format_options) + json_arr_pop_impl::(manager, &mut redis_key, ctx, path.get_path(), index, &format_options) } } fn json_arr_pop_impl( + manager: M, redis_key: &mut M::WriteHolder, ctx: &Context, path: &str, @@ -1466,12 +1495,14 @@ where }); } if need_notify { - redis_key.apply_changes(ctx, "json.arrpop")?; + redis_key.notify_keyspace_event(ctx, "json.arrpop")?; + manager.apply_changes(ctx); } Ok(res.into()) } fn json_arr_pop_legacy( + manager: M, redis_key: &mut M::WriteHolder, ctx: &Context, path: &str, @@ -1493,7 +1524,8 @@ where None => Ok(().into()), })?); } - redis_key.apply_changes(ctx, "json.arrpop")?; + redis_key.notify_keyspace_event(ctx, "json.arrpop")?; + manager.apply_changes(ctx); res } else { Err(RedisError::String( @@ -1516,12 +1548,13 @@ pub fn json_arr_trim(manager: M, ctx: &Context, args: Vec(&mut redis_key, ctx, path.get_path(), start, stop) + json_arr_trim_legacy::(manager, &mut redis_key, ctx, path.get_path(), start, stop) } else { - json_arr_trim_impl::(&mut redis_key, ctx, path.get_path(), start, stop) + json_arr_trim_impl::(manager, &mut redis_key, ctx, path.get_path(), start, stop) } } fn json_arr_trim_impl( + manager: M, redis_key: &mut M::WriteHolder, ctx: &Context, path: &str, @@ -1548,12 +1581,14 @@ where }); } if need_notify { - redis_key.apply_changes(ctx, "json.arrtrim")?; + redis_key.notify_keyspace_event(ctx, "json.arrtrim")?; + manager.apply_changes(ctx); } Ok(res.into()) } fn json_arr_trim_legacy( + manager: M, redis_key: &mut M::WriteHolder, ctx: &Context, path: &str, @@ -1577,7 +1612,8 @@ where for p in paths { res = Some(redis_key.arr_trim(p, start, stop)?); } - redis_key.apply_changes(ctx, "json.arrtrim")?; + redis_key.notify_keyspace_event(ctx, "json.arrtrim")?; + manager.apply_changes(ctx); Ok(res.unwrap().into()) } } @@ -1736,7 +1772,8 @@ pub fn json_clear(manager: M, ctx: &Context, args: Vec) } } if cleared > 0 { - redis_key.apply_changes(ctx, "json.clear")?; + redis_key.notify_keyspace_event(ctx, "json.clear")?; + manager.apply_changes(ctx); } Ok(cleared.into()) } diff --git a/redis_json/src/ivalue_manager.rs b/redis_json/src/ivalue_manager.rs index bd1424283..a2026235b 100644 --- a/redis_json/src/ivalue_manager.rs +++ b/redis_json/src/ivalue_manager.rs @@ -281,11 +281,6 @@ impl<'a> IValueKeyHolderWrite<'a> { } impl<'a> WriteHolder for IValueKeyHolderWrite<'a> { - fn apply_changes(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError> { - self.notify_keyspace_event(ctx, command) - .inspect(|_| ctx.replicate_verbatim()) - } - fn notify_keyspace_event(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError> { if ctx.notify_keyspace_event(NotifyEvent::MODULE, command, &self.key_name) != Status::Ok { Err(RedisError::Str("failed notify key space event")) diff --git a/redis_json/src/manager.rs b/redis_json/src/manager.rs index 30bf52d52..96b782163 100644 --- a/redis_json/src/manager.rs +++ b/redis_json/src/manager.rs @@ -63,7 +63,6 @@ pub trait WriteHolder { ) -> RedisResult; fn arr_trim(&mut self, path: Vec, start: i64, stop: i64) -> Result; fn clear(&mut self, path: Vec) -> Result; - fn apply_changes(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError>; fn notify_keyspace_event(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError>; } From fd5adbabde8e83796e551981dc2d87c73fd84282 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Wed, 3 Jul 2024 10:57:45 +0000 Subject: [PATCH 049/112] lint --- redis_json/src/commands.rs | 61 ++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/redis_json/src/commands.rs b/redis_json/src/commands.rs index 20a86f8f7..429d5276b 100644 --- a/redis_json/src/commands.rs +++ b/redis_json/src/commands.rs @@ -725,24 +725,48 @@ where // check context flags to see if RESP3 is enabled if is_resp3(ctx) { - let res = json_num_op_impl::(manager, &mut redis_key, ctx, path.get_path(), number, op, cmd)? - .drain(..) - .map(|v| { - v.map_or(RedisValue::Null, |v| { - if let Some(i) = v.as_i64() { - RedisValue::Integer(i) - } else { - RedisValue::Float(v.as_f64().unwrap_or_default()) - } - }) + let res = json_num_op_impl::( + manager, + &mut redis_key, + ctx, + path.get_path(), + number, + op, + cmd, + )? + .drain(..) + .map(|v| { + v.map_or(RedisValue::Null, |v| { + if let Some(i) = v.as_i64() { + RedisValue::Integer(i) + } else { + RedisValue::Float(v.as_f64().unwrap_or_default()) + } }) - .collect::>() - .into(); + }) + .collect::>() + .into(); Ok(res) } else if path.is_legacy() { - json_num_op_legacy::(manager, &mut redis_key, ctx, path.get_path(), number, op, cmd) + json_num_op_legacy::( + manager, + &mut redis_key, + ctx, + path.get_path(), + number, + op, + cmd, + ) } else { - let results = json_num_op_impl::(manager, &mut redis_key, ctx, path.get_path(), number, op, cmd)?; + let results = json_num_op_impl::( + manager, + &mut redis_key, + ctx, + path.get_path(), + number, + op, + cmd, + )?; // Convert to RESP2 format return as one JSON array let values = to_json_value::(results, Value::Null); @@ -1457,7 +1481,14 @@ pub fn json_arr_pop(manager: M, ctx: &Context, args: Vec(manager, &mut redis_key, ctx, path.get_path(), index) } else { - json_arr_pop_impl::(manager, &mut redis_key, ctx, path.get_path(), index, &format_options) + json_arr_pop_impl::( + manager, + &mut redis_key, + ctx, + path.get_path(), + index, + &format_options, + ) } } From 0a1de826d8f67925298a86c6ad2380fa0d3c1a39 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Tue, 9 Jul 2024 14:30:59 +0300 Subject: [PATCH 050/112] MOD-7342 fix centos7&amzn2 build (#1214) --- deps/readies | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/readies b/deps/readies index a1d34e41d..7fc8e62b2 160000 --- a/deps/readies +++ b/deps/readies @@ -1 +1 @@ -Subproject commit a1d34e41d644bf5c32ec41d609a61996eb031301 +Subproject commit 7fc8e62b24d3f7ff01096d83f14cbc216ac0e2f0 From e8d9b9a252ec2f6812ee7f3e708eb240d3ec9098 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Thu, 11 Jul 2024 09:36:31 +0300 Subject: [PATCH 051/112] MOD-7368 make c_api public function (#1217) --- redis_json/src/c_api.rs | 72 ++++++++++++++--------------------------- redis_json/src/lib.rs | 1 + 2 files changed, 25 insertions(+), 48 deletions(-) diff --git a/redis_json/src/c_api.rs b/redis_json/src/c_api.rs index b4d399aed..fea81f207 100644 --- a/redis_json/src/c_api.rs +++ b/redis_json/src/c_api.rs @@ -47,7 +47,7 @@ struct ResultsIterator<'a, V: SelectValue> { pub static mut LLAPI_CTX: Option<*mut rawmod::RedisModuleCtx> = None; #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub(crate) fn create_rmstring( +pub fn create_rmstring( ctx: *mut rawmod::RedisModuleCtx, from_str: &str, str: *mut *mut rawmod::RedisModuleString, @@ -61,7 +61,7 @@ pub(crate) fn create_rmstring( Status::Err as c_int } -pub(crate) fn json_api_open_key_internal( +pub fn json_api_open_key_internal( manager: M, ctx: *mut rawmod::RedisModuleCtx, key: RedisString, @@ -75,7 +75,7 @@ pub(crate) fn json_api_open_key_internal( null() } -pub(crate) fn json_api_open_key_with_flags_internal( +pub fn json_api_open_key_with_flags_internal( manager: M, ctx: *mut rawmod::RedisModuleCtx, key: RedisString, @@ -90,11 +90,7 @@ pub(crate) fn json_api_open_key_with_flags_internal( null() } -pub(crate) fn json_api_get_at( - _: M, - json: *const c_void, - index: size_t, -) -> *const c_void { +pub fn json_api_get_at(_: M, json: *const c_void, index: size_t) -> *const c_void { let json = unsafe { &*(json.cast::()) }; match json.get_type() { SelectValueType::Array => json @@ -105,11 +101,7 @@ pub(crate) fn json_api_get_at( } #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub(crate) fn json_api_get_len( - _: M, - json: *const c_void, - count: *mut libc::size_t, -) -> c_int { +pub fn json_api_get_len(_: M, json: *const c_void, count: *mut libc::size_t) -> c_int { let json = unsafe { &*(json.cast::()) }; let len = match json.get_type() { SelectValueType::String => Some(json.get_str().len()), @@ -125,11 +117,11 @@ pub(crate) fn json_api_get_len( } } -pub(crate) fn json_api_get_type(_: M, json: *const c_void) -> c_int { +pub fn json_api_get_type(_: M, json: *const c_void) -> c_int { json_api_get_type_internal(unsafe { &*(json.cast::()) }) as c_int } -pub(crate) fn json_api_get_string( +pub fn json_api_get_string( _: M, json: *const c_void, str: *mut *const c_char, @@ -146,7 +138,7 @@ pub(crate) fn json_api_get_string( } } -pub(crate) fn json_api_get_json( +pub fn json_api_get_json( _: M, json: *const c_void, ctx: *mut rawmod::RedisModuleCtx, @@ -157,7 +149,7 @@ pub(crate) fn json_api_get_json( create_rmstring(ctx, &res, str) } -pub(crate) fn json_api_get_json_from_iter( +pub fn json_api_get_json_from_iter( _: M, iter: *mut c_void, ctx: *mut rawmod::RedisModuleCtx, @@ -174,11 +166,7 @@ pub(crate) fn json_api_get_json_from_iter( } #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub(crate) fn json_api_get_int( - _: M, - json: *const c_void, - val: *mut c_longlong, -) -> c_int { +pub fn json_api_get_int(_: M, json: *const c_void, val: *mut c_longlong) -> c_int { let json = unsafe { &*(json.cast::()) }; match json.get_type() { SelectValueType::Long => { @@ -190,11 +178,7 @@ pub(crate) fn json_api_get_int( } #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub(crate) fn json_api_get_double( - _: M, - json: *const c_void, - val: *mut c_double, -) -> c_int { +pub fn json_api_get_double(_: M, json: *const c_void, val: *mut c_double) -> c_int { let json = unsafe { &*(json.cast::()) }; match json.get_type() { SelectValueType::Double => { @@ -210,11 +194,7 @@ pub(crate) fn json_api_get_double( } #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub(crate) fn json_api_get_boolean( - _: M, - json: *const c_void, - val: *mut c_int, -) -> c_int { +pub fn json_api_get_boolean(_: M, json: *const c_void, val: *mut c_int) -> c_int { let json = unsafe { &*(json.cast::()) }; match json.get_type() { SelectValueType::Bool => { @@ -228,7 +208,7 @@ pub(crate) fn json_api_get_boolean( //--------------------------------------------------------------------------------------------- #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub(crate) fn set_string(from_str: &str, str: *mut *const c_char, len: *mut size_t) -> c_int { +pub fn set_string(from_str: &str, str: *mut *const c_char, len: *mut size_t) -> c_int { if !str.is_null() { unsafe { *str = from_str.as_ptr().cast::(); @@ -251,7 +231,7 @@ fn json_api_get_type_internal(v: &V) -> JSONType { } } -pub(crate) fn json_api_next(_: M, iter: *mut c_void) -> *const c_void { +pub fn json_api_next(_: M, iter: *mut c_void) -> *const c_void { let iter = unsafe { &mut *(iter.cast::>()) }; if iter.pos >= iter.results.len() { null_mut() @@ -262,28 +242,24 @@ pub(crate) fn json_api_next(_: M, iter: *mut c_void) -> *const c_voi } } -pub(crate) fn json_api_len(_: M, iter: *const c_void) -> size_t { +pub fn json_api_len(_: M, iter: *const c_void) -> size_t { let iter = unsafe { &*(iter.cast::>()) }; iter.results.len() as size_t } -pub(crate) fn json_api_free_iter(_: M, iter: *mut c_void) { +pub fn json_api_free_iter(_: M, iter: *mut c_void) { unsafe { drop(Box::from_raw(iter.cast::>())); } } -pub(crate) fn json_api_reset_iter(_: M, iter: *mut c_void) { +pub fn json_api_reset_iter(_: M, iter: *mut c_void) { let iter = unsafe { &mut *(iter.cast::>()) }; iter.pos = 0; } #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub(crate) fn json_api_get( - _: M, - val: *const c_void, - path: *const c_char, -) -> *const c_void { +pub fn json_api_get(_: M, val: *const c_void, path: *const c_char) -> *const c_void { let v = unsafe { &*(val.cast::()) }; let path = unsafe { CStr::from_ptr(path).to_str().unwrap() }; let query = match compile(path) { @@ -299,11 +275,11 @@ pub(crate) fn json_api_get( .cast::() } -pub(crate) fn json_api_is_json(m: M, key: *mut rawmod::RedisModuleKey) -> c_int { +pub fn json_api_is_json(m: M, key: *mut rawmod::RedisModuleKey) -> c_int { m.is_json(key).map_or(0, |res| res as c_int) } -pub(crate) fn json_api_get_key_value(_: M, val: *const c_void) -> *const c_void { +pub fn json_api_get_key_value(_: M, val: *const c_void) -> *const c_void { let json = unsafe { &*(val.cast::()) }; match json.get_type() { SelectValueType::Object => Box::into_raw(Box::new(json.items().unwrap())).cast::(), @@ -311,7 +287,7 @@ pub(crate) fn json_api_get_key_value(_: M, val: *const c_void) -> *c } } -pub(crate) fn json_api_next_key_value<'a, M: Manager>( +pub fn json_api_next_key_value<'a, M: Manager>( _: M, iter: *mut c_void, str: *mut *mut rawmod::RedisModuleString, @@ -328,7 +304,7 @@ where } } -pub(crate) fn json_api_free_key_values_iter<'a, M: Manager>(_: M, iter: *mut c_void) +pub fn json_api_free_key_values_iter<'a, M: Manager>(_: M, iter: *mut c_void) where M::V: 'a, { @@ -338,7 +314,7 @@ where } } -pub(crate) fn get_llapi_ctx() -> Context { +pub fn get_llapi_ctx() -> Context { Context::new(unsafe { LLAPI_CTX.unwrap() }) } @@ -539,7 +515,7 @@ macro_rules! redis_json_module_export_shared_api { match json_path::compile(path) { Ok(q) => Box::into_raw(Box::new(q)).cast::(), Err(e) => { - crate::c_api::create_rmstring(ctx, &format!("{}", e), err_msg); + create_rmstring(ctx, &format!("{}", e), err_msg); std::ptr::null() } } diff --git a/redis_json/src/lib.rs b/redis_json/src/lib.rs index adbf1502a..1570e0537 100644 --- a/redis_json/src/lib.rs +++ b/redis_json/src/lib.rs @@ -128,6 +128,7 @@ macro_rules! redis_json_module_create {( }; use libc::size_t; use std::collections::HashMap; + use $crate::c_api::create_rmstring; macro_rules! json_command { ($cmd:ident) => { From 52aa9a682ab48d122271dfb57cb5ed30cf3f2cc8 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Sun, 14 Jul 2024 15:07:31 +0300 Subject: [PATCH 052/112] initial commit --- .circleci/config.yml | 64 +++++++++++++++++++++---------------------- sbin/upload-artifacts | 21 ++++++++------ 2 files changed, 44 insertions(+), 41 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d9f3aa609..e06c663f7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -319,27 +319,27 @@ jobs: - vm-build-platforms-steps: platform: <> - build-macos-x64: - macos: - xcode: 13.4.1 - resource_class: macos.x86.medium.gen2 - parameters: - upload: - type: string - default: "yes" - steps: - - early-returns - - build-steps - - test-steps - - persist-artifacts - - run: - name: Upload artifacts to S3 - command: | - if [[ -n $CIRCLE_BRANCH ]]; then - make upload-artifacts SHOW=1 - else - make upload-release SHOW=1 - fi + # build-macos-x64: + # macos: + # xcode: 13.4.1 + # resource_class: macos.x86.medium.gen2 + # parameters: + # upload: + # type: string + # default: "yes" + # steps: + # - early-returns + # - build-steps + # - test-steps + # - persist-artifacts + # - run: + # name: Upload artifacts to S3 + # command: | + # if [[ -n $CIRCLE_BRANCH ]]; then + # make upload-artifacts SHOW=1 + # else + # make upload-release SHOW=1 + # fi build-macos-m1: macos: @@ -532,9 +532,9 @@ workflows: matrix: parameters: platform: [jammy, focal, bionic] - - build-macos-x64: - <<: *on-integ-and-version-tags - context: common + # - build-macos-x64: + # <<: *on-integ-and-version-tags + # context: common - build-macos-m1: context: common <<: *on-integ-and-version-tags @@ -554,7 +554,7 @@ workflows: requires: - build-platforms - build-arm-platforms - - build-macos-x64 + # - build-macos-x64 - build-macos-m1 - upload-artifacts: name: upload-release-artifacts @@ -563,7 +563,7 @@ workflows: requires: - build-platforms - build-arm-platforms - - build-macos-x64 + # - build-macos-x64 - build-macos-m1 - release-qa-tests: <<: *on-version-tags @@ -588,9 +588,9 @@ workflows: when: << pipeline.parameters.run_nightly_twice_a_week_flow_label >> jobs: - - build-macos-x64: - context: common - upload: "yes" + # - build-macos-x64: + # context: common + # upload: "yes" - build-macos-m1: context: common upload: "yes" @@ -601,9 +601,9 @@ workflows: cron: "20 17 * * 0,3" <<: *on-integ-branch-cron jobs: - - build-macos-x64: - context: common - upload: "yes" + # - build-macos-x64: + # context: common + # upload: "yes" - build-macos-m1: context: common upload: "yes" diff --git a/sbin/upload-artifacts b/sbin/upload-artifacts index d303d1045..7fdd1eff6 100755 --- a/sbin/upload-artifacts +++ b/sbin/upload-artifacts @@ -54,15 +54,16 @@ OS=$($READIES/bin/platform --os) if [[ $OS == macos ]]; then # as we don't build on macOS for every platform, we converge to a least common denominator - if [[ $ARCH == x86_64 ]]; then - OSNICK=catalina # to be aligned with the rest of the modules in redis stack - else - [[ $OSNICK == ventura ]] && OSNICK=monterey - fi + # if [[ $ARCH == x86_64 ]]; then + # OSNICK=catalina # to be aligned with the rest of the modules in redis stack + # else + # [[ $OSNICK == ventura ]] && OSNICK=monterey + # fi + PLATFORMS=("$OS-catalina-x86_64" "$OS-monterey-aarch64") +else + PLATFORMS=("$OS-$OSNICK-$ARCH") fi -PLATFORM="$OS-$OSNICK-$ARCH" - #---------------------------------------------------------------------------------------------- OP="" @@ -113,8 +114,10 @@ s3_upload() { local upload_dir="${S3_URL}/${prod_subdir}${MAYBE_SNAP}" local file if [[ $SNAPSHOT == 1 ]]; then - for file in `ls ${prefix}.*${PLATFORM}*.zip ${prefix}.*${PLATFORM}*.tgz 2> /dev/null`; do - s3_upload_file $file $upload_dir + for platform in ${PLATFORMS[@]}; do + for file in `ls ${prefix}.*${platform}*.zip ${prefix}.*${platform}*.tgz 2> /dev/null`; do + s3_upload_file $file $upload_dir + done done else for file in `ls ${prefix}.*.zip ${prefix}.*.tgz 2> /dev/null`; do From 7c65f7e52da871d914c52a4b5759a6192c1297af Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Sun, 14 Jul 2024 16:23:08 +0300 Subject: [PATCH 053/112] modify default flow to check that run build scripts for all platforms works --- .circleci/config.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e06c663f7..348f0eb6b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -517,17 +517,19 @@ workflows: when: << pipeline.parameters.run_default_flow >> jobs: - - build-linux-debian: - name: build - <<: *not-on-integ-branch + # - build-linux-debian: + # name: build + # <<: *not-on-integ-branch - build-platforms: - <<: *on-integ-and-version-tags + <<: *not-on-integ-branch + # <<: *on-integ-and-version-tags context: common matrix: parameters: platform: [jammy, focal, bionic, centos7, rocky8, rocky9, bullseye, amzn2] - build-arm-platforms: - <<: *on-integ-and-version-tags + <<: *not-on-integ-branch + # <<: *on-integ-and-version-tags context: common matrix: parameters: @@ -537,7 +539,8 @@ workflows: # context: common - build-macos-m1: context: common - <<: *on-integ-and-version-tags + <<: *not-on-integ-branch + # <<: *on-integ-and-version-tags - coverage: <<: *always - sanitize: From 9bf11a1a8755c705a959226182390a4ecfec460e Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Sun, 14 Jul 2024 16:24:34 +0300 Subject: [PATCH 054/112] modify default flow to check that uploading artifacts to staging lab works --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 348f0eb6b..a903048e0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -551,7 +551,8 @@ workflows: san-type: [address] - upload-artifacts: name: upload-artifacts-to-staging-lab - <<: *on-integ-branch + <<: *not-on-integ-branch + # <<: *on-integ-branch staging-lab: "1" context: common requires: From 917cae57622956cd649345e3feb1a3457f474639 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Mon, 15 Jul 2024 16:23:10 +0300 Subject: [PATCH 055/112] moving to GHA --- .github/workflows/macos-build-and-deploy.yml | 56 +++++++ .install/macos-12.sh | 2 + sbin/upload-artifacts | 165 +++++++++++++++++-- 3 files changed, 211 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/macos-build-and-deploy.yml create mode 100644 .install/macos-12.sh diff --git a/.github/workflows/macos-build-and-deploy.yml b/.github/workflows/macos-build-and-deploy.yml new file mode 100644 index 000000000..daa2c032d --- /dev/null +++ b/.github/workflows/macos-build-and-deploy.yml @@ -0,0 +1,56 @@ +name: Build for macos + +on: + push: + workflow_dispatch: + # branches: + # - master + # tags: + # - 'v[0-9]+.[0-9]+.[0-9]+' + +jobs: + build: + runs-on: macos-12 + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - name: Install prereqs + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Install dependencies + run: | + bash .install/macos-12.sh + - name: Get Redis + uses: actions/checkout@v4 + with: + repository: redis/redis + ref: 'unstable' # todo change per version/tag + path: redis + - name: Build Redis + working-directory: redis + run: make install + - name: Build module + run: | + make build + - name: Test + run: | + make test + - name: Pack module + run: | + make pack BRANCH=${{ github.ref_name }} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: "us-east-1" + - name: Upload artifacts to S3 - staging + run: | + make upload-artifacts SHOW=1 VERBOSE=1 + make upload-release SHOW=1 STAGING=1 VERBOSE=1 + - name: Upload artifacts to S3 - release # todo: trigger this manually instead + if: ${{ github.ref != 'refs/heads/master' }} + run: make upload-release SHOW=1 VERBOSE=1 diff --git a/.install/macos-12.sh b/.install/macos-12.sh new file mode 100644 index 000000000..05a7907cf --- /dev/null +++ b/.install/macos-12.sh @@ -0,0 +1,2 @@ +#!/bin/bash + diff --git a/sbin/upload-artifacts b/sbin/upload-artifacts index 7fdd1eff6..99b7c57f7 100755 --- a/sbin/upload-artifacts +++ b/sbin/upload-artifacts @@ -54,16 +54,15 @@ OS=$($READIES/bin/platform --os) if [[ $OS == macos ]]; then # as we don't build on macOS for every platform, we converge to a least common denominator - # if [[ $ARCH == x86_64 ]]; then - # OSNICK=catalina # to be aligned with the rest of the modules in redis stack - # else - # [[ $OSNICK == ventura ]] && OSNICK=monterey - # fi - PLATFORMS=("$OS-catalina-x86_64" "$OS-monterey-aarch64") -else - PLATFORMS=("$OS-$OSNICK-$ARCH") + if [[ $ARCH == x86_64 ]]; then + OSNICK=catalina # to be aligned with the rest of the modules in redis stack + else + [[ $OSNICK == ventura ]] && OSNICK=monterey + fi fi +PLATFORM="$OS-$OSNICK-$ARCH" + #---------------------------------------------------------------------------------------------- OP="" @@ -114,10 +113,8 @@ s3_upload() { local upload_dir="${S3_URL}/${prod_subdir}${MAYBE_SNAP}" local file if [[ $SNAPSHOT == 1 ]]; then - for platform in ${PLATFORMS[@]}; do - for file in `ls ${prefix}.*${platform}*.zip ${prefix}.*${platform}*.tgz 2> /dev/null`; do - s3_upload_file $file $upload_dir - done + for file in `ls ${prefix}.*${PLATFORM}*.zip ${prefix}.*${PLATFORM}*.tgz 2> /dev/null`; do + s3_upload_file $file $upload_dir done else for file in `ls ${prefix}.*.zip ${prefix}.*.tgz 2> /dev/null`; do @@ -138,3 +135,147 @@ if [[ $NOP != 1 ]]; then fi PROD=rejson-oss PREFIX=rejson-oss s3_upload + + + + +# #!/bin/bash + +# PROGNAME="${BASH_SOURCE[0]}" +# HERE="$(cd "$(dirname "$PROGNAME")" &>/dev/null && pwd)" +# ROOT=$(cd $HERE/.. && pwd) +# READIES=$ROOT/deps/readies +# . $READIES/shibumi/defs + +# #---------------------------------------------------------------------------------------------- + +# if [[ $1 == --help || $1 == help || $HELP == 1 ]]; then +# cat <<-END +# Upload packages to S3 + +# upload-artifacts [--help|help] artifacts... + +# Argument variables: +# SNAPSHOT=1 Upload snapshots (default) +# OSNICK=nick Operate on packages for given OSNICK (default: host) + +# RELEASE=1 Upload release artifacts +# STAGING=1 Upload into staging area + +# FORCE=1 Allow uploading outside of CI +# NOP=1 No operation +# VERBOSE=1 Show artifacts details +# HELP=1 Show help + +# END +# exit 0 +# fi + +# #---------------------------------------------------------------------------------------------- + +# ARCH=$($READIES/bin/platform --arch) +# [[ $ARCH == x64 ]] && ARCH="x86_64" +# [[ $ARCH == arm64v8 ]] && ARCH="aarch64" + +# OS=$($READIES/bin/platform --os) +# [[ $OS == linux ]] && OS="Linux" + +# [[ -z $OSNICK ]] && OSNICK=$($READIES/bin/platform --osnick) +# [[ $OSNICK == trusty ]] && OSNICK=ubuntu14.04 +# [[ $OSNICK == xenial ]] && OSNICK=ubuntu16.04 +# [[ $OSNICK == bionic ]] && OSNICK=ubuntu18.04 +# [[ $OSNICK == focal ]] && OSNICK=ubuntu20.04 +# [[ $OSNICK == jammy ]] && OSNICK=ubuntu22.04 +# [[ $OSNICK == centos7 ]] && OSNICK=rhel7 +# [[ $OSNICK == centos8 ]] && OSNICK=rhel8 +# [[ $OSNICK == centos9 ]] && OSNICK=rhel9 +# [[ $OSNICK == ol8 ]] && OSNICK=rhel8 +# [[ $OSNICK == rocky8 ]] && OSNICK=rhel8 +# [[ $OSNICK == rocky9 ]] && OSNICK=rhel9 + +# if [[ $OS == macos ]]; then +# # as we don't build on macOS for every platform, we converge to a least common denominator +# # if [[ $ARCH == x86_64 ]]; then +# # OSNICK=catalina # to be aligned with the rest of the modules in redis stack +# # else +# # [[ $OSNICK == ventura ]] && OSNICK=monterey +# # fi +# PLATFORMS=("$OS-catalina-x86_64" "$OS-monterey-aarch64") +# else +# PLATFORMS=("$OS-$OSNICK-$ARCH") +# fi + +# #---------------------------------------------------------------------------------------------- + +# OP="" +# [[ $NOP == 1 ]] && OP=echo + +# if [[ $STAGING == 1 ]]; then +# S3_URL=s3://redismodules/lab/staging +# else +# S3_URL=s3://redismodules +# fi + +# if [[ -z $AWS_ACCESS_KEY_ID || -z $AWS_SECRET_ACCESS_KEY ]]; then +# eprint "No credentials for S3 upload." +# exit 1 +# fi + +# cd $ROOT/bin + +# if [[ $SNAPSHOT == 1 ]]; then +# MAYBE_SNAP=/snapshots +# else +# MAYBE_SNAP= +# fi + +# cd artifacts${MAYBE_SNAP} +# [[ $VERBOSE == 1 ]] && du -ah --apparent-size * + +# #---------------------------------------------------------------------------------------------- + +# s3_upload_file() { +# local file="$1" +# local s3_dir="$2" +# [[ $s3_dir != */ ]] && s3_dir="${s3_dir}/" + +# $OP aws --debug s3 cp $file $s3_dir --acl public-read --no-progress +# } + +# s3_ls() { +# local s3_dir="$1" +# [[ $s3_dir != */ ]] && s3_dir="${s3_dir}/" + +# $OP aws s3 ls $s3_dir +# } + +# s3_upload() { +# local prod_subdir="$PROD" +# local prefix="$PREFIX" +# local upload_dir="${S3_URL}/${prod_subdir}${MAYBE_SNAP}" +# local file +# if [[ $SNAPSHOT == 1 ]]; then +# for platform in ${PLATFORMS[@]}; do +# for file in `ls ${prefix}.*${platform}*.zip ${prefix}.*${platform}*.tgz 2> /dev/null`; do +# s3_upload_file $file $upload_dir +# done +# done +# else +# for file in `ls ${prefix}.*.zip ${prefix}.*.tgz 2> /dev/null`; do +# s3_upload_file $file $upload_dir +# done +# fi +# [[ $VERBOSE == 1 ]] && s3_ls $upload_dir + +# return 0 +# } + +# #---------------------------------------------------------------------------------------------- + +# if [[ $NOP != 1 ]]; then +# if ! is_command aws; then +# $READIES/bin/getaws +# fi +# fi + +# PROD=rejson-oss PREFIX=rejson-oss s3_upload From f7bfdde81c0019fbc5c603e14956d3c58bef2048 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Mon, 15 Jul 2024 16:26:05 +0300 Subject: [PATCH 056/112] fix --- .github/workflows/macos-build-and-deploy.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/macos-build-and-deploy.yml b/.github/workflows/macos-build-and-deploy.yml index daa2c032d..feb73e540 100644 --- a/.github/workflows/macos-build-and-deploy.yml +++ b/.github/workflows/macos-build-and-deploy.yml @@ -15,7 +15,6 @@ jobs: run: shell: bash -l -eo pipefail {0} steps: - - name: Install prereqs - name: Checkout uses: actions/checkout@v4 with: From e8700ae372d06f9c8087650e60911898fd071789 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Mon, 15 Jul 2024 16:33:58 +0300 Subject: [PATCH 057/112] fix --- .install/macos-12.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.install/macos-12.sh b/.install/macos-12.sh index 05a7907cf..06876bd96 100644 --- a/.install/macos-12.sh +++ b/.install/macos-12.sh @@ -1,2 +1,28 @@ #!/bin/bash +echo "::error make --version" + +#!/bin/bash +version=3.25.1 +processor=$(uname -m) +OS_TYPE=$(uname -s) +MODE=$1 # whether to install using sudo or not + +if [[ $OS_TYPE = 'Darwin' ]] +then + brew install cmake +else + if [[ $processor = 'x86_64' ]] + then + filename=cmake-${version}-linux-x86_64.sh + else + filename=cmake-${version}-linux-aarch64.sh + fi + + wget https://github.com/Kitware/CMake/releases/download/v${version}/${filename} + chmod u+x ./${filename} + $MODE ./${filename} --skip-license --prefix=/usr/local --exclude-subdir + cmake --version +fi + +echo "::error make --version" \ No newline at end of file From e16669ed700066b3b290d3a499e0eff15c30b97f Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Mon, 15 Jul 2024 16:36:32 +0300 Subject: [PATCH 058/112] fix --- .install/macos-12.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.install/macos-12.sh b/.install/macos-12.sh index 06876bd96..8ae41cb97 100644 --- a/.install/macos-12.sh +++ b/.install/macos-12.sh @@ -1,8 +1,7 @@ #!/bin/bash -echo "::error make --version" +echo "::error $(make --version)" -#!/bin/bash version=3.25.1 processor=$(uname -m) OS_TYPE=$(uname -s) @@ -25,4 +24,4 @@ else cmake --version fi -echo "::error make --version" \ No newline at end of file +echo "::error $(make --version)" \ No newline at end of file From 1553712430adc51667ac8c675151e023a92b8490 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Mon, 15 Jul 2024 16:52:00 +0300 Subject: [PATCH 059/112] fix --- .install/macos-12.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.install/macos-12.sh b/.install/macos-12.sh index 8ae41cb97..a6534e7b2 100644 --- a/.install/macos-12.sh +++ b/.install/macos-12.sh @@ -2,6 +2,24 @@ echo "::error $(make --version)" +export HOMEBREW_NO_AUTO_UPDATE=1 +BREW_PREFIX=$(brew --prefix) +GNUBIN=$BREW_PREFIX/opt/make/libexec/gnubin +LLVM=$BREW_PREFIX/opt/llvm@16/bin +COREUTILS=$BREW_PREFIX/opt/coreutils/libexec/gnubin + +brew update +brew install coreutils +brew install make +brew install llvm@16 + +echo "export PATH=$COREUTILS:$LLVM:$GNUBIN:$PATH" >> ~/.bashrc +echo "export PATH=$COREUTILS:$LLVM:$GNUBIN:$PATH" >> ~/.zshrc +source ~/.bashrc +source ~/.zshrc + +brew install openssl + version=3.25.1 processor=$(uname -m) OS_TYPE=$(uname -s) From b38e2ef3372f7671297493d5fe64028931608ed7 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Mon, 15 Jul 2024 17:13:24 +0300 Subject: [PATCH 060/112] add RLTest --- .github/workflows/macos-build-and-deploy.yml | 11 ++++++++ .install/install_cmake.sh | 22 ++++++++++++++++ .install/install_script.sh | 22 ++++++++++++++++ .install/macos-12.sh | 27 +------------------- 4 files changed, 56 insertions(+), 26 deletions(-) create mode 100644 .install/install_cmake.sh create mode 100755 .install/install_script.sh diff --git a/.github/workflows/macos-build-and-deploy.yml b/.github/workflows/macos-build-and-deploy.yml index feb73e540..5cfd0dff3 100644 --- a/.github/workflows/macos-build-and-deploy.yml +++ b/.github/workflows/macos-build-and-deploy.yml @@ -34,6 +34,17 @@ jobs: - name: Build module run: | make build + - name: Deps checkout + uses: actions/checkout@v3 + with: + path: setup + sparse-checkout-cone-mode: false + sparse-checkout: | + .install + tests/pytests/requirements.* + - name: Setup specific + working-directory: setup/.install + run: ./install_script.sh ${{ steps.mode.outputs.mode }} - name: Test run: | make test diff --git a/.install/install_cmake.sh b/.install/install_cmake.sh new file mode 100644 index 000000000..418a8df22 --- /dev/null +++ b/.install/install_cmake.sh @@ -0,0 +1,22 @@ +#!/bin/bash +version=3.25.1 +processor=$(uname -m) +OS_TYPE=$(uname -s) +MODE=$1 # whether to install using sudo or not + +if [[ $OS_TYPE = 'Darwin' ]] +then + brew install cmake +else + if [[ $processor = 'x86_64' ]] + then + filename=cmake-${version}-linux-x86_64.sh + else + filename=cmake-${version}-linux-aarch64.sh + fi + + wget https://github.com/Kitware/CMake/releases/download/v${version}/${filename} + chmod u+x ./${filename} + $MODE ./${filename} --skip-license --prefix=/usr/local --exclude-subdir + cmake --version +fi diff --git a/.install/install_script.sh b/.install/install_script.sh new file mode 100755 index 000000000..b66f8effd --- /dev/null +++ b/.install/install_script.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +OS_TYPE=$(uname -s) +MODE=$1 # whether to install using sudo or not + +if [[ $OS_TYPE = 'Darwin' ]] +then + OS='macos' +else + VERSION=$(grep '^VERSION_ID=' /etc/os-release | sed 's/"//g') + VERSION=${VERSION#"VERSION_ID="} + OS_NAME=$(grep '^NAME=' /etc/os-release | sed 's/"//g') + OS_NAME=${OS_NAME#"NAME="} + [[ $OS_NAME == 'Rocky Linux' ]] && VERSION=${VERSION%.*} # remove minor version for Rocky Linux + OS=${OS_NAME,,}_${VERSION} + OS=$(echo $OS | sed 's/[/ ]/_/g') # replace spaces and slashes with underscores +fi +echo $OS + +source ${OS}.sh $MODE + +git config --global --add safe.directory '*' diff --git a/.install/macos-12.sh b/.install/macos-12.sh index a6534e7b2..2db98a96d 100644 --- a/.install/macos-12.sh +++ b/.install/macos-12.sh @@ -1,7 +1,5 @@ #!/bin/bash -echo "::error $(make --version)" - export HOMEBREW_NO_AUTO_UPDATE=1 BREW_PREFIX=$(brew --prefix) GNUBIN=$BREW_PREFIX/opt/make/libexec/gnubin @@ -19,27 +17,4 @@ source ~/.bashrc source ~/.zshrc brew install openssl - -version=3.25.1 -processor=$(uname -m) -OS_TYPE=$(uname -s) -MODE=$1 # whether to install using sudo or not - -if [[ $OS_TYPE = 'Darwin' ]] -then - brew install cmake -else - if [[ $processor = 'x86_64' ]] - then - filename=cmake-${version}-linux-x86_64.sh - else - filename=cmake-${version}-linux-aarch64.sh - fi - - wget https://github.com/Kitware/CMake/releases/download/v${version}/${filename} - chmod u+x ./${filename} - $MODE ./${filename} --skip-license --prefix=/usr/local --exclude-subdir - cmake --version -fi - -echo "::error $(make --version)" \ No newline at end of file +source install_cmake.sh From 037d8e7cd5d0ff22a53214c6cfa84479f6f049ef Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Mon, 15 Jul 2024 17:20:19 +0300 Subject: [PATCH 061/112] fix --- .github/workflows/macos-build-and-deploy.yml | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/macos-build-and-deploy.yml b/.github/workflows/macos-build-and-deploy.yml index 5cfd0dff3..16439b653 100644 --- a/.github/workflows/macos-build-and-deploy.yml +++ b/.github/workflows/macos-build-and-deploy.yml @@ -19,9 +19,20 @@ jobs: uses: actions/checkout@v4 with: submodules: 'recursive' - - name: Install dependencies - run: | - bash .install/macos-12.sh + - name: Deps checkout + uses: actions/checkout@v3 + with: + path: setup + sparse-checkout-cone-mode: false + sparse-checkout: | + .install + tests/pytests/requirements.* + - name: Setup specific + working-directory: setup/.install + run: ./install_script.sh ${{ steps.mode.outputs.mode }} + # - name: Install dependencies + # run: | + # bash .install/macos-12.sh - name: Get Redis uses: actions/checkout@v4 with: @@ -34,17 +45,6 @@ jobs: - name: Build module run: | make build - - name: Deps checkout - uses: actions/checkout@v3 - with: - path: setup - sparse-checkout-cone-mode: false - sparse-checkout: | - .install - tests/pytests/requirements.* - - name: Setup specific - working-directory: setup/.install - run: ./install_script.sh ${{ steps.mode.outputs.mode }} - name: Test run: | make test From 9ba1f236c9861ee01367f2f50b301db90b61fb88 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Mon, 15 Jul 2024 17:23:23 +0300 Subject: [PATCH 062/112] fix name --- .install/{macos-12.sh => macos.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .install/{macos-12.sh => macos.sh} (100%) diff --git a/.install/macos-12.sh b/.install/macos.sh similarity index 100% rename from .install/macos-12.sh rename to .install/macos.sh From 2c3e79b088cfc3facfbc7f70cf5c26f93a8287d4 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Mon, 15 Jul 2024 17:32:18 +0300 Subject: [PATCH 063/112] actually do the installation --- .github/workflows/macos-build-and-deploy.yml | 6 +++++ .install/common_installations.sh | 28 ++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100755 .install/common_installations.sh diff --git a/.github/workflows/macos-build-and-deploy.yml b/.github/workflows/macos-build-and-deploy.yml index 16439b653..b5a48c3e8 100644 --- a/.github/workflows/macos-build-and-deploy.yml +++ b/.github/workflows/macos-build-and-deploy.yml @@ -30,6 +30,12 @@ jobs: - name: Setup specific working-directory: setup/.install run: ./install_script.sh ${{ steps.mode.outputs.mode }} + - name: Full checkout + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Setup common + run: .install/common_installations.sh ${{ steps.mode.outputs.mode }} # - name: Install dependencies # run: | # bash .install/macos-12.sh diff --git a/.install/common_installations.sh b/.install/common_installations.sh new file mode 100755 index 000000000..6dab3f92a --- /dev/null +++ b/.install/common_installations.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -e +OS_TYPE=$(uname -s) +MODE=$1 # whether to install using sudo or not + +activate_venv() { + echo "copy ativation script to shell config" + if [[ $OS_TYPE == Darwin ]]; then + echo "source venv/bin/activate" >> ~/.bashrc + echo "source venv/bin/activate" >> ~/.zshrc + else + echo "source $PWD/venv/bin/activate" >> ~/.bash_profile + fi +} + +python3 -m venv venv +activate_venv +source venv/bin/activate + +pip install --upgrade pip +pip install -q --upgrade setuptools +echo "pip version: $(pip --version)" +echo "pip path: $(which pip)" + +pip install -q -r tests/pytests/requirements.txt + +# List installed packages +pip list From 659024bd3b71d9dea4d6b7754a2f378fdab0741b Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Mon, 15 Jul 2024 17:42:09 +0300 Subject: [PATCH 064/112] json calls it pytest (singular) --- .github/workflows/macos-build-and-deploy.yml | 4 +--- .install/common_installations.sh | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/macos-build-and-deploy.yml b/.github/workflows/macos-build-and-deploy.yml index b5a48c3e8..dbe6a8d1e 100644 --- a/.github/workflows/macos-build-and-deploy.yml +++ b/.github/workflows/macos-build-and-deploy.yml @@ -36,9 +36,7 @@ jobs: submodules: recursive - name: Setup common run: .install/common_installations.sh ${{ steps.mode.outputs.mode }} - # - name: Install dependencies - # run: | - # bash .install/macos-12.sh + - name: Get Redis uses: actions/checkout@v4 with: diff --git a/.install/common_installations.sh b/.install/common_installations.sh index 6dab3f92a..4c915168c 100755 --- a/.install/common_installations.sh +++ b/.install/common_installations.sh @@ -22,7 +22,7 @@ pip install -q --upgrade setuptools echo "pip version: $(pip --version)" echo "pip path: $(which pip)" -pip install -q -r tests/pytests/requirements.txt +pip install -q -r tests/pytest/requirements.txt # List installed packages pip list From 4356ea8b89ac3ecdb8f7fc63f26c8d3c26dfe4c7 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Mon, 15 Jul 2024 17:42:32 +0300 Subject: [PATCH 065/112] json calls it pytest (singular) --- .github/workflows/macos-build-and-deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos-build-and-deploy.yml b/.github/workflows/macos-build-and-deploy.yml index dbe6a8d1e..ddf977077 100644 --- a/.github/workflows/macos-build-and-deploy.yml +++ b/.github/workflows/macos-build-and-deploy.yml @@ -26,7 +26,7 @@ jobs: sparse-checkout-cone-mode: false sparse-checkout: | .install - tests/pytests/requirements.* + tests/pytest/requirements.* - name: Setup specific working-directory: setup/.install run: ./install_script.sh ${{ steps.mode.outputs.mode }} @@ -36,7 +36,7 @@ jobs: submodules: recursive - name: Setup common run: .install/common_installations.sh ${{ steps.mode.outputs.mode }} - + - name: Get Redis uses: actions/checkout@v4 with: From af7ef5ebbb531194612647ba2a04ed9429bd09e9 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Mon, 15 Jul 2024 19:55:05 +0300 Subject: [PATCH 066/112] more dependencies --- .install/common_installations.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.install/common_installations.sh b/.install/common_installations.sh index 4c915168c..821e25269 100755 --- a/.install/common_installations.sh +++ b/.install/common_installations.sh @@ -23,6 +23,8 @@ echo "pip version: $(pip --version)" echo "pip path: $(which pip)" pip install -q -r tests/pytest/requirements.txt +# These packages are needed to build the package +pip install -q addict toml jinja2 ramp-packer # List installed packages pip list From 4999232f2ae1d655a0f842993a3b0c0d640c0329 Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Tue, 16 Jul 2024 15:08:46 +0300 Subject: [PATCH 067/112] removing macos-x64 flow from circleci --- .circleci/config.yml | 51 ++------------- sbin/upload-artifacts | 144 ------------------------------------------ 2 files changed, 7 insertions(+), 188 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a903048e0..ba3c2afb3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -319,28 +319,6 @@ jobs: - vm-build-platforms-steps: platform: <> - # build-macos-x64: - # macos: - # xcode: 13.4.1 - # resource_class: macos.x86.medium.gen2 - # parameters: - # upload: - # type: string - # default: "yes" - # steps: - # - early-returns - # - build-steps - # - test-steps - # - persist-artifacts - # - run: - # name: Upload artifacts to S3 - # command: | - # if [[ -n $CIRCLE_BRANCH ]]; then - # make upload-artifacts SHOW=1 - # else - # make upload-release SHOW=1 - # fi - build-macos-m1: macos: xcode: 14.2.0 @@ -517,30 +495,24 @@ workflows: when: << pipeline.parameters.run_default_flow >> jobs: - # - build-linux-debian: - # name: build - # <<: *not-on-integ-branch - - build-platforms: + - build-linux-debian: + name: build <<: *not-on-integ-branch - # <<: *on-integ-and-version-tags + - build-platforms: + <<: *on-integ-and-version-tags context: common matrix: parameters: platform: [jammy, focal, bionic, centos7, rocky8, rocky9, bullseye, amzn2] - build-arm-platforms: - <<: *not-on-integ-branch - # <<: *on-integ-and-version-tags + <<: *on-integ-and-version-tags context: common matrix: parameters: platform: [jammy, focal, bionic] - # - build-macos-x64: - # <<: *on-integ-and-version-tags - # context: common - build-macos-m1: context: common - <<: *not-on-integ-branch - # <<: *on-integ-and-version-tags + <<: *on-integ-and-version-tags - coverage: <<: *always - sanitize: @@ -551,14 +523,12 @@ workflows: san-type: [address] - upload-artifacts: name: upload-artifacts-to-staging-lab - <<: *not-on-integ-branch - # <<: *on-integ-branch + <<: *on-integ-branch staging-lab: "1" context: common requires: - build-platforms - build-arm-platforms - # - build-macos-x64 - build-macos-m1 - upload-artifacts: name: upload-release-artifacts @@ -567,7 +537,6 @@ workflows: requires: - build-platforms - build-arm-platforms - # - build-macos-x64 - build-macos-m1 - release-qa-tests: <<: *on-version-tags @@ -592,9 +561,6 @@ workflows: when: << pipeline.parameters.run_nightly_twice_a_week_flow_label >> jobs: - # - build-macos-x64: - # context: common - # upload: "yes" - build-macos-m1: context: common upload: "yes" @@ -605,9 +571,6 @@ workflows: cron: "20 17 * * 0,3" <<: *on-integ-branch-cron jobs: - # - build-macos-x64: - # context: common - # upload: "yes" - build-macos-m1: context: common upload: "yes" diff --git a/sbin/upload-artifacts b/sbin/upload-artifacts index 99b7c57f7..d303d1045 100755 --- a/sbin/upload-artifacts +++ b/sbin/upload-artifacts @@ -135,147 +135,3 @@ if [[ $NOP != 1 ]]; then fi PROD=rejson-oss PREFIX=rejson-oss s3_upload - - - - -# #!/bin/bash - -# PROGNAME="${BASH_SOURCE[0]}" -# HERE="$(cd "$(dirname "$PROGNAME")" &>/dev/null && pwd)" -# ROOT=$(cd $HERE/.. && pwd) -# READIES=$ROOT/deps/readies -# . $READIES/shibumi/defs - -# #---------------------------------------------------------------------------------------------- - -# if [[ $1 == --help || $1 == help || $HELP == 1 ]]; then -# cat <<-END -# Upload packages to S3 - -# upload-artifacts [--help|help] artifacts... - -# Argument variables: -# SNAPSHOT=1 Upload snapshots (default) -# OSNICK=nick Operate on packages for given OSNICK (default: host) - -# RELEASE=1 Upload release artifacts -# STAGING=1 Upload into staging area - -# FORCE=1 Allow uploading outside of CI -# NOP=1 No operation -# VERBOSE=1 Show artifacts details -# HELP=1 Show help - -# END -# exit 0 -# fi - -# #---------------------------------------------------------------------------------------------- - -# ARCH=$($READIES/bin/platform --arch) -# [[ $ARCH == x64 ]] && ARCH="x86_64" -# [[ $ARCH == arm64v8 ]] && ARCH="aarch64" - -# OS=$($READIES/bin/platform --os) -# [[ $OS == linux ]] && OS="Linux" - -# [[ -z $OSNICK ]] && OSNICK=$($READIES/bin/platform --osnick) -# [[ $OSNICK == trusty ]] && OSNICK=ubuntu14.04 -# [[ $OSNICK == xenial ]] && OSNICK=ubuntu16.04 -# [[ $OSNICK == bionic ]] && OSNICK=ubuntu18.04 -# [[ $OSNICK == focal ]] && OSNICK=ubuntu20.04 -# [[ $OSNICK == jammy ]] && OSNICK=ubuntu22.04 -# [[ $OSNICK == centos7 ]] && OSNICK=rhel7 -# [[ $OSNICK == centos8 ]] && OSNICK=rhel8 -# [[ $OSNICK == centos9 ]] && OSNICK=rhel9 -# [[ $OSNICK == ol8 ]] && OSNICK=rhel8 -# [[ $OSNICK == rocky8 ]] && OSNICK=rhel8 -# [[ $OSNICK == rocky9 ]] && OSNICK=rhel9 - -# if [[ $OS == macos ]]; then -# # as we don't build on macOS for every platform, we converge to a least common denominator -# # if [[ $ARCH == x86_64 ]]; then -# # OSNICK=catalina # to be aligned with the rest of the modules in redis stack -# # else -# # [[ $OSNICK == ventura ]] && OSNICK=monterey -# # fi -# PLATFORMS=("$OS-catalina-x86_64" "$OS-monterey-aarch64") -# else -# PLATFORMS=("$OS-$OSNICK-$ARCH") -# fi - -# #---------------------------------------------------------------------------------------------- - -# OP="" -# [[ $NOP == 1 ]] && OP=echo - -# if [[ $STAGING == 1 ]]; then -# S3_URL=s3://redismodules/lab/staging -# else -# S3_URL=s3://redismodules -# fi - -# if [[ -z $AWS_ACCESS_KEY_ID || -z $AWS_SECRET_ACCESS_KEY ]]; then -# eprint "No credentials for S3 upload." -# exit 1 -# fi - -# cd $ROOT/bin - -# if [[ $SNAPSHOT == 1 ]]; then -# MAYBE_SNAP=/snapshots -# else -# MAYBE_SNAP= -# fi - -# cd artifacts${MAYBE_SNAP} -# [[ $VERBOSE == 1 ]] && du -ah --apparent-size * - -# #---------------------------------------------------------------------------------------------- - -# s3_upload_file() { -# local file="$1" -# local s3_dir="$2" -# [[ $s3_dir != */ ]] && s3_dir="${s3_dir}/" - -# $OP aws --debug s3 cp $file $s3_dir --acl public-read --no-progress -# } - -# s3_ls() { -# local s3_dir="$1" -# [[ $s3_dir != */ ]] && s3_dir="${s3_dir}/" - -# $OP aws s3 ls $s3_dir -# } - -# s3_upload() { -# local prod_subdir="$PROD" -# local prefix="$PREFIX" -# local upload_dir="${S3_URL}/${prod_subdir}${MAYBE_SNAP}" -# local file -# if [[ $SNAPSHOT == 1 ]]; then -# for platform in ${PLATFORMS[@]}; do -# for file in `ls ${prefix}.*${platform}*.zip ${prefix}.*${platform}*.tgz 2> /dev/null`; do -# s3_upload_file $file $upload_dir -# done -# done -# else -# for file in `ls ${prefix}.*.zip ${prefix}.*.tgz 2> /dev/null`; do -# s3_upload_file $file $upload_dir -# done -# fi -# [[ $VERBOSE == 1 ]] && s3_ls $upload_dir - -# return 0 -# } - -# #---------------------------------------------------------------------------------------------- - -# if [[ $NOP != 1 ]]; then -# if ! is_command aws; then -# $READIES/bin/getaws -# fi -# fi - -# PROD=rejson-oss PREFIX=rejson-oss s3_upload From 1aae0556247888cc5831257d5e48bfcf17a15ceb Mon Sep 17 00:00:00 2001 From: Ephraim Feldblum Date: Tue, 16 Jul 2024 15:10:40 +0300 Subject: [PATCH 068/112] set new flow to run only on master and tag branches --- .github/workflows/macos-build-and-deploy.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/macos-build-and-deploy.yml b/.github/workflows/macos-build-and-deploy.yml index ddf977077..0ee501792 100644 --- a/.github/workflows/macos-build-and-deploy.yml +++ b/.github/workflows/macos-build-and-deploy.yml @@ -2,11 +2,10 @@ name: Build for macos on: push: - workflow_dispatch: - # branches: - # - master - # tags: - # - 'v[0-9]+.[0-9]+.[0-9]+' + branches: + - master + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' jobs: build: From 04551a9a307cba65128ecdc431aebf720d1ff532 Mon Sep 17 00:00:00 2001 From: efeldblum <109016390+ephraimfeldblum@users.noreply.github.com> Date: Wed, 7 Aug 2024 15:15:18 +0300 Subject: [PATCH 069/112] MOD-7279 Fix `SET` commands with overlapping paths (#1225) * not the solution. still brainstorming * some refactoring * more refactoring. rustification * more refactoring. rustification * missed variable renaming * making cargo fmt shut up * seriously, i wrote "if let None ="? * found where i changed logic. reverted changed logic without reverting refactor * some more refactoring * didn't even need try_for_each * "else { false }" grates on me * turbofish soup is disgusting * lazily index * fix the case of merge * fmt * per review * resolve unused using * turbofish soup is disgusting * hate fmt --- Cargo.lock | 1 + json_path/Cargo.toml | 1 + json_path/src/json_path.rs | 5 +- redis_json/src/commands.rs | 162 +++++++++++++------------------ redis_json/src/ivalue_manager.rs | 106 ++++++++------------ redis_json/src/key_value.rs | 153 ++++++++++++----------------- redis_json/src/redisjson.rs | 1 + tests/pytest/test.py | 6 +- 8 files changed, 182 insertions(+), 253 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c0eb2969b..11fc870dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -483,6 +483,7 @@ dependencies = [ "bson", "env_logger", "ijson", + "itertools", "log", "pest", "pest_derive", diff --git a/json_path/Cargo.toml b/json_path/Cargo.toml index b409d46c3..dd7bbdc53 100644 --- a/json_path/Cargo.toml +++ b/json_path/Cargo.toml @@ -14,6 +14,7 @@ serde.workspace = true ijson.workspace = true log = "0.4" regex = "1" +itertools = "0.13" [dev-dependencies] env_logger = "0.11" diff --git a/json_path/src/json_path.rs b/json_path/src/json_path.rs index 05f7dd35b..427e42cbe 100644 --- a/json_path/src/json_path.rs +++ b/json_path/src/json_path.rs @@ -4,6 +4,7 @@ * the Server Side Public License v1 (SSPLv1). */ +use itertools::Itertools; use pest::iterators::{Pair, Pairs}; use pest::Parser; use pest_derive::Parser; @@ -161,7 +162,7 @@ pub(crate) fn compile(path: &str) -> Result { positives .iter() .map(|v| format!("{v}")) - .collect::>() + .collect_vec() .join(", "), ) }; @@ -172,7 +173,7 @@ pub(crate) fn compile(path: &str) -> Result { negatives .iter() .map(|v| format!("{v}")) - .collect::>() + .collect_vec() .join(", "), ) }; diff --git a/redis_json/src/commands.rs b/redis_json/src/commands.rs index 429d5276b..430520182 100644 --- a/redis_json/src/commands.rs +++ b/redis_json/src/commands.rs @@ -7,10 +7,11 @@ use crate::error::Error; use crate::formatter::ReplyFormatOptions; use crate::key_value::KeyValue; -use crate::manager::err_msg_json_path_doesnt_exist_with_param; -use crate::manager::err_msg_json_path_doesnt_exist_with_param_or; -use crate::manager::{Manager, ReadHolder, UpdateInfo, WriteHolder}; -use crate::redisjson::{Format, Path, ReplyFormat}; +use crate::manager::{ + err_msg_json_path_doesnt_exist_with_param, err_msg_json_path_doesnt_exist_with_param_or, + Manager, ReadHolder, UpdateInfo, WriteHolder, +}; +use crate::redisjson::{Format, Path, ReplyFormat, SetOptions}; use json_path::select_value::{SelectValue, SelectValueType}; use redis_module::{Context, RedisValue}; use redis_module::{NextArg, RedisError, RedisResult, RedisString, REDIS_OK}; @@ -19,8 +20,6 @@ use std::str::FromStr; use json_path::{calc_once_with_paths, compile, json_path::UserPathTracker}; -use crate::redisjson::SetOptions; - use serde_json::{Number, Value}; use itertools::FoldWhile::{Continue, Done}; @@ -147,7 +146,7 @@ pub fn json_get(manager: M, ctx: &Context, args: Vec) - let key = manager.open_key_read(ctx, &key)?; let value = match key.get_value()? { - Some(doc) => KeyValue::new(doc).to_json(&mut paths, &format_options)?, + Some(doc) => KeyValue::new(doc).to_json(paths, &format_options)?, None => RedisValue::Null, }; @@ -199,23 +198,18 @@ pub fn json_set(manager: M, ctx: &Context, args: Vec) - Ok(RedisValue::Null) } } else { - let update_info = KeyValue::new(doc).find_paths(path.get_path(), &op)?; - if !update_info.is_empty() { - let updated = apply_updates::(&mut redis_key, val, update_info); - if updated { - redis_key.notify_keyspace_event(ctx, "json.set")?; - manager.apply_changes(ctx); - REDIS_OK - } else { - Ok(RedisValue::Null) - } + let update_info = KeyValue::new(doc).find_paths(path.get_path(), op)?; + if !update_info.is_empty() && apply_updates::(&mut redis_key, val, update_info) { + redis_key.notify_keyspace_event(ctx, "json.set")?; + manager.apply_changes(ctx); + REDIS_OK } else { Ok(RedisValue::Null) } } } (None, SetOptions::AlreadyExists) => Ok(RedisValue::Null), - (None, _) => { + _ => { if path.get_path() == JSON_ROOT_PATH { redis_key.set_value(Vec::new(), val)?; redis_key.notify_keyspace_event(ctx, "json.set")?; @@ -265,7 +259,7 @@ pub fn json_merge(manager: M, ctx: &Context, args: Vec) REDIS_OK } else { let mut update_info = - KeyValue::new(doc).find_paths(path.get_path(), &SetOptions::None)?; + KeyValue::new(doc).find_paths(path.get_path(), SetOptions::MergeExisting)?; if !update_info.is_empty() { let mut res = false; if update_info.len() == 1 { @@ -336,7 +330,7 @@ pub fn json_mset(manager: M, ctx: &Context, args: Vec) let update_info = if path.get_path() == JSON_ROOT_PATH { None } else if let Some(value) = key_value { - Some(KeyValue::new(value).find_paths(path.get_path(), &SetOptions::None)?) + Some(KeyValue::new(value).find_paths(path.get_path(), SetOptions::None)?) } else { return Err(RedisError::Str( "ERR new objects must be created at the root", @@ -377,24 +371,18 @@ fn apply_updates( // If there is only one update info, we can avoid cloning the value if update_info.len() == 1 { match update_info.pop().unwrap() { - UpdateInfo::SUI(sui) => redis_key.set_value(sui.path, value).unwrap_or(false), - UpdateInfo::AUI(aui) => redis_key - .dict_add(aui.path, &aui.key, value) - .unwrap_or(false), + UpdateInfo::SUI(sui) => redis_key.set_value(sui.path, value), + UpdateInfo::AUI(aui) => redis_key.dict_add(aui.path, &aui.key, value), } + .unwrap_or(false) } else { - let mut updated = false; - for ui in update_info { - updated = match ui { - UpdateInfo::SUI(sui) => redis_key - .set_value(sui.path, value.clone()) - .unwrap_or(false), - UpdateInfo::AUI(aui) => redis_key - .dict_add(aui.path, &aui.key, value.clone()) - .unwrap_or(false), - } || updated - } - updated + update_info.into_iter().fold(false, |updated, ui| { + match ui { + UpdateInfo::SUI(sui) => redis_key.set_value(sui.path, value.clone()), + UpdateInfo::AUI(aui) => redis_key.dict_add(aui.path, &aui.key, value.clone()), + } + .unwrap_or(updated) + }) } } @@ -438,11 +426,8 @@ where { values_and_paths .into_iter() - .map(|(v, p)| match f(v) { - true => Some(p), - _ => None, - }) - .collect::>>>() + .map(|(v, p)| f(v).then_some(p)) + .collect() } /// Returns a Vec of Values with `None` for Values that do not match the filter @@ -452,11 +437,8 @@ where { values_and_paths .into_iter() - .map(|(v, _)| match f(v) { - true => Some(v), - _ => None, - }) - .collect::>>() + .map(|(v, _)| f(v).then_some(v)) + .collect() } fn find_all_paths( @@ -496,13 +478,13 @@ where values .into_iter() .map(|n| n.map_or_else(|| none_value.clone(), |t| t.into())) - .collect::>() + .collect() } /// Sort the paths so higher indices precede lower indices on the same array, /// And longer paths precede shorter paths /// And if a path is a sub-path of the other, then only paths with shallower hierarchy (closer to the top-level) remain -fn prepare_paths_for_deletion(paths: &mut Vec>) { +pub fn prepare_paths_for_updating(paths: &mut Vec>) { if paths.len() < 2 { // No need to reorder when there are less than 2 paths return; @@ -542,21 +524,17 @@ fn prepare_paths_for_deletion(paths: &mut Vec>) { }); // Remove paths which are nested by others (on each sub-tree only top most ancestor should be deleted) // (TODO: Add a mode in which the jsonpath selector will already skip nested paths) - let mut string_paths = Vec::new(); - paths.iter().for_each(|v| { - string_paths.push(v.join(",")); - }); + let mut string_paths = paths.iter().map(|v| v.join(",")).collect_vec(); string_paths.sort(); paths.retain(|v| { let path = v.join(","); - let found = string_paths.binary_search(&path).unwrap(); - for p in string_paths.iter().take(found) { - if path.starts_with(p.as_str()) { - return false; - } - } - true + string_paths + .iter() + .skip_while(|p| !path.starts_with(*p)) + .next() + .map(|found| path == *found) + .unwrap_or(false) }); } @@ -580,7 +558,7 @@ pub fn json_del(manager: M, ctx: &Context, args: Vec) - 1 } else { let mut paths = find_paths(path.get_path(), doc, |_| true)?; - prepare_paths_for_deletion(&mut paths); + prepare_paths_for_updating(&mut paths); let mut changed = 0; for p in paths { if redis_key.delete_path(p)? { @@ -676,8 +654,8 @@ where Some(root) => KeyValue::new(root) .get_values(path)? .iter() - .map(|v| (KeyValue::value_name(*v)).into()) - .collect::>() + .map(|v| RedisValue::from(KeyValue::value_name(*v))) + .collect_vec() .into(), None => RedisValue::Null, }; @@ -688,14 +666,11 @@ fn json_type_legacy(redis_key: &M::ReadHolder, path: &str) -> RedisResult where M: Manager, { - let value = redis_key.get_value()?.map_or_else( - || RedisValue::Null, - |doc| { - KeyValue::new(doc) - .get_type(path) - .map_or(RedisValue::Null, |s| s.into()) - }, - ); + let value = redis_key.get_value()?.map_or(RedisValue::Null, |doc| { + KeyValue::new(doc) + .get_type(path) + .map_or(RedisValue::Null, |s| s.into()) + }); Ok(value) } @@ -734,7 +709,7 @@ where op, cmd, )? - .drain(..) + .into_iter() .map(|v| { v.map_or(RedisValue::Null, |v| { if let Some(i) = v.as_i64() { @@ -744,7 +719,7 @@ where } }) }) - .collect::>() + .collect_vec() .into(); Ok(res) } else if path.is_legacy() { @@ -796,21 +771,21 @@ where ) })?; - let mut res = vec![]; let mut need_notify = false; - for p in paths { - res.push(match p { - Some(p) => { + let res = paths + .into_iter() + .map(|p| { + p.map(|p| { need_notify = true; - Some(match op { - NumOp::Incr => redis_key.incr_by(p, number)?, - NumOp::Mult => redis_key.mult_by(p, number)?, - NumOp::Pow => redis_key.pow_by(p, number)?, - }) - } - _ => None, - }); - } + match op { + NumOp::Incr => redis_key.incr_by(p, number), + NumOp::Mult => redis_key.mult_by(p, number), + NumOp::Pow => redis_key.pow_by(p, number), + } + }) + .transpose() + }) + .try_collect()?; if need_notify { redis_key.notify_keyspace_event(ctx, cmd)?; manager.apply_changes(ctx); @@ -1676,9 +1651,7 @@ where let values = find_all_values(path, root, |v| v.get_type() == SelectValueType::Object)?; let mut res: Vec = vec![]; for v in values { - res.push(v.map_or(RedisValue::Null, |v| { - v.keys().unwrap().collect::>().into() - })); + res.push(v.map_or(RedisValue::Null, |v| v.keys().unwrap().collect_vec().into())); } res.into() }; @@ -1695,7 +1668,7 @@ where }; let value = match KeyValue::new(root).get_first(path) { Ok(v) => match v.get_type() { - SelectValueType::Object => v.keys().unwrap().collect::>().into(), + SelectValueType::Object => v.keys().unwrap().collect_vec().into(), _ => { return Err(RedisError::String( err_msg_json_path_doesnt_exist_with_param_or(path, "not an object"), @@ -1737,7 +1710,7 @@ where RedisValue::Integer(v.len().unwrap() as i64) }) }) - .collect::>() + .collect_vec() .into(), None => { return Err(RedisError::String( @@ -1838,7 +1811,7 @@ pub fn json_debug(manager: M, ctx: &Context, args: Vec) .get_values(path.get_path())? .iter() .map(|v| manager.get_memory(v).unwrap()) - .collect::>(), + .collect(), None => vec![], } .into()) @@ -1870,8 +1843,7 @@ pub fn json_resp(manager: M, ctx: &Context, args: Vec) }; let key = manager.open_key_read(ctx, &key)?; - key.get_value()?.map_or_else( - || Ok(RedisValue::Null), - |doc| KeyValue::new(doc).resp_serialize(path), - ) + key.get_value()?.map_or(Ok(RedisValue::Null), |doc| { + KeyValue::new(doc).resp_serialize(path) + }) } diff --git a/redis_json/src/ivalue_manager.rs b/redis_json/src/ivalue_manager.rs index a2026235b..32700b0b4 100644 --- a/redis_json/src/ivalue_manager.rs +++ b/redis_json/src/ivalue_manager.rs @@ -59,7 +59,7 @@ fn replace Result, Error>>( if is_last { if let Entry::Occupied(mut e) = obj.entry(token) { let v = e.get_mut(); - if let Some(res) = (func)(v)? { + if let Some(res) = func(v)? { *v = res; } else { e.remove(); @@ -71,22 +71,22 @@ fn replace Result, Error>>( } ValueType::Array => { let arr = target_once.as_array_mut().unwrap(); - if let Ok(x) = token.parse::() { - if is_last { - if x < arr.len() { - let v = &mut arr.as_mut_slice()[x]; - if let Some(res) = (func)(v)? { - *v = res; - } else { - arr.remove(x); - } + let idx = token.parse::().expect(&format!( + "An array index is parsed successfully. Array = {:?}, index = {:?}", + arr, token + )); + if is_last { + if idx < arr.len() { + let v = &mut arr.as_mut_slice()[idx]; + if let Some(res) = func(v)? { + *v = res; + } else { + arr.remove(idx); } - return Ok(()); } - arr.get_mut(x) - } else { - panic!("Array index should have been parsed successfully before reaching here") + return Ok(()); } + arr.get_mut(idx) } _ => None, }; @@ -125,13 +125,8 @@ fn update Result, Error>>( if is_last { if let Entry::Occupied(mut e) = obj.entry(token) { let v = e.get_mut(); - match (func)(v) { - Ok(res) => { - if res.is_none() { - e.remove(); - } - } - Err(err) => return Err(err), + if func(v)?.is_none() { + e.remove(); } } return Ok(()); @@ -140,25 +135,20 @@ fn update Result, Error>>( } ValueType::Array => { let arr = target_once.as_array_mut().unwrap(); - if let Ok(x) = token.parse::() { - if is_last { - if x < arr.len() { - let v = &mut arr.as_mut_slice()[x]; - match (func)(v) { - Ok(res) => { - if res.is_none() { - arr.remove(x); - } - } - Err(err) => return Err(err), - } + let idx = token.parse::().expect(&format!( + "An array index is parsed successfully. Array = {:?}, index = {:?}", + arr, token + )); + if is_last { + if idx < arr.len() { + let v = &mut arr.as_mut_slice()[idx]; + if func(v)?.is_none() { + arr.remove(idx); } - return Ok(()); } - arr.get_mut(x) - } else { - panic!("Array index should have been parsed successfully before reaching here") + return Ok(()); } + arr.get_mut(idx) } _ => None, }; @@ -181,16 +171,11 @@ impl<'a> IValueKeyHolderWrite<'a> { if paths.is_empty() { // updating the root require special treatment let root = self.get_value().unwrap().unwrap(); - let res = (op_fun)(root); - match res { - Ok(res) => { - if res.is_none() { - root.take(); - } - } - Err(err) => { - return Err(RedisError::String(err.msg)); - } + if op_fun(root) + .map_err(|err| RedisError::String(err.msg))? + .is_none() + { + root.take(); } } else { update(paths, self.get_value().unwrap().unwrap(), op_fun)?; @@ -633,24 +618,17 @@ impl<'a> Manager for RedisIValueJsonKeyManager<'a> { .map_or_else( |e| Err(e.to_string().into()), |docs: Document| { - let v = if docs.is_empty() { - IValue::NULL - } else { - docs.iter().next().map_or_else( - || IValue::NULL, - |(_, b)| { - let v: serde_json::Value = b.clone().into(); - let mut out = serde_json::Serializer::new(Vec::new()); - v.serialize(&mut out).unwrap(); - self.from_str( - &String::from_utf8(out.into_inner()).unwrap(), - Format::JSON, - limit_depth, - ) - .unwrap() - }, + let v = docs.iter().next().map_or(IValue::NULL, |(_, b)| { + let v: serde_json::Value = b.clone().into(); + let mut out = serde_json::Serializer::new(Vec::new()); + v.serialize(&mut out).unwrap(); + self.from_str( + &String::from_utf8(out.into_inner()).unwrap(), + Format::JSON, + limit_depth, ) - }; + .unwrap() + }); Ok(v) }, ), diff --git a/redis_json/src/key_value.rs b/redis_json/src/key_value.rs index 4c05ac434..dc4d49176 100644 --- a/redis_json/src/key_value.rs +++ b/redis_json/src/key_value.rs @@ -1,3 +1,4 @@ +use itertools::Itertools; use std::collections::HashMap; use json_path::{ @@ -10,7 +11,7 @@ use serde::Serialize; use serde_json::Value; use crate::{ - commands::{FoundIndex, ObjectLen, Values}, + commands::{prepare_paths_for_updating, FoundIndex, ObjectLen, Values}, error::Error, formatter::{RedisJsonFormatter, ReplyFormatOptions}, manager::{ @@ -46,9 +47,9 @@ impl<'a, V: SelectValue + 'a> KeyValue<'a, V> { } else { Ok(self .get_values(path.get_path())? - .iter() + .into_iter() .map(|v| Self::resp_serialize_inner(v)) - .collect::>() + .collect_vec() .into()) } } @@ -112,7 +113,7 @@ impl<'a, V: SelectValue + 'a> KeyValue<'a, V> { fn to_json_multi( &self, - paths: &mut Vec, + paths: Vec, format: &ReplyFormatOptions, is_legacy: bool, ) -> Result { @@ -124,32 +125,23 @@ impl<'a, V: SelectValue + 'a> KeyValue<'a, V> { let path_len = paths.len(); let temp_doc = paths - .drain(..) + .into_iter() .fold(HashMap::with_capacity(path_len), |mut acc, path: Path| { - let query = compile(path.get_path()); - // If we can't compile the path, we can't continue - if query.is_err() { - return acc; - } - - let query = query.unwrap(); - let results = calc_once(query, self.val); + if let Ok(query) = compile(path.get_path()) { + let results = calc_once(query, self.val); - let value = if is_legacy { - if results.is_empty() { - None + let value = if is_legacy { + (!results.is_empty()).then(|| Values::Single(results[0])) } else { - Some(Values::Single(results[0])) - } - } else { - Some(Values::Multi(results)) - }; + Some(Values::Multi(results)) + }; - if value.is_none() && missing_path.is_none() { - missing_path = Some(path.get_original().to_string()); + if value.is_none() && missing_path.is_none() { + missing_path = Some(path.get_original().to_string()); + } + acc.insert(path.get_original(), value); } - acc.insert(path.get_original(), value); acc }); if let Some(p) = missing_path { @@ -159,17 +151,17 @@ impl<'a, V: SelectValue + 'a> KeyValue<'a, V> { // If we're using RESP3, we need to convert the HashMap to a RedisValue::Map unless we're using the legacy format let res = if format.is_resp3_reply() { let map = temp_doc - .iter() + .into_iter() .map(|(k, v)| { let key = RedisValueKey::String(k.to_string()); let value = match v { Some(Values::Single(value)) => Self::value_to_resp3(value, format), - Some(Values::Multi(values)) => Self::values_to_resp3(values, format), + Some(Values::Multi(values)) => Self::values_to_resp3(&values, format), None => RedisValue::Null, }; (key, value) }) - .collect::>(); + .collect(); RedisValue::Map(map) } else { Self::serialize_object(&temp_doc, format).into() @@ -177,24 +169,18 @@ impl<'a, V: SelectValue + 'a> KeyValue<'a, V> { Ok(res) } - fn to_resp3( - &self, - paths: &mut Vec, - format: &ReplyFormatOptions, - ) -> Result { + fn to_resp3(&self, paths: Vec, format: &ReplyFormatOptions) -> Result { let results = paths - .drain(..) + .into_iter() .map(|path: Path| self.to_resp3_path(&path, format)) - .collect::>(); - + .collect(); Ok(RedisValue::Array(results)) } pub fn to_resp3_path(&self, path: &Path, format: &ReplyFormatOptions) -> RedisValue { - compile(path.get_path()).map_or_else( - |_| RedisValue::Array(vec![]), - |q| Self::values_to_resp3(&calc_once(q, self.val), format), - ) + compile(path.get_path()).map_or(RedisValue::Array(vec![]), |q| { + Self::values_to_resp3(&calc_once(q, self.val), format) + }) } fn to_json_single( @@ -218,7 +204,7 @@ impl<'a, V: SelectValue + 'a> KeyValue<'a, V> { values .iter() .map(|v| Self::value_to_resp3(v, format)) - .collect::>() + .collect_vec() .into() } @@ -235,7 +221,7 @@ impl<'a, V: SelectValue + 'a> KeyValue<'a, V> { .values() .unwrap() .map(|v| Self::value_to_resp3(v, format)) - .collect::>(), + .collect(), ), SelectValueType::Object => RedisValue::Map( value @@ -247,7 +233,7 @@ impl<'a, V: SelectValue + 'a> KeyValue<'a, V> { Self::value_to_resp3(v, format), ) }) - .collect::>(), + .collect(), ), } } else { @@ -263,7 +249,7 @@ impl<'a, V: SelectValue + 'a> KeyValue<'a, V> { pub fn to_json( &self, - paths: &mut Vec, + paths: Vec, format: &ReplyFormatOptions, ) -> Result { let is_legacy = !paths.iter().any(|p| !p.is_legacy()); @@ -329,15 +315,13 @@ impl<'a, V: SelectValue + 'a> KeyValue<'a, V> { } } - pub fn find_paths( - &mut self, - path: &str, - option: &SetOptions, - ) -> Result, Error> { - if SetOptions::NotExists != *option { + pub fn find_paths(&mut self, path: &str, option: SetOptions) -> Result, Error> { + if option != SetOptions::NotExists { let query = compile(path)?; - let res = calc_once_paths(query, self.val); - + let mut res = calc_once_paths(query, self.val); + if option != SetOptions::MergeExisting { + prepare_paths_for_updating(&mut res); + } if !res.is_empty() { return Ok(res .into_iter() @@ -345,7 +329,7 @@ impl<'a, V: SelectValue + 'a> KeyValue<'a, V> { .collect()); } } - if SetOptions::AlreadyExists == *option { + if option == SetOptions::AlreadyExists { Ok(Vec::new()) // empty vector means no updates } else { self.find_add_paths(path) @@ -422,45 +406,30 @@ impl<'a, V: SelectValue + 'a> KeyValue<'a, V> { } pub fn is_equal(a: &T1, b: &T2) -> bool { - match (a.get_type(), b.get_type()) { - (SelectValueType::Null, SelectValueType::Null) => true, - (SelectValueType::Bool, SelectValueType::Bool) => a.get_bool() == b.get_bool(), - (SelectValueType::Long, SelectValueType::Long) => a.get_long() == b.get_long(), - (SelectValueType::Double, SelectValueType::Double) => a.get_double() == b.get_double(), - (SelectValueType::String, SelectValueType::String) => a.get_str() == b.get_str(), - (SelectValueType::Array, SelectValueType::Array) => { - if a.len().unwrap() == b.len().unwrap() { - for (i, e) in a.values().unwrap().enumerate() { - if !Self::is_equal(e, b.get_index(i).unwrap()) { - return false; - } - } - true - } else { - false + a.get_type() == b.get_type() + && match a.get_type() { + SelectValueType::Null => true, + SelectValueType::Bool => a.get_bool() == b.get_bool(), + SelectValueType::Long => a.get_long() == b.get_long(), + SelectValueType::Double => a.get_double() == b.get_double(), + SelectValueType::String => a.get_str() == b.get_str(), + SelectValueType::Array => { + a.len().unwrap() == b.len().unwrap() + && a.values() + .unwrap() + .zip(b.values().unwrap()) + .all(|(a, b)| Self::is_equal(a, b)) } - } - (SelectValueType::Object, SelectValueType::Object) => { - if a.len().unwrap() == b.len().unwrap() { - for k in a.keys().unwrap() { - let temp1 = a.get_key(k); - let temp2 = b.get_key(k); - match (temp1, temp2) { - (Some(a1), Some(b1)) => { - if !Self::is_equal(a1, b1) { - return false; - } - } - (_, _) => return false, - } - } - true - } else { - false + SelectValueType::Object => { + a.len().unwrap() == b.len().unwrap() + && a.keys() + .unwrap() + .all(|k| match (a.get_key(k), b.get_key(k)) { + (Some(a), Some(b)) => Self::is_equal(a, b), + _ => false, + }) } } - (_, _) => false, - } } pub fn arr_index( @@ -472,9 +441,11 @@ impl<'a, V: SelectValue + 'a> KeyValue<'a, V> { ) -> Result { let res = self .get_values(path)? - .iter() - .map(|value| Self::arr_first_index_single(value, &json_value, start, end).into()) - .collect::>(); + .into_iter() + .map(|value| { + RedisValue::from(Self::arr_first_index_single(value, &json_value, start, end)) + }) + .collect_vec(); Ok(res.into()) } diff --git a/redis_json/src/redisjson.rs b/redis_json/src/redisjson.rs index a8c56c9c1..1c1263c8b 100644 --- a/redis_json/src/redisjson.rs +++ b/redis_json/src/redisjson.rs @@ -53,6 +53,7 @@ pub fn normalize_arr_indices(start: i64, end: i64, len: i64) -> (i64, i64) { pub enum SetOptions { NotExists, AlreadyExists, + MergeExisting, None, } diff --git a/tests/pytest/test.py b/tests/pytest/test.py index 0b18b2df4..515fcd06a 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -1528,7 +1528,11 @@ def test_mset_replication_in_aof(env): assert(len(aof_content) == 1) - +def test_recursive_descent(env): + r = env + r.expect('JSON.SET', 'k', '$', '[{"a":1}]').ok() + r.expect('JSON.SET', 'k', '$..*', '[{"a":1}]').ok() + r.expect('JSON.GET', 'k', '$').equal('[[[{"a":1}]]]') # class CacheTestCase(BaseReJSONTest): # @property From bfb657b52799ab1e7b5e73c658eb54afc911ac7f Mon Sep 17 00:00:00 2001 From: Shockingly Good Date: Wed, 21 Aug 2024 13:22:41 +0200 Subject: [PATCH 070/112] Fix the CI issues. (#1231) Fix the CI by updating the macos version. --- .circleci/config.yml | 7 ++++--- deps/readies | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba3c2afb3..8a869af3b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -76,6 +76,7 @@ commands: echo "python3: $(command -v python3)" python3 --version python3 -m pip list + python3 -m pip install --upgrade setuptools six pip install-prerequisites: parameters: @@ -321,7 +322,7 @@ jobs: build-macos-m1: macos: - xcode: 14.2.0 + xcode: 15.4.0 resource_class: macos.m1.large.gen1 parameters: upload: @@ -544,7 +545,7 @@ workflows: requires: - upload-release-artifacts - + nightly: triggers: - schedule: @@ -555,7 +556,7 @@ workflows: name: build-with-redis-<> matrix: parameters: - redis_version: ["7", "unstable"] + redis_version: ["7", "unstable"] nightly-twice-a-week-by-param: when: diff --git a/deps/readies b/deps/readies index 7fc8e62b2..580e221a0 160000 --- a/deps/readies +++ b/deps/readies @@ -1 +1 @@ -Subproject commit 7fc8e62b24d3f7ff01096d83f14cbc216ac0e2f0 +Subproject commit 580e221a0a7ce686433090254e377046007c1d78 From 8d8471d5ac16335a48ebf21553cbc5ce494b7fd1 Mon Sep 17 00:00:00 2001 From: Shockingly Good Date: Mon, 26 Aug 2024 11:26:54 +0200 Subject: [PATCH 071/112] Add alpine CI workflow. (#1230) --- .github/workflows/alpine.yml | 52 ++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/alpine.yml diff --git a/.github/workflows/alpine.yml b/.github/workflows/alpine.yml new file mode 100644 index 000000000..31868e4c2 --- /dev/null +++ b/.github/workflows/alpine.yml @@ -0,0 +1,52 @@ +name: alpine + +on: + push: + branches: + - master + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + +jobs: + build: + runs-on: ubuntu-latest + defaults: + run: + shell: bash + container: + image: alpine:3 + steps: + - name: Install prerequisites + shell: sh + run: | + apk add bash make tar cargo python3 python3-dev py3-pip gcc git curl build-base autoconf automake py3-cryptography linux-headers musl-dev libffi-dev openssl-dev openssh py-virtualenv clang18-libclang + - name: Checkout the module + uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Install python dependencies + run: | + pip install -q --upgrade setuptools + pip install -q --upgrade pip + pip install -q -r tests/pytest/requirements.txt + pip install -q addict toml jinja2 ramp-packer + env: + PIP_BREAK_SYSTEM_PACKAGES: 1 + - name: Checkout Redis + uses: actions/checkout@v4 + with: + repository: redis/redis + ref: '7.2.1' + path: redis + - name: Build Redis + working-directory: redis + run: make install + - name: Build module + run: | + make build + - name: Test + run: | + make test + - name: Pack module + run: | + make pack BRANCH=${{ github.ref_name }} From cd75d781d79493a19c0ef9e9134f5b17c7ba82d9 Mon Sep 17 00:00:00 2001 From: Shockingly Good Date: Mon, 26 Aug 2024 18:55:32 +0200 Subject: [PATCH 072/112] Move ubuntu docker builds from CircleCI to GHA. (#1234) * Move ubuntu docker builds from CircleCI to GHA. * Try building Redis in parallel and use newer AWS creds * Restrict the ubuntu docker CI only for merges/tags. --- .circleci/config.yml | 32 +++++------ .github/workflows/ubuntu.yml | 102 +++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/ubuntu.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 8a869af3b..46e76c6df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -309,16 +309,16 @@ jobs: - vm-build-platforms-steps: platform: <> - build-arm-platforms: - parameters: - platform: - type: string - machine: - image: ubuntu-2004:202101-01 - resource_class: arm.medium - steps: - - vm-build-platforms-steps: - platform: <> + # build-arm-platforms: + # parameters: + # platform: + # type: string + # machine: + # image: ubuntu-2004:202101-01 + # resource_class: arm.medium + # steps: + # - vm-build-platforms-steps: + # platform: <> build-macos-m1: macos: @@ -504,13 +504,7 @@ workflows: context: common matrix: parameters: - platform: [jammy, focal, bionic, centos7, rocky8, rocky9, bullseye, amzn2] - - build-arm-platforms: - <<: *on-integ-and-version-tags - context: common - matrix: - parameters: - platform: [jammy, focal, bionic] + platform: [centos7, rocky8, rocky9, bullseye, amzn2] - build-macos-m1: context: common <<: *on-integ-and-version-tags @@ -529,7 +523,7 @@ workflows: context: common requires: - build-platforms - - build-arm-platforms + # - build-arm-platforms - build-macos-m1 - upload-artifacts: name: upload-release-artifacts @@ -537,7 +531,7 @@ workflows: context: common requires: - build-platforms - - build-arm-platforms + # - build-arm-platforms - build-macos-m1 - release-qa-tests: <<: *on-version-tags diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml new file mode 100644 index 000000000..a9e60d6e9 --- /dev/null +++ b/.github/workflows/ubuntu.yml @@ -0,0 +1,102 @@ +name: Build for Ubuntu + +on: + push: + branches: + - master + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + paths-ignore: + - '.circleci/**' + - 'docs/**' + - '*.md' + +# https://github.com/actions/checkout/issues/1487 +# A workaround for the old runtime is to use these environment +# variables, install the latest version of git within the container, +# and use the actions/checkout@v3 instead of @v4, until it is fixed. +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16 + ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 + DEBIAN_FRONTEND: noninteractive + +jobs: + build: + strategy: + matrix: + docker_image: ["ubuntu:bionic", "ubuntu:focal", "ubuntu:jammy"] + runs-on: "ubuntu-latest" + container: + image: ${{ matrix.docker_image }} + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - name: Update git + run: | + apt update + apt install -y software-properties-common + add-apt-repository ppa:git-core/ppa -y + apt update + apt install -y git + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: 'recursive' + - name: Install python 3.12 + run: | + apt install -y wget build-essential libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev + wget https://www.python.org/ftp/python/3.12.2/Python-3.12.2.tgz + tar xzf Python-3.12.2.tgz + cd Python-3.12.2 + ./configure --enable-optimizations + make -j + make altinstall + python3 --version + python3.12 --version + pip3.12 --version + update-alternatives --install /usr/bin/python3 python3 `which python3.12` 1 + python3 --version + - name: Install prerequisites + run: | + apt install -y make build-essential pkg-config python3-pip libclang-dev clang + ./deps/readies/bin/getaws + ./deps/readies/bin/getrust + python3 -m pip list + python3 -m pip install --upgrade setuptools six pip addict toml jinja2 ramp-packer + python3 -m pip install -r tests/pytest/requirements.txt + - name: Get Redis + uses: actions/checkout@v3 + with: + repository: redis/redis + ref: '7.2.1' + path: redis + - name: Build Redis + working-directory: redis + run: | + make -j + make install + - name: Build module + run: | + make build OSNICK=jammy VERSION=${{ github.ref_name }} TEST=1 OFFICIAL=1 SHOW=1 + - name: Test + run: | + make test + - name: Pack module + run: | + make pack BRANCH=${{ github.ref_name }} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: "us-east-1" + - name: Upload artifacts to S3 - staging + run: | + make upload-artifacts SHOW=1 VERBOSE=1 + make upload-release SHOW=1 STAGING=1 VERBOSE=1 + - name: Upload artifacts to S3 - release # todo: trigger this manually instead + if: ${{ github.ref != 'refs/heads/master' }} + run: make upload-release SHOW=1 VERBOSE=1 + From 89b4dad11f66edfc72fc19bc69437b334f18dc64 Mon Sep 17 00:00:00 2001 From: Shockingly Good Date: Tue, 27 Aug 2024 15:46:01 +0200 Subject: [PATCH 073/112] Remove the release-qa-tests (#1243) The opereto is long gone and is not used by all other projects, not to mention it has been inaccessible for a long time and only prevents from following the release pipeline. --- .circleci/config.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 46e76c6df..ae1e842c4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -401,18 +401,6 @@ jobs: make upload-artifacts SHOW=1 fi - release-qa-tests: - docker: - - image: redisfab/rmbuilder:6.2.7-x64-bullseye - steps: - - early-returns - - early-return-for-forked-pull-requests - - checkout - - setup-automation - - run: - name: Run QA Automation - command: ./tests/qa/qatests -m "$CIRCLE_TAG" - #---------------------------------------------------------------------------------------------------------------------------------- on-any-branch: &on-any-branch @@ -533,12 +521,6 @@ workflows: - build-platforms # - build-arm-platforms - build-macos-m1 - - release-qa-tests: - <<: *on-version-tags - context: common - requires: - - upload-release-artifacts - nightly: triggers: From 997c43fc749bb346d64bdd0b1fb7ce3496a2ca93 Mon Sep 17 00:00:00 2001 From: Shockingly Good Date: Wed, 28 Aug 2024 11:13:39 +0200 Subject: [PATCH 074/112] Update the machine for CircleCI. (#1249) The previously used one has been removed from CircleCI. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ae1e842c4..e05bd01bc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -303,7 +303,7 @@ jobs: # - image: debian:bullseye machine: enabled: true - image: ubuntu-2004:202010-01 + image: ubuntu-2204:edge resource_class: large steps: - vm-build-platforms-steps: From b7ae77b30d8f0aa3d4293ad00f6eb147f9af0ac3 Mon Sep 17 00:00:00 2001 From: Lior Kogan Date: Wed, 28 Aug 2024 14:36:39 +0300 Subject: [PATCH 075/112] Update SECURITY.md --- SECURITY.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index ec125ed77..b0c5ab6f6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,9 +11,9 @@ unless this is not possible or feasible with a reasonable effort. | Version | Supported | | ------- | ------------------ | +| 2.8 | :white_check_mark: | | 2.6 | :white_check_mark: | -| 2.4 | :white_check_mark: | -| < 2.4 | :x: | +| < 2.6 | :x: | ## Reporting a Vulnerability From 460662bc5f4eeb564c1c4bfc06b205561e62b992 Mon Sep 17 00:00:00 2001 From: Shockingly Good Date: Thu, 29 Aug 2024 14:06:58 +0200 Subject: [PATCH 076/112] Always use the cargo package version as the module version. (#1254) Changes the way the module version is specified to always get the crate version from the Cargo.toml. This greatly reduces the chances of accidentally forgetting to update the module version. --- Cargo.lock | 273 ++++++++++++++++++++++-------------------- redis_json/src/lib.rs | 30 ++++- 2 files changed, 173 insertions(+), 130 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 11fc870dc..71bfa8ea8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,9 +41,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.14" +version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" dependencies = [ "anstyle", "anstyle-parse", @@ -56,33 +56,33 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" [[package]] name = "anstyle-parse" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" dependencies = [ "windows-sys", ] [[package]] name = "anstyle-wincon" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" dependencies = [ "anstyle", "windows-sys", @@ -130,11 +130,11 @@ dependencies = [ "peeking_take_while", "prettyplease", "proc-macro2", - "quote 1.0.36", + "quote 1.0.37", "regex", "rustc-hash", "shlex", - "syn 2.0.68", + "syn 2.0.76", "which", ] @@ -198,11 +198,20 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "cc" -version = "1.0.101" +version = "1.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac367972e516d45567c7eafc73d24e1c193dcf200a8d94e9db7b3d38b349572d" +checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" +dependencies = [ + "shlex", +] [[package]] name = "cexpr" @@ -232,15 +241,15 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" dependencies = [ "libc", ] @@ -306,9 +315,9 @@ dependencies = [ [[package]] name = "env_filter" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" dependencies = [ "log", "regex", @@ -316,9 +325,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.3" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" +checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" dependencies = [ "anstream", "anstyle", @@ -438,9 +447,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -448,9 +457,9 @@ dependencies = [ [[package]] name = "is_terminal_polyfill" -version = "1.70.0" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" @@ -469,9 +478,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] @@ -506,15 +515,15 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.155" +version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" [[package]] name = "libloading" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", "windows-targets", @@ -522,22 +531,22 @@ dependencies = [ [[package]] name = "linkme" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccb76662d78edc9f9bf56360d6919bdacc8b7761227727e5082f128eeb90bbf5" +checksum = "3c943daedff228392b791b33bba32e75737756e80a613e32e246c6ce9cbab20a" dependencies = [ "linkme-impl", ] [[package]] name = "linkme-impl" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dccda732e04fa3baf2e17cf835bfe2601c7c2edafd64417c627dabae3a8cda" +checksum = "cb26336e6dc7cc76e7927d2c9e7e3bb376d7af65a6f56a0b16c47d18a9b1abc5" dependencies = [ "proc-macro2", - "quote 1.0.36", - "syn 2.0.68", + "quote 1.0.37", + "syn 2.0.76", ] [[package]] @@ -558,9 +567,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "memchr" @@ -641,9 +650,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.0" +version = "0.36.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" +checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" dependencies = [ "memchr", ] @@ -675,9 +684,9 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "pest" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" +checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" dependencies = [ "memchr", "thiserror", @@ -686,9 +695,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459" +checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" dependencies = [ "pest", "pest_generator", @@ -696,22 +705,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" +checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" dependencies = [ "pest", "pest_meta", "proc-macro2", - "quote 1.0.36", - "syn 2.0.68", + "quote 1.0.37", + "syn 2.0.76", ] [[package]] name = "pest_meta" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" +checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" dependencies = [ "once_cell", "pest", @@ -732,18 +741,21 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "prettyplease" -version = "0.2.20" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" dependencies = [ "proc-macro2", - "syn 2.0.68", + "syn 2.0.76", ] [[package]] @@ -763,9 +775,9 @@ checksum = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -836,7 +848,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9c92438cbeaaba0f99e2944ceeb2e34fc8fa7955576296570575df046b81197" dependencies = [ "proc-macro2", - "quote 1.0.36", + "quote 1.0.37", "serde", "serde_syn", "syn 1.0.109", @@ -850,7 +862,7 @@ checksum = "bd1ba5724bf8bfd0c9d6f1169e45255957175b7d81944b325909024a159bc74c" dependencies = [ "lazy_static", "proc-macro2", - "quote 1.0.36", + "quote 1.0.37", "syn 1.0.109", ] @@ -874,18 +886,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "regex" -version = "1.10.5" +version = "1.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" dependencies = [ "aho-corasick", "memchr", @@ -924,9 +936,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "a85d50532239da68e9addb745ba38ff4612a242c1c7ceea689c4bc7c2f43c36f" dependencies = [ "bitflags 2.6.0", "errno", @@ -955,9 +967,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.203" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" dependencies = [ "serde_derive", ] @@ -973,23 +985,24 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" dependencies = [ "proc-macro2", - "quote 1.0.36", - "syn 2.0.68", + "quote 1.0.37", + "syn 2.0.76", ] [[package]] name = "serde_json" -version = "1.0.118" +version = "1.0.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4" +checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" dependencies = [ "indexmap", "itoa", + "memchr", "ryu", "serde", ] @@ -1037,7 +1050,7 @@ checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ "heck", "proc-macro2", - "quote 1.0.36", + "quote 1.0.37", "rustversion", "syn 1.0.109", ] @@ -1060,18 +1073,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", - "quote 1.0.36", + "quote 1.0.37", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.68" +version = "2.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" dependencies = [ "proc-macro2", - "quote 1.0.36", + "quote 1.0.37", "unicode-ident", ] @@ -1092,22 +1105,22 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "thiserror" -version = "1.0.61" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.61" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", - "quote 1.0.36", - "syn 2.0.68", + "quote 1.0.37", + "syn 2.0.76", ] [[package]] @@ -1173,9 +1186,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.9.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ "getrandom", "serde", @@ -1183,9 +1196,9 @@ dependencies = [ [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wasi" @@ -1195,57 +1208,58 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", - "quote 1.0.36", - "syn 2.0.68", + "quote 1.0.37", + "syn 2.0.76", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ - "quote 1.0.36", + "quote 1.0.37", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", - "quote 1.0.36", - "syn 2.0.68", + "quote 1.0.37", + "syn 2.0.76", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "which" @@ -1270,9 +1284,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -1286,51 +1300,51 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "wyz" @@ -1343,20 +1357,21 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", - "quote 1.0.36", - "syn 2.0.68", + "quote 1.0.37", + "syn 2.0.76", ] diff --git a/redis_json/src/lib.rs b/redis_json/src/lib.rs index 1570e0537..c240a74c3 100644 --- a/redis_json/src/lib.rs +++ b/redis_json/src/lib.rs @@ -236,12 +236,40 @@ const fn dummy_init(_ctx: &Context, _args: &[RedisString]) -> Status { #[cfg(not(feature = "as-library"))] const fn dummy_info(_ctx: &InfoContext, _for_crash_report: bool) {} +const fn version() -> i32 { + let string = env!("CARGO_PKG_VERSION"); + let mut bytes = string.as_bytes(); + let mut value: i32 = 0; + let mut result = 0; + let mut multiplier = 10000; + + while let [byte, rest @ ..] = bytes { + bytes = rest; + match byte { + b'0'..=b'9' => { + value = value * 10 + (*byte - b'0') as i32; + } + b'.' => { + result += value * multiplier; + multiplier /= 100; + value = 0; + } + _ => { + // The provided string is not a valid version specification. + unreachable!() + } + } + } + + result + value +} + #[cfg(not(feature = "as-library"))] redis_json_module_create! { data_types: [REDIS_JSON_TYPE], pre_command_function: pre_command, get_manage: Some(ivalue_manager::RedisIValueJsonKeyManager{phantom:PhantomData}), - version: 99_99_99, + version: version(), init: dummy_init, info: dummy_info, } From dda8ad6d6de43e033c6f9847d7100c6952a085f9 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Tue, 3 Sep 2024 12:45:37 +0300 Subject: [PATCH 077/112] Pin ramp-packer. to 2.5.10 (#1259) --- .github/workflows/alpine.yml | 2 +- .github/workflows/ubuntu.yml | 3 ++- .install/build_package_requirements.txt | 4 ++++ .install/common_installations.sh | 2 +- .install/mariner2.sh | 2 +- deps/readies | 2 +- 6 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 .install/build_package_requirements.txt diff --git a/.github/workflows/alpine.yml b/.github/workflows/alpine.yml index 31868e4c2..1bc681970 100644 --- a/.github/workflows/alpine.yml +++ b/.github/workflows/alpine.yml @@ -29,7 +29,7 @@ jobs: pip install -q --upgrade setuptools pip install -q --upgrade pip pip install -q -r tests/pytest/requirements.txt - pip install -q addict toml jinja2 ramp-packer + pip install -q -r .install/build_package_requirements.txt env: PIP_BREAK_SYSTEM_PACKAGES: 1 - name: Checkout Redis diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index a9e60d6e9..41690ccae 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -64,7 +64,8 @@ jobs: ./deps/readies/bin/getaws ./deps/readies/bin/getrust python3 -m pip list - python3 -m pip install --upgrade setuptools six pip addict toml jinja2 ramp-packer + python3 -m pip install --upgrade setuptools six pip + pip install -q -r .install/build_package_requirements.txt python3 -m pip install -r tests/pytest/requirements.txt - name: Get Redis uses: actions/checkout@v3 diff --git a/.install/build_package_requirements.txt b/.install/build_package_requirements.txt new file mode 100644 index 000000000..464a0f4e5 --- /dev/null +++ b/.install/build_package_requirements.txt @@ -0,0 +1,4 @@ +addict +toml +jinja2 +ramp-packer==2.5.10 \ No newline at end of file diff --git a/.install/common_installations.sh b/.install/common_installations.sh index 821e25269..1174bbff6 100755 --- a/.install/common_installations.sh +++ b/.install/common_installations.sh @@ -24,7 +24,7 @@ echo "pip path: $(which pip)" pip install -q -r tests/pytest/requirements.txt # These packages are needed to build the package -pip install -q addict toml jinja2 ramp-packer +pip install -q -r .install/build_package_requirements.txt # List installed packages pip list diff --git a/.install/mariner2.sh b/.install/mariner2.sh index d31239876..8936d807f 100644 --- a/.install/mariner2.sh +++ b/.install/mariner2.sh @@ -8,7 +8,7 @@ pip install -q --upgrade pip pip install -q -r tests/pytest/requirements.txt # These packages are needed to build the package -pip install -q addict toml jinja2 ramp-packer +pip install -q -r .install/build_package_requirements.txt # Install aws-cli for uploading artifacts to s3 curdir="$PWD" diff --git a/deps/readies b/deps/readies index 580e221a0..7fc8e62b2 160000 --- a/deps/readies +++ b/deps/readies @@ -1 +1 @@ -Subproject commit 580e221a0a7ce686433090254e377046007c1d78 +Subproject commit 7fc8e62b24d3f7ff01096d83f14cbc216ac0e2f0 From 1e826223133ba675b5b91ea73b63b7b0218014a8 Mon Sep 17 00:00:00 2001 From: efeldblum <109016390+ephraimfeldblum@users.noreply.github.com> Date: Mon, 9 Sep 2024 11:33:21 +0300 Subject: [PATCH 078/112] reduce complexity (#1264) --- redis_json/src/ivalue_manager.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/redis_json/src/ivalue_manager.rs b/redis_json/src/ivalue_manager.rs index 32700b0b4..2bb97e21b 100644 --- a/redis_json/src/ivalue_manager.rs +++ b/redis_json/src/ivalue_manager.rs @@ -432,13 +432,10 @@ impl<'a> WriteHolder for IValueKeyHolderWrite<'a> { if !(0..=len).contains(&index) { return Err("ERR index out of bounds".into()); } - let mut index = index as usize; + let index = index as usize; let curr = v.as_array_mut().unwrap(); - curr.reserve(args.len()); - for a in args { - curr.insert(index, a.clone()); - index += 1; - } + curr.extend(args.iter().cloned()); + curr[index..].rotate_right(args.len()); res = Some(curr.len()); Ok(Some(())) })?; From 6539e1ee34fcb79a1ed894f9a96da8ccc217d2a2 Mon Sep 17 00:00:00 2001 From: "Meir Shpilraien (Spielrein)" Date: Tue, 10 Sep 2024 14:28:37 +0300 Subject: [PATCH 079/112] RED-134847 support for active defrag for RedisJSON. (#1262) The PR adds support for active defrag on RedisJSON. A pre condition for this PR is that the following PR's will be megred: * [Redis defrad module API extentions](https://github.com/redis/redis/pull/13509) * [redismodule-rs support for active defrag API](https://github.com/RedisLabsModules/redismodule-rs/pull/387) * [IJSON support for defrag](https://github.com/RedisJSON/ijson/pull/1) The PR register defrag function on the json datatype and uses the new capability of ISON to defrag the key. **Notice**: * Increamental defrag of the json key is **not** support. In order to support it we need to implement the free_effort callback. This is not trivial and even if it was it would have cause the json object to potentially be freed when the GIL is not hold, which violate the assumption of our shared string implementation (it is not thread safe). We leave it for future improvment. * If we run on a Redis version that do not support the defrag start callback, we can still partially support defrag. In that case the IJSON object will be defraged but the shared strings dictionatrywill not be reinitialize. This basically means that shared strings will not be defraged. Tests were added to cover the new functionality. --- Cargo.lock | 18 +++-- Cargo.toml | 2 +- redis_json/Cargo.toml | 5 +- redis_json/src/commands.rs | 2 + redis_json/src/defrag.rs | 106 ++++++++++++++++++++++++++++ redis_json/src/lib.rs | 3 +- tests/pytest/test_defrag.py | 133 ++++++++++++++++++++++++++++++++++++ 7 files changed, 255 insertions(+), 14 deletions(-) create mode 100644 redis_json/src/defrag.rs create mode 100644 tests/pytest/test_defrag.py diff --git a/Cargo.lock b/Cargo.lock index 71bfa8ea8..c10bb0623 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -436,7 +436,7 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "ijson" version = "0.1.3" -source = "git+https://github.com/RedisJSON/ijson?rev=e0119ac74f6c4ee918718ee122c3948b74ebeba8#e0119ac74f6c4ee918718ee122c3948b74ebeba8" +source = "git+https://github.com/RedisJSON/ijson?rev=eede48fad51b4ace5043d3e0714f5a65481a065d#eede48fad51b4ace5043d3e0714f5a65481a065d" dependencies = [ "dashmap", "hashbrown 0.13.2", @@ -820,9 +820,8 @@ dependencies = [ [[package]] name = "redis-module" -version = "2.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d3b95d9fe5b8681bacea6532bbc7632590ae4499cecc8e019685b515fddda71" +version = "99.99.99" +source = "git+https://github.com/RedisLabsModules/redismodule-rs?tag=v2.0.8#bce557d5ec9381d8b70eaee30c47f74bb66f458b" dependencies = [ "backtrace", "bindgen", @@ -843,9 +842,8 @@ dependencies = [ [[package]] name = "redis-module-macros" -version = "2.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c92438cbeaaba0f99e2944ceeb2e34fc8fa7955576296570575df046b81197" +version = "99.99.99" +source = "git+https://github.com/RedisLabsModules/redismodule-rs?tag=v2.0.8#bce557d5ec9381d8b70eaee30c47f74bb66f458b" dependencies = [ "proc-macro2", "quote 1.0.37", @@ -856,9 +854,8 @@ dependencies = [ [[package]] name = "redis-module-macros-internals" -version = "2.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd1ba5724bf8bfd0c9d6f1169e45255957175b7d81944b325909024a159bc74c" +version = "99.99.99" +source = "git+https://github.com/RedisLabsModules/redismodule-rs?tag=v2.0.8#bce557d5ec9381d8b70eaee30c47f74bb66f458b" dependencies = [ "lazy_static", "proc-macro2", @@ -876,6 +873,7 @@ dependencies = [ "ijson", "itertools", "json_path", + "lazy_static", "libc", "linkme", "redis-module", diff --git a/Cargo.toml b/Cargo.toml index 6a98cf077..535926cee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ ] [workspace.dependencies] -ijson = { git="https://github.com/RedisJSON/ijson", rev="e0119ac74f6c4ee918718ee122c3948b74ebeba8", default_features=false} +ijson = { git="https://github.com/RedisJSON/ijson", rev="eede48fad51b4ace5043d3e0714f5a65481a065d", default_features=false} serde_json = { version="1", features = ["unbounded_depth"]} serde = { version = "1", features = ["derive"] } serde_derive = "1" diff --git a/redis_json/Cargo.toml b/redis_json/Cargo.toml index 4065b356c..16d36b360 100644 --- a/redis_json/Cargo.toml +++ b/redis_json/Cargo.toml @@ -25,11 +25,12 @@ ijson.workspace = true serde_json.workspace = true serde.workspace = true libc = "0.2" -redis-module ={ version = "^2.0.7", default-features = false, features = ["min-redis-compatibility-version-7-2"] } -redis-module-macros = "^2.0.7" +redis-module ={ git="https://github.com/RedisLabsModules/redismodule-rs", tag="v2.0.8", default-features = false, features = ["min-redis-compatibility-version-7-2"] } +redis-module-macros = { git="https://github.com/RedisLabsModules/redismodule-rs", tag="v2.0.8" } itertools = "0.13" json_path = {path="../json_path"} linkme = "0.3" +lazy_static = "1" [features] as-library = [] diff --git a/redis_json/src/commands.rs b/redis_json/src/commands.rs index 430520182..f18c886b6 100644 --- a/redis_json/src/commands.rs +++ b/redis_json/src/commands.rs @@ -4,6 +4,7 @@ * the Server Side Public License v1 (SSPLv1). */ +use crate::defrag::defrag_info; use crate::error::Error; use crate::formatter::ReplyFormatOptions; use crate::key_value::KeyValue; @@ -1817,6 +1818,7 @@ pub fn json_debug(manager: M, ctx: &Context, args: Vec) .into()) } } + "DEFRAG_INFO" => defrag_info(ctx), "HELP" => { let results = vec![ "MEMORY [path] - reports memory usage", diff --git a/redis_json/src/defrag.rs b/redis_json/src/defrag.rs new file mode 100644 index 000000000..49e6ea24c --- /dev/null +++ b/redis_json/src/defrag.rs @@ -0,0 +1,106 @@ +use std::{ + alloc::Layout, + os::raw::{c_int, c_void}, +}; + +use ijson::{Defrag, DefragAllocator}; +use lazy_static::lazy_static; +use redis_module::{ + defrag::DefragContext, raw, redisvalue::RedisValueKey, Context, RedisGILGuard, RedisResult, + RedisValue, +}; +use redis_module_macros::{defrag_end_function, defrag_start_function}; + +use crate::redisjson::RedisJSON; + +#[derive(Default)] +pub(crate) struct DefragStats { + defrag_started: usize, + defrag_ended: usize, + keys_defrag: usize, +} + +lazy_static! { + pub(crate) static ref DEFRAG_STATS: RedisGILGuard = RedisGILGuard::default(); +} + +struct DefragCtxAllocator<'dc> { + defrag_ctx: &'dc DefragContext, +} + +impl<'dc> DefragAllocator for DefragCtxAllocator<'dc> { + unsafe fn realloc_ptr(&mut self, ptr: *mut T, _layout: Layout) -> *mut T { + self.defrag_ctx.defrag_realloc(ptr) + } + + /// Allocate memory for defrag + unsafe fn alloc(&mut self, layout: Layout) -> *mut u8 { + self.defrag_ctx.defrag_alloc(layout) + } + + /// Free memory for defrag + unsafe fn free(&mut self, ptr: *mut T, layout: Layout) { + self.defrag_ctx.defrag_dealloc(ptr, layout) + } +} + +#[defrag_start_function] +fn defrag_start(defrag_ctx: &DefragContext) { + let mut defrag_stats = DEFRAG_STATS.lock(defrag_ctx); + defrag_stats.defrag_started += 1; + ijson::reinit_shared_string_cache(); +} + +#[defrag_end_function] +fn defrag_end(defrag_ctx: &DefragContext) { + let mut defrag_stats = DEFRAG_STATS.lock(defrag_ctx); + defrag_stats.defrag_ended += 1; +} + +#[allow(non_snake_case, unused)] +pub unsafe extern "C" fn defrag( + ctx: *mut raw::RedisModuleDefragCtx, + key: *mut raw::RedisModuleString, + value: *mut *mut c_void, +) -> c_int { + let defrag_ctx = DefragContext::new(ctx); + + let mut defrag_stats = DEFRAG_STATS.lock(&defrag_ctx); + defrag_stats.keys_defrag += 1; + + let mut defrag_allocator = DefragCtxAllocator { + defrag_ctx: &defrag_ctx, + }; + let value = value.cast::<*mut RedisJSON>(); + let new_val = defrag_allocator.realloc_ptr(*value, Layout::new::>()); + if !new_val.is_null() { + std::ptr::write(value, new_val); + } + std::ptr::write( + &mut (**value).data as *mut ijson::IValue, + std::ptr::read(*value).data.defrag(&mut defrag_allocator), + ); + 0 +} + +pub(crate) fn defrag_info(ctx: &Context) -> RedisResult { + let defrag_stats = DEFRAG_STATS.lock(ctx); + Ok(RedisValue::OrderedMap( + [ + ( + RedisValueKey::String("defrag_started".to_owned()), + RedisValue::Integer(defrag_stats.defrag_started as i64), + ), + ( + RedisValueKey::String("defrag_ended".to_owned()), + RedisValue::Integer(defrag_stats.defrag_ended as i64), + ), + ( + RedisValueKey::String("keys_defrag".to_owned()), + RedisValue::Integer(defrag_stats.keys_defrag as i64), + ), + ] + .into_iter() + .collect(), + )) +} diff --git a/redis_json/src/lib.rs b/redis_json/src/lib.rs index c240a74c3..f5b75caf7 100644 --- a/redis_json/src/lib.rs +++ b/redis_json/src/lib.rs @@ -36,6 +36,7 @@ mod array_index; mod backward; pub mod c_api; pub mod commands; +pub mod defrag; pub mod error; mod formatter; pub mod ivalue_manager; @@ -73,7 +74,7 @@ pub static REDIS_JSON_TYPE: RedisType = RedisType::new( free_effort: None, unlink: None, copy: Some(redisjson::type_methods::copy), - defrag: None, + defrag: Some(defrag::defrag), free_effort2: None, unlink2: None, diff --git a/tests/pytest/test_defrag.py b/tests/pytest/test_defrag.py new file mode 100644 index 000000000..71dcd80f3 --- /dev/null +++ b/tests/pytest/test_defrag.py @@ -0,0 +1,133 @@ +import time +import json +from RLTest import Defaults + +Defaults.decode_responses = True + +def enableDefrag(env): + # make defrag as aggressive as possible + env.cmd('CONFIG', 'SET', 'hz', '100') + env.cmd('CONFIG', 'SET', 'active-defrag-ignore-bytes', '1') + env.cmd('CONFIG', 'SET', 'active-defrag-threshold-lower', '0') + env.cmd('CONFIG', 'SET', 'active-defrag-cycle-min', '99') + + try: + env.cmd('CONFIG', 'SET', 'activedefrag', 'yes') + except Exception: + # If active defrag is not supported by the current Redis, simply skip the test. + env.skip() + +def defragOnObj(env, obj): + enableDefrag(env) + json_str = json.dumps(obj) + env.expect('JSON.SET', 'test', '$', json_str).ok() + for i in range(10000): + env.expect('JSON.SET', 'test%d' % i, '$', json_str).ok() + i += 1 + env.expect('JSON.SET', 'test%d' % i, '$', json_str).ok() + for i in range(10000): + env.expect('DEL', 'test%d' % i).equal(1) + i += 1 + _, _, _, _, _, keysDefrag = env.cmd('JSON.DEBUG', 'DEFRAG_INFO') + startTime = time.time() + # Wait for at least 2 defrag full cycles + # We verify only the 'keysDefrag' value because the other values + # are not promised to be updated. It depends if Redis support + # the start/end defrag callbacks. + while keysDefrag < 2: + time.sleep(0.1) + _, _, _, _, _, keysDefrag = env.cmd('JSON.DEBUG', 'DEFRAG_INFO') + if time.time() - startTime > 30: + # We will wait for up to 30 seconds and then we consider it a failure + env.assertTrue(False, message='Failed waiting for defrag to run') + return + # make sure json is still valid. + res = json.loads(env.cmd('JSON.GET', 'test%d' % i, '$'))[0] + env.assertEqual(res, obj) + env.assertGreater(env.cmd('info', 'Stats')['active_defrag_key_hits'], 0) + +def testDefragNumber(env): + defragOnObj(env, 1) + +def testDefragBigNumber(env): + defragOnObj(env, 100000000000000000000) + +def testDefragDouble(env): + defragOnObj(env, 1.111111111111) + +def testDefragNegativeNumber(env): + defragOnObj(env, -100000000000000000000) + +def testDefragNegativeDouble(env): + defragOnObj(env, -1.111111111111) + +def testDefragTrue(env): + defragOnObj(env, True) + +def testDefragFalse(env): + defragOnObj(env, True) + +def testDefragNone(env): + defragOnObj(env, None) + +def testDefragEmptyString(env): + defragOnObj(env, "") + +def testDefragString(env): + defragOnObj(env, "foo") + +def testDefragEmptyArray(env): + defragOnObj(env, []) + +def testDefragArray(env): + defragOnObj(env, [1, 2, 3]) + +def testDefragEmptyObject(env): + defragOnObj(env, {}) + +def testDefragObject(env): + defragOnObj(env, {"foo": "bar"}) + +def testDefragComplex(env): + defragOnObj(env, {"foo": ["foo", 1, None, True, False, {}, {"foo": [], "bar": 1}]}) + +def testDefragBigJsons(env): + enableDefrag(env) + + # Disable defrag so we can actually create fragmentation + env.cmd('CONFIG', 'SET', 'activedefrag', 'no') + + env.expect('JSON.SET', 'key1', '$', "[]").ok() + env.expect('JSON.SET', 'key2', '$', "[]").ok() + + for i in range(100000): + env.cmd('JSON.ARRAPPEND', 'key1', '$', "[1.11111111111]") + env.cmd('JSON.ARRAPPEND', 'key2', '$', "[1.11111111111]") + + # Now we delete key2 which should cause fragmenation + env.expect('DEL', 'key2').equal(1) + + # wait for fragmentation for up to 30 seconds + frag = env.cmd('info', 'memory')['allocator_frag_ratio'] + startTime = time.time() + while frag < 1.4: + time.sleep(0.1) + frag = env.cmd('info', 'memory')['allocator_frag_ratio'] + if time.time() - startTime > 30: + # We will wait for up to 30 seconds and then we consider it a failure + env.assertTrue(False, message='Failed waiting for fragmentation, current value %s which is expected to be above 1.4.' % frag) + return + + #enable active defrag + env.cmd('CONFIG', 'SET', 'activedefrag', 'yes') + + # wait for fragmentation for go down for up to 30 seconds + frag = env.cmd('info', 'memory')['allocator_frag_ratio'] + startTime = time.time() + while frag > 1.1: + time.sleep(0.1) + frag = env.cmd('info', 'memory')['allocator_frag_ratio'] + if time.time() - startTime > 30: + # We will wait for up to 30 seconds and then we consider it a failure + env.assertTrue(False, message='Failed waiting for fragmentation to go down, current value %s which is expected to be bellow 1.1.' % frag) + return From 1b8ab1575eaa38bb7e9b4f02a6b487fd2b0c072c Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Tue, 17 Sep 2024 21:51:23 +0300 Subject: [PATCH 080/112] MOD-7644 Support Alpine ARM64 (#1265) Create github workflow which builds and tests alpine arm64. In addition update alpine workflow to upload module artifacts to S3 --- .../build-json-module-and-redis/action.yml | 22 ++++ .github/actions/setup-env/action.yml | 68 ++++++++++ .../actions/upload-artifacts-to-s3/action.yml | 48 +++++++ .github/workflows/alpine-arm.yml | 121 ++++++++++++++++++ .github/workflows/alpine.yml | 6 + sbin/upload-artifacts | 3 +- 6 files changed, 267 insertions(+), 1 deletion(-) create mode 100644 .github/actions/build-json-module-and-redis/action.yml create mode 100644 .github/actions/setup-env/action.yml create mode 100644 .github/actions/upload-artifacts-to-s3/action.yml create mode 100644 .github/workflows/alpine-arm.yml diff --git a/.github/actions/build-json-module-and-redis/action.yml b/.github/actions/build-json-module-and-redis/action.yml new file mode 100644 index 000000000..f6c30157c --- /dev/null +++ b/.github/actions/build-json-module-and-redis/action.yml @@ -0,0 +1,22 @@ +name: Build JSON module and Redis Server +description: | + Build JSON module and Redis Server + +inputs: + redis-ref: + description: 'Redis version to build' + required: true + default: '7.2.1' + +runs: + using: composite + steps: + - name: Build Redis + shell: bash + working-directory: redis + run: | + make install -j `nproc` + - name: Build module + shell: bash + run: | + make build -j `nproc` \ No newline at end of file diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml new file mode 100644 index 000000000..fb8467a00 --- /dev/null +++ b/.github/actions/setup-env/action.yml @@ -0,0 +1,68 @@ +name: Setup environment +description: | + setup environment for the build and output the branch and tag information + +inputs: + github-ref: + description: GitHub ref + required: true + redis-ref: + description: Redis ref + required: false + +outputs: + TAGGED: + description: 'Is this a tagged build, actual value is 1 or 0' + value: ${{ steps.set-tagged.outputs.TAGGED }} + TAG: + description: 'The tag name' + value: ${{ steps.set-git-info.outputs.TAG }} + BRANCH: + description: 'The branch name' + value: ${{ steps.set-git-info.outputs.TAG }} + TAG_OR_BRANCH: + description: 'The tag or branch name' + value: ${{ steps.set-git-info.outputs.TAG }}${{ steps.set-git-info.outputs.BRANCH }} + redis-ref: + description: 'The redis ref' + value: ${{ steps.set-redis-ref.outputs.REDIS_REF }} + +runs: + using: composite + steps: + - name: Set the branch and tag outputs + shell: bash + id: set-git-info + run: | + export REF="${{ inputs.github-ref }}" + export BRANCH_PATTERN="^refs/heads/(.*)$" + export TAG_PATTERN="^refs/tags/(.*)$" + + if [[ $REF =~ $BRANCH_PATTERN ]]; then + echo "BRANCH=${BASH_REMATCH[1]}" >> $GITHUB_OUTPUT + fi + + if [[ $REF =~ $TAG_PATTERN ]]; then + echo "TAG=${BASH_REMATCH[1]}" >> $GITHUB_OUTPUT + fi + - name: Set the tagged flag + shell: bash + id: set-tagged + run: | + # If this is a version tag, then set to false, meaning this + # is not a production build. + export REF="${{ inputs.github-ref }}" + export PATTERN="refs/tags/v[0-9]+.*" + if [[ $REF =~ $PATTERN ]]; then + echo "This is a tagged build" + echo "TAGGED=1" >> $GITHUB_OUTPUT + else + echo "This is not a tagged build" + echo "TAGGED=0" >> $GITHUB_OUTPUT + fi + - name: Set redis ref + shell: bash + id: set-redis-ref + run: | + export REDIS_REF="${{ inputs.redis-ref || '7.2.1'}}" + echo "REDIS_REF=${REDIS_REF}" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/actions/upload-artifacts-to-s3/action.yml b/.github/actions/upload-artifacts-to-s3/action.yml new file mode 100644 index 000000000..b0276085e --- /dev/null +++ b/.github/actions/upload-artifacts-to-s3/action.yml @@ -0,0 +1,48 @@ +name: Upload Artifacts to S3 +description: | + Uploads module artifacts to S3 bucket. + +inputs: + aws-access-key-id: + description: 'AWS Access Key ID' + required: true + aws-secret-access-key: + description: 'AWS Secret Access Key' + required: true + github-ref: + description: 'GitHub ref' + required: true + osnick: + description: 'OS Nickname' + required: false + default: '' + +runs: + using: composite + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v3 + with: # todo: use role instead of access key + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: "us-east-1" + - name: Upload artifacts to S3 - staging + shell: bash + run: | + echo ::group::install aws cli + python3 -m venv .aws-cli-venv && source .aws-cli-venv/bin/activate && + pip3 install --upgrade pip && pip3 install --no-cache-dir awscli && rm -rf /var/cache/apk/* + echo ::endgroup:: + echo ::group::upload artifacts + make upload-artifacts SHOW=1 VERBOSE=1 + echo ::endgroup:: + echo ::group::upload staging release + make upload-release SHOW=1 STAGING=1 VERBOSE=1 + echo ::endgroup:: + + echo ::group::upload production release + # todo: trigger this manually instead + if [[ "${{ inputs.github-ref}}" != 'refs/heads/master' ]]; then + make upload-release SHOW=1 VERBOSE=1 + fi + echo ::endgroup:: \ No newline at end of file diff --git a/.github/workflows/alpine-arm.yml b/.github/workflows/alpine-arm.yml new file mode 100644 index 000000000..878b32553 --- /dev/null +++ b/.github/workflows/alpine-arm.yml @@ -0,0 +1,121 @@ +name: alpine ARM64 + +on: + push: + branches: + - master + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + pull_request: + types: + - opened + - reopened + - review_requested + workflow_dispatch: # Allows you to run this workflow manually from the Actions tab + inputs: + redis-ref: + description: 'Redis ref to checkout' + required: true + default: 'unstable' + workflow_call: # Allows to run this workflow from another workflow + inputs: + redis-ref: + description: 'Redis ref to checkout' + type: string + required: true + +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + +jobs: + setup-environment: + runs-on: ubuntu-latest + outputs: + TAGGED: ${{ steps.set-envtagged.outputs.TAGGED }} + TAG: ${{ steps.set-env.outputs.TAG }} + BRANCH: ${{ steps.set-env.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ steps.set-env.outputs.TAG }}${{ steps.set-env.outputs.BRANCH }} + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + steps: + - name: checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: set env + id: set-env + uses: ./.github/actions/setup-env + with: + github-ref: ${{ github.ref }} + redis-ref: ${{ inputs.redis-ref }} + + alpine-arm64: + runs-on: ubuntu24-arm64-4-16 # ubuntu24-arm64-2-8 + needs: setup-environment + defaults: + run: + shell: bash + env: + TAGGED: ${{ needs.setup-environment.outputs.TAGGED }} + VERSION: ${{ needs.setup-environment.outputs.TAG }} + BRANCH: ${{ needs.setup-environment.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ needs.setup-environment.outputs.TAG_OR_BRANCH}} + + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16 + ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 + container: + image: alpine:3 # required to run the job on the ARM instance + steps: + - name: Workaround alpine-arm64 GHA issues + shell: sh + run: | + cp /etc/os-release /etc/os-release.bak + sed -i 's/ID=alpine/ID=NotpineForGHA/g' /etc/os-release + - name: Install prerequisites + shell: sh + run: | + echo ::group::install packages + apk add bash make libtool tar cmake python3 python3-dev \ + py3-pip gcc git curl build-base autoconf automake py3-cryptography \ + linux-headers musl-dev libffi-dev openssl-dev openssh py-virtualenv \ + clang18-libclang gcompat libstdc++ libgcc g++ openblas-dev \ + xsimd git xz bsd-compat-headers clang18 cargo + echo ::endgroup:: + - name: Checkout the module + uses: actions/checkout@v3 + with: + submodules: 'recursive' + - name: Checkout Redis + uses: actions/checkout@v3 + with: + repository: 'redis/redis' + ref: ${{needs.setup-environment.outputs.redis-ref}} + path: 'redis' + - name: Install python dependencies + run: | + echo ::group::install requirements + pip install -q --upgrade setuptools + pip install -q --upgrade pip + pip install -q -r tests/pytest/requirements.txt + pip install -q -r .install/build_package_requirements.txt + echo ::endgroup:: + env: + PIP_BREAK_SYSTEM_PACKAGES: 1 + - name: build + uses: ./.github/actions/build-json-module-and-redis + - name: Test + run: | + make test + - name: Pack module + run: | + git config --global --add safe.directory $GITHUB_WORKSPACE # to avoid git error + mv /etc/os-release.bak /etc/os-release + make pack BRANCH=$TAG_OR_BRANCH SHOW=1 + sed -i 's/ID=alpine/ID=NotpineForGHA/g' /etc/os-release + - name: Upload artifacts to S3 + uses: ./.github/actions/upload-artifacts-to-s3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + github-ref: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/alpine.yml b/.github/workflows/alpine.yml index 1bc681970..091ad88c3 100644 --- a/.github/workflows/alpine.yml +++ b/.github/workflows/alpine.yml @@ -50,3 +50,9 @@ jobs: - name: Pack module run: | make pack BRANCH=${{ github.ref_name }} + - name: Upload artifacts to S3 + uses: ./.github/actions/upload-artifacts-to-s3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + github-ref: ${{ github.ref }} diff --git a/sbin/upload-artifacts b/sbin/upload-artifacts index d303d1045..d589418c5 100755 --- a/sbin/upload-artifacts +++ b/sbin/upload-artifacts @@ -88,7 +88,8 @@ else fi cd artifacts${MAYBE_SNAP} -[[ $VERBOSE == 1 ]] && du -ah --apparent-size * +# in some versions --apparent-size is not supported +[[ $VERBOSE == 1 ]] && du -ah --apparent-size * || du -ah -s * #---------------------------------------------------------------------------------------------- From b68c846945aefca1fa665c887e5eded066b46414 Mon Sep 17 00:00:00 2001 From: efeldblum <109016390+ephraimfeldblum@users.noreply.github.com> Date: Thu, 19 Sep 2024 10:35:30 +0300 Subject: [PATCH 081/112] MOD-7733 Support Ubuntu[18, 20, 22] ARM64 (#1270) * initial commit * fix * fix * is cmake even needed * get rust * using cargo instead of make * reroute to correct file * i'm not selecting the geographic area in which i live * fix * activate venv * venv * now to fix alpine * to check that alpine works on all commits * fix alpine * fix alpine * fix alpine * fix alpine * fix alpine * fix outputs * stop using makefile * split build action into two * check that alpine works * only run on master and tags --- .../action.yml | 23 +++ .github/workflows/ubuntu-arm.yml | 169 ++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 .github/actions/build-json-module-and-redis-with-cargo/action.yml create mode 100644 .github/workflows/ubuntu-arm.yml diff --git a/.github/actions/build-json-module-and-redis-with-cargo/action.yml b/.github/actions/build-json-module-and-redis-with-cargo/action.yml new file mode 100644 index 000000000..70d2a1c8c --- /dev/null +++ b/.github/actions/build-json-module-and-redis-with-cargo/action.yml @@ -0,0 +1,23 @@ +name: Build JSON module and Redis Server +description: | + Build JSON module and Redis Server + +inputs: + redis-ref: + description: 'Redis version to build' + required: true + default: '7.2.1' + +runs: + using: composite + steps: + - name: Build Redis + shell: bash + working-directory: redis + run: | + make install -j `nproc` + - name: Build module + shell: bash + run: | + . "$HOME/.cargo/env" + cargo build --release \ No newline at end of file diff --git a/.github/workflows/ubuntu-arm.yml b/.github/workflows/ubuntu-arm.yml new file mode 100644 index 000000000..1ade42bbb --- /dev/null +++ b/.github/workflows/ubuntu-arm.yml @@ -0,0 +1,169 @@ +name: Build and Test ubuntu ARM instances + +on: + push: + branches: + - master + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + pull_request: + types: + - opened + - reopened + - review_requested + workflow_dispatch: # Allows you to run this workflow manually from the Actions tab + inputs: + redis-ref: + description: 'Redis ref to checkout' + required: true + default: 'unstable' + workflow_call: # Allows to run this workflow from another workflow + inputs: + redis-ref: + description: 'Redis ref to checkout' + type: string + required: true + +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + + +jobs: + setup-environment: + runs-on: ubuntu-latest + outputs: + TAGGED: ${{ steps.set-env.outputs.TAGGED }} + TAG: ${{ steps.set-env.outputs.TAG }} + BRANCH: ${{ steps.set-env.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ steps.set-env.outputs.TAG }}${{ steps.set-env.outputs.BRANCH }} + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + steps: + - name: checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: set env + id: set-env + uses: ./.github/actions/setup-env + with: + github-ref: ${{ github.ref }} + redis-ref: ${{ inputs.redis-ref }} + + ubuntu-arm64: + runs-on: ubuntu24-arm64-4-16 # ubuntu24-arm64-2-8 + needs: setup-environment + strategy: + matrix: + docker: + - image: "ubuntu:bionic" + nick: "bionic" + install_git: | + apt-get update && apt-get install -y software-properties-common + add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git + install_deps: | + apt update -qq + apt upgrade -yqq + apt dist-upgrade -yqq + apt install -yqq software-properties-common unzip rsync + add-apt-repository ppa:ubuntu-toolchain-r/test -y + apt update + apt install -yqq build-essential wget curl make gcc-10 g++-10 openssl libssl-dev cargo binfmt-support \ + lsb-core awscli libclang-dev clang curl + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 60 --slave /usr/bin/g++ g++ /usr/bin/g++-10 + apt -y install python3.8 python3.8-venv python3.8-dev python3-venv python3-dev python3-pip + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2 + - image: "ubuntu:focal" + nick: focal + install_git: | + apt-get update && apt-get install -y software-properties-common + add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git + install_deps: | + apt update -qq + apt upgrade -yqq + apt install -yqq wget make clang-format gcc python3 python3-venv python3-pip lcov git openssl libssl-dev \ + unzip rsync build-essential gcc-10 g++-10 cargo libclang-dev clang curl + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 60 --slave /usr/bin/g++ g++ /usr/bin/g++-10 + - image: "ubuntu:jammy" + nick: "jammy" + install_git: | + apt-get update && apt-get install -y git + install_deps: | + apt update -qq + apt upgrade -yqq + apt install -yqq git wget build-essential lcov openssl libssl-dev \ + python3 python3-pip python3-venv python3-dev unzip rsync libclang-dev clang curl + defaults: + run: + shell: bash + env: + TAGGED: ${{ needs.setup-environment.outputs.TAGGED }} + VERSION: ${{ needs.setup-environment.outputs.TAG }} + BRANCH: ${{ needs.setup-environment.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ needs.setup-environment.outputs.TAG_OR_BRANCH}} + + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16 + ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 + container: + image: ${{ matrix.docker.image }} + steps: + - name: Install git + run: | + ${{ matrix.docker.install_git }} + - name: Checkout the module + uses: actions/checkout@v3 + with: + submodules: 'recursive' + - name: Install dependencies + run: | + ${{ matrix.docker.install_deps }} + env: + DEBIAN_FRONTEND: noninteractive + - name: Checkout Redis + uses: actions/checkout@v3 + with: + repository: 'redis/redis' + ref: ${{ needs.setup-environment.outputs.redis-ref }} + path: 'redis' + - name: Get Rust + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + source "$HOME/.cargo/env" + rustup update + rustup update nightly + rustup component add rust-src --toolchain nightly + - name: Install python dependencies + run: | + echo ::group::activate venv + python3 -m venv venv + echo "source $PWD/venv/bin/activate" >> ~/.bash_profile + source venv/bin/activate + echo ::endgroup:: + echo ::group::install requirements + pip install -q --upgrade setuptools + pip install -q --upgrade pip + pip install -q -r tests/pytest/requirements.txt + pip install -q -r .install/build_package_requirements.txt + echo ::endgroup:: + env: + PIP_BREAK_SYSTEM_PACKAGES: 1 + - name: build + uses: ./.github/actions/build-json-module-and-redis-with-cargo + - name: Test + run: | + source venv/bin/activate + MODULE=$(realpath ./target/release/librejson.so) RLTEST_ARGS='--no-progress' ./tests/pytest/tests.sh + - name: Pack module + run: | + source venv/bin/activate + git config --global --add safe.directory /__w/RedisJSON/RedisJSON # to avoid git error + MODULE=$(realpath ./target/release/librejson.so) BRANCH=$TAG_OR_BRANCH \ + SHOW=1 OSNICK=${{ matrix.docker.nick }} ./sbin/pack.sh + - name: Upload artifacts to S3 + uses: ./.github/actions/upload-artifacts-to-s3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + github-ref: ${{ github.ref }} + osnick: ${{ matrix.docker.nick }} \ No newline at end of file From 5e9d4cca5020ca653a028511ab563bc43fc9f789 Mon Sep 17 00:00:00 2001 From: efeldblum <109016390+ephraimfeldblum@users.noreply.github.com> Date: Sun, 22 Sep 2024 17:05:08 +0300 Subject: [PATCH 082/112] Fix module lib name (#1271) * update name from librjson.so/.dylib to rejson.so * only one file * run on commit but no test * revert to only run on master/tag --- .../actions/build-json-module-and-redis-with-cargo/action.yml | 3 ++- .github/workflows/ubuntu-arm.yml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/actions/build-json-module-and-redis-with-cargo/action.yml b/.github/actions/build-json-module-and-redis-with-cargo/action.yml index 70d2a1c8c..7ca75dd9e 100644 --- a/.github/actions/build-json-module-and-redis-with-cargo/action.yml +++ b/.github/actions/build-json-module-and-redis-with-cargo/action.yml @@ -20,4 +20,5 @@ runs: shell: bash run: | . "$HOME/.cargo/env" - cargo build --release \ No newline at end of file + cargo build --release + cp $(realpath ./target/release)/librejson.so $(realpath ./target/release)/rejson.so diff --git a/.github/workflows/ubuntu-arm.yml b/.github/workflows/ubuntu-arm.yml index 1ade42bbb..2a0b5e36d 100644 --- a/.github/workflows/ubuntu-arm.yml +++ b/.github/workflows/ubuntu-arm.yml @@ -153,12 +153,12 @@ jobs: - name: Test run: | source venv/bin/activate - MODULE=$(realpath ./target/release/librejson.so) RLTEST_ARGS='--no-progress' ./tests/pytest/tests.sh + MODULE=$(realpath ./target/release/rejson.so) RLTEST_ARGS='--no-progress' ./tests/pytest/tests.sh - name: Pack module run: | source venv/bin/activate git config --global --add safe.directory /__w/RedisJSON/RedisJSON # to avoid git error - MODULE=$(realpath ./target/release/librejson.so) BRANCH=$TAG_OR_BRANCH \ + MODULE=$(realpath ./target/release/rejson.so) BRANCH=$TAG_OR_BRANCH \ SHOW=1 OSNICK=${{ matrix.docker.nick }} ./sbin/pack.sh - name: Upload artifacts to S3 uses: ./.github/actions/upload-artifacts-to-s3 From e1c30cfed679ec7f79a8bee93eeef5f65ef9ceac Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Tue, 24 Sep 2024 13:34:29 +0300 Subject: [PATCH 083/112] MOD-7429 GHA step build & test x86 (#1258) * 1. create install script for each os 2. build setup a GHA yaml file 3. update deps * add verbose to pytest * add verbose to pytest * add verbose to pytest * add verbose to pytest * add verbose to pytest * add verbose to pytest * add verbose to pytest * add verbose to pytest * add verbose to pytest * add verbose to pytest * need to find the abspath * realpath should do * typo * why is it not finding RLTest? * activate venv prior to running tests * path * perms? * perms * is packaging even used at all? * reordering python installations to when needed * consolidation * . * . * is there a reason to have three checkouts? * . * nope * wrong pip * -r flag was already used * . * . * revert * try this again * ty this again * try this again * revert * temporary removal of focal * apply to ci-basic * . * . * node16 * . * - * get pip * upgrade * recursive checkout submodules * lock * review * . * cargo * cargo * venv issue * cargo * venv? * removing unecessary rust verion * upload to s3 * pip * 2->3 * venv * temporarily skip tests * split venv to its own step * and dependencies * what happens if i try installing make * without make * add jammy * readding tests * reintegrate venv and deps into test step * return focal to matrix * reorder tests * more venv activation * and if we reintegrate them back together * only run on master and tags --------- Co-authored-by: Ephraim Feldblum --- .../action.yml | 48 ++++++ .github/workflows/build-linux-platforms.yml | 137 ++++++++++++++++++ .github/workflows/ci-basic.yml | 89 ++++++++++++ .install/activate_venv.sh | 15 ++ .install/amazon_linux_2.sh | 29 ++++ .install/common_installations.sh | 28 +--- .install/debian_gnu_linux_11.sh | 11 ++ .install/getrust.sh | 17 +++ .install/install_aws.sh | 14 ++ .install/install_cmake.sh | 0 .install/macos.sh | 0 .install/mariner2.sh | 0 .install/rocky_linux_8.sh | 24 +++ .install/rocky_linux_9.sh | 15 ++ .install/ubuntu_18.04.sh | 23 +++ .install/ubuntu_20.04.sh | 14 ++ .install/ubuntu_22.04.sh | 10 ++ Cargo.lock | 117 +++++---------- Cargo.toml | 2 +- json_path/Cargo.toml | 2 +- redis_json/Cargo.toml | 3 - tests/pytest/common.py | 1 - tests/pytest/requirements.txt | 2 +- 23 files changed, 493 insertions(+), 108 deletions(-) create mode 100644 .github/actions/upload-artifacts-to-s3-without-make/action.yml create mode 100644 .github/workflows/build-linux-platforms.yml create mode 100644 .github/workflows/ci-basic.yml create mode 100755 .install/activate_venv.sh create mode 100755 .install/amazon_linux_2.sh create mode 100755 .install/debian_gnu_linux_11.sh create mode 100755 .install/getrust.sh create mode 100755 .install/install_aws.sh mode change 100644 => 100755 .install/install_cmake.sh mode change 100644 => 100755 .install/macos.sh mode change 100644 => 100755 .install/mariner2.sh create mode 100755 .install/rocky_linux_8.sh create mode 100755 .install/rocky_linux_9.sh create mode 100755 .install/ubuntu_18.04.sh create mode 100755 .install/ubuntu_20.04.sh create mode 100755 .install/ubuntu_22.04.sh diff --git a/.github/actions/upload-artifacts-to-s3-without-make/action.yml b/.github/actions/upload-artifacts-to-s3-without-make/action.yml new file mode 100644 index 000000000..9a9a43833 --- /dev/null +++ b/.github/actions/upload-artifacts-to-s3-without-make/action.yml @@ -0,0 +1,48 @@ +name: Upload Artifacts to S3 +description: | + Uploads module artifacts to S3 bucket. + +inputs: + aws-access-key-id: + description: 'AWS Access Key ID' + required: true + aws-secret-access-key: + description: 'AWS Secret Access Key' + required: true + github-ref: + description: 'GitHub ref' + required: true + osnick: + description: 'OS Nickname' + required: false + default: '' + +runs: + using: composite + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v3 + with: # todo: use role instead of access key + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: "us-east-1" + - name: Upload artifacts to S3 - staging + shell: bash + run: | + echo ::group::install aws cli + python3 -m venv .aws-cli-venv && source .aws-cli-venv/bin/activate && + pip3 install --upgrade pip && pip3 install --no-cache-dir awscli && rm -rf /var/cache/apk/* + echo ::endgroup:: + echo ::group::upload artifacts + SNAPSHOT=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts + echo ::endgroup:: + echo ::group::upload staging release + RELEASE=1 SHOW=1 STAGING=1 VERBOSE=1 ./sbin/upload-artifacts + echo ::endgroup:: + + echo ::group::upload production release + # todo: trigger this manually instead + if [[ "${{ inputs.github-ref}}" != 'refs/heads/master' ]]; then + RELEASE=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts + fi + echo ::endgroup:: \ No newline at end of file diff --git a/.github/workflows/build-linux-platforms.yml b/.github/workflows/build-linux-platforms.yml new file mode 100644 index 000000000..f9136b20c --- /dev/null +++ b/.github/workflows/build-linux-platforms.yml @@ -0,0 +1,137 @@ +name: Build all supported linux platforms + +on: + push: + branches: + - master + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + pull_request: + types: + - opened + - reopened + - review_requested + +jobs: + build-linux-matrix: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + docker_image: + - image: "ubuntu:bionic" + pre_req_install_cmd: | + # https://github.com/actions/checkout/issues/1809 + echo "ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION=node16" >> $GITHUB_ENV + echo "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION=node16" >> $GITHUB_ENV + # https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/ + echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV + apt-get update && apt-get install -y software-properties-common + add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git + install_deps_script: ./ubuntu_18.04.sh + - image: "ubuntu:focal" + # https://github.com/actions/checkout/issues/1386 - sparse checkout does not work on got 2.25 which is the default in focal + pre_req_install_cmd: | + apt-get update && apt-get install -y software-properties-common + add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git + install_deps_script: ./ubuntu_20.04.sh + - image: "ubuntu:jammy" + pre_req_install_cmd: | + apt-get update && apt-get install -y git + install_deps_script: ./ubuntu_22.04.sh + - image: "rockylinux:8" + pre_req_install_cmd: | + dnf install -y git + install_deps_script: ./rocky_linux_8.sh + - image: "rockylinux:9" + pre_req_install_cmd: | + dnf install -y git + install_deps_script: ./rocky_linux_9.sh + - image: "debian:bullseye" + pre_req_install_cmd: | + apt-get update && apt-get install -y git + install_deps_script: ./debian_gnu_linux_11.sh + - image: "amazonlinux:2" + pre_req_install_cmd: | + # https://github.com/actions/checkout/issues/1809 + echo "ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION=node16" >> $GITHUB_ENV + echo "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION=node16" >> $GITHUB_ENV + # https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/ + echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV + yum install -y git + install_deps_script: ./amazon_linux_2.sh + - image: "mcr.microsoft.com/cbl-mariner/base/core:2.0" + pre_req_install_cmd: tdnf install --noplugins --skipsignature -y ca-certificates git + install_deps_script: ./mariner2.sh + container: + image: ${{ matrix.docker_image.image }} + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - name: Install git + run: ${{ matrix.docker_image.pre_req_install_cmd }} + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: 'recursive' + - name: Setup + working-directory: .install + run: | + echo ::group::Install dependencies + ${{ matrix.docker_image.install_deps_script }} + echo ::endgroup:: + echo ::group::Install rust + ./getrust.sh + echo ::endgroup:: + - name: Get Redis + uses: actions/checkout@v3 + with: + repository: redis/redis + ref: 'unstable' # todo change per version/tag + path: redis + submodules: 'recursive' + - name: build + uses: ./.github/actions/build-json-module-and-redis-with-cargo + - name: Set Artifact Names + # Artifact names have to be unique, so we base them on the environment. + # We also remove invalid characters from the name. + id: artifact-names + run: | # Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ? + echo "name=$(echo "${{ matrix.docker_image.image }} x86-64, Redis unstable" | \ + sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT + - name: Run tests + run: | + echo ::group::Activate virtual environment + python3 -m venv venv + echo "source $PWD/venv/bin/activate" >> ~/.bash_profile + source venv/bin/activate + echo ::endgroup:: + echo ::group::Install python dependencies + ./.install/common_installations.sh + echo ::endgroup:: + echo ::group::Unit tests + cargo test + echo ::endgroup:: + echo ::group::Flow tests + MODULE=$(realpath ./target/release/rejson.so) RLTEST_ARGS='--no-progress' ./tests/pytest/tests.sh + echo ::endgroup:: + env: + PIP_BREAK_SYSTEM_PACKAGES: 1 + - name: Upload test artifacts + if: failure() + uses: actions/upload-artifact@v3 + with: + name: Test logs ${{ steps.artifact-names.outputs.name }} + path: tests/**/logs/*.log* + if-no-files-found: ignore + - name: Pack module + run: | + git config --global --add safe.directory /__w/RedisJSON/RedisJSON # to avoid git error + MODULE=$(realpath ./target/release/rejson.so) ./sbin/pack.sh + - name: Upload artifacts to S3 + uses: ./.github/actions/upload-artifacts-to-s3-without-make + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + github-ref: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/ci-basic.yml b/.github/workflows/ci-basic.yml new file mode 100644 index 000000000..e400a4e25 --- /dev/null +++ b/.github/workflows/ci-basic.yml @@ -0,0 +1,89 @@ +name: CI Basic + +on: + push: + paths-ignore: + - '.circleci/**' + - 'docs/**' + - '*.md' + branches-ignore: + - main + - master + - '[0-9]+.[0-9]+.[0-9]+' + - '[0-9]+.[0-9]+' + - 'feature-*' + tags-ignore: + - 'v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+-m[0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+' + +jobs: + build-linux-jammy: + runs-on: "ubuntu-latest" + container: + image: "ubuntu:jammy" + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - name: Install git + run: | + apt-get update && apt-get install -y git + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: 'recursive' + - name: Setup + working-directory: .install + run: | + echo ::group::Install dependencies + ./ubuntu_22.04.sh + echo ::endgroup:: + echo ::group::Install rust + ./getrust.sh + echo ::endgroup:: + - name: Get Redis + uses: actions/checkout@v3 + with: + repository: redis/redis + ref: 'unstable' # todo change per version/tag + path: redis + - name: Build Redis + working-directory: redis + run: | + make install + - name: Build module + run: | + . "$HOME/.cargo/env" + cargo --version + cargo build --release + - name: Set Artifact Names + # Artifact names have to be unique, so we base them on the environment. + # We also remove invalid characters from the name. + id: artifact-names + run: | # Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ? + echo "name=$(echo "ubuntu22 x86-64, Redis unstable" | \ + sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT + - name: Run tests + run: | + echo ::group::Activate virtual environment + ./.install/activate_venv.sh + echo ::endgroup:: + echo ::group::Install python dependencies + ./.install/common_installations.sh + echo ::endgroup:: + echo ::group::Flow tests + MODULE=$(realpath ./target/release/librejson.so) RLTEST_ARGS='--no-progress' ./tests/pytest/tests.sh + echo ::endgroup:: + echo ::group::Unit tests + cargo test + echo ::endgroup:: + env: + PIP_BREAK_SYSTEM_PACKAGES: 1 + - name: Upload artifacts + if: failure() + uses: actions/upload-artifact@v3 + with: + name: Test logs ${{ steps.artifact-names.outputs.name }} + path: tests/**/logs/*.log* + if-no-files-found: ignore \ No newline at end of file diff --git a/.install/activate_venv.sh b/.install/activate_venv.sh new file mode 100755 index 000000000..6d6993a65 --- /dev/null +++ b/.install/activate_venv.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +activate_venv() { + echo "copy ativation script to shell config" + if [[ $OS_TYPE == Darwin ]]; then + echo "source venv/bin/activate" >> ~/.bashrc + echo "source venv/bin/activate" >> ~/.zshrc + else + echo "source $PWD/venv/bin/activate" >> ~/.bash_profile + fi +} + +python3 -m venv venv +activate_venv +source venv/bin/activate \ No newline at end of file diff --git a/.install/amazon_linux_2.sh b/.install/amazon_linux_2.sh new file mode 100755 index 000000000..c7f3c8611 --- /dev/null +++ b/.install/amazon_linux_2.sh @@ -0,0 +1,29 @@ +#!/bin/bash +MODE=$1 # whether to install using sudo or not +set -e +export DEBIAN_FRONTEND=noninteractive + +$MODE yum update -y +# Install the RPM package that provides the Software Collections (SCL) required for devtoolset-11 +$MODE yum install -y https://vault.centos.org/centos/7/extras/x86_64/Packages/centos-release-scl-rh-2-3.el7.centos.noarch.rpm + +# http://mirror.centos.org/centos/7/ is deprecated, so we changed the above link to `https://vault.centos.org`, +# and we have to change the baseurl in the repo file to the working mirror (from mirror.centos.org to vault.centos.org) +$MODE sed -i 's/mirrorlist=/#mirrorlist=/g' /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo # Disable mirrorlist +$MODE sed -i 's/#baseurl=http:\/\/mirror/baseurl=http:\/\/vault/g' /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo # Enable a working baseurl + +$MODE yum install -y wget git which devtoolset-11-gcc devtoolset-11-gcc-c++ devtoolset-11-make \ + rsync python3 unzip tar python3-devel python3-pip awscli + +source /opt/rh/devtoolset-11/enable + +cp /opt/rh/devtoolset-11/enable /etc/profile.d/scl-devtoolset-11.sh + +$MODE yum install -y curl +$MODE yum install -y openssl11 openssl11-devel +$MODE ln -s `which openssl11` /usr/bin/openssl + +# Install clang +$MODE yum install -y clang + +source install_cmake.sh $MODE diff --git a/.install/common_installations.sh b/.install/common_installations.sh index 1174bbff6..dc781d70f 100755 --- a/.install/common_installations.sh +++ b/.install/common_installations.sh @@ -3,28 +3,14 @@ set -e OS_TYPE=$(uname -s) MODE=$1 # whether to install using sudo or not -activate_venv() { - echo "copy ativation script to shell config" - if [[ $OS_TYPE == Darwin ]]; then - echo "source venv/bin/activate" >> ~/.bashrc - echo "source venv/bin/activate" >> ~/.zshrc - else - echo "source $PWD/venv/bin/activate" >> ~/.bash_profile - fi -} +pip3 install --upgrade pip +pip3 install -q --upgrade setuptools +echo "pip version: $(pip3 --version)" +echo "pip path: $(which pip3)" -python3 -m venv venv -activate_venv -source venv/bin/activate - -pip install --upgrade pip -pip install -q --upgrade setuptools -echo "pip version: $(pip --version)" -echo "pip path: $(which pip)" - -pip install -q -r tests/pytest/requirements.txt +pip3 install -q -r tests/pytest/requirements.txt # These packages are needed to build the package -pip install -q -r .install/build_package_requirements.txt +pip3 install -q -r .install/build_package_requirements.txt # List installed packages -pip list +pip3 list diff --git a/.install/debian_gnu_linux_11.sh b/.install/debian_gnu_linux_11.sh new file mode 100755 index 000000000..f7db0cc66 --- /dev/null +++ b/.install/debian_gnu_linux_11.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e +export DEBIAN_FRONTEND=noninteractive +MODE=$1 # whether to install using sudo or not + +$MODE apt update -qq +$MODE apt upgrade -yqq +$MODE apt install -yqq git wget curl build-essential lcov openssl libssl-dev python3 python3-venv python3-pip \ + rsync unzip cargo libclang-dev clang + +source install_cmake.sh $MODE diff --git a/.install/getrust.sh b/.install/getrust.sh new file mode 100755 index 000000000..05ea5e234 --- /dev/null +++ b/.install/getrust.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Download and install rustup +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + +# Source the cargo environment script to update the PATH +source "$HOME/.cargo/env" + +# Update rustup and install nightly toolchain +rustup update +rustup update nightly + +# for RedisJSON build with addess santizer +rustup component add rust-src --toolchain nightly + +# Verify cargo installation +cargo --version \ No newline at end of file diff --git a/.install/install_aws.sh b/.install/install_aws.sh new file mode 100755 index 000000000..0fcf40036 --- /dev/null +++ b/.install/install_aws.sh @@ -0,0 +1,14 @@ +#!/bin/bash +ARCH=$(uname -m) +OS_TYPE=$(uname -s) +MODE=$1 # whether to install using sudo or not + +if [[ $OS_TYPE = 'Darwin' ]] +then + curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg" + $MODE installer -pkg AWSCLIV2.pkg -target / +else + wget -O awscliv2.zip https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip + unzip awscliv2.zip + $MODE ./aws/install +fi diff --git a/.install/install_cmake.sh b/.install/install_cmake.sh old mode 100644 new mode 100755 diff --git a/.install/macos.sh b/.install/macos.sh old mode 100644 new mode 100755 diff --git a/.install/mariner2.sh b/.install/mariner2.sh old mode 100644 new mode 100755 diff --git a/.install/rocky_linux_8.sh b/.install/rocky_linux_8.sh new file mode 100755 index 000000000..9753f1852 --- /dev/null +++ b/.install/rocky_linux_8.sh @@ -0,0 +1,24 @@ +#!/bin/bash +MODE=$1 # whether to install using sudo or not +set -e + +$MODE dnf update -y + +# Development Tools includes python11 and config-manager +$MODE dnf groupinstall "Development Tools" -yqq +# install pip +$MODE dnf install python3.11-pip -y + +# powertools is needed to install epel +$MODE dnf config-manager --set-enabled powertools + +# get epel to install gcc11 +$MODE dnf install epel-release -yqq + + +$MODE dnf install -y gcc-toolset-11-gcc gcc-toolset-11-gcc-c++ gcc-toolset-11-libatomic-devel make wget git openssl openssl-devel \ + bzip2-devel libffi-devel zlib-devel tar xz which rsync cargo clang curl + +cp /opt/rh/gcc-toolset-11/enable /etc/profile.d/gcc-toolset-11.sh + +source install_cmake.sh $MODE diff --git a/.install/rocky_linux_9.sh b/.install/rocky_linux_9.sh new file mode 100755 index 000000000..4f85741fe --- /dev/null +++ b/.install/rocky_linux_9.sh @@ -0,0 +1,15 @@ +#!/bin/bash +MODE=$1 # whether to install using sudo or not +set -e +export DEBIAN_FRONTEND=noninteractive +$MODE dnf update -y + +$MODE dnf install -y gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ make wget git openssl openssl-devel python3 which \ + rsync unzip cargo clang + +# install pip +$MODE dnf install python3-pip -y + +cp /opt/rh/gcc-toolset-13/enable /etc/profile.d/gcc-toolset-13.sh + +source install_cmake.sh $MODE diff --git a/.install/ubuntu_18.04.sh b/.install/ubuntu_18.04.sh new file mode 100755 index 000000000..244e049c9 --- /dev/null +++ b/.install/ubuntu_18.04.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e +export DEBIAN_FRONTEND=noninteractive +MODE=$1 # whether to install using sudo or not + +$MODE apt update -qq +$MODE apt upgrade -yqq +$MODE apt dist-upgrade -yqq +$MODE apt install -yqq software-properties-common unzip rsync + +# ppa for modern python and gcc10 +$MODE add-apt-repository ppa:ubuntu-toolchain-r/test -y +$MODE apt update +$MODE apt install -yqq build-essential wget curl make gcc-10 g++-10 openssl libssl-dev cargo binfmt-support lsb-core awscli libclang-dev clang curl +$MODE update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 60 --slave /usr/bin/g++ g++ /usr/bin/g++-10 + +# Install Python 3.8 +$MODE apt -y install python3.8 python3.8-venv python3.8-dev python3-venv python3-dev python3-pip + +# Set python3 to point to python3.8 +$MODE update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2 + +source install_cmake.sh $MODE diff --git a/.install/ubuntu_20.04.sh b/.install/ubuntu_20.04.sh new file mode 100755 index 000000000..b084abf59 --- /dev/null +++ b/.install/ubuntu_20.04.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e +export DEBIAN_FRONTEND=noninteractive +MODE=$1 # whether to install using sudo or not + +$MODE apt update -qq +$MODE apt upgrade -yqq + +$MODE apt install -yqq wget make clang-format gcc python3 python3-venv python3-pip lcov git openssl libssl-dev \ + unzip rsync build-essential gcc-10 g++-10 cargo libclang-dev clang curl + +$MODE update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 60 --slave /usr/bin/g++ g++ /usr/bin/g++-10 + +source install_cmake.sh $MODE diff --git a/.install/ubuntu_22.04.sh b/.install/ubuntu_22.04.sh new file mode 100755 index 000000000..3fb4f2344 --- /dev/null +++ b/.install/ubuntu_22.04.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e +export DEBIAN_FRONTEND=noninteractive +MODE=$1 # whether to install using sudo or not + +$MODE apt update -qq +$MODE apt upgrade -yqq +$MODE apt install -yqq git wget build-essential lcov openssl libssl-dev \ + python3 python3-pip python3-venv python3-dev unzip rsync libclang-dev clang curl +source install_cmake.sh $MODE diff --git a/Cargo.lock b/Cargo.lock index c10bb0623..02167b4a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,55 +39,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "anstream" -version = "0.6.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" - -[[package]] -name = "anstyle-parse" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" -dependencies = [ - "anstyle", - "windows-sys", -] - [[package]] name = "autocfg" version = "1.3.0" @@ -239,12 +190,6 @@ dependencies = [ "libloading", ] -[[package]] -name = "colorchoice" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" - [[package]] name = "cpufeatures" version = "0.2.13" @@ -313,27 +258,17 @@ dependencies = [ "syn 0.11.11", ] -[[package]] -name = "env_filter" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" -dependencies = [ - "log", - "regex", -] - [[package]] name = "env_logger" -version = "0.11.5" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ - "anstream", - "anstyle", - "env_filter", "humantime", + "is-terminal", "log", + "regex", + "termcolor", ] [[package]] @@ -412,6 +347,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex" version = "0.4.3" @@ -456,10 +397,15 @@ dependencies = [ ] [[package]] -name = "is_terminal_polyfill" -version = "1.70.1" +name = "is-terminal" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] [[package]] name = "itertools" @@ -869,7 +815,6 @@ version = "99.99.99" dependencies = [ "bitflags 2.6.0", "bson", - "env_logger", "ijson", "itertools", "json_path", @@ -1101,6 +1046,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "thiserror" version = "1.0.63" @@ -1176,12 +1130,6 @@ version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - [[package]] name = "uuid" version = "1.10.0" @@ -1271,6 +1219,15 @@ dependencies = [ "rustix", ] +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys", +] + [[package]] name = "windows-sys" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index 535926cee..cdb7c91d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ ijson = { git="https://github.com/RedisJSON/ijson", rev="eede48fad51b4ace5043d3e serde_json = { version="1", features = ["unbounded_depth"]} serde = { version = "1", features = ["derive"] } serde_derive = "1" -bson = "2" +bson = "2.11" [workspace.package] edition = "2021" diff --git a/json_path/Cargo.toml b/json_path/Cargo.toml index dd7bbdc53..ba7d0853c 100644 --- a/json_path/Cargo.toml +++ b/json_path/Cargo.toml @@ -17,7 +17,7 @@ regex = "1" itertools = "0.13" [dev-dependencies] -env_logger = "0.11" +env_logger = "0.10" # do not change this version without running a full ci cycle [[bin]] name = "jsonpath" diff --git a/redis_json/Cargo.toml b/redis_json/Cargo.toml index 16d36b360..483dc8f27 100644 --- a/redis_json/Cargo.toml +++ b/redis_json/Cargo.toml @@ -15,9 +15,6 @@ homepage = "https://redis.io/docs/stack/json/" crate-type = ["cdylib", "rlib"] name = "rejson" -[dev-dependencies] -env_logger = "0.11" - [dependencies] bitflags = "2" bson.workspace = true diff --git a/tests/pytest/common.py b/tests/pytest/common.py index adeee7c91..47f1f09a1 100644 --- a/tests/pytest/common.py +++ b/tests/pytest/common.py @@ -2,7 +2,6 @@ from contextlib import contextmanager from functools import wraps from includes import * -from packaging import version @contextmanager def TimeLimit(timeout): diff --git a/tests/pytest/requirements.txt b/tests/pytest/requirements.txt index bde78a3c6..bdba68604 100644 --- a/tests/pytest/requirements.txt +++ b/tests/pytest/requirements.txt @@ -1,3 +1,3 @@ RLTest ~= 0.7.2 six >= 1.10.0 -gevent >= 23.9.1 +gevent >= 22.10.2 From 698a2624ea803cb378ac36719c73755b3490666b Mon Sep 17 00:00:00 2001 From: efeldblum <109016390+ephraimfeldblum@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:40:04 +0300 Subject: [PATCH 084/112] MOD-7428: GHA step: Code coverage (#1266) * initial commit * run on push * now only when not docs * . * getclang * perms * move to install * . * reorganization * pass sudo along * more sudo * after install python dependencies * . * run from sbin * doubling * more sudo * clang++ cannot be slave of clang * review --- .github/workflows/event-pull-request.yml | 19 ++++ .github/workflows/flow-coverage.yml | 39 ++++++++ .github/workflows/task-check-docs.yml | 30 +++++++ .install/install_clang.sh | 19 ++++ .install/update_clang_alternatives.sh | 108 +++++++++++++++++++++++ 5 files changed, 215 insertions(+) create mode 100644 .github/workflows/event-pull-request.yml create mode 100644 .github/workflows/flow-coverage.yml create mode 100644 .github/workflows/task-check-docs.yml create mode 100755 .install/install_clang.sh create mode 100755 .install/update_clang_alternatives.sh diff --git a/.github/workflows/event-pull-request.yml b/.github/workflows/event-pull-request.yml new file mode 100644 index 000000000..b15b59713 --- /dev/null +++ b/.github/workflows/event-pull-request.yml @@ -0,0 +1,19 @@ +name: Pull Request Flow + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] # Defaults + ready_for_review + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + docs-only: # Check whether the PR is only modifying docs + uses: ./.github/workflows/task-check-docs.yml + + coverage: + needs: docs-only + if: ${{ needs.docs-only.outputs.only-docs-changed == 'false' && !github.event.pull_request.draft }} + uses: ./.github/workflows/flow-coverage.yml + secrets: inherit diff --git a/.github/workflows/flow-coverage.yml b/.github/workflows/flow-coverage.yml new file mode 100644 index 000000000..1792992d9 --- /dev/null +++ b/.github/workflows/flow-coverage.yml @@ -0,0 +1,39 @@ +name: Coverage analysis + +on: + workflow_call: + +jobs: + coverage: + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 # fetch all history for accurate results + - name: Setup specific + working-directory: .install + run: | + ./install_script.sh sudo + - name: Setup common + run: | + ./.install/common_installations.sh sudo + - name: Install clang 18 + working-directory: .install + run: | + ./install_clang.sh sudo + - name: Get Redis + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: ./deps/readies/bin/getredis --with-github-token + - name: Build and test + run: make coverage QUICK=1 SHOW=1 + - name: Upload coverage + uses: codecov/codecov-action@v3 + with: + file: bin/linux-x64-debug-cov/cov.info + fail_ci_if_error: true # Fail on upload errors + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/task-check-docs.yml b/.github/workflows/task-check-docs.yml new file mode 100644 index 000000000..14b137b15 --- /dev/null +++ b/.github/workflows/task-check-docs.yml @@ -0,0 +1,30 @@ +name: Checks if Only Documentation Files were Changed + +# Documentation: https://redislabs.atlassian.net/wiki/spaces/DX/pages/3967844669/RediSearch+CI+refactor + +on: + workflow_call: + outputs: + only-docs-changed: + value: ${{ jobs.check-only-docs-changed.outputs.only-docs-changed }} + +jobs: + check-only-docs-changed: + runs-on: ubuntu-latest + outputs: + only-docs-changed: ${{ steps.check-docs.outputs.only_modified }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 # required for changed-files action to work + - name: Check if only docs were changed + id: check-docs + uses: tj-actions/changed-files@v44 + with: + # List of files we allow to be changed without running the CI. Modify as needed. + files: | + **.md + docs/** + licenses/** + LICENSE.txt diff --git a/.install/install_clang.sh b/.install/install_clang.sh new file mode 100755 index 000000000..116d06871 --- /dev/null +++ b/.install/install_clang.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env sh + +export CWD=$(dirname `which "${0}"`) +export CLANG_VERSION=18 +export DEBIAN_FRONTEND=noninteractive +MODE=$1 # whether to install using sudo or not + +wget https://apt.llvm.org/llvm.sh -O llvm.sh + +chmod u+x llvm.sh + +# expected to fail: +$MODE ./llvm.sh $CLANG_VERSION + +$MODE apt-get install python3-lldb-18 --yes --force-yes + +$MODE ./llvm.sh $CLANG_VERSION + +$MODE $CWD/update_clang_alternatives.sh $CLANG_VERSION 1 diff --git a/.install/update_clang_alternatives.sh b/.install/update_clang_alternatives.sh new file mode 100755 index 000000000..3388e0def --- /dev/null +++ b/.install/update_clang_alternatives.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash + +# This script registers a specific version of Clang and LLVM tools using the `update-alternatives` command. +# It creates symlinks for the given version, allowing users to switch between multiple versions of Clang and LLVM. + + +# Function: register_clang_version +# Arguments: +# 1. version - The version of Clang/LLVM to be registered (e.g., 18). +# 2. priority - The priority of the version. A higher priority number indicates a preferred version. +# +# Sets up a primary symlink and several slave symlinks that correspond to related tools for Clang and LLVM. +# +# `update-alternatives --install` creates or updates the alternative for the given tool. +# The first argument after `--install` is the path to the main symlink (e.g., /usr/bin/clang). +# The second argument is the name of the alternative (e.g., clang), and the third argument is the +# path to the actual binary for the specified version (e.g., /usr/bin/clang-18). +# The `--slave` options are used to link other related binaries (like clang-format, llvm-nm, etc.) +# so that all tools are switched consistently when the main tool (e.g., clang) is switched. + +function register_clang_version { + local version=$1 + local priority=$2 + + # Register LLVM tools + update-alternatives \ + --verbose \ + --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${version} ${priority} \ + --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-${version} \ + --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-${version} \ + --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-${version} \ + --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-${version} \ + --slave /usr/bin/llvm-cat llvm-cat /usr/bin/llvm-cat-${version} \ + --slave /usr/bin/llvm-cfi-verify llvm-cfi-verify /usr/bin/llvm-cfi-verify-${version} \ + --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-${version} \ + --slave /usr/bin/llvm-cvtres llvm-cvtres /usr/bin/llvm-cvtres-${version} \ + --slave /usr/bin/llvm-cxxdump llvm-cxxdump /usr/bin/llvm-cxxdump-${version} \ + --slave /usr/bin/llvm-cxxfilt llvm-cxxfilt /usr/bin/llvm-cxxfilt-${version} \ + --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-${version} \ + --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-${version} \ + --slave /usr/bin/llvm-dlltool llvm-dlltool /usr/bin/llvm-dlltool-${version} \ + --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-${version} \ + --slave /usr/bin/llvm-dwp llvm-dwp /usr/bin/llvm-dwp-${version} \ + --slave /usr/bin/llvm-exegesis llvm-exegesis /usr/bin/llvm-exegesis-${version} \ + --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-${version} \ + --slave /usr/bin/llvm-lib llvm-lib /usr/bin/llvm-lib-${version} \ + --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-${version} \ + --slave /usr/bin/llvm-lto llvm-lto /usr/bin/llvm-lto-${version} \ + --slave /usr/bin/llvm-lto2 llvm-lto2 /usr/bin/llvm-lto2-${version} \ + --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-${version} \ + --slave /usr/bin/llvm-mca llvm-mca /usr/bin/llvm-mca-${version} \ + --slave /usr/bin/llvm-modextract llvm-modextract /usr/bin/llvm-modextract-${version} \ + --slave /usr/bin/llvm-mt llvm-mt /usr/bin/llvm-mt-${version} \ + --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-${version} \ + --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-${version} \ + --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-${version} \ + --slave /usr/bin/llvm-opt-report llvm-opt-report /usr/bin/llvm-opt-report-${version} \ + --slave /usr/bin/llvm-pdbutil llvm-pdbutil /usr/bin/llvm-pdbutil-${version} \ + --slave /usr/bin/llvm-PerfectShuffle llvm-PerfectShuffle /usr/bin/llvm-PerfectShuffle-${version} \ + --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-${version} \ + --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-${version} \ + --slave /usr/bin/llvm-rc llvm-rc /usr/bin/llvm-rc-${version} \ + --slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-${version} \ + --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-${version} \ + --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-${version} \ + --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-${version} \ + --slave /usr/bin/llvm-split llvm-split /usr/bin/llvm-split-${version} \ + --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-${version} \ + --slave /usr/bin/llvm-strings llvm-strings /usr/bin/llvm-strings-${version} \ + --slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-${version} \ + --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-${version} \ + --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-${version} \ + --slave /usr/bin/llvm-undname llvm-undname /usr/bin/llvm-undname-${version} \ + --slave /usr/bin/llvm-xray llvm-xray /usr/bin/llvm-xray-${version} + + # Register Clang tools + update-alternatives \ + --verbose \ + --install /usr/bin/clang clang /usr/bin/clang-${version} ${priority} \ + --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-${version} \ + --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-${version} \ + --slave /usr/bin/clang-cl clang-cl /usr/bin/clang-cl-${version} \ + --slave /usr/bin/clangd clangd /usr/bin/clangd-${version} \ + --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-${version} \ + --slave /usr/bin/clang-check clang-check /usr/bin/clang-check-${version} \ + --slave /usr/bin/clang-query clang-query /usr/bin/clang-query-${version} \ + --slave /usr/bin/asan_symbolize asan_symbolize /usr/bin/asan_symbolize-${version} \ + --slave /usr/bin/bugpoint bugpoint /usr/bin/bugpoint-${version} \ + --slave /usr/bin/dsymutil dsymutil /usr/bin/dsymutil-${version} \ + --slave /usr/bin/lld lld /usr/bin/lld-${version} \ + --slave /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-${version} \ + --slave /usr/bin/lld-link lld-link /usr/bin/lld-link-${version} \ + --slave /usr/bin/llc llc /usr/bin/llc-${version} \ + --slave /usr/bin/lli lli /usr/bin/lli-${version} \ + --slave /usr/bin/obj2yaml obj2yaml /usr/bin/obj2yaml-${version} \ + --slave /usr/bin/opt opt /usr/bin/opt-${version} \ + --slave /usr/bin/sanstats sanstats /usr/bin/sanstats-${version} \ + --slave /usr/bin/verify-uselistorder verify-uselistorder /usr/bin/verify-uselistorder-${version} \ + --slave /usr/bin/wasm-ld wasm-ld /usr/bin/wasm-ld-${version} \ + --slave /usr/bin/yaml2obj yaml2obj /usr/bin/yaml2obj-${version} + + # Register clang++ + update-alternatives \ + --verbose \ + --install /usr/bin/clang++ clang++ /usr/bin/clang++-${version} ${priority} +} + +register_clang_version $1 $2 From c94ee010735dda3997cba7ede72aefae1d3f8a6c Mon Sep 17 00:00:00 2001 From: efeldblum <109016390+ephraimfeldblum@users.noreply.github.com> Date: Tue, 1 Oct 2024 15:17:55 +0300 Subject: [PATCH 085/112] GHA step: Sanitizer (#1274) * initial commit * tryst * actions/checkout * shell: bash * shell: bash * id * shell: bash * sudo * no install git * container * container * delay common_installations * getrust * remove extraneous cargo download? * uninstall and reinstall * profile_d * bullseye -> jammy * focal * rocky9? * move installing cargo to immediately before making redisjson * jammy * dir * revert * no default dir * no default dir * unneeded if * rust * profile.d * working dir * env * echo to bash_profile * source * echo * pwd * bashrc * echo groups * no sudo * $HOME -> ~ * revert * hardcoding it in * fix alpine arm * fix alpine arm * fix alpine arm * fix alpine arm * review * review --- .../build-json-module-and-redis/action.yml | 13 +- .github/actions/node20-supported/action.yml | 30 +++++ .github/actions/run-tests/action.yml | 122 ++++++++++++++++++ .github/workflows/alpine-arm.yml | 2 +- .github/workflows/build-linux-platforms.yml | 12 +- .github/workflows/event-pull-request.yml | 6 + .github/workflows/flow-sanitizer.yml | 29 +++++ ...r2.sh => common_base_linux_mariner_2.0.sh} | 0 .install/getrust.sh | 17 ++- 9 files changed, 212 insertions(+), 19 deletions(-) create mode 100644 .github/actions/node20-supported/action.yml create mode 100644 .github/actions/run-tests/action.yml create mode 100644 .github/workflows/flow-sanitizer.yml rename .install/{mariner2.sh => common_base_linux_mariner_2.0.sh} (100%) diff --git a/.github/actions/build-json-module-and-redis/action.yml b/.github/actions/build-json-module-and-redis/action.yml index f6c30157c..9f3d7d31f 100644 --- a/.github/actions/build-json-module-and-redis/action.yml +++ b/.github/actions/build-json-module-and-redis/action.yml @@ -7,6 +7,8 @@ inputs: description: 'Redis version to build' required: true default: '7.2.1' + sanitizer: + type: string runs: using: composite @@ -15,8 +17,15 @@ runs: shell: bash working-directory: redis run: | - make install -j `nproc` + echo ::group::Build Redis + make install -j `nproc` + echo ::endgroup:: - name: Build module shell: bash run: | - make build -j `nproc` \ No newline at end of file + echo ::group::Build RedisJSON + if [[ -f "$HOME/.cargo/env" ]]; then + . "$HOME/.cargo/env" + fi + make build SAN=${{ inputs.sanitizer }} -j `nproc` + echo ::endgroup:: diff --git a/.github/actions/node20-supported/action.yml b/.github/actions/node20-supported/action.yml new file mode 100644 index 000000000..65db7653f --- /dev/null +++ b/.github/actions/node20-supported/action.yml @@ -0,0 +1,30 @@ +name: Check if node20 is supported +description: Check if node20 is supported + +inputs: + container: + type: string + +outputs: + supported: + value: ${{ steps.node20.outputs.supported }} + +runs: + using: composite + steps: + - name: Check if node20 is Supported + id: node20 + shell: bash + run: | + for platform in ubuntu:bionic centos:7 amazonlinux:2 alpine:3; do + if [[ "${{ inputs.container }}" == "$platform" ]]; then + echo "supported=false" >> $GITHUB_OUTPUT + # https://github.com/actions/checkout/issues/1809 + echo "ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION=node16" >> $GITHUB_ENV + echo "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION=node16" >> $GITHUB_ENV + # https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/ + echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV + exit 0 + fi + done + echo "supported=true" >> $GITHUB_OUTPUT diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml new file mode 100644 index 000000000..7ceeb5289 --- /dev/null +++ b/.github/actions/run-tests/action.yml @@ -0,0 +1,122 @@ +name: Common Flow for Tests + +# Documentation: https://redislabs.atlassian.net/wiki/spaces/DX/pages/3967844669/RediSearch+CI+refactor + +inputs: + env: + default: "ubuntu-22.04" + type: string + container: + type: string + sanitizer: + type: string + test-config: + description: 'Test configuration environment variable (e.g. "CONFIG=tls" or "QUICK=1")' + required: true + type: string + +runs: + using: composite + steps: + - name: Get Installation Mode + shell: bash + id: mode + run: | + [[ -z "${{ inputs.container }}" ]] && echo "mode=sudo" >> $GITHUB_OUTPUT || echo "mode=" >> $GITHUB_OUTPUT + - name: Check if node20 is Supported + id: node20 + uses: ./.github/actions/node20-supported + with: + container: ${{ inputs.container }} + - name: Install git + shell: bash + run: | + # TODO: must be changed to run a script based on the input env + echo ::group::Install git + ${{ steps.mode.outputs.mode }} apt-get update && apt-get install -y git + echo ::endgroup:: + - name: Setup specific + shell: bash + working-directory: .install + run: | + echo ::group::OS-Specific Setup + ./install_script.sh ${{ steps.mode.outputs.mode }} + echo ::endgroup:: + echo ::group::Get Rust + ./getrust.sh ${{ steps.mode.outputs.mode }} + echo ::endgroup:: + + - name: Full checkout (node20 supported) + if: steps.node20.outputs.supported == 'true' + uses: actions/checkout@v4 + with: + submodules: recursive + - name: Full checkout (node20 unsupported) + if: steps.node20.outputs.supported == 'false' + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Get Redis + uses: actions/checkout@v3 + with: + repository: redis/redis + ref: 'unstable' # todo change per version/tag + path: redis + submodules: 'recursive' + - name: Build + uses: ./.github/actions/build-json-module-and-redis + with: + sanitizer: ${{ inputs.san }} + + - name: Set Artifact Names + shell: bash + # Artifact names have to be unique, so we base them on the environment. + # We also remove invalid characters from the name. + id: artifact-names + run: | + # Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ? + echo "name=$(echo "${{ inputs.container || inputs.env }} ${{ runner.arch }}, Redis ${{ inputs.get-redis || 'unstable' }}" | \ + sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT + + - name: Run tests + shell: bash + id: test + run: | + echo ::group::Activate virtual environment + python3 -m venv venv + echo "source $PWD/venv/bin/activate" >> ~/.bash_profile + source venv/bin/activate + echo ::endgroup:: + echo ::group::Install python dependencies + ./.install/common_installations.sh + echo ::endgroup:: + echo ::group::Unit tests + . "$HOME/.cargo/env" + make cargo_test LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }} + echo ::endgroup:: + echo ::group::Flow tests + make pytest LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }} ${{ inputs.test-config }} + echo ::endgroup:: + env: + PIP_BREAK_SYSTEM_PACKAGES: 1 + + - name: Upload test artifacts (node20 supported) + if: steps.node20.outputs.supported == 'true' && steps.test.outcome == 'failure' + uses: actions/upload-artifact@v4 + with: + name: Test logs ${{ steps.artifact-names.outputs.name }} + path: tests/**/logs/*.log* + if-no-files-found: ignore + - name: Upload test artifacts (node20 unsupported) + if: steps.node20.outputs.supported == 'false' && steps.test.outcome == 'failure' + uses: actions/upload-artifact@v3 + with: + name: Test logs ${{ steps.artifact-names.outputs.name }} + path: tests/**/logs/*.log* + if-no-files-found: ignore + + - name: Fail flow if tests failed + shell: bash + if: steps.test.outcome == 'failure' + run: exit 1 diff --git a/.github/workflows/alpine-arm.yml b/.github/workflows/alpine-arm.yml index 878b32553..37a59e57d 100644 --- a/.github/workflows/alpine-arm.yml +++ b/.github/workflows/alpine-arm.yml @@ -32,7 +32,7 @@ jobs: setup-environment: runs-on: ubuntu-latest outputs: - TAGGED: ${{ steps.set-envtagged.outputs.TAGGED }} + TAGGED: ${{ steps.set-env.outputs.TAGGED }} TAG: ${{ steps.set-env.outputs.TAG }} BRANCH: ${{ steps.set-env.outputs.BRANCH }} TAG_OR_BRANCH: ${{ steps.set-env.outputs.TAG }}${{ steps.set-env.outputs.BRANCH }} diff --git a/.github/workflows/build-linux-platforms.yml b/.github/workflows/build-linux-platforms.yml index f9136b20c..59fd9aced 100644 --- a/.github/workflows/build-linux-platforms.yml +++ b/.github/workflows/build-linux-platforms.yml @@ -28,29 +28,23 @@ jobs: echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV apt-get update && apt-get install -y software-properties-common add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git - install_deps_script: ./ubuntu_18.04.sh - image: "ubuntu:focal" # https://github.com/actions/checkout/issues/1386 - sparse checkout does not work on got 2.25 which is the default in focal pre_req_install_cmd: | apt-get update && apt-get install -y software-properties-common add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git - install_deps_script: ./ubuntu_20.04.sh - image: "ubuntu:jammy" pre_req_install_cmd: | apt-get update && apt-get install -y git - install_deps_script: ./ubuntu_22.04.sh - image: "rockylinux:8" pre_req_install_cmd: | dnf install -y git - install_deps_script: ./rocky_linux_8.sh - image: "rockylinux:9" pre_req_install_cmd: | dnf install -y git - install_deps_script: ./rocky_linux_9.sh - image: "debian:bullseye" pre_req_install_cmd: | apt-get update && apt-get install -y git - install_deps_script: ./debian_gnu_linux_11.sh - image: "amazonlinux:2" pre_req_install_cmd: | # https://github.com/actions/checkout/issues/1809 @@ -59,10 +53,8 @@ jobs: # https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/ echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV yum install -y git - install_deps_script: ./amazon_linux_2.sh - image: "mcr.microsoft.com/cbl-mariner/base/core:2.0" pre_req_install_cmd: tdnf install --noplugins --skipsignature -y ca-certificates git - install_deps_script: ./mariner2.sh container: image: ${{ matrix.docker_image.image }} defaults: @@ -79,10 +71,10 @@ jobs: working-directory: .install run: | echo ::group::Install dependencies - ${{ matrix.docker_image.install_deps_script }} + ./install_script.sh echo ::endgroup:: echo ::group::Install rust - ./getrust.sh + ./getrust.sh echo ::endgroup:: - name: Get Redis uses: actions/checkout@v3 diff --git a/.github/workflows/event-pull-request.yml b/.github/workflows/event-pull-request.yml index b15b59713..1968c92fc 100644 --- a/.github/workflows/event-pull-request.yml +++ b/.github/workflows/event-pull-request.yml @@ -17,3 +17,9 @@ jobs: if: ${{ needs.docs-only.outputs.only-docs-changed == 'false' && !github.event.pull_request.draft }} uses: ./.github/workflows/flow-coverage.yml secrets: inherit + + sanitize: + needs: docs-only + if: ${{ needs.docs-only.outputs.only-docs-changed == 'false' && !github.event.pull_request.draft }} + uses: ./.github/workflows/flow-sanitizer.yml + secrets: inherit diff --git a/.github/workflows/flow-sanitizer.yml b/.github/workflows/flow-sanitizer.yml new file mode 100644 index 000000000..bd47a3c6f --- /dev/null +++ b/.github/workflows/flow-sanitizer.yml @@ -0,0 +1,29 @@ +name: Clang Sanitizer + +# Documentation: https://redislabs.atlassian.net/wiki/spaces/DX/pages/3967844669/RediSearch+CI+refactor + +on: + workflow_call: + inputs: + container: + default: "ubuntu:jammy" + type: string + +jobs: + clang-sanitizer: + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash -l -eo pipefail {0} + container: + image: ${{ inputs.container }} + steps: + - name: checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: ./.github/actions/run-tests + with: + container: ${{ inputs.container }} + test-config: QUICK=1 + sanitizer: addr diff --git a/.install/mariner2.sh b/.install/common_base_linux_mariner_2.0.sh similarity index 100% rename from .install/mariner2.sh rename to .install/common_base_linux_mariner_2.0.sh diff --git a/.install/getrust.sh b/.install/getrust.sh index 05ea5e234..62b1cb261 100755 --- a/.install/getrust.sh +++ b/.install/getrust.sh @@ -1,17 +1,22 @@ #!/bin/bash +MODE=$1 # whether to install using sudo or not + # Download and install rustup -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +$MODE curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y # Source the cargo environment script to update the PATH -source "$HOME/.cargo/env" +echo "source $HOME/.cargo/env" >> $HOME/.bashrc +source $HOME/.cargo/env # Update rustup and install nightly toolchain -rustup update -rustup update nightly +$MODE rustup update +$MODE rustup update nightly # for RedisJSON build with addess santizer -rustup component add rust-src --toolchain nightly +$MODE rustup component add --toolchain nightly rust-src # Verify cargo installation -cargo --version \ No newline at end of file +cargo --version + +rustup show From 19f4a8668203b63827b2d048167b6eb53e8862ba Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Sun, 13 Oct 2024 11:38:42 +0300 Subject: [PATCH 086/112] Pin redis-py 5.0.8 (#1279) --- tests/pytest/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/requirements.txt b/tests/pytest/requirements.txt index bdba68604..04e6bb889 100644 --- a/tests/pytest/requirements.txt +++ b/tests/pytest/requirements.txt @@ -1,3 +1,4 @@ RLTest ~= 0.7.2 +redis ~= 5.0.8 six >= 1.10.0 gevent >= 22.10.2 From 554e4d859e5eaccc5d3d9add9e8fa7c8ad8acc12 Mon Sep 17 00:00:00 2001 From: efeldblum <109016390+ephraimfeldblum@users.noreply.github.com> Date: Sun, 13 Oct 2024 17:26:29 +0300 Subject: [PATCH 087/112] MOD-7858 MOD-7859 Add workflow dispatch to monterrey (#1283) * initial * fix --- .github/workflows/macos-build-and-deploy.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos-build-and-deploy.yml b/.github/workflows/macos-build-and-deploy.yml index 0ee501792..da2ada70a 100644 --- a/.github/workflows/macos-build-and-deploy.yml +++ b/.github/workflows/macos-build-and-deploy.yml @@ -6,6 +6,7 @@ on: - master tags: - 'v[0-9]+.[0-9]+.[0-9]+' + workflow_dispatch: # Allows you to run this workflow manually from the Actions tab jobs: build: @@ -28,13 +29,13 @@ jobs: tests/pytest/requirements.* - name: Setup specific working-directory: setup/.install - run: ./install_script.sh ${{ steps.mode.outputs.mode }} + run: ./install_script.sh - name: Full checkout uses: actions/checkout@v3 with: submodules: recursive - name: Setup common - run: .install/common_installations.sh ${{ steps.mode.outputs.mode }} + run: .install/common_installations.sh - name: Get Redis uses: actions/checkout@v4 From 32c2def9870ed505fe461e374843864210165311 Mon Sep 17 00:00:00 2001 From: efeldblum <109016390+ephraimfeldblum@users.noreply.github.com> Date: Mon, 14 Oct 2024 15:26:02 +0300 Subject: [PATCH 088/112] MOD-7858 MOD-7859 Add macos 13 to GHA (#1286) * initial * directly on arch * typo * update macos.sh * i hate ai tools * revert * venv * llvm18 is breaking macos12? * macos13 is monterey? * replace 12 with 13 * venv * revert action usage * use the correct name by force * readd pr filtering and tests --- .github/workflows/macos-build-and-deploy.yml | 80 +++++++++++++++++++- .install/activate_venv.sh | 2 +- .install/macos.sh | 33 +++++--- sbin/pack.sh | 1 + sbin/upload-artifacts | 1 + 5 files changed, 103 insertions(+), 14 deletions(-) diff --git a/.github/workflows/macos-build-and-deploy.yml b/.github/workflows/macos-build-and-deploy.yml index da2ada70a..7850c2f69 100644 --- a/.github/workflows/macos-build-and-deploy.yml +++ b/.github/workflows/macos-build-and-deploy.yml @@ -7,10 +7,15 @@ on: tags: - 'v[0-9]+.[0-9]+.[0-9]+' workflow_dispatch: # Allows you to run this workflow manually from the Actions tab + inputs: + redis-ref: + description: 'Redis ref to checkout' + required: true + default: 'unstable' jobs: - build: - runs-on: macos-12 + build-macos-x64: + runs-on: macos-13 defaults: run: shell: bash -l -eo pipefail {0} @@ -41,7 +46,7 @@ jobs: uses: actions/checkout@v4 with: repository: redis/redis - ref: 'unstable' # todo change per version/tag + ref: ${{ inputs.redis-ref }} path: redis - name: Build Redis working-directory: redis @@ -68,3 +73,72 @@ jobs: - name: Upload artifacts to S3 - release # todo: trigger this manually instead if: ${{ github.ref != 'refs/heads/master' }} run: make upload-release SHOW=1 VERBOSE=1 + + build-macos-m1: + runs-on: macos-latest-xlarge + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Deps checkout + uses: actions/checkout@v3 + with: + path: setup + sparse-checkout-cone-mode: false + sparse-checkout: | + .install + tests/pytest/requirements.* + - name: Setup specific + working-directory: setup/.install + run: ./install_script.sh + - name: Full checkout + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Setup common + run: | + echo ::group::Activate virtual environment + python3 -m venv venv + echo "source venv/bin/activate" >> ~/.bashrc + echo "source venv/bin/activate" >> ~/.zshrc + source venv/bin/activate + echo ::endgroup:: + echo ::group::Install python dependencies + ./.install/common_installations.sh + echo ::endgroup:: + + - name: Get Redis + uses: actions/checkout@v4 + with: + repository: redis/redis + ref: ${{ inputs.redis-ref }} + path: redis + - name: Build Redis + working-directory: redis + run: make install + - name: Build module + run: | + make build + - name: Test + run: | + make test + - name: Pack module + run: | + make pack BRANCH=${{ github.ref_name }} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: "us-east-1" + - name: Upload artifacts to S3 - staging + run: | + make upload-artifacts SHOW=1 VERBOSE=1 + make upload-release SHOW=1 STAGING=1 VERBOSE=1 + - name: Upload artifacts to S3 - release # todo: trigger this manually instead + if: ${{ github.ref != 'refs/heads/master' }} + run: make upload-release SHOW=1 VERBOSE=1 \ No newline at end of file diff --git a/.install/activate_venv.sh b/.install/activate_venv.sh index 6d6993a65..03e84ff58 100755 --- a/.install/activate_venv.sh +++ b/.install/activate_venv.sh @@ -12,4 +12,4 @@ activate_venv() { python3 -m venv venv activate_venv -source venv/bin/activate \ No newline at end of file +source venv/bin/activate diff --git a/.install/macos.sh b/.install/macos.sh index 2db98a96d..01108f543 100755 --- a/.install/macos.sh +++ b/.install/macos.sh @@ -1,20 +1,33 @@ #!/bin/bash +if ! which brew &> /dev/null; then + echo "Brew is not installed. Install from https://brew.sh" + exit 1 +fi + export HOMEBREW_NO_AUTO_UPDATE=1 -BREW_PREFIX=$(brew --prefix) -GNUBIN=$BREW_PREFIX/opt/make/libexec/gnubin -LLVM=$BREW_PREFIX/opt/llvm@16/bin -COREUTILS=$BREW_PREFIX/opt/coreutils/libexec/gnubin + +LLVM_VERSION="18" brew update brew install coreutils brew install make -brew install llvm@16 +brew install openssl +brew install llvm@$LLVM_VERSION -echo "export PATH=$COREUTILS:$LLVM:$GNUBIN:$PATH" >> ~/.bashrc -echo "export PATH=$COREUTILS:$LLVM:$GNUBIN:$PATH" >> ~/.zshrc -source ~/.bashrc -source ~/.zshrc +BREW_PREFIX=$(brew --prefix) +GNUBIN=$BREW_PREFIX/opt/make/libexec/gnubin +LLVM="$BREW_PREFIX/opt/llvm@$LLVM_VERSION/bin" +COREUTILS=$BREW_PREFIX/opt/coreutils/libexec/gnubin + +update_profile() { + local profile_path=$1 + local newpath="export PATH=$COREUTILS:$LLVM:$GNUBIN:\$PATH" + grep -qxF "$newpath" "$profile_path" || echo "$newpath" >> "$profile_path" + source $profile_path +} + +[[ -f ~/.bash_profile ]] && update_profile ~/.bash_profile +[[ -f ~/.zshrc ]] && update_profile ~/.zshrc -brew install openssl source install_cmake.sh diff --git a/sbin/pack.sh b/sbin/pack.sh index aeeae98bf..7abd60929 100755 --- a/sbin/pack.sh +++ b/sbin/pack.sh @@ -79,6 +79,7 @@ if [[ $OS == macos ]]; then OSNICK=catalina # to be aligned with the rest of the modules in redis stack else [[ $OSNICK == ventura ]] && OSNICK=monterey + [[ $OSNICK == sonoma ]] && OSNICK=monterey fi fi diff --git a/sbin/upload-artifacts b/sbin/upload-artifacts index d589418c5..99818dd6d 100755 --- a/sbin/upload-artifacts +++ b/sbin/upload-artifacts @@ -58,6 +58,7 @@ if [[ $OS == macos ]]; then OSNICK=catalina # to be aligned with the rest of the modules in redis stack else [[ $OSNICK == ventura ]] && OSNICK=monterey + [[ $OSNICK == sonoma ]] && OSNICK=monterey fi fi From 61826d8ec7529d19dc8c178f7d15ccc2718d30ea Mon Sep 17 00:00:00 2001 From: efeldblum <109016390+ephraimfeldblum@users.noreply.github.com> Date: Tue, 15 Oct 2024 11:10:44 +0300 Subject: [PATCH 089/112] Enabling the compiler to better see through `do_op` (#1267) * initial commit * . --- redis_json/src/ivalue_manager.rs | 558 +++++++++++-------------------- 1 file changed, 187 insertions(+), 371 deletions(-) diff --git a/redis_json/src/ivalue_manager.rs b/redis_json/src/ivalue_manager.rs index 2bb97e21b..90abd50b3 100644 --- a/redis_json/src/ivalue_manager.rs +++ b/redis_json/src/ivalue_manager.rs @@ -11,7 +11,6 @@ use crate::redisjson::normalize_arr_start_index; use crate::Format; use crate::REDIS_JSON_TYPE; use bson::{from_document, Document}; -use ijson::object::Entry; use ijson::{DestructuredMut, INumber, IObject, IString, IValue, ValueType}; use json_path::select_value::{SelectValue, SelectValueType}; use redis_module::key::{verify_type, KeyFlags, RedisKey, RedisKeyWritable}; @@ -34,205 +33,89 @@ pub struct IValueKeyHolderWrite<'a> { val: Option<&'a mut RedisJSON>, } -/// -/// Replaces a value at a given `path`, starting from `root` -/// -/// The new value is the value returned from `func`, which is called on the current value. -/// -/// If the returned value from `func` is [`None`], the current value is removed. -/// If the returned value from `func` is [`Err`], the current value remains (although it could be modified by `func`) -/// -fn replace Result, Error>>( - path: &[String], - root: &mut IValue, - mut func: F, -) -> Result<(), Error> { - let mut target = root; - - let last_index = path.len().saturating_sub(1); - for (i, token) in path.iter().enumerate() { - let target_once = target; - let is_last = i == last_index; - let target_opt = match target_once.type_() { - ValueType::Object => { - let obj = target_once.as_object_mut().unwrap(); - if is_last { - if let Entry::Occupied(mut e) = obj.entry(token) { - let v = e.get_mut(); - if let Some(res) = func(v)? { - *v = res; - } else { - e.remove(); - } - } - return Ok(()); - } - obj.get_mut(token.as_str()) - } - ValueType::Array => { - let arr = target_once.as_array_mut().unwrap(); - let idx = token.parse::().expect(&format!( - "An array index is parsed successfully. Array = {:?}, index = {:?}", - arr, token - )); - if is_last { - if idx < arr.len() { - let v = &mut arr.as_mut_slice()[idx]; - if let Some(res) = func(v)? { - *v = res; - } else { - arr.remove(idx); - } - } - return Ok(()); - } - arr.get_mut(idx) - } +fn follow_path(path: Vec, root: &mut IValue) -> Option<&mut IValue> { + path.into_iter() + .try_fold(root, |target, token| match target.destructure_mut() { + DestructuredMut::Object(obj) => obj.get_mut(token.as_str()), + DestructuredMut::Array(arr) => arr.get_mut(token.parse::().ok()?), _ => None, - }; - - if let Some(t) = target_opt { - target = t; - } else { - break; - } - } - - Ok(()) + }) } /// /// Updates a value at a given `path`, starting from `root` /// /// The value is modified by `func`, which is called on the current value. -/// If the returned value from `func` is [`None`], the current value is removed. /// If the returned value from `func` is [`Err`], the current value remains (although it could be modified by `func`) /// -fn update Result, Error>>( - path: &[String], - root: &mut IValue, - mut func: F, -) -> Result<(), Error> { - let mut target = root; - - let last_index = path.len().saturating_sub(1); - for (i, token) in path.iter().enumerate() { - let target_once = target; - let is_last = i == last_index; - let target_opt = match target_once.type_() { - ValueType::Object => { - let obj = target_once.as_object_mut().unwrap(); - if is_last { - if let Entry::Occupied(mut e) = obj.entry(token) { - let v = e.get_mut(); - if func(v)?.is_none() { - e.remove(); - } - } - return Ok(()); - } - obj.get_mut(token.as_str()) - } - ValueType::Array => { - let arr = target_once.as_array_mut().unwrap(); - let idx = token.parse::().expect(&format!( - "An array index is parsed successfully. Array = {:?}, index = {:?}", - arr, token - )); - if is_last { - if idx < arr.len() { - let v = &mut arr.as_mut_slice()[idx]; - if func(v)?.is_none() { - arr.remove(idx); - } - } - return Ok(()); - } - arr.get_mut(idx) - } - _ => None, - }; - - if let Some(t) = target_opt { - target = t; - } else { - break; - } - } +fn update(path: Vec, root: &mut IValue, func: F) -> RedisResult +where + F: FnOnce(&mut IValue) -> RedisResult, +{ + follow_path(path, root).map_or_else( + || Err(RedisError::String(err_msg_json_path_doesnt_exist())), + func, + ) +} - Ok(()) +/// +/// Removes a value at a given `path`, starting from `root` +/// +fn remove(mut path: Vec, root: &mut IValue) -> bool { + let token = path.pop().unwrap(); + follow_path(path, root) + .and_then(|target| match target.destructure_mut() { + DestructuredMut::Object(obj) => obj.remove(token.as_str()), + DestructuredMut::Array(arr) => arr.remove(token.parse::().ok()?), + _ => None, + }) + .is_some() } impl<'a> IValueKeyHolderWrite<'a> { - fn do_op(&mut self, paths: &[String], mut op_fun: F) -> Result<(), RedisError> + fn do_op(&mut self, paths: Vec, op_fun: F) -> RedisResult where - F: FnMut(&mut IValue) -> Result, Error>, + F: FnOnce(&mut IValue) -> RedisResult, { - if paths.is_empty() { - // updating the root require special treatment - let root = self.get_value().unwrap().unwrap(); - if op_fun(root) - .map_err(|err| RedisError::String(err.msg))? - .is_none() - { - root.take(); - } - } else { - update(paths, self.get_value().unwrap().unwrap(), op_fun)?; - } - - Ok(()) + let root = self.get_value()?.unwrap(); + update(paths, root, op_fun) } fn do_num_op( &mut self, path: Vec, num: &str, - mut op1_fun: F1, - mut op2_fun: F2, - ) -> Result + op1: F1, + op2: F2, + ) -> RedisResult where - F1: FnMut(i64, i64) -> i64, - F2: FnMut(f64, f64) -> f64, + F1: FnOnce(i64, i64) -> i64, + F2: FnOnce(f64, f64) -> f64, { let in_value = &serde_json::from_str(num)?; if let serde_json::Value::Number(in_value) = in_value { - let mut res = None; - self.do_op(&path, |v| { - let num_res = match (v.get_type(), in_value.as_i64()) { + let n = self.do_op(path, |v| { + let new_val = match (v.get_type(), in_value.as_i64()) { (SelectValueType::Long, Some(num2)) => { let num1 = v.get_long(); - let res = op1_fun(num1, num2); - Ok(res.into()) + Ok(op1(num1, num2).into()) } _ => { let num1 = v.get_double(); let num2 = in_value.as_f64().unwrap(); - INumber::try_from(op2_fun(num1, num2)) + INumber::try_from(op2(num1, num2)) .map_err(|_| RedisError::Str("result is not a number")) } - }; - let new_val = IValue::from(num_res?); - *v = new_val.clone(); - res = Some(new_val); - Ok(Some(())) + }?; + *v = IValue::from(new_val.clone()); + Ok(new_val) })?; - match res { - None => Err(RedisError::String(err_msg_json_path_doesnt_exist())), - Some(n) => { - if let Some(n) = n.as_number() { - if !n.has_decimal_point() { - Ok(n.to_i64().unwrap().into()) - } else if let Some(f) = n.to_f64() { - Ok(serde_json::Number::from_f64(f).unwrap()) - } else { - Err(RedisError::Str("result is not a number")) - } - } else { - Err(RedisError::Str("result is not a number")) - } - } + if n.has_decimal_point() { + n.to_f64().and_then(serde_json::Number::from_f64) + } else { + n.to_i64().map(Into::into) } + .ok_or_else(|| RedisError::Str("result is not a number")) } else { Err(RedisError::Str("bad input number")) } @@ -245,23 +128,14 @@ impl<'a> IValueKeyHolderWrite<'a> { Ok(()) } - fn set_root(&mut self, v: Option) -> Result<(), RedisError> { - match v { - Some(inner) => { - self.get_json_holder()?; - match &mut self.val { - Some(v) => v.data = inner, - None => self - .key - .set_value(&REDIS_JSON_TYPE, RedisJSON { data: inner })?, - } - } - None => { - self.val = None; - self.key.delete()?; - } + fn set_root(&mut self, data: IValue) -> RedisResult { + self.get_json_holder()?; + if let Some(val) = &mut self.val { + val.data = data + } else { + self.key.set_value(&REDIS_JSON_TYPE, RedisJSON { data })? } - Ok(()) + Ok(true) } } @@ -288,235 +162,170 @@ impl<'a> WriteHolder for IValueKeyHolderWrite<'a> { } } - fn set_value(&mut self, path: Vec, mut v: IValue) -> Result { - let mut updated = false; + fn set_value(&mut self, path: Vec, mut v: IValue) -> RedisResult { if path.is_empty() { // update the root - self.set_root(Some(v))?; - updated = true; + self.set_root(v) } else { - replace(&path, self.get_value()?.unwrap(), |_v| { - updated = true; - Ok(Some(v.take())) - })?; + let root = self.get_value()?.unwrap(); + Ok(update(path, root, |val| Ok(*val = v.take())).is_ok()) } - Ok(updated) } - fn merge_value(&mut self, path: Vec, v: IValue) -> Result { - let mut updated = false; - if path.is_empty() { - merge(self.get_value()?.unwrap(), &v); - // update the root - updated = true; - } else { - replace(&path, self.get_value()?.unwrap(), |current| { - updated = true; - merge(current, &v); - Ok(Some(current.take())) - })?; - } - Ok(updated) + fn merge_value(&mut self, path: Vec, mut v: IValue) -> RedisResult { + let root = self.get_value()?.unwrap(); + Ok(update(path, root, |current| Ok(merge(current, v.take()))).is_ok()) } - fn dict_add( - &mut self, - path: Vec, - key: &str, - mut v: IValue, - ) -> Result { - let mut updated = false; - if path.is_empty() { - // update the root - let root = self.get_value().unwrap().unwrap(); - if let Some(o) = root.as_object_mut() { - if !o.contains_key(key) { - updated = true; + fn dict_add(&mut self, path: Vec, key: &str, mut v: IValue) -> RedisResult { + self.do_op(path, |val| { + val.as_object_mut().map_or(Ok(false), |o| { + let res = !o.contains_key(key); + if res { o.insert(key.to_string(), v.take()); } - } - } else { - update(&path, self.get_value().unwrap().unwrap(), |val| { - if val.is_object() { - let o = val.as_object_mut().unwrap(); - if !o.contains_key(key) { - updated = true; - o.insert(key.to_string(), v.take()); - } - } - Ok(Some(())) - })?; - } - Ok(updated) + Ok(res) + }) + }) } - fn delete_path(&mut self, path: Vec) -> Result { - let mut deleted = false; - update(&path, self.get_value().unwrap().unwrap(), |_v| { - deleted = true; // might delete more than a single value - Ok(None) - })?; - Ok(deleted) + fn delete_path(&mut self, path: Vec) -> RedisResult { + self.get_value().map(|root| remove(path, root.unwrap())) } - fn incr_by(&mut self, path: Vec, num: &str) -> Result { + fn incr_by(&mut self, path: Vec, num: &str) -> RedisResult { self.do_num_op(path, num, i64::wrapping_add, |f1, f2| f1 + f2) } - fn mult_by(&mut self, path: Vec, num: &str) -> Result { + fn mult_by(&mut self, path: Vec, num: &str) -> RedisResult { self.do_num_op(path, num, i64::wrapping_mul, |f1, f2| f1 * f2) } - fn pow_by(&mut self, path: Vec, num: &str) -> Result { + fn pow_by(&mut self, path: Vec, num: &str) -> RedisResult { self.do_num_op(path, num, |i1, i2| i1.pow(i2 as u32), f64::powf) } - fn bool_toggle(&mut self, path: Vec) -> Result { - let mut res = None; - self.do_op(&path, |v| { + fn bool_toggle(&mut self, path: Vec) -> RedisResult { + self.do_op(path, |v| { if let DestructuredMut::Bool(mut bool_mut) = v.destructure_mut() { //Using DestructuredMut in order to modify a `Bool` variant let val = bool_mut.get() ^ true; bool_mut.set(val); - res = Some(val); + Ok(val) + } else { + Err(err_json(v, "bool").into()) } - Ok(Some(())) - })?; - res.ok_or_else(|| RedisError::String(err_msg_json_path_doesnt_exist())) - } - - fn str_append(&mut self, path: Vec, val: String) -> Result { - let json = serde_json::from_str(&val)?; - if let serde_json::Value::String(s) = json { - let mut res = None; - self.do_op(&path, |v| { - let v_str = v.as_string_mut().unwrap(); - let new_str = [v_str.as_str(), s.as_str()].concat(); - res = Some(new_str.len()); - *v_str = IString::intern(&new_str); - Ok(Some(())) - })?; - res.ok_or_else(|| RedisError::String(err_msg_json_path_doesnt_exist())) - } else { - Err(RedisError::String(err_msg_json_expected( + }) + } + + fn str_append(&mut self, path: Vec, val: String) -> RedisResult { + match serde_json::from_str(&val)? { + serde_json::Value::String(s) => self.do_op(path, |v| { + v.as_string_mut() + .map(|v_str| { + let new_str = [v_str.as_str(), s.as_str()].concat(); + *v_str = IString::intern(&new_str); + Ok(new_str.len()) + }) + .unwrap_or_else(|| Err(err_json(v, "string").into())) + }), + _ => Err(RedisError::String(err_msg_json_expected( "string", val.as_str(), - ))) + ))), } } - fn arr_append(&mut self, path: Vec, args: Vec) -> Result { - let mut res = None; - self.do_op(&path, |v| { - let arr = v.as_array_mut().unwrap(); - for a in &args { - arr.push(a.clone()); - } - res = Some(arr.len()); - Ok(Some(())) - })?; - res.ok_or_else(|| RedisError::String(err_msg_json_path_doesnt_exist())) + fn arr_append(&mut self, path: Vec, args: Vec) -> RedisResult { + self.do_op(path, |v| { + v.as_array_mut() + .map(|arr| { + arr.extend(args); + Ok(arr.len()) + }) + .unwrap_or_else(|| Err(err_json(v, "array").into())) + }) } - fn arr_insert( - &mut self, - paths: Vec, - args: &[IValue], - index: i64, - ) -> Result { - let mut res = None; - self.do_op(&paths, |v: &mut IValue| { - // Verify legal index in bounds - let len = v.len().unwrap() as i64; - let index = if index < 0 { len + index } else { index }; - if !(0..=len).contains(&index) { - return Err("ERR index out of bounds".into()); - } - let index = index as usize; - let curr = v.as_array_mut().unwrap(); - curr.extend(args.iter().cloned()); - curr[index..].rotate_right(args.len()); - res = Some(curr.len()); - Ok(Some(())) - })?; - res.ok_or_else(|| RedisError::String(err_msg_json_path_doesnt_exist())) + fn arr_insert(&mut self, paths: Vec, args: &[IValue], idx: i64) -> RedisResult { + self.do_op(paths, |v| { + v.as_array_mut() + .map(|arr| { + // Verify legal index in bounds + let len = arr.len() as _; + let idx = if idx < 0 { len + idx } else { idx }; + if !(0..=len).contains(&idx) { + return Err(RedisError::Str("ERR index out of bounds")); + } + arr.extend(args.iter().cloned()); + arr[idx as _..].rotate_right(args.len()); + Ok(arr.len()) + }) + .unwrap_or_else(|| Err(err_json(v, "array").into())) + }) } - fn arr_pop) -> RedisResult>( - &mut self, - path: Vec, - index: i64, - serialize_callback: C, - ) -> RedisResult { - let mut res = None; - self.do_op(&path, |v| { - if let Some(array) = v.as_array_mut() { - if array.is_empty() { - return Ok(Some(())); - } - // Verify legal index in bounds - let len = array.len() as i64; - let index = normalize_arr_start_index(index, len) as usize; - res = Some(array.remove(index).unwrap()); - Ok(Some(())) - } else { - Err(err_json(v, "array")) - } + fn arr_pop(&mut self, path: Vec, index: i64, serialize_callback: C) -> RedisResult + where + C: FnOnce(Option<&IValue>) -> RedisResult, + { + let res = self.do_op(path, |v| { + v.as_array_mut() + .map(|array| { + if array.is_empty() { + return None; + } + // Verify legal index in bounds + let len = array.len() as i64; + let index = normalize_arr_start_index(index, len) as usize; + array.remove(index) + }) + .ok_or_else(|| err_json(v, "array").into()) })?; serialize_callback(res.as_ref()) } - fn arr_trim(&mut self, path: Vec, start: i64, stop: i64) -> Result { - let mut res = None; - self.do_op(&path, |v| { - if let Some(array) = v.as_array_mut() { - let len = array.len() as i64; - let stop = stop.normalize(len); - let start = if start < 0 || start < len { - start.normalize(len) - } else { - stop + 1 // start >=0 && start >= len - }; - let range = if start > stop || len == 0 { - 0..0 // Return an empty array - } else { - start..(stop + 1) - }; - - array.rotate_left(range.start); - array.truncate(range.end - range.start); - res = Some(array.len()); - Ok(Some(())) - } else { - Err(err_json(v, "array")) - } - })?; - res.ok_or_else(|| RedisError::String(err_msg_json_path_doesnt_exist())) + fn arr_trim(&mut self, path: Vec, start: i64, stop: i64) -> RedisResult { + self.do_op(path, |v| { + v.as_array_mut() + .map(|array| { + let len = array.len() as i64; + let stop = stop.normalize(len); + let start = if start < 0 || start < len { + start.normalize(len) + } else { + stop + 1 // start >=0 && start >= len + }; + let range = if start > stop || len == 0 { + 0..0 // Return an empty array + } else { + start..(stop + 1) + }; + + array.rotate_left(range.start); + array.truncate(range.end - range.start); + array.len() + }) + .ok_or_else(|| err_json(v, "array").into()) + }) } - fn clear(&mut self, path: Vec) -> Result { - let mut cleared = 0; - self.do_op(&path, |v| match v.type_() { - ValueType::Object => { - let obj = v.as_object_mut().unwrap(); + fn clear(&mut self, path: Vec) -> RedisResult { + self.do_op(path, |v| match v.destructure_mut() { + DestructuredMut::Object(obj) => { obj.clear(); - cleared += 1; - Ok(Some(())) + Ok(1) } - ValueType::Array => { - let arr = v.as_array_mut().unwrap(); + DestructuredMut::Array(arr) => { arr.clear(); - cleared += 1; - Ok(Some(())) + Ok(1) } - ValueType::Number => { - *v = IValue::from(0); - cleared += 1; - Ok(Some(())) + DestructuredMut::Number(n) => { + *n = INumber::from(0); + Ok(1) } - _ => Ok(Some(())), - })?; - Ok(cleared) + _ => Ok(0), + }) } } @@ -531,9 +340,9 @@ impl ReadHolder for IValueKeyHolderRead { } } -fn merge(doc: &mut IValue, patch: &IValue) { +fn merge(doc: &mut IValue, mut patch: IValue) { if !patch.is_object() { - *doc = patch.clone(); + *doc = patch; return; } @@ -541,13 +350,20 @@ fn merge(doc: &mut IValue, patch: &IValue) { *doc = IObject::new().into(); } let map = doc.as_object_mut().unwrap(); - for (key, value) in patch.as_object().unwrap() { - if value.is_null() { - map.remove(key.as_str()); - } else { - merge(map.entry(key.as_str()).or_insert(IValue::NULL), value); - } - } + patch + .as_object_mut() + .unwrap() + .into_iter() + .for_each(|(key, value)| { + if value.is_null() { + map.remove(key.as_str()); + } else { + merge( + map.entry(key.as_str()).or_insert(IValue::NULL), + value.take(), + ) + } + }) } pub struct RedisIValueJsonKeyManager<'a> { From d65d9a482ef7c2d00c582b025cca827be6d3d516 Mon Sep 17 00:00:00 2001 From: "Meir Shpilraien (Spielrein)" Date: Thu, 7 Nov 2024 20:15:00 +0200 Subject: [PATCH 090/112] RED-137916 Enable thread safe on bigredis (#1293) --- Cargo.lock | 2 +- Cargo.toml | 2 +- redis_json/src/lib.rs | 12 ++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 02167b4a1..a4fb86fbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -377,7 +377,7 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "ijson" version = "0.1.3" -source = "git+https://github.com/RedisJSON/ijson?rev=eede48fad51b4ace5043d3e0714f5a65481a065d#eede48fad51b4ace5043d3e0714f5a65481a065d" +source = "git+https://github.com/RedisJSON/ijson?rev=881b96e7941b8dc335863746ac108f6d9ed0b98a#881b96e7941b8dc335863746ac108f6d9ed0b98a" dependencies = [ "dashmap", "hashbrown 0.13.2", diff --git a/Cargo.toml b/Cargo.toml index cdb7c91d6..fb0919361 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ ] [workspace.dependencies] -ijson = { git="https://github.com/RedisJSON/ijson", rev="eede48fad51b4ace5043d3e0714f5a65481a065d", default_features=false} +ijson = { git="https://github.com/RedisJSON/ijson", rev="881b96e7941b8dc335863746ac108f6d9ed0b98a", default_features=false} serde_json = { version="1", features = ["unbounded_depth"]} serde = { version = "1", features = ["derive"] } serde_derive = "1" diff --git a/redis_json/src/lib.rs b/redis_json/src/lib.rs index f5b75caf7..c38e3c846 100644 --- a/redis_json/src/lib.rs +++ b/redis_json/src/lib.rs @@ -122,6 +122,7 @@ macro_rules! redis_json_module_create {( use rawmod::ModuleOptions; use redis_module::redis_module; use redis_module::logging::RedisLogLevel; + use redis_module::RedisValue; use std::{ ffi::{CStr, CString}, @@ -171,6 +172,17 @@ macro_rules! redis_json_module_create {( export_shared_api(ctx); ctx.set_module_options(ModuleOptions::HANDLE_IO_ERRORS); ctx.log_notice("Enabled diskless replication"); + let is_bigredis = + ctx.call("config", &["get", "bigredis-enabled"]) + .map_or(false, |res| match res { + RedisValue::Array(a) => !a.is_empty(), + _ => false, + }); + ctx.log_notice(&format!("Initialized shared string cache, thread safe: {is_bigredis}.")); + if let Err(e) = ijson::init_shared_string_cache(is_bigredis) { + ctx.log(RedisLogLevel::Warning, &format!("Failed initializing shared string cache, {e}.")); + return Status::Err; + } $init_func(ctx, args) } From fa4f4546960e65a22551e01a7a2b7bf41b7a8689 Mon Sep 17 00:00:00 2001 From: efeldblum <109016390+ephraimfeldblum@users.noreply.github.com> Date: Sun, 10 Nov 2024 16:59:24 +0200 Subject: [PATCH 091/112] MOD-7499 Add JSON ACL category (#1291) * Add JSON ACL category * delete redundant file * .into() * extend temporary * skip on redis version < 7.4 * skip on redis version < 7.4 * skip on redis version < 7.4 * skip on redis version < 7.4 * skip on redis version < 7.4 * using latest `redismodule-rs` * add optional ACL category * skip test because Redis is broken * using latest redismodule-rs * 7.4.1 -> 7.4 * undo circleci change * udpate redismodule-rs version to 2.1.0 * remove .circleci * update github to stop running circleci * reintroducing config.yml --------- Co-authored-by: Eran Hadad --- .circleci/config.yml | 1106 ++++++++--------- .../action.yml | 2 +- .../build-json-module-and-redis/action.yml | 2 +- .github/actions/setup-env/action.yml | 2 +- .github/workflows/alpine.yml | 2 +- .github/workflows/mariner2.yml | 2 +- .github/workflows/ubuntu.yml | 2 +- Cargo.lock | 227 ++-- build/docker/Makefile | 2 +- ramp.yml | 4 +- redis_json/Cargo.toml | 4 +- redis_json/src/lib.rs | 53 +- tests/pytest/common.py | 33 + tests/pytest/test_acl.py | 83 ++ tests/pytest/tests.sh | 6 +- 15 files changed, 832 insertions(+), 698 deletions(-) create mode 100644 tests/pytest/test_acl.py diff --git a/.circleci/config.yml b/.circleci/config.yml index e05bd01bc..b2b6c1008 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,553 +1,553 @@ -version: 2.1 - -parameters: - run_default_flow: - default: true - type: boolean - run_nightly_twice_a_week_flow_label: - default: false - type: boolean - -commands: - early-returns: - steps: - - run: - name: Early return if this is a docs build - command: | - if [[ $CIRCLE_BRANCH == *docs ]]; then - echo "Identifies as documents PR, no testing required." - circleci step halt - fi - - run: - name: Early return if this branch should ignore CI - command: | - if [[ $CIRCLE_BRANCH == *noci ]]; then - echo "Identifies as actively ignoring CI, no testing required." - circleci step halt - fi - - early-return-for-forked-pull-requests: - description: >- - If this build is from a fork, stop executing the current job and return success. - This is useful to avoid steps that will fail due to missing credentials. - steps: - - run: - name: Early return if this build is from a forked PR - command: | - if [[ -n "$CIRCLE_PR_NUMBER" ]]; then - echo "Nothing to do for forked PRs, so marking this step successful" - circleci step halt - fi - - setup-executor: - steps: - - run: - name: Setup executor - command: | - apt-get -qq update - apt-get -q install -y git openssh-client curl ca-certificates make tar gzip - bash <(curl -fsSL https://raw.githubusercontent.com/docker/docker-install/master/install.sh) - - setup_remote_docker: - version: 20.10.14 - docker_layer_caching: true - - checkout-all: - steps: - - checkout - - run: - name: Checkout submodules - command: git submodule update --init --recursive - - setup-automation: - steps: - - run: - name: Setup automation - command: | - git submodule update --init deps/readies - if [[ $(uname -s) == Darwin ]]; then rm -f /usr/local/bin/python3; fi - ./deps/readies/bin/getpy3 - - run: - name: Setup automation (part 2) - shell: /bin/bash -l -eo pipefail - command: | - export HOMEBREW_NO_AUTO_UPDATE=1 - ./deps/readies/bin/getaws - ls -l /usr/local/bin/python* || true - echo "python3: $(command -v python3)" - python3 --version - python3 -m pip list - python3 -m pip install --upgrade setuptools six pip - - install-prerequisites: - parameters: - redis_version: - type: string - default: "7.2" - getredis_params: - type: string - default: "" - system_setup_params: - type: string - default: "" - steps: - - setup-automation - - run: - name: System setup - shell: /bin/bash -l -eo pipefail - command: | - ./sbin/system-setup.py <> - - run: - name: Install Redis - shell: /bin/bash -l -eo pipefail - command: | - export HOMEBREW_NO_AUTO_UPDATE=1 - ./deps/readies/bin/getredis -v '<>' --force <> - - run: - name: System report - shell: /bin/bash -l -eo pipefail - command: | - source $HOME/.cargo/env - make info - - save-tests-logs: - steps: - - run: - name: Cleanup test log dir - command: | - rm -f tests/pytest/logs/*.{aof,rdb} - when: always - - store_artifacts: - path: tests/pytest/logs - - persist-artifacts: - steps: - - early-return-for-forked-pull-requests - - run: - name: List artifacts - command: | - cd bin/artifacts - du -ah --apparent-size * - - persist_to_workspace: - root: bin/ - paths: - - artifacts/*.zip - - artifacts/*.tgz - - artifacts/*.tar - - artifacts/snapshots/* - - build-steps: - parameters: - build_params: - type: string - default: "" - test_params: - type: string - default: "" - redis_version: - type: string - default: "7" - getredis_params: - type: string - default: "" - steps: - - early-returns - - checkout-all - - install-prerequisites: - redis_version: <> - getredis_params: <> - - restore_cache: - keys: - - v3-dependencies-{{ arch }}-{{ checksum "Cargo.lock" }} - - run: - name: Check formatting - shell: /bin/bash -l -eo pipefail - command: make lint - - run: - name: Build debug - shell: /bin/bash -l -eo pipefail - command: make build DEBUG=1 <> - - run: - name: Build release - shell: /bin/bash -l -eo pipefail - command: make build pack <> - - save_cache: - key: v3-dependencies-{{ arch }}-{{ checksum "Cargo.lock" }} - paths: - - "~/.cargo" - - "./target" - - test-steps: - steps: - - run: - name: Run tests - shell: /bin/bash -l -eo pipefail - command: | - python3 -m RLTest --version - make test - timeout: 30m - no_output_timeout: 30m - - save_cache: - key: v2-dependencies-{{ arch }}-{{ checksum "Cargo.lock" }} - paths: - - "~/.cargo" - - "./target" - - save-tests-logs - - build-platforms-steps: - parameters: - platform: - type: string - steps: - - early-returns - - setup-executor - - checkout-all - - setup-automation - - run: - name: Build for platform - shell: /bin/bash -l -eo pipefail - command: | - ROOT=$PWD - cd build/docker - make build OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH TEST=1 OFFICIAL=1 SHOW=1 - cd $ROOT - mkdir -p tests/pytest/logs - tar -C tests/pytest/logs -xzf bin/artifacts/pytest-logs*.tgz - timeout: 60m - no_output_timeout: 30m - - save-tests-logs - - early-return-for-forked-pull-requests - - run: - name: Upload artifacts to S3 - shell: /bin/bash -l -eo pipefail - command: | - if [[ -n $CIRCLE_BRANCH ]]; then - make upload-artifacts OSNICK=<> SHOW=1 - fi - - run: - name: Publish container - shell: /bin/bash -l -eo pipefail - command: | - docker login -u redisfab -p $DOCKER_REDISFAB_PWD - cd build/docker - make publish OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH OFFICIAL=1 SHOW=1 - - persist-artifacts - - vm-build-platforms-steps: - parameters: - platform: - type: string - steps: - - early-returns - - checkout - - setup-automation - - run: - name: Install Docker - shell: /bin/bash -l -eo pipefail - command: ./deps/readies/bin/getdocker - - run: - name: Build for platform - command: | - ROOT=$PWD - cd build/docker - make build OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH TEST=1 OFFICIAL=1 SHOW=1 - cd $ROOT - mkdir -p tests/pytest/logs - tar -C tests/pytest/logs -xzf bin/artifacts/pytest-logs*.tgz - no_output_timeout: 30m - - save-tests-logs - - early-return-for-forked-pull-requests - - run: - name: Upload artifacts to S3 - command: | - if [[ -n $CIRCLE_BRANCH ]]; then - make upload-artifacts OSNICK=<> SHOW=1 - fi - - run: - name: Publish container - command: | - docker login -u redisfab -p $DOCKER_REDISFAB_PWD - cd build/docker - make publish OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH OFFICIAL=1 SHOW=1 - - persist-artifacts - -#---------------------------------------------------------------------------------------------------------------------------------- - -jobs: - build-linux-debian: - docker: - - image: redisfab/rmbuilder:6.2.7-x64-bullseye - parameters: - redis_version: - type: string - default: "7" - persist: - type: string - default: "yes" - steps: - - build-steps: - redis_version: <> - - test-steps - - run: - name: Persist artifacts? - command: | - if [[ "<>" != "yes" ]]; then - circleci step halt - fi - - persist-artifacts - - build-platforms: - parameters: - platform: - type: string - # docker: - # - image: debian:bullseye - machine: - enabled: true - image: ubuntu-2204:edge - resource_class: large - steps: - - vm-build-platforms-steps: - platform: <> - - # build-arm-platforms: - # parameters: - # platform: - # type: string - # machine: - # image: ubuntu-2004:202101-01 - # resource_class: arm.medium - # steps: - # - vm-build-platforms-steps: - # platform: <> - - build-macos-m1: - macos: - xcode: 15.4.0 - resource_class: macos.m1.large.gen1 - parameters: - upload: - type: string - default: "yes" - steps: - - early-returns - - build-steps - - test-steps - - persist-artifacts - - run: - name: Upload artifacts to S3 - command: | - if [[ -n $CIRCLE_BRANCH ]]; then - make upload-artifacts SHOW=1 VERBOSE=1 - else - make upload-release SHOW=1 - fi - - coverage: - docker: - - image: redisfab/rmbuilder:6.2.7-x64-focal - steps: - - early-returns - - checkout-all - - install-prerequisites - - run: - name: Build & Test - shell: /bin/bash -l -eo pipefail - command: | - make clang-install - make coverage SHOW=1 - make upload-cov SHOW=1 - no_output_timeout: 30m - - save-tests-logs - - sanitize: - docker: - - image: redisfab/clang:16-x64-focal - parameters: - san-type: - type: string - steps: - - early-returns - - checkout-all - - install-prerequisites - - run: - name: Build & test - shell: /bin/bash -l -eo pipefail - command: make SAN=<> build test SHOW=1 - no_output_timeout: 30m - - save-tests-logs - - upload-artifacts: - parameters: - staging-lab: - type: string - default: "0" - docker: - - image: redisfab/rmbuilder:6.2.7-x64-bullseye - steps: - - early-returns - - early-return-for-forked-pull-requests - - checkout - - setup-automation - - attach_workspace: - at: ~/workspace - - run: - name: Upload artifacts to S3 - command: | - mkdir -p bin - ln -s ~/workspace/artifacts bin/artifacts - if [[ -n $CIRCLE_TAG ]]; then - make upload-release SHOW=1 - else - make upload-artifacts SHOW=1 - fi - -#---------------------------------------------------------------------------------------------------------------------------------- - -on-any-branch: &on-any-branch - filters: - branches: - only: /.*/ - tags: - only: /.*/ - -always: &always - filters: - branches: - only: /.*/ - tags: - only: /.*/ - -never: &never - filters: - branches: - ignore: /.*/ - tags: - ignore: /.*/ - -on-master: &on-master - filters: - branches: - only: master - tags: - ignore: /.*/ - -on-integ-branch: &on-integ-branch - filters: - branches: - only: - - master - - /^\d+\.\d+.*$/ - - /^feature.*$/ - tags: - ignore: /.*/ - -on-integ-branch-cron: &on-integ-branch-cron - filters: - branches: - only: - - master - - /^\d+\.\d+.*$/ - - /^feature.*$/ - -not-on-integ-branch: ¬-on-integ-branch - filters: - branches: - ignore: - - master - - /^\d+\.\d+.*$/ - - /^feature.*$/ - tags: - ignore: /.*/ - -on-version-tags: &on-version-tags - filters: - branches: - ignore: /.*/ - tags: - only: /^v[0-9].*/ - -on-integ-and-version-tags: &on-integ-and-version-tags - filters: - branches: - only: - - master - - /^\d+\.\d+.*$/ - - /^feature.*$/ - tags: - only: /^v[0-9].*/ - -#---------------------------------------------------------------------------------------------------------------------------------- - -workflows: - version: 2 - default-flow: - when: - << pipeline.parameters.run_default_flow >> - jobs: - - build-linux-debian: - name: build - <<: *not-on-integ-branch - - build-platforms: - <<: *on-integ-and-version-tags - context: common - matrix: - parameters: - platform: [centos7, rocky8, rocky9, bullseye, amzn2] - - build-macos-m1: - context: common - <<: *on-integ-and-version-tags - - coverage: - <<: *always - - sanitize: - name: sanitize-<< matrix.san-type >> - <<: *always - matrix: - parameters: - san-type: [address] - - upload-artifacts: - name: upload-artifacts-to-staging-lab - <<: *on-integ-branch - staging-lab: "1" - context: common - requires: - - build-platforms - # - build-arm-platforms - - build-macos-m1 - - upload-artifacts: - name: upload-release-artifacts - <<: *on-version-tags - context: common - requires: - - build-platforms - # - build-arm-platforms - - build-macos-m1 - - nightly: - triggers: - - schedule: - cron: "07 20 * * *" - <<: *on-integ-branch-cron - jobs: - - build-linux-debian: - name: build-with-redis-<> - matrix: - parameters: - redis_version: ["7", "unstable"] - - nightly-twice-a-week-by-param: - when: - << pipeline.parameters.run_nightly_twice_a_week_flow_label >> - jobs: - - build-macos-m1: - context: common - upload: "yes" - - nightly-twice-a-week: - triggers: - - schedule: - cron: "20 17 * * 0,3" - <<: *on-integ-branch-cron - jobs: - - build-macos-m1: - context: common - upload: "yes" +# version: 2.1 + +# parameters: +# run_default_flow: +# default: true +# type: boolean +# run_nightly_twice_a_week_flow_label: +# default: false +# type: boolean + +# commands: +# early-returns: +# steps: +# - run: +# name: Early return if this is a docs build +# command: | +# if [[ $CIRCLE_BRANCH == *docs ]]; then +# echo "Identifies as documents PR, no testing required." +# circleci step halt +# fi +# - run: +# name: Early return if this branch should ignore CI +# command: | +# if [[ $CIRCLE_BRANCH == *noci ]]; then +# echo "Identifies as actively ignoring CI, no testing required." +# circleci step halt +# fi + +# early-return-for-forked-pull-requests: +# description: >- +# If this build is from a fork, stop executing the current job and return success. +# This is useful to avoid steps that will fail due to missing credentials. +# steps: +# - run: +# name: Early return if this build is from a forked PR +# command: | +# if [[ -n "$CIRCLE_PR_NUMBER" ]]; then +# echo "Nothing to do for forked PRs, so marking this step successful" +# circleci step halt +# fi + +# setup-executor: +# steps: +# - run: +# name: Setup executor +# command: | +# apt-get -qq update +# apt-get -q install -y git openssh-client curl ca-certificates make tar gzip +# bash <(curl -fsSL https://raw.githubusercontent.com/docker/docker-install/master/install.sh) +# - setup_remote_docker: +# version: 20.10.14 +# docker_layer_caching: true + +# checkout-all: +# steps: +# - checkout +# - run: +# name: Checkout submodules +# command: git submodule update --init --recursive + +# setup-automation: +# steps: +# - run: +# name: Setup automation +# command: | +# git submodule update --init deps/readies +# if [[ $(uname -s) == Darwin ]]; then rm -f /usr/local/bin/python3; fi +# ./deps/readies/bin/getpy3 +# - run: +# name: Setup automation (part 2) +# shell: /bin/bash -l -eo pipefail +# command: | +# export HOMEBREW_NO_AUTO_UPDATE=1 +# ./deps/readies/bin/getaws +# ls -l /usr/local/bin/python* || true +# echo "python3: $(command -v python3)" +# python3 --version +# python3 -m pip list +# python3 -m pip install --upgrade setuptools six pip + +# install-prerequisites: +# parameters: +# redis_version: +# type: string +# default: "7.2" +# getredis_params: +# type: string +# default: "" +# system_setup_params: +# type: string +# default: "" +# steps: +# - setup-automation +# - run: +# name: System setup +# shell: /bin/bash -l -eo pipefail +# command: | +# ./sbin/system-setup.py <> +# - run: +# name: Install Redis +# shell: /bin/bash -l -eo pipefail +# command: | +# export HOMEBREW_NO_AUTO_UPDATE=1 +# ./deps/readies/bin/getredis -v '<>' --force <> +# - run: +# name: System report +# shell: /bin/bash -l -eo pipefail +# command: | +# source $HOME/.cargo/env +# make info + +# save-tests-logs: +# steps: +# - run: +# name: Cleanup test log dir +# command: | +# rm -f tests/pytest/logs/*.{aof,rdb} +# when: always +# - store_artifacts: +# path: tests/pytest/logs + +# persist-artifacts: +# steps: +# - early-return-for-forked-pull-requests +# - run: +# name: List artifacts +# command: | +# cd bin/artifacts +# du -ah --apparent-size * +# - persist_to_workspace: +# root: bin/ +# paths: +# - artifacts/*.zip +# - artifacts/*.tgz +# - artifacts/*.tar +# - artifacts/snapshots/* + +# build-steps: +# parameters: +# build_params: +# type: string +# default: "" +# test_params: +# type: string +# default: "" +# redis_version: +# type: string +# default: "7" +# getredis_params: +# type: string +# default: "" +# steps: +# - early-returns +# - checkout-all +# - install-prerequisites: +# redis_version: <> +# getredis_params: <> +# - restore_cache: +# keys: +# - v3-dependencies-{{ arch }}-{{ checksum "Cargo.lock" }} +# - run: +# name: Check formatting +# shell: /bin/bash -l -eo pipefail +# command: make lint +# - run: +# name: Build debug +# shell: /bin/bash -l -eo pipefail +# command: make build DEBUG=1 <> +# - run: +# name: Build release +# shell: /bin/bash -l -eo pipefail +# command: make build pack <> +# - save_cache: +# key: v3-dependencies-{{ arch }}-{{ checksum "Cargo.lock" }} +# paths: +# - "~/.cargo" +# - "./target" + +# test-steps: +# steps: +# - run: +# name: Run tests +# shell: /bin/bash -l -eo pipefail +# command: | +# python3 -m RLTest --version +# make test +# timeout: 30m +# no_output_timeout: 30m +# - save_cache: +# key: v2-dependencies-{{ arch }}-{{ checksum "Cargo.lock" }} +# paths: +# - "~/.cargo" +# - "./target" +# - save-tests-logs + +# build-platforms-steps: +# parameters: +# platform: +# type: string +# steps: +# - early-returns +# - setup-executor +# - checkout-all +# - setup-automation +# - run: +# name: Build for platform +# shell: /bin/bash -l -eo pipefail +# command: | +# ROOT=$PWD +# cd build/docker +# make build OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH TEST=1 OFFICIAL=1 SHOW=1 +# cd $ROOT +# mkdir -p tests/pytest/logs +# tar -C tests/pytest/logs -xzf bin/artifacts/pytest-logs*.tgz +# timeout: 60m +# no_output_timeout: 30m +# - save-tests-logs +# - early-return-for-forked-pull-requests +# - run: +# name: Upload artifacts to S3 +# shell: /bin/bash -l -eo pipefail +# command: | +# if [[ -n $CIRCLE_BRANCH ]]; then +# make upload-artifacts OSNICK=<> SHOW=1 +# fi +# - run: +# name: Publish container +# shell: /bin/bash -l -eo pipefail +# command: | +# docker login -u redisfab -p $DOCKER_REDISFAB_PWD +# cd build/docker +# make publish OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH OFFICIAL=1 SHOW=1 +# - persist-artifacts + +# vm-build-platforms-steps: +# parameters: +# platform: +# type: string +# steps: +# - early-returns +# - checkout +# - setup-automation +# - run: +# name: Install Docker +# shell: /bin/bash -l -eo pipefail +# command: ./deps/readies/bin/getdocker +# - run: +# name: Build for platform +# command: | +# ROOT=$PWD +# cd build/docker +# make build OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH TEST=1 OFFICIAL=1 SHOW=1 +# cd $ROOT +# mkdir -p tests/pytest/logs +# tar -C tests/pytest/logs -xzf bin/artifacts/pytest-logs*.tgz +# no_output_timeout: 30m +# - save-tests-logs +# - early-return-for-forked-pull-requests +# - run: +# name: Upload artifacts to S3 +# command: | +# if [[ -n $CIRCLE_BRANCH ]]; then +# make upload-artifacts OSNICK=<> SHOW=1 +# fi +# - run: +# name: Publish container +# command: | +# docker login -u redisfab -p $DOCKER_REDISFAB_PWD +# cd build/docker +# make publish OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH OFFICIAL=1 SHOW=1 +# - persist-artifacts + +# #---------------------------------------------------------------------------------------------------------------------------------- + +# jobs: +# build-linux-debian: +# docker: +# - image: redisfab/rmbuilder:6.2.7-x64-bullseye +# parameters: +# redis_version: +# type: string +# default: "7" +# persist: +# type: string +# default: "yes" +# steps: +# - build-steps: +# redis_version: <> +# - test-steps +# - run: +# name: Persist artifacts? +# command: | +# if [[ "<>" != "yes" ]]; then +# circleci step halt +# fi +# - persist-artifacts + +# build-platforms: +# parameters: +# platform: +# type: string +# # docker: +# # - image: debian:bullseye +# machine: +# enabled: true +# image: ubuntu-2204:edge +# resource_class: large +# steps: +# - vm-build-platforms-steps: +# platform: <> + +# # build-arm-platforms: +# # parameters: +# # platform: +# # type: string +# # machine: +# # image: ubuntu-2004:202101-01 +# # resource_class: arm.medium +# # steps: +# # - vm-build-platforms-steps: +# # platform: <> + +# build-macos-m1: +# macos: +# xcode: 15.4.0 +# resource_class: macos.m1.large.gen1 +# parameters: +# upload: +# type: string +# default: "yes" +# steps: +# - early-returns +# - build-steps +# - test-steps +# - persist-artifacts +# - run: +# name: Upload artifacts to S3 +# command: | +# if [[ -n $CIRCLE_BRANCH ]]; then +# make upload-artifacts SHOW=1 VERBOSE=1 +# else +# make upload-release SHOW=1 +# fi + +# coverage: +# docker: +# - image: redisfab/rmbuilder:6.2.7-x64-focal +# steps: +# - early-returns +# - checkout-all +# - install-prerequisites +# - run: +# name: Build & Test +# shell: /bin/bash -l -eo pipefail +# command: | +# make clang-install +# make coverage SHOW=1 +# make upload-cov SHOW=1 +# no_output_timeout: 30m +# - save-tests-logs + +# sanitize: +# docker: +# - image: redisfab/clang:16-x64-focal +# parameters: +# san-type: +# type: string +# steps: +# - early-returns +# - checkout-all +# - install-prerequisites +# - run: +# name: Build & test +# shell: /bin/bash -l -eo pipefail +# command: make SAN=<> build test SHOW=1 +# no_output_timeout: 30m +# - save-tests-logs + +# upload-artifacts: +# parameters: +# staging-lab: +# type: string +# default: "0" +# docker: +# - image: redisfab/rmbuilder:6.2.7-x64-bullseye +# steps: +# - early-returns +# - early-return-for-forked-pull-requests +# - checkout +# - setup-automation +# - attach_workspace: +# at: ~/workspace +# - run: +# name: Upload artifacts to S3 +# command: | +# mkdir -p bin +# ln -s ~/workspace/artifacts bin/artifacts +# if [[ -n $CIRCLE_TAG ]]; then +# make upload-release SHOW=1 +# else +# make upload-artifacts SHOW=1 +# fi + +# #---------------------------------------------------------------------------------------------------------------------------------- + +# on-any-branch: &on-any-branch +# filters: +# branches: +# only: /.*/ +# tags: +# only: /.*/ + +# always: &always +# filters: +# branches: +# only: /.*/ +# tags: +# only: /.*/ + +# never: &never +# filters: +# branches: +# ignore: /.*/ +# tags: +# ignore: /.*/ + +# on-master: &on-master +# filters: +# branches: +# only: master +# tags: +# ignore: /.*/ + +# on-integ-branch: &on-integ-branch +# filters: +# branches: +# only: +# - master +# - /^\d+\.\d+.*$/ +# - /^feature.*$/ +# tags: +# ignore: /.*/ + +# on-integ-branch-cron: &on-integ-branch-cron +# filters: +# branches: +# only: +# - master +# - /^\d+\.\d+.*$/ +# - /^feature.*$/ + +# not-on-integ-branch: ¬-on-integ-branch +# filters: +# branches: +# ignore: +# - master +# - /^\d+\.\d+.*$/ +# - /^feature.*$/ +# tags: +# ignore: /.*/ + +# on-version-tags: &on-version-tags +# filters: +# branches: +# ignore: /.*/ +# tags: +# only: /^v[0-9].*/ + +# on-integ-and-version-tags: &on-integ-and-version-tags +# filters: +# branches: +# only: +# - master +# - /^\d+\.\d+.*$/ +# - /^feature.*$/ +# tags: +# only: /^v[0-9].*/ + +# #---------------------------------------------------------------------------------------------------------------------------------- + +# workflows: +# version: 2 +# default-flow: +# when: +# << pipeline.parameters.run_default_flow >> +# jobs: +# - build-linux-debian: +# name: build +# <<: *not-on-integ-branch +# - build-platforms: +# <<: *on-integ-and-version-tags +# context: common +# matrix: +# parameters: +# platform: [centos7, rocky8, rocky9, bullseye, amzn2] +# - build-macos-m1: +# context: common +# <<: *on-integ-and-version-tags +# - coverage: +# <<: *always +# - sanitize: +# name: sanitize-<< matrix.san-type >> +# <<: *always +# matrix: +# parameters: +# san-type: [address] +# - upload-artifacts: +# name: upload-artifacts-to-staging-lab +# <<: *on-integ-branch +# staging-lab: "1" +# context: common +# requires: +# - build-platforms +# # - build-arm-platforms +# - build-macos-m1 +# - upload-artifacts: +# name: upload-release-artifacts +# <<: *on-version-tags +# context: common +# requires: +# - build-platforms +# # - build-arm-platforms +# - build-macos-m1 + +# nightly: +# triggers: +# - schedule: +# cron: "07 20 * * *" +# <<: *on-integ-branch-cron +# jobs: +# - build-linux-debian: +# name: build-with-redis-<> +# matrix: +# parameters: +# redis_version: ["7", "unstable"] + +# nightly-twice-a-week-by-param: +# when: +# << pipeline.parameters.run_nightly_twice_a_week_flow_label >> +# jobs: +# - build-macos-m1: +# context: common +# upload: "yes" + +# nightly-twice-a-week: +# triggers: +# - schedule: +# cron: "20 17 * * 0,3" +# <<: *on-integ-branch-cron +# jobs: +# - build-macos-m1: +# context: common +# upload: "yes" \ No newline at end of file diff --git a/.github/actions/build-json-module-and-redis-with-cargo/action.yml b/.github/actions/build-json-module-and-redis-with-cargo/action.yml index 7ca75dd9e..0bea4d0fa 100644 --- a/.github/actions/build-json-module-and-redis-with-cargo/action.yml +++ b/.github/actions/build-json-module-and-redis-with-cargo/action.yml @@ -6,7 +6,7 @@ inputs: redis-ref: description: 'Redis version to build' required: true - default: '7.2.1' + default: '7.4' runs: using: composite diff --git a/.github/actions/build-json-module-and-redis/action.yml b/.github/actions/build-json-module-and-redis/action.yml index 9f3d7d31f..5239223fb 100644 --- a/.github/actions/build-json-module-and-redis/action.yml +++ b/.github/actions/build-json-module-and-redis/action.yml @@ -6,7 +6,7 @@ inputs: redis-ref: description: 'Redis version to build' required: true - default: '7.2.1' + default: '7.4' sanitizer: type: string diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index fb8467a00..c9b607ab4 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -64,5 +64,5 @@ runs: shell: bash id: set-redis-ref run: | - export REDIS_REF="${{ inputs.redis-ref || '7.2.1'}}" + export REDIS_REF="${{ inputs.redis-ref || '7.4'}}" echo "REDIS_REF=${REDIS_REF}" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/workflows/alpine.yml b/.github/workflows/alpine.yml index 091ad88c3..05c9266d7 100644 --- a/.github/workflows/alpine.yml +++ b/.github/workflows/alpine.yml @@ -36,7 +36,7 @@ jobs: uses: actions/checkout@v4 with: repository: redis/redis - ref: '7.2.1' + ref: '7.4' path: redis - name: Build Redis working-directory: redis diff --git a/.github/workflows/mariner2.yml b/.github/workflows/mariner2.yml index ecec78652..79510d72a 100644 --- a/.github/workflows/mariner2.yml +++ b/.github/workflows/mariner2.yml @@ -30,7 +30,7 @@ jobs: uses: actions/checkout@v4 with: repository: redis/redis - ref: '7.2.1' + ref: '7.4' path: redis - name: Build Redis working-directory: redis diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 41690ccae..d1753b202 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -71,7 +71,7 @@ jobs: uses: actions/checkout@v3 with: repository: redis/redis - ref: '7.2.1' + ref: '7.4' path: redis - name: Build Redis working-directory: redis diff --git a/Cargo.lock b/Cargo.lock index a4fb86fbd..4ab135ca9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,18 +4,18 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "ahash" @@ -41,23 +41,23 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets", ] [[package]] @@ -85,7 +85,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.76", + "syn 2.0.87", "which", ] @@ -124,9 +124,9 @@ dependencies = [ [[package]] name = "bson" -version = "2.11.0" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a88e82b9106923b5c4d6edfca9e7db958d4e98a478ec115022e81b9b38e2c8" +checksum = "068208f2b6fcfa27a7f1ee37488d2bb8ba2640f68f5475d08e1d9130696aba59" dependencies = [ "ahash", "base64", @@ -157,9 +157,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "cc" -version = "1.1.15" +version = "1.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" +checksum = "baee610e9452a8f6f0a1b6194ec09ff9e2d85dea54432acdae41aa0761c95d70" dependencies = [ "shlex", ] @@ -192,9 +192,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -284,7 +284,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -316,9 +316,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -341,6 +341,12 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "hashbrown" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" + [[package]] name = "heck" version = "0.4.1" @@ -365,7 +371,7 @@ version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -388,12 +394,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.4.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.1", ] [[package]] @@ -404,7 +410,7 @@ checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" dependencies = [ "hermit-abi", "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -424,9 +430,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] @@ -461,9 +467,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.158" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "libloading" @@ -477,22 +483,22 @@ dependencies = [ [[package]] name = "linkme" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c943daedff228392b791b33bba32e75737756e80a613e32e246c6ce9cbab20a" +checksum = "70fe496a7af8c406f877635cbf3cd6a9fac9d6f443f58691cd8afe6ce0971af4" dependencies = [ "linkme-impl", ] [[package]] name = "linkme-impl" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb26336e6dc7cc76e7927d2c9e7e3bb376d7af65a6f56a0b16c47d18a9b1abc5" +checksum = "b01f197a15988fb5b2ec0a5a9800c97e70771499c456ad757d63b3c5e9b96e75" dependencies = [ "proc-macro2", "quote 1.0.37", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -540,11 +546,11 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] @@ -596,18 +602,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.3" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "parking_lot_core" @@ -630,9 +636,9 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "pest" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" +checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" dependencies = [ "memchr", "thiserror", @@ -641,9 +647,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" +checksum = "d214365f632b123a47fd913301e14c946c61d1c183ee245fa76eb752e59a02dd" dependencies = [ "pest", "pest_generator", @@ -651,22 +657,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" +checksum = "eb55586734301717aea2ac313f50b2eb8f60d2fc3dc01d190eefa2e625f60c4e" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote 1.0.37", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] name = "pest_meta" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" +checksum = "b75da2a70cf4d9cb76833c990ac9cd3923c9a8905a8929789ce347c84564d03d" dependencies = [ "once_cell", "pest", @@ -696,19 +702,19 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.22" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -766,8 +772,8 @@ dependencies = [ [[package]] name = "redis-module" -version = "99.99.99" -source = "git+https://github.com/RedisLabsModules/redismodule-rs?tag=v2.0.8#bce557d5ec9381d8b70eaee30c47f74bb66f458b" +version = "2.1.0" +source = "git+https://github.com/RedisLabsModules/redismodule-rs?tag=v2.1.0#2aa3463e03f15b89697587231a8582fdcf875a01" dependencies = [ "backtrace", "bindgen", @@ -789,7 +795,7 @@ dependencies = [ [[package]] name = "redis-module-macros" version = "99.99.99" -source = "git+https://github.com/RedisLabsModules/redismodule-rs?tag=v2.0.8#bce557d5ec9381d8b70eaee30c47f74bb66f458b" +source = "git+https://github.com/RedisLabsModules/redismodule-rs?tag=v2.1.0#2aa3463e03f15b89697587231a8582fdcf875a01" dependencies = [ "proc-macro2", "quote 1.0.37", @@ -801,7 +807,7 @@ dependencies = [ [[package]] name = "redis-module-macros-internals" version = "99.99.99" -source = "git+https://github.com/RedisLabsModules/redismodule-rs?tag=v2.0.8#bce557d5ec9381d8b70eaee30c47f74bb66f458b" +source = "git+https://github.com/RedisLabsModules/redismodule-rs?tag=v2.1.0#2aa3463e03f15b89697587231a8582fdcf875a01" dependencies = [ "lazy_static", "proc-macro2", @@ -829,18 +835,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "regex" -version = "1.10.6" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -850,9 +856,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", @@ -861,9 +867,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rustc-demangle" @@ -879,22 +885,22 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.35" +version = "0.38.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a85d50532239da68e9addb745ba38ff4612a242c1c7ceea689c4bc7c2f43c36f" +checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" dependencies = [ "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "ryu" @@ -910,9 +916,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" dependencies = [ "serde_derive", ] @@ -928,20 +934,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote 1.0.37", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] name = "serde_json" -version = "1.0.127" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "indexmap", "itoa", @@ -1022,9 +1028,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.76" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote 1.0.37", @@ -1057,22 +1063,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2", "quote 1.0.37", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -1114,15 +1120,15 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ucd-trie" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-xid" @@ -1132,9 +1138,9 @@ checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" [[package]] name = "uuid" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom", "serde", @@ -1154,9 +1160,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", "once_cell", @@ -1165,24 +1171,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote 1.0.37", - "syn 2.0.76", + "syn 2.0.87", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote 1.0.37", "wasm-bindgen-macro-support", @@ -1190,22 +1196,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote 1.0.37", - "syn 2.0.76", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "which" @@ -1225,7 +1231,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -1237,6 +1243,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -1328,5 +1343,5 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote 1.0.37", - "syn 2.0.76", + "syn 2.0.87", ] diff --git a/build/docker/Makefile b/build/docker/Makefile index ab1fc8527..a57acced5 100644 --- a/build/docker/Makefile +++ b/build/docker/Makefile @@ -5,7 +5,7 @@ include $(ROOT)/deps/readies/mk/main REPO=rejson -REDIS_VERSION=7.2.1 +REDIS_VERSION=7.4 OSNICK.official=bullseye diff --git a/ramp.yml b/ramp.yml index fbf7f6040..4fc0ae981 100644 --- a/ramp.yml +++ b/ramp.yml @@ -7,8 +7,8 @@ homepage: http://redisjson.io license: Redis Source Available License 2.0 (RSALv2) or the Server Side Public License v1 (SSPLv1) command_line_args: "" min_redis_version: "7.1" -min_redis_pack_version: "7.2" -compatible_redis_version: "7.2" +min_redis_pack_version: "7.4" +compatible_redis_version: "7.4" capabilities: - types - no_multi_key diff --git a/redis_json/Cargo.toml b/redis_json/Cargo.toml index 483dc8f27..21daa03cc 100644 --- a/redis_json/Cargo.toml +++ b/redis_json/Cargo.toml @@ -22,8 +22,8 @@ ijson.workspace = true serde_json.workspace = true serde.workspace = true libc = "0.2" -redis-module ={ git="https://github.com/RedisLabsModules/redismodule-rs", tag="v2.0.8", default-features = false, features = ["min-redis-compatibility-version-7-2"] } -redis-module-macros = { git="https://github.com/RedisLabsModules/redismodule-rs", tag="v2.0.8" } +redis-module ={ git="https://github.com/RedisLabsModules/redismodule-rs", tag="v2.1.0", default-features = false, features = ["min-redis-compatibility-version-7-4"] } +redis-module-macros = { git="https://github.com/RedisLabsModules/redismodule-rs", tag="v2.1.0" } itertools = "0.13" json_path = {path="../json_path"} linkme = "0.3" diff --git a/redis_json/src/lib.rs b/redis_json/src/lib.rs index c38e3c846..e1ce832ef 100644 --- a/redis_json/src/lib.rs +++ b/redis_json/src/lib.rs @@ -10,6 +10,7 @@ extern crate redis_module; use commands::*; use redis_module::native_types::RedisType; use redis_module::raw::RedisModuleTypeMethods; +use redis_module::AclCategory; #[cfg(not(feature = "as-library"))] use redis_module::InfoContext; @@ -199,40 +200,42 @@ macro_rules! redis_json_module_create {( Status::Ok } + use AclCategory as ACL; redis_module! { name: $crate::MODULE_NAME, version: $version, allocator: (get_allocator!(), get_allocator!()), data_types: [$($data_type,)*], + acl_categories: [ACL::from("json"), ], init: json_init_config, init: initialize, info: $info_func, commands: [ - ["json.del", json_command!(json_del), "write", 1,1,1], - ["json.get", json_command!(json_get), "readonly", 1,1,1], - ["json.mget", json_command!(json_mget), "readonly", 1,1,1], - ["json.set", json_command!(json_set), "write deny-oom", 1,1,1], - ["json.mset", json_command!(json_mset), "write deny-oom", 1,-1,3], - ["json.type", json_command!(json_type), "readonly", 1,1,1], - ["json.numincrby", json_command!(json_num_incrby), "write", 1,1,1], - ["json.toggle", json_command!(json_bool_toggle), "write deny-oom", 1,1,1], - ["json.nummultby", json_command!(json_num_multby), "write", 1,1,1], - ["json.numpowby", json_command!(json_num_powby), "write", 1,1,1], - ["json.strappend", json_command!(json_str_append), "write deny-oom", 1,1,1], - ["json.strlen", json_command!(json_str_len), "readonly", 1,1,1], - ["json.arrappend", json_command!(json_arr_append), "write deny-oom", 1,1,1], - ["json.arrindex", json_command!(json_arr_index), "readonly", 1,1,1], - ["json.arrinsert", json_command!(json_arr_insert), "write deny-oom", 1,1,1], - ["json.arrlen", json_command!(json_arr_len), "readonly", 1,1,1], - ["json.arrpop", json_command!(json_arr_pop), "write", 1,1,1], - ["json.arrtrim", json_command!(json_arr_trim), "write", 1,1,1], - ["json.objkeys", json_command!(json_obj_keys), "readonly", 1,1,1], - ["json.objlen", json_command!(json_obj_len), "readonly", 1,1,1], - ["json.clear", json_command!(json_clear), "write", 1,1,1], - ["json.debug", json_command!(json_debug), "readonly", 2,2,1], - ["json.forget", json_command!(json_del), "write", 1,1,1], - ["json.resp", json_command!(json_resp), "readonly", 1,1,1], - ["json.merge", json_command!(json_merge), "write deny-oom", 1,1,1], + ["json.del", json_command!(json_del), "write", 1,1,1, ACL::Write, ACL::from("json")], + ["json.get", json_command!(json_get), "readonly", 1,1,1, ACL::Read, ACL::from("json")], + ["json.mget", json_command!(json_mget), "readonly", 1,1,1, ACL::Read, ACL::from("json")], + ["json.set", json_command!(json_set), "write deny-oom", 1,1,1, ACL::Write, ACL::from("json")], + ["json.mset", json_command!(json_mset), "write deny-oom", 1,-1,3, ACL::Write, ACL::from("json")], + ["json.type", json_command!(json_type), "readonly", 1,1,1, ACL::Read, ACL::from("json")], + ["json.numincrby", json_command!(json_num_incrby), "write", 1,1,1, ACL::Write, ACL::from("json")], + ["json.toggle", json_command!(json_bool_toggle), "write deny-oom", 1,1,1, ACL::Write, ACL::from("json")], + ["json.nummultby", json_command!(json_num_multby), "write", 1,1,1, ACL::Write, ACL::from("json")], + ["json.numpowby", json_command!(json_num_powby), "write", 1,1,1, ACL::Write, ACL::from("json")], + ["json.strappend", json_command!(json_str_append), "write deny-oom", 1,1,1, ACL::Write, ACL::from("json")], + ["json.strlen", json_command!(json_str_len), "readonly", 1,1,1, ACL::Read, ACL::from("json")], + ["json.arrappend", json_command!(json_arr_append), "write deny-oom", 1,1,1, ACL::Write, ACL::from("json")], + ["json.arrindex", json_command!(json_arr_index), "readonly", 1,1,1, ACL::Read, ACL::from("json")], + ["json.arrinsert", json_command!(json_arr_insert), "write deny-oom", 1,1,1, ACL::Write, ACL::from("json")], + ["json.arrlen", json_command!(json_arr_len), "readonly", 1,1,1, ACL::Read, ACL::from("json")], + ["json.arrpop", json_command!(json_arr_pop), "write", 1,1,1, ACL::Write, ACL::from("json")], + ["json.arrtrim", json_command!(json_arr_trim), "write", 1,1,1, ACL::Write, ACL::from("json")], + ["json.objkeys", json_command!(json_obj_keys), "readonly", 1,1,1, ACL::Read, ACL::from("json")], + ["json.objlen", json_command!(json_obj_len), "readonly", 1,1,1, ACL::Read, ACL::from("json")], + ["json.clear", json_command!(json_clear), "write", 1,1,1, ACL::Write, ACL::from("json")], + ["json.debug", json_command!(json_debug), "readonly", 2,2,1, ACL::Read, ACL::from("json")], + ["json.forget", json_command!(json_del), "write", 1,1,1, ACL::Write, ACL::from("json")], + ["json.resp", json_command!(json_resp), "readonly", 1,1,1, ACL::Read, ACL::from("json")], + ["json.merge", json_command!(json_merge), "write deny-oom", 1,1,1, ACL::Write, ACL::from("json")], ], } } diff --git a/tests/pytest/common.py b/tests/pytest/common.py index 47f1f09a1..42bcbd330 100644 --- a/tests/pytest/common.py +++ b/tests/pytest/common.py @@ -2,6 +2,10 @@ from contextlib import contextmanager from functools import wraps from includes import * +from packaging import version +from unittest import SkipTest +from RLTest import Env +import inspect @contextmanager def TimeLimit(timeout): @@ -43,3 +47,32 @@ def wrapper(env, *args, **kwargs): return return f(env, *args, **kwargs) return wrapper + +def skip_redis_less_than(redis_less_than=None): + def decorate(f): + def wrapper(): + if redis_less_than and server_version_is_less_than(redis_less_than): + raise SkipTest() + if len(inspect.signature(f).parameters) > 0: + env = Env() + return f(env) + else: + return f() + return wrapper + return decorate + + +server_ver = None +def server_version_is_at_least(ver): + global server_ver + if server_ver is None: + import subprocess + # Expecting something like "Redis server v=7.2.3 sha=******** malloc=jemalloc-5.3.0 bits=64 build=***************" + v = subprocess.run([Defaults.binary, '--version'], stdout=subprocess.PIPE).stdout.decode().split()[2].split('=')[1] + server_ver = version.parse(v) + if not isinstance(ver, version.Version): + ver = version.parse(ver) + return server_ver >= ver + +def server_version_is_less_than(ver): + return not server_version_is_at_least(ver) diff --git a/tests/pytest/test_acl.py b/tests/pytest/test_acl.py new file mode 100644 index 000000000..ee0a772b5 --- /dev/null +++ b/tests/pytest/test_acl.py @@ -0,0 +1,83 @@ +from common import * + + +@skip_redis_less_than(redis_less_than="7.4.1") +def test_acl_category(env): + """Test that the `json` category was added appropriately in module load""" + res = env.cmd('ACL', 'CAT') + print(res) + env.assertTrue('json' in res) + +@skip_redis_less_than(redis_less_than="7.99.99") +def test_acl_json_commands(env): + """Tests that the RedisJSON commands are registered to the `json` ACL category""" + res = env.cmd('ACL', 'CAT', 'json') + COMMANDS = [ + 'json.del', 'json.get', 'json.mget', 'json.set', 'json.mset', 'json.type', 'json.numincrby', 'json.toggle', + 'json.nummultby', 'json.numpowby', 'json.strappend', 'json.strlen', 'json.arrappend', 'json.arrindex', + 'json.arrinsert', 'json.arrlen', 'json.arrpop', 'json.arrtrim', 'json.objkeys', 'json.objlen', 'json.clear', + 'json.debug', 'json.forget', 'json.resp', 'json.merge', + ] + + # Use a set since the order of the response is not consistent. + env.assertEqual(set(res), set(COMMANDS)) + + # Check that one of our commands is listed in a non-json category + res = env.cmd('ACL', 'CAT', 'read') + env.assertTrue('json.get' in res) + +@skip_redis_less_than(redis_less_than="7.4.1") +def test_acl_non_default_user(env): + """Tests that a user with a non-default ACL can't access the json category""" + + # Create a user with no command permissions (full keyspace and pubsub access) + env.expect('ACL', 'SETUSER', 'testusr', 'on', '>123', '~*', '&*').ok() + + env.expect('AUTH', 'testusr', '123').true() + + # Such a user shouldn't be able to run any RedisJSON commands (or any other commands) + env.expect('json.get', 'idx', '$', '*').error().contains( + "User testusr has no permissions to run the 'json.get' command") + + # Add `testusr` read permissions + env.expect('AUTH', 'default', '').true() + env.expect('ACL', 'SETUSER', 'testusr', '+@read').ok() + env.expect('AUTH', 'testusr', '123').true() + + READ_JSON_COMMANDS = [ + 'json.get', 'json.type', 'json.strlen', 'json.arrlen', 'json.objlen', + 'json.debug', 'json.arrindex', 'json.objkeys', + ] + + # `testusr` should now be able to run `read` commands like `json.get' + for cmd in READ_JSON_COMMANDS: + env.expect(cmd).error().notContains( + "User testusr has no permissions") + + # `testusr` should not be able to run `json` commands that are not `read` + env.expect('json.set', 'idx', '$', '0').error().contains( + "User testusr has no permissions to run the 'json.set' command") + + # Add `write` permissions to `testusr` + env.expect('AUTH', 'default', '').true() + env.expect('ACL', 'SETUSER', 'testusr', '+@write').ok() + env.expect('AUTH', 'testusr', '123').true() + + WRITE_JSON_COMMANDS = [ + 'json.del', 'json.set', 'json.merge', 'json.clear', 'json.forget', 'json.strappend', + 'json.arrappend', 'json.arrinsert', 'json.arrpop', 'json.arrtrim', + ] + + # `testusr` should now be able to run `write` commands + for cmd in WRITE_JSON_COMMANDS: + env.expect(cmd).error().notContains( + "User testusr has no permissions") + + # Add `testusr` `json` permissions + env.expect('AUTH', 'default', '').true() + env.expect('ACL', 'SETUSER', 'testusr', '+@json').ok() + env.expect('AUTH', 'testusr', '123').true() + + # `testusr` should now be able to run `json` commands like `json.set` + env.expect('json.set', 'idx', '$', '0').ok() + env.expect('json.get', 'idx', '$').equal('[0]') diff --git a/tests/pytest/tests.sh b/tests/pytest/tests.sh index d4fc3f9a8..7d5a31b13 100755 --- a/tests/pytest/tests.sh +++ b/tests/pytest/tests.sh @@ -10,9 +10,9 @@ READIES=$ROOT/deps/readies export PYTHONUNBUFFERED=1 -VALGRIND_REDIS_VER=7.2 -SAN_REDIS_VER=7.2 -SAN_REDIS_SUFFIX=7.2 +VALGRIND_REDIS_VER=7.4 +SAN_REDIS_VER=7.4 +SAN_REDIS_SUFFIX=7.4 # SAN_REDIS_VER=6.2 # SAN_REDIS_SUFFIX=6.2 From a5d8d1009183c9adb331844b84ed81b581b78b36 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Wed, 13 Nov 2024 13:42:07 +0200 Subject: [PATCH 092/112] MOD-7888 bigstore_version_2 capability added in ramp capabilities (#1292) --- ramp.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ramp.yml b/ramp.yml index 4fc0ae981..8eeb11472 100644 --- a/ramp.yml +++ b/ramp.yml @@ -6,9 +6,9 @@ description: Native JSON Data Type for Redis homepage: http://redisjson.io license: Redis Source Available License 2.0 (RSALv2) or the Server Side Public License v1 (SSPLv1) command_line_args: "" -min_redis_version: "7.1" -min_redis_pack_version: "7.4" compatible_redis_version: "7.4" +min_redis_version: "7.4" +min_redis_pack_version: "7.6.0" capabilities: - types - no_multi_key @@ -25,3 +25,4 @@ capabilities: - intershard_tls - intershard_tls_pass - ipv6 + - bigstore_version_2 From 21bac46cdff741cfddea1786f0475c6fec05e6d7 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Sun, 17 Nov 2024 10:10:12 +0000 Subject: [PATCH 093/112] update mariner install script path --- .github/workflows/mariner2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mariner2.yml b/.github/workflows/mariner2.yml index 79510d72a..7dc479098 100644 --- a/.github/workflows/mariner2.yml +++ b/.github/workflows/mariner2.yml @@ -25,7 +25,7 @@ jobs: submodules: 'recursive' - name: Install dependencies run: | - bash .install/mariner2.sh + bash .install/.install/common_base_linux_mariner_2.0.sh - name: Get Redis uses: actions/checkout@v4 with: From 7db5c39535c40c2b9276651cbf3a6ba046c6976d Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Sun, 17 Nov 2024 10:30:44 +0000 Subject: [PATCH 094/112] delete mariner2.yaml --- .github/workflows/mariner2.yml | 59 ---------------------------------- 1 file changed, 59 deletions(-) delete mode 100644 .github/workflows/mariner2.yml diff --git a/.github/workflows/mariner2.yml b/.github/workflows/mariner2.yml deleted file mode 100644 index 7dc479098..000000000 --- a/.github/workflows/mariner2.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Build for mariner2 - -on: - push: - branches: - - master - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - -jobs: - build: - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l -eo pipefail {0} - container: - image: mcr.microsoft.com/cbl-mariner/base/core:2.0 - steps: - - name: Install prereqs - run: | - tdnf install --noplugins --skipsignature -y ca-certificates git - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - name: Install dependencies - run: | - bash .install/.install/common_base_linux_mariner_2.0.sh - - name: Get Redis - uses: actions/checkout@v4 - with: - repository: redis/redis - ref: '7.4' - path: redis - - name: Build Redis - working-directory: redis - run: make install - - name: Build module - run: | - make build - - name: Test - run: | - make test - - name: Pack module - run: | - make pack BRANCH=${{ github.ref_name }} - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: "us-east-1" - - name: Upload artifacts to S3 - staging - run: | - make upload-artifacts SHOW=1 VERBOSE=1 - make upload-release SHOW=1 STAGING=1 VERBOSE=1 - - name: Upload artifacts to S3 - release # todo: trigger this manually instead - if: ${{ github.ref != 'refs/heads/master' }} - run: make upload-release SHOW=1 VERBOSE=1 From 904ae5613e1997aa8bef680298ab014f5bca6138 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Thu, 21 Nov 2024 08:45:07 +0200 Subject: [PATCH 095/112] Refactor shared string cache initialization into a separate function (#1299) --- redis_json/src/lib.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/redis_json/src/lib.rs b/redis_json/src/lib.rs index e1ce832ef..1df8f1883 100644 --- a/redis_json/src/lib.rs +++ b/redis_json/src/lib.rs @@ -10,6 +10,7 @@ extern crate redis_module; use commands::*; use redis_module::native_types::RedisType; use redis_module::raw::RedisModuleTypeMethods; +#[cfg(not(feature = "as-library"))] use redis_module::AclCategory; #[cfg(not(feature = "as-library"))] use redis_module::InfoContext; @@ -180,7 +181,7 @@ macro_rules! redis_json_module_create {( _ => false, }); ctx.log_notice(&format!("Initialized shared string cache, thread safe: {is_bigredis}.")); - if let Err(e) = ijson::init_shared_string_cache(is_bigredis) { + if let Err(e) = $crate::init_ijson_shared_string_cache(is_bigredis) { ctx.log(RedisLogLevel::Warning, &format!("Failed initializing shared string cache, {e}.")); return Status::Err; } @@ -249,6 +250,10 @@ const fn dummy_init(_ctx: &Context, _args: &[RedisString]) -> Status { Status::Ok } +pub fn init_ijson_shared_string_cache(is_bigredis: bool) -> Result<(), String> { + ijson::init_shared_string_cache(is_bigredis) +} + #[cfg(not(feature = "as-library"))] const fn dummy_info(_ctx: &InfoContext, _for_crash_report: bool) {} From 6c391a76435a277f12e5a51c897538bf5813caa0 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Thu, 21 Nov 2024 16:55:41 +0200 Subject: [PATCH 096/112] Update ramp-packer to version 2.6.0 and enable bigstore_version_2 support in ramp.yml (#1301) --- .install/build_package_requirements.txt | 2 +- ramp.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.install/build_package_requirements.txt b/.install/build_package_requirements.txt index 464a0f4e5..763279a74 100644 --- a/.install/build_package_requirements.txt +++ b/.install/build_package_requirements.txt @@ -1,4 +1,4 @@ addict toml jinja2 -ramp-packer==2.5.10 \ No newline at end of file +ramp-packer==2.5.11 diff --git a/ramp.yml b/ramp.yml index 8eeb11472..677ea07dc 100644 --- a/ramp.yml +++ b/ramp.yml @@ -9,6 +9,7 @@ command_line_args: "" compatible_redis_version: "7.4" min_redis_version: "7.4" min_redis_pack_version: "7.6.0" +bigstore_version_2_support: true capabilities: - types - no_multi_key @@ -25,4 +26,3 @@ capabilities: - intershard_tls - intershard_tls_pass - ipv6 - - bigstore_version_2 From 93d8c182f65fe7fed1379dc1108efda25a20a57c Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Sun, 24 Nov 2024 12:03:29 +0200 Subject: [PATCH 097/112] update ramp packer to 2.5.12 (#1303) --- .install/build_package_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.install/build_package_requirements.txt b/.install/build_package_requirements.txt index 763279a74..a698dc981 100644 --- a/.install/build_package_requirements.txt +++ b/.install/build_package_requirements.txt @@ -1,4 +1,4 @@ addict toml jinja2 -ramp-packer==2.5.11 +ramp-packer==2.5.12 From fd0882b73d1fe37ca31b86482e2f1f7189c7b67f Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Tue, 24 Dec 2024 19:39:30 +0200 Subject: [PATCH 098/112] MOD-8197 Unified CI workflows (#1304) - Updated and added new GitHub Actions workflows for build, test, and deployment - Introduced event-based workflows for specific triggers: - `event-ci.yml`: Runs CI checks on pull requests and pushes to specific branches - `event-nightly.yml`: Executes nightly builds and tests - `event-tag.yml`: Triggers workflows on new tags for release processes - Differentiated flow-based workflows for various environments: - `flow-alpine.yml`: Handles builds and tests for Alpine Linux - `flow-linux-x86.yml`: Manages workflows for x86 Linux distributions - `flow-macos.yml`: Manages workflows for macOS environments - `flow-ubuntu-arm.yml`: Handles builds and tests for Ubuntu on ARM architecture - Removed obsolete workflows - Added new installation scripts for various Linux distributions and macOS - Modified existing scripts and configurations for better compatibility and performance --- .circleci/config.yml | 553 ------------------ .../action.yml | 3 +- .../build-json-module-and-redis/action.yml | 4 - .github/actions/make-pack/action.yml | 9 + .github/actions/pack-module/action.yml | 17 + .github/actions/run-tests/action.yml | 109 +--- .github/actions/san-run-tests/action.yml | 126 ++++ .github/actions/setup-env/action.yml | 2 +- .../action.yml | 56 +- .../actions/upload-artifacts-to-s3/action.yml | 24 +- .github/actions/upload-artifacts/action.yml | 26 + .github/workflows/alpine-arm.yml | 121 ---- .github/workflows/alpine.yml | 58 -- .github/workflows/backport_pr.yml | 2 +- .github/workflows/build-linux-platforms.yml | 129 ---- .github/workflows/ci-basic.yml | 89 --- .github/workflows/event-ci.yml | 59 ++ .github/workflows/event-nightly.yml | 71 +++ .github/workflows/event-pull-request.yml | 25 - .github/workflows/event-tag.yml | 58 ++ .github/workflows/event-weekly.yml | 63 +- .github/workflows/flow-alpine.yml | 110 ++++ .github/workflows/flow-linter.yml | 15 + .github/workflows/flow-linux-x86.yml | 185 ++++++ ...os-build-and-deploy.yml => flow-macos.yml} | 94 +-- .github/workflows/flow-sanitizer.yml | 10 +- .../{spellcheck.yml => flow-spellcheck.yml} | 10 +- .../{ubuntu-arm.yml => flow-ubuntu-arm.yml} | 57 +- .github/workflows/freebsd.yml | 2 +- .github/workflows/macos.yml | 19 - .github/workflows/trigger-build.yml | 22 - .github/workflows/ubuntu.yml | 103 ---- sbin/upload-artifacts | 6 +- 33 files changed, 905 insertions(+), 1332 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/actions/make-pack/action.yml create mode 100644 .github/actions/pack-module/action.yml create mode 100644 .github/actions/san-run-tests/action.yml create mode 100644 .github/actions/upload-artifacts/action.yml delete mode 100644 .github/workflows/alpine-arm.yml delete mode 100644 .github/workflows/alpine.yml delete mode 100644 .github/workflows/build-linux-platforms.yml delete mode 100644 .github/workflows/ci-basic.yml create mode 100644 .github/workflows/event-ci.yml create mode 100644 .github/workflows/event-nightly.yml delete mode 100644 .github/workflows/event-pull-request.yml create mode 100644 .github/workflows/event-tag.yml create mode 100644 .github/workflows/flow-alpine.yml create mode 100644 .github/workflows/flow-linter.yml create mode 100644 .github/workflows/flow-linux-x86.yml rename .github/workflows/{macos-build-and-deploy.yml => flow-macos.yml} (58%) rename .github/workflows/{spellcheck.yml => flow-spellcheck.yml} (55%) rename .github/workflows/{ubuntu-arm.yml => flow-ubuntu-arm.yml} (82%) delete mode 100644 .github/workflows/macos.yml delete mode 100644 .github/workflows/trigger-build.yml delete mode 100644 .github/workflows/ubuntu.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index b2b6c1008..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,553 +0,0 @@ -# version: 2.1 - -# parameters: -# run_default_flow: -# default: true -# type: boolean -# run_nightly_twice_a_week_flow_label: -# default: false -# type: boolean - -# commands: -# early-returns: -# steps: -# - run: -# name: Early return if this is a docs build -# command: | -# if [[ $CIRCLE_BRANCH == *docs ]]; then -# echo "Identifies as documents PR, no testing required." -# circleci step halt -# fi -# - run: -# name: Early return if this branch should ignore CI -# command: | -# if [[ $CIRCLE_BRANCH == *noci ]]; then -# echo "Identifies as actively ignoring CI, no testing required." -# circleci step halt -# fi - -# early-return-for-forked-pull-requests: -# description: >- -# If this build is from a fork, stop executing the current job and return success. -# This is useful to avoid steps that will fail due to missing credentials. -# steps: -# - run: -# name: Early return if this build is from a forked PR -# command: | -# if [[ -n "$CIRCLE_PR_NUMBER" ]]; then -# echo "Nothing to do for forked PRs, so marking this step successful" -# circleci step halt -# fi - -# setup-executor: -# steps: -# - run: -# name: Setup executor -# command: | -# apt-get -qq update -# apt-get -q install -y git openssh-client curl ca-certificates make tar gzip -# bash <(curl -fsSL https://raw.githubusercontent.com/docker/docker-install/master/install.sh) -# - setup_remote_docker: -# version: 20.10.14 -# docker_layer_caching: true - -# checkout-all: -# steps: -# - checkout -# - run: -# name: Checkout submodules -# command: git submodule update --init --recursive - -# setup-automation: -# steps: -# - run: -# name: Setup automation -# command: | -# git submodule update --init deps/readies -# if [[ $(uname -s) == Darwin ]]; then rm -f /usr/local/bin/python3; fi -# ./deps/readies/bin/getpy3 -# - run: -# name: Setup automation (part 2) -# shell: /bin/bash -l -eo pipefail -# command: | -# export HOMEBREW_NO_AUTO_UPDATE=1 -# ./deps/readies/bin/getaws -# ls -l /usr/local/bin/python* || true -# echo "python3: $(command -v python3)" -# python3 --version -# python3 -m pip list -# python3 -m pip install --upgrade setuptools six pip - -# install-prerequisites: -# parameters: -# redis_version: -# type: string -# default: "7.2" -# getredis_params: -# type: string -# default: "" -# system_setup_params: -# type: string -# default: "" -# steps: -# - setup-automation -# - run: -# name: System setup -# shell: /bin/bash -l -eo pipefail -# command: | -# ./sbin/system-setup.py <> -# - run: -# name: Install Redis -# shell: /bin/bash -l -eo pipefail -# command: | -# export HOMEBREW_NO_AUTO_UPDATE=1 -# ./deps/readies/bin/getredis -v '<>' --force <> -# - run: -# name: System report -# shell: /bin/bash -l -eo pipefail -# command: | -# source $HOME/.cargo/env -# make info - -# save-tests-logs: -# steps: -# - run: -# name: Cleanup test log dir -# command: | -# rm -f tests/pytest/logs/*.{aof,rdb} -# when: always -# - store_artifacts: -# path: tests/pytest/logs - -# persist-artifacts: -# steps: -# - early-return-for-forked-pull-requests -# - run: -# name: List artifacts -# command: | -# cd bin/artifacts -# du -ah --apparent-size * -# - persist_to_workspace: -# root: bin/ -# paths: -# - artifacts/*.zip -# - artifacts/*.tgz -# - artifacts/*.tar -# - artifacts/snapshots/* - -# build-steps: -# parameters: -# build_params: -# type: string -# default: "" -# test_params: -# type: string -# default: "" -# redis_version: -# type: string -# default: "7" -# getredis_params: -# type: string -# default: "" -# steps: -# - early-returns -# - checkout-all -# - install-prerequisites: -# redis_version: <> -# getredis_params: <> -# - restore_cache: -# keys: -# - v3-dependencies-{{ arch }}-{{ checksum "Cargo.lock" }} -# - run: -# name: Check formatting -# shell: /bin/bash -l -eo pipefail -# command: make lint -# - run: -# name: Build debug -# shell: /bin/bash -l -eo pipefail -# command: make build DEBUG=1 <> -# - run: -# name: Build release -# shell: /bin/bash -l -eo pipefail -# command: make build pack <> -# - save_cache: -# key: v3-dependencies-{{ arch }}-{{ checksum "Cargo.lock" }} -# paths: -# - "~/.cargo" -# - "./target" - -# test-steps: -# steps: -# - run: -# name: Run tests -# shell: /bin/bash -l -eo pipefail -# command: | -# python3 -m RLTest --version -# make test -# timeout: 30m -# no_output_timeout: 30m -# - save_cache: -# key: v2-dependencies-{{ arch }}-{{ checksum "Cargo.lock" }} -# paths: -# - "~/.cargo" -# - "./target" -# - save-tests-logs - -# build-platforms-steps: -# parameters: -# platform: -# type: string -# steps: -# - early-returns -# - setup-executor -# - checkout-all -# - setup-automation -# - run: -# name: Build for platform -# shell: /bin/bash -l -eo pipefail -# command: | -# ROOT=$PWD -# cd build/docker -# make build OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH TEST=1 OFFICIAL=1 SHOW=1 -# cd $ROOT -# mkdir -p tests/pytest/logs -# tar -C tests/pytest/logs -xzf bin/artifacts/pytest-logs*.tgz -# timeout: 60m -# no_output_timeout: 30m -# - save-tests-logs -# - early-return-for-forked-pull-requests -# - run: -# name: Upload artifacts to S3 -# shell: /bin/bash -l -eo pipefail -# command: | -# if [[ -n $CIRCLE_BRANCH ]]; then -# make upload-artifacts OSNICK=<> SHOW=1 -# fi -# - run: -# name: Publish container -# shell: /bin/bash -l -eo pipefail -# command: | -# docker login -u redisfab -p $DOCKER_REDISFAB_PWD -# cd build/docker -# make publish OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH OFFICIAL=1 SHOW=1 -# - persist-artifacts - -# vm-build-platforms-steps: -# parameters: -# platform: -# type: string -# steps: -# - early-returns -# - checkout -# - setup-automation -# - run: -# name: Install Docker -# shell: /bin/bash -l -eo pipefail -# command: ./deps/readies/bin/getdocker -# - run: -# name: Build for platform -# command: | -# ROOT=$PWD -# cd build/docker -# make build OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH TEST=1 OFFICIAL=1 SHOW=1 -# cd $ROOT -# mkdir -p tests/pytest/logs -# tar -C tests/pytest/logs -xzf bin/artifacts/pytest-logs*.tgz -# no_output_timeout: 30m -# - save-tests-logs -# - early-return-for-forked-pull-requests -# - run: -# name: Upload artifacts to S3 -# command: | -# if [[ -n $CIRCLE_BRANCH ]]; then -# make upload-artifacts OSNICK=<> SHOW=1 -# fi -# - run: -# name: Publish container -# command: | -# docker login -u redisfab -p $DOCKER_REDISFAB_PWD -# cd build/docker -# make publish OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH OFFICIAL=1 SHOW=1 -# - persist-artifacts - -# #---------------------------------------------------------------------------------------------------------------------------------- - -# jobs: -# build-linux-debian: -# docker: -# - image: redisfab/rmbuilder:6.2.7-x64-bullseye -# parameters: -# redis_version: -# type: string -# default: "7" -# persist: -# type: string -# default: "yes" -# steps: -# - build-steps: -# redis_version: <> -# - test-steps -# - run: -# name: Persist artifacts? -# command: | -# if [[ "<>" != "yes" ]]; then -# circleci step halt -# fi -# - persist-artifacts - -# build-platforms: -# parameters: -# platform: -# type: string -# # docker: -# # - image: debian:bullseye -# machine: -# enabled: true -# image: ubuntu-2204:edge -# resource_class: large -# steps: -# - vm-build-platforms-steps: -# platform: <> - -# # build-arm-platforms: -# # parameters: -# # platform: -# # type: string -# # machine: -# # image: ubuntu-2004:202101-01 -# # resource_class: arm.medium -# # steps: -# # - vm-build-platforms-steps: -# # platform: <> - -# build-macos-m1: -# macos: -# xcode: 15.4.0 -# resource_class: macos.m1.large.gen1 -# parameters: -# upload: -# type: string -# default: "yes" -# steps: -# - early-returns -# - build-steps -# - test-steps -# - persist-artifacts -# - run: -# name: Upload artifacts to S3 -# command: | -# if [[ -n $CIRCLE_BRANCH ]]; then -# make upload-artifacts SHOW=1 VERBOSE=1 -# else -# make upload-release SHOW=1 -# fi - -# coverage: -# docker: -# - image: redisfab/rmbuilder:6.2.7-x64-focal -# steps: -# - early-returns -# - checkout-all -# - install-prerequisites -# - run: -# name: Build & Test -# shell: /bin/bash -l -eo pipefail -# command: | -# make clang-install -# make coverage SHOW=1 -# make upload-cov SHOW=1 -# no_output_timeout: 30m -# - save-tests-logs - -# sanitize: -# docker: -# - image: redisfab/clang:16-x64-focal -# parameters: -# san-type: -# type: string -# steps: -# - early-returns -# - checkout-all -# - install-prerequisites -# - run: -# name: Build & test -# shell: /bin/bash -l -eo pipefail -# command: make SAN=<> build test SHOW=1 -# no_output_timeout: 30m -# - save-tests-logs - -# upload-artifacts: -# parameters: -# staging-lab: -# type: string -# default: "0" -# docker: -# - image: redisfab/rmbuilder:6.2.7-x64-bullseye -# steps: -# - early-returns -# - early-return-for-forked-pull-requests -# - checkout -# - setup-automation -# - attach_workspace: -# at: ~/workspace -# - run: -# name: Upload artifacts to S3 -# command: | -# mkdir -p bin -# ln -s ~/workspace/artifacts bin/artifacts -# if [[ -n $CIRCLE_TAG ]]; then -# make upload-release SHOW=1 -# else -# make upload-artifacts SHOW=1 -# fi - -# #---------------------------------------------------------------------------------------------------------------------------------- - -# on-any-branch: &on-any-branch -# filters: -# branches: -# only: /.*/ -# tags: -# only: /.*/ - -# always: &always -# filters: -# branches: -# only: /.*/ -# tags: -# only: /.*/ - -# never: &never -# filters: -# branches: -# ignore: /.*/ -# tags: -# ignore: /.*/ - -# on-master: &on-master -# filters: -# branches: -# only: master -# tags: -# ignore: /.*/ - -# on-integ-branch: &on-integ-branch -# filters: -# branches: -# only: -# - master -# - /^\d+\.\d+.*$/ -# - /^feature.*$/ -# tags: -# ignore: /.*/ - -# on-integ-branch-cron: &on-integ-branch-cron -# filters: -# branches: -# only: -# - master -# - /^\d+\.\d+.*$/ -# - /^feature.*$/ - -# not-on-integ-branch: ¬-on-integ-branch -# filters: -# branches: -# ignore: -# - master -# - /^\d+\.\d+.*$/ -# - /^feature.*$/ -# tags: -# ignore: /.*/ - -# on-version-tags: &on-version-tags -# filters: -# branches: -# ignore: /.*/ -# tags: -# only: /^v[0-9].*/ - -# on-integ-and-version-tags: &on-integ-and-version-tags -# filters: -# branches: -# only: -# - master -# - /^\d+\.\d+.*$/ -# - /^feature.*$/ -# tags: -# only: /^v[0-9].*/ - -# #---------------------------------------------------------------------------------------------------------------------------------- - -# workflows: -# version: 2 -# default-flow: -# when: -# << pipeline.parameters.run_default_flow >> -# jobs: -# - build-linux-debian: -# name: build -# <<: *not-on-integ-branch -# - build-platforms: -# <<: *on-integ-and-version-tags -# context: common -# matrix: -# parameters: -# platform: [centos7, rocky8, rocky9, bullseye, amzn2] -# - build-macos-m1: -# context: common -# <<: *on-integ-and-version-tags -# - coverage: -# <<: *always -# - sanitize: -# name: sanitize-<< matrix.san-type >> -# <<: *always -# matrix: -# parameters: -# san-type: [address] -# - upload-artifacts: -# name: upload-artifacts-to-staging-lab -# <<: *on-integ-branch -# staging-lab: "1" -# context: common -# requires: -# - build-platforms -# # - build-arm-platforms -# - build-macos-m1 -# - upload-artifacts: -# name: upload-release-artifacts -# <<: *on-version-tags -# context: common -# requires: -# - build-platforms -# # - build-arm-platforms -# - build-macos-m1 - -# nightly: -# triggers: -# - schedule: -# cron: "07 20 * * *" -# <<: *on-integ-branch-cron -# jobs: -# - build-linux-debian: -# name: build-with-redis-<> -# matrix: -# parameters: -# redis_version: ["7", "unstable"] - -# nightly-twice-a-week-by-param: -# when: -# << pipeline.parameters.run_nightly_twice_a_week_flow_label >> -# jobs: -# - build-macos-m1: -# context: common -# upload: "yes" - -# nightly-twice-a-week: -# triggers: -# - schedule: -# cron: "20 17 * * 0,3" -# <<: *on-integ-branch-cron -# jobs: -# - build-macos-m1: -# context: common -# upload: "yes" \ No newline at end of file diff --git a/.github/actions/build-json-module-and-redis-with-cargo/action.yml b/.github/actions/build-json-module-and-redis-with-cargo/action.yml index 0bea4d0fa..18da16fdd 100644 --- a/.github/actions/build-json-module-and-redis-with-cargo/action.yml +++ b/.github/actions/build-json-module-and-redis-with-cargo/action.yml @@ -19,6 +19,7 @@ runs: - name: Build module shell: bash run: | - . "$HOME/.cargo/env" + . $HOME/.cargo/env + echo "source $HOME/.cargo/env" >> $HOME/.bash_profile cargo build --release cp $(realpath ./target/release)/librejson.so $(realpath ./target/release)/rejson.so diff --git a/.github/actions/build-json-module-and-redis/action.yml b/.github/actions/build-json-module-and-redis/action.yml index 5239223fb..f317c9c97 100644 --- a/.github/actions/build-json-module-and-redis/action.yml +++ b/.github/actions/build-json-module-and-redis/action.yml @@ -3,10 +3,6 @@ description: | Build JSON module and Redis Server inputs: - redis-ref: - description: 'Redis version to build' - required: true - default: '7.4' sanitizer: type: string diff --git a/.github/actions/make-pack/action.yml b/.github/actions/make-pack/action.yml new file mode 100644 index 000000000..27170a11b --- /dev/null +++ b/.github/actions/make-pack/action.yml @@ -0,0 +1,9 @@ +name: Run make pack module script + +runs: + using: composite + steps: + - name: Pack module + shell: bash + run: | + make pack BRANCH=$TAG_OR_BRANCH diff --git a/.github/actions/pack-module/action.yml b/.github/actions/pack-module/action.yml new file mode 100644 index 000000000..ceb84b30c --- /dev/null +++ b/.github/actions/pack-module/action.yml @@ -0,0 +1,17 @@ +name: Run pack module script + +runs: + using: composite + steps: + - name: Pack module + shell: bash + run: | + if command -v scl_source &> /dev/null + then + . scl_source enable devtoolset-11 || true + fi + . venv/bin/activate + git config --global --add safe.directory $GITHUB_WORKSPACE + export PATH="$GITHUB_WORKSPACE/redis/src:$PATH" + MODULE=$(realpath ./target/release/rejson.so) BRANCH=$TAG_OR_BRANCH \ + SHOW=1 OSNICK=${{ matrix.docker.nick }} ./sbin/pack.sh diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 7ceeb5289..3cd82ea15 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -1,122 +1,29 @@ -name: Common Flow for Tests - -# Documentation: https://redislabs.atlassian.net/wiki/spaces/DX/pages/3967844669/RediSearch+CI+refactor +name: Run module tests inputs: - env: - default: "ubuntu-22.04" - type: string - container: - type: string - sanitizer: - type: string - test-config: - description: 'Test configuration environment variable (e.g. "CONFIG=tls" or "QUICK=1")' - required: true + run_valgrind: + description: 'Run valgrind on the tests' type: string + default: 0 runs: using: composite steps: - - name: Get Installation Mode - shell: bash - id: mode - run: | - [[ -z "${{ inputs.container }}" ]] && echo "mode=sudo" >> $GITHUB_OUTPUT || echo "mode=" >> $GITHUB_OUTPUT - - name: Check if node20 is Supported - id: node20 - uses: ./.github/actions/node20-supported - with: - container: ${{ inputs.container }} - - name: Install git - shell: bash - run: | - # TODO: must be changed to run a script based on the input env - echo ::group::Install git - ${{ steps.mode.outputs.mode }} apt-get update && apt-get install -y git - echo ::endgroup:: - - name: Setup specific - shell: bash - working-directory: .install - run: | - echo ::group::OS-Specific Setup - ./install_script.sh ${{ steps.mode.outputs.mode }} - echo ::endgroup:: - echo ::group::Get Rust - ./getrust.sh ${{ steps.mode.outputs.mode }} - echo ::endgroup:: - - - name: Full checkout (node20 supported) - if: steps.node20.outputs.supported == 'true' - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Full checkout (node20 unsupported) - if: steps.node20.outputs.supported == 'false' - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Get Redis - uses: actions/checkout@v3 - with: - repository: redis/redis - ref: 'unstable' # todo change per version/tag - path: redis - submodules: 'recursive' - - name: Build - uses: ./.github/actions/build-json-module-and-redis - with: - sanitizer: ${{ inputs.san }} - - - name: Set Artifact Names - shell: bash - # Artifact names have to be unique, so we base them on the environment. - # We also remove invalid characters from the name. - id: artifact-names - run: | - # Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ? - echo "name=$(echo "${{ inputs.container || inputs.env }} ${{ runner.arch }}, Redis ${{ inputs.get-redis || 'unstable' }}" | \ - sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT - - name: Run tests shell: bash - id: test run: | echo ::group::Activate virtual environment python3 -m venv venv echo "source $PWD/venv/bin/activate" >> ~/.bash_profile - source venv/bin/activate + . venv/bin/activate echo ::endgroup:: echo ::group::Install python dependencies ./.install/common_installations.sh echo ::endgroup:: echo ::group::Unit tests - . "$HOME/.cargo/env" - make cargo_test LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }} + . $HOME/.cargo/env + cargo test echo ::endgroup:: echo ::group::Flow tests - make pytest LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }} ${{ inputs.test-config }} + MODULE=$(realpath ./target/release/rejson.so) RLTEST_ARGS='--no-progress' ./tests/pytest/tests.sh VG=${{inputs.run_valgrind}} echo ::endgroup:: - env: - PIP_BREAK_SYSTEM_PACKAGES: 1 - - - name: Upload test artifacts (node20 supported) - if: steps.node20.outputs.supported == 'true' && steps.test.outcome == 'failure' - uses: actions/upload-artifact@v4 - with: - name: Test logs ${{ steps.artifact-names.outputs.name }} - path: tests/**/logs/*.log* - if-no-files-found: ignore - - name: Upload test artifacts (node20 unsupported) - if: steps.node20.outputs.supported == 'false' && steps.test.outcome == 'failure' - uses: actions/upload-artifact@v3 - with: - name: Test logs ${{ steps.artifact-names.outputs.name }} - path: tests/**/logs/*.log* - if-no-files-found: ignore - - - name: Fail flow if tests failed - shell: bash - if: steps.test.outcome == 'failure' - run: exit 1 diff --git a/.github/actions/san-run-tests/action.yml b/.github/actions/san-run-tests/action.yml new file mode 100644 index 000000000..4673fbce9 --- /dev/null +++ b/.github/actions/san-run-tests/action.yml @@ -0,0 +1,126 @@ +name: Common Flow for Tests + +# Documentation: https://redislabs.atlassian.net/wiki/spaces/DX/pages/3967844669/RediSearch+CI+refactor + +inputs: + env: + default: "ubuntu-22.04" + type: string + container: + type: string + sanitizer: + type: string + test-config: + description: 'Test configuration environment variable (e.g. "CONFIG=tls" or "QUICK=1")' + required: true + type: string + redis-ref: + description: 'Redis ref to checkout' + type: string + required: true + +runs: + using: composite + steps: + - name: Get Installation Mode + shell: bash + id: mode + run: | + [[ -z "${{ inputs.container }}" ]] && echo "mode=sudo" >> $GITHUB_OUTPUT || echo "mode=" >> $GITHUB_OUTPUT + - name: Check if node20 is Supported + id: node20 + uses: ./.github/actions/node20-supported + with: + container: ${{ inputs.container }} + - name: Install git + shell: bash + run: | + # TODO: must be changed to run a script based on the input env + echo ::group::Install git + ${{ steps.mode.outputs.mode }} apt-get update && apt-get install -y git + echo ::endgroup:: + - name: Setup specific + shell: bash + working-directory: .install + run: | + echo ::group::OS-Specific Setup + ./install_script.sh ${{ steps.mode.outputs.mode }} + echo ::endgroup:: + echo ::group::Get Rust + ./getrust.sh ${{ steps.mode.outputs.mode }} + echo ::endgroup:: + + - name: Full checkout (node20 supported) + if: steps.node20.outputs.supported == 'true' + uses: actions/checkout@v4 + with: + submodules: recursive + - name: Full checkout (node20 unsupported) + if: steps.node20.outputs.supported == 'false' + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Get Redis + uses: actions/checkout@v4 + with: + repository: redis/redis + ref: ${{ inputs.redis-ref }} + path: redis + submodules: 'recursive' + - name: Build + uses: ./.github/actions/build-json-module-and-redis + with: + sanitizer: ${{ inputs.san }} + + - name: Set Artifact Names + shell: bash + # Artifact names have to be unique, so we base them on the environment. + # We also remove invalid characters from the name. + id: artifact-names + run: | + # Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ? + echo "name=$(echo "${{ inputs.container || inputs.env }} ${{ runner.arch }}, Redis ${{ inputs.redis-ref }}" | \ + sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT + + - name: Run tests + shell: bash + id: test + run: | + echo ::group::Activate virtual environment + python3 -m venv venv + echo "source $PWD/venv/bin/activate" >> ~/.bash_profile + source venv/bin/activate + echo ::endgroup:: + echo ::group::Install python dependencies + ./.install/common_installations.sh + echo ::endgroup:: + echo ::group::Unit tests + . "$HOME/.cargo/env" + make cargo_test LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }} + echo ::endgroup:: + echo ::group::Flow tests + make pytest LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }} ${{ inputs.test-config }} + echo ::endgroup:: + env: + PIP_BREAK_SYSTEM_PACKAGES: 1 + + - name: Upload test artifacts (node20 supported) + if: steps.node20.outputs.supported == 'true' && steps.test.outcome == 'failure' + uses: actions/upload-artifact@v4 + with: + name: Test logs ${{ steps.artifact-names.outputs.name }} + path: tests/**/logs/*.log* + if-no-files-found: ignore + - name: Upload test artifacts (node20 unsupported) + if: steps.node20.outputs.supported == 'false' && steps.test.outcome == 'failure' + uses: actions/upload-artifact@v4 + with: + name: Test logs ${{ steps.artifact-names.outputs.name }} + path: tests/**/logs/*.log* + if-no-files-found: ignore + + - name: Fail flow if tests failed + shell: bash + if: steps.test.outcome == 'failure' + run: exit 1 diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index c9b607ab4..a24cfba86 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -65,4 +65,4 @@ runs: id: set-redis-ref run: | export REDIS_REF="${{ inputs.redis-ref || '7.4'}}" - echo "REDIS_REF=${REDIS_REF}" >> $GITHUB_OUTPUT \ No newline at end of file + echo "REDIS_REF=${REDIS_REF}" >> $GITHUB_OUTPUT diff --git a/.github/actions/upload-artifacts-to-s3-without-make/action.yml b/.github/actions/upload-artifacts-to-s3-without-make/action.yml index 9a9a43833..b28937dcf 100644 --- a/.github/actions/upload-artifacts-to-s3-without-make/action.yml +++ b/.github/actions/upload-artifacts-to-s3-without-make/action.yml @@ -9,9 +9,6 @@ inputs: aws-secret-access-key: description: 'AWS Secret Access Key' required: true - github-ref: - description: 'GitHub ref' - required: true osnick: description: 'OS Nickname' required: false @@ -20,29 +17,40 @@ inputs: runs: using: composite steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: # todo: use role instead of access key - aws-access-key-id: ${{ inputs.aws-access-key-id }} - aws-secret-access-key: ${{ inputs.aws-secret-access-key }} - aws-region: "us-east-1" - - name: Upload artifacts to S3 - staging + - name: Configure AWS credentials and upload artifcats # todo - use aws role instead shell: bash run: | - echo ::group::install aws cli + echo ::group::install aws cli python3 -m venv .aws-cli-venv && source .aws-cli-venv/bin/activate && pip3 install --upgrade pip && pip3 install --no-cache-dir awscli && rm -rf /var/cache/apk/* - echo ::endgroup:: - echo ::group::upload artifacts - SNAPSHOT=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts - echo ::endgroup:: - echo ::group::upload staging release - RELEASE=1 SHOW=1 STAGING=1 VERBOSE=1 ./sbin/upload-artifacts - echo ::endgroup:: - - echo ::group::upload production release - # todo: trigger this manually instead - if [[ "${{ inputs.github-ref}}" != 'refs/heads/master' ]]; then - RELEASE=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts + echo ::endgroup:: + + # Variables from the workflow + export AWS_ACCESS_KEY_ID="${{ inputs.aws-access-key-id }}" + export AWS_SECRET_ACCESS_KEY="${{ inputs.aws-secret-access-key }}" + export AWS_REGION="us-east-1" + # Check if the required environment variables are set + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ] || [ -z "$AWS_REGION" ]; then + echo "Missing AWS credentials or region configuration." + exit 1 fi - echo ::endgroup:: \ No newline at end of file + # Configure AWS CLI with provided credentials and region + echo "Configuring AWS CLI with access keys..." + aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" + aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" + aws configure set region "$AWS_REGION" + + echo ::group::upload artifacts + SNAPSHOT=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts + echo ::endgroup:: + echo ::group::upload staging release + RELEASE=1 SHOW=1 STAGING=1 VERBOSE=1 ./sbin/upload-artifacts + echo ::endgroup:: + + echo ::group::upload production release + REF="${{ inputs.github-ref }}" + PATTERN="refs/tags/v[0-9]+.*" + if [[ $REF =~ $PATTERN ]]; then + RELEASE=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts + fi + echo ::endgroup:: diff --git a/.github/actions/upload-artifacts-to-s3/action.yml b/.github/actions/upload-artifacts-to-s3/action.yml index b0276085e..dfed5d609 100644 --- a/.github/actions/upload-artifacts-to-s3/action.yml +++ b/.github/actions/upload-artifacts-to-s3/action.yml @@ -20,12 +20,6 @@ inputs: runs: using: composite steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: # todo: use role instead of access key - aws-access-key-id: ${{ inputs.aws-access-key-id }} - aws-secret-access-key: ${{ inputs.aws-secret-access-key }} - aws-region: "us-east-1" - name: Upload artifacts to S3 - staging shell: bash run: | @@ -33,6 +27,22 @@ runs: python3 -m venv .aws-cli-venv && source .aws-cli-venv/bin/activate && pip3 install --upgrade pip && pip3 install --no-cache-dir awscli && rm -rf /var/cache/apk/* echo ::endgroup:: + + # Variables from the workflow + export AWS_ACCESS_KEY_ID="${{ inputs.aws-access-key-id }}" + export AWS_SECRET_ACCESS_KEY="${{ inputs.aws-secret-access-key }}" + export AWS_REGION="us-east-1" + # Check if the required environment variables are set + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ] || [ -z "$AWS_REGION" ]; then + echo "Missing AWS credentials or region configuration." + exit 1 + fi + # Configure AWS CLI with provided credentials and region + echo "Configuring AWS CLI with access keys..." + aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" + aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" + aws configure set region "$AWS_REGION" + echo ::group::upload artifacts make upload-artifacts SHOW=1 VERBOSE=1 echo ::endgroup:: @@ -45,4 +55,4 @@ runs: if [[ "${{ inputs.github-ref}}" != 'refs/heads/master' ]]; then make upload-release SHOW=1 VERBOSE=1 fi - echo ::endgroup:: \ No newline at end of file + echo ::endgroup:: diff --git a/.github/actions/upload-artifacts/action.yml b/.github/actions/upload-artifacts/action.yml new file mode 100644 index 000000000..f3f9ca630 --- /dev/null +++ b/.github/actions/upload-artifacts/action.yml @@ -0,0 +1,26 @@ +name: Pack the module and upload it to S3 +description: Pack the module and upload it to S3 + +inputs: + image: # The Docker image to use for the build + description: 'The Docker image to use for the build' + required: true + +runs: + using: composite + steps: + - name: Set Artifact Names + # Artifact names have to be unique, so we base them on the environment. + # We also remove invalid characters from the name. + id: artifact-names + shell: bash + run: | # Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ? + echo "name=$(echo "${{ inputs.image }} x86-64, Redis unstable" | \ + sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT + - name: Upload test artifacts + if: inputs.image != 'amazonlinux:2' && inputs.image != 'ubuntu:bionic' + uses: actions/upload-artifact@v4 + with: + name: Test logs ${{ steps.artifact-names.outputs.name }} + path: tests/**/logs/*.log* + if-no-files-found: ignore diff --git a/.github/workflows/alpine-arm.yml b/.github/workflows/alpine-arm.yml deleted file mode 100644 index 37a59e57d..000000000 --- a/.github/workflows/alpine-arm.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: alpine ARM64 - -on: - push: - branches: - - master - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - pull_request: - types: - - opened - - reopened - - review_requested - workflow_dispatch: # Allows you to run this workflow manually from the Actions tab - inputs: - redis-ref: - description: 'Redis ref to checkout' - required: true - default: 'unstable' - workflow_call: # Allows to run this workflow from another workflow - inputs: - redis-ref: - description: 'Redis ref to checkout' - type: string - required: true - -permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout - -jobs: - setup-environment: - runs-on: ubuntu-latest - outputs: - TAGGED: ${{ steps.set-env.outputs.TAGGED }} - TAG: ${{ steps.set-env.outputs.TAG }} - BRANCH: ${{ steps.set-env.outputs.BRANCH }} - TAG_OR_BRANCH: ${{ steps.set-env.outputs.TAG }}${{ steps.set-env.outputs.BRANCH }} - redis-ref: ${{ steps.set-env.outputs.redis-ref }} - steps: - - name: checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: set env - id: set-env - uses: ./.github/actions/setup-env - with: - github-ref: ${{ github.ref }} - redis-ref: ${{ inputs.redis-ref }} - - alpine-arm64: - runs-on: ubuntu24-arm64-4-16 # ubuntu24-arm64-2-8 - needs: setup-environment - defaults: - run: - shell: bash - env: - TAGGED: ${{ needs.setup-environment.outputs.TAGGED }} - VERSION: ${{ needs.setup-environment.outputs.TAG }} - BRANCH: ${{ needs.setup-environment.outputs.BRANCH }} - TAG_OR_BRANCH: ${{ needs.setup-environment.outputs.TAG_OR_BRANCH}} - - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16 - ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 - container: - image: alpine:3 # required to run the job on the ARM instance - steps: - - name: Workaround alpine-arm64 GHA issues - shell: sh - run: | - cp /etc/os-release /etc/os-release.bak - sed -i 's/ID=alpine/ID=NotpineForGHA/g' /etc/os-release - - name: Install prerequisites - shell: sh - run: | - echo ::group::install packages - apk add bash make libtool tar cmake python3 python3-dev \ - py3-pip gcc git curl build-base autoconf automake py3-cryptography \ - linux-headers musl-dev libffi-dev openssl-dev openssh py-virtualenv \ - clang18-libclang gcompat libstdc++ libgcc g++ openblas-dev \ - xsimd git xz bsd-compat-headers clang18 cargo - echo ::endgroup:: - - name: Checkout the module - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - name: Checkout Redis - uses: actions/checkout@v3 - with: - repository: 'redis/redis' - ref: ${{needs.setup-environment.outputs.redis-ref}} - path: 'redis' - - name: Install python dependencies - run: | - echo ::group::install requirements - pip install -q --upgrade setuptools - pip install -q --upgrade pip - pip install -q -r tests/pytest/requirements.txt - pip install -q -r .install/build_package_requirements.txt - echo ::endgroup:: - env: - PIP_BREAK_SYSTEM_PACKAGES: 1 - - name: build - uses: ./.github/actions/build-json-module-and-redis - - name: Test - run: | - make test - - name: Pack module - run: | - git config --global --add safe.directory $GITHUB_WORKSPACE # to avoid git error - mv /etc/os-release.bak /etc/os-release - make pack BRANCH=$TAG_OR_BRANCH SHOW=1 - sed -i 's/ID=alpine/ID=NotpineForGHA/g' /etc/os-release - - name: Upload artifacts to S3 - uses: ./.github/actions/upload-artifacts-to-s3 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - github-ref: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/alpine.yml b/.github/workflows/alpine.yml deleted file mode 100644 index 05c9266d7..000000000 --- a/.github/workflows/alpine.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: alpine - -on: - push: - branches: - - master - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - -jobs: - build: - runs-on: ubuntu-latest - defaults: - run: - shell: bash - container: - image: alpine:3 - steps: - - name: Install prerequisites - shell: sh - run: | - apk add bash make tar cargo python3 python3-dev py3-pip gcc git curl build-base autoconf automake py3-cryptography linux-headers musl-dev libffi-dev openssl-dev openssh py-virtualenv clang18-libclang - - name: Checkout the module - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - name: Install python dependencies - run: | - pip install -q --upgrade setuptools - pip install -q --upgrade pip - pip install -q -r tests/pytest/requirements.txt - pip install -q -r .install/build_package_requirements.txt - env: - PIP_BREAK_SYSTEM_PACKAGES: 1 - - name: Checkout Redis - uses: actions/checkout@v4 - with: - repository: redis/redis - ref: '7.4' - path: redis - - name: Build Redis - working-directory: redis - run: make install - - name: Build module - run: | - make build - - name: Test - run: | - make test - - name: Pack module - run: | - make pack BRANCH=${{ github.ref_name }} - - name: Upload artifacts to S3 - uses: ./.github/actions/upload-artifacts-to-s3 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - github-ref: ${{ github.ref }} diff --git a/.github/workflows/backport_pr.yml b/.github/workflows/backport_pr.yml index 8ac05f0b3..43156c1e4 100644 --- a/.github/workflows/backport_pr.yml +++ b/.github/workflows/backport_pr.yml @@ -26,7 +26,7 @@ jobs: contains(github.event.comment.body, '/backport') ) steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Create backport pull requests uses: korthout/backport-action@v1 with: diff --git a/.github/workflows/build-linux-platforms.yml b/.github/workflows/build-linux-platforms.yml deleted file mode 100644 index 59fd9aced..000000000 --- a/.github/workflows/build-linux-platforms.yml +++ /dev/null @@ -1,129 +0,0 @@ -name: Build all supported linux platforms - -on: - push: - branches: - - master - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - pull_request: - types: - - opened - - reopened - - review_requested - -jobs: - build-linux-matrix: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - docker_image: - - image: "ubuntu:bionic" - pre_req_install_cmd: | - # https://github.com/actions/checkout/issues/1809 - echo "ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION=node16" >> $GITHUB_ENV - echo "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION=node16" >> $GITHUB_ENV - # https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/ - echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV - apt-get update && apt-get install -y software-properties-common - add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git - - image: "ubuntu:focal" - # https://github.com/actions/checkout/issues/1386 - sparse checkout does not work on got 2.25 which is the default in focal - pre_req_install_cmd: | - apt-get update && apt-get install -y software-properties-common - add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git - - image: "ubuntu:jammy" - pre_req_install_cmd: | - apt-get update && apt-get install -y git - - image: "rockylinux:8" - pre_req_install_cmd: | - dnf install -y git - - image: "rockylinux:9" - pre_req_install_cmd: | - dnf install -y git - - image: "debian:bullseye" - pre_req_install_cmd: | - apt-get update && apt-get install -y git - - image: "amazonlinux:2" - pre_req_install_cmd: | - # https://github.com/actions/checkout/issues/1809 - echo "ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION=node16" >> $GITHUB_ENV - echo "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION=node16" >> $GITHUB_ENV - # https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/ - echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV - yum install -y git - - image: "mcr.microsoft.com/cbl-mariner/base/core:2.0" - pre_req_install_cmd: tdnf install --noplugins --skipsignature -y ca-certificates git - container: - image: ${{ matrix.docker_image.image }} - defaults: - run: - shell: bash -l -eo pipefail {0} - steps: - - name: Install git - run: ${{ matrix.docker_image.pre_req_install_cmd }} - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - name: Setup - working-directory: .install - run: | - echo ::group::Install dependencies - ./install_script.sh - echo ::endgroup:: - echo ::group::Install rust - ./getrust.sh - echo ::endgroup:: - - name: Get Redis - uses: actions/checkout@v3 - with: - repository: redis/redis - ref: 'unstable' # todo change per version/tag - path: redis - submodules: 'recursive' - - name: build - uses: ./.github/actions/build-json-module-and-redis-with-cargo - - name: Set Artifact Names - # Artifact names have to be unique, so we base them on the environment. - # We also remove invalid characters from the name. - id: artifact-names - run: | # Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ? - echo "name=$(echo "${{ matrix.docker_image.image }} x86-64, Redis unstable" | \ - sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT - - name: Run tests - run: | - echo ::group::Activate virtual environment - python3 -m venv venv - echo "source $PWD/venv/bin/activate" >> ~/.bash_profile - source venv/bin/activate - echo ::endgroup:: - echo ::group::Install python dependencies - ./.install/common_installations.sh - echo ::endgroup:: - echo ::group::Unit tests - cargo test - echo ::endgroup:: - echo ::group::Flow tests - MODULE=$(realpath ./target/release/rejson.so) RLTEST_ARGS='--no-progress' ./tests/pytest/tests.sh - echo ::endgroup:: - env: - PIP_BREAK_SYSTEM_PACKAGES: 1 - - name: Upload test artifacts - if: failure() - uses: actions/upload-artifact@v3 - with: - name: Test logs ${{ steps.artifact-names.outputs.name }} - path: tests/**/logs/*.log* - if-no-files-found: ignore - - name: Pack module - run: | - git config --global --add safe.directory /__w/RedisJSON/RedisJSON # to avoid git error - MODULE=$(realpath ./target/release/rejson.so) ./sbin/pack.sh - - name: Upload artifacts to S3 - uses: ./.github/actions/upload-artifacts-to-s3-without-make - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - github-ref: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/ci-basic.yml b/.github/workflows/ci-basic.yml deleted file mode 100644 index e400a4e25..000000000 --- a/.github/workflows/ci-basic.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: CI Basic - -on: - push: - paths-ignore: - - '.circleci/**' - - 'docs/**' - - '*.md' - branches-ignore: - - main - - master - - '[0-9]+.[0-9]+.[0-9]+' - - '[0-9]+.[0-9]+' - - 'feature-*' - tags-ignore: - - 'v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+-m[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+' - -jobs: - build-linux-jammy: - runs-on: "ubuntu-latest" - container: - image: "ubuntu:jammy" - defaults: - run: - shell: bash -l -eo pipefail {0} - steps: - - name: Install git - run: | - apt-get update && apt-get install -y git - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - name: Setup - working-directory: .install - run: | - echo ::group::Install dependencies - ./ubuntu_22.04.sh - echo ::endgroup:: - echo ::group::Install rust - ./getrust.sh - echo ::endgroup:: - - name: Get Redis - uses: actions/checkout@v3 - with: - repository: redis/redis - ref: 'unstable' # todo change per version/tag - path: redis - - name: Build Redis - working-directory: redis - run: | - make install - - name: Build module - run: | - . "$HOME/.cargo/env" - cargo --version - cargo build --release - - name: Set Artifact Names - # Artifact names have to be unique, so we base them on the environment. - # We also remove invalid characters from the name. - id: artifact-names - run: | # Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ? - echo "name=$(echo "ubuntu22 x86-64, Redis unstable" | \ - sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT - - name: Run tests - run: | - echo ::group::Activate virtual environment - ./.install/activate_venv.sh - echo ::endgroup:: - echo ::group::Install python dependencies - ./.install/common_installations.sh - echo ::endgroup:: - echo ::group::Flow tests - MODULE=$(realpath ./target/release/librejson.so) RLTEST_ARGS='--no-progress' ./tests/pytest/tests.sh - echo ::endgroup:: - echo ::group::Unit tests - cargo test - echo ::endgroup:: - env: - PIP_BREAK_SYSTEM_PACKAGES: 1 - - name: Upload artifacts - if: failure() - uses: actions/upload-artifact@v3 - with: - name: Test logs ${{ steps.artifact-names.outputs.name }} - path: tests/**/logs/*.log* - if-no-files-found: ignore \ No newline at end of file diff --git a/.github/workflows/event-ci.yml b/.github/workflows/event-ci.yml new file mode 100644 index 000000000..a47cc12f7 --- /dev/null +++ b/.github/workflows/event-ci.yml @@ -0,0 +1,59 @@ +name: Event CI + +permissions: + id-token: write + contents: read + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + docs-only: + uses: ./.github/workflows/task-check-docs.yml + prepare-values: + runs-on: ubuntu-latest + outputs: + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + steps: + - name: set env + id: set-env + run: | + echo "redis-ref=unstable" >> $GITHUB_OUTPUT # todo change per version/tag + linux: + uses: ./.github/workflows/flow-linux-x86.yml + needs: [prepare-values] + with: + os: jammy rocky9 amazonlinux2 + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + linux-valgrind: + uses: ./.github/workflows/flow-linux-x86.yml + needs: [prepare-values] + with: + os: jammy + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + run_valgrind: true + secrets: inherit + linux-sanitizer: + needs: [prepare-values, docs-only] + if: ${{ needs.docs-only.outputs.only-docs-changed == 'false' && !github.event.pull_request.draft }} + uses: ./.github/workflows/flow-sanitizer.yml + with: + container: ubuntu:jammy + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + linux-coverage: + needs: [prepare-values, docs-only] + if: ${{ needs.docs-only.outputs.only-docs-changed == 'false' && !github.event.pull_request.draft }} + uses: ./.github/workflows/flow-coverage.yml + secrets: inherit + spellcheck: + uses: ./.github/workflows/flow-spellcheck.yml + secrets: inherit + linter: + uses: ./.github/workflows/flow-linter.yml + secrets: inherit diff --git a/.github/workflows/event-nightly.yml b/.github/workflows/event-nightly.yml new file mode 100644 index 000000000..efbcdc84d --- /dev/null +++ b/.github/workflows/event-nightly.yml @@ -0,0 +1,71 @@ +name: Event Nightly + +permissions: + id-token: write + contents: read + +on: + schedule: + - cron: '20 20 * * *' # 20:20 UTC every day + workflow_dispatch: + inputs: + redis-ref: + description: 'Redis ref to checkout' + required: true + default: 'unstable' +jobs: + prepare-values: + runs-on: ubuntu-latest + outputs: + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + steps: + - name: set env + id: set-env + run: | + echo "redis-ref=unstable" >> $GITHUB_OUTPUT # todo change per version/tag + linux: + uses: ./.github/workflows/flow-linux-x86.yml + needs: [prepare-values] + with: + os: bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2 + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + ubuntu-arm64: + uses: ./.github/workflows/flow-ubuntu-arm.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + alpine: + uses: ./.github/workflows/flow-alpine.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + macos: + uses: ./.github/workflows/flow-macos.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + linux-valgrind: + uses: ./.github/workflows/flow-linux-x86.yml + needs: [prepare-values] + with: + os: jammy + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + run_valgrind: true + secrets: inherit + linux-sanitizer: + uses: ./.github/workflows/flow-sanitizer.yml + needs: [prepare-values] + with: + container: ubuntu:jammy + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + spellcheck: + uses: ./.github/workflows/flow-spellcheck.yml + secrets: inherit + linter: + uses: ./.github/workflows/flow-linter.yml + secrets: inherit diff --git a/.github/workflows/event-pull-request.yml b/.github/workflows/event-pull-request.yml deleted file mode 100644 index 1968c92fc..000000000 --- a/.github/workflows/event-pull-request.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Pull Request Flow - -on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] # Defaults + ready_for_review - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - docs-only: # Check whether the PR is only modifying docs - uses: ./.github/workflows/task-check-docs.yml - - coverage: - needs: docs-only - if: ${{ needs.docs-only.outputs.only-docs-changed == 'false' && !github.event.pull_request.draft }} - uses: ./.github/workflows/flow-coverage.yml - secrets: inherit - - sanitize: - needs: docs-only - if: ${{ needs.docs-only.outputs.only-docs-changed == 'false' && !github.event.pull_request.draft }} - uses: ./.github/workflows/flow-sanitizer.yml - secrets: inherit diff --git a/.github/workflows/event-tag.yml b/.github/workflows/event-tag.yml new file mode 100644 index 000000000..68dd5ff46 --- /dev/null +++ b/.github/workflows/event-tag.yml @@ -0,0 +1,58 @@ +name: Event TAG + +permissions: + id-token: write + contents: read + +on: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + workflow_dispatch: + inputs: + redis-ref: + description: 'Redis ref to checkout' + required: true + default: 'unstable' + +jobs: + prepare-values: + runs-on: ubuntu-latest + outputs: + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + steps: + - name: set env + id: set-env + run: | + echo "redis-ref=unstable" >> $GITHUB_OUTPUT # todo change per version/tag + linux: + uses: ./.github/workflows/flow-linux-x86.yml + needs: [prepare-values] + with: + os: bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2 + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + ubuntu-arm64: + uses: ./.github/workflows/flow-ubuntu-arm.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + alpine: + uses: ./.github/workflows/flow-alpine.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + macos: + uses: ./.github/workflows/flow-macos.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + spellcheck: + uses: ./.github/workflows/flow-spellcheck.yml + secrets: inherit + linter: + uses: ./.github/workflows/flow-linter.yml + secrets: inherit diff --git a/.github/workflows/event-weekly.yml b/.github/workflows/event-weekly.yml index e26cf60a0..8845fb00f 100644 --- a/.github/workflows/event-weekly.yml +++ b/.github/workflows/event-weekly.yml @@ -1,12 +1,73 @@ -name: Weekly Flow +name: Event Weekly on: schedule: - cron: "0 0 * * 0" +permissions: + id-token: write + contents: read + jobs: run-benchmarks: uses: ./.github/workflows/benchmark-runner.yml secrets: inherit with: extended: true + + prepare-values: + runs-on: ubuntu-latest + outputs: + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + steps: + - name: set env + id: set-env + run: | + echo "redis-ref=unstable" >> $GITHUB_OUTPUT # todo change per version/tag + linux: + uses: ./.github/workflows/flow-linux-x86.yml + needs: [prepare-values] + with: + # os: jammy rocky9 amazonlinux2 + os: bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2 + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + ubuntu-arm64: + uses: ./.github/workflows/flow-ubuntu-arm.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + alpine: + uses: ./.github/workflows/flow-alpine.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + macos: + uses: ./.github/workflows/flow-macos.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + linux-valgrind: + uses: ./.github/workflows/flow-linux-x86.yml + needs: [prepare-values] + with: + os: jammy + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + run_valgrind: true + secrets: inherit + linux-sanitizer: + uses: ./.github/workflows/flow-sanitizer.yml + needs: [prepare-values] + with: + container: ubuntu:jammy + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + spellcheck: + uses: ./.github/workflows/flow-spellcheck.yml + secrets: inherit + linter: + uses: ./.github/workflows/flow-linter.yml + secrets: inherit diff --git a/.github/workflows/flow-alpine.yml b/.github/workflows/flow-alpine.yml new file mode 100644 index 000000000..fe4a5d684 --- /dev/null +++ b/.github/workflows/flow-alpine.yml @@ -0,0 +1,110 @@ +name: Flow alpine + +permissions: + id-token: write + contents: read + +on: + workflow_dispatch: # Allows you to run this workflow manually from the Actions tab + inputs: + redis-ref: + description: 'Redis ref to checkout' + required: true + default: 'unstable' + run-test: + type: boolean + default: true + workflow_call: # Allows to run this workflow from another workflow + inputs: + redis-ref: + description: 'Redis ref to checkout' + type: string + required: true + run-test: + type: boolean + default: true + +jobs: + setup-environment: + runs-on: ubuntu-latest + outputs: + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + TAGGED: ${{ steps.set-env.outputs.TAGGED }} + TAG: ${{ steps.set-env.outputs.TAG }} + BRANCH: ${{ steps.set-env.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ steps.set-env.outputs.TAG }}${{ steps.set-env.outputs.BRANCH }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: set env + id: set-env + uses: ./.github/actions/setup-env + with: + github-ref: ${{ github.ref }} + redis-ref: ${{ inputs.redis-ref }} + build: + runs-on: ${{matrix.runs_on}} + needs: setup-environment + defaults: + run: + shell: bash + container: + image: alpine:3 + env: + TAGGED: ${{ needs.setup-environment.outputs.TAGGED }} + VERSION: ${{ needs.setup-environment.outputs.TAG }} + BRANCH: ${{ needs.setup-environment.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ needs.setup-environment.outputs.TAG_OR_BRANCH}} + PIP_BREAK_SYSTEM_PACKAGES: 1 + strategy: + matrix: + runs_on: [ubuntu24-arm64-4-16, ubuntu-latest] + steps: + - name: Install prerequisites + shell: sh + run: | + apk add bash make tar cargo python3 python3-dev py3-pip gcc git curl build-base autoconf automake py3-cryptography linux-headers musl-dev libffi-dev openssl-dev openssh py-virtualenv clang18-libclang + - name: git checkout + run: | + # Perform checkout + REPO_URL="https://github.com/${{ github.repository }}.git" + # Clone the repository to the current directory + git clone --recurse-submodules --depth=1 $REPO_URL . + git config --global --add safe.directory /__w/${{ github.repository }} + REF=${{github.sha}} + git fetch origin ${REF} + git checkout ${REF} + - name: Install python dependencies + run: | + pip install -q --upgrade setuptools + pip install -q --upgrade pip + pip install -q -r tests/pytest/requirements.txt + pip install -q -r .install/build_package_requirements.txt + env: + PIP_BREAK_SYSTEM_PACKAGES: 1 + - name: checkout redis + run: | + # Perform checkout + REPO_URL="https://github.com/redis/redis.git" + # Clone the repository to the current directory + git clone --recurse-submodules --depth=1 $REPO_URL redis + cd redis + git fetch origin ${{inputs.redis-ref}} + git checkout ${{inputs.redis-ref}} + - name: Build Redis + working-directory: redis + run: make install + - name: Build module + run: | + make build + - name: Run tests + if: ${{inputs.run-test}} + run: | + make test + - name: Pack module + uses: ./.github/actions/make-pack + - name: Upload artifacts to S3 + uses: ./.github/actions/upload-artifacts-to-s3-without-make + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/flow-linter.yml b/.github/workflows/flow-linter.yml new file mode 100644 index 000000000..0e0ee2114 --- /dev/null +++ b/.github/workflows/flow-linter.yml @@ -0,0 +1,15 @@ +name: Flow linter +on: + workflow_call: # Allows to run this workflow from another workflow + +jobs: + linter: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install build dependencies + run: sudo apt-get update && sudo apt-get install -y build-essential + - name: Linter + run: make lint diff --git a/.github/workflows/flow-linux-x86.yml b/.github/workflows/flow-linux-x86.yml new file mode 100644 index 000000000..ab1f13f20 --- /dev/null +++ b/.github/workflows/flow-linux-x86.yml @@ -0,0 +1,185 @@ +name: Build all supported linux platforms + +on: + workflow_dispatch: # Allows to run this workflow from another workflow + inputs: + redis-ref: + description: 'Redis ref to checkout' # todo change per version/tag + type: string + required: true + run-test: + type: boolean + default: true + os: + description: 'OS to build on, bash array style' + type: string + required: true + run_valgrind: + description: 'Run valgrind on the tests' + type: boolean + default: false + workflow_call: # Allows to run this workflow from another workflow + inputs: + redis-ref: + description: 'Redis ref to checkout' # todo change per version/tag + type: string + required: true + run-test: + type: boolean + default: true + os: + description: 'OS to build on, bash array style' + type: string + required: true + run_valgrind: + description: 'Run valgrind on the tests' + type: boolean + default: false + +jobs: + setup-environment: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + TAGGED: ${{ steps.set-env.outputs.TAGGED }} + TAG: ${{ steps.set-env.outputs.TAG }} + BRANCH: ${{ steps.set-env.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ steps.set-env.outputs.TAG }}${{ steps.set-env.outputs.BRANCH }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: set env + id: set-env + uses: ./.github/actions/setup-env + with: + github-ref: ${{ github.ref }} + redis-ref: ${{ inputs.redis-ref }} + - name: Set matrix + id: set-matrix + run: | + OS="${{ inputs.os }}" + if [ -z "${OS}" ]; then + OS="bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2" + fi + MATRIX="[" + for os in $OS; do + case $os in + bionic) + MATRIX="${MATRIX}{\"image\": \"ubuntu:bionic\", \"pre_req_install_cmd\": \"apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git\"}," + ;; + focal) + MATRIX="${MATRIX}{\"image\": \"ubuntu:focal\", \"pre_req_install_cmd\": \"apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git\"}," + ;; + jammy) + MATRIX="${MATRIX}{\"image\": \"ubuntu:jammy\", \"pre_req_install_cmd\": \"apt-get update && apt-get install -y git\"}," + ;; + rocky8) + MATRIX="${MATRIX}{\"image\": \"rockylinux:8\", \"pre_req_install_cmd\": \"dnf install -y git\"}," + ;; + rocky9) + MATRIX="${MATRIX}{\"image\": \"rockylinux:9\", \"pre_req_install_cmd\": \"dnf install -y git\"}," + ;; + bullseye) + MATRIX="${MATRIX}{\"image\": \"debian:bullseye\", \"pre_req_install_cmd\": \"apt-get update && apt-get install -y git\"}," + ;; + amazonlinux2) + MATRIX="${MATRIX}{\"image\": \"amazonlinux:2\", \"pre_req_install_cmd\": \"yum update -y && yum install -y git\"}," + ;; + mariner2) + MATRIX="${MATRIX}{\"image\": \"mcr.microsoft.com/cbl-mariner/base/core:2.0\", \"pre_req_install_cmd\": \"tdnf install --noplugins --skipsignature -y ca-certificates git\"}," + ;; + *) + echo "Unknown OS: $os" + exit 1 + ;; + esac + done + MATRIX="${MATRIX%?}]" + echo "${MATRIX}" + echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT + + build-linux-matrix: + name: ${{matrix.docker_image.image}}, ${{needs.setup-environment.outputs.redis-ref}} + runs-on: ubuntu-latest + needs: setup-environment + strategy: + fail-fast: false + matrix: + docker_image: ${{fromJson(needs.setup-environment.outputs.matrix)}} + env: + TAGGED: ${{ needs.setup-environment.outputs.TAGGED }} + VERSION: ${{ needs.setup-environment.outputs.TAG }} + BRANCH: ${{ needs.setup-environment.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ needs.setup-environment.outputs.TAG_OR_BRANCH}} + PIP_BREAK_SYSTEM_PACKAGES: 1 + container: + image: ${{ matrix.docker_image.image }} + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - name: Install git + run: ${{ matrix.docker_image.pre_req_install_cmd }} + - name: Checkout + if: matrix.docker_image.image != 'amazonlinux:2' && matrix.docker_image.image != 'ubuntu:bionic' + uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Checkout Redis + if: matrix.docker_image.image != 'amazonlinux:2' && matrix.docker_image.image != 'ubuntu:bionic' + uses: actions/checkout@v4 + with: + repository: redis/redis + ref: ${{needs.setup-environment.outputs.redis-ref}} + path: redis + submodules: 'recursive' + - name: git checkout + if: matrix.docker_image.image == 'amazonlinux:2' || matrix.docker_image.image == 'ubuntu:bionic' + run: | + # Perform checkout + REPO_URL="https://github.com/${{ github.repository }}.git" + # Clone the repository to the current directory + git config --global --add safe.directory /__w/${{ github.repository }} + git clone --recurse-submodules --depth=1 $REPO_URL . + REF=${{github.sha}} + git fetch origin ${REF} + git checkout ${REF} + + # Perform checkout + REPO_URL="https://github.com/redis/redis.git" + # Clone the repository to the current directory + git clone --recurse-submodules --depth=1 $REPO_URL redis + cd redis + git fetch origin ${{needs.setup-environment.outputs.redis-ref}} + git checkout ${{needs.setup-environment.outputs.redis-ref}} + - name: Setup + working-directory: .install + run: | + echo ::group::Install dependencies + ./install_script.sh + echo ::endgroup:: + echo ::group::Install rust + ./getrust.sh + echo ::endgroup:: + - name: build + uses: ./.github/actions/build-json-module-and-redis-with-cargo + - name: Run tests + if: ${{inputs.run-test}} + uses: ./.github/actions/run-tests + with: + run_valgrind: ${{inputs.run_valgrind && '1' || '0'}} + - name: Upload test artifacts + if: failure() + uses: ./.github/actions/upload-artifacts + with: + image: ${{ matrix.docker_image.image }} + - name: Pack module + if: ${{!inputs.run_valgrind}} + uses: ./.github/actions/pack-module + - name: Upload artifacts to S3 + if: ${{!inputs.run_valgrind}} + uses: ./.github/actions/upload-artifacts-to-s3-without-make + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/macos-build-and-deploy.yml b/.github/workflows/flow-macos.yml similarity index 58% rename from .github/workflows/macos-build-and-deploy.yml rename to .github/workflows/flow-macos.yml index 7850c2f69..8bf300a22 100644 --- a/.github/workflows/macos-build-and-deploy.yml +++ b/.github/workflows/flow-macos.yml @@ -1,21 +1,52 @@ name: Build for macos on: - push: - branches: - - master - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' workflow_dispatch: # Allows you to run this workflow manually from the Actions tab inputs: redis-ref: description: 'Redis ref to checkout' required: true default: 'unstable' + run-test: + type: boolean + default: true + workflow_call: # Allows you to run this workflow manually from the Actions tab + inputs: + redis-ref: + description: 'Redis ref to checkout' + type: string + default: 'unstable' + run-test: + type: boolean + default: true jobs: + setup-environment: + runs-on: ubuntu-latest + outputs: + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + TAGGED: ${{ steps.set-env.outputs.TAGGED }} + TAG: ${{ steps.set-env.outputs.TAG }} + BRANCH: ${{ steps.set-env.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ steps.set-env.outputs.TAG }}${{ steps.set-env.outputs.BRANCH }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: set env + id: set-env + uses: ./.github/actions/setup-env + with: + github-ref: ${{ github.ref }} + redis-ref: ${{ inputs.redis-ref }} build-macos-x64: runs-on: macos-13 + needs: setup-environment + env: + TAGGED: ${{ needs.setup-environment.outputs.TAGGED }} + VERSION: ${{ needs.setup-environment.outputs.TAG }} + BRANCH: ${{ needs.setup-environment.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ needs.setup-environment.outputs.TAG_OR_BRANCH}} + PIP_BREAK_SYSTEM_PACKAGES: 1 defaults: run: shell: bash -l -eo pipefail {0} @@ -25,7 +56,7 @@ jobs: with: submodules: 'recursive' - name: Deps checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: setup sparse-checkout-cone-mode: false @@ -33,15 +64,14 @@ jobs: .install tests/pytest/requirements.* - name: Setup specific - working-directory: setup/.install + working-directory: .install run: ./install_script.sh - name: Full checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Setup common run: .install/common_installations.sh - - name: Get Redis uses: actions/checkout@v4 with: @@ -55,27 +85,26 @@ jobs: run: | make build - name: Test + if: ${{inputs.run-test}} run: | make test - name: Pack module - run: | - make pack BRANCH=${{ github.ref_name }} - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 + run: make pack BRANCH=$TAG_OR_BRANCH + - name: Upload artifacts to S3 + uses: ./.github/actions/upload-artifacts-to-s3-without-make with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: "us-east-1" - - name: Upload artifacts to S3 - staging - run: | - make upload-artifacts SHOW=1 VERBOSE=1 - make upload-release SHOW=1 STAGING=1 VERBOSE=1 - - name: Upload artifacts to S3 - release # todo: trigger this manually instead - if: ${{ github.ref != 'refs/heads/master' }} - run: make upload-release SHOW=1 VERBOSE=1 build-macos-m1: runs-on: macos-latest-xlarge + needs: setup-environment + env: + TAGGED: ${{ needs.setup-environment.outputs.TAGGED }} + VERSION: ${{ needs.setup-environment.outputs.TAG }} + BRANCH: ${{ needs.setup-environment.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ needs.setup-environment.outputs.TAG_OR_BRANCH}} + PIP_BREAK_SYSTEM_PACKAGES: 1 defaults: run: shell: bash -l -eo pipefail {0} @@ -85,7 +114,7 @@ jobs: with: submodules: 'recursive' - name: Deps checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: setup sparse-checkout-cone-mode: false @@ -96,7 +125,7 @@ jobs: working-directory: setup/.install run: ./install_script.sh - name: Full checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Setup common @@ -105,12 +134,11 @@ jobs: python3 -m venv venv echo "source venv/bin/activate" >> ~/.bashrc echo "source venv/bin/activate" >> ~/.zshrc - source venv/bin/activate + . venv/bin/activate echo ::endgroup:: echo ::group::Install python dependencies ./.install/common_installations.sh echo ::endgroup:: - - name: Get Redis uses: actions/checkout@v4 with: @@ -124,21 +152,13 @@ jobs: run: | make build - name: Test + if: ${{inputs.run-test}} run: | make test - name: Pack module - run: | - make pack BRANCH=${{ github.ref_name }} - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 + run: make pack BRANCH=$TAG_OR_BRANCH + - name: Upload artifacts to S3 + uses: ./.github/actions/upload-artifacts-to-s3-without-make with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: "us-east-1" - - name: Upload artifacts to S3 - staging - run: | - make upload-artifacts SHOW=1 VERBOSE=1 - make upload-release SHOW=1 STAGING=1 VERBOSE=1 - - name: Upload artifacts to S3 - release # todo: trigger this manually instead - if: ${{ github.ref != 'refs/heads/master' }} - run: make upload-release SHOW=1 VERBOSE=1 \ No newline at end of file diff --git a/.github/workflows/flow-sanitizer.yml b/.github/workflows/flow-sanitizer.yml index bd47a3c6f..9c59d6eb0 100644 --- a/.github/workflows/flow-sanitizer.yml +++ b/.github/workflows/flow-sanitizer.yml @@ -8,6 +8,11 @@ on: container: default: "ubuntu:jammy" type: string + required: true + redis-ref: + description: 'Redis ref to checkout' + type: string + required: true jobs: clang-sanitizer: @@ -19,11 +24,12 @@ jobs: image: ${{ inputs.container }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: ./.github/actions/run-tests + - uses: ./.github/actions/san-run-tests with: container: ${{ inputs.container }} test-config: QUICK=1 sanitizer: addr + redis-ref: ${{inputs.redis-ref}} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/flow-spellcheck.yml similarity index 55% rename from .github/workflows/spellcheck.yml rename to .github/workflows/flow-spellcheck.yml index 8b445ccb1..bd8a5ba0b 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/flow-spellcheck.yml @@ -1,16 +1,14 @@ name: Spellcheck on: - push: - branches: [master] - pull_request: - branches: [master] + workflow_call: # Allows to run this workflow from another workflow + jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Spellcheck - uses: rojopolis/spellcheck-github-actions@0.30.0 + uses: rojopolis/spellcheck-github-actions@0.45.0 with: config_path: .github/spellcheck-settings.yml task_name: Markdown diff --git a/.github/workflows/ubuntu-arm.yml b/.github/workflows/flow-ubuntu-arm.yml similarity index 82% rename from .github/workflows/ubuntu-arm.yml rename to .github/workflows/flow-ubuntu-arm.yml index 2a0b5e36d..183dac731 100644 --- a/.github/workflows/ubuntu-arm.yml +++ b/.github/workflows/flow-ubuntu-arm.yml @@ -1,28 +1,24 @@ name: Build and Test ubuntu ARM instances on: - push: - branches: - - master - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - pull_request: - types: - - opened - - reopened - - review_requested workflow_dispatch: # Allows you to run this workflow manually from the Actions tab inputs: redis-ref: description: 'Redis ref to checkout' required: true default: 'unstable' + run-test: + type: boolean + default: true workflow_call: # Allows to run this workflow from another workflow inputs: redis-ref: description: 'Redis ref to checkout' type: string required: true + run-test: + type: boolean + default: true permissions: id-token: write # This is required for requesting the JWT @@ -40,7 +36,7 @@ jobs: redis-ref: ${{ steps.set-env.outputs.redis-ref }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: set env @@ -101,18 +97,26 @@ jobs: VERSION: ${{ needs.setup-environment.outputs.TAG }} BRANCH: ${{ needs.setup-environment.outputs.BRANCH }} TAG_OR_BRANCH: ${{ needs.setup-environment.outputs.TAG_OR_BRANCH}} - - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16 - ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 container: image: ${{ matrix.docker.image }} steps: - name: Install git run: | ${{ matrix.docker.install_git }} + - name: git checkout + if: matrix.docker.image == 'ubuntu:bionic' + run: | + # Perform checkout + REPO_URL="https://github.com/${{ github.repository }}.git" + # Clone the repository to the current directory + git config --global --add safe.directory /__w/${{ github.repository }} + git clone --recurse-submodules --depth=1 $REPO_URL . + REF=${{github.sha}} + git fetch origin ${REF} + git checkout ${REF} - name: Checkout the module - uses: actions/checkout@v3 + if: matrix.docker.image != 'ubuntu:bionic' + uses: actions/checkout@v4 with: submodules: 'recursive' - name: Install dependencies @@ -121,11 +125,19 @@ jobs: env: DEBIAN_FRONTEND: noninteractive - name: Checkout Redis - uses: actions/checkout@v3 + if: matrix.docker.image != 'ubuntu:bionic' + uses: actions/checkout@v4 with: repository: 'redis/redis' ref: ${{ needs.setup-environment.outputs.redis-ref }} path: 'redis' + - name: Get Redis + if: matrix.docker.image == 'ubuntu:bionic' + run: | + # Perform checkout + REPO_URL="https://github.com/redis/redis.git" + # Clone the repository to the current directory + git clone --recurse-submodules --depth=1 $REPO_URL redis - name: Get Rust run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y @@ -151,19 +163,14 @@ jobs: - name: build uses: ./.github/actions/build-json-module-and-redis-with-cargo - name: Test + if: ${{inputs.run-test}} run: | source venv/bin/activate MODULE=$(realpath ./target/release/rejson.so) RLTEST_ARGS='--no-progress' ./tests/pytest/tests.sh - name: Pack module - run: | - source venv/bin/activate - git config --global --add safe.directory /__w/RedisJSON/RedisJSON # to avoid git error - MODULE=$(realpath ./target/release/rejson.so) BRANCH=$TAG_OR_BRANCH \ - SHOW=1 OSNICK=${{ matrix.docker.nick }} ./sbin/pack.sh + uses: ./.github/actions/pack-module - name: Upload artifacts to S3 - uses: ./.github/actions/upload-artifacts-to-s3 + uses: ./.github/actions/upload-artifacts-to-s3-without-make with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - github-ref: ${{ github.ref }} - osnick: ${{ matrix.docker.nick }} \ No newline at end of file diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml index 99ec6078a..fab2384be 100644 --- a/.github/workflows/freebsd.yml +++ b/.github/workflows/freebsd.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: build uses: vmactions/freebsd-vm@v1 with: diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml deleted file mode 100644 index da24a8143..000000000 --- a/.github/workflows/macos.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: macos - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - - runs-on: macos-latest - - steps: - - uses: actions/checkout@v2 - - name: Build - run: cargo build --all --all-targets --verbose - - name: Run tests - run: cargo test --all --all-targets --verbose diff --git a/.github/workflows/trigger-build.yml b/.github/workflows/trigger-build.yml deleted file mode 100644 index 8c5dc3574..000000000 --- a/.github/workflows/trigger-build.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Trigger website deploy -on: - push: - branches: - - master - - '[0-9]+.[0-9]+' - paths: - - 'docs/**' - - 'commands.json' - -jobs: - trigger: - runs-on: ubuntu-latest - steps: - - run: | - echo "'$DATA'" | xargs \ - curl \ - -X POST https://api.netlify.com/build_hooks/${NETLIFY_BUILD_HOOK_ID} \ - -d - env: - NETLIFY_BUILD_HOOK_ID: ${{ secrets.NETLIFY_BUILD_HOOK_ID }} - DATA: '{"repository":"${{ github.repository }}", "sha":"${{ github.sha }}", "ref":"${{ github.ref }}"}}' diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml deleted file mode 100644 index d1753b202..000000000 --- a/.github/workflows/ubuntu.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: Build for Ubuntu - -on: - push: - branches: - - master - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - paths-ignore: - - '.circleci/**' - - 'docs/**' - - '*.md' - -# https://github.com/actions/checkout/issues/1487 -# A workaround for the old runtime is to use these environment -# variables, install the latest version of git within the container, -# and use the actions/checkout@v3 instead of @v4, until it is fixed. -env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16 - ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 - DEBIAN_FRONTEND: noninteractive - -jobs: - build: - strategy: - matrix: - docker_image: ["ubuntu:bionic", "ubuntu:focal", "ubuntu:jammy"] - runs-on: "ubuntu-latest" - container: - image: ${{ matrix.docker_image }} - defaults: - run: - shell: bash -l -eo pipefail {0} - steps: - - name: Update git - run: | - apt update - apt install -y software-properties-common - add-apt-repository ppa:git-core/ppa -y - apt update - apt install -y git - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - name: Install python 3.12 - run: | - apt install -y wget build-essential libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev - wget https://www.python.org/ftp/python/3.12.2/Python-3.12.2.tgz - tar xzf Python-3.12.2.tgz - cd Python-3.12.2 - ./configure --enable-optimizations - make -j - make altinstall - python3 --version - python3.12 --version - pip3.12 --version - update-alternatives --install /usr/bin/python3 python3 `which python3.12` 1 - python3 --version - - name: Install prerequisites - run: | - apt install -y make build-essential pkg-config python3-pip libclang-dev clang - ./deps/readies/bin/getaws - ./deps/readies/bin/getrust - python3 -m pip list - python3 -m pip install --upgrade setuptools six pip - pip install -q -r .install/build_package_requirements.txt - python3 -m pip install -r tests/pytest/requirements.txt - - name: Get Redis - uses: actions/checkout@v3 - with: - repository: redis/redis - ref: '7.4' - path: redis - - name: Build Redis - working-directory: redis - run: | - make -j - make install - - name: Build module - run: | - make build OSNICK=jammy VERSION=${{ github.ref_name }} TEST=1 OFFICIAL=1 SHOW=1 - - name: Test - run: | - make test - - name: Pack module - run: | - make pack BRANCH=${{ github.ref_name }} - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: "us-east-1" - - name: Upload artifacts to S3 - staging - run: | - make upload-artifacts SHOW=1 VERBOSE=1 - make upload-release SHOW=1 STAGING=1 VERBOSE=1 - - name: Upload artifacts to S3 - release # todo: trigger this manually instead - if: ${{ github.ref != 'refs/heads/master' }} - run: make upload-release SHOW=1 VERBOSE=1 - diff --git a/sbin/upload-artifacts b/sbin/upload-artifacts index 99818dd6d..2a4e681e9 100755 --- a/sbin/upload-artifacts +++ b/sbin/upload-artifacts @@ -89,8 +89,10 @@ else fi cd artifacts${MAYBE_SNAP} -# in some versions --apparent-size is not supported -[[ $VERBOSE == 1 ]] && du -ah --apparent-size * || du -ah -s * +if du --help | grep -q -- --apparent-size; then + DU_ARGS='--apparent-size' +fi +[[ $VERBOSE == 1 ]] && du -ah ${DU_ARGS} * #---------------------------------------------------------------------------------------------- From 6a78868e97f1b0d3befbfcfb943c59883c625636 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Tue, 24 Dec 2024 17:57:28 +0000 Subject: [PATCH 099/112] pass github-ref to upload artifacts action --- .../actions/upload-artifacts-to-s3-without-make/action.yml | 4 ++++ .github/workflows/benchmark-trigger.yml | 2 +- .github/workflows/flow-alpine.yml | 1 + .github/workflows/flow-linux-x86.yml | 1 + .github/workflows/flow-macos.yml | 1 + .github/workflows/flow-ubuntu-arm.yml | 1 + 6 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/actions/upload-artifacts-to-s3-without-make/action.yml b/.github/actions/upload-artifacts-to-s3-without-make/action.yml index b28937dcf..4e510ed40 100644 --- a/.github/actions/upload-artifacts-to-s3-without-make/action.yml +++ b/.github/actions/upload-artifacts-to-s3-without-make/action.yml @@ -13,6 +13,10 @@ inputs: description: 'OS Nickname' required: false default: '' + github-ref: + description: 'github ref' + required: false + default: '' runs: using: composite diff --git a/.github/workflows/benchmark-trigger.yml b/.github/workflows/benchmark-trigger.yml index 6f256c13b..4a9b949cc 100644 --- a/.github/workflows/benchmark-trigger.yml +++ b/.github/workflows/benchmark-trigger.yml @@ -1,4 +1,4 @@ -name: Check if needs trigger CircleCI benchmark +name: Check if needs trigger benchmark on: pull_request: diff --git a/.github/workflows/flow-alpine.yml b/.github/workflows/flow-alpine.yml index fe4a5d684..8a9a940ef 100644 --- a/.github/workflows/flow-alpine.yml +++ b/.github/workflows/flow-alpine.yml @@ -108,3 +108,4 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + github-ref: ${{ github.ref }} diff --git a/.github/workflows/flow-linux-x86.yml b/.github/workflows/flow-linux-x86.yml index ab1f13f20..c2191e59b 100644 --- a/.github/workflows/flow-linux-x86.yml +++ b/.github/workflows/flow-linux-x86.yml @@ -183,3 +183,4 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + github-ref: ${{ github.ref }} diff --git a/.github/workflows/flow-macos.yml b/.github/workflows/flow-macos.yml index 8bf300a22..f1f10d15e 100644 --- a/.github/workflows/flow-macos.yml +++ b/.github/workflows/flow-macos.yml @@ -162,3 +162,4 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + github-ref: ${{ github.ref }} diff --git a/.github/workflows/flow-ubuntu-arm.yml b/.github/workflows/flow-ubuntu-arm.yml index 183dac731..0f9272bb0 100644 --- a/.github/workflows/flow-ubuntu-arm.yml +++ b/.github/workflows/flow-ubuntu-arm.yml @@ -174,3 +174,4 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + github-ref: ${{ github.ref }} From a0bcd9d29ce4fc86b6373c91de02256285db834a Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Tue, 24 Dec 2024 18:12:45 +0000 Subject: [PATCH 100/112] update github-ref --- .github/workflows/flow-macos.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/flow-macos.yml b/.github/workflows/flow-macos.yml index f1f10d15e..a1dccb313 100644 --- a/.github/workflows/flow-macos.yml +++ b/.github/workflows/flow-macos.yml @@ -95,6 +95,8 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + github-ref: ${{ github.ref }} + build-macos-m1: runs-on: macos-latest-xlarge From fece5bfd453a99fdcda1294ad9fc559d4adcabb4 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Wed, 25 Dec 2024 08:00:49 +0000 Subject: [PATCH 101/112] update submodule after git checkout on CI --- .github/workflows/flow-alpine.yml | 2 ++ .github/workflows/flow-linux-x86.yml | 2 ++ .github/workflows/flow-ubuntu-arm.yml | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/.github/workflows/flow-alpine.yml b/.github/workflows/flow-alpine.yml index 8a9a940ef..ec5b7c305 100644 --- a/.github/workflows/flow-alpine.yml +++ b/.github/workflows/flow-alpine.yml @@ -74,6 +74,7 @@ jobs: REF=${{github.sha}} git fetch origin ${REF} git checkout ${REF} + git submodule update --init --recursive - name: Install python dependencies run: | pip install -q --upgrade setuptools @@ -91,6 +92,7 @@ jobs: cd redis git fetch origin ${{inputs.redis-ref}} git checkout ${{inputs.redis-ref}} + git submodule update --init --recursive - name: Build Redis working-directory: redis run: make install diff --git a/.github/workflows/flow-linux-x86.yml b/.github/workflows/flow-linux-x86.yml index c2191e59b..843a8c45c 100644 --- a/.github/workflows/flow-linux-x86.yml +++ b/.github/workflows/flow-linux-x86.yml @@ -145,6 +145,7 @@ jobs: REF=${{github.sha}} git fetch origin ${REF} git checkout ${REF} + git submodule update --init --recursive # Perform checkout REPO_URL="https://github.com/redis/redis.git" @@ -153,6 +154,7 @@ jobs: cd redis git fetch origin ${{needs.setup-environment.outputs.redis-ref}} git checkout ${{needs.setup-environment.outputs.redis-ref}} + git submodule update --init --recursive - name: Setup working-directory: .install run: | diff --git a/.github/workflows/flow-ubuntu-arm.yml b/.github/workflows/flow-ubuntu-arm.yml index 0f9272bb0..3a7edd44b 100644 --- a/.github/workflows/flow-ubuntu-arm.yml +++ b/.github/workflows/flow-ubuntu-arm.yml @@ -114,6 +114,7 @@ jobs: REF=${{github.sha}} git fetch origin ${REF} git checkout ${REF} + git submodule update --init --recursive - name: Checkout the module if: matrix.docker.image != 'ubuntu:bionic' uses: actions/checkout@v4 @@ -138,6 +139,10 @@ jobs: REPO_URL="https://github.com/redis/redis.git" # Clone the repository to the current directory git clone --recurse-submodules --depth=1 $REPO_URL redis + cd redis + git fetch origin ${{inputs.redis-ref}} + git checkout ${{inputs.redis-ref}} + git submodule update --init --recursive - name: Get Rust run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y From 12c30d30824af3bede0c191d806711b4cc14d955 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Thu, 26 Dec 2024 14:48:24 +0000 Subject: [PATCH 102/112] use the right redis-ref on CI --- .github/workflows/flow-alpine.yml | 6 +++--- .github/workflows/flow-linux-x86.yml | 2 +- .github/workflows/flow-macos.yml | 4 ++-- .github/workflows/flow-ubuntu-arm.yml | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/flow-alpine.yml b/.github/workflows/flow-alpine.yml index ec5b7c305..5ab4d4de0 100644 --- a/.github/workflows/flow-alpine.yml +++ b/.github/workflows/flow-alpine.yml @@ -88,10 +88,10 @@ jobs: # Perform checkout REPO_URL="https://github.com/redis/redis.git" # Clone the repository to the current directory - git clone --recurse-submodules --depth=1 $REPO_URL redis + git clone --recurse-submodules $REPO_URL redis cd redis - git fetch origin ${{inputs.redis-ref}} - git checkout ${{inputs.redis-ref}} + git fetch origin ${{ needs.setup-environment.outputs.redis-ref }} + git checkout ${{ needs.setup-environment.outputs.redis-ref }} git submodule update --init --recursive - name: Build Redis working-directory: redis diff --git a/.github/workflows/flow-linux-x86.yml b/.github/workflows/flow-linux-x86.yml index 843a8c45c..52a85f849 100644 --- a/.github/workflows/flow-linux-x86.yml +++ b/.github/workflows/flow-linux-x86.yml @@ -150,7 +150,7 @@ jobs: # Perform checkout REPO_URL="https://github.com/redis/redis.git" # Clone the repository to the current directory - git clone --recurse-submodules --depth=1 $REPO_URL redis + git clone --recurse-submodules $REPO_URL redis cd redis git fetch origin ${{needs.setup-environment.outputs.redis-ref}} git checkout ${{needs.setup-environment.outputs.redis-ref}} diff --git a/.github/workflows/flow-macos.yml b/.github/workflows/flow-macos.yml index a1dccb313..244fd4731 100644 --- a/.github/workflows/flow-macos.yml +++ b/.github/workflows/flow-macos.yml @@ -76,7 +76,7 @@ jobs: uses: actions/checkout@v4 with: repository: redis/redis - ref: ${{ inputs.redis-ref }} + ref: ${{ needs.setup-environment.outputs.redis-ref }} path: redis - name: Build Redis working-directory: redis @@ -145,7 +145,7 @@ jobs: uses: actions/checkout@v4 with: repository: redis/redis - ref: ${{ inputs.redis-ref }} + ref: ${{ needs.setup-environment.outputs.redis-ref }} path: redis - name: Build Redis working-directory: redis diff --git a/.github/workflows/flow-ubuntu-arm.yml b/.github/workflows/flow-ubuntu-arm.yml index 3a7edd44b..bcb106515 100644 --- a/.github/workflows/flow-ubuntu-arm.yml +++ b/.github/workflows/flow-ubuntu-arm.yml @@ -138,10 +138,10 @@ jobs: # Perform checkout REPO_URL="https://github.com/redis/redis.git" # Clone the repository to the current directory - git clone --recurse-submodules --depth=1 $REPO_URL redis + git clone --recurse-submodules $REPO_URL redis cd redis - git fetch origin ${{inputs.redis-ref}} - git checkout ${{inputs.redis-ref}} + git fetch origin ${{ needs.setup-environment.outputs.redis-ref }} + git checkout ${{ needs.setup-environment.outputs.redis-ref }} git submodule update --init --recursive - name: Get Rust run: | From c15c0f00f868bb7cea01170e5685afc5753eb61c Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Mon, 30 Dec 2024 10:20:12 +0200 Subject: [PATCH 103/112] MOD-8436 update upload artifacts script to use Ventura and Sonoma instead catalina and monterey (#1314) --- sbin/pack.sh | 10 ---------- sbin/upload-artifacts | 10 ---------- 2 files changed, 20 deletions(-) diff --git a/sbin/pack.sh b/sbin/pack.sh index 7abd60929..3738b905c 100755 --- a/sbin/pack.sh +++ b/sbin/pack.sh @@ -73,16 +73,6 @@ OSNICK=$($READIES/bin/platform --osnick) [[ $OSNICK == rocky8 ]] && OSNICK=rhel8 [[ $OSNICK == rocky9 ]] && OSNICK=rhel9 -if [[ $OS == macos ]]; then - # as we don't build on macOS for every platform, we converge to a least common denominator - if [[ $ARCH == x86_64 ]]; then - OSNICK=catalina # to be aligned with the rest of the modules in redis stack - else - [[ $OSNICK == ventura ]] && OSNICK=monterey - [[ $OSNICK == sonoma ]] && OSNICK=monterey - fi -fi - PLATFORM="$OS-$OSNICK-$ARCH" #---------------------------------------------------------------------------------------------- diff --git a/sbin/upload-artifacts b/sbin/upload-artifacts index 2a4e681e9..f0949f9cd 100755 --- a/sbin/upload-artifacts +++ b/sbin/upload-artifacts @@ -52,16 +52,6 @@ OS=$($READIES/bin/platform --os) [[ $OSNICK == rocky8 ]] && OSNICK=rhel8 [[ $OSNICK == rocky9 ]] && OSNICK=rhel9 -if [[ $OS == macos ]]; then - # as we don't build on macOS for every platform, we converge to a least common denominator - if [[ $ARCH == x86_64 ]]; then - OSNICK=catalina # to be aligned with the rest of the modules in redis stack - else - [[ $OSNICK == ventura ]] && OSNICK=monterey - [[ $OSNICK == sonoma ]] && OSNICK=monterey - fi -fi - PLATFORM="$OS-$OSNICK-$ARCH" #---------------------------------------------------------------------------------------------- From f611a961ff38fa18d555de28c3540e06ed45897b Mon Sep 17 00:00:00 2001 From: efeldblum <109016390+ephraimfeldblum@users.noreply.github.com> Date: Mon, 30 Dec 2024 15:46:27 +0200 Subject: [PATCH 104/112] fix get_memory to prevent crash on CRDT (#1313) --- redis_json/src/commands.rs | 10 ++-- redis_json/src/ivalue_manager.rs | 89 ++++++++++++++------------------ redis_json/src/manager.rs | 2 +- redis_json/src/redisjson.rs | 5 +- 4 files changed, 44 insertions(+), 62 deletions(-) diff --git a/redis_json/src/commands.rs b/redis_json/src/commands.rs index f18c886b6..05080fca2 100644 --- a/redis_json/src/commands.rs +++ b/redis_json/src/commands.rs @@ -1800,9 +1800,7 @@ pub fn json_debug(manager: M, ctx: &Context, args: Vec) let key = manager.open_key_read(ctx, &key)?; if path.is_legacy() { Ok(match key.get_value()? { - Some(doc) => { - manager.get_memory(KeyValue::new(doc).get_first(path.get_path())?)? - } + Some(doc) => M::get_memory(KeyValue::new(doc).get_first(path.get_path())?)?, None => 0, } .into()) @@ -1810,9 +1808,9 @@ pub fn json_debug(manager: M, ctx: &Context, args: Vec) Ok(match key.get_value()? { Some(doc) => KeyValue::new(doc) .get_values(path.get_path())? - .iter() - .map(|v| manager.get_memory(v).unwrap()) - .collect(), + .into_iter() + .map(M::get_memory) + .try_collect()?, None => vec![], } .into()) diff --git a/redis_json/src/ivalue_manager.rs b/redis_json/src/ivalue_manager.rs index 90abd50b3..7ba5b4eca 100644 --- a/redis_json/src/ivalue_manager.rs +++ b/redis_json/src/ivalue_manager.rs @@ -11,7 +11,7 @@ use crate::redisjson::normalize_arr_start_index; use crate::Format; use crate::REDIS_JSON_TYPE; use bson::{from_document, Document}; -use ijson::{DestructuredMut, INumber, IObject, IString, IValue, ValueType}; +use ijson::{DestructuredMut, DestructuredRef, INumber, IObject, IString, IValue}; use json_path::select_value::{SelectValue, SelectValueType}; use redis_module::key::{verify_type, KeyFlags, RedisKey, RedisKeyWritable}; use redis_module::raw::{RedisModuleKey, Status}; @@ -451,55 +451,45 @@ impl<'a> Manager for RedisIValueJsonKeyManager<'a> { /// /// following https://github.com/Diggsey/ijson/issues/23#issuecomment-1377270111 /// - fn get_memory(&self, v: &Self::V) -> Result { - let res = size_of::() - + match v.type_() { - ValueType::Null | ValueType::Bool => 0, - ValueType::Number => { - let num = v.as_number().unwrap(); - if num.has_decimal_point() { - // 64bit float - 16 - } else if num >= &INumber::from(-128) && num <= &INumber::from(383) { - // 8bit - 0 - } else if num > &INumber::from(-8_388_608) && num <= &INumber::from(8_388_607) { - // 24bit - 4 - } else { - // 64bit - 16 - } + fn get_memory(v: &Self::V) -> Result { + Ok(match v.destructure_ref() { + DestructuredRef::Null | DestructuredRef::Bool(_) => 0, + DestructuredRef::Number(num) => { + const STATIC_LO: i32 = -1 << 7; // INumber::STATIC_LOWER + const STATIC_HI: i32 = 0b11 << 7; // INumber::STATIC_UPPER + const SHORT_LO: i32 = -1 << 23; // INumber::SHORT_LOWER + const SHORT_HI: i32 = 1 << 23; // INumber::SHORT_UPPER + + if num.has_decimal_point() { + 16 // 64bit float + } else if &INumber::from(STATIC_LO) <= num && num < &INumber::from(STATIC_HI) { + 0 // 8bit + } else if &INumber::from(SHORT_LO) <= num && num < &INumber::from(SHORT_HI) { + 4 // 24bit + } else { + 16 // 64bit } - ValueType::String => v.as_string().unwrap().len(), - ValueType::Array => { - let arr = v.as_array().unwrap(); - let capacity = arr.capacity(); - if capacity == 0 { - 0 - } else { - size_of::() * (capacity + 2) - + arr - .into_iter() - .map(|v| self.get_memory(v).unwrap()) - .sum::() - } + } + DestructuredRef::String(s) => s.len(), + DestructuredRef::Array(arr) => match arr.capacity() { + 0 => 0, + capacity => { + arr.into_iter() // IValueManager::get_memory() always returns OK, safe to unwrap here + .map(|val| Self::get_memory(val).unwrap()) + .sum::() + + (capacity + 2) * size_of::() } - ValueType::Object => { - let val = v.as_object().unwrap(); - let capacity = val.capacity(); - if capacity == 0 { - 0 - } else { - size_of::() * (capacity * 3 + 2) - + val - .into_iter() - .map(|(s, v)| s.len() + self.get_memory(v).unwrap()) - .sum::() - } + }, + DestructuredRef::Object(obj) => match obj.capacity() { + 0 => 0, + capacity => { + obj.into_iter() // IValueManager::get_memory() always returns OK, safe to unwrap here + .map(|(s, val)| s.len() + Self::get_memory(val).unwrap()) + .sum::() + + (capacity * 3 + 2) * size_of::() } - }; - Ok(res) + }, + } + size_of::()) } fn is_json(&self, key: *mut RedisModuleKey) -> Result { @@ -521,9 +511,6 @@ mod tests { fn test_get_memory() { let _guard = SINGLE_THREAD_TEST_MUTEX.lock(); - let manager = RedisIValueJsonKeyManager { - phantom: PhantomData, - }; let json = r#"{ "a": 100.12, "b": "foo", @@ -540,7 +527,7 @@ mod tests { "m": {"t": "f"} }"#; let value = serde_json::from_str(json).unwrap(); - let res = manager.get_memory(&value).unwrap(); + let res = RedisIValueJsonKeyManager::get_memory(&value).unwrap(); assert_eq!(res, 903); } diff --git a/redis_json/src/manager.rs b/redis_json/src/manager.rs index 96b782163..6473040b5 100644 --- a/redis_json/src/manager.rs +++ b/redis_json/src/manager.rs @@ -95,7 +95,7 @@ pub trait Manager { fn apply_changes(&self, ctx: &Context); #[allow(clippy::wrong_self_convention)] fn from_str(&self, val: &str, format: Format, limit_depth: bool) -> Result; - fn get_memory(&self, v: &Self::V) -> Result; + fn get_memory(v: &Self::V) -> Result; fn is_json(&self, key: *mut RedisModuleKey) -> Result; } diff --git a/redis_json/src/redisjson.rs b/redis_json/src/redisjson.rs index 1c1263c8b..7fcff2a63 100644 --- a/redis_json/src/redisjson.rs +++ b/redis_json/src/redisjson.rs @@ -251,9 +251,6 @@ pub mod type_methods { #[allow(non_snake_case, unused)] pub unsafe extern "C" fn mem_usage(value: *const c_void) -> usize { let json = unsafe { &*(value as *mut RedisJSON) }; - let manager = RedisIValueJsonKeyManager { - phantom: PhantomData, - }; - manager.get_memory(&json.data).unwrap_or(0) + RedisIValueJsonKeyManager::get_memory(&json.data).unwrap_or(0) } } From b47f0d75a050a52c3d7adc77f473223e917f3d08 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Thu, 2 Jan 2025 20:45:05 +0000 Subject: [PATCH 105/112] fix setup-env action to correctly output branch name and update event triggers in workflow --- .github/actions/setup-env/action.yml | 25 ++++++++++++++++--------- .github/workflows/event-nightly.yml | 6 ++++++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index a24cfba86..3e047a91e 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -19,7 +19,7 @@ outputs: value: ${{ steps.set-git-info.outputs.TAG }} BRANCH: description: 'The branch name' - value: ${{ steps.set-git-info.outputs.TAG }} + value: ${{ steps.set-git-info.outputs.BRANCH }} TAG_OR_BRANCH: description: 'The tag or branch name' value: ${{ steps.set-git-info.outputs.TAG }}${{ steps.set-git-info.outputs.BRANCH }} @@ -34,17 +34,24 @@ runs: shell: bash id: set-git-info run: | - export REF="${{ inputs.github-ref }}" - export BRANCH_PATTERN="^refs/heads/(.*)$" - export TAG_PATTERN="^refs/tags/(.*)$" + if [[ "${{github.event_name}}" == "pull_request" ]]; then + BRANCH="${{github.event.pull_request.base.ref}}" + else + REF="${{ github.ref }}" + BRANCH_PATTERN="^refs/heads/(.*)$" + TAG_PATTERN="^refs/tags/(.*)$" - if [[ $REF =~ $BRANCH_PATTERN ]]; then - echo "BRANCH=${BASH_REMATCH[1]}" >> $GITHUB_OUTPUT - fi + if [[ $REF =~ $BRANCH_PATTERN ]]; then + BRANCH=${BASH_REMATCH[1]} + fi - if [[ $REF =~ $TAG_PATTERN ]]; then - echo "TAG=${BASH_REMATCH[1]}" >> $GITHUB_OUTPUT + if [[ $REF =~ $TAG_PATTERN ]]; then + TAG=${BASH_REMATCH[1]} + fi fi + echo "TAG=${TAG}" >> $GITHUB_OUTPUT + echo "BRANCH=${BRANCH}" >> $GITHUB_OUTPUT + echo "TAG=${TAG}, BRANCH=${BRANCH}" - name: Set the tagged flag shell: bash id: set-tagged diff --git a/.github/workflows/event-nightly.yml b/.github/workflows/event-nightly.yml index efbcdc84d..bb2101cad 100644 --- a/.github/workflows/event-nightly.yml +++ b/.github/workflows/event-nightly.yml @@ -5,6 +5,12 @@ permissions: contents: read on: + push: + branches: + - main + - master + - '[0-9]+.[0-9]+.[0-9]+' + - '[0-9]+.[0-9]+' schedule: - cron: '20 20 * * *' # 20:20 UTC every day workflow_dispatch: From d69b6d43dfcd1384e3276a239319c21b0afd0bf0 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Fri, 10 Jan 2025 00:48:54 +0200 Subject: [PATCH 106/112] enhance pack.sh to support module path and improve package generation options (#1319) --- .github/actions/make-pack/action.yml | 2 +- .github/actions/pack-module/action.yml | 3 +- .github/actions/setup-env/action.yml | 14 +- Makefile | 2 +- ramp.yml => pack/ramp.yml | 0 sbin/pack.sh | 215 +++++++++++++++++-------- sbin/upload-artifacts | 18 ++- 7 files changed, 164 insertions(+), 90 deletions(-) rename ramp.yml => pack/ramp.yml (100%) diff --git a/.github/actions/make-pack/action.yml b/.github/actions/make-pack/action.yml index 27170a11b..4b541eae0 100644 --- a/.github/actions/make-pack/action.yml +++ b/.github/actions/make-pack/action.yml @@ -6,4 +6,4 @@ runs: - name: Pack module shell: bash run: | - make pack BRANCH=$TAG_OR_BRANCH + BRANCH=$TAG_OR_BRANCH make pack diff --git a/.github/actions/pack-module/action.yml b/.github/actions/pack-module/action.yml index ceb84b30c..3b958449a 100644 --- a/.github/actions/pack-module/action.yml +++ b/.github/actions/pack-module/action.yml @@ -13,5 +13,4 @@ runs: . venv/bin/activate git config --global --add safe.directory $GITHUB_WORKSPACE export PATH="$GITHUB_WORKSPACE/redis/src:$PATH" - MODULE=$(realpath ./target/release/rejson.so) BRANCH=$TAG_OR_BRANCH \ - SHOW=1 OSNICK=${{ matrix.docker.nick }} ./sbin/pack.sh + BRANCH=$TAG_OR_BRANCH SHOW=1 OSNICK=${{ matrix.docker.nick }} ./sbin/pack.sh $(realpath ./target/release/rejson.so) diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index 3e047a91e..dc460200d 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -34,21 +34,17 @@ runs: shell: bash id: set-git-info run: | - if [[ "${{github.event_name}}" == "pull_request" ]]; then - BRANCH="${{github.event.pull_request.base.ref}}" - else + if [[ "${{github.event_name}}" != "pull_request" ]]; then REF="${{ github.ref }}" - BRANCH_PATTERN="^refs/heads/(.*)$" TAG_PATTERN="^refs/tags/(.*)$" - - if [[ $REF =~ $BRANCH_PATTERN ]]; then - BRANCH=${BASH_REMATCH[1]} - fi - if [[ $REF =~ $TAG_PATTERN ]]; then TAG=${BASH_REMATCH[1]} fi fi + + if [[ -z $TAG ]]; then + BRANCH=${{ github.head_ref || github.ref_name }} + fi echo "TAG=${TAG}" >> $GITHUB_OUTPUT echo "BRANCH=${BRANCH}" >> $GITHUB_OUTPUT echo "TAG=${TAG}, BRANCH=${BRANCH}" diff --git a/Makefile b/Makefile index bd04658c2..d95adb322 100644 --- a/Makefile +++ b/Makefile @@ -222,7 +222,7 @@ bench benchmark: $(TARGET) #---------------------------------------------------------------------------------------------- pack: - $(SHOW)MODULE=$(abspath $(TARGET)) ./sbin/pack.sh + $(SHOW) BINDIR=$(BINDIR) ./sbin/pack.sh $(abspath $(TARGET)) upload-release: $(SHOW)RELEASE=1 ./sbin/upload-artifacts diff --git a/ramp.yml b/pack/ramp.yml similarity index 100% rename from ramp.yml rename to pack/ramp.yml diff --git a/sbin/pack.sh b/sbin/pack.sh index 3738b905c..91d029c28 100755 --- a/sbin/pack.sh +++ b/sbin/pack.sh @@ -18,14 +18,17 @@ if [[ $1 == --help || $1 == help || $HELP == 1 ]]; then cat <<-END Generate RedisJSON distribution packages. - [ARGVARS...] pack.sh [--help|help] + [ARGVARS...] pack.sh [--help|help] [] Argument variables: - MODULE=path Path of module .so - RAMP=0|1 Build RAMP package DEPS=0|1 Build dependencies files SYM=0|1 Build debug symbols file + RELEASE=1 Generate "release" packages (artifacts/release/) + SNAPSHOT=1 Generate "shapshot" packages (artifacts/snapshot/) + + MODULE_NAME=name Module name (default: bf) + PACKAGE_NAME=name Package stem name BRANCH=name Branch name for snapshot packages WITH_GITSHA=1 Append Git SHA to shapshot package names @@ -33,7 +36,7 @@ if [[ $1 == --help || $1 == help || $HELP == 1 ]]; then RAMP_VARIANT=name RAMP variant (e.g. ramp-{name}.yml) ARTDIR=dir Directory in which packages are created (default: bin/artifacts) - + RAMP_YAML=path RAMP configuration file path RAMP_ARGS=args Extra arguments to RAMP @@ -53,8 +56,10 @@ OP="" # RLEC naming conventions -ARCH=$($READIES/bin/platform --arch) -[[ $ARCH == x64 ]] && ARCH=x86_64 +ARCH=$(uname -m) + +[[ $ARCH == x64 ]] && ARCH=x86_64 +[[ $ARCH == arm64 ]] && ARCH=aarch64 [[ $ARCH == arm64v8 ]] && ARCH=aarch64 OS=$($READIES/bin/platform --os) @@ -73,23 +78,26 @@ OSNICK=$($READIES/bin/platform --osnick) [[ $OSNICK == rocky8 ]] && OSNICK=rhel8 [[ $OSNICK == rocky9 ]] && OSNICK=rhel9 + PLATFORM="$OS-$OSNICK-$ARCH" #---------------------------------------------------------------------------------------------- -if [[ -z $MODULE || ! -f $MODULE ]]; then - eprint "MODULE is not defined or does not refer to a file" - exit 1 -fi +MODULE="$1" RAMP=${RAMP:-1} DEPS=${DEPS:-1} SYM=${SYM:-1} +RELEASE=${RELEASE:-1} +SNAPSHOT=${SNAPSHOT:-1} + [[ -z $ARTDIR ]] && ARTDIR=bin/artifacts mkdir -p $ARTDIR $ARTDIR/snapshots ARTDIR=$(cd $ARTDIR && pwd) +#---------------------------------------------------------------------------------------------- + MODULE_NAME=${MODULE_NAME:-ReJSON} PACKAGE_NAME=rejson-oss @@ -103,24 +111,43 @@ pack_ramp() { cd $ROOT local stem=${PACKAGE_NAME}.${PLATFORM} + local stem_debug=${PACKAGE_NAME}.debug.${PLATFORM} + + if [[ $SNAPSHOT == 0 ]]; then + local verspec=${SEMVER}${VARIANT} + local packdir=. + local s3base="" + else + local verspec=${BRANCH}${VARIANT} + local packdir=snapshots + local s3base=snapshots/ + fi - local verspec=${SEMVER}${_VARIANT} - local fq_package=$stem.${verspec}.zip + local fq_package_debug=$stem_debug.${verspec}.zip - [[ ! -d $ARTDIR ]] && mkdir -p $ARTDIR + [[ ! -d $ARTDIR/$packdir ]] && mkdir -p $ARTDIR/$packdir - local packfile="$ARTDIR/$fq_package" + local packfile=$ARTDIR/$packdir/$fq_package + local packfile_debug=$ARTDIR/$packdir/$fq_package_debug local xtx_vars="" - local dep_fname="${PACKAGE_NAME}.${PLATFORM}.${verspec}.tgz" + for dep in $DEP_NAMES; do + eval "export NAME_${dep}=${PACKAGE_NAME}_${dep}" + local dep_fname="${PACKAGE_NAME}.${dep}.${PLATFORM}.${verspec}.tgz" + eval "export PATH_${dep}=${s3base}${dep_fname}" + local dep_sha256="$ARTDIR/$packdir/${dep_fname}.sha256" + eval "export SHA256_${dep}=$(cat $dep_sha256)" + + xtx_vars+=" -e NAME_$dep -e PATH_$dep -e SHA256_$dep" + done - if [[ -z $RAMP_YAML ]]; then - RAMP_YAML=$ROOT/ramp.yml + if [[ -n $RAMP_YAML ]]; then + RAMP_YAML="$(realpath $RAMP_YAML)" elif [[ -z $RAMP_VARIANT ]]; then - RAMP_YAML=$ROOT/ramp.yml + RAMP_YAML="$ROOT/pack/ramp.yml" else - RAMP_YAML=$ROOT/ramp${_RAMP_VARIANT}.yml + RAMP_YAML="$ROOT/pack/ramp${_RAMP_VARIANT}.yml" fi python3 $READIES/bin/xtx \ @@ -133,7 +160,7 @@ pack_ramp() { fi runn rm -f /tmp/ramp.fname $packfile - + # ROOT is required so ramp will detect the right git commit cd $ROOT runn @ <<-EOF @@ -155,18 +182,35 @@ pack_ramp() { exit 1 else local packname=`cat /tmp/ramp.fname` - echo "# Created $packname" + echo "# Created $(realpath $packname)" fi fi - cd $ARTDIR/snapshots - if [[ ! -z $BRANCH ]]; then - local snap_package=$stem.${BRANCH}${_VARIANT}.zip - ln -sf ../$fq_package $snap_package + if [[ -f $MODULE.debug ]]; then + runn @ <<-EOF + $RAMP_CMD pack -m /tmp/ramp.yml \ + $RAMP_ARGS \ + -n $MODULE_NAME \ + --verbose \ + --debug \ + --packname-file /tmp/ramp.fname \ + -o $packfile_debug \ + $MODULE.debug \ + >/tmp/ramp.err 2>&1 || true + EOF + + if [[ $NOP != 1 ]]; then + if [[ ! -e $packfile_debug ]]; then + eprint "Error generating RAMP file:" + >&2 cat /tmp/ramp.err + exit 1 + else + local packname=`cat /tmp/ramp.fname` + echo "# Created $(realpath $packname)" + fi + fi fi - local packname=`cat /tmp/ramp.fname` - echo "Created $packname" cd $ROOT } @@ -175,37 +219,47 @@ pack_ramp() { pack_deps() { local dep="$1" + cd $ROOT + local stem=${PACKAGE_NAME}.${dep}.${PLATFORM} - local verspec=${SEMVER}${_VARIANT} + local verspec=${SEMVER}${VARIANT} + local fq_package=$stem.${verspec}.tgz local depdir=$(cat $ARTDIR/$dep.dir) - - local fq_dep=$stem.${verspec}.tgz - local tar_path=$ARTDIR/$fq_dep + local tar_path=$ARTDIR/$fq_package local dep_prefix_dir=$(cat $ARTDIR/$dep.prefix) - - { cd $depdir ;\ - cat $ARTDIR/$dep.files | \ - xargs tar -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' \ - --transform "s,^,$dep_prefix_dir," 2> /tmp/pack.err | \ - gzip -n - > $tar_path ; E=$?; } || true - rm -f $ARTDIR/$dep.prefix $ARTDIR/$dep.files $ARTDIR/$dep.dir - cd $ROOT - if [[ $E != 0 || -s /tmp/pack.err ]]; then - eprint "Error creating $tar_path:" - cat /tmp/pack.err >&2 - exit 1 + rm -f $tar_path + if [[ $NOP != 1 ]]; then + { cd $depdir ;\ + cat $ARTDIR/$dep.files | \ + xargs tar -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' \ + --transform "s,^,$dep_prefix_dir," 2> /tmp/pack.err | \ + gzip -n - > $tar_path ; E=$?; } || true + if [[ ! -e $tar_path || -z $(tar tzf $tar_path) ]]; then + eprint "Count not create $tar_path. Aborting." + rm -f $tar_path + exit 1 + fi + else + runn @ <<-EOF + cd $depdir + cat $ARTDIR/$dep.files | \ + xargs tar -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' \ + --transform "s,^,$dep_prefix_dir," 2> /tmp/pack.err | \ + gzip -n - > $tar_path ; E=$?; } || true + EOF fi runn @ <<-EOF - sha256sum $tar_path | awk '{print $1}' > $tar_path.sha256 + sha256sum $tar_path | gawk '{print $1}' > $tar_path.sha256 EOF + mkdir -p $ARTDIR/snapshots cd $ARTDIR/snapshots if [[ -n $BRANCH ]]; then - local snap_dep=$stem.${BRANCH}${_VARIANT}.tgz - runn ln -sf ../$fq_dep $snap_dep - runn ln -sf ../$fq_dep.sha256 $snap_dep.sha256 + local snap_package=$stem.${BRANCH}${VARIANT}.tgz + runn ln -sf ../$fq_package $snap_package + runn ln -sf ../$fq_package.sha256 $snap_package.sha256 fi cd $ROOT @@ -213,18 +267,6 @@ pack_deps() { #---------------------------------------------------------------------------------------------- -prepare_symbols_dep() { - if [[ ! -f $MODULE.debug ]]; then return 0; fi - echo "# Preparing debug symbols dependencies ..." - echo $(cd "$(dirname $MODULE)" && pwd) > $ARTDIR/debug.dir - echo $(basename $MODULE.debug) > $ARTDIR/debug.files - echo "" > $ARTDIR/debug.prefix - pack_deps debug - echo "# Done." -} - -#---------------------------------------------------------------------------------------------- - NUMVER="$(NUMERIC=1 $SBIN/getver)" SEMVER="$($SBIN/getver)" @@ -237,7 +279,16 @@ fi #---------------------------------------------------------------------------------------------- +git_config_add_ifnx() { + local key="$1" + local val="$2" + if [[ -z $(git config --global --get $key $val) ]]; then + git config --global --add $key $val + fi +} + if [[ -z $BRANCH ]]; then + git_config_add_ifnx safe.directory $ROOT BRANCH=$(git rev-parse --abbrev-ref HEAD) # this happens of detached HEAD if [[ $BRANCH == HEAD ]]; then @@ -246,21 +297,33 @@ if [[ -z $BRANCH ]]; then fi BRANCH=${BRANCH//[^A-Za-z0-9._-]/_} if [[ $WITH_GITSHA == 1 ]]; then + git_config_add_ifnx safe.directory $ROOT GIT_COMMIT=$(git rev-parse --short HEAD) BRANCH="${BRANCH}-${GIT_COMMIT}" fi -export BRANCH + +#---------------------------------------------------------------------------------------------- + +RELEASE_ramp=${PACKAGE_NAME}.$OS-$OSNICK-$ARCH.$SEMVER${VARIANT}.zip +SNAPSHOT_ramp=${PACKAGE_NAME}.$OS-$OSNICK-$ARCH.${BRANCH}${VARIANT}.zip + +RELEASE_deps= +SNAPSHOT_deps= +for dep in $DEP_NAMES; do + RELEASE_deps+=" ${PACKAGE_NAME}.${dep}.$OS-$OSNICK-$ARCH.$SEMVER${VARIANT}.tgz" + SNAPSHOT_deps+=" ${PACKAGE_NAME}.${dep}.$OS-$OSNICK-$ARCH.${BRANCH}${VARIANT}.tgz" +done #---------------------------------------------------------------------------------------------- if [[ $JUST_PRINT == 1 ]]; then if [[ $RAMP == 1 ]]; then - echo "${PACKAGE_NAME}.${OS}-${OSNICK}-${ARCH}.${SEMVER}${VARIANT}.zip" + [[ $RELEASE == 1 ]] && echo $RELEASE_ramp + [[ $SNAPSHOT == 1 ]] && echo $SNAPSHOT_ramp fi if [[ $DEPS == 1 ]]; then - for dep in $DEP_NAMES; do - echo "${PACKAGE_NAME}.${dep}.${OS}-${OSNICK}-${ARCH}.${SEMVER}${VARIANT}.tgz" - done + [[ $RELEASE == 1 ]] && echo $RELEASE_deps + [[ $SNAPSHOT == 1 ]] && echo $SNAPSHOT_deps fi exit 0 fi @@ -269,14 +332,19 @@ fi mkdir -p $ARTDIR -if [[ $DEPS == 1 ]]; then - echo "# Building dependencies ..." +if [[ $DEPS == 1 && -n $DEP_NAMES ]]; then + # set up `debug` dep + dirname "$(realpath "$MODULE")" > "$ARTDIR/debug.dir" + echo "$(basename "$(realpath "$MODULE")").debug" > "$ARTDIR/debug.files" + echo "" > $ARTDIR/debug.prefix - [[ $SYM == 1 ]] && prepare_symbols_dep + echo "# Building dependencies ..." for dep in $DEP_NAMES; do - echo "# $dep ..." - pack_deps $dep + if [[ $OS != macos ]]; then + echo "# $dep ..." + pack_deps $dep + fi done echo "# Done." fi @@ -292,7 +360,14 @@ if [[ $RAMP == 1 ]]; then fi echo "# Building RAMP $RAMP_VARIANT files ..." - pack_ramp + + [[ -z $MODULE ]] && { eprint "Nothing to pack. Aborting."; exit 1; } + [[ ! -f $MODULE ]] && { eprint "$MODULE does not exist. Aborting."; exit 1; } + MODULE=$(realpath $MODULE) + + [[ $RELEASE == 1 ]] && SNAPSHOT=0 pack_ramp + [[ $SNAPSHOT == 1 ]] && pack_ramp + echo "# Done." fi diff --git a/sbin/upload-artifacts b/sbin/upload-artifacts index f0949f9cd..be4f4620a 100755 --- a/sbin/upload-artifacts +++ b/sbin/upload-artifacts @@ -6,6 +6,7 @@ ROOT=$(cd $HERE/.. && pwd) READIES=$ROOT/deps/readies . $READIES/shibumi/defs +set -e #---------------------------------------------------------------------------------------------- if [[ $1 == --help || $1 == help || $HELP == 1 ]]; then @@ -21,7 +22,6 @@ if [[ $1 == --help || $1 == help || $HELP == 1 ]]; then RELEASE=1 Upload release artifacts STAGING=1 Upload into staging area - FORCE=1 Allow uploading outside of CI NOP=1 No operation VERBOSE=1 Show artifacts details HELP=1 Show help @@ -65,9 +65,11 @@ else S3_URL=s3://redismodules fi -if [[ -z $AWS_ACCESS_KEY_ID || -z $AWS_SECRET_ACCESS_KEY ]]; then - eprint "No credentials for S3 upload." - exit 1 +if [[ $FORCE != 1 ]]; then + if [[ -z $AWS_ACCESS_KEY_ID || -z $AWS_SECRET_ACCESS_KEY ]]; then + eprint "No credentials for S3 upload." + exit 1 + fi fi cd $ROOT/bin @@ -90,15 +92,17 @@ s3_upload_file() { local file="$1" local s3_dir="$2" [[ $s3_dir != */ ]] && s3_dir="${s3_dir}/" - - $OP aws --debug s3 cp $file $s3_dir --acl public-read --no-progress + + $OP aws s3 cp $file $s3_dir --acl public-read --no-progress } s3_ls() { local s3_dir="$1" [[ $s3_dir != */ ]] && s3_dir="${s3_dir}/" - + + echo "::group::S3 ls $s3_dir" $OP aws s3 ls $s3_dir + echo "::endgroup::" } s3_upload() { From 1da355d309bdd36729d016ae833b8011e0791fb4 Mon Sep 17 00:00:00 2001 From: Omer Shadmi <76992134+oshadmi@users.noreply.github.com> Date: Thu, 6 Feb 2025 13:39:26 +0200 Subject: [PATCH 107/112] MOD-8480: update redismodule-rs to 2.1.2 and fix codecov (#1326) * MOD-8480: update redismodule-rs to 2.1.2 on master * * try building with newer readies * turn on shell debug mode * update readies commit * * print coverage tools version * * fix clang-format not being a slave in update_clang_alternatives.sh * * print clang++ version * * cleanup a bit * * use all option in llvm.sh * * more generic update alternatives script * * test * * small fix * * try removing before installing * * fix deprecated default_features by moving it to be default-features * * print clang version * * update rust * * update coverage flag * * use rustc llvm version to avoid versioning errors in code coverage --------- Co-authored-by: jonathan keinan --- .github/workflows/flow-coverage.yml | 7 + .install/install_clang.sh | 19 ++- .install/update_clang_alternatives.sh | 187 +++++++++++++++----------- Cargo.lock | 10 +- Cargo.toml | 2 +- Makefile | 2 +- deps/readies | 2 +- redis_json/Cargo.toml | 4 +- 8 files changed, 136 insertions(+), 97 deletions(-) diff --git a/.github/workflows/flow-coverage.yml b/.github/workflows/flow-coverage.yml index 1792992d9..a0d84d3ea 100644 --- a/.github/workflows/flow-coverage.yml +++ b/.github/workflows/flow-coverage.yml @@ -21,6 +21,13 @@ jobs: - name: Setup common run: | ./.install/common_installations.sh sudo + - name: Get Rust + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + source "$HOME/.cargo/env" + rustup update + rustup update nightly + rustup component add rust-src --toolchain nightly - name: Install clang 18 working-directory: .install run: | diff --git a/.install/install_clang.sh b/.install/install_clang.sh index 116d06871..18b8e8b1f 100755 --- a/.install/install_clang.sh +++ b/.install/install_clang.sh @@ -1,7 +1,9 @@ #!/usr/bin/env sh +rust_llvm_major_version=$(rustc --version --verbose | grep "LLVM version" | awk '{print $3}' | cut -d. -f1) + export CWD=$(dirname `which "${0}"`) -export CLANG_VERSION=18 +export CLANG_VERSION=${rust_llvm_major_version} export DEBIAN_FRONTEND=noninteractive MODE=$1 # whether to install using sudo or not @@ -9,11 +11,16 @@ wget https://apt.llvm.org/llvm.sh -O llvm.sh chmod u+x llvm.sh -# expected to fail: -$MODE ./llvm.sh $CLANG_VERSION - -$MODE apt-get install python3-lldb-18 --yes --force-yes +$MODE ./llvm.sh $CLANG_VERSION all -$MODE ./llvm.sh $CLANG_VERSION +$MODE apt-get install python3-lldb-${rust_llvm_major_version} --yes $MODE $CWD/update_clang_alternatives.sh $CLANG_VERSION 1 + +ls /bin/ + +$MODE clang --version +$MODE clang++ --version +$MODE llvm-cov --version +$MODE llvm-profdata --version + diff --git a/.install/update_clang_alternatives.sh b/.install/update_clang_alternatives.sh index 3388e0def..c023ae07e 100755 --- a/.install/update_clang_alternatives.sh +++ b/.install/update_clang_alternatives.sh @@ -18,91 +18,116 @@ # The `--slave` options are used to link other related binaries (like clang-format, llvm-nm, etc.) # so that all tools are switched consistently when the main tool (e.g., clang) is switched. -function register_clang_version { +# Function to register alternatives as slave first and fall back to master if it fails +register_alternative() { + local tool=$1 + local tool_with_version=$2 + local version=$3 + local priority=$4 + + # Try registering as slave first + update-alternatives --remove-all "${tool}" + update-alternatives --verbose --install "/usr/bin/${tool}" "${tool}" "/usr/bin/${tool_with_version}" "${priority}" + + # Check if the previous command resulted in an error indicating that the tool is a master alternative + #if [ $? -ne 0 ]; then + # echo "Error: Failed to set up ${tool} as an alternative." +# + # # Force reinstallation in case of broken symlink group + # echo "Forcing reinstallation of ${tool}." + # update-alternatives --remove "${tool}" "/usr/bin/${tool}-${version}" + # update-alternatives --install "/usr/bin/${tool}" "${tool}" "/usr/bin/${tool_with_version}" ${priority} + #fi +} + +# Function to register Clang tools +register_clang_version() { local version=$1 local priority=$2 - # Register LLVM tools - update-alternatives \ - --verbose \ - --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${version} ${priority} \ - --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-${version} \ - --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-${version} \ - --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-${version} \ - --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-${version} \ - --slave /usr/bin/llvm-cat llvm-cat /usr/bin/llvm-cat-${version} \ - --slave /usr/bin/llvm-cfi-verify llvm-cfi-verify /usr/bin/llvm-cfi-verify-${version} \ - --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-${version} \ - --slave /usr/bin/llvm-cvtres llvm-cvtres /usr/bin/llvm-cvtres-${version} \ - --slave /usr/bin/llvm-cxxdump llvm-cxxdump /usr/bin/llvm-cxxdump-${version} \ - --slave /usr/bin/llvm-cxxfilt llvm-cxxfilt /usr/bin/llvm-cxxfilt-${version} \ - --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-${version} \ - --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-${version} \ - --slave /usr/bin/llvm-dlltool llvm-dlltool /usr/bin/llvm-dlltool-${version} \ - --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-${version} \ - --slave /usr/bin/llvm-dwp llvm-dwp /usr/bin/llvm-dwp-${version} \ - --slave /usr/bin/llvm-exegesis llvm-exegesis /usr/bin/llvm-exegesis-${version} \ - --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-${version} \ - --slave /usr/bin/llvm-lib llvm-lib /usr/bin/llvm-lib-${version} \ - --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-${version} \ - --slave /usr/bin/llvm-lto llvm-lto /usr/bin/llvm-lto-${version} \ - --slave /usr/bin/llvm-lto2 llvm-lto2 /usr/bin/llvm-lto2-${version} \ - --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-${version} \ - --slave /usr/bin/llvm-mca llvm-mca /usr/bin/llvm-mca-${version} \ - --slave /usr/bin/llvm-modextract llvm-modextract /usr/bin/llvm-modextract-${version} \ - --slave /usr/bin/llvm-mt llvm-mt /usr/bin/llvm-mt-${version} \ - --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-${version} \ - --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-${version} \ - --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-${version} \ - --slave /usr/bin/llvm-opt-report llvm-opt-report /usr/bin/llvm-opt-report-${version} \ - --slave /usr/bin/llvm-pdbutil llvm-pdbutil /usr/bin/llvm-pdbutil-${version} \ - --slave /usr/bin/llvm-PerfectShuffle llvm-PerfectShuffle /usr/bin/llvm-PerfectShuffle-${version} \ - --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-${version} \ - --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-${version} \ - --slave /usr/bin/llvm-rc llvm-rc /usr/bin/llvm-rc-${version} \ - --slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-${version} \ - --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-${version} \ - --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-${version} \ - --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-${version} \ - --slave /usr/bin/llvm-split llvm-split /usr/bin/llvm-split-${version} \ - --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-${version} \ - --slave /usr/bin/llvm-strings llvm-strings /usr/bin/llvm-strings-${version} \ - --slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-${version} \ - --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-${version} \ - --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-${version} \ - --slave /usr/bin/llvm-undname llvm-undname /usr/bin/llvm-undname-${version} \ - --slave /usr/bin/llvm-xray llvm-xray /usr/bin/llvm-xray-${version} - - # Register Clang tools - update-alternatives \ - --verbose \ - --install /usr/bin/clang clang /usr/bin/clang-${version} ${priority} \ - --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-${version} \ - --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-${version} \ - --slave /usr/bin/clang-cl clang-cl /usr/bin/clang-cl-${version} \ - --slave /usr/bin/clangd clangd /usr/bin/clangd-${version} \ - --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-${version} \ - --slave /usr/bin/clang-check clang-check /usr/bin/clang-check-${version} \ - --slave /usr/bin/clang-query clang-query /usr/bin/clang-query-${version} \ - --slave /usr/bin/asan_symbolize asan_symbolize /usr/bin/asan_symbolize-${version} \ - --slave /usr/bin/bugpoint bugpoint /usr/bin/bugpoint-${version} \ - --slave /usr/bin/dsymutil dsymutil /usr/bin/dsymutil-${version} \ - --slave /usr/bin/lld lld /usr/bin/lld-${version} \ - --slave /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-${version} \ - --slave /usr/bin/lld-link lld-link /usr/bin/lld-link-${version} \ - --slave /usr/bin/llc llc /usr/bin/llc-${version} \ - --slave /usr/bin/lli lli /usr/bin/lli-${version} \ - --slave /usr/bin/obj2yaml obj2yaml /usr/bin/obj2yaml-${version} \ - --slave /usr/bin/opt opt /usr/bin/opt-${version} \ - --slave /usr/bin/sanstats sanstats /usr/bin/sanstats-${version} \ - --slave /usr/bin/verify-uselistorder verify-uselistorder /usr/bin/verify-uselistorder-${version} \ - --slave /usr/bin/wasm-ld wasm-ld /usr/bin/wasm-ld-${version} \ - --slave /usr/bin/yaml2obj yaml2obj /usr/bin/yaml2obj-${version} + # List of all Clang and LLVM tools and their binary names + declare -a tools=( + # Clang tools + "clang" "clang-${version}" + "clang-cpp" "clang-cpp-${version}" + "clang-cl" "clang-cl-${version}" + "clangd" "clangd-${version}" + "clang-check" "clang-check-${version}" + "clang-query" "clang-query-${version}" + "asan_symbolize" "asan_symbolize-${version}" + "bugpoint" "bugpoint-${version}" + "dsymutil" "dsymutil-${version}" + "lld" "lld-${version}" + "ld.lld" "ld.lld-${version}" + "lld-link" "lld-link-${version}" + "llc" "llc-${version}" + "lli" "lli-${version}" + "opt" "opt-${version}" + "sanstats" "sanstats-${version}" + "verify-uselistorder" "verify-uselistorder-${version}" + "wasm-ld" "wasm-ld-${version}" + "yaml2obj" "yaml2obj-${version}" + "clang++" "clang++-${version}" + "clang-tidy" "clang-tidy-${version}" + "clang-format" "clang-format-${version}" + + # LLVM tools + "llvm-config" "llvm-config-${version}" + "llvm-ar" "llvm-ar-${version}" + "llvm-as" "llvm-as-${version}" + "llvm-bcanalyzer" "llvm-bcanalyzer-${version}" + "llvm-c-test" "llvm-c-test-${version}" + "llvm-cat" "llvm-cat-${version}" + "llvm-cfi-verify" "llvm-cfi-verify-${version}" + "llvm-cov" "llvm-cov-${version}" + "llvm-cvtres" "llvm-cvtres-${version}" + "llvm-cxxdump" "llvm-cxxdump-${version}" + "llvm-cxxfilt" "llvm-cxxfilt-${version}" + "llvm-diff" "llvm-diff-${version}" + "llvm-dis" "llvm-dis-${version}" + "llvm-dlltool" "llvm-dlltool-${version}" + "llvm-dwarfdump" "llvm-dwarfdump-${version}" + "llvm-dwp" "llvm-dwp-${version}" + "llvm-exegesis" "llvm-exegesis-${version}" + "llvm-extract" "llvm-extract-${version}" + "llvm-lib" "llvm-lib-${version}" + "llvm-link" "llvm-link-${version}" + "llvm-lto" "llvm-lto-${version}" + "llvm-lto2" "llvm-lto2-${version}" + "llvm-mc" "llvm-mc-${version}" + "llvm-mca" "llvm-mca-${version}" + "llvm-modextract" "llvm-modextract-${version}" + "llvm-mt" "llvm-mt-${version}" + "llvm-nm" "llvm-nm-${version}" + "llvm-objcopy" "llvm-objcopy-${version}" + "llvm-objdump" "llvm-objdump-${version}" + "llvm-opt-report" "llvm-opt-report-${version}" + "llvm-pdbutil" "llvm-pdbutil-${version}" + "llvm-PerfectShuffle" "llvm-PerfectShuffle-${version}" + "llvm-profdata" "llvm-profdata-${version}" + "llvm-ranlib" "llvm-ranlib-${version}" + "llvm-rc" "llvm-rc-${version}" + "llvm-readelf" "llvm-readelf-${version}" + "llvm-readobj" "llvm-readobj-${version}" + "llvm-rtdyld" "llvm-rtdyld-${version}" + "llvm-size" "llvm-size-${version}" + "llvm-split" "llvm-split-${version}" + "llvm-stress" "llvm-stress-${version}" + "llvm-strings" "llvm-strings-${version}" + "llvm-strip" "llvm-strip-${version}" + "llvm-symbolizer" "llvm-symbolizer-${version}" + "llvm-tblgen" "llvm-tblgen-${version}" + "llvm-undname" "llvm-undname-${version}" + "llvm-xray" "llvm-xray-${version}" + ) - # Register clang++ - update-alternatives \ - --verbose \ - --install /usr/bin/clang++ clang++ /usr/bin/clang++-${version} ${priority} + # Loop through the tools list and register them + for ((i=0; i<${#tools[@]}; i+=2)); do + tool="${tools[$i]}" + tool_bin="${tools[$i+1]}" + register_alternative "$tool" "$tool_bin" "$version" "$priority" + done } +# Call the function to register clang version (replace with actual version and priority) register_clang_version $1 $2 diff --git a/Cargo.lock b/Cargo.lock index 4ab135ca9..9d7543cca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -772,8 +772,8 @@ dependencies = [ [[package]] name = "redis-module" -version = "2.1.0" -source = "git+https://github.com/RedisLabsModules/redismodule-rs?tag=v2.1.0#2aa3463e03f15b89697587231a8582fdcf875a01" +version = "2.1.2" +source = "git+https://github.com/RedisLabsModules/redismodule-rs?tag=v2.1.2#556f420685b0b9662cdb816e22b1c55bc696a6dc" dependencies = [ "backtrace", "bindgen", @@ -795,7 +795,7 @@ dependencies = [ [[package]] name = "redis-module-macros" version = "99.99.99" -source = "git+https://github.com/RedisLabsModules/redismodule-rs?tag=v2.1.0#2aa3463e03f15b89697587231a8582fdcf875a01" +source = "git+https://github.com/RedisLabsModules/redismodule-rs?tag=v2.1.2#556f420685b0b9662cdb816e22b1c55bc696a6dc" dependencies = [ "proc-macro2", "quote 1.0.37", @@ -807,7 +807,7 @@ dependencies = [ [[package]] name = "redis-module-macros-internals" version = "99.99.99" -source = "git+https://github.com/RedisLabsModules/redismodule-rs?tag=v2.1.0#2aa3463e03f15b89697587231a8582fdcf875a01" +source = "git+https://github.com/RedisLabsModules/redismodule-rs?tag=v2.1.2#556f420685b0b9662cdb816e22b1c55bc696a6dc" dependencies = [ "lazy_static", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index fb0919361..362a3058d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ ] [workspace.dependencies] -ijson = { git="https://github.com/RedisJSON/ijson", rev="881b96e7941b8dc335863746ac108f6d9ed0b98a", default_features=false} +ijson = { git="https://github.com/RedisJSON/ijson", rev="881b96e7941b8dc335863746ac108f6d9ed0b98a", default-features=false} serde_json = { version="1", features = ["unbounded_depth"]} serde = { version = "1", features = ["derive"] } serde_derive = "1" diff --git a/Makefile b/Makefile index d95adb322..be8c4a0a2 100644 --- a/Makefile +++ b/Makefile @@ -98,7 +98,7 @@ else endif ifeq ($(COV),1) -RUST_FLAGS += -C instrument_coverage +RUST_FLAGS += -C instrument-coverage endif ifeq ($(PROFILE),1) diff --git a/deps/readies b/deps/readies index 7fc8e62b2..28ddde9e6 160000 --- a/deps/readies +++ b/deps/readies @@ -1 +1 @@ -Subproject commit 7fc8e62b24d3f7ff01096d83f14cbc216ac0e2f0 +Subproject commit 28ddde9e66d72289b3d3e700dc7114b27de20888 diff --git a/redis_json/Cargo.toml b/redis_json/Cargo.toml index 21daa03cc..fba96ac3e 100644 --- a/redis_json/Cargo.toml +++ b/redis_json/Cargo.toml @@ -22,8 +22,8 @@ ijson.workspace = true serde_json.workspace = true serde.workspace = true libc = "0.2" -redis-module ={ git="https://github.com/RedisLabsModules/redismodule-rs", tag="v2.1.0", default-features = false, features = ["min-redis-compatibility-version-7-4"] } -redis-module-macros = { git="https://github.com/RedisLabsModules/redismodule-rs", tag="v2.1.0" } +redis-module ={ git="https://github.com/RedisLabsModules/redismodule-rs", tag="v2.1.2", default-features = false, features = ["min-redis-compatibility-version-7-4"] } +redis-module-macros = { git="https://github.com/RedisLabsModules/redismodule-rs", tag="v2.1.2" } itertools = "0.13" json_path = {path="../json_path"} linkme = "0.3" From b39aec13d67866ebf73ec19ccd04639d2ae2d3cf Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Tue, 4 Mar 2025 06:43:41 +0200 Subject: [PATCH 108/112] MOD-8713 Remove outdated documentation(#1328) --- .github/workflows/task-check-docs.yml | 1 - TODO.md | 152 -------------- docs/commands/json.arrappend.md | 68 ------ docs/commands/json.arrindex.md | 112 ---------- docs/commands/json.arrinsert.md | 93 -------- docs/commands/json.arrlen.md | 72 ------- docs/commands/json.arrpop.md | 84 -------- docs/commands/json.arrtrim.md | 95 --------- docs/commands/json.clear.md | 65 ------ docs/commands/json.debug-help.md | 14 -- docs/commands/json.debug-memory.md | 52 ----- docs/commands/json.debug.md | 1 - docs/commands/json.del.md | 67 ------ docs/commands/json.forget.md | 1 - docs/commands/json.get.md | 99 --------- docs/commands/json.merge.md | 120 ----------- docs/commands/json.mget.md | 55 ----- docs/commands/json.mset.md | 60 ------ docs/commands/json.numincrby.md | 62 ------ docs/commands/json.nummultby.md | 47 ----- docs/commands/json.objkeys.md | 43 ---- docs/commands/json.objlen.md | 42 ---- docs/commands/json.resp.md | 82 -------- docs/commands/json.set.md | 89 -------- docs/commands/json.strappend.md | 54 ----- docs/commands/json.strlen.md | 42 ---- docs/commands/json.toggle.md | 74 ------- docs/commands/json.type.md | 46 ---- docs/docs/_index.md | 278 ------------------------ docs/docs/developer.md | 162 -------------- docs/docs/indexing_JSON.md | 14 -- docs/docs/path.md | 285 ------------------------- docs/docs/performance/_index.md | 291 -------------------------- docs/docs/ram.md | 107 ---------- docs/docs/resp3.md | 23 -- docs/docs/use_cases.md | 23 -- 36 files changed, 2975 deletions(-) delete mode 100644 TODO.md delete mode 100644 docs/commands/json.arrappend.md delete mode 100644 docs/commands/json.arrindex.md delete mode 100644 docs/commands/json.arrinsert.md delete mode 100644 docs/commands/json.arrlen.md delete mode 100644 docs/commands/json.arrpop.md delete mode 100644 docs/commands/json.arrtrim.md delete mode 100644 docs/commands/json.clear.md delete mode 100644 docs/commands/json.debug-help.md delete mode 100644 docs/commands/json.debug-memory.md delete mode 100644 docs/commands/json.debug.md delete mode 100644 docs/commands/json.del.md delete mode 100644 docs/commands/json.forget.md delete mode 100644 docs/commands/json.get.md delete mode 100644 docs/commands/json.merge.md delete mode 100644 docs/commands/json.mget.md delete mode 100644 docs/commands/json.mset.md delete mode 100644 docs/commands/json.numincrby.md delete mode 100644 docs/commands/json.nummultby.md delete mode 100644 docs/commands/json.objkeys.md delete mode 100644 docs/commands/json.objlen.md delete mode 100644 docs/commands/json.resp.md delete mode 100644 docs/commands/json.set.md delete mode 100644 docs/commands/json.strappend.md delete mode 100644 docs/commands/json.strlen.md delete mode 100644 docs/commands/json.toggle.md delete mode 100644 docs/commands/json.type.md delete mode 100644 docs/docs/_index.md delete mode 100644 docs/docs/developer.md delete mode 100644 docs/docs/indexing_JSON.md delete mode 100644 docs/docs/path.md delete mode 100644 docs/docs/performance/_index.md delete mode 100644 docs/docs/ram.md delete mode 100644 docs/docs/resp3.md delete mode 100644 docs/docs/use_cases.md diff --git a/.github/workflows/task-check-docs.yml b/.github/workflows/task-check-docs.yml index 14b137b15..951dca656 100644 --- a/.github/workflows/task-check-docs.yml +++ b/.github/workflows/task-check-docs.yml @@ -25,6 +25,5 @@ jobs: # List of files we allow to be changed without running the CI. Modify as needed. files: | **.md - docs/** licenses/** LICENSE.txt diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 76023b32e..000000000 --- a/TODO.md +++ /dev/null @@ -1,152 +0,0 @@ -# RedisJSON TODOs - ---- - -# MVP milestone - -This is what RedisJSON (https://github.com/RedisJSON/RedisJSON) currently has ready for the MVP: - -* Code is under src/ -* Building with CMake - - Need to verify on OSX - - Currently does not have an `install` option - needed? -* Documentation - - docs/commands.md - - docs/design.md ~ 30% done - - README.md ~ 85% done - - Missing about/what is ReJSON - - Some notes about performance - - Perhaps a Node.js example - - Source code is about 90% documented -* AGPLv3 license -* Copyright - -## Missing misc - -1. Peer review of project CTO->RnD/QA? -1. Number overflows in number operations -1. Something is printing "inf" - -## Source code/style - -1. Review and standardize use of int/size_t/uint32... -1. Improve/remove error reporting and logging in case of module internal errors - -## Benchmarks - -1. Need to include a simple standalone "benchmark", either w/ redis-benchmark or not ~ 30% done, need to complete some suites and generate graphs from output - -## Examples - -TBD - -1. A session token that also has a list of last seen times, what stack though -1. Node.js example perhaps - -## Blog post - -References: - -* [Parsing JSON Is A Minefield](http://seriot.ch/parsing_json.php) - ---- - -# Post MVP - -## Profiling - -1. Memory usage: implemented JSON.MEMORY, need to compile an automated reporting tool -1. Performance with callgrind and/or gperftools - -## Build/test - -1. Run https://github.com/nst/JSONTestSuite and report like http://seriot.ch/parsing_json.php -1. Need a dependable cycle to check for memory leaks -1. Once we have a way to check baseline performance, add regression -1. Fuzz all module commands with a mix of keys paths and values -1. Memory leaks suite to run - `valgrind --tool=memcheck --suppressions=../redis/src/valgrind.sup ../redis/src/redis-server --loadmodule ./lib/rejson.so` -1. Verify each command's syntax - need a YAML -1. Add CI to repo? - -## Path parsing - -1. Add array slice - -## Dictionary optimiztions - -Encode as trie over a certain size threshold to save memory and increase lookup performance. Alternatively, use a hash dictionary. - -## Secondary indexing - -Integrate with @dvirsky's `secondary` library. - -## Schema - -Support [JSON Schema](http://json-schema.org/). - -JSON.SETSCHEMA - -Notes: -1. Could be replaced by a JSON.SET modifier -2. Indexing will be specified in the schema -3. Cluster needs to be taken into account as well - -JSON.VALIDATE - -## Expiry - -JSON.EXPIRE - -# Cache serialized objects - -Manage a cache inside the module for frequently accessed object in order to avoid repetitive -serialization. - -## KeyRef nodes - -Add a node type that references a Redis key that is either a JSON data type or a regular Redis key. -The module's API can transparently support resolving referenced keys and querying the data in them. -KeyRefs in cluster mode will only be allowed if in the same slot. - -Redis core data types can be mapped to flat (i.e. non-nested) JSON structure as follows: -* A Redis String is a JSON String (albeit some escaping may be needed) -* A List: a JSON Array of JSON Strings, where the indices are the identical -* A Hash: a JSON Object where the Hash fields are the Object's keys and the values are JSON Strings -* A Set: a JSON Object where the Set members are the keys and their values are always a JSON Null -* A Sorted Set: a JSON Array that is made from two elements: - * A JSON Object where each key is a member and value is the score - * A JSON Array of all members ordered by score in ascending order - -## Compression - -Compress (string only? entire objects?) values over a (configureable?) size threshold with zstd. - -## Additions to API - -JSON.STATS -Print statistics about encountered values, parsing performance and such - -JSON.OBJSET -An alias for 'JSON.SET' - -JSON.COUNT -P: count JS: ? R: N/A -Counts the number of occurances for scalar in the array - -JSON.REMOVE [count] -P: builtin del JS: ? R: LREM (but also has a count and direction) -Removes the first `count` occurances (default 1) of value from array. If index is negative, -traversal is reversed. - -JSON.EXISTS -P: in JS: ? R: HEXISTS/LINDEX -Checks if path key or array index exists. Syntactic sugar for JSON.TYPE. - -JSON.REVERSE -P: reverse JS: ? R: N/A -Reverses the array. Nice to have. - -JSON.SORT -P: sort JS: ? R: SORT -Sorts the values in an array. Nice to have. diff --git a/docs/commands/json.arrappend.md b/docs/commands/json.arrappend.md deleted file mode 100644 index 7cc9e3a9f..000000000 --- a/docs/commands/json.arrappend.md +++ /dev/null @@ -1,68 +0,0 @@ -Append the `json` values into the array at `path` after the last element in it - -[Examples](#examples) - -## Required arguments - -
key - -is key to modify. -
- -
value - -is one or more values to append to one or more arrays. - -{{% alert title="About using strings with JSON commands" color="warning" %}} -To specify a string as an array value to append, wrap the quoted string with an additional set of single quotes. Example: `'"silver"'`. For more detailed use, see [Examples](#examples). -{{% /alert %}} -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`. -
- -## Return value - -`JSON.ARRAPEND` returns an [array](/docs/reference/protocol-spec/#resp-arrays) of integer replies for each path, the array's new size, or `nil`, if the matching JSON value is not an array. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -
-Add a new color to a list of product colors - -Create a document for noise-cancelling headphones in black and silver colors. - -{{< highlight bash >}} -redis> JSON.SET item:1 $ '{"name":"Noise-cancelling Bluetooth headphones","description":"Wireless Bluetooth headphones with noise-cancelling technology","connection":{"wireless":true,"type":"Bluetooth"},"price":99.98,"stock":25,"colors":["black","silver"]}' -OK -{{< / highlight >}} - -Add color `blue` to the end of the `colors` array. `JSON.ARRAPEND` returns the array's new size. - -{{< highlight bash >}} -redis> JSON.ARRAPPEND item:1 $.colors '"blue"' -1) (integer) 3 -{{< / highlight >}} - -Return the new length of the `colors` array. - -{{< highlight bash >}} -redis> JSON.GET item:1 -"{\"name\":\"Noise-cancelling Bluetooth headphones\",\"description\":\"Wireless Bluetooth headphones with noise-cancelling technology\",\"connection\":{\"wireless\":true,\"type\":\"Bluetooth\"},\"price\":99.98,\"stock\":25,\"colors\":[\"black\",\"silver\",\"blue\"]}" -{{< / highlight >}} - -
- -## See also - -`JSON.ARRINDEX` | `JSON.ARRINSERT` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/commands/json.arrindex.md b/docs/commands/json.arrindex.md deleted file mode 100644 index 77840ff16..000000000 --- a/docs/commands/json.arrindex.md +++ /dev/null @@ -1,112 +0,0 @@ -Search for the first occurrence of a JSON value in an array - -[Examples](#examples) - -## Required arguments - -
key - -is key to parse. -
- -
path - -is JSONPath to specify. -
- -
value - -is value to find its index in one or more arrays. - -{{% alert title="About using strings with JSON commands" color="warning" %}} -To specify a string as an array value to index, wrap the quoted string with an additional set of single quotes. Example: `'"silver"'`. For more detailed use, see [Examples](#examples). -{{% /alert %}} -
- -## Optional arguments - -
start - -is inclusive start value to specify in a slice of the array to search. Default is `0`. -
- - -
stop - -is exclusive stop value to specify in a slice of the array to search, including the last element. Default is `0`. Negative values are interpreted as starting from the end. -
- -{{% alert title="About out-of-range indexes" color="warning" %}} - -Out-of-range indexes round to the array's start and end. An inverse index range (such as the range from 1 to 0) returns unfound or `-1`. -{{% /alert %}} - -## Return value - -`JSON.ARRINDEX` returns an [array](/docs/reference/protocol-spec/#resp-arrays) of integer replies for each path, the first position in the array of each JSON value that matches the path, `-1` if unfound in the array, or `nil`, if the matching JSON value is not an array. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -
-Find the specific place of a color in a list of product colors - -Create a document for noise-cancelling headphones in black and silver colors. - -{{< highlight bash >}} -redis> JSON.SET item:1 $ '{"name":"Noise-cancelling Bluetooth headphones","description":"Wireless Bluetooth headphones with noise-cancelling technology","connection":{"wireless":true,"type":"Bluetooth"},"price":99.98,"stock":25,"colors":["black","silver"]}' -OK -{{< / highlight >}} - -Add color `blue` to the end of the `colors` array. `JSON.ARRAPEND` returns the array's new size. - -{{< highlight bash >}} -redis> JSON.ARRAPPEND item:1 $.colors '"blue"' -1) (integer) 3 -{{< / highlight >}} - -Return the new length of the `colors` array. - -{{< highlight bash >}} -redis> JSON.GET item:1 -"{\"name\":\"Noise-cancelling Bluetooth headphones\",\"description\":\"Wireless Bluetooth headphones with noise-cancelling technology\",\"connection\":{\"wireless\":true,\"type\":\"Bluetooth\"},\"price\":99.98,\"stock\":25,\"colors\":[\"black\",\"silver\",\"blue\"]}" -{{< / highlight >}} - -Get the list of colors for the product. - -{{< highlight bash >}} -redis> JSON.GET item:1 '$.colors[*]' -"[\"black\",\"silver\",\"blue\"]" -{{< / highlight >}} - -Insert two more colors after the second color. You now have five colors. - -{{< highlight bash >}} -redis> JSON.ARRINSERT item:1 $.colors 2 '"yellow"' '"gold"' -1) (integer) 5 -{{< / highlight >}} - -Get the updated list of colors. - -{{< highlight bash >}} -redis> JSON.GET item:1 $.colors -"[[\"black\",\"silver\",\"yellow\",\"gold\",\"blue\"]]" -{{< / highlight >}} - -Find the place where color `silver` is located. - -{{< highlight bash >}} -redis> JSON.ARRINDEX item:1 $..colors '"silver"' -1) (integer) 1 -{{< / highlight >}} -
- -## See also - -`JSON.ARRAPPEND` | `JSON.ARRINSERT` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) - diff --git a/docs/commands/json.arrinsert.md b/docs/commands/json.arrinsert.md deleted file mode 100644 index 04d91f4f7..000000000 --- a/docs/commands/json.arrinsert.md +++ /dev/null @@ -1,93 +0,0 @@ -Insert the `json` values into the array at `path` before the `index` (shifts to the right) - -[Examples](#examples) - -## Required arguments - -
key - -is key to modify. -
- -
value - -is one or more values to insert in one or more arrays. - -{{% alert title="About using strings with JSON commands" color="warning" %}} -To specify a string as an array value to insert, wrap the quoted string with an additional set of single quotes. Example: `'"silver"'`. For more detailed use, see [Examples](#examples). -{{% /alert %}} -
- -
index - -is position in the array where you want to insert a value. The index must be in the array's range. Inserting at `index` 0 prepends to the array. Negative index values start from the end of the array. -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`. -
- -## Return value - -`JSON.ARRINSERT` returns an [array](/docs/reference/protocol-spec/#resp-arrays) of integer replies for each path, the array's new size, or `nil`, if the matching JSON value is not an array. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -
-Add new colors to a specific place in a list of product colors - -Create a document for noise-cancelling headphones in black and silver colors. - -{{< highlight bash >}} -redis> JSON.SET item:1 $ '{"name":"Noise-cancelling Bluetooth headphones","description":"Wireless Bluetooth headphones with noise-cancelling technology","connection":{"wireless":true,"type":"Bluetooth"},"price":99.98,"stock":25,"colors":["black","silver"]}' -OK -{{< / highlight >}} - -Add color `blue` to the end of the `colors` array. `JSON.ARRAPEND` returns the array's new size. - -{{< highlight bash >}} -redis> JSON.ARRAPPEND item:1 $.colors '"blue"' -1) (integer) 3 -{{< / highlight >}} - -Return the new length of the `colors` array. - -{{< highlight bash >}} -redis> JSON.GET item:1 -"{\"name\":\"Noise-cancelling Bluetooth headphones\",\"description\":\"Wireless Bluetooth headphones with noise-cancelling technology\",\"connection\":{\"wireless\":true,\"type\":\"Bluetooth\"},\"price\":99.98,\"stock\":25,\"colors\":[\"black\",\"silver\",\"blue\"]}" -{{< / highlight >}} - -Get the list of colors for the product. - -{{< highlight bash >}} -redis> JSON.GET item:1 '$.colors[*]' -"[\"black\",\"silver\",\"blue\"]" -{{< / highlight >}} - -Insert two more colors after the second color. You now have five colors. - -{{< highlight bash >}} -redis> JSON.ARRINSERT item:1 $.colors 2 '"yellow"' '"gold"' -1) (integer) 5 -{{< / highlight >}} - -Get the updated list of colors. - -{{< highlight bash >}} -redis> JSON.GET item:1 $.colors -"[[\"black\",\"silver\",\"yellow\",\"gold\",\"blue\"]]" -{{< / highlight >}} -
- -## See also - -`JSON.ARRAPPEND` | `JSON.ARRINDEX` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/commands/json.arrlen.md b/docs/commands/json.arrlen.md deleted file mode 100644 index 069fbaf16..000000000 --- a/docs/commands/json.arrlen.md +++ /dev/null @@ -1,72 +0,0 @@ -Report the length of the JSON array at `path` in `key` - -[Examples](#examples) - -## Required arguments - -
key - -is key to parse. -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`, if not provided. Returns null if the `key` or `path` do not exist. -
- -## Return - -`JSON.ARRLEN` returns an [array](/docs/reference/protocol-spec/#resp-arrays) of integer replies, an integer for each matching value, each is the array's length, or `nil`, if the matching value is not an array. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -
-Get lengths of JSON arrays in a document - -Create a document for wireless earbuds. - -{{< highlight bash >}} -redis> JSON.SET item:2 $ '{"name":"Wireless earbuds","description":"Wireless Bluetooth in-ear headphones","connection":{"wireless":true,"type":"Bluetooth"},"price":64.99,"stock":17,"colors":["black","white"], "max_level":[80, 100, 120]}' -OK -{{< / highlight >}} - -Find lengths of arrays in all objects of the document. - -{{< highlight bash >}} -redis> JSON.ARRLEN item:2 '$.[*]' -1) (nil) -2) (nil) -3) (nil) -4) (nil) -5) (nil) -6) (integer) 2 -7) (integer) 3 -{{< / highlight >}} - -Return the length of the `max_level` array. - -{{< highlight bash >}} -redis> JSON.ARRLEN item:2 '$..max_level' -1) (integer) 3 -{{< / highlight >}} - -Get all the maximum level values. - -{{< highlight bash >}} -redis> JSON.GET item:2 '$..max_level' -"[[80,100,120]]" -{{< / highlight >}} - -
- -## See also - -`JSON.ARRINDEX` | `JSON.ARRINSERT` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/commands/json.arrpop.md b/docs/commands/json.arrpop.md deleted file mode 100644 index 67b0f78b3..000000000 --- a/docs/commands/json.arrpop.md +++ /dev/null @@ -1,84 +0,0 @@ -Remove and return an element from the index in the array - -[Examples](#examples) - -## Required arguments - -
key - -is key to modify. -
- -
index - -is position in the array to start popping from. Default is `-1`, meaning the last element. Out-of-range indexes round to their respective array ends. Popping an empty array returns null. -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`. -
- -## Return - -`JSON.ARRPOP` returns an [array](/docs/reference/protocol-spec/#resp-arrays) of bulk string replies for each path, each reply is the popped JSON value, or `nil`, if the matching JSON value is not an array. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -
-Pop a value from an index and insert a new value - -Create two headphone products with maximum sound levels. - -{{< highlight bash >}} -redis> JSON.SET key $ '[{"name":"Healthy headphones","description":"Wireless Bluetooth headphones with noise-cancelling technology","connection":{"wireless":true,"type":"Bluetooth"},"price":99.98,"stock":25,"colors":["black","silver"],"max_level":[60,70,80]},{"name":"Noisy headphones","description":"Wireless Bluetooth headphones with noise-cancelling technology","connection":{"wireless":true,"type":"Bluetooth"},"price":99.98,"stock":25,"colors":["black","silver"],"max_level":[80,90,100,120]}]' -OK -{{< / highlight >}} - -Get all maximum values for the second product. - -{{< highlight bash >}} -redis> JSON.GET key $.[1].max_level -"[[80,90,100,120]]" -{{< / highlight >}} - -Update the `max_level` field of the product: remove an unavailable value and add a newly available value. - -{{< highlight bash >}} -redis> JSON.ARRPOP key $.[1].max_level 0 -1) "80" -{{< / highlight >}} - -Get the updated array. - -{{< highlight bash >}} -redis> JSON.GET key $.[1].max_level -"[[90,100,120]]" -{{< / highlight >}} - -Now insert a new lowest value. - -{{< highlight bash >}} -redis> JSON.ARRINSERT key $.[1].max_level 0 85 -1) (integer) 4 -{{< / highlight >}} - -Get the updated array. - -{{< highlight bash >}} -redis> JSON.GET key $.[1].max_level -"[[85,90,100,120]]" -{{< / highlight >}} -
- -## See also - -`JSON.ARRAPPEND` | `JSON.ARRINDEX` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/commands/json.arrtrim.md b/docs/commands/json.arrtrim.md deleted file mode 100644 index 0a9abbcc5..000000000 --- a/docs/commands/json.arrtrim.md +++ /dev/null @@ -1,95 +0,0 @@ -Trim an array so that it contains only the specified inclusive range of elements - -[Examples](#examples) - -## Required arguments - -
key - -is key to modify. -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`. -
- -
start - -is index of the first element to keep (previous elements are trimmed). Default is 0. -
- -
stop - -is the index of the last element to keep (following elements are trimmed), including the last element. Default is 0. Negative values are interpreted as starting from the end. -
- -{{% alert title="About out-of-range indexes" color="warning" %}} - -JSON.ARRTRIM is extremely forgiving, and using it with out-of-range indexes does not produce an error. Note a few differences between how RedisJSON v2.0 and legacy versions handle out-of-range indexes. - -Behavior as of RedisJSON v2.0: - -* If `start` is larger than the array's size or `start` > `stop`, returns 0 and an empty array. -* If `start` is < 0, then start from the end of the array. -* If `stop` is larger than the end of the array, it is treated like the last element. -{{% /alert %}} - -## Return - -JSON.ARRTRIM returns an array of integer replies for each path, the array's new size, or `nil`, if the matching JSON value is not an array. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -
-Trim an array to a specific set of values - -Create two headphone products with maximum sound levels. - -{{< highlight bash >}} -redis> JSON.GET key $ -"[[{\"name\":\"Healthy headphones\",\"description\":\"Wireless Bluetooth headphones with noise-cancelling technology\",\"connection\":{\"wireless\":true,\"type\":\"Bluetooth\"},\"price\":99.98,\"stock\":25,\"colors\":[\"black\",\"silver\"],\"max_level\":[60,70,80]},{\"name\":\"Noisy headphones\",\"description\":\"Wireless Bluetooth headphones with noise-cancelling technology\",\"connection\":{\"wireless\":true,\"type\":\"Bluetooth\"},\"price\":99.98,\"stock\":25,\"colors\":[\"black\",\"silver\"],\"max_level\":[85,90,100,120]}]]" -OK -{{< / highlight >}} - -Add new sound level values to the second product. - -{{< highlight bash >}} -redis> JSON.ARRAPPEND key $.[1].max_level 140 160 180 200 220 240 260 280 -1) (integer) 12 -{{< / highlight >}} - -Get the updated array. - -{{< highlight bash >}} -redis> JSON.GET key $.[1].max_level -"[[85,90,100,120,140,160,180,200,220,240,260,280]]" -{{< / highlight >}} - -Keep only the values between the fifth and the ninth element, inclusive of that last element. - -{{< highlight bash >}} -redis> JSON.ARRTRIM key $.[1].max_level 4 8 -1) (integer) 5 -{{< / highlight >}} - -Get the updated array. - -{{< highlight bash >}} -redis> JSON.GET key $.[1].max_level -"[[140,160,180,200,220]]" -{{< / highlight >}} -
- -## See also - -`JSON.ARRINDEX` | `JSON.ARRINSERT` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) - diff --git a/docs/commands/json.clear.md b/docs/commands/json.clear.md deleted file mode 100644 index c217637cd..000000000 --- a/docs/commands/json.clear.md +++ /dev/null @@ -1,65 +0,0 @@ -Clear container values (arrays/objects) and set numeric values to `0` - -[Examples](#examples) - -## Required arguments - -
key - -is key to parse. -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`. Nonexisting paths are ignored. -
- -## Return - -JSON.CLEAR returns an integer reply specifying the number of matching JSON arrays and objects cleared + number of matching JSON numerical values zeroed. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -{{% alert title="Note" color="warning" %}} - -Already cleared values are ignored for empty containers and zero numbers. - -{{% /alert %}} - -## Examples - -
-Clear container values and set numeric values to 0 - -Create a JSON document. - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"obj":{"a":1, "b":2}, "arr":[1,2,3], "str": "foo", "bool": true, "int": 42, "float": 3.14}' -OK -{{< / highlight >}} - -Clear all container values. This returns the number of objects with cleared values. - -{{< highlight bash >}} -redis> JSON.CLEAR doc $.* -(integer) 4 -{{< / highlight >}} - -Get the updated document. Note that numeric values have been set to `0`. - -{{< highlight bash >}} -redis> JSON.GET doc $ -"[{\"obj\":{},\"arr\":[],\"str\":\"foo\",\"bool\":true,\"int\":0,\"float\":0}]" -{{< / highlight >}} -
- -## See also - -`JSON.ARRINDEX` | `JSON.ARRINSERT` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) - diff --git a/docs/commands/json.debug-help.md b/docs/commands/json.debug-help.md deleted file mode 100644 index e7f116c29..000000000 --- a/docs/commands/json.debug-help.md +++ /dev/null @@ -1,14 +0,0 @@ -Return helpful information about the `JSON.DEBUG` command - -## Return - -JSON.DEBUG HELP returns an array with helpful messages. - -## See also - -`JSON.DEBUG` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/commands/json.debug-memory.md b/docs/commands/json.debug-memory.md deleted file mode 100644 index dbe301c81..000000000 --- a/docs/commands/json.debug-memory.md +++ /dev/null @@ -1,52 +0,0 @@ -Report a value's memory usage in bytes - -[Examples](#examples) - -## Required arguments - -
key - -is key to parse. -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`. -
- -## Return - -JSON.DEBUG MEMORY returns an integer reply specified as the value size in bytes. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -
-Report a value's memory usage in bytes - -Create a JSON document. - -{{< highlight bash >}} -redis> JSON.SET item:2 $ '{"name":"Wireless earbuds","description":"Wireless Bluetooth in-ear headphones","connection":{"wireless":true,"type":"Bluetooth"},"price":64.99,"stock":17,"colors":["black","white"], "max_level":[80, 100, 120]}' -OK -{{< / highlight >}} - -Get the values' memory usage in bytes. - -{{< highlight bash >}} -redis> JSON.DEBUG MEMORY item:2 -(integer) 253 -{{< / highlight >}} -
- -## See also - -`JSON.SET` | `JSON.ARRLEN` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) - diff --git a/docs/commands/json.debug.md b/docs/commands/json.debug.md deleted file mode 100644 index 1bd976f5d..000000000 --- a/docs/commands/json.debug.md +++ /dev/null @@ -1 +0,0 @@ -This is a container command for debugging related tasks. diff --git a/docs/commands/json.del.md b/docs/commands/json.del.md deleted file mode 100644 index 5d572dd7a..000000000 --- a/docs/commands/json.del.md +++ /dev/null @@ -1,67 +0,0 @@ -Delete a value - -[Examples](#examples) - -## Required arguments - -
key - -is key to modify. -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`. Nonexisting paths are ignored. - -{{% alert title="Note" color="warning" %}} - -Deleting an object's root is equivalent to deleting the key from Redis. - -{{% /alert %}} -
- -## Return - -JSON.DEL returns an integer reply specified as the number of paths deleted (0 or more). -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -
-Delete a value - -Create a JSON document. - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"a": 1, "nested": {"a": 2, "b": 3}}' -OK -{{< / highlight >}} - -Delete specified values. - -{{< highlight bash >}} -redis> JSON.DEL doc $..a -(integer) 2 -{{< / highlight >}} - -Get the updated document. - -{{< highlight bash >}} -redis> JSON.GET doc $ -"[{\"nested\":{\"b\":3}}]" -{{< / highlight >}} -
- -## See also - -`JSON.SET` | `JSON.ARRLEN` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) - - - diff --git a/docs/commands/json.forget.md b/docs/commands/json.forget.md deleted file mode 100644 index 9ac8cdc52..000000000 --- a/docs/commands/json.forget.md +++ /dev/null @@ -1 +0,0 @@ -See `JSON.DEL`. \ No newline at end of file diff --git a/docs/commands/json.get.md b/docs/commands/json.get.md deleted file mode 100644 index 6aeebb879..000000000 --- a/docs/commands/json.get.md +++ /dev/null @@ -1,99 +0,0 @@ -Return the value at `path` in JSON serialized form - -[Examples](#examples) - -## Required arguments - -
key - -is key to parse. -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`. JSON.GET accepts multiple `path` arguments. - -{{% alert title="Note" color="warning" %}} - -When using a single JSONPath, the root of the matching values is a JSON string with a top-level **array** of serialized JSON value. -In contrast, a legacy path returns a single value. - -When using multiple JSONPath arguments, the root of the matching values is a JSON string with a top-level **object**, with each object value being a top-level array of serialized JSON value. -In contrast, if all paths are legacy paths, each object value is a single serialized JSON value. -If there are multiple paths that include both legacy path and JSONPath, the returned value conforms to the JSONPath version (an array of values). - -{{% /alert %}} - -
- -
INDENT - -sets the indentation string for nested levels. -
- -
NEWLINE - -sets the string that's printed at the end of each line. -
- -
SPACE - -sets the string that's put between a key and a value. -
- -{{% alert title="Note" color="warning" %}} - -Produce pretty-formatted JSON with `redis-cli` by following this example: - -{{< highlight bash >}} -~/$ redis-cli --raw -redis> JSON.GET myjsonkey INDENT "\t" NEWLINE "\n" SPACE " " path.to.value[1] -{{< / highlight >}} - -{{% /alert %}} - -## Return - -JSON.GET returns a bulk string representing a JSON array of string replies. -Each string is the JSON serialization of each JSON value that matches a path. -Using multiple paths, JSON.GET returns a bulk string representing a JSON object with string values. -Each string value is an array of the JSON serialization of each JSON value that matches a path. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -
-Return the value at path in JSON serialized form - -Create a JSON document. - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"a":2, "b": 3, "nested": {"a": 4, "b": null}}' -OK -{{< / highlight >}} - -With a single JSONPath (JSON array bulk string): - -{{< highlight bash >}} -redis> JSON.GET doc $..b -"[3,null]" -{{< / highlight >}} - -Using multiple paths with at least one JSONPath returns a JSON string with a top-level object with an array of JSON values per path: - -{{< highlight bash >}} -redis> JSON.GET doc ..a $..b -"{\"$..b\":[3,null],\"..a\":[2,4]}" -{{< / highlight >}} -
- -## See also - -`JSON.SET` | `JSON.MGET` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/commands/json.merge.md b/docs/commands/json.merge.md deleted file mode 100644 index a83d65555..000000000 --- a/docs/commands/json.merge.md +++ /dev/null @@ -1,120 +0,0 @@ -Merge a given JSON value into matching paths. Consequently, JSON values at matching paths are updated, deleted, or expanded with new children. - -This command complies with [RFC7396](https://datatracker.ietf.org/doc/html/rfc7396) Json Merge Patch - -[Examples](#examples) - -## Required arguments - -
key - -is key to merge into. -
- -
path - -is JSONPath to specify. For non-existing keys the `path` must be `$`. For existing keys, for each matched `path`, the value that matches the `path` is being merged with the JSON `value`. For existing keys, when the path exists, except for the last element, a new child is added with the JSON `value`. - -
- -
value - -is JSON value to merge with at the specified path. Merging is done according to the following rules per JSON value in the `value` argument while considering the corresponding original value if it exists: -* merging an existing object key with a `null` value deletes the key -* merging an existing object key with non-null value updates the value -* merging a non-existing object key adds the key and value -* merging an existing array with any merged value, replaces the entire array with the value -
- -## Return value - -JSET.MERGE returns a simple string reply: `OK` if executed correctly or `error` if fails to set the new values - -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -The JSON.MERGE provide four different behaviours to merge changes on a given key: create unexistent path, update an existing path with a new value, delete a existing path or replace an array with a new array - -
-Create a unexistent path-value - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"a":2}' -OK -redis> JSON.MERGE doc $.b '8' -OK -redis> JSON.GET doc $ -"[{\"a\":2,\"b\":8}]" -{{< / highlight >}} - -
- -
-Replace an existing value - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"a":2}' -OK -redis> JSON.MERGE doc $.a '3' -OK -redis> JSON.GET doc $ -"[{\"a\":3}]" -{{< / highlight >}} - -
- -
-Delete on existing value - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"a":2}' -OK -redis> JSON.MERGE doc $ '{"a":null}' -OK -redis> JSON.GET doc $ -"[{}]" -{{< / highlight >}} - -
- -
-Replace an Array - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"a":[2,4,6,8]}' -OK -redis> JSON.MERGE doc $.a '[10,12]' -OK -redis> JSON.GET doc $ -"[{\"a\":[10,12]}]" -{{< / highlight >}} - -
- - -
-Merge changes in multi-paths - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"f1": {"a":1}, "f2":{"a":2}}' -OK -redis> JSON.GET doc -"{\"f1\":{\"a\":1},\"f2\":{\"a\":2}}" -redis> JSON.MERGE doc $ '{"f1": null, "f2":{"a":3, "b":4}, "f3":[2,4,6]}' -OK -redis> JSON.GET doc -"{\"f2\":{\"a\":3,\"b\":4},\"f3\":[2,4,6]}" -{{< / highlight >}} - -
- -## See also - -`JSON.GET` | `JSON.MGET` | `JSON.SET` | `JSON.MSET` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) - diff --git a/docs/commands/json.mget.md b/docs/commands/json.mget.md deleted file mode 100644 index b266b3cbd..000000000 --- a/docs/commands/json.mget.md +++ /dev/null @@ -1,55 +0,0 @@ -Return the values at `path` from multiple `key` arguments - -[Examples](#examples) - -## Required arguments - -
key - -is key to parse. Returns `null` for nonexistent keys. -
- -## Optional arguments - -
path - -is JSONPath to specify. Returns `null` for nonexistent paths. - -
- -## Return - -JSON.MGET returns an array of bulk string replies specified as the JSON serialization of the value at each key's path. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -
-Return the values at path from multiple key arguments - -Create two JSON documents. - -{{< highlight bash >}} -redis> JSON.SET doc1 $ '{"a":1, "b": 2, "nested": {"a": 3}, "c": null}' -OK -redis> JSON.SET doc2 $ '{"a":4, "b": 5, "nested": {"a": 6}, "c": null}' -OK -{{< / highlight >}} - -Get values from all arguments in the documents. - -{{< highlight bash >}} -redis> JSON.MGET doc1 doc2 $..a -1) "[1,3]" -2) "[4,6]" -{{< / highlight >}} -
- -## See also - -`JSON.SET` | `JSON.GET` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/commands/json.mset.md b/docs/commands/json.mset.md deleted file mode 100644 index c8ee61775..000000000 --- a/docs/commands/json.mset.md +++ /dev/null @@ -1,60 +0,0 @@ -Set or update one or more JSON values according to the specified `key`-`path`-`value` triplets - -`JSON.MSET` is atomic, hence, all given additions or updates are either applied or not. It is not possible for clients to see that some of the keys were updated while others are unchanged. - -A JSON value is a hierarchical structure. If you change a value in a specific path - nested values are affected. - - -[Examples](#examples) - -## Required arguments - -
key - -is key to modify. -
- -
path - -is JSONPath to specify. For new Redis keys the `path` must be the root. For existing keys, when the entire `path` exists, the value that it contains is replaced with the `json` value. For existing keys, when the `path` exists, except for the last element, a new child is added with the `json` value. - -
- -
value - -is value to set at the specified path -
- -## Return value - -JSET.MSET returns a simple string reply: `OK` if executed correctly or `error` if fails to set the new values - -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -
-Add a new values in multiple keys - -{{< highlight bash >}} -redis> JSON.MSET doc1 $ '{"a":1}' doc2 $ '{"f":{"a":2}}' doc3 $ '{"f1":{"a":0},"f2":{"a":0}}' -OK -redis> JSON.MSET doc1 $ '{"a":2}' doc2 $.f.a '3' doc3 $ '{"f1":{"a":1},"f2":{"a":2}}' -OK -redis> JSON.GET doc1 $ -"[{\"a\":2}]" -redis> JSON.GET doc2 $ -"[{\"f\":{\"a\":3}}]" -redis> JSON.GET doc3 -"{\"f1\":{\"a\":1},\"f2\":{\"a\":2}}" -{{< / highlight >}} -
- -## See also - -`JSON.SET` | `JSON.MGET` | `JSON.GET` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/commands/json.numincrby.md b/docs/commands/json.numincrby.md deleted file mode 100644 index cda825421..000000000 --- a/docs/commands/json.numincrby.md +++ /dev/null @@ -1,62 +0,0 @@ -Increment the number value stored at `path` by `number` - -[Examples](#examples) - -## Required arguments - -
key - -is key to modify. -
- -
path - -is JSONPath to specify. -
- -
value - -is number value to increment. -
- -## Return - -JSON.NUMINCRBY returns a bulk string reply specified as a stringified new value for each path, or `nil`, if the matching JSON value is not a number. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -
-Increment number values - -Create a document. - -{{< highlight bash >}} -redis> JSON.SET doc . '{"a":"b","b":[{"a":2}, {"a":5}, {"a":"c"}]}' -OK -{{< / highlight >}} - -Increment a value of `a` object by 2. The command fails to find a number and returns `null`. - -{{< highlight bash >}} -redis> JSON.NUMINCRBY doc $.a 2 -"[null]" -{{< / highlight >}} - -Recursively find and increment a value of all `a` objects. The command increments numbers it finds and returns `null` for nonnumber values. - -{{< highlight bash >}} -redis> JSON.NUMINCRBY doc $..a 2 -"[null,4,7,null]" -{{< / highlight >}} - -
- -## See also - -`JSON.ARRINDEX` | `JSON.ARRINSERT` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/commands/json.nummultby.md b/docs/commands/json.nummultby.md deleted file mode 100644 index ac112d0a6..000000000 --- a/docs/commands/json.nummultby.md +++ /dev/null @@ -1,47 +0,0 @@ -Multiply the number value stored at `path` by `number` - -[Examples](#examples) - -## Required arguments - -
key - -is key to modify. -
- -
value - -is number value to multiply. -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`. -
- -## Return - -JSON.NUMMULTBY returns a bulk string reply specified as a stringified new values for each path, or `nil` element if the matching JSON value is not a number. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -{{< highlight bash >}} -redis> JSON.SET doc . '{"a":"b","b":[{"a":2}, {"a":5}, {"a":"c"}]}' -OK -redis> JSON.NUMMULTBY doc $.a 2 -"[null]" -redis> JSON.NUMMULTBY doc $..a 2 -"[null,4,10,null]" -{{< / highlight >}} - -## See also - -`JSON.NUMINCRBY` | `JSON.ARRINSERT` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/commands/json.objkeys.md b/docs/commands/json.objkeys.md deleted file mode 100644 index a91cde25e..000000000 --- a/docs/commands/json.objkeys.md +++ /dev/null @@ -1,43 +0,0 @@ -Return the keys in the object that's referenced by `path` - -[Examples](#examples) - -## Required arguments - -
key - -is key to parse. Returns `null` for nonexistent keys. -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`. Returns `null` for nonexistant path. - -
- -## Return - -JSON.OBJKEYS returns an array of array replies for each path, an array of the key names in the object as a bulk string reply, or `nil` if the matching JSON value is not an object. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"a":[3], "nested": {"a": {"b":2, "c": 1}}}' -OK -redis> JSON.OBJKEYS doc $..a -1) (nil) -2) 1) "b" - 2) "c" -{{< / highlight >}} - -## See also - -`JSON.ARRINDEX` | `JSON.ARRINSERT` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/commands/json.objlen.md b/docs/commands/json.objlen.md deleted file mode 100644 index 41408b048..000000000 --- a/docs/commands/json.objlen.md +++ /dev/null @@ -1,42 +0,0 @@ -Report the number of keys in the JSON object at `path` in `key` - -[Examples](#examples) - -## Required arguments - -
key - -is key to parse. Returns `null` for nonexistent keys. -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`. Returns `null` for nonexistant path. - -
- -## Return - -JSON.OBJLEN returns an array of integer replies for each path specified as the number of keys in the object or `nil`, if the matching JSON value is not an object. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"a":[3], "nested": {"a": {"b":2, "c": 1}}}' -OK -redis> JSON.OBJLEN doc $..a -1) (nil) -2) (integer) 2 -{{< / highlight >}} - -## See also - -`JSON.ARRINDEX` | `JSON.ARRINSERT` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/commands/json.resp.md b/docs/commands/json.resp.md deleted file mode 100644 index a4892134c..000000000 --- a/docs/commands/json.resp.md +++ /dev/null @@ -1,82 +0,0 @@ -Return the JSON in `key` in [Redis serialization protocol specification](/docs/reference/protocol-spec) form - -[Examples](#examples) - -## Required arguments - -
key - -is key to parse. -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`. This command uses the following mapping from JSON to RESP: - -* JSON `null` maps to the bulk string reply. -* JSON `false` and `true` values map to the simple string reply. -* JSON number maps to the integer reply or bulk string reply, depending on type. -* JSON string maps to the bulk string reply. -* JSON array is represented as an array reply in which the first element is the simple string reply `[`, followed by the array's elements. -* JSON object is represented as an array reply in which the first element is the simple string reply `{`. Each successive entry represents a key-value pair as a two-entry array reply of the bulk string reply. - -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). -
- -## Return - -JSON.RESP returns an array reply specified as the JSON's RESP form detailed in [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -
-Return an array of RESP details about a document - -Create a JSON document. - -{{< highlight bash >}} -redis> JSON.SET item:2 $ '{"name":"Wireless earbuds","description":"Wireless Bluetooth in-ear headphones","connection":{"wireless":true,"type":"Bluetooth"},"price":64.99,"stock":17,"colors":["black","white"], "max_level":[80, 100, 120]}' -OK -{{< / highlight >}} - -Get all RESP details about the document. - -{{< highlight bash >}} -redis> JSON.RESP item:2 - 1) { - 2) "name" - 3) "Wireless earbuds" - 4) "description" - 5) "Wireless Bluetooth in-ear headphones" - 6) "connection" - 7) 1) { - 2) "wireless" - 3) true - 4) "type" - 5) "Bluetooth" - 8) "price" - 9) "64.989999999999995" -10) "stock" -11) (integer) 17 -12) "colors" -13) 1) [ - 2) "black" - 3) "white" -14) "max_level" -15) 1) [ - 2) (integer) 80 - 3) (integer) 100 - 4) (integer) 120 -{{< / highlight >}} -
- -## See also - -`JSON.SET` | `JSON.ARRLEN` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/commands/json.set.md b/docs/commands/json.set.md deleted file mode 100644 index 6e68f5e8a..000000000 --- a/docs/commands/json.set.md +++ /dev/null @@ -1,89 +0,0 @@ -Set the JSON value at `path` in `key` - -[Examples](#examples) - -## Required arguments - -
key - -is key to modify. -
- -
path - -is JSONPath to specify. Default is root `$`. For new Redis keys the `path` must be the root. For existing keys, when the entire `path` exists, the value that it contains is replaced with the `json` value. For existing keys, when the `path` exists, except for the last element, a new child is added with the `json` value. - -Adds a key (with its respective value) to a JSON Object (in a RedisJSON data type key) only if it is the last child in the `path`, or it is the parent of a new child being added in the `path`. Optional arguments `NX` and `XX` modify this behavior for both new RedisJSON data type keys as well as the JSON Object keys in them. -
- -
value - -is value to set at the specified path -
- -## Optional arguments - -
NX - -sets the key only if it does not already exist. -
- -
XX - -sets the key only if it already exists. -
- -## Return value - -JSET.SET returns a simple string reply: `OK` if executed correctly or `nil` if the specified `NX` or `XX` conditions were not met. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -
-Replace an existing value - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"a":2}' -OK -redis> JSON.SET doc $.a '3' -OK -redis> JSON.GET doc $ -"[{\"a\":3}]" -{{< / highlight >}} -
- -
-Add a new value - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"a":2}' -OK -redis> JSON.SET doc $.b '8' -OK -redis> JSON.GET doc $ -"[{\"a\":2,\"b\":8}]" -{{< / highlight >}} -
- -
-Update multi-paths - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"f1": {"a":1}, "f2":{"a":2}}' -OK -redis> JSON.SET doc $..a 3 -OK -redis> JSON.GET doc -"{\"f1\":{\"a\":3},\"f2\":{\"a\":3}}" -{{< / highlight >}} -
- -## See also - -`JSON.GET` | `JSON.MGET` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/commands/json.strappend.md b/docs/commands/json.strappend.md deleted file mode 100644 index 44293e4fe..000000000 --- a/docs/commands/json.strappend.md +++ /dev/null @@ -1,54 +0,0 @@ -Append the `json-string` values to the string at `path` - -[Examples](#examples) - -## Required arguments - -
key - -is key to modify. -
- -
value - -is value to append to one or more strings. - -{{% alert title="About using strings with JSON commands" color="warning" %}} -To specify a string as an array value to append, wrap the quoted string with an additional set of single quotes. Example: `'"silver"'`. For more detailed use, see [Examples](#examples). -{{% /alert %}} -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`. -
- -## Return value - -JSON.STRAPPEND returns an array of integer replies for each path, the string's new length, or `nil`, if the matching JSON value is not a string. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"a":"foo", "nested": {"a": "hello"}, "nested2": {"a": 31}}' -OK -redis> JSON.STRAPPEND doc $..a '"baz"' -1) (integer) 6 -2) (integer) 8 -3) (nil) -redis> JSON.GET doc $ -"[{\"a\":\"foobaz\",\"nested\":{\"a\":\"hellobaz\"},\"nested2\":{\"a\":31}}]" -{{< / highlight >}} - -## See also - -`JSON.ARRAPEND` | `JSON.ARRINSERT` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) - diff --git a/docs/commands/json.strlen.md b/docs/commands/json.strlen.md deleted file mode 100644 index 701d572c6..000000000 --- a/docs/commands/json.strlen.md +++ /dev/null @@ -1,42 +0,0 @@ -Report the length of the JSON String at `path` in `key` - -[Examples](#examples) - -## Required arguments - -
key - -is key to parse. -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`, if not provided. Returns null if the `key` or `path` do not exist. -
- -## Return - -JSON.STRLEN returns by recursive descent an array of integer replies for each path, the array's length, or `nil`, if the matching JSON value is not a string. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"a":"foo", "nested": {"a": "hello"}, "nested2": {"a": 31}}' -OK -redis> JSON.STRLEN doc $..a -1) (integer) 3 -2) (integer) 5 -3) (nil) -{{< / highlight >}} - -## See also - -`JSON.ARRLEN` | `JSON.ARRINSERT` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/commands/json.toggle.md b/docs/commands/json.toggle.md deleted file mode 100644 index c778bfa86..000000000 --- a/docs/commands/json.toggle.md +++ /dev/null @@ -1,74 +0,0 @@ -Toggle a Boolean value stored at `path` - -[Examples](#examples) - -## Required arguments - -
key - -is key to modify. -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`. - -
- -## Return - -JSON.TOGGLE returns an array of integer replies for each path, the new value (`0` if `false` or `1` if `true`), or `nil` for JSON values matching the path that are not Boolean. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -
-Toogle a Boolean value stored at path - -Create a JSON document. - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"bool": true}' -OK -{{< / highlight >}} - -Toggle the Boolean value. - -{{< highlight bash >}} -redis> JSON.TOGGLE doc $.bool -1) (integer) 0 -{{< / highlight >}} - -Get the updated document. - -{{< highlight bash >}} -redis> JSON.GET doc $ -"[{\"bool\":false}]" -{{< / highlight >}} - -Toggle the Boolean value. - -{{< highlight bash >}} -redis> JSON.TOGGLE doc $.bool -1) (integer) 1 -{{< / highlight >}} - -Get the updated document. - -{{< highlight bash >}} -redis> JSON.GET doc $ -"[{\"bool\":true}]" -{{< / highlight >}} -
- -## See also - -`JSON.SET` | `JSON.GET` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) - diff --git a/docs/commands/json.type.md b/docs/commands/json.type.md deleted file mode 100644 index 3d5a93a9b..000000000 --- a/docs/commands/json.type.md +++ /dev/null @@ -1,46 +0,0 @@ -Report the type of JSON value at `path` - -[Examples](#examples) - -## Required arguments - -
key - -is key to parse. -
- -## Optional arguments - -
path - -is JSONPath to specify. Default is root `$`. Returns null if the `key` or `path` do not exist. - -
- -## Return - -JSON.TYPE returns an array of string replies for each path, specified as the value's type. -For more information about replies, see [Redis serialization protocol specification](/docs/reference/protocol-spec). - -## Examples - -{{< highlight bash >}} -redis> JSON.SET doc $ '{"a":2, "nested": {"a": true}, "foo": "bar"}' -OK -redis> JSON.TYPE doc $..foo -1) "string" -redis> JSON.TYPE doc $..a -1) "integer" -2) "boolean" -redis> JSON.TYPE doc $..dummy -(empty array) -{{< / highlight >}} - -## See also - -`JSON.SET` | `JSON.ARRLEN` - -## Related topics - -* [RedisJSON](/docs/stack/json) -* [Index and search JSON documents](/docs/stack/search/indexing_json) diff --git a/docs/docs/_index.md b/docs/docs/_index.md deleted file mode 100644 index 2f929790e..000000000 --- a/docs/docs/_index.md +++ /dev/null @@ -1,278 +0,0 @@ ---- -title: JSON -description: JSON support for Redis -linkTitle: JSON -type: docs -stack: true -aliases: - - /docs/stack/json ---- - -[![Discord](https://img.shields.io/discord/697882427875393627?style=flat-square)](https://discord.gg/QUkjSsk) -[![Github](https://img.shields.io/static/v1?label=&message=repository&color=5961FF&logo=github)](https://github.com/RedisJSON/RedisJSON/) - -The JSON capability of Redis Stack provides JavaScript Object Notation (JSON) support for Redis. It lets you store, update, and retrieve JSON values in a Redis database, similar to any other Redis data type. Redis JSON also works seamlessly with [Search and Query](https://redis.io/docs/stack/search/) to let you [index and query JSON documents](https://redis.io/docs/stack/search/indexing_json). - -## Primary features - -* Full support for the JSON standard -* A [JSONPath](http://goessner.net/articles/JsonPath/) syntax for selecting/updating elements inside documents (see [JSONPath syntax](/redisjson/path#jsonpath-syntax)) -* Documents stored as binary data in a tree structure, allowing fast access to sub-elements -* Typed atomic operations for all JSON value types - -## Use Redis JSON - -To learn how to use JSON, it's best to start with the Redis CLI. The following examples assume that you're connected to a Redis server with JSON enabled. - -### `redis-cli` examples - -First, start [`redis-cli`](http://redis.io/topics/rediscli) in interactive mode. - -The first JSON command to try is `JSON.SET`, which sets a Redis key with a JSON value. `JSON.SET` accepts all JSON value types. This example creates a JSON string: - -```sh -> JSON.SET animal $ '"dog"' -"OK" -> JSON.GET animal $ -"[\"dog\"]" -> JSON.TYPE animal $ -1) "string" -``` - -Note how the commands include the dollar sign character `$`. This is the [path](/redisjson/path) to the value in the JSON document (in this case it just means the root). - -Here are a few more string operations. `JSON.STRLEN` tells you the length of the string, and you can append another string to it with `JSON.STRAPPEND`. - -```sh -> JSON.STRLEN animal $ -1) "3" -> JSON.STRAPPEND animal $ '" (Canis familiaris)"' -1) "22" -> JSON.GET animal $ -"[\"dog (Canis familiaris)\"]" -``` - -Numbers can be [incremented](/commands/json.numincrby) and [multiplied](/commands/json.nummultby): - -``` -> JSON.SET num $ 0 -OK -> JSON.NUMINCRBY num $ 1 -"[1]" -> JSON.NUMINCRBY num $ 1.5 -"[2.5]" -> JSON.NUMINCRBY num $ -0.75 -"[1.75]" -> JSON.NUMMULTBY num $ 24 -"[42]" -``` - -Here's a more interesting example that includes JSON arrays and objects: - -``` -> JSON.SET example $ '[ true, { "answer": 42 }, null ]' -OK -> JSON.GET example $ -"[[true,{\"answer\":42},null]]" -> JSON.GET example $[1].answer -"[42]" -> JSON.DEL example $[-1] -(integer) 1 -> JSON.GET example $ -"[[true,{\"answer\":42}]]" -``` - -The `JSON.DEL` command deletes any JSON value you specify with the `path` parameter. - -You can manipulate arrays with a dedicated subset of JSON commands: - -``` -> JSON.SET arr $ [] -OK -> JSON.ARRAPPEND arr $ 0 -1) (integer) 1 -> JSON.GET arr $ -"[[0]]" -> JSON.ARRINSERT arr $ 0 -2 -1 -1) (integer) 3 -> JSON.GET arr $ -"[[-2,-1,0]]" -> JSON.ARRTRIM arr $ 1 1 -1) (integer) 1 -> JSON.GET arr $ -"[[-1]]" -> JSON.ARRPOP arr $ -1) "-1" -> JSON.ARRPOP arr $ -1) (nil) -``` - -JSON objects also have their own commands: - -``` -> JSON.SET obj $ '{"name":"Leonard Cohen","lastSeen":1478476800,"loggedOut": true}' -OK -> JSON.OBJLEN obj $ -1) (integer) 3 -> JSON.OBJKEYS obj $ -1) 1) "name" - 2) "lastSeen" - 3) "loggedOut" -``` - -To return a JSON response in a more human-readable format, run `redis-cli` in raw output mode and include formatting keywords such as `INDENT`, `NEWLINE`, and `SPACE` with the `JSON.GET` command: - -```sh -$ redis-cli --raw -> JSON.GET obj INDENT "\t" NEWLINE "\n" SPACE " " $ -[ - { - "name": "Leonard Cohen", - "lastSeen": 1478476800, - "loggedOut": true - } -] -``` - -### Python example - -This code snippet shows how to use JSON with raw Redis commands from Python with [redis-py](https://github.com/redis/redis-py): - -```Python -import redis - -data = { - 'dog': { - 'scientific-name' : 'Canis familiaris' - } -} - -r = redis.Redis() -r.json().set('doc', '$', data) -doc = r.json().get('doc', '$') -dog = r.json().get('doc', '$.dog') -scientific_name = r.json().get('doc', '$..scientific-name') -``` - -### Run with Docker - -To run RedisJSON with Docker, use the `redis-stack-server` Docker image: - -```sh -$ docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest -``` - -For more information about running Redis Stack in a Docker container, see [Run Redis Stack on Docker](/docs/getting-started/install-stack/docker). - -### Download binaries - -To download and run the RedisJSON module that provides the JSON data structure from a precompiled binary: - -1. Download a precompiled version from the [Redis download center](https://redis.com/download-center/modules/). - -2. Load the module it in Redis - - ```sh - $ redis-server --loadmodule /path/to/module/src/rejson.so - ``` - -### Build from source - -To build RedisJSON from the source code: - -1. Clone the [repository](https://github.com/RedisJSON/RedisJSON) (make sure you include the `--recursive` option to properly clone submodules): - - ```sh - $ git clone --recursive https://github.com/RedisJSON/RedisJSON.git - $ cd RedisJSON - ``` - -2. Install dependencies: - - ```sh - $ ./sbin/setup - ``` - -3. Build: - ```sh - $ make build - ``` - -### Load the module to Redis - -Requirements: - -Generally, it is best to run the latest Redis version. - -If your OS has a [Redis 6.x package or later](http://redis.io/download), you can install it using the OS package manager. - -Otherwise, you can invoke - -```sh -$ ./deps/readies/bin/getredis -``` - -To load the RedisJSON module, use one of the following methods: - -* [Makefile recipe](#makefile-recipe) -* [Configuration file](#configuration-file) -* [Command-line option](#command-line-option) -* [MODULE LOAD command](/commands/module-load/) - -#### Makefile recipe - -Run Redis with RedisJSON: - -```sh -$ make run -``` - -#### Configuration file - -Or you can have Redis load the module during startup by adding the following to your `redis.conf` file: - -``` -loadmodule /path/to/module/target/release/librejson.so -``` - -On Mac OS, if this module was built as a dynamic library, run: - -``` -loadmodule /path/to/module/target/release/librejson.dylib -``` - -In the above lines replace `/path/to/module/` with the actual path to the module. - -Alternatively, you can download and run Redis from a precompiled binary: - -1. Download a precompiled version of RedisJSON from the [Redis download center](https://redis.com/download-center/modules/). - -#### Command-line option - -Alternatively, you can have Redis load the module using the following command-line argument syntax: - - ```bash - $ redis-server --loadmodule /path/to/module/librejson.so - ``` - -In the above lines replace `/path/to/module/` with the actual path to the module's library. - -#### `MODULE LOAD` command - -You can also use the `MODULE LOAD` command to load RedisJSON. Note that `MODULE LOAD` is a **dangerous command** and may be blocked/deprecated in the future due to security considerations. - -After the module has been loaded successfully, the Redis log should have lines similar to: - -``` -... -9:M 11 Aug 2022 16:24:06.701 * version: 20009 git sha: d8d4b19 branch: HEAD -9:M 11 Aug 2022 16:24:06.701 * Exported RedisJSON_V1 API -9:M 11 Aug 2022 16:24:06.701 * Enabled diskless replication -9:M 11 Aug 2022 16:24:06.701 * Created new data type 'ReJSON-RL' -9:M 11 Aug 2022 16:24:06.701 * Module 'ReJSON' loaded from /opt/redis-stack/lib/rejson.so -... -``` - -### Limitation - -A JSON value passed to a command can have a depth of up to 128. If you pass to a command a JSON value that contains an object or an array with a nesting level of more than 128, the command returns an error. diff --git a/docs/docs/developer.md b/docs/docs/developer.md deleted file mode 100644 index b7f5de0b0..000000000 --- a/docs/docs/developer.md +++ /dev/null @@ -1,162 +0,0 @@ ---- -title: "Developer notes" -linkTitle: "Developer notes" -weight: 7 -description: > - Notes on debugging, testing and documentation -aliases: - - /docs/stack/json/developer ---- - -# Developing Redis JSON - -Developing Redis JSON involves setting up the development environment (which can be either Linux-based or macOS-based), building RedisJSON (the Redis module providing JSON), running tests and benchmarks, and debugging both the JSON module and its tests. - -## Cloning the git repository -To clone the RedisJSON module and its submodules, run: -```sh -git clone --recursive https://github.com/RedisJSON/RedisJSON.git -``` -## Working in an isolated environment -There are several reasons to use an isolated environment for development, like keeping your workstation clean and developing for a different Linux distribution. - -You can use a virtual machine as an isolated development environment. To set one up, you can use [Vagrant](https://www.vagrantup.com) or Docker. - -To set up a virtual machine with Docker: - -``` -rejson=$(docker run -d -it -v $PWD:/build debian:bullseye bash) -docker exec -it $rejson bash -``` -Then run ```cd /build``` from within the container. - -In this mode, all installations remain in the scope of the Docker container. -After you exit the container, you can either restart it with the previous ```docker exec``` command or save the state of the container to an image and resume it at a later time: - -``` -docker commit $rejson redisjson1 -docker stop $rejson -rejson=$(docker run -d -it -v $PWD:/build redisjson1 bash) -docker exec -it $rejson bash -``` - -You can replace `debian:bullseye` with your OS of choice. If you use the same OS as your host machine, you can run the RedisJSON binary on your host after it is built. - -## Installing prerequisites - -To build and test RedisJSON one needs to install several packages, depending on the underlying OS. Currently, we support the Ubuntu/Debian, CentOS, Fedora, and macOS. - -Enter the `RedisJSON` directory and run: - -```sh -$ ./sbin/setup -``` - -**This will install various packages on your system** using the native package manager and pip. It will invoke `sudo` on its own, prompting for permission. - -If you prefer to avoid that, you can: - -* Review `system-setup.py` and install packages manually, -* Use `system-setup.py --nop` to display installation commands without executing them, -* Use an isolated environment like explained above, -* Use a Python virtual environment, as Python installations are known to be sensitive when not used in isolation: `python -m virtualenv venv; . ./venv/bin/activate` - -## Installing Redis -Generally, it is best to run the latest Redis version. - -If your OS has a Redis 6.x package, you can install it using the OS package manager. - -Otherwise, you can invoke -```sh -$ ./deps/readies/bin/getredis -``` - -## Getting help -```make help``` provides a quick summary of the development features: - -``` -make setup # install prerequisites - -make build - DEBUG=1 # build debug variant - SAN=type # build with LLVM sanitizer (type=address|memory|leak|thread) - VALGRIND|VG=1 # build for testing with Valgrind -make clean # remove binary files - ALL=1 # remove binary directories - -make all # build all libraries and packages - -make test # run both cargo and python tests -make cargo_test # run inbuilt rust unit tests -make pytest # run flow tests using RLTest - TEST=file:name # run test matching `name` from `file` - TEST_ARGS="..." # RLTest arguments - QUICK=1 # run only general tests - GEN=1 # run general tests on a standalone Redis topology - AOF=1 # run AOF persistency tests on a standalone Redis topology - SLAVES=1 # run replication tests on standalone Redis topology - CLUSTER=1 # run general tests on a OSS Redis Cluster topology - VALGRIND|VG=1 # run specified tests with Valgrind - VERBOSE=1 # display more RLTest-related information - -make pack # build package (RAMP file) -make upload-artifacts # copy snapshot packages to S3 - OSNICK=nick # copy snapshots for specific OSNICK -make upload-release # copy release packages to S3 - -common options for upload operations: - STAGING=1 # copy to staging lab area (for validation) - FORCE=1 # allow operation outside CI environment - VERBOSE=1 # show more details - NOP=1 # do not copy, just print commands - -make coverage # perform coverage analysis -make show-cov # show coverage analysis results (implies COV=1) -make upload-cov # upload coverage analysis results to codecov.io (implies COV=1) - -make docker # build for specific Linux distribution - OSNICK=nick # Linux distribution to build for - REDIS_VER=ver # use Redis version `ver` - TEST=1 # test after build - PACK=1 # create packages - ARTIFACTS=1 # copy artifacts from docker image - PUBLISH=1 # publish (i.e. docker push) after build - -make sanbox # create container for CLang Sanitizer tests -``` - -## Building from source -Run ```make build``` to build RedisJSON. - -Notes: - -* Binary files are placed under `target/release/`, according to platform and build variant. - -* RedisJSON uses [Cargo](https://github.com/rust-lang/cargo) as its build system. ```make build``` will invoke both Cargo and the subsequent `make` command that's required to complete the build. - -Use ```make clean``` to remove built artifacts. ```make clean ALL=1``` will remove the entire bin subdirectory. - -## Running tests -There are several sets of unit tests: -* Rust tests, integrated in the source code, run by ```make cargo_test```. -* Python tests (enabled by RLTest), located in ```tests/pytests```, run by ```make pytest```. - -You can run all tests with ```make test```. -To run only specific tests, use the ```TEST``` parameter. For example, run ```make test TEST=regex```. - -You can run the module's tests against an "embedded" disposable Redis instance or against an instance -you provide. To use the "embedded" mode, you must include the `redis-server` executable in your `PATH`. - -You can override the spawning of the embedded server by specifying a Redis port via the `REDIS_PORT` -environment variable, e.g.: - -```bash -$ # use an existing local Redis instance for testing the module -$ REDIS_PORT=6379 make test -``` - -## Debugging -To include debugging information, you need to set the `DEBUG` environment variable before you compile RedisJSON. For example, run `export DEBUG=1`. - -You can add breakpoints to Python tests in single-test mode. To set a breakpoint, call the ```BB()``` function inside a test. - diff --git a/docs/docs/indexing_JSON.md b/docs/docs/indexing_JSON.md deleted file mode 100644 index 14c8130d5..000000000 --- a/docs/docs/indexing_JSON.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -title: Index/Search JSON documents -linkTitle: Index/Search -weight: 2 -description: Combine Redis JSON and Search and Query to index and search JSON documents -aliases: - - /docs/stack/json/indexing_json ---- - -In addition to storing JSON documents, you can also index them using the [Search and Query](/docs/stack/search) feature. This enables full-text search capabilities and document retrieval based on their content. - -To use these features, you must install two modules: RedisJSON and RediSearch. [Redis Stack](/docs/stack) automatically includes both modules. - -See the [tutorial](/docs/stack/search/indexing_json) to learn how to search and query your JSON. \ No newline at end of file diff --git a/docs/docs/path.md b/docs/docs/path.md deleted file mode 100644 index e5a4c537c..000000000 --- a/docs/docs/path.md +++ /dev/null @@ -1,285 +0,0 @@ ---- -title: "Path" -linkTitle: "Path" -weight: 3 -description: Access specific elements within a JSON document -aliases: - - /docs/stack/json/path ---- - -Paths help you access specific elements within a JSON document. Since no standard for JSON path syntax exists, Redis JSON implements its own. JSON's syntax is based on common best practices and intentionally resembles [JSONPath](http://goessner.net/articles/JsonPath/). - -JSON supports two query syntaxes: [JSONPath syntax](#jsonpath-syntax) and the [legacy path syntax](#legacy-path-syntax) from the first version of JSON. - -JSON knows which syntax to use depending on the first character of the path query. If the query starts with the character `$`, it uses JSONPath syntax. Otherwise, it defaults to the legacy path syntax. - -The returned value is a JSON string with a top-level array of JSON serialized strings. -And if multi-paths are used, the return value is a JSON string with a top-level object with values that are arrays of serialized JSON values. - -## JSONPath support - -RedisJSON v2.0 introduced [JSONPath](http://goessner.net/articles/JsonPath/) support. It follows the syntax described by Goessner in his [article](http://goessner.net/articles/JsonPath/). - -A JSONPath query can resolve to several locations in a JSON document. In this case, the JSON commands apply the operation to every possible location. This is a major improvement over [legacy path](#legacy-path-syntax) queries, which only operate on the first path. - -Notice that the structure of the command response often differs when using JSONPath. See the [Commands](/commands/?group=json) page for more details. - -The new syntax supports bracket notation, which allows the use of special characters like colon ":" or whitespace in key names. - -If you want to include double quotes in your query, enclose the JSONPath within single quotes. For example: - -```sh -JSON.GET store '$.inventory["headphones"]' -``` - -### JSONPath syntax - -The following JSONPath syntax table was adapted from Goessner's [path syntax comparison](https://goessner.net/articles/JsonPath/index.html#e2). - -| Syntax element | Description | -|----------------|-------------| -| $ | The root (outermost JSON element), starts the path. | -| . or [] | Selects a child element. | -| .. | Recursively descends through the JSON document. | -| * | Wildcard, returns all elements. | -| [] | Subscript operator, accesses an array element. | -| [,] | Union, selects multiple elements. | -| [start\:end\:step] | Array slice where start, end, and step are indexes. | -| ?() | Filters a JSON object or array. Supports comparison operators (`==`, `!=`, `<`, `<=`, `>`, `>=`, `=~`), logical operators (`&&`, `\|\|`), and parenthesis (`(`, `)`). | -| () | Script expression. | -| @ | The current element, used in filter or script expressions. | - -### JSONPath examples - -The following JSONPath examples use this JSON document, which stores details about items in a store's inventory: - -```json -{ - "inventory": { - "headphones": [ - { - "id": 12345, - "name": "Noise-cancelling Bluetooth headphones", - "description": "Wireless Bluetooth headphones with noise-cancelling technology", - "wireless": true, - "connection": "Bluetooth", - "price": 99.98, - "stock": 25, - "free-shipping": false, - "colors": ["black", "silver"] - }, - { - "id": 12346, - "name": "Wireless earbuds", - "description": "Wireless Bluetooth in-ear headphones", - "wireless": true, - "connection": "Bluetooth", - "price": 64.99, - "stock": 17, - "free-shipping": false, - "colors": ["black", "white"] - }, - { - "id": 12347, - "name": "Mic headset", - "description": "Headset with built-in microphone", - "wireless": false, - "connection": "USB", - "price": 35.01, - "stock": 28, - "free-shipping": false - } - ], - "keyboards": [ - { - "id": 22345, - "name": "Wireless keyboard", - "description": "Wireless Bluetooth keyboard", - "wireless": true, - "connection": "Bluetooth", - "price": 44.99, - "stock": 23, - "free-shipping": false, - "colors": ["black", "silver"] - }, - { - "id": 22346, - "name": "USB-C keyboard", - "description": "Wired USB-C keyboard", - "wireless": false, - "connection": "USB-C", - "price": 29.99, - "stock": 30, - "free-shipping": false - } - ] - } -} -``` - -First, create the JSON document in your database: - -```sh -JSON.SET store $ '{"inventory":{"headphones":[{"id":12345,"name":"Noise-cancelling Bluetooth headphones","description":"Wireless Bluetooth headphones with noise-cancelling technology","wireless":true,"connection":"Bluetooth","price":99.98,"stock":25,"free-shipping":false,"colors":["black","silver"]},{"id":12346,"name":"Wireless earbuds","description":"Wireless Bluetooth in-ear headphones","wireless":true,"connection":"Bluetooth","price":64.99,"stock":17,"free-shipping":false,"colors":["black","white"]},{"id":12347,"name":"Mic headset","description":"Headset with built-in microphone","wireless":false,"connection":"USB","price":35.01,"stock":28,"free-shipping":false}],"keyboards":[{"id":22345,"name":"Wireless keyboard","description":"Wireless Bluetooth keyboard","wireless":true,"connection":"Bluetooth","price":44.99,"stock":23,"free-shipping":false,"colors":["black","silver"]},{"id":22346,"name":"USB-C keyboard","description":"Wired USB-C keyboard","wireless":false,"connection":"USB-C","price":29.99,"stock":30,"free-shipping":false}]}}' -``` - -#### Access JSON examples - -The following examples use the `JSON.GET` command to retrieve data from various paths in the JSON document. - -You can use the wildcard operator `*` to return a list of all items in the inventory: - -```sh -127.0.0.1:6379> JSON.GET store $.inventory.* -"[[{\"id\":12345,\"name\":\"Noise-cancelling Bluetooth headphones\",\"description\":\"Wireless Bluetooth headphones with noise-cancelling technology\",\"wireless\":true,\"connection\":\"Bluetooth\",\"price\":99.98,\"stock\":25,\"free-shipping\":false,\"colors\":[\"black\",\"silver\"]},{\"id\":12346,\"name\":\"Wireless earbuds\",\"description\":\"Wireless Bluetooth in-ear headphones\",\"wireless\":true,\"connection\":\"Bluetooth\",\"price\":64.99,\"stock\":17,\"free-shipping\":false,\"colors\":[\"black\",\"white\"]},{\"id\":12347,\"name\":\"Mic headset\",\"description\":\"Headset with built-in microphone\",\"wireless\":false,\"connection\":\"USB\",\"price\":35.01,\"stock\":28,\"free-shipping\":false}],[{\"id\":22345,\"name\":\"Wireless keyboard\",\"description\":\"Wireless Bluetooth keyboard\",\"wireless\":true,\"connection\":\"Bluetooth\",\"price\":44.99,\"stock\":23,\"free-shipping\":false,\"colors\":[\"black\",\"silver\"]},{\"id\":22346,\"name\":\"USB-C keyboard\",\"description\":\"Wired USB-C keyboard\",\"wireless\":false,\"connection\":\"USB-C\",\"price\":29.99,\"stock\":30,\"free-shipping\":false}]]" -``` - -For some queries, multiple paths can produce the same results. For example, the following paths return the names of all headphones: - -```sh -127.0.0.1:6379> JSON.GET store $.inventory.headphones[*].name -"[\"Noise-cancelling Bluetooth headphones\",\"Wireless earbuds\",\"Mic headset\"]" -127.0.0.1:6379> JSON.GET store '$.inventory["headphones"][*].name' -"[\"Noise-cancelling Bluetooth headphones\",\"Wireless earbuds\",\"Mic headset\"]" -127.0.0.1:6379> JSON.GET store $..headphones[*].name -"[\"Noise-cancelling Bluetooth headphones\",\"Wireless earbuds\",\"Mic headset\"]" -``` - -The recursive descent operator `..` can retrieve a field from multiple sections of a JSON document. The following example returns the names of all inventory items: - -```sh -127.0.0.1:6379> JSON.GET store $..name -"[\"Noise-cancelling Bluetooth headphones\",\"Wireless earbuds\",\"Mic headset\",\"Wireless keyboard\",\"USB-C keyboard\"]" -``` - -You can use an array slice to select a range of elements from an array. This example returns the names of the first two headphones: - -```sh -127.0.0.1:6379> JSON.GET store $..headphones[0:2].name -"[\"Noise-cancelling Bluetooth headphones\",\"Wireless earbuds\"]" -``` - -Filter expressions `?()` let you select JSON elements based on certain conditions. You can use comparison operators (`==`, `!=`, `<`, `<=`, `>`, `>=`, and starting with version v2.4.2, also `=~`), logical operators (`&&`, `||`), and parenthesis (`(`, `)`) within these expressions. A filter expression can be applied on an array or on an object, iterating over all the **elements** in the array or all the **values** in the object, retrieving only the ones that match the filter condition. - -Paths within the filter condition are using the dot notation with either `@` to denote the current array element or the current object value, or `$` to denote the top-level element. For example, use `@.key_name` to refer to a nested value and `$.top_level_key_name` to refer to a top-level value. - -Starting with version v2.4.2, the comparison operator `=~` can be used for matching a path of a string value on the left side against a regular expression pattern on the right side. For more information, see the [supported regular expression syntax docs](https://docs.rs/regex/latest/regex/#syntax). - -Non-string values do not match. A match can only occur when the left side is a path of a string value and the right side is either a hard-coded string, or a path of a string value. See [examples](#json-filter-examples) below. - -The regex match is partial, meaning `"foo"` regex pattern matches a string such as `"barefoots"`. -To make it exact, use the regex pattern `"^foo$"`. - -Other JSONPath engines may use regex pattern between slashes, e.g., `/foo/`, and their match is exact. -They can perform partial matches using a regex pattern such as `/.*foo.*/`. - -#### JSON Filter examples - -In the following example, the filter only returns wireless headphones with a price less than 70: - -```sh -127.0.0.1:6379> JSON.GET store $..headphones[?(@.price<70&&@.wireless==true)] -"[{\"id\":12346,\"name\":\"Wireless earbuds\",\"description\":\"Wireless Bluetooth in-ear headphones\",\"wireless\":true,\"connection\":\"Bluetooth\",\"price\":64.99,\"stock\":17,\"free-shipping\":false,\"colors\":[\"black\",\"white\"]}]" -``` - -This example filters the inventory for the names of items that support Bluetooth connections: - -```sh -127.0.0.1:6379> JSON.GET store '$.inventory.*[?(@.connection=="Bluetooth")].name' -"[\"Noise-cancelling Bluetooth headphones\",\"Wireless earbuds\",\"Wireless keyboard\"]" -``` - -This example, starting with version v2.4.2, filters only keyboards with some sort of USB connection using regex match. Notice this match is case-insensitive thanks to the prefix `(?i)` in the regular expression pattern `"(?i)usb"`: - -```sh -127.0.0.1:6379> JSON.GET store '$.inventory.keyboards[?(@.connection =~ "(?i)usb")]' -"[{\"id\":22346,\"name\":\"USB-C keyboard\",\"description\":\"Wired USB-C keyboard\",\"wireless\":false,\"connection\":\"USB-C\",\"price\":29.99,\"stock\":30,\"free-shipping\":false}]" -``` -The regular expression pattern can also be specified using a path of a string value on the right side. - -For example, let's add each keybaord object with a string value named `regex_pat`: - -```sh -127.0.0.1:6379> JSON.SET store '$.inventory.keyboards[0].regex_pat' '"(?i)bluetooth"' -OK -127.0.0.1:6379> JSON.SET store '$.inventory.keyboards[1].regex_pat' '"usb"' -OK -``` - -Now we can match against the value of `regex_pat` instead of a hard-coded regular expression pattern, and get the keyboard with the `Bluetooth` string in its `connection` key. Notice the one with `USB-C` does not match since its regular expression pattern is case-sensitive and the regular expression pattern is using lowercase: - -```sh -127.0.0.1:6379> JSON.GET store '$.inventory.keyboards[?(@.connection =~ @.regex_pat)]' -"[{\"id\":22345,\"name\":\"Wireless keyboard\",\"description\":\"Wireless Bluetooth keyboard\",\"wireless\":true,\"connection\":\"Bluetooth\",\"price\":44.99,\"stock\":23,\"free-shipping\":false,\"colors\":[\"black\",\"silver\"],\"regex\":\"(?i)Bluetooth\",\"regex_pat\":\"(?i)bluetooth\"}]" -``` - -#### Update JSON examples - -You can also use JSONPath queries when you want to update specific sections of a JSON document. - -For example, you can pass a JSONPath to the `JSON.SET` command to update a specific field. This example changes the price of the first item in the headphones list: - -```sh -127.0.0.1:6379> JSON.GET store $..headphones[0].price -"[99.98]" -127.0.0.1:6379> JSON.SET store $..headphones[0].price 78.99 -"OK" -127.0.0.1:6379> JSON.GET store $..headphones[0].price -"[78.99]" -``` - -You can use filter expressions to update only JSON elements that match certain conditions. The following example changes `free-shipping` to `true` for any items with a price greater than 49: - -```sh -127.0.0.1:6379> JSON.SET store $.inventory.*[?(@.price>49)].free-shipping true -"OK" -127.0.0.1:6379> JSON.GET store $.inventory.*[?(@.free-shipping==true)].name -"[\"Noise-cancelling Bluetooth headphones\",\"Wireless earbuds\"]" -``` - -JSONPath queries also work with other JSON commands that accept a path as an argument. For example, you can add a new color option for a set of headphones with `JSON.ARRAPPEND`: - -```sh -127.0.0.1:6379> JSON.GET store $..headphones[0].colors -"[[\"black\",\"silver\"]]" -127.0.0.1:6379> JSON.ARRAPPEND store $..headphones[0].colors '"pink"' -1) "3" -127.0.0.1:6379> JSON.GET store $..headphones[0].colors -"[[\"black\",\"silver\",\"pink\"]]" -``` - -## Legacy path syntax - -RedisJSON v1 had the following path implementation. JSON v2 still supports this legacy path in addition to JSONPath. - -Paths always begin at the root of a Redis JSON value. The root is denoted by a period character (`.`). For paths that reference the root's children, it is optional to prefix the path with the root. - -Redis JSON supports both dot notation and bracket notation for object key access. The following paths refer to _headphones_, which is a child of _inventory_ under the root: - -* `.inventory.headphones` -* `inventory["headphones"]` -* `['inventory']["headphones"]` - -To access an array element, enclose its index within a pair of square brackets. The index is 0-based, with 0 being the first element of the array, 1 being the next element, and so on. You can use negative offsets to access elements starting from the end of the array. For example, -1 is the last element in the array, -2 is the second to last element, and so on. - -### JSON key names and path compatibility - -By definition, a JSON key can be any valid JSON string. Paths, on the other hand, are traditionally based on JavaScript's (and Java's) variable naming conventions. - -Although JSON can store objects that contain arbitrary key names, you can only use a legacy path to access these keys if they conform to these naming syntax rules: - -1. Names must begin with a letter, a dollar sign (`$`), or an underscore (`_`) character -2. Names can contain letters, digits, dollar signs, and underscores -3. Names are case-sensitive - -## Time complexity of path evaluation - -The time complexity of searching (navigating to) an element in the path is calculated from: - -1. Child level - every level along the path adds an additional search -2. Key search - O(N)†, where N is the number of keys in the parent object -3. Array search - O(1) - -This means that the overall time complexity of searching a path is _O(N*M)_, where N is the depth and M is the number of parent object keys. - -† While this is acceptable for objects where N is small, access can be optimized for larger objects. diff --git a/docs/docs/performance/_index.md b/docs/docs/performance/_index.md deleted file mode 100644 index d78e25350..000000000 --- a/docs/docs/performance/_index.md +++ /dev/null @@ -1,291 +0,0 @@ ---- -title: "Performance" -linkTitle: "Performance" -weight: 5 -description: > - Performance benchmarks -aliases: - - /docs/stack/json/performance ---- - -To get an early sense of what Redis JSON is capable of, you can test it with `redis-benchmark` just like -any other Redis command. However, in order to have more control over the tests, we'll use a -a tool written in Go called _ReJSONBenchmark_ that we expect to release in the near future. - -The following figures were obtained from an AWS EC2 c4.8xlarge instance that ran both the Redis -server as well the as the benchmarking tool. Connections to the server are via the networking stack. -All tests are non-pipelined. - -> NOTE: The results below are measured using the preview version of Redis JSON, which is still very much unoptimized. - -## Redis JSON baseline - -### A smallish object - -We test a JSON value that, while purely synthetic, is interesting. The test subject is -[/tests/files/pass-100.json](https://github.com/RedisLabsModules/redisjson/blob/master/tests/files/pass-100.json), -who weighs in at 380 bytes and is nested. We first test SETting it, then GETting it using several -different paths: - -![ReJSONBenchmark pass-100.json](images/bench_pass_100.png) - -![ReJSONBenchmark pass-100.json percentiles](images/bench_pass_100_p.png) - -### A bigger array - -Moving on to bigger values, we use the 1.4 kB array in -[/tests/files/pass-jsonsl-1.json](https://github.com/RedisLabsModules/redisjson/blob/master/tests/files/pass-jsonsl-1.json): - - -![ReJSONBenchmark pass-jsonsl-1.json](images/bench_pass_jsonsl_1.png) - -![ReJSONBenchmark pass-jsonsl-1.json percentiles](images/bench_pass_jsonsl_1_p.png) - -### A largish object - -More of the same to wrap up, now we'll take on a behemoth of no less than 3.5 kB as given by -[/tests/files/pass-json-parser-0000.json](https://github.com/RedisLabsModules/redisjson/blob/master/tests/files/pass-json-parser-0000.json): - -![ReJSONBenchmark pass-json-parser-0000.json](images/bench_pass_json_parser_0000.png) - -![ReJSONBenchmark pass-json-parser-0000.json percentiles](images/bench_pass_json_parser_0000_p.png) - -### Number operations - -Last but not least, some adding and multiplying: - -![ReJSONBenchmark number operations](images/bench_numbers.png) - -![ReJSONBenchmark number operations percentiles](images/bench_numbers_p.png) - -### Baseline - -To establish a baseline, we'll use the Redis `PING` command. -First, lets see what `redis-benchmark` reports: - -``` -~$ redis/src/redis-benchmark -n 1000000 ping -====== ping ====== - 1000000 requests completed in 7.11 seconds - 50 parallel clients - 3 bytes payload - keep alive: 1 - -99.99% <= 1 milliseconds -100.00% <= 1 milliseconds -140587.66 requests per second -``` - -ReJSONBenchmark's concurrency is configurable, so we'll test a few settings to find a good one. Here -are the results, which indicate that 16 workers yield the best throughput: - -![ReJSONBenchmark PING](images/bench_ping.png) - -![ReJSONBenchmark PING percentiles](images/bench_ping_p.png) - -Note how our benchmarking tool does slightly worse in PINGing - producing only 116K ops, compared to -`redis-cli`'s 140K. - -### The empty string - -Another JSON benchmark is that of setting and getting an empty string - a value that's only two -bytes long (i.e. `""`). Granted, that's not very useful, but it teaches us something about the basic -performance of the module: - -![ReJSONBenchmark empty string](images/bench_empty_string.png) - -![ReJSONBenchmark empty string percentiles](images/bench_empty_string_p.png) - -## Comparison vs. server-side Lua scripting - -We compare Redis Stack's JSON performance with Redis' embedded Lua engine. For this purpose, we use the Lua -scripts at [/benchmarks/lua](https://github.com/RedisLabsModules/redisjson/tree/master/benchmarks/lua). -These scripts provide JSON's GET and SET functionality on values stored in JSON or MessagePack -formats. Each of the different operations (set root, get root, set path and get path) is executed -with each "engine" on objects of varying sizes. - -### Setting and getting the root - -Storing raw JSON performs best in this test, but that isn't really surprising as all it does is -serve unprocessed strings. While you can and should use Redis for caching opaque data, and JSON -"blobs" are just one example, this does not allow any updates other than these of the entire value. - -A more meaningful comparison therefore is between JSON and the MessagePack variant, since both -process the incoming JSON value before actually storing it. While the rates and latencies of these -two behave in a very similar way, the absolute measurements suggest that Redis JSON's performance may be -further improved. - -![VS. Lua set root](images/bench_lua_set_root.png) - -![VS. Lua set root latency](images/bench_lua_set_root_l.png) - -![VS. Lua get root](images/bench_lua_get_root.png) - -![VS. Lua get root latency](images/bench_lua_get_root_l.png) - -### Setting and getting parts of objects - -This test shows why Redis JSON exists. Not only does it outperform the Lua variants, it retains constant -rates and latencies regardless the object's overall size. There's no magic here - JSON keeps the -value deserialized so that accessing parts of it is a relatively inexpensive operation. In deep contrast -are both raw JSON as well as MessagePack, which require decoding the entire object before anything can -be done with it (a process that becomes more expensive the larger the object is). - -![VS. Lua set path to scalar](images/bench_lua_set_path.png) - -![VS. Lua set path to scalar latency](images/bench_lua_set_path_l.png) - -![VS. Lua get scalar from path](images/bench_lua_get_path.png) - -![VS. Lua get scalar from path latency](images/bench_lua_get_path_l.png) - -### Even more charts - -These charts are more of the same but independent for each file (value): - -![VS. Lua pass-100.json rate](images/bench_lua_pass_100.png) - -![VS. Lua pass-100.json average latency](images/bench_lua_pass_100_l.png) - -![VS. Lua pass-jsonsl-1.json rate](images/bench_lua_pass_jsonsl_1.png) - -![VS. Lua pass-jsonsl-1.json average latency](images/bench_lua_pass_jsonsl_1_l.png) - -![VS. Lua pass-json-parser-0000.json rate](images/bench_lua_pass_json_parser_0000.png) - -![VS. Lua pass-json-parser-0000.json latency](images/bench_lua_pass_json_parser_0000_l.png) - -![VS. Lua pass-jsonsl-yahoo2.json rate](images/bench_lua_pass_jsonsl_yahoo2.png) - -![VS. Lua pass-jsonsl-yahoo2.json latency](images/bench_lua_pass_jsonsl_yahoo2_l.png) - -![VS. Lua pass-jsonsl-yelp.json rate](images/bench_lua_pass_jsonsl_yelp.png) - -![VS. Lua pass-jsonsl-yelp.json latency](images/bench_lua_pass_jsonsl_yelp_l.png) - -## Raw results - -The following are the raw results from the benchmark in CSV format. - -### JSON results - -``` -title,concurrency,rate,average latency,50.00%-tile,90.00%-tile,95.00%-tile,99.00%-tile,99.50%-tile,100.00%-tile -[ping],1,22128.12,0.04,0.04,0.04,0.05,0.05,0.05,1.83 -[ping],2,54641.13,0.04,0.03,0.05,0.05,0.06,0.07,2.14 -[ping],4,76000.18,0.05,0.05,0.07,0.07,0.09,0.10,2.10 -[ping],8,106750.99,0.07,0.07,0.10,0.11,0.14,0.16,2.99 -[ping],12,111297.33,0.11,0.10,0.15,0.16,0.20,0.22,6.81 -[ping],16,116292.19,0.14,0.13,0.19,0.21,0.27,0.33,7.50 -[ping],20,110622.82,0.18,0.17,0.24,0.27,0.38,0.47,12.21 -[ping],24,107468.51,0.22,0.20,0.31,0.38,0.58,0.71,13.86 -[ping],28,102827.35,0.27,0.25,0.38,0.44,0.66,0.79,12.87 -[ping],32,105733.51,0.30,0.28,0.42,0.50,0.79,0.97,10.56 -[ping],36,102046.43,0.35,0.33,0.48,0.56,0.90,1.13,14.66 -JSON.SET {key} . {empty string size: 2 B},16,80276.63,0.20,0.18,0.28,0.32,0.41,0.45,6.48 -JSON.GET {key} .,16,92191.23,0.17,0.16,0.24,0.27,0.34,0.38,9.80 -JSON.SET {key} . {pass-100.json size: 380 B},16,41512.77,0.38,0.35,0.50,0.62,0.81,0.86,9.56 -JSON.GET {key} .,16,48374.10,0.33,0.29,0.47,0.56,0.72,0.79,9.36 -JSON.GET {key} sclr,16,94801.23,0.17,0.15,0.24,0.27,0.35,0.39,13.21 -JSON.SET {key} sclr 1,16,82032.08,0.19,0.18,0.27,0.31,0.40,0.44,8.97 -JSON.GET {key} sub_doc,16,81633.51,0.19,0.18,0.27,0.32,0.43,0.49,9.88 -JSON.GET {key} sub_doc.sclr,16,95052.35,0.17,0.15,0.24,0.27,0.35,0.39,7.39 -JSON.GET {key} array_of_docs,16,68223.05,0.23,0.22,0.29,0.31,0.44,0.50,8.84 -JSON.GET {key} array_of_docs[1],16,76390.57,0.21,0.19,0.30,0.34,0.44,0.49,9.99 -JSON.GET {key} array_of_docs[1].sclr,16,90202.13,0.18,0.16,0.25,0.29,0.36,0.39,7.87 -JSON.SET {key} . {pass-jsonsl-1.json size: 1.4 kB},16,16117.11,0.99,0.91,1.22,1.55,2.17,2.35,9.27 -JSON.GET {key} .,16,15193.51,1.05,0.94,1.41,1.75,2.33,2.42,7.19 -JSON.GET {key} [0],16,78198.90,0.20,0.19,0.29,0.33,0.42,0.47,10.87 -"JSON.SET {key} [0] ""foo""",16,80156.90,0.20,0.18,0.28,0.32,0.40,0.44,12.03 -JSON.GET {key} [7],16,99013.98,0.16,0.15,0.23,0.26,0.34,0.38,7.67 -JSON.GET {key} [8].zero,16,90562.19,0.17,0.16,0.25,0.28,0.35,0.38,7.03 -JSON.SET {key} . {pass-json-parser-0000.json size: 3.5 kB},16,14239.25,1.12,1.06,1.21,1.48,2.35,2.59,11.91 -JSON.GET {key} .,16,8366.31,1.91,1.86,2.00,2.04,2.92,3.51,12.92 -"JSON.GET {key} [""web-app""].servlet",16,9339.90,1.71,1.68,1.74,1.78,2.68,3.26,10.47 -"JSON.GET {key} [""web-app""].servlet[0]",16,13374.88,1.19,1.07,1.54,1.95,2.69,2.82,12.15 -"JSON.GET {key} [""web-app""].servlet[0][""servlet-name""]",16,81267.36,0.20,0.18,0.28,0.31,0.38,0.42,9.67 -"JSON.SET {key} [""web-app""].servlet[0][""servlet-name""] ""bar""",16,79955.04,0.20,0.18,0.27,0.33,0.42,0.46,6.72 -JSON.SET {key} . {pass-jsonsl.yahoo2-json size: 18 kB},16,3394.07,4.71,4.62,4.72,4.79,7.35,9.03,17.78 -JSON.GET {key} .,16,891.46,17.92,17.33,17.56,20.12,31.77,42.87,66.64 -JSON.SET {key} ResultSet.totalResultsAvailable 1,16,75513.03,0.21,0.19,0.30,0.34,0.42,0.46,9.21 -JSON.GET {key} ResultSet.totalResultsAvailable,16,91202.84,0.17,0.16,0.24,0.28,0.35,0.38,5.30 -JSON.SET {key} . {pass-jsonsl-yelp.json size: 40 kB},16,1624.86,9.84,9.67,9.86,9.94,15.86,19.36,31.94 -JSON.GET {key} .,16,442.55,36.08,35.62,37.78,38.14,55.23,81.33,88.40 -JSON.SET {key} message.code 1,16,77677.25,0.20,0.19,0.28,0.33,0.42,0.45,11.07 -JSON.GET {key} message.code,16,89206.61,0.18,0.16,0.25,0.28,0.36,0.39,8.60 -[JSON.SET num . 0],16,84498.21,0.19,0.17,0.26,0.30,0.39,0.43,8.08 -[JSON.NUMINCRBY num . 1],16,78640.20,0.20,0.18,0.28,0.33,0.44,0.48,11.05 -[JSON.NUMMULTBY num . 2],16,77170.85,0.21,0.19,0.28,0.33,0.43,0.47,6.85 -``` - -### Lua using cjson - -``` -json-set-root.lua empty string,16,86817.84,0.18,0.17,0.26,0.31,0.39,0.42,9.36 -json-get-root.lua,16,90795.08,0.17,0.16,0.25,0.28,0.36,0.39,8.75 -json-set-root.lua pass-100.json,16,84190.26,0.19,0.17,0.27,0.30,0.38,0.41,12.00 -json-get-root.lua,16,87170.45,0.18,0.17,0.26,0.29,0.38,0.45,9.81 -json-get-path.lua sclr,16,54556.80,0.29,0.28,0.35,0.38,0.57,0.64,7.53 -json-set-path.lua sclr 1,16,35907.30,0.44,0.42,0.53,0.67,0.93,1.00,8.57 -json-get-path.lua sub_doc,16,51158.84,0.31,0.30,0.36,0.39,0.50,0.62,7.22 -json-get-path.lua sub_doc sclr,16,51054.47,0.31,0.29,0.39,0.47,0.66,0.74,7.43 -json-get-path.lua array_of_docs,16,39103.77,0.41,0.37,0.57,0.68,0.87,0.94,8.02 -json-get-path.lua array_of_docs 1,16,45811.31,0.35,0.32,0.45,0.56,0.77,0.83,8.17 -json-get-path.lua array_of_docs 1 sclr,16,47346.83,0.34,0.31,0.44,0.54,0.72,0.79,8.07 -json-set-root.lua pass-jsonsl-1.json,16,82100.90,0.19,0.18,0.28,0.31,0.39,0.43,12.43 -json-get-root.lua,16,77922.14,0.20,0.18,0.30,0.34,0.66,0.86,8.71 -json-get-path.lua 0,16,38162.83,0.42,0.40,0.49,0.59,0.88,0.96,6.16 -"json-set-path.lua 0 ""foo""",16,21205.52,0.75,0.70,0.84,1.07,1.60,1.74,5.77 -json-get-path.lua 7,16,37254.89,0.43,0.39,0.55,0.69,0.92,0.98,10.24 -json-get-path.lua 8 zero,16,33772.43,0.47,0.43,0.63,0.77,1.01,1.09,7.89 -json-set-root.lua pass-json-parser-0000.json,16,76314.18,0.21,0.19,0.29,0.33,0.41,0.44,8.16 -json-get-root.lua,16,65177.87,0.24,0.21,0.35,0.42,0.89,1.01,9.02 -json-get-path.lua web-app servlet,16,15938.62,1.00,0.88,1.45,1.71,2.11,2.20,8.07 -json-get-path.lua web-app servlet 0,16,19469.27,0.82,0.78,0.90,1.07,1.67,1.84,7.59 -json-get-path.lua web-app servlet 0 servlet-name,16,24694.26,0.65,0.63,0.71,0.74,1.07,1.31,8.60 -"json-set-path.lua web-app servlet 0 servlet-name ""bar""",16,16555.74,0.96,0.92,1.05,1.25,1.98,2.20,9.08 -json-set-root.lua pass-jsonsl-yahoo2.json,16,47544.65,0.33,0.31,0.41,0.47,0.59,0.64,10.52 -json-get-root.lua,16,25369.92,0.63,0.57,0.91,1.05,1.37,1.56,9.95 -json-set-path.lua ResultSet totalResultsAvailable 1,16,5077.32,3.15,3.09,3.20,3.24,5.12,6.26,14.98 -json-get-path.lua ResultSet totalResultsAvailable,16,7652.56,2.09,2.05,2.13,2.17,3.23,3.95,9.65 -json-set-root.lua pass-jsonsl-yelp.json,16,29575.20,0.54,0.52,0.64,0.75,0.94,1.00,12.66 -json-get-root.lua,16,18424.29,0.87,0.84,1.25,1.40,1.82,1.95,7.35 -json-set-path.lua message code 1,16,2251.07,7.10,6.98,7.14,7.22,11.00,12.79,21.14 -json-get-path.lua message code,16,3380.72,4.73,4.44,5.03,6.82,10.28,11.06,14.93 -``` - -### Lua using cmsgpack - -``` -msgpack-set-root.lua empty string,16,82592.66,0.19,0.18,0.27,0.31,0.38,0.42,10.18 -msgpack-get-root.lua,16,89561.41,0.18,0.16,0.25,0.29,0.37,0.40,9.52 -msgpack-set-root.lua pass-100.json,16,44326.47,0.36,0.34,0.43,0.54,0.78,0.86,6.45 -msgpack-get-root.lua,16,41036.58,0.39,0.36,0.51,0.62,0.84,0.91,7.21 -msgpack-get-path.lua sclr,16,55845.56,0.28,0.26,0.36,0.44,0.64,0.70,11.29 -msgpack-set-path.lua sclr 1,16,43608.26,0.37,0.34,0.47,0.58,0.78,0.85,10.27 -msgpack-get-path.lua sub_doc,16,50153.07,0.32,0.29,0.41,0.50,0.69,0.75,8.56 -msgpack-get-path.lua sub_doc sclr,16,54016.35,0.29,0.27,0.38,0.46,0.62,0.67,6.38 -msgpack-get-path.lua array_of_docs,16,45394.79,0.35,0.32,0.45,0.56,0.78,0.85,11.88 -msgpack-get-path.lua array_of_docs 1,16,48336.48,0.33,0.30,0.42,0.52,0.71,0.76,7.69 -msgpack-get-path.lua array_of_docs 1 sclr,16,53689.41,0.30,0.27,0.38,0.46,0.64,0.69,11.16 -msgpack-set-root.lua pass-jsonsl-1.json,16,28956.94,0.55,0.51,0.65,0.82,1.17,1.26,8.39 -msgpack-get-root.lua,16,26045.44,0.61,0.58,0.68,0.83,1.28,1.42,8.56 -"msgpack-set-path.lua 0 ""foo""",16,29813.56,0.53,0.49,0.67,0.83,1.15,1.22,6.82 -msgpack-get-path.lua 0,16,44827.58,0.36,0.32,0.48,0.58,0.76,0.81,9.19 -msgpack-get-path.lua 7,16,47529.14,0.33,0.31,0.42,0.53,0.73,0.79,7.47 -msgpack-get-path.lua 8 zero,16,44442.72,0.36,0.33,0.45,0.56,0.77,0.85,8.11 -msgpack-set-root.lua pass-json-parser-0000.json,16,19585.82,0.81,0.78,0.85,1.05,1.66,1.86,4.33 -msgpack-get-root.lua,16,19014.08,0.84,0.73,1.23,1.45,1.76,1.84,13.52 -msgpack-get-path.lua web-app servlet,16,18992.61,0.84,0.73,1.23,1.45,1.75,1.82,8.19 -msgpack-get-path.lua web-app servlet 0,16,24328.78,0.66,0.64,0.73,0.77,1.15,1.34,8.81 -msgpack-get-path.lua web-app servlet 0 servlet-name,16,31012.81,0.51,0.49,0.57,0.65,1.02,1.13,8.11 -"msgpack-set-path.lua web-app servlet 0 servlet-name ""bar""",16,20388.54,0.78,0.73,0.88,1.08,1.63,1.78,7.22 -msgpack-set-root.lua pass-jsonsl-yahoo2.json,16,5597.60,2.85,2.81,2.89,2.94,4.57,5.59,10.19 -msgpack-get-root.lua,16,6585.01,2.43,2.39,2.52,2.66,3.76,4.80,10.59 -msgpack-set-path.lua ResultSet totalResultsAvailable 1,16,6666.95,2.40,2.35,2.43,2.47,3.78,4.59,12.08 -msgpack-get-path.lua ResultSet totalResultsAvailable,16,10733.03,1.49,1.45,1.60,1.66,2.36,2.93,13.15 -msgpack-set-root-lua pass-jsonsl-yelp.json,16,2291.53,6.97,6.87,7.01,7.12,10.54,12.89,21.75 -msgpack-get-root.lua,16,2889.59,5.53,5.45,5.71,5.86,8.80,10.48,25.55 -msgpack-set-path.lua message code 1,16,2847.85,5.61,5.44,5.56,6.01,10.58,11.90,16.91 -msgpack-get-path.lua message code,16,5030.95,3.18,3.07,3.24,3.57,6.08,6.92,12.44 -``` diff --git a/docs/docs/ram.md b/docs/docs/ram.md deleted file mode 100644 index ffc9f32ba..000000000 --- a/docs/docs/ram.md +++ /dev/null @@ -1,107 +0,0 @@ ---- -title: "Redis JSON RAM Usage" -linkTitle: "Memory Usage" -weight: 6 -description: > - Debugging memory consumption -aliases: - - /docs/stack/json/ram ---- - -Every key in Redis takes memory and requires at least the amount of RAM to store the key name, as -well as some per-key overhead that Redis uses. On top of that, the value in the key also requires -RAM. - -Redis JSON stores JSON values as binary data after deserializing them. This representation is often more -expensive, size-wize, than the serialized form. The JSON data type uses at least 24 bytes (on -64-bit architectures) for every value, as can be seen by sampling an empty string with the -`JSON.DEBUG MEMORY` command: - -``` -127.0.0.1:6379> JSON.SET emptystring . '""' -OK -127.0.0.1:6379> JSON.DEBUG MEMORY emptystring -(integer) 24 -``` - -This RAM requirement is the same for all scalar values, but strings require additional space -depending on their actual length. For example, a 3-character string will use 3 additional bytes: - -``` -127.0.0.1:6379> JSON.SET foo . '"bar"' -OK -127.0.0.1:6379> JSON.DEBUG MEMORY foo -(integer) 27 -``` - -Empty containers take up 32 bytes to set up: - -``` -127.0.0.1:6379> JSON.SET arr . '[]' -OK -127.0.0.1:6379> JSON.DEBUG MEMORY arr -(integer) 32 -127.0.0.1:6379> JSON.SET obj . '{}' -OK -127.0.0.1:6379> JSON.DEBUG MEMORY obj -(integer) 32 -``` - -The actual size of a container is the sum of sizes of all items in it on top of its own -overhead. To avoid expensive memory reallocations, containers' capacity is scaled by multiples of 2 -until a treshold size is reached, from which they grow by fixed chunks. - -A container with a single scalar is made up of 32 and 24 bytes, respectively: -``` -127.0.0.1:6379> JSON.SET arr . '[""]' -OK -127.0.0.1:6379> JSON.DEBUG MEMORY arr -(integer) 56 -``` - -A container with two scalars requires 40 bytes for the container (each pointer to an entry in the -container is 8 bytes), and 2 * 24 bytes for the values themselves: -``` -127.0.0.1:6379> JSON.SET arr . '["", ""]' -OK -127.0.0.1:6379> JSON.DEBUG MEMORY arr -(integer) 88 -``` - -A 3-item (each 24 bytes) container will be allocated with capacity for 4 items, i.e. 56 bytes: - -``` -127.0.0.1:6379> JSON.SET arr . '["", "", ""]' -OK -127.0.0.1:6379> JSON.DEBUG MEMORY arr -(integer) 128 -``` - -The next item will not require an allocation in the container, so usage will increase only by that -scalar's requirement, but another value will scale the container again: - -``` -127.0.0.1:6379> JSON.SET arr . '["", "", "", ""]' -OK -127.0.0.1:6379> JSON.DEBUG MEMORY arr -(integer) 152 -127.0.0.1:6379> JSON.SET arr . '["", "", "", "", ""]' -OK -127.0.0.1:6379> JSON.DEBUG MEMORY arr -(integer) 208 -``` - -This table gives the size (in bytes) of a few of the test files on disk and when stored using -JSON. The _MessagePack_ column is for reference purposes and reflects the length of the value -when stored using MessagePack. - -| File | Filesize | Redis JSON | MessagePack | -| -------------------------------------- | --------- | ------ | ----------- | -| /tests/files/pass-100.json | 380 | 1079 | 140 | -| /tests/files/pass-jsonsl-1.json | 1441 | 3666 | 753 | -| /tests/files/pass-json-parser-0000.json | 3468 | 7209 | 2393 | -| /tests/files/pass-jsonsl-yahoo2.json | 18446 | 37469 | 16869 | -| /tests/files/pass-jsonsl-yelp.json | 39491 | 75341 | 35469 | - -> Note: In the current version, deleting values from containers **does not** free the container's -allocated memory. diff --git a/docs/docs/resp3.md b/docs/docs/resp3.md deleted file mode 100644 index 2ef61dc3a..000000000 --- a/docs/docs/resp3.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -title: "Guide for migrating from RESP2 to RESP3 replies" -linkTitle: "RESP3 migration guide" -weight: 6 -description: JSON RESP2 to RESP3 replies reference for client developers ---- - -In RESP3, the default value of the optional path argument was changed from `.` to `$`. -Due to this change, the replies of some commands have slightly changed. -This page provides a brief comparison between RESP2 and RESP3 responses for JSON commands to help developers in migrating their clients from RESP2 to RESP3. - -### JSON command replies comparison - -The types are described using a [“TypeScript-like” syntax](https://www.typescriptlang.org/docs/handbook/2/everyday-types.html). `Array` denotes an [array](https://www.typescriptlang.org/docs/handbook/2/everyday-types.html#arrays) where the type of elements is known, but the number of elements is not. - -| Command | RESP2 | RESP3 | -|---------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| All JSON commands | **Default value of optional `path` argument**: `.` | **Default value of optional `path` argument:** `$` | -| JSON.ARRAPPEND
JSON.ARRINDEX
JSON.ARRINSERT
JSON.ARRLEN
JSON.ARRTRIM
JSON.OBJLEN
JSON.STRAPPEND
JSON.STRLEN
JSON.TOGGLE | *`$`-based path argument:*
Reply: Array\

*`.`-based path argument :* 
Reply: BulkString | *`$`-based path argument:* 
Reply: Array\

*`.`-based path argument :*
Reply: number | -| JSON.GET | Reply: JSON encoded string
Example:
```> JSON.SET k $ "[1,2,3]"```
```> JSON.GET k```
```"[1,2,3]"``` | Reply: JSON encoded string with a top-level array
Example:
```> JSON.SET k $ "[1,2,3]"```
```> JSON.GET k```
```"[[1,2,3]]"``` | -| JSON.NUMINCRBY
JSON.NUMMULTBY | *`$`-based path argument:*
Reply: JSON-encoded BulkString | null

*`.`-based path argument :* 
Reply: BulkString | null | error | *`$`-based path argument:*
Reply: Array\ | error

*`.`-based path argument :* 
Reply: number | null | error | -| JSON.OBJKEYS | *`$`-based path argument:*
Reply: Array\>

*`.`-based path argument :* 
Reply: Array\ | *`$`-based path argument:*
Reply: Array\>

*`.`-based path argument :* 
Reply: Array\ | -| JSON.TYPE | *`$`-based path argument:*
Reply: Array\
Example:
```> JSON.TYPE k $```
```1) "array"```

*`.`-based path argument :* 
Reply: BulkString | *`$`-based path argument:*
Reply: Array\>
Example:
```> JSON.TYPE k $```
```1) 1) "array"```

*`.`-based path argument :* 
Reply: Array\ | diff --git a/docs/docs/use_cases.md b/docs/docs/use_cases.md deleted file mode 100644 index d935a1df0..000000000 --- a/docs/docs/use_cases.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -title: "Use cases" -linkTitle: "Use cases" -weight: 4 -description: > - JSON use cases ---- - -You can of course use Redis native data structures to store JSON objects, and that's a common practice. For example, you can serialize JSON and save it in a Redis String. - -However, Redis JSON provides several benefits over this approach. - -**Access and retrieval of subvalues** - -With JSON, you can get nested values without having to transmit the entire object over the network. Being able to access sub-objects can lead to greater efficiencies when you're storing large JSON objects in Redis. - -**Atomic partial updates** - -JSON allows you to atomically run operations like incrementing a value, adding, or removing elements from an array, append strings, and so on. To do the same with a serialized object, you have to retrieve and then reserialize the entire object, which can be expensive and also lack atomicity. - -**Indexing and querying** - -When you store JSON objects as Redis strings, there's no good way to query those objects. On the other hand, storing these objects as JSON using Redis Stack lets you index and query them. This is provided by the search and query capabilities of Redis Stack. \ No newline at end of file From e75f9af22476612f9afcde66351e40fb3983657d Mon Sep 17 00:00:00 2001 From: efeldblum <109016390+ephraimfeldblum@users.noreply.github.com> Date: Tue, 25 Mar 2025 16:06:45 +0200 Subject: [PATCH 109/112] MOD-9169 Do not double count used capacity of containers in memory stats (#1329) * do not double count used capacity of containers in memory stats * unit test * flow tests * separate root from recursive calls * rustic --- redis_json/src/ivalue_manager.rs | 88 +++++++++++++++++--------------- tests/pytest/test_multi.py | 2 +- tests/pytest/test_resp3.py | 2 +- 3 files changed, 48 insertions(+), 44 deletions(-) diff --git a/redis_json/src/ivalue_manager.rs b/redis_json/src/ivalue_manager.rs index 7ba5b4eca..bb9b81da7 100644 --- a/redis_json/src/ivalue_manager.rs +++ b/redis_json/src/ivalue_manager.rs @@ -370,6 +370,50 @@ pub struct RedisIValueJsonKeyManager<'a> { pub phantom: PhantomData<&'a u64>, } +impl RedisIValueJsonKeyManager<'_> { + /// + /// following https://github.com/Diggsey/ijson/issues/23#issuecomment-1377270111 + /// + fn get_memory_impl(v: &IValue) -> usize { + match v.destructure_ref() { + DestructuredRef::Null | DestructuredRef::Bool(_) => 0, + DestructuredRef::Number(num) => { + const STATIC_LO: i32 = -1 << 7; // INumber::STATIC_LOWER + const STATIC_HI: i32 = 0b11 << 7; // INumber::STATIC_UPPER + const SHORT_LO: i32 = -1 << 23; // INumber::SHORT_LOWER + const SHORT_HI: i32 = 1 << 23; // INumber::SHORT_UPPER + + if num.has_decimal_point() { + 16 // 64bit float + } else if &INumber::from(STATIC_LO) <= num && num < &INumber::from(STATIC_HI) { + 0 // 8bit + } else if &INumber::from(SHORT_LO) <= num && num < &INumber::from(SHORT_HI) { + 4 // 24bit + } else { + 16 // 64bit + } + } + DestructuredRef::String(s) => s.len(), + DestructuredRef::Array(arr) => match arr.capacity() { + 0 => 0, + capacity => { + arr.into_iter().map(Self::get_memory_impl).sum::() + + (capacity + 2) * size_of::() + } + }, + DestructuredRef::Object(obj) => match obj.capacity() { + 0 => 0, + capacity => { + obj.into_iter() + .map(|(s, val)| s.len() + Self::get_memory_impl(val)) + .sum::() + + (capacity * 3 + 2) * size_of::() + } + }, + } + } +} + impl<'a> Manager for RedisIValueJsonKeyManager<'a> { type WriteHolder = IValueKeyHolderWrite<'a>; type ReadHolder = IValueKeyHolderRead; @@ -448,48 +492,8 @@ impl<'a> Manager for RedisIValueJsonKeyManager<'a> { } } - /// - /// following https://github.com/Diggsey/ijson/issues/23#issuecomment-1377270111 - /// fn get_memory(v: &Self::V) -> Result { - Ok(match v.destructure_ref() { - DestructuredRef::Null | DestructuredRef::Bool(_) => 0, - DestructuredRef::Number(num) => { - const STATIC_LO: i32 = -1 << 7; // INumber::STATIC_LOWER - const STATIC_HI: i32 = 0b11 << 7; // INumber::STATIC_UPPER - const SHORT_LO: i32 = -1 << 23; // INumber::SHORT_LOWER - const SHORT_HI: i32 = 1 << 23; // INumber::SHORT_UPPER - - if num.has_decimal_point() { - 16 // 64bit float - } else if &INumber::from(STATIC_LO) <= num && num < &INumber::from(STATIC_HI) { - 0 // 8bit - } else if &INumber::from(SHORT_LO) <= num && num < &INumber::from(SHORT_HI) { - 4 // 24bit - } else { - 16 // 64bit - } - } - DestructuredRef::String(s) => s.len(), - DestructuredRef::Array(arr) => match arr.capacity() { - 0 => 0, - capacity => { - arr.into_iter() // IValueManager::get_memory() always returns OK, safe to unwrap here - .map(|val| Self::get_memory(val).unwrap()) - .sum::() - + (capacity + 2) * size_of::() - } - }, - DestructuredRef::Object(obj) => match obj.capacity() { - 0 => 0, - capacity => { - obj.into_iter() // IValueManager::get_memory() always returns OK, safe to unwrap here - .map(|(s, val)| s.len() + Self::get_memory(val).unwrap()) - .sum::() - + (capacity * 3 + 2) * size_of::() - } - }, - } + size_of::()) + Ok(Self::get_memory_impl(v) + size_of::()) } fn is_json(&self, key: *mut RedisModuleKey) -> Result { @@ -528,7 +532,7 @@ mod tests { }"#; let value = serde_json::from_str(json).unwrap(); let res = RedisIValueJsonKeyManager::get_memory(&value).unwrap(); - assert_eq!(res, 903); + assert_eq!(res, 759); } /// Tests the deserialiser of IValue for a string with unicode diff --git a/tests/pytest/test_multi.py b/tests/pytest/test_multi.py index fd1a0b5b1..476c6ac56 100644 --- a/tests/pytest/test_multi.py +++ b/tests/pytest/test_multi.py @@ -855,7 +855,7 @@ def testDebugCommand(env): # Test missing path (defaults to root) res = r.execute_command('JSON.DEBUG', 'MEMORY', 'doc1') - r.assertEqual(res, 1187) + r.assertEqual(res, 1075) # Test missing subcommand r.expect('JSON.DEBUG', 'non_existing_doc', '$..a').raiseError() diff --git a/tests/pytest/test_resp3.py b/tests/pytest/test_resp3.py index a27bf3cf2..1843432fd 100644 --- a/tests/pytest/test_resp3.py +++ b/tests/pytest/test_resp3.py @@ -325,7 +325,7 @@ def test_resp_default_path(self): r.assertEqual(r.execute_command('JSON.OBJKEYS', 'test_resp3'), ['a']) r.assertEqual(r.execute_command('JSON.OBJLEN', 'test_resp3'), 1) r.assertEqual(r.execute_command('JSON.TYPE', 'test_resp3'), ['object']) - r.assertEqual(r.execute_command('JSON.DEBUG', 'MEMORY', 'test_resp3'), 507) + r.assertEqual(r.execute_command('JSON.DEBUG', 'MEMORY', 'test_resp3'), 443) r.assertEqual(r.execute_command('JSON.DEL', 'test_resp3'), 1) # Test JSON.strX commands on object type when default path is used From 03d47a7b6033f4cd9e0df8aa011f5a00dd3f8935 Mon Sep 17 00:00:00 2001 From: Eran Hadad Date: Sun, 6 Apr 2025 22:43:38 +0300 Subject: [PATCH 110/112] bump ramp-packer version to 2.5.13 (#1334) --- .install/build_package_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.install/build_package_requirements.txt b/.install/build_package_requirements.txt index a698dc981..bd647bb41 100644 --- a/.install/build_package_requirements.txt +++ b/.install/build_package_requirements.txt @@ -1,4 +1,4 @@ addict toml jinja2 -ramp-packer==2.5.12 +ramp-packer==2.5.13 From 47b4430d13f7258e607ecf4e3eb295e85317214f Mon Sep 17 00:00:00 2001 From: Gal Cohen <38293267+galcohen-redislabs@users.noreply.github.com> Date: Thu, 10 Apr 2025 13:31:53 +0300 Subject: [PATCH 111/112] MOD-9172 Add Azure Linux 3 support (#1335) Also disabled the freebsd github workflow (the code doesn't even compile there) --- .github/workflows/event-ci.yml | 2 +- .github/workflows/event-nightly.yml | 2 +- .github/workflows/event-tag.yml | 2 +- .github/workflows/event-weekly.yml | 2 +- .github/workflows/flow-linux-x86.yml | 5 ++++- .github/workflows/freebsd.yml | 6 +----- .install/microsoft_azure_linux_3.0.sh | 29 +++++++++++++++++++++++++++ 7 files changed, 38 insertions(+), 10 deletions(-) create mode 100644 .install/microsoft_azure_linux_3.0.sh diff --git a/.github/workflows/event-ci.yml b/.github/workflows/event-ci.yml index a47cc12f7..cc54b0f50 100644 --- a/.github/workflows/event-ci.yml +++ b/.github/workflows/event-ci.yml @@ -27,7 +27,7 @@ jobs: uses: ./.github/workflows/flow-linux-x86.yml needs: [prepare-values] with: - os: jammy rocky9 amazonlinux2 + os: jammy rocky9 amazonlinux2 azurelinux3 redis-ref: ${{needs.prepare-values.outputs.redis-ref}} secrets: inherit linux-valgrind: diff --git a/.github/workflows/event-nightly.yml b/.github/workflows/event-nightly.yml index bb2101cad..8b48f65de 100644 --- a/.github/workflows/event-nightly.yml +++ b/.github/workflows/event-nightly.yml @@ -33,7 +33,7 @@ jobs: uses: ./.github/workflows/flow-linux-x86.yml needs: [prepare-values] with: - os: bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2 + os: bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2 azurelinux3 redis-ref: ${{needs.prepare-values.outputs.redis-ref}} secrets: inherit ubuntu-arm64: diff --git a/.github/workflows/event-tag.yml b/.github/workflows/event-tag.yml index 68dd5ff46..fad9ca757 100644 --- a/.github/workflows/event-tag.yml +++ b/.github/workflows/event-tag.yml @@ -29,7 +29,7 @@ jobs: uses: ./.github/workflows/flow-linux-x86.yml needs: [prepare-values] with: - os: bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2 + os: bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2 azurelinux3 redis-ref: ${{needs.prepare-values.outputs.redis-ref}} secrets: inherit ubuntu-arm64: diff --git a/.github/workflows/event-weekly.yml b/.github/workflows/event-weekly.yml index 8845fb00f..5fce226b3 100644 --- a/.github/workflows/event-weekly.yml +++ b/.github/workflows/event-weekly.yml @@ -29,7 +29,7 @@ jobs: needs: [prepare-values] with: # os: jammy rocky9 amazonlinux2 - os: bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2 + os: bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2 azurelinux3 redis-ref: ${{needs.prepare-values.outputs.redis-ref}} secrets: inherit ubuntu-arm64: diff --git a/.github/workflows/flow-linux-x86.yml b/.github/workflows/flow-linux-x86.yml index 52a85f849..9ebffc28a 100644 --- a/.github/workflows/flow-linux-x86.yml +++ b/.github/workflows/flow-linux-x86.yml @@ -60,7 +60,7 @@ jobs: run: | OS="${{ inputs.os }}" if [ -z "${OS}" ]; then - OS="bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2" + OS="bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2 azurelinux3" fi MATRIX="[" for os in $OS; do @@ -89,6 +89,9 @@ jobs: mariner2) MATRIX="${MATRIX}{\"image\": \"mcr.microsoft.com/cbl-mariner/base/core:2.0\", \"pre_req_install_cmd\": \"tdnf install --noplugins --skipsignature -y ca-certificates git\"}," ;; + azurelinux3) + MATRIX="${MATRIX}{\"image\": \"mcr.microsoft.com/azurelinux/base/core:3.0\", \"pre_req_install_cmd\": \"tdnf install --noplugins --skipsignature -y ca-certificates git\"}," + ;; *) echo "Unknown OS: $os" exit 1 diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml index fab2384be..276946d95 100644 --- a/.github/workflows/freebsd.yml +++ b/.github/workflows/freebsd.yml @@ -1,10 +1,6 @@ name: freebsd -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] +on: [] jobs: build: diff --git a/.install/microsoft_azure_linux_3.0.sh b/.install/microsoft_azure_linux_3.0.sh new file mode 100644 index 000000000..1a3279851 --- /dev/null +++ b/.install/microsoft_azure_linux_3.0.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# This script automates the process of setting up a development environment for RedisJSON on a Microsoft Azure Linux virtual machine. + +set -e + +# Update and install dev tools needed for building and testing +tdnf -y update && \ +tdnf install -y \ + git \ + wget \ + gcc \ + clang-devel \ + llvm-devel \ + make \ + cmake \ + libffi-devel \ + openssl-devel \ + build-essential \ + zlib-devel \ + bzip2-devel \ + python3-devel \ + which \ + unzip \ + ca-certificates \ + python3-pip + +# Install AWS CLI for uploading to S3 +pip3 install awscli --upgrade + From f3d6cd9a2e6809aedf92c3a25948729500f69b6c Mon Sep 17 00:00:00 2001 From: efeldblum <109016390+ephraimfeldblum@users.noreply.github.com> Date: Thu, 10 Apr 2025 23:55:00 +0300 Subject: [PATCH 112/112] MOD-9117 DEL key upon JSON.DEL emptying root (#1336) * DEL key upon JSON.DEL emptying root * fix tests --- json_path/src/json_node.rs | 43 +++++++-------- redis_json/src/commands.rs | 104 +++++++++++++++++------------------- redis_json/src/redisjson.rs | 25 +++++++++ tests/pytest/test.py | 9 ++-- 4 files changed, 95 insertions(+), 86 deletions(-) diff --git a/json_path/src/json_node.rs b/json_path/src/json_node.rs index 22193de56..6fd649d07 100644 --- a/json_path/src/json_node.rs +++ b/json_path/src/json_node.rs @@ -6,7 +6,7 @@ /// Use `SelectValue` use crate::select_value::{SelectValue, SelectValueType}; -use ijson::{IValue, ValueType}; +use ijson::{DestructuredRef, IString, IValue, ValueType}; use serde_json::Value; impl SelectValue for Value { @@ -96,27 +96,21 @@ impl SelectValue for Value { fn get_str(&self) -> String { match self { Self::String(s) => s.to_string(), - _ => { - panic!("not a string"); - } + _ => panic!("not a string"), } } fn as_str(&self) -> &str { match self { Self::String(s) => s.as_str(), - _ => { - panic!("not a string"); - } + _ => panic!("not a string"), } } fn get_bool(&self) -> bool { match self { Self::Bool(b) => *b, - _ => { - panic!("not a bool"); - } + _ => panic!("not a bool"), } } @@ -160,32 +154,33 @@ impl SelectValue for IValue { } fn values<'a>(&'a self) -> Option + 'a>> { - if let Some(arr) = self.as_array() { - Some(Box::new(arr.iter())) - } else if let Some(o) = self.as_object() { - Some(Box::new(o.values())) - } else { - None + match self.destructure_ref() { + DestructuredRef::Array(arr) => Some(Box::new(arr.iter())), + DestructuredRef::Object(o) => Some(Box::new(o.values())), + _ => None, } } fn keys<'a>(&'a self) -> Option + 'a>> { - self.as_object() - .map_or(None, |o| Some(Box::new(o.keys().map(|k| &k[..])))) + match self.destructure_ref() { + DestructuredRef::Object(o) => Some(Box::new(o.keys().map(IString::as_str))), + _ => None, + } } fn items<'a>(&'a self) -> Option + 'a>> { - match self.as_object() { - Some(o) => Some(Box::new(o.iter().map(|(k, v)| (&k[..], v)))), + match self.destructure_ref() { + DestructuredRef::Object(o) => Some(Box::new(o.iter().map(|(k, v)| (k.as_str(), v)))), _ => None, } } fn len(&self) -> Option { - self.as_array().map_or_else( - || self.as_object().map(ijson::IObject::len), - |arr| Some(arr.len()), - ) + match self.destructure_ref() { + DestructuredRef::Array(arr) => Some(arr.len()), + DestructuredRef::Object(o) => Some(o.len()), + _ => None, + } } fn is_empty(&self) -> Option { diff --git a/redis_json/src/commands.rs b/redis_json/src/commands.rs index 05080fca2..eeccb368b 100644 --- a/redis_json/src/commands.rs +++ b/redis_json/src/commands.rs @@ -12,7 +12,7 @@ use crate::manager::{ err_msg_json_path_doesnt_exist_with_param, err_msg_json_path_doesnt_exist_with_param_or, Manager, ReadHolder, UpdateInfo, WriteHolder, }; -use crate::redisjson::{Format, Path, ReplyFormat, SetOptions}; +use crate::redisjson::{Format, Path, ReplyFormat, SetOptions, JSON_ROOT_PATH}; use json_path::select_value::{SelectValue, SelectValueType}; use redis_module::{Context, RedisValue}; use redis_module::{NextArg, RedisError, RedisResult, RedisString, REDIS_OK}; @@ -27,8 +27,6 @@ use itertools::FoldWhile::{Continue, Done}; use itertools::{EitherOrBoth, Itertools}; use serde::{Serialize, Serializer}; -const JSON_ROOT_PATH_LEGACY: &str = "."; -const JSON_ROOT_PATH: &str = "$"; const CMD_ARG_NOESCAPE: &str = "NOESCAPE"; const CMD_ARG_INDENT: &str = "INDENT"; const CMD_ARG_NEWLINE: &str = "NEWLINE"; @@ -85,11 +83,6 @@ fn is_resp3(ctx: &Context) -> bool { .contains(redis_module::ContextFlags::FLAGS_RESP3) } -/// Returns the deault path for the given RESP version -const fn default_path() -> &'static str { - JSON_ROOT_PATH_LEGACY -} - /// /// JSON.GET /// [INDENT indentation-string] @@ -103,7 +96,7 @@ pub fn json_get(manager: M, ctx: &Context, args: Vec) - let key = args.next_arg()?; // Set Capacity to 1 assuming the common case has one path - let mut paths: Vec = Vec::with_capacity(1); + let mut paths = Vec::with_capacity(1); let mut format_options = ReplyFormatOptions::new(is_resp3(ctx), ReplyFormat::STRING); @@ -142,7 +135,7 @@ pub fn json_get(manager: M, ctx: &Context, args: Vec) - // path is optional -> no path found we use legacy root "." if paths.is_empty() { - paths.push(Path::new(default_path())); + paths.push(Path::default()); } let key = manager.open_key_read(ctx, &key)?; @@ -189,9 +182,9 @@ pub fn json_set(manager: M, ctx: &Context, args: Vec) - match (current, set_option) { (Some(doc), op) => { - if path.get_path() == JSON_ROOT_PATH { + if path == JSON_ROOT_PATH { if op != SetOptions::NotExists { - redis_key.set_value(Vec::new(), val)?; + redis_key.set_value(vec![], val)?; redis_key.notify_keyspace_event(ctx, "json.set")?; manager.apply_changes(ctx); REDIS_OK @@ -211,7 +204,7 @@ pub fn json_set(manager: M, ctx: &Context, args: Vec) - } (None, SetOptions::AlreadyExists) => Ok(RedisValue::Null), _ => { - if path.get_path() == JSON_ROOT_PATH { + if path == JSON_ROOT_PATH { redis_key.set_value(Vec::new(), val)?; redis_key.notify_keyspace_event(ctx, "json.set")?; manager.apply_changes(ctx); @@ -253,7 +246,7 @@ pub fn json_merge(manager: M, ctx: &Context, args: Vec) match current { Some(doc) => { - if path.get_path() == JSON_ROOT_PATH { + if path == JSON_ROOT_PATH { redis_key.merge_value(Vec::new(), val)?; redis_key.notify_keyspace_event(ctx, "json.merge")?; manager.apply_changes(ctx); @@ -293,7 +286,7 @@ pub fn json_merge(manager: M, ctx: &Context, args: Vec) } } None => { - if path.get_path() == JSON_ROOT_PATH { + if path == JSON_ROOT_PATH { // Nothing to merge with it's a new doc redis_key.set_value(Vec::new(), val)?; redis_key.notify_keyspace_event(ctx, "json.merge")?; @@ -328,7 +321,7 @@ pub fn json_mset(manager: M, ctx: &Context, args: Vec) // Verify the path is valid and get all the update info let path = Path::new(args.next_str()?); - let update_info = if path.get_path() == JSON_ROOT_PATH { + let update_info = if path == JSON_ROOT_PATH { None } else if let Some(value) = key_value { Some(KeyValue::new(value).find_paths(path.get_path(), SetOptions::None)?) @@ -547,36 +540,38 @@ pub fn json_del(manager: M, ctx: &Context, args: Vec) - let key = args.next_arg()?; let path = match args.next() { - None => Path::new(default_path()), + None => Path::default(), Some(s) => Path::new(s.try_as_str()?), }; let mut redis_key = manager.open_key_write(ctx, key)?; - let deleted = match redis_key.get_value()? { - Some(doc) => { - let res = if path.get_path() == JSON_ROOT_PATH { - redis_key.delete()?; - 1 - } else { - let mut paths = find_paths(path.get_path(), doc, |_| true)?; - prepare_paths_for_updating(&mut paths); - let mut changed = 0; - for p in paths { - if redis_key.delete_path(p)? { - changed += 1; - } - } - changed - }; - if res > 0 { - redis_key.notify_keyspace_event(ctx, "json.del")?; - manager.apply_changes(ctx); - } - res + let deleted = if let Some(doc) = redis_key.get_value()? { + if path != JSON_ROOT_PATH { + let mut paths = find_paths(path.get_path(), doc, |_| true)?; + prepare_paths_for_updating(&mut paths); + paths + .into_iter() + .try_fold(0, |acc, p| redis_key.delete_path(p).map(|v| acc + v as i64))? + } else { + 1 } - None => 0, + } else { + 0 }; - Ok((deleted as i64).into()) + + if deleted > 0 { + let is_empty = redis_key + .get_value()? + .and_then(|v| v.is_empty()) + .unwrap_or(false); + if is_empty || path == JSON_ROOT_PATH { + redis_key.delete()?; + } + redis_key.notify_keyspace_event(ctx, "json.del")?; + manager.apply_changes(ctx); + } + + Ok(deleted.into()) } /// @@ -628,7 +623,7 @@ pub fn json_mget(manager: M, ctx: &Context, args: Vec) pub fn json_type(manager: M, ctx: &Context, args: Vec) -> RedisResult { let mut args = args.into_iter().skip(1); let key = args.next_arg()?; - let path = Path::new(args.next_str().unwrap_or(default_path())); + let path = args.next_str().map(Path::new).unwrap_or_default(); let key = manager.open_key_read(ctx, &key)?; @@ -964,7 +959,7 @@ pub fn json_str_append( path = Path::new(path_or_json); json = val.try_as_str()?; } else { - path = Path::new(default_path()); + path = Path::default(); json = path_or_json; } @@ -1047,7 +1042,7 @@ where pub fn json_str_len(manager: M, ctx: &Context, args: Vec) -> RedisResult { let mut args = args.into_iter().skip(1); let key = args.next_arg()?; - let path = Path::new(args.next_str().unwrap_or(default_path())); + let path = args.next_str().map(Path::new).unwrap_or_default(); let key = manager.open_key_read(ctx, &key)?; @@ -1353,7 +1348,7 @@ where pub fn json_arr_len(manager: M, ctx: &Context, args: Vec) -> RedisResult { let mut args = args.into_iter().skip(1); let key = args.next_arg()?; - let path = Path::new(args.next_str().unwrap_or(default_path())); + let path = args.next_str().map(Path::new).unwrap_or_default(); let is_legacy = path.is_legacy(); let key = manager.open_key_read(ctx, &key)?; let root = match key.get_value()? { @@ -1438,7 +1433,7 @@ pub fn json_arr_pop(manager: M, ctx: &Context, args: Vec (Path::new(default_path()), i64::MAX), + None => (Path::default(), i64::MAX), Some(s) => { let path = Path::new(s.try_as_str()?); let index = args.next_i64().unwrap_or(-1); @@ -1631,7 +1626,7 @@ where pub fn json_obj_keys(manager: M, ctx: &Context, args: Vec) -> RedisResult { let mut args = args.into_iter().skip(1); let key = args.next_arg()?; - let path = Path::new(args.next_str().unwrap_or(default_path())); + let path = args.next_str().map(Path::new).unwrap_or_default(); let mut key = manager.open_key_read(ctx, &key)?; if path.is_legacy() { @@ -1688,7 +1683,7 @@ pub fn json_obj_len(manager: M, ctx: &Context, args: Vec(manager: M, ctx: &Context, args: Vec) )?; let paths = if paths.is_empty() { - vec![Path::new(default_path())] + vec![Path::default()] } else { paths }; @@ -1770,12 +1765,9 @@ pub fn json_clear(manager: M, ctx: &Context, args: Vec) SelectValueType::Double => v.get_double() != 0.0, _ => false, })?; - let mut cleared = 0; - if !paths.is_empty() { - for p in paths { - cleared += redis_key.clear(p)?; - } - } + let cleared = paths + .into_iter() + .try_fold(0, |acc, p| redis_key.clear(p).map(|v| acc + v))?; if cleared > 0 { redis_key.notify_keyspace_event(ctx, "json.clear")?; manager.apply_changes(ctx); @@ -1795,7 +1787,7 @@ pub fn json_debug(manager: M, ctx: &Context, args: Vec) match args.next_str()?.to_uppercase().as_str() { "MEMORY" => { let key = args.next_arg()?; - let path = Path::new(args.next_str().unwrap_or(default_path())); + let path = args.next_str().map(Path::new).unwrap_or_default(); let key = manager.open_key_read(ctx, &key)?; if path.is_legacy() { @@ -1838,7 +1830,7 @@ pub fn json_resp(manager: M, ctx: &Context, args: Vec) let key = args.next_arg()?; let path = match args.next() { - None => Path::new(default_path()), + None => Path::default(), Some(s) => Path::new(s.try_as_str()?), }; diff --git a/redis_json/src/redisjson.rs b/redis_json/src/redisjson.rs index 7fcff2a63..357c80e54 100644 --- a/redis_json/src/redisjson.rs +++ b/redis_json/src/redisjson.rs @@ -146,6 +146,31 @@ impl<'a> Path<'a> { } } +const JSON_ROOT_PATH_LEGACY: &str = "."; +const JSON_ROOT_PATH_S: &str = "$"; +pub const JSON_ROOT_PATH: Path = Path { + original_path: JSON_ROOT_PATH_S, + fixed_path: None, +}; + +/// Returns the deault path for the given RESP version +impl Default for Path<'_> { + fn default() -> Self { + Path { + original_path: JSON_ROOT_PATH_LEGACY, + fixed_path: Some(JSON_ROOT_PATH_S.into()), + } + } +} + +impl PartialEq for Path<'_> { + fn eq(&self, other: &Self) -> bool { + self.get_path() == other.get_path() + } +} + +impl Eq for Path<'_> {} + impl Display for Path<'_> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}", self.get_path()) diff --git a/tests/pytest/test.py b/tests/pytest/test.py index 515fcd06a..2b5bfcca7 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -445,9 +445,7 @@ def testDelCommand(env): r.assertEqual(r.execute_command('JSON.OBJLEN', 'test', '.'), 1) r.assertIsNone(r.execute_command('JSON.TYPE', 'test', '.baz')) r.assertEqual(r.execute_command('JSON.DEL', 'test', '.foo'), 1) - r.assertEqual(r.execute_command('JSON.OBJLEN', 'test', '.'), 0) - r.assertIsNone(r.execute_command('JSON.TYPE', 'test', '.foo')) - r.assertEqual(r.execute_command('JSON.TYPE', 'test', '.'), 'object') + r.assertIsNone(r.execute_command('JSON.GET', 'test')) # Test deleting some keys from an object r.assertOk(r.execute_command('JSON.SET', 'test', '.', '{}')) @@ -457,11 +455,10 @@ def testDelCommand(env): r.assertEqual(r.execute_command('JSON.OBJLEN', 'test', '.'), 1) r.assertIsNone(r.execute_command('JSON.TYPE', 'test', '.baz')) r.assertEqual(r.execute_command('JSON.DEL', 'test', '.foo'), 1) - r.assertEqual(r.execute_command('JSON.OBJLEN', 'test', '.'), 0) - r.assertIsNone(r.execute_command('JSON.TYPE', 'test', '.foo')) - r.assertEqual(r.execute_command('JSON.TYPE', 'test', '.'), 'object') + r.assertIsNone(r.execute_command('JSON.GET', 'test')) # Test with an array + r.assertOk(r.execute_command('JSON.SET', 'test', '.', '{}')) r.assertOk(r.execute_command('JSON.SET', 'test', '.foo', '"bar"')) r.assertOk(r.execute_command('JSON.SET', 'test', '.baz', '"qux"')) r.assertOk(r.execute_command('JSON.SET', 'test', '.arr', '[1.2,1,2]'))