MOD-4432: Add to JSONPath filter the regexp match operator (RedisJSON#848)

oshadmi · web-flow · commit c4483f915e88 · 2022-11-08T18:44:32.000+02:00
* Add to JSONPath filter the regexp match operator

* Improve coverage

* Minor cosmetics

* Allow match using regex pattern from a field and add end-to-end test

* Test with numeric combined in filter

* Add more tests and documentation
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -28,6 +28,7 @@ serde = "1.0"
 libc = "0.2"
 redis-module = { version="1.0", features = ["experimental-api"]}
 itertools = "0.10"
+regex = "1"
 [features]
 # Workaround to allow cfg(feature = "test") in redismodue-rs dependencies:
 # https://github.com/RedisLabsModules/redismodule-rs/pull/68
diff --git a/docs/docs/path.md b/docs/docs/path.md
@@ -13,7 +13,7 @@ RedisJSON knows which syntax to use depending on the first character of the path
 
 ## JSONPath support
 
-RedisJSON v2.0 introduces [JSONPath](http://goessner.net/articles/JsonPath/) support. It follows the syntax described by Goessner in his article.
+RedisJSON v2.0 introduces [JSONPath](http://goessner.net/articles/JsonPath/) support. It follows the syntax described by Goessner in his [article](http://goessner.net/articles/JsonPath/).
 
 A JSONPath query can resolve to several locations in a JSON document. In this case, the JSON commands apply the operation to every possible location. This is a major improvement over [legacy path](#legacy-path-syntax) queries, which only operate on the first path.
 
@@ -40,7 +40,7 @@ The following JSONPath syntax table was adapted from Goessner's [path syntax com
 | [] | Subscript operator, accesses an array element. |
 | [,] | Union, selects multiple elements. |
 | [start\:end\:step] | Array slice where start, end, and step are indexes. |
-| ?() | Filters a JSON object or array. Supports comparison operators <nobr>(==, !=, <, <=, >, >=)</nobr> and logical operators <nobr>(&&, \|\|)</nobr>. |
+| ?() | Filters a JSON object or array. Supports comparison operators <nobr>(`==`, `!=`, `<`, `<=`, `>`, `>=`, `=~`)</nobr>, logical operators <nobr>(`&&`, `\|\|`)</nobr>, and parenthesis <nobr>(`(`, `)`)</nobr>. |
 | () | Script expression. |
 | @ | The current element, used in filter or script expressions. |
 
@@ -154,7 +154,7 @@ You can use an array slice to select a range of elements from an array. This exa
 "[\"Noise-cancelling Bluetooth headphones\",\"Wireless earbuds\"]"
 ```
 
-Filter expressions `?()` let you select JSON elements based on certain conditions. You can use comparison operators (==, !=, <, <=, >, >=) and logical operators (&&, \|\|) within these expressions.
+Filter expressions `?()` let you select JSON elements based on certain conditions. You can use comparison operators (`==`, `!=`, `<`, `<=`, `>`, `>=`, `=~`), logical operators (`&&`, `||`), and parenthesis (`(`, `)`) within these expressions. A filter expression can be applied on an array or on an object, iterating all the elements in the array or all the key-value pair in the object, retrieving only the ones that match the filter condition. The filter condition is using `@` to denote the current array element or the current object. Use `@.key_name` to refer to a specific value. Use `$` to denote the top-level, e.g., `$.top_level_key_name`
 
 For example, this filter only returns wireless headphones with a price less than 70:
 
@@ -170,6 +170,32 @@ This example filters the inventory for the names of items that support Bluetooth
 "[\"Noise-cancelling Bluetooth headphones\",\"Wireless earbuds\",\"Wireless keyboard\"]"
 ```
 
+The comparison operator `=~` is matching a string value of the left-hand side against a regular expression pattern on the right-hand side. The supported regular expression syntax is detailed [here](https://docs.rs/regex/latest/regex/#syntax).
+
+For example, this filters only keyboards with some sort of USB connection (notice this match is case-insensitive thanks to the prefix `(?i)` in the regular expression pattern `"(?i)usb"` ):
+
+```sh
+127.0.0.1:6379> JSON.GET store '$.inventory.keyboards[?(@.connection =~ "(?i)usb")]'
+"[{\"id\":22346,\"name\":\"USB-C keyboard\",\"description\":\"Wired USB-C keyboard\",\"wireless\":false,\"connection\":\"USB-C\",\"price\":29.99,\"stock\":30,\"free-shipping\":false}]"
+```
+The regular expression pattern can also be taken from a JSON string key on the right-hand side.
+
+For example, let's add each keybaord object with a `regex_pat` key:
+
+```sh
+127.0.0.1:6379> JSON.SET store '$.inventory.keyboards[0].regex_pat' '"(?i)bluetooth"'
+OK
+127.0.0.1:6379> JSON.SET store '$.inventory.keyboards[1].regex' '"usb"'
+OK
+```
+
+Now we can match against this `regex_pat` key instead of a hard-coded regular expression pattern, and get the keyboard with the `Bluetooth` string in its `connection` key (notice the one with `USB-C` did not match since its regular expression pattern is case-sensitive and the regular expression pattern is using lower case):
+
+```sh
+127.0.0.1:6379> JSON.GET store '$.inventory.keyboards[?(@.connection =~ @.regex_pat)]'
+"[{\"id\":22345,\"name\":\"Wireless keyboard\",\"description\":\"Wireless Bluetooth keyboard\",\"wireless\":true,\"connection\":\"Bluetooth\",\"price\":44.99,\"stock\":23,\"free-shipping\":false,\"colors\":[\"black\",\"silver\"],\"regex\":\"(?i)Bluetooth\",\"regex_pat\":\"(?i)bluetooth\"}]"
+```
+
 #### Update JSON examples
 
 You can also use JSONPath queries when you want to update specific sections of a JSON document.
diff --git a/src/jsonpath/json_path.rs b/src/jsonpath/json_path.rs
@@ -3,6 +3,7 @@ use pest::Parser;
 use std::cmp::Ordering;
 
 use crate::jsonpath::select_value::{SelectValue, SelectValueType};
+use regex::Regex;
 use std::fmt::Debug;
 
 #[derive(Parser)]
@@ -433,8 +434,32 @@ impl<'i, 'j, S: SelectValue> TermEvaluationResult<'i, 'j, S> {
         !self.eq(s)
     }
 
-    fn re(&self, _s: &Self) -> bool {
-        false
+    fn re_is_match(regex: &str, s: &str) -> bool {
+        Regex::new(regex).map_or_else(|_| false, |re| Regex::is_match(&re, s))
+    }
+
+    fn re_match(&self, s: &Self) -> bool {
+        match (self, s) {
+            (TermEvaluationResult::Value(v), TermEvaluationResult::Str(regex)) => {
+                match v.get_type() {
+                    SelectValueType::String => Self::re_is_match(regex, v.as_str()),
+                    _ => false,
+                }
+            }
+            (TermEvaluationResult::Value(v1), TermEvaluationResult::Value(v2)) => {
+                match (v1.get_type(), v2.get_type()) {
+                    (SelectValueType::String, SelectValueType::String) => {
+                        Self::re_is_match(v2.as_str(), v1.as_str())
+                    }
+                    (_, _) => false,
+                }
+            }
+            (_, _) => false,
+        }
+    }
+
+    fn re(&self, s: &Self) -> bool {
+        self.re_match(s)
     }
 }
 
diff --git a/tests/pytest/test.py b/tests/pytest/test.py
@@ -1173,6 +1173,50 @@ def testEscape(env):
     r.expect('JSON.SET', 'doc', '$', '{"val": "escaped unicode here:\u2B50"}').ok()
     r.expect('JSON.GET', 'doc', '$.val').equal('["escaped unicode here:⭐"]')
 
+def testFilter(env):
+    # Test JSONPath filter
+    r = env
+
+    doc = {
+        "arr": ["kaboom", "kafoosh", "four", "bar", 7.0, "foolish", ["food", "foo", "FoO", "fight"], -9, {"in" : "fooctious"}, "ffool", "(?i)^[f][o][o]$", False, None],
+        "pat_regex": ".*foo",
+        "pat_plain": "(?i)^[f][o][o]$",
+        "pat_bad": "[f.*",
+        "pat_not_str1": 42,
+        "pat_not_str2": None,
+        "pat_not_str3": True,
+        "pat_not_str4": {"p":".*foo"},
+        "pat_not_str5": [".*foo"],
+    }
+    r.expect('JSON.SET', 'doc', '$', json.dumps(doc)).ok()
+    
+    # regex match using a static regex pattern
+    r.expect('JSON.GET', 'doc', '$.arr[?(@ =~ ".*foo")]').equal('["kafoosh","foolish","ffool"]')
+    
+    # regex match using a field
+    r.expect('JSON.GET', 'doc', '$.arr[?(@ =~ $.pat_regex)]').equal('["kafoosh","foolish","ffool"]')
+    
+    # regex case-insensitive match using a field (notice the `.*` before the filter)
+    r.expect('JSON.GET', 'doc', '$.arr.*[?(@ =~ $.pat_plain)]').equal('["foo","FoO"]')
+
+    # regex match using field after being modified
+    r.expect('JSON.SET', 'doc', '$.pat_regex', '"k.*foo"').ok()
+    r.expect('JSON.GET', 'doc', '$.arr[?(@ =~ $.pat_regex)]').equal('["kafoosh"]')
+
+    # regex mismatch (illegal pattern)
+    r.expect('JSON.GET', 'doc', '$.arr[?(@ == $.pat_bad)]').equal('[]')
+    r.expect('JSON.GET', 'doc', '$.arr[?(@ == $.pat_bad || @>4.5)]').equal('[7.0]')
+
+    # regex mismatch (missing pattern)
+    r.expect('JSON.GET', 'doc', '$.arr[?(@ =~ $.pat_missing)]').equal('[]')
+
+    # regex mismatch (not a string pattern)
+    for i in range(1, 6):
+        r.expect('JSON.GET', 'doc', '$.arr[?(@ =~ $.pat_not_str{})]'.format(i)).equal('[]')
+
+    # plain string match
+    r.expect('JSON.GET', 'doc', '$.arr[?(@ == $.pat_plain)]').equal('["(?i)^[f][o][o]$"]')
+    
 
 # class CacheTestCase(BaseReJSONTest):
 #     @property
diff --git a/tests/rust_tests/op.rs b/tests/rust_tests/op.rs
@@ -366,3 +366,89 @@ fn op_for_same_type() {
         ]),
     );
 }
+
+#[test]
+fn op_string_regexp_match() {
+    setup();
+
+    select_and_then_compare(
+        r#"$.tags[?(@ =~ "^[a-z]{4}$")]"#,
+        read_json("./json_examples/data_obj.json"),
+        json!(["aute", "elit", "esse"]),
+    );
+
+    select_and_then_compare(
+        r#"$.tags[?(@ =~ "^[ec].*")]"#,
+        read_json("./json_examples/data_obj.json"),
+        json!(["elit", "esse", "culpa"]),
+    );
+
+    select_and_then_compare(
+        r#"$.arr[?(@ =~ "^[ec.*")]"#, //erroneous regexp
+        json!([{
+            "arr": ["eclectic", 54, "elit", "esse", "culpa"],
+        }]),
+        json!([]),
+    );
+
+    select_and_then_compare(
+        // Flat visit all JSON types
+        r#"$.arr[?(@ =~ "^[Ee]c.*")]"#,
+        json!({
+            "arr": ["eclectic", 54, "elit", 96.33, {"eclipse":"ecstatic"}, "esse", true, "culpa", "echo", ["ecu", "eching"], "Ecuador", null, "etc"],
+        }),
+        json!(["eclectic", "echo", "Ecuador"]),
+    );
+
+    select_and_then_compare(
+        // Recursive visit all JSON types
+        r#"$..[?(@ =~ "^[Ee]c.*")]"#,
+        json!({
+            "arr": ["eclectic", 54, "elit", 96.33, {"eclipse":"ecstatic"}, "esse", true, "culpa", "echo", ["ecu", "eching", "plan"], "Ecuador", null, "etc"],
+        }),
+        json!([
+            "eclectic", "echo", "Ecuador",  // "arr" filtered
+            "eclectic", // "arr" content flattened - 1st level
+            "ecstatic", // "arr" content flattened - "eclipse" filtered
+            "ecstatic", // "arr" content flattened - "eclipse" flattened
+            "echo",     // "arr" content flattened - 1st level
+            "ecu", "eching", // "arr" content flattened - anonymous array filtered
+            "ecu", "eching",  // "arr" content flattened - anonymous array flattened
+            "Ecuador"  // "arr" content flattened - 1st level
+        ]),
+    );
+}
+
+#[test]
+fn op_string_regexp_field_match() {
+    setup();
+
+    select_and_then_compare(
+        r#"$.arr[?(@ =~ $.pat1)]"#, //regex
+        json!({
+            "arr": ["kaboom", "kafoosh", "four", "bar", 7.0, -9, false, null, "foolish", "ffool", "[f][o][o]"],
+            "pat1":"foo",
+            "pat2":"k.*foo"
+        }),
+        json!(["kafoosh", "foolish", "ffool"]),
+    );
+
+    select_and_then_compare(
+        r#"$.arr[?(@ =~ $.pat2)]"#, //regex
+        json!({
+            "arr": ["kaboom", "kafoosh", "four", "bar", 7.0, -9, false, null, "foolish", "ffool", "[f][o][o]"],
+            "pat1":"foo",
+            "pat2":"k.*foo"
+        }),
+        json!(["kafoosh"]),
+    );
+
+    select_and_then_compare(
+        r#"$.arr[?(@ == $.pat1)]"#, //plain string
+        json!({
+            "arr": ["kaboom", "kafoosh", "four", "bar", 7.0, -9, false, null, "foolish", "ffool", "[f][o][o]"],
+            "pat1":"[f][o][o]"
+        }),
+        json!(["[f][o][o]"]),
+    );
+}
