From a2159c801c2706b6dbf01f73be9bc043f3ec9707 Mon Sep 17 00:00:00 2001 From: Robin Jarry Date: Wed, 24 Jan 2024 13:50:41 +0100 Subject: [PATCH 01/71] tox: fix lint with python 3.12 Fix the following error when running pylint with python 3.12: Exception on node ImportFrom in file 'cffi/build.py' Traceback (most recent call last): File ".../python3.12/site-packages/pylint/checkers/imports.py", line 846, in _get_imported_module return importnode.do_import_module(modname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/nodes/_base_nodes.py", line 146, in do_import_module return mymodule.import_module( ^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/nodes/scoped_nodes/scoped_nodes.py", line 530, in import_module return AstroidManager().ast_from_module_name( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/manager.py", line 246, in ast_from_module_name return self.ast_from_file(found_spec.location, modname, fallback=False) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/manager.py", line 138, in ast_from_file return AstroidBuilder(self).file_build(filepath, modname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/builder.py", line 144, in file_build module, builder = self._data_build(data, modname, path) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/builder.py", line 204, in _data_build module = builder.visit_module(node, modname, node_file, package) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/rebuilder.py", line 254, in visit_module [self.visit(child, newnode) for child in node.body], ^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/rebuilder.py", line 609, in visit visit_method = getattr(self, visit_name) ^^^^^^^^^^^^^^^^^^^^^^^^^ AttributeError: 'TreeRebuilder' object has no attribute 'visit_typealias' Update all lint and format dependencies to their latest stable releases. Ignore the new use-implicit-booleaness-not-comparison-to-string and use-implicit-booleaness-not-comparison-to-zero pylint rules. Use the correct Hashable reference from collections.abc instead of the deprecated typing alias. Signed-off-by: Robin Jarry --- .github/workflows/ci.yml | 2 ++ libyang/keyed_list.py | 3 ++- pylintrc | 2 ++ tox.ini | 18 +++++++++++------- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cfbb845..4c308738 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,8 @@ jobs: toxenv: py310 - python: "3.11" toxenv: py311 + - python: "3.12" + toxenv: py312 - python: pypy3.9 toxenv: pypy3 steps: diff --git a/libyang/keyed_list.py b/libyang/keyed_list.py index 02b030fa..1a98af32 100644 --- a/libyang/keyed_list.py +++ b/libyang/keyed_list.py @@ -1,8 +1,9 @@ # Copyright (c) 2020 6WIND S.A. # SPDX-License-Identifier: MIT +from collections.abc import Hashable import copy -from typing import Any, Hashable, Iterable, Optional, Tuple, Union +from typing import Any, Iterable, Optional, Tuple, Union # ------------------------------------------------------------------------------------- diff --git a/pylintrc b/pylintrc index 97a7cec1..16a9f0ae 100644 --- a/pylintrc +++ b/pylintrc @@ -77,6 +77,8 @@ disable= too-many-return-statements, too-many-statements, unused-argument, + use-implicit-booleaness-not-comparison-to-string, + use-implicit-booleaness-not-comparison-to-zero, wrong-import-order, [REPORTS] diff --git a/tox.ini b/tox.ini index f8e3c972..9fd15d15 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = format,lint,py{36,37,38,39,310,311,py3,3},lydevel,coverage +envlist = format,lint,py{36,37,38,39,310,311,312,py3,3},lydevel,coverage skip_missing_interpreters = true isolated_build = true distdir = {toxinidir}/dist @@ -36,8 +36,8 @@ basepython = python3 description = Format python code using isort and black. changedir = . deps = - black~=23.1.0 - isort~=5.12.0 + black~=23.12.1 + isort~=5.13.2 skip_install = true install_command = python3 -m pip install {opts} {packages} allowlist_externals = @@ -52,10 +52,14 @@ basepython = python3 description = Run coding style checks. changedir = . deps = - black~=23.1.0 - flake8~=6.0.0 - isort~=5.12.0 - pylint~=2.16.2 + astroid~=3.0.2 + black~=23.12.1 + flake8~=7.0.0 + isort~=5.13.2 + pycodestyle~=2.11.1 + pyflakes~=3.2.0 + pylint~=3.0.3 + setuptools~=69.0.3 allowlist_externals = /bin/sh /usr/bin/sh From 097412cdb8ff40993755661f9760dadb8a5c9bbc Mon Sep 17 00:00:00 2001 From: Matthieu Ternisien d'Ouville Date: Wed, 17 Jan 2024 08:33:29 +0100 Subject: [PATCH 02/71] schema: enable getting node data path without list key There is no way to get a node data path (path without choice/case) without list keys. Adds the path_type parameter to the SNode.schema_path() method. This parameter can takes 3 values: - SNode.PATH_LOG: returns the path with schema-only nodes (choice, case) included, the default - SNode.PATH_DATA: returns the path without schema-only nodes - SNode.PATH_DATA_PATTERN: similar to PATH_DATA with list keys added (the one used by data_path()) The SNode.PATH_LOG is set by default to not change the original behavior. The SNode.data_path() method now calls SNode.schema_path() with self.PATH_DATA_PATTERN instead of lib.lysc_path(). Here is an example of the output difference between schema_path(), data_path(), and schema_path(path_type=SNode.PATH_DATA) with a node included in a choice and a list: node.schema_path(): /ietf-keystore:keystore/asymmetric-keys/asymmetric-key/private-key-type/private-key/private-key node.data_path() or node.schema_path(SNode.PATH_DATA_PATTERN): /ietf-keystore:keystore/asymmetric-keys/asymmetric-key[name='%s']/private-key node.schema_path(SNode.PATH_DATA): /ietf-keystore:keystore/asymmetric-keys/asymmetric-key/private-key Tests have been updated accordingly. Signed-off-by: Matthieu Ternisien d'Ouville --- libyang/schema.py | 21 +++++++++++---------- tests/test_schema.py | 15 ++++++++++----- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index c6d822c2..6b0c803b 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1034,6 +1034,10 @@ class SNode: ANYDATA: "anydata", } + PATH_LOG = lib.LYSC_PATH_LOG + PATH_DATA = lib.LYSC_PATH_DATA + PATH_DATA_PATTERN = lib.LYSC_PATH_DATA_PATTERN + def __init__(self, context: "libyang.Context", cdata): self.context = context self.cdata = cdata # C type: "struct lysc_node *" @@ -1079,22 +1083,19 @@ def status(self) -> str: def module(self) -> Module: return Module(self.context, self.cdata.module) - def schema_path(self) -> str: + def schema_path(self, path_type: int = PATH_LOG) -> str: try: - s = lib.lysc_path(self.cdata, lib.LYSC_PATH_LOG, ffi.NULL, 0) + s = lib.lysc_path(self.cdata, path_type, ffi.NULL, 0) return c2str(s) finally: lib.free(s) def data_path(self, key_placeholder: str = "'%s'") -> str: - try: - s = lib.lysc_path(self.cdata, lib.LYSC_PATH_DATA_PATTERN, ffi.NULL, 0) - val = c2str(s) - if key_placeholder != "'%s'": - val = val.replace("'%s'", key_placeholder) - return val - finally: - lib.free(s) + val = self.schema_path(self.PATH_DATA_PATTERN) + + if key_placeholder != "'%s'": + val = val.replace("'%s'", key_placeholder) + return val def extensions(self) -> Iterator[ExtensionCompiled]: ext = ffi.cast("struct lysc_ext_instance *", self.cdata.exts) diff --git a/tests/test_schema.py b/tests/test_schema.py index b88f0092..f493ba14 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -282,13 +282,16 @@ def test_iter_tree(self): # ------------------------------------------------------------------------------------- class ListTest(unittest.TestCase): - SCHEMA_PATH = "/yolo-system:conf/url" - DATA_PATH = "/yolo-system:conf/url[host='%s'][proto='%s']" + PATH = { + "LOG": "/yolo-system:conf/url", + "DATA": "/yolo-system:conf/url", + "DATA_PATTERN": "/yolo-system:conf/url[host='%s'][proto='%s']", + } def setUp(self): self.ctx = Context(YANG_DIR) self.ctx.load_module("yolo-system") - self.list = next(self.ctx.find_path(self.SCHEMA_PATH)) + self.list = next(self.ctx.find_path(self.PATH["LOG"])) def tearDown(self): self.list = None @@ -300,9 +303,11 @@ def test_list_attrs(self): self.assertEqual(self.list.nodetype(), SNode.LIST) self.assertEqual(self.list.keyword(), "list") - self.assertEqual(self.list.schema_path(), self.SCHEMA_PATH) + self.assertEqual(self.list.schema_path(), self.PATH["LOG"]) - self.assertEqual(self.list.data_path(), self.DATA_PATH) + self.assertEqual(self.list.schema_path(SNode.PATH_DATA), self.PATH["DATA"]) + + self.assertEqual(self.list.data_path(), self.PATH["DATA_PATTERN"]) self.assertFalse(self.list.ordered()) def test_list_keys(self): From cccb9d6b90d955a8a7c678a878b5eb2b5ff9a0c1 Mon Sep 17 00:00:00 2001 From: Wataru Ishida Date: Sat, 29 Jan 2022 06:57:20 +0000 Subject: [PATCH 03/71] test: add keyless list test Keyless list is allowed for operational data. (see RFC7950 7.8.2.) It is used in ietf-netconf-notifications:netconf-config-change, for example. It is already supported, but untested, add a test case. Signed-off-by: Wataru Ishida Signed-off-by: Samuel Gauthier --- tests/test_data.py | 16 ++++++++++++++++ tests/test_diff.py | 3 +++ tests/test_schema.py | 2 +- tests/yang/yolo/yolo-system.yang | 8 ++++++++ 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/test_data.py b/tests/test_data.py index becb5d0c..32c605fe 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -704,6 +704,22 @@ def test_notification_from_dict_module(self): dnotif.free() self.assertEqual(json.loads(j), json.loads(self.JSON_NOTIF)) + DICT_NOTIF_KEYLESS_LIST = { + "config-change": {"edit": [{"target": "a"}, {"target": "b"}]}, + } + + def test_data_to_dict_keyless_list(self): + module = self.ctx.get_module("yolo-system") + dnotif = module.parse_data_dict( + self.DICT_NOTIF_KEYLESS_LIST, strict=True, notification=True + ) + self.assertIsInstance(dnotif, DNotif) + try: + dic = dnotif.print_dict() + finally: + dnotif.free() + self.assertEqual(dic, self.DICT_NOTIF_KEYLESS_LIST) + XML_DIFF_STATE1 = """ foo 1234 diff --git a/tests/test_diff.py b/tests/test_diff.py index d3edadc9..49bf77a2 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -75,6 +75,9 @@ class DiffTest(unittest.TestCase): (SNodeAdded, "/yolo-system:alarm-triggered"), (SNodeAdded, "/yolo-system:alarm-triggered/severity"), (SNodeAdded, "/yolo-system:alarm-triggered/description"), + (SNodeAdded, "/yolo-system:config-change"), + (SNodeAdded, "/yolo-system:config-change/edit"), + (SNodeAdded, "/yolo-system:config-change/edit/target"), (EnumRemoved, "/yolo-system:conf/url/proto"), (EnumRemoved, "/yolo-system:state/url/proto"), (EnumStatusAdded, "/yolo-system:conf/url/proto"), diff --git a/tests/test_schema.py b/tests/test_schema.py index f493ba14..770a476c 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -56,7 +56,7 @@ def test_mod_filepath(self): def test_mod_iter(self): children = list(iter(self.module)) - self.assertEqual(len(children), 5) + self.assertEqual(len(children), 6) def test_mod_children_rpcs(self): rpcs = list(self.module.children(types=(SNode.RPC,))) diff --git a/tests/yang/yolo/yolo-system.yang b/tests/yang/yolo/yolo-system.yang index 44196b47..a34c421c 100644 --- a/tests/yang/yolo/yolo-system.yang +++ b/tests/yang/yolo/yolo-system.yang @@ -204,4 +204,12 @@ module yolo-system { type uint32; } } + + notification config-change { + list edit { + leaf target { + type string; + } + } + } } From 3b4d5099a04c16a93037cc49ca66a9f2ec14aa60 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Fri, 26 Jan 2024 00:23:52 +0100 Subject: [PATCH 04/71] schema: fix leaf-list defaults Instanciate a Type module to be able to compare with Type enum. The next commit will add unit tests. Fixes: 806be4c3bc4f ("Port to libyang 2") Fixes: #80 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libyang/schema.py b/libyang/schema.py index 6b0c803b..3aef4176 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1279,7 +1279,7 @@ def defaults(self) -> Iterator[str]: if not val: yield None ret = c2str(val) - val_type = self.cdata_leaflist.dflts[i].realtype + val_type = Type(self.context, self.cdata_leaflist.dflts[i].realtype, None) if val_type == Type.BOOL: ret = val == "true" elif val_type in Type.NUM_TYPES: From 3428ea055cd47d7f33e061c0d1ea8116af21b3d1 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Tue, 28 Nov 2023 14:59:21 +0100 Subject: [PATCH 05/71] schema: return float for decimal64 in default function This patch changes the output for leaf/leaf-list nodes using decimal64 type, and adds new unit test for this purpose. The leaf-list defaults function is reworked to be similar to the leaf default function. A new yang schema is added for clarity, and to avoid modifying all the tests using yolo-system.yang. Fixes: #80 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/schema.py | 17 +++++++++------ tests/test_schema.py | 31 ++++++++++++++++++++++++++++ tests/yang/yolo/yolo-nodetypes.yang | 32 +++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 tests/yang/yolo/yolo-nodetypes.yang diff --git a/libyang/schema.py b/libyang/schema.py index 3aef4176..d85912d1 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1209,7 +1209,7 @@ def __init__(self, context: "libyang.Context", cdata): self.cdata_leaf = ffi.cast("struct lysc_node_leaf *", cdata) self.cdata_leaf_parsed = ffi.cast("struct lysp_node_leaf *", self.cdata_parsed) - def default(self) -> Union[None, bool, int, str]: + def default(self) -> Union[None, bool, int, str, float]: if not self.cdata_leaf.dflt: return None val = lib.lyd_value_get_canonical(self.context.cdata, self.cdata_leaf.dflt) @@ -1221,6 +1221,8 @@ def default(self) -> Union[None, bool, int, str]: return val == "true" if val_type.base() in Type.NUM_TYPES: return int(val) + if val_type.base() == Type.DEC64: + return float(val) return val def units(self) -> Optional[str]: @@ -1268,7 +1270,7 @@ def type(self) -> Type: self.context, self.cdata_leaflist.type, self.cdata_leaflist_parsed.type ) - def defaults(self) -> Iterator[str]: + def defaults(self) -> Iterator[Union[None, bool, int, str, float]]: if self.cdata_leaflist.dflts == ffi.NULL: return arr_length = ffi.cast("uint64_t *", self.cdata_leaflist.dflts)[-1] @@ -1278,13 +1280,16 @@ def defaults(self) -> Iterator[str]: ) if not val: yield None - ret = c2str(val) + val = c2str(val) val_type = Type(self.context, self.cdata_leaflist.dflts[i].realtype, None) if val_type == Type.BOOL: - ret = val == "true" + yield val == "true" elif val_type in Type.NUM_TYPES: - ret = int(val) - yield ret + yield int(val) + elif val_type.base() == Type.DEC64: + yield float(val) + else: + yield val def must_conditions(self) -> Iterator[str]: pdata = self.cdata_leaflist_parsed diff --git a/tests/test_schema.py b/tests/test_schema.py index 770a476c..434618bf 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -474,3 +474,34 @@ def test_leaf_parent(self): def test_iter_tree(self): leaf = next(self.ctx.find_path("/yolo-system:conf")) self.assertEqual(len(list(leaf.iter_tree(full=True))), 23) + + +# ------------------------------------------------------------------------------------- +class LeafTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.ctx.load_module("yolo-nodetypes") + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_leaf_default(self): + leaf = next(self.ctx.find_path("/yolo-nodetypes:conf/percentage")) + self.assertIsInstance(leaf.default(), float) + + +# ------------------------------------------------------------------------------------- +class LeafListTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.ctx.load_module("yolo-nodetypes") + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_leaflist_defaults(self): + leaflist = next(self.ctx.find_path("/yolo-nodetypes:conf/ratios")) + for d in leaflist.defaults(): + self.assertIsInstance(d, float) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang new file mode 100644 index 00000000..3dd76832 --- /dev/null +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -0,0 +1,32 @@ +module yolo-nodetypes { + yang-version 1.1; + namespace "urn:yang:yolo:nodetypes"; + prefix sys; + + description + "YOLO Nodetypes."; + + revision 2024-01-25 { + description + "Initial version."; + } + + container conf { + description + "Configuration."; + leaf percentage { + type decimal64 { + fraction-digits 2; + } + default 10.2; + } + + leaf-list ratios { + type decimal64 { + fraction-digits 2; + } + default 2.5; + default 2.6; + } + } +} From 6502d47d62154da80472ae95f0b36fa77efb13f3 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sat, 11 Nov 2023 23:42:12 +0100 Subject: [PATCH 06/71] context: add root_node to find_path This patch enhances the context find_path function to allow using relative paths as well, by adding an optional root_node attribute, from which the relative path is being evaluated. Fixes: #82 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/context.py | 14 ++++++++++++-- tests/test_context.py | 8 +++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/libyang/context.py b/libyang/context.py index 57432941..0d8d0a67 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -175,17 +175,27 @@ def get_module(self, name: str) -> Module: return Module(self, mod) - def find_path(self, path: str, output: bool = False) -> Iterator[SNode]: + def find_path( + self, + path: str, + output: bool = False, + root_node: Optional["libyang.SNode"] = None, + ) -> Iterator[SNode]: if self.cdata is None: raise RuntimeError("context already destroyed") + if root_node is not None: + ctx_node = root_node.cdata + else: + ctx_node = ffi.NULL + flags = 0 if output: flags |= lib.LYS_FIND_XP_OUTPUT node_set = ffi.new("struct ly_set **") if ( - lib.lys_find_xpath(self.cdata, ffi.NULL, str2c(path), 0, node_set) + lib.lys_find_xpath(self.cdata, ctx_node, str2c(path), 0, node_set) != lib.LY_SUCCESS ): raise self.error("cannot find path") diff --git a/tests/test_context.py b/tests/test_context.py index 6e88a261..5650ecde 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -4,7 +4,7 @@ import os import unittest -from libyang import Context, LibyangError, Module, SRpc +from libyang import Context, LibyangError, Module, SLeaf, SLeafList YANG_DIR = os.path.join(os.path.dirname(__file__), "yang") @@ -82,8 +82,10 @@ def test_ctx_load_invalid_module(self): def test_ctx_find_path(self): with Context(YANG_DIR) as ctx: ctx.load_module("yolo-system") - node = next(ctx.find_path("/yolo-system:format-disk")) - self.assertIsInstance(node, SRpc) + node = next(ctx.find_path("/yolo-system:conf/offline")) + self.assertIsInstance(node, SLeaf) + node2 = next(ctx.find_path("../number", root_node=node)) + self.assertIsInstance(node2, SLeafList) def test_ctx_iter_modules(self): with Context(YANG_DIR) as ctx: From c6caf46e08c084bc558fc271d1a6a93fd8430d21 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sat, 11 Nov 2023 23:48:04 +0100 Subject: [PATCH 07/71] schema: add a with_typedefs option to union_types This patch adds a new with_typedefs option to the union_types function that will return the typedefs instead of the resolved types when available. A unit test is added as well. Fixes: #83 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/schema.py | 13 ++++++++++++- tests/test_schema.py | 2 ++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/libyang/schema.py b/libyang/schema.py index d85912d1..7e72e809 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -555,16 +555,27 @@ def typedef(self) -> "Typedef": return import_module.get_typedef(type_name) return None - def union_types(self) -> Iterator["Type"]: + def union_types(self, with_typedefs: bool = False) -> Iterator["Type"]: if self.cdata.basetype != self.UNION: return + typedef = self.typedef() t = ffi.cast("struct lysc_type_union *", self.cdata) if self.cdata_parsed and self.cdata_parsed.types != ffi.NULL: for union_type, union_type_parsed in zip( ly_array_iter(t.types), ly_array_iter(self.cdata_parsed.types) ): yield Type(self.context, union_type, union_type_parsed) + elif ( + with_typedefs + and typedef + and typedef.cdata + and typedef.cdata.type.types != ffi.NULL + ): + for union_type, union_type_parsed in zip( + ly_array_iter(t.types), ly_array_iter(typedef.cdata.type.types) + ): + yield Type(self.context, union_type, union_type_parsed) else: for union_type in ly_array_iter(t.types): yield Type(self.context, union_type, None) diff --git a/tests/test_schema.py b/tests/test_schema.py index 434618bf..5bf5f888 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -420,7 +420,9 @@ def test_leaf_type_union(self): self.assertEqual(t.name(), "types:number") self.assertEqual(t.base(), Type.UNION) types = set(u.name() for u in t.union_types()) + types2 = set(u.name() for u in t.union_types(with_typedefs=True)) self.assertEqual(types, set(["int16", "int32", "uint16", "uint32"])) + self.assertEqual(types2, set(["signed", "unsigned"])) for u in t.union_types(): ext = u.get_extension( "type-desc", prefix="omg-extensions", arg_value=f"<{u.name()}>" From 6e42fb3dd9e448ac6e8e3c6599d4e864372dfebe Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Thu, 25 Jan 2024 23:15:15 +0100 Subject: [PATCH 08/71] data: fix DNode double free Calling free twice on a dnode is failing with libyang-python, when it is supported with libyang. That is because cdata is set to None in the free function. Set it to ffi.NULL so that libyang can see the NULL pointer. Fixes: #84 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/data.py | 2 +- tests/test_data.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/libyang/data.py b/libyang/data.py index 69649eba..c253c57e 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -881,7 +881,7 @@ def free(self, with_siblings: bool = True) -> None: else: self.free_internal(with_siblings) finally: - self.cdata = None + self.cdata = ffi.NULL def __repr__(self): cls = self.__class__ diff --git a/tests/test_data.py b/tests/test_data.py index 32c605fe..bf1a44fb 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -876,3 +876,8 @@ def test_add_defaults(self): node = dnode.find_path("/yolo-system:conf/speed") self.assertIsInstance(node, DLeaf) self.assertEqual(node.value(), 4321) + + def test_dnode_double_free(self): + dnode = self.ctx.parse_data_mem(self.JSON_CONFIG, "json", validate_present=True) + dnode.free() + dnode.free() From 6423babd1a5b26af3ce297477cdad6c90362e15f Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sat, 11 Nov 2023 23:10:04 +0100 Subject: [PATCH 09/71] data: add unlink function This patch adds unlink function from libyang to allow proper node cleanup procedure. Fixes: #84 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 2 ++ libyang/data.py | 6 ++++++ tests/test_data.py | 16 ++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index c1eaee39..23edb262 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -267,6 +267,8 @@ LY_ERR lys_set_implemented(struct lys_module *, const char **); #define LYD_NEW_PATH_CANON_VALUE ... LY_ERR lyd_new_path(struct lyd_node *, const struct ly_ctx *, const char *, const char *, uint32_t, struct lyd_node **); LY_ERR lyd_find_xpath(const struct lyd_node *, const char *, struct ly_set **); +void lyd_unlink_siblings(struct lyd_node *node); +void lyd_unlink_tree(struct lyd_node *node); void lyd_free_all(struct lyd_node *node); void lyd_free_tree(struct lyd_node *node); diff --git a/libyang/data.py b/libyang/data.py index c253c57e..0ed18750 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -868,6 +868,12 @@ def merge_data_dict( rpcreply=rpcreply, ) + def unlink(self, with_siblings: bool = False) -> None: + if with_siblings: + lib.lyd_unlink_siblings(self.cdata) + else: + lib.lyd_unlink_tree(self.cdata) + def free_internal(self, with_siblings: bool = True) -> None: if with_siblings: lib.lyd_free_all(self.cdata) diff --git a/tests/test_data.py b/tests/test_data.py index bf1a44fb..84e68f82 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -881,3 +881,19 @@ def test_dnode_double_free(self): dnode = self.ctx.parse_data_mem(self.JSON_CONFIG, "json", validate_present=True) dnode.free() dnode.free() + + def test_dnode_unlink(self): + dnode = self.ctx.parse_data_mem(self.JSON_CONFIG, "json", validate_present=True) + self.assertIsInstance(dnode, DContainer) + try: + child = dnode.find_one("hostname") + self.assertIsInstance(child, DNode) + child.unlink(with_siblings=False) + self.assertIsNone(dnode.find_one("hostname")) + child = next(dnode.children(), None) + self.assertIsNot(child, None) + child.unlink(with_siblings=True) + child = next(dnode.children(), None) + self.assertIsNone(child, None) + finally: + dnode.free() From d5f48d6f4f7767148bb2e217aec414766fd643f6 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sat, 11 Nov 2023 23:11:28 +0100 Subject: [PATCH 10/71] context: add LY_CTX_LEAFREF_EXTENDED option This patch adds the ability to create context with an additional option for extended leafrefs, which can use deref() functions. Fixes: #85 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 1 + cffi/source.c | 4 +-- libyang/context.py | 3 +++ tests/test_context.py | 5 ++++ tests/yang/yolo/yolo-leafref-extended.yang | 31 ++++++++++++++++++++++ 5 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 tests/yang/yolo/yolo-leafref-extended.yang diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 23edb262..0e4bd97f 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -14,6 +14,7 @@ struct ly_ctx; #define LY_CTX_EXPLICIT_COMPILE ... #define LY_CTX_REF_IMPLEMENTED ... #define LY_CTX_SET_PRIV_PARSED ... +#define LY_CTX_LEAFREF_EXTENDED ... typedef enum { diff --git a/cffi/source.c b/cffi/source.c index f7fe18a2..2682dd88 100644 --- a/cffi/source.c +++ b/cffi/source.c @@ -9,6 +9,6 @@ #if (LY_VERSION_MAJOR != 2) #error "This version of libyang bindings only works with libyang 2.x" #endif -#if (LY_VERSION_MINOR < 25) -#error "Need at least libyang 2.25" +#if (LY_VERSION_MINOR < 37) +#error "Need at least libyang 2.37" #endif diff --git a/libyang/context.py b/libyang/context.py index 0d8d0a67..dc3e0052 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -28,6 +28,7 @@ def __init__( search_path: Optional[str] = None, disable_searchdir_cwd: bool = True, explicit_compile: Optional[bool] = False, + leafref_extended: bool = False, yanglib_path: Optional[str] = None, yanglib_fmt: str = "json", cdata=None, # C type: "struct ly_ctx *" @@ -41,6 +42,8 @@ def __init__( options |= lib.LY_CTX_DISABLE_SEARCHDIR_CWD if explicit_compile: options |= lib.LY_CTX_EXPLICIT_COMPILE + if leafref_extended: + options |= lib.LY_CTX_LEAFREF_EXTENDED # force priv parsed options |= lib.LY_CTX_SET_PRIV_PARSED diff --git a/tests/test_context.py b/tests/test_context.py index 5650ecde..59839284 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -106,3 +106,8 @@ def test_ctx_parse_module(self): with Context(YANG_DIR) as ctx: mod = ctx.parse_module_file(f, features=["turbo-boost", "networking"]) self.assertIsInstance(mod, Module) + + def test_ctx_leafref_extended(self): + with Context(YANG_DIR, leafref_extended=True) as ctx: + mod = ctx.load_module("yolo-leafref-extended") + self.assertIsInstance(mod, Module) diff --git a/tests/yang/yolo/yolo-leafref-extended.yang b/tests/yang/yolo/yolo-leafref-extended.yang new file mode 100644 index 00000000..0aa8bd2f --- /dev/null +++ b/tests/yang/yolo/yolo-leafref-extended.yang @@ -0,0 +1,31 @@ +module yolo-leafref-extended { + yang-version 1.1; + namespace "urn:yang:yolo:leafref-extended"; + prefix leafref-ext; + + revision 2025-01-25 { + description + "Initial version."; + } + + list list1 { + key leaf1; + leaf leaf1 { + type string; + } + leaf-list leaflist2 { + type string; + } + } + + leaf ref1 { + type leafref { + path "../list1/leaf1"; + } + } + leaf ref2 { + type leafref { + path "deref(../ref1)/../leaflist2"; + } + } +} From b62fb0d52498d9749918f1609b44b75b98c376c8 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sat, 11 Nov 2023 23:51:41 +0100 Subject: [PATCH 11/71] schema: add fraction_digits support This patch introduces fraction_digits() and all_fraction_digits() functions for SLeaf and SLeafList. Fixes: #86 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/schema.py | 16 ++++++++++++++++ tests/test_schema.py | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/libyang/schema.py b/libyang/schema.py index 7e72e809..bd315596 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -622,6 +622,22 @@ def all_ranges(self) -> Iterator[str]: if rng is not None: yield rng + def fraction_digits(self) -> Optional[int]: + if not self.cdata_parsed: + return None + if self.cdata.basetype != self.DEC64: + return None + return self.cdata_parsed.fraction_digits + + def all_fraction_digits(self) -> Iterator[int]: + if self.cdata.basetype == lib.LY_TYPE_UNION: + for t in self.union_types(): + yield from t.all_fraction_digits() + else: + fd = self.fraction_digits() + if fd is not None: + yield fd + STR_TYPES = frozenset((STRING, BINARY, ENUM, IDENT, BITS)) def length(self) -> Optional[str]: diff --git a/tests/test_schema.py b/tests/test_schema.py index 5bf5f888..1287012a 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -477,6 +477,14 @@ def test_iter_tree(self): leaf = next(self.ctx.find_path("/yolo-system:conf")) self.assertEqual(len(list(leaf.iter_tree(full=True))), 23) + def test_leaf_type_fraction_digits(self): + self.ctx.load_module("yolo-nodetypes") + leaf = next(self.ctx.find_path("/yolo-nodetypes:conf/percentage")) + self.assertIsInstance(leaf, SLeaf) + t = leaf.type() + self.assertIsInstance(t, Type) + self.assertEqual(next(t.all_fraction_digits(), None), 2) + # ------------------------------------------------------------------------------------- class LeafTest(unittest.TestCase): From 3e3af68211a1087df778b9d6dfe6a978c7edbcc0 Mon Sep 17 00:00:00 2001 From: Sergei Markov Date: Thu, 25 Jan 2024 13:57:28 +0800 Subject: [PATCH 12/71] context: add support for LYD_VALIDATE_MULTI_ERROR Thanks to the LYD_VALIDATE_MULTI_ERROR, the validation does not stop at the first error, but generates all the detected errors. Add a dedicated test. Fixes: #79 Signed-off-by: Sergei Markov Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 1 + libyang/context.py | 9 ++++++++- libyang/data.py | 3 +++ tests/test_data.py | 25 +++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 0e4bd97f..ba6bd5d4 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -314,6 +314,7 @@ LY_ERR lyd_print_all(struct ly_out *, const struct lyd_node *, LYD_FORMAT, uint3 #define LYD_VALIDATE_NO_STATE ... #define LYD_VALIDATE_PRESENT ... #define LYD_VALIDATE_OPTS_MASK ... +#define LYD_VALIDATE_MULTI_ERROR ... LY_ERR lyd_parse_data_mem(const struct ly_ctx *, const char *, LYD_FORMAT, uint32_t, uint32_t, struct lyd_node **); diff --git a/libyang/context.py b/libyang/context.py index dc3e0052..eefc4e05 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -339,6 +339,7 @@ def parse_data( ordered: bool = False, strict: bool = False, validate_present: bool = False, + validate_multi_error: bool = False, ) -> Optional[DNode]: if self.cdata is None: raise RuntimeError("context already destroyed") @@ -351,7 +352,9 @@ def parse_data( strict=strict, ) validation_flgs = validation_flags( - no_state=no_state, validate_present=validate_present + no_state=no_state, + validate_present=validate_present, + validate_multi_error=validate_multi_error, ) fmt = data_format(fmt) encode = True @@ -403,6 +406,7 @@ def parse_data_mem( ordered: bool = False, strict: bool = False, validate_present: bool = False, + validate_multi_error: bool = False, ) -> Optional[DNode]: return self.parse_data( fmt, @@ -416,6 +420,7 @@ def parse_data_mem( ordered=ordered, strict=strict, validate_present=validate_present, + validate_multi_error=validate_multi_error, ) def parse_data_file( @@ -430,6 +435,7 @@ def parse_data_file( ordered: bool = False, strict: bool = False, validate_present: bool = False, + validate_multi_error: bool = False, ) -> Optional[DNode]: return self.parse_data( fmt, @@ -443,6 +449,7 @@ def parse_data_file( ordered=ordered, strict=strict, validate_present=validate_present, + validate_multi_error=validate_multi_error, ) def __iter__(self) -> Iterator[Module]: diff --git a/libyang/data.py b/libyang/data.py index 0ed18750..c2bf95f5 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -171,12 +171,15 @@ def data_type(dtype): def validation_flags( no_state: bool = False, validate_present: bool = False, + validate_multi_error: bool = False, ) -> int: flags = 0 if no_state: flags |= lib.LYD_VALIDATE_NO_STATE if validate_present: flags |= lib.LYD_VALIDATE_PRESENT + if validate_multi_error: + flags |= lib.LYD_VALIDATE_MULTI_ERROR return flags diff --git a/tests/test_data.py b/tests/test_data.py index 84e68f82..25bec67d 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -256,6 +256,31 @@ def test_data_parse_config_xml(self): finally: dnode.free() + XML_CONFIG_MULTI_ERROR = """ + foo + + https + /CESNET/libyang-python + abcd + + 2000 + +""" + + def test_data_parse_config_xml_multi_error(self): + with self.assertRaises(Exception) as cm: + self.ctx.parse_data_mem( + self.XML_CONFIG_MULTI_ERROR, + "xml", + validate_present=True, + validate_multi_error=True, + ) + self.assertEqual( + str(cm.exception), + 'failed to parse data tree: Invalid boolean value "abcd".: ' + 'List instance is missing its key "host".', + ) + XML_STATE = """ foo From 95012d95d0aa8996fbf14a8099deaf84ec87e41c Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Fri, 26 Jan 2024 09:29:00 +0100 Subject: [PATCH 13/71] schema: add uniques function for SList This patch add ability get a list of SList unique definitions. Fixes: #88 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/schema.py | 9 ++++++++- tests/test_schema.py | 16 ++++++++++++++++ tests/yang/yolo/yolo-nodetypes.yang | 21 +++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/libyang/schema.py b/libyang/schema.py index bd315596..b1fd416c 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: MIT from contextlib import suppress -from typing import IO, Any, Dict, Iterator, Optional, Tuple, Union +from typing import IO, Any, Dict, Iterator, List, Optional, Tuple, Union from _libyang import ffi, lib from .util import IOType, LibyangError, c2str, init_output, ly_array_iter, str2c @@ -1416,6 +1416,13 @@ def must_conditions(self) -> Iterator[str]: for must in ly_array_iter(pdata.musts): yield c2str(must.arg.str) + def uniques(self) -> Iterator[List[SNode]]: + for unique in ly_array_iter(self.cdata_list.uniques): + nodes = [] + for node in ly_array_iter(unique): + nodes.append(SNode.new(self.context, node)) + yield nodes + def __str__(self): return "%s [%s]" % (self.name(), ", ".join(k.name() for k in self.keys())) diff --git a/tests/test_schema.py b/tests/test_schema.py index 1287012a..4524a0a6 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -328,6 +328,22 @@ def test_list_parent(self): self.assertIsInstance(parent, SContainer) self.assertEqual(parent.name(), "conf") + def test_list_uniques(self): + self.ctx.load_module("yolo-nodetypes") + list1 = next(self.ctx.find_path("/yolo-nodetypes:conf/list1")) + self.assertIsInstance(list1, SList) + uniques = list(list1.uniques()) + self.assertEqual(len(uniques), 1) + elements = [u.name() for u in uniques[0]] + self.assertEqual(len(elements), 2) + self.assertTrue("leaf2" in elements) + self.assertTrue("leaf3" in elements) + + list2 = next(self.ctx.find_path("/yolo-nodetypes:conf/list2")) + self.assertIsInstance(list2, SList) + uniques = list(list2.uniques()) + self.assertEqual(len(uniques), 0) + # ------------------------------------------------------------------------------------- class RpcTest(unittest.TestCase): diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index 3dd76832..4ef7d3d6 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -28,5 +28,26 @@ module yolo-nodetypes { default 2.5; default 2.6; } + + list list1 { + key leaf1; + unique "leaf2 leaf3"; + leaf leaf1 { + type string; + } + leaf leaf2 { + type string; + } + leaf leaf3 { + type string; + } + } + + list list2 { + key leaf1; + leaf leaf1 { + type string; + } + } } } From 4b5b425cdef1647a4b6ea6b50765cbccf61c442d Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sat, 11 Nov 2023 23:57:17 +0100 Subject: [PATCH 14/71] schema: add {min,max}_elements for SList and SLeafList This patch add ability to get minumum and maximum number of elements, and add unit tests. The yolo-nodetypes module is used twice and is now loaded in the list test setUp. Fixes: #89 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/schema.py | 16 ++++++++++++++++ tests/test_schema.py | 24 +++++++++++++++++++++++- tests/yang/yolo/yolo-nodetypes.yang | 12 ++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/libyang/schema.py b/libyang/schema.py index b1fd416c..c6ffa607 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1325,6 +1325,16 @@ def must_conditions(self) -> Iterator[str]: for must in ly_array_iter(pdata.musts): yield c2str(must.arg.str) + def max_elements(self) -> int: + return ( + self.cdata_leaflist.max + if self.cdata_leaflist.max != (2**32 - 1) + else None + ) + + def min_elements(self) -> int: + return self.cdata_leaflist.min + def __str__(self): return "%s %s" % (self.name(), self.type().name()) @@ -1423,6 +1433,12 @@ def uniques(self) -> Iterator[List[SNode]]: nodes.append(SNode.new(self.context, node)) yield nodes + def max_elements(self) -> int: + return self.cdata_list.max if self.cdata_list.max != (2**32 - 1) else None + + def min_elements(self) -> int: + return self.cdata_list.min + def __str__(self): return "%s [%s]" % (self.name(), ", ".join(k.name() for k in self.keys())) diff --git a/tests/test_schema.py b/tests/test_schema.py index 4524a0a6..ad093c66 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -291,6 +291,7 @@ class ListTest(unittest.TestCase): def setUp(self): self.ctx = Context(YANG_DIR) self.ctx.load_module("yolo-system") + self.ctx.load_module("yolo-nodetypes") self.list = next(self.ctx.find_path(self.PATH["LOG"])) def tearDown(self): @@ -329,7 +330,6 @@ def test_list_parent(self): self.assertEqual(parent.name(), "conf") def test_list_uniques(self): - self.ctx.load_module("yolo-nodetypes") list1 = next(self.ctx.find_path("/yolo-nodetypes:conf/list1")) self.assertIsInstance(list1, SList) uniques = list(list1.uniques()) @@ -344,6 +344,17 @@ def test_list_uniques(self): uniques = list(list2.uniques()) self.assertEqual(len(uniques), 0) + def test_list_min_max(self): + list1 = next(self.ctx.find_path("/yolo-nodetypes:conf/list1")) + self.assertIsInstance(list1, SList) + self.assertEqual(list1.min_elements(), 2) + self.assertEqual(list1.max_elements(), 10) + + list2 = next(self.ctx.find_path("/yolo-nodetypes:conf/list2")) + self.assertIsInstance(list2, SList) + self.assertEqual(list2.min_elements(), 0) + self.assertEqual(list2.max_elements(), None) + # ------------------------------------------------------------------------------------- class RpcTest(unittest.TestCase): @@ -531,3 +542,14 @@ def test_leaflist_defaults(self): leaflist = next(self.ctx.find_path("/yolo-nodetypes:conf/ratios")) for d in leaflist.defaults(): self.assertIsInstance(d, float) + + def test_leaf_list_min_max(self): + leaflist1 = next(self.ctx.find_path("/yolo-nodetypes:conf/leaf-list1")) + self.assertIsInstance(leaflist1, SLeafList) + self.assertEqual(leaflist1.min_elements(), 3) + self.assertEqual(leaflist1.max_elements(), 11) + + leaflist2 = next(self.ctx.find_path("/yolo-nodetypes:conf/leaf-list2")) + self.assertIsInstance(leaflist2, SLeafList) + self.assertEqual(leaflist2.min_elements(), 0) + self.assertEqual(leaflist2.max_elements(), None) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index 4ef7d3d6..785c4bc7 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -32,6 +32,8 @@ module yolo-nodetypes { list list1 { key leaf1; unique "leaf2 leaf3"; + min-elements 2; + max-elements 10; leaf leaf1 { type string; } @@ -49,5 +51,15 @@ module yolo-nodetypes { type string; } } + + leaf-list leaf-list1 { + type string; + min-elements 3; + max-elements 11; + } + + leaf-list leaf-list2 { + type string; + } } } From 21849df456b587cf556a3f7ec459a014f0f4ff02 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Tue, 28 Nov 2023 15:02:58 +0100 Subject: [PATCH 15/71] data: add insert_sibling function to DNode This patch introduce the insert_sibling function usable for inserting data nodes on the module level. Add a unit test. A presence container is added on the conf container of the yolo-nodetypes module to avoid this error: > libyang.util.LibyangError: failed to parse data tree: Too few "list1" instances. Fixes: #95 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 1 + libyang/data.py | 5 +++++ tests/test_data.py | 17 +++++++++++++++++ tests/yang/yolo/yolo-nodetypes.yang | 5 +++++ 4 files changed, 28 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index ba6bd5d4..7ab29e50 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -971,6 +971,7 @@ LY_ERR lyd_any_value_str(const struct lyd_node *, char **); LY_ERR lyd_merge_tree(struct lyd_node **, const struct lyd_node *, uint16_t); LY_ERR lyd_merge_siblings(struct lyd_node **, const struct lyd_node *, uint16_t); LY_ERR lyd_insert_child(struct lyd_node *, struct lyd_node *); +LY_ERR lyd_insert_sibling(struct lyd_node *, struct lyd_node *, struct lyd_node **); LY_ERR lyd_diff_apply_all(struct lyd_node **, const struct lyd_node *); #define LYD_DUP_NO_META ... diff --git a/libyang/data.py b/libyang/data.py index c2bf95f5..633cf74a 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -322,6 +322,11 @@ def insert_child(self, node): if ret != lib.LY_SUCCESS: raise self.context.error("cannot insert node") + def insert_sibling(self, node): + ret = lib.lyd_insert_sibling(self.cdata, node.cdata, ffi.NULL) + if ret != lib.LY_SUCCESS: + raise self.context.error("cannot insert sibling") + def name(self) -> str: return c2str(self.cdata.schema.name) diff --git a/tests/test_data.py b/tests/test_data.py index 25bec67d..9b774a2e 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -20,6 +20,7 @@ IOType, LibyangError, ) +from libyang.data import dict_to_dnode YANG_DIR = os.path.join(os.path.dirname(__file__), "yang") @@ -32,6 +33,7 @@ def setUp(self): modules = [ self.ctx.load_module("ietf-netconf"), self.ctx.load_module("yolo-system"), + self.ctx.load_module("yolo-nodetypes"), ] for mod in modules: @@ -922,3 +924,18 @@ def test_dnode_unlink(self): self.assertIsNone(child, None) finally: dnode.free() + + def test_dnode_insert_sibling(self): + MAIN = {"yolo-nodetypes:conf": {"percentage": "20.2"}} + SIBLING = {"yolo-nodetypes:test1": 10} + module = self.ctx.get_module("yolo-nodetypes") + dnode1 = dict_to_dnode(MAIN, module, None, validate=False) + dnode2 = dict_to_dnode(SIBLING, module, None, validate=False) + self.assertEqual(len(list(dnode1.siblings(include_self=False))), 0) + self.assertEqual(len(list(dnode2.siblings(include_self=False))), 0) + dnode2.insert_sibling(dnode1) + self.assertEqual(len(list(dnode1.siblings(include_self=False))), 1) + self.assertEqual(len(list(dnode2.siblings(include_self=False))), 1) + sibling = next(dnode1.siblings(include_self=False), None) + self.assertIsInstance(sibling, DLeaf) + self.assertEqual(sibling.cdata, dnode2.cdata) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index 785c4bc7..6daab475 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -12,6 +12,7 @@ module yolo-nodetypes { } container conf { + presence "enable conf"; description "Configuration."; leaf percentage { @@ -62,4 +63,8 @@ module yolo-nodetypes { type string; } } + + leaf test1 { + type uint8; + } } From 06d486da18e2a22b9ede0dfb4a28e1bbef6b7a56 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Tue, 28 Nov 2023 15:04:28 +0100 Subject: [PATCH 16/71] data: add only_node option to add_defaults This patch allows user to use only_node option within add_defaults function. This option limits the scope of creating and adding implicit default data nodes to just given tree where DNode is considered as root. Fixes: #96 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 1 + libyang/data.py | 11 ++++++++--- tests/test_data.py | 20 +++++++++++++++----- tests/yang/yolo/yolo-nodetypes.yang | 11 +++++++++++ 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 7ab29e50..8a62118b 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1021,6 +1021,7 @@ LY_ERR lyd_merge_module(struct lyd_node **, const struct lyd_node *, const struc #define LYD_IMPLICIT_OUTPUT ... #define LYD_IMPLICIT_NO_DEFAULTS ... +LY_ERR lyd_new_implicit_tree(struct lyd_node *, uint32_t, struct lyd_node **); LY_ERR lyd_new_implicit_all(struct lyd_node **, const struct ly_ctx *, uint32_t, struct lyd_node **); LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_meta **); diff --git a/libyang/data.py b/libyang/data.py index 633cf74a..f9d3301d 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -260,6 +260,7 @@ def add_defaults( no_defaults: bool = False, no_state: bool = False, output: bool = False, + only_node: bool = False, ): flags = implicit_flags( no_config=no_config, @@ -267,9 +268,13 @@ def add_defaults( no_state=no_state, output=output, ) - node_p = ffi.new("struct lyd_node **") - node_p[0] = self.cdata - ret = lib.lyd_new_implicit_all(node_p, self.context.cdata, flags, ffi.NULL) + if only_node: + node_p = ffi.cast("struct lyd_node *", self.cdata) + ret = lib.lyd_new_implicit_tree(node_p, flags, ffi.NULL) + else: + node_p = ffi.new("struct lyd_node **") + node_p[0] = self.cdata + ret = lib.lyd_new_implicit_all(node_p, self.context.cdata, flags, ffi.NULL) if ret != lib.LY_SUCCESS: raise self.context.error("cannot get module") diff --git a/tests/test_data.py b/tests/test_data.py index 9b774a2e..1824acba 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -14,6 +14,7 @@ DataType, DContainer, DLeaf, + DList, DNode, DNotif, DRpc, @@ -893,13 +894,22 @@ def test_find_all(self): dnode.free() def test_add_defaults(self): - dnode = self.ctx.parse_data_mem(self.JSON_CONFIG, "json", validate_present=True) - node = dnode.find_path("/yolo-system:conf/speed") + JSON = '{"yolo-nodetypes:records": [{"id": "rec1"}]}' + dnode = self.ctx.parse_data_mem( + JSON, "json", validate_present=True, parse_only=True + ) + self.assertIsInstance(dnode, DList) + node = dnode.find_one("id") self.assertIsInstance(node, DLeaf) - node.free(with_siblings=False) - node = dnode.find_path("/yolo-system:conf/speed") + node = dnode.find_one("name") + self.assertIsNone(node) + dnode.add_defaults(only_node=True) + node = dnode.find_one("name") + self.assertIsInstance(node, DLeaf) + self.assertEqual(node.value(), "ASD") + node = dnode.find_path("/yolo-nodetypes:conf/speed") self.assertIsNone(node) - dnode.add_defaults() + dnode.add_defaults(only_node=False) node = dnode.find_path("/yolo-system:conf/speed") self.assertIsInstance(node, DLeaf) self.assertEqual(node.value(), 4321) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index 6daab475..6da6933a 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -11,6 +11,17 @@ module yolo-nodetypes { "Initial version."; } + list records { + key id; + leaf id { + type string; + } + leaf name { + type string; + default "ASD"; + } + } + container conf { presence "enable conf"; description From f26b56a67de204d82c3bc7cf42888d4fc317a88e Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sun, 12 Nov 2023 01:11:23 +0100 Subject: [PATCH 17/71] schema: add require_instance function for leafrefs This patch introduces require_instance function, to allow user to get information whether the leafref requires valid instace prior being instanciated. Fixes: #93 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/schema.py | 6 ++++++ tests/test_schema.py | 7 +++++++ tests/yang/yolo/yolo-system.yang | 1 + 3 files changed, 14 insertions(+) diff --git a/libyang/schema.py b/libyang/schema.py index c6ffa607..30d4d744 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -681,6 +681,12 @@ def all_patterns(self) -> Iterator[Tuple[str, bool]]: else: yield from self.patterns() + def require_instance(self) -> Optional[bool]: + if self.cdata.basetype != self.LEAFREF: + return None + t = ffi.cast("struct lysc_type_leafref *", self.cdata) + return bool(t.require_instance) + def module(self) -> Module: if not self.cdata_parsed: return None diff --git a/tests/test_schema.py b/tests/test_schema.py index ad093c66..a7a68f54 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -512,6 +512,13 @@ def test_leaf_type_fraction_digits(self): self.assertIsInstance(t, Type) self.assertEqual(next(t.all_fraction_digits(), None), 2) + def test_leaf_type_require_instance(self): + leaf = next(self.ctx.find_path("/yolo-system:conf/hostname-ref")) + self.assertIsInstance(leaf, SLeaf) + t = leaf.type() + self.assertIsInstance(t, Type) + self.assertFalse(t.require_instance()) + # ------------------------------------------------------------------------------------- class LeafTest(unittest.TestCase): diff --git a/tests/yang/yolo/yolo-system.yang b/tests/yang/yolo/yolo-system.yang index a34c421c..78a0e20f 100644 --- a/tests/yang/yolo/yolo-system.yang +++ b/tests/yang/yolo/yolo-system.yang @@ -43,6 +43,7 @@ module yolo-system { leaf hostname-ref { type leafref { path "../hostname"; + require-instance false; } } From f5b4e7a084a83a44d1996dd152e1a675eeff9cb5 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sun, 12 Nov 2023 00:07:23 +0100 Subject: [PATCH 18/71] schema: add Must class This patches introduces a new Must class that represents a must statement. A new musts method is added to access the must statements of a node. The must_conditions method is updated to use this new class, and factorized in the SNode class. Fixes: #90 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/__init__.py | 2 ++ libyang/schema.py | 55 ++++++++++++++--------------- tests/test_schema.py | 10 ++++++ tests/yang/yolo/yolo-nodetypes.yang | 3 ++ 4 files changed, 41 insertions(+), 29 deletions(-) diff --git a/libyang/__init__.py b/libyang/__init__.py index aa9dcca9..482a0ad0 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -74,6 +74,7 @@ IfNotFeature, IfOrFeatures, Module, + Must, Revision, SContainer, SLeaf, @@ -138,6 +139,7 @@ "MandatoryAdded", "MandatoryRemoved", "Module", + "Must", "MustAdded", "MustRemoved", "NodeTypeAdded", diff --git a/libyang/schema.py b/libyang/schema.py index 30d4d744..b06623c4 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1036,6 +1036,21 @@ def __str__(self): return "(%s OR %s)" % (self.a, self.b) +# ------------------------------------------------------------------------------------- +class Must: + __slots__ = ("context", "cdata") + + def __init__(self, context: "libyang.Context", cdata): + self.context = context + self.cdata = cdata # C type: "struct lysc_must *" + + def condition(self) -> str: + return c2str(lib.lyxp_get_expr(self.cdata.cond)) + + def error_message(self) -> Optional[str]: + return c2str(self.cdata.emsg) if self.cdata.emsg != ffi.NULL else None + + # ------------------------------------------------------------------------------------- class SNode: __slots__ = ("context", "cdata", "cdata_parsed", "__dict__") @@ -1138,7 +1153,17 @@ def extensions(self) -> Iterator[ExtensionCompiled]: yield ExtensionCompiled(self.context, extension) def must_conditions(self) -> Iterator[str]: - return iter(()) + for must in self.musts(): + yield must.condition() + + def musts(self) -> Iterator[Must]: + mc = lib.lysc_node_musts(self.cdata) + if mc == ffi.NULL: + return + for m in ly_array_iter(mc): + if not m: + continue + yield Must(self.context, m) def get_extension( self, name: str, prefix: Optional[str] = None, arg_value: Optional[str] = None @@ -1269,13 +1294,6 @@ def is_key(self) -> bool: return True return False - def must_conditions(self) -> Iterator[str]: - pdata = self.cdata_leaf_parsed - if pdata.musts == ffi.NULL: - return - for must in ly_array_iter(pdata.musts): - yield c2str(must.arg.str) - def __str__(self): return "%s %s" % (self.name(), self.type().name()) @@ -1324,13 +1342,6 @@ def defaults(self) -> Iterator[Union[None, bool, int, str, float]]: else: yield val - def must_conditions(self) -> Iterator[str]: - pdata = self.cdata_leaflist_parsed - if pdata.musts == ffi.NULL: - return - for must in ly_array_iter(pdata.musts): - yield c2str(must.arg.str) - def max_elements(self) -> int: return ( self.cdata_leaflist.max @@ -1363,13 +1374,6 @@ def presence(self) -> Optional[str]: return c2str(self.cdata_container_parsed.presence) - def must_conditions(self) -> Iterator[str]: - pdata = self.cdata_container_parsed - if pdata.musts == ffi.NULL: - return - for must in ly_array_iter(pdata.musts): - yield c2str(must.arg.str) - def __iter__(self) -> Iterator[SNode]: return self.children() @@ -1425,13 +1429,6 @@ def keys(self) -> Iterator[SNode]: yield SLeaf(self.context, node) node = node.next - def must_conditions(self) -> Iterator[str]: - pdata = self.cdata_list_parsed - if pdata.musts == ffi.NULL: - return - for must in ly_array_iter(pdata.musts): - yield c2str(must.arg.str) - def uniques(self) -> Iterator[List[SNode]]: for unique in ly_array_iter(self.cdata_list.uniques): nodes = [] diff --git a/tests/test_schema.py b/tests/test_schema.py index a7a68f54..c83bde7f 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -12,6 +12,7 @@ IOType, LibyangError, Module, + Must, Revision, SContainer, SLeaf, @@ -530,6 +531,15 @@ def tearDown(self): self.ctx.destroy() self.ctx = None + def test_must(self): + leaf = next(self.ctx.find_path("/yolo-nodetypes:conf/percentage")) + self.assertIsInstance(leaf, SLeaf) + must = next(leaf.musts(), None) + self.assertIsInstance(must, Must) + self.assertEqual(must.error_message(), "ERROR1") + must = next(leaf.must_conditions(), None) + self.assertIsInstance(must, str) + def test_leaf_default(self): leaf = next(self.ctx.find_path("/yolo-nodetypes:conf/percentage")) self.assertIsInstance(leaf.default(), float) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index 6da6933a..a456ae1d 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -31,6 +31,9 @@ module yolo-nodetypes { fraction-digits 2; } default 10.2; + must ". = 10.6" { + error-message "ERROR1"; + } } leaf-list ratios { From 0f438382dabd6e1f0850ba5c28cb5f2a3442b80c Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sun, 12 Nov 2023 00:08:42 +0100 Subject: [PATCH 19/71] schema: add Pattern class This patches introduces a new Pattern class that represents a pattern statement. A new pattern_details method is added to access the pattern statements of a node. Fixes: #92 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 15 +++++++++++++ libyang/__init__.py | 2 ++ libyang/schema.py | 36 ++++++++++++++++++++++++++++++++ tests/test_schema.py | 10 +++++++++ tests/yang/yolo/yolo-system.yang | 4 +++- 5 files changed, 66 insertions(+), 1 deletion(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 8a62118b..53e91695 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -792,6 +792,21 @@ struct lysc_must { struct lysc_ext_instance *exts; }; +struct pcre2_real_code; +typedef struct pcre2_real_code pcre2_code; + +struct lysc_pattern { + const char *expr; + pcre2_code *code; + const char *dsc; + const char *ref; + const char *emsg; + const char *eapptag; + struct lysc_ext_instance *exts; + uint32_t inverted : 1; + uint32_t refcount : 31; +}; + #define LYSP_RESTR_PATTERN_ACK ... #define LYSP_RESTR_PATTERN_NACK ... diff --git a/libyang/__init__.py b/libyang/__init__.py index 482a0ad0..f9dfeccd 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -75,6 +75,7 @@ IfOrFeatures, Module, Must, + Pattern, Revision, SContainer, SLeaf, @@ -146,6 +147,7 @@ "NodeTypeRemoved", "OrderedByUserAdded", "OrderedByUserRemoved", + "Pattern", "PatternAdded", "PatternRemoved", "PresenceAdded", diff --git a/libyang/schema.py b/libyang/schema.py index b06623c4..3ba6d996 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -448,6 +448,24 @@ class Bit(_EnumBit): pass +# ------------------------------------------------------------------------------------- +class Pattern: + __slots__ = ("context", "cdata") + + def __init__(self, context: "libyang.Context", cdata): + self.context = context + self.cdata = cdata # C type: "struct lysc_pattern *" + + def expression(self) -> str: + return c2str(self.cdata.expr) + + def inverted(self) -> bool: + return self.cdata.inverted + + def error_message(self) -> Optional[str]: + return c2str(self.cdata.emsg) if self.cdata.emsg != ffi.NULL else None + + # ------------------------------------------------------------------------------------- class Type: __slots__ = ("context", "cdata", "cdata_parsed", "__dict__") @@ -681,6 +699,24 @@ def all_patterns(self) -> Iterator[Tuple[str, bool]]: else: yield from self.patterns() + def pattern_details(self) -> Iterator[Pattern]: + if self.cdata.basetype != self.STRING: + return + t = ffi.cast("struct lysc_type_str *", self.cdata) + if t.patterns == ffi.NULL: + return + for p in ly_array_iter(t.patterns): + if not p: + continue + yield Pattern(self.context, p) + + def all_pattern_details(self) -> Iterator[Pattern]: + if self.cdata.basetype == lib.LY_TYPE_UNION: + for t in self.union_types(): + yield from t.all_pattern_details() + else: + yield from self.pattern_details() + def require_instance(self) -> Optional[bool]: if self.cdata.basetype != self.LEAFREF: return None diff --git a/tests/test_schema.py b/tests/test_schema.py index c83bde7f..78b1229b 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -13,6 +13,7 @@ LibyangError, Module, Must, + Pattern, Revision, SContainer, SLeaf, @@ -439,6 +440,15 @@ def test_leaf_type_pattern(self): t = leaf.type() self.assertIsInstance(t, Type) self.assertEqual(list(t.patterns()), [("[a-z.]+", False), ("1", True)]) + patterns = list(t.all_pattern_details()) + self.assertEqual(len(patterns), 2) + self.assertIsInstance(patterns[0], Pattern) + self.assertEqual(patterns[0].expression(), "[a-z.]+") + self.assertFalse(patterns[0].inverted()) + self.assertEqual(patterns[0].error_message(), "ERROR1") + self.assertEqual(patterns[1].expression(), "1") + self.assertTrue(patterns[1].inverted()) + self.assertIsNone(patterns[1].error_message()) def test_leaf_type_union(self): leaf = next(self.ctx.find_path("/yolo-system:conf/yolo-system:number")) diff --git a/tests/yang/yolo/yolo-system.yang b/tests/yang/yolo/yolo-system.yang index 78a0e20f..5aa633ad 100644 --- a/tests/yang/yolo/yolo-system.yang +++ b/tests/yang/yolo/yolo-system.yang @@ -85,7 +85,9 @@ module yolo-system { } leaf host { type string { - pattern "[a-z.]+"; + pattern "[a-z.]+" { + error-message "ERROR1"; + } pattern "1" { modifier "invert-match"; } From a6c7164b743d83041cb0db5363b0863f9a311be1 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Mon, 29 Jan 2024 13:28:21 +0100 Subject: [PATCH 20/71] schema: use Pattern class for patterns method Now that we have the Pattern class, use it for the patterns() method. Fixes: #92 Signed-off-by: Samuel Gauthier --- libyang/schema.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index 3ba6d996..c3f0c931 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -677,20 +677,8 @@ def all_lengths(self) -> Iterator[str]: yield length def patterns(self) -> Iterator[Tuple[str, bool]]: - if not self.cdata_parsed or self.cdata.basetype != self.STRING: - return - if self.cdata_parsed.patterns == ffi.NULL: - return - for p in ly_array_iter(self.cdata_parsed.patterns): - if not p: - continue - # in case of pattern restriction, the first byte has a special meaning: - # 0x06 (ACK) for regular match and 0x15 (NACK) for invert-match - invert_match = p.arg.str[0] == b"\x15" - # yield tuples like: - # ('[a-zA-Z_][a-zA-Z0-9\-_.]*', False) - # ('[xX][mM][lL].*', True) - yield c2str(p.arg.str + 1), invert_match + for pattern in self.pattern_details(): + yield pattern.expression(), pattern.inverted() def all_patterns(self) -> Iterator[Tuple[str, bool]]: if self.cdata.basetype == lib.LY_TYPE_UNION: From 5681ef0f50ee14ceaa811c6b36c30f84de0d0f1c Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Fri, 26 Jan 2024 11:13:49 +0100 Subject: [PATCH 21/71] schema: support LYS_GETNEXT_WITH{CASE,CHOICE} Add support for LYS_GETNEXT_WITH{CASE,CHOICE} for Module, Container, Choice and Case. To do so, introduce a new iter_children_options function that translates booleans to lys_getnext options. The Rpc children function is updated to use the iter_children_options mecanism. The lib.LYS_CHOICE and lib.LYS_CASE are added by to the default types in iter_children, as it does not change the behavior. Fixes: #91 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 1 + libyang/schema.py | 89 +++++++++++++++++++++++++++++++++++++------- tests/test_schema.py | 4 ++ 3 files changed, 80 insertions(+), 14 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 53e91695..78e500a4 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -691,6 +691,7 @@ struct lysc_ext { #define LYS_GETNEXT_WITHCASE ... #define LYS_GETNEXT_INTONPCONT ... #define LYS_GETNEXT_OUTPUT ... +#define LYS_GETNEXT_WITHSCHEMAMOUNT ... const struct lysc_node* lys_find_child(const struct lysc_node *, const struct lys_module *, const char *, size_t, uint16_t, uint32_t); const struct lysc_node* lysc_node_child(const struct lysc_node *); diff --git a/libyang/schema.py b/libyang/schema.py index c3f0c931..b257c8f6 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -137,8 +137,12 @@ def get_module_from_prefix(self, prefix: str) -> Optional["Module"]: def __iter__(self) -> Iterator["SNode"]: return self.children() - def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator["SNode"]: - return iter_children(self.context, self.cdata, types=types) + def children( + self, types: Optional[Tuple[int, ...]] = None, with_choice: bool = False + ) -> Iterator["SNode"]: + return iter_children( + self.context, self.cdata, types=types, with_choice=with_choice + ) def __str__(self) -> str: return self.name() @@ -1401,8 +1405,12 @@ def presence(self) -> Optional[str]: def __iter__(self) -> Iterator[SNode]: return self.children() - def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator[SNode]: - return iter_children(self.context, self.cdata, types=types) + def children( + self, types: Optional[Tuple[int, ...]] = None, with_choice: bool = False + ) -> Iterator[SNode]: + return iter_children( + self.context, self.cdata, types=types, with_choice=with_choice + ) # ------------------------------------------------------------------------------------- @@ -1411,8 +1419,10 @@ class SChoice(SNode): def __iter__(self) -> Iterator[SNode]: return self.children() - def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator[SNode]: - return iter_children(self.context, self.cdata, types=types) + def children( + self, types: Optional[Tuple[int, ...]] = None, with_case: bool = False + ) -> Iterator[SNode]: + return iter_children(self.context, self.cdata, types=types, with_case=with_case) # ------------------------------------------------------------------------------------- @@ -1421,8 +1431,12 @@ class SCase(SNode): def __iter__(self) -> Iterator[SNode]: return self.children() - def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator[SNode]: - return iter_children(self.context, self.cdata, types=types) + def children( + self, types: Optional[Tuple[int, ...]] = None, with_choice: bool = False + ) -> Iterator[SNode]: + return iter_children( + self.context, self.cdata, types=types, with_choice=with_choice + ) # ------------------------------------------------------------------------------------- @@ -1442,9 +1456,18 @@ def __iter__(self) -> Iterator[SNode]: return self.children() def children( - self, skip_keys: bool = False, types: Optional[Tuple[int, ...]] = None + self, + skip_keys: bool = False, + types: Optional[Tuple[int, ...]] = None, + with_choice: bool = False, ) -> Iterator[SNode]: - return iter_children(self.context, self.cdata, skip_keys=skip_keys, types=types) + return iter_children( + self.context, + self.cdata, + skip_keys=skip_keys, + types=types, + with_choice=with_choice, + ) def keys(self) -> Iterator[SNode]: node = lib.lysc_node_child(self.cdata) @@ -1512,9 +1535,7 @@ def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator[SNode]: yield from iter_children(self.context, self.cdata, types=types) # With libyang2, you can get only input or output # To keep behavior, we iter 2 times witt output options - yield from iter_children( - self.context, self.cdata, types=types, options=lib.LYS_GETNEXT_OUTPUT - ) + yield from iter_children(self.context, self.cdata, types=types, output=True) # ------------------------------------------------------------------------------------- @@ -1539,13 +1560,43 @@ class SAnydata(SNode): pass +# ------------------------------------------------------------------------------------- +def iter_children_options( + with_choice: bool = False, + no_choice: bool = False, + with_case: bool = False, + into_non_presence_container: bool = False, + output: bool = False, + with_schema_mount: bool = False, +) -> int: + options = 0 + if with_choice: + options |= lib.LYS_GETNEXT_WITHCHOICE + if no_choice: + options |= lib.LYS_GETNEXT_NOCHOICE + if with_case: + options |= lib.LYS_GETNEXT_WITHCASE + if into_non_presence_container: + options |= lib.LYS_GETNEXT_INTONPCONT + if output: + options |= lib.LYS_GETNEXT_OUTPUT + if with_schema_mount: + options |= lib.LYS_GETNEXT_WITHSCHEMAMOUNT + return options + + # ------------------------------------------------------------------------------------- def iter_children( context: "libyang.Context", parent, # C type: Union["struct lys_module *", "struct lys_node *"] skip_keys: bool = False, types: Optional[Tuple[int, ...]] = None, - options: int = 0, + with_choice: bool = False, + no_choice: bool = False, + with_case: bool = False, + into_non_presence_container: bool = False, + output: bool = False, + with_schema_mount: bool = False, ) -> Iterator[SNode]: if types is None: types = ( @@ -1556,6 +1607,8 @@ def iter_children( lib.LYS_LEAF, lib.LYS_LEAFLIST, lib.LYS_NOTIF, + lib.LYS_CHOICE, + lib.LYS_CASE, ) def _skip(node) -> bool: @@ -1576,6 +1629,14 @@ def _skip(node) -> bool: else: module = ffi.NULL + options = iter_children_options( + with_choice=with_choice, + no_choice=no_choice, + with_case=with_case, + into_non_presence_container=into_non_presence_container, + output=output, + with_schema_mount=with_schema_mount, + ) child = lib.lys_getnext(ffi.NULL, parent, module, options) while child: if not _skip(child): diff --git a/tests/test_schema.py b/tests/test_schema.py index 78b1229b..5b947cd8 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -271,6 +271,10 @@ def test_cont_iter(self): def test_cont_children_leafs(self): leafs = list(self.container.children(types=(SNode.LEAF,))) self.assertEqual(len(leafs), 9) + without_choice = [c.name() for c in self.container.children(with_choice=False)] + with_choice = [c.name() for c in self.container.children(with_choice=True)] + self.assertTrue("pill" not in without_choice) + self.assertTrue("pill" in with_choice) def test_cont_parent(self): self.assertIsNone(self.container.parent()) From ae3b32023ba5f9a5fbf52299785528e49ab8251a Mon Sep 17 00:00:00 2001 From: nvxf <68589039+nvxf@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:23:47 +0100 Subject: [PATCH 22/71] data: fix DNode.new schema handling In case of a call to DNode.new with cdata containing an opaque node the new method tries to access cdata.schema.nodetype, which results in a NULL pointer dereference. To get the schema for an opaque node retrieve the schema from the context using the path of the node. Fixes: #73 Signed-off-by: nvxf <68589039+nvxf@users.noreply.github.com> Acked-by: Samuel Gauthier --- libyang/data.py | 22 ++++++++++++++++------ tests/test_data.py | 11 +++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/libyang/data.py b/libyang/data.py index f9d3301d..abc66cd6 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -417,11 +417,7 @@ def eval_xpath(self, xpath: str): return False def path(self) -> str: - path = lib.lyd_path(self.cdata, lib.LYD_PATH_STD, ffi.NULL, 0) - try: - return c2str(path) - finally: - lib.free(path) + return self._get_path(self.cdata) def validate( self, @@ -923,11 +919,25 @@ def _decorator(nodeclass): @classmethod def new(cls, context: "libyang.Context", cdata) -> "DNode": cdata = ffi.cast("struct lyd_node *", cdata) - nodecls = cls.NODETYPE_CLASS.get(cdata.schema.nodetype, None) + if not cdata.schema: + schemas = list(context.find_path(cls._get_path(cdata))) + if len(schemas) != 1: + raise LibyangError("Unable to determine schema") + nodecls = cls.NODETYPE_CLASS.get(schemas[0].nodetype(), None) + else: + nodecls = cls.NODETYPE_CLASS.get(cdata.schema.nodetype, None) if nodecls is None: raise TypeError("node type %s not implemented" % cdata.schema.nodetype) return nodecls(context, cdata) + @staticmethod + def _get_path(cdata) -> str: + path = lib.lyd_path(cdata, lib.LYD_PATH_STD, ffi.NULL, 0) + try: + return c2str(path) + finally: + lib.free(path) + # ------------------------------------------------------------------------------------- @DNode.register(SNode.CONTAINER) diff --git a/tests/test_data.py b/tests/test_data.py index 1824acba..6a160359 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -949,3 +949,14 @@ def test_dnode_insert_sibling(self): sibling = next(dnode1.siblings(include_self=False), None) self.assertIsInstance(sibling, DLeaf) self.assertEqual(sibling.cdata, dnode2.cdata) + + def test_dnode_new_opaq_find_one(self): + root = self.ctx.create_data_path(path="/yolo-system:conf") + root.new_path( + "hostname", + None, + opt_opaq=True, + ) + dnode = root.find_one("/yolo-system:conf/hostname") + + self.assertIsInstance(dnode, DLeaf) From 1a069b9c8188f329dcc05371d9288b1ad318d936 Mon Sep 17 00:00:00 2001 From: nvxf <68589039+nvxf@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:25:32 +0100 Subject: [PATCH 23/71] data: add support for lyd_attr to DNode Add new DNodeAttrs class mapped to access the lyd_attr structure in a DNode, and unit tests for it. Fixes: #77 Signed-off-by: nvxf <68589039+nvxf@users.noreply.github.com> Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 51 +++++++++++++++++++++++++++++ libyang/__init__.py | 1 + libyang/data.py | 66 ++++++++++++++++++++++++++++++++++++-- tests/test_data.py | 78 +++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 192 insertions(+), 4 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 78e500a4..b9d3b773 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1042,5 +1042,56 @@ LY_ERR lyd_new_implicit_all(struct lyd_node **, const struct ly_ctx *, uint32_t, LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_meta **); +struct ly_opaq_name { + const char *name; + const char *prefix; + + union { + const char *module_ns; + const char *module_name; + }; +}; + +struct lyd_node_opaq { + union { + struct lyd_node node; + + struct { + uint32_t hash; + uint32_t flags; + const struct lysc_node *schema; + struct lyd_node_inner *parent; + struct lyd_node *next; + struct lyd_node *prev; + struct lyd_meta *meta; + void *priv; + }; + }; + + struct lyd_node *child; + + struct ly_opaq_name name; + const char *value; + uint32_t hints; + LY_VALUE_FORMAT format; + void *val_prefix_data; + + struct lyd_attr *attr; + const struct ly_ctx *ctx; +}; + +struct lyd_attr { + struct lyd_node_opaq *parent; + struct lyd_attr *next; + struct ly_opaq_name name; + const char *value; + uint32_t hints; + LY_VALUE_FORMAT format; + void *val_prefix_data; +}; + +LY_ERR lyd_new_attr(struct lyd_node *, const char *, const char *, const char *, struct lyd_attr **); +void lyd_free_attr_single(const struct ly_ctx *ctx, struct lyd_attr *attr); + /* from libc, needed to free allocated strings */ void free(void *); diff --git a/libyang/__init__.py b/libyang/__init__.py index f9dfeccd..5e3854ad 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -13,6 +13,7 @@ DLeafList, DList, DNode, + DNodeAttrs, DNotif, DRpc, ) diff --git a/libyang/data.py b/libyang/data.py index abc66cd6..f0caf240 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: MIT import logging -from typing import IO, Any, Dict, Iterator, Optional, Union +from typing import IO, Any, Dict, Iterator, Optional, Tuple, Union from _libyang import ffi, lib from .keyed_list import KeyedList @@ -190,13 +190,69 @@ def diff_flags(with_defaults: bool = False) -> int: return flags +# ------------------------------------------------------------------------------------- +class DNodeAttrs: + __slots__ = ("context", "parent", "cdata", "__dict__") + + def __init__(self, context: "libyang.Context", parent: "libyang.DNode"): + self.context = context + self.parent = parent + self.cdata = [] # C type: "struct lyd_attr *" + + def get(self, name: str) -> Optional[str]: + for attr_name, attr_value in self: + if attr_name == name: + return attr_value + return None + + def set(self, name: str, value: str): + attrs = ffi.new("struct lyd_attr **") + ret = lib.lyd_new_attr( + self.parent.cdata, + ffi.NULL, + str2c(name), + str2c(value), + attrs, + ) + if ret != lib.LY_SUCCESS: + raise self.context.error("cannot create attr") + self.cdata.append(attrs[0]) + + def remove(self, name: str): + for attr in self.cdata: + if self._get_attr_name(attr) == name: + lib.lyd_free_attr_single(self.context.cdata, attr) + self.cdata.remove(attr) + break + + def __contains__(self, name: str) -> bool: + for attr_name, _ in self: + if attr_name == name: + return True + return False + + def __iter__(self) -> Iterator[Tuple[str, str]]: + for attr in self.cdata: + name = self._get_attr_name(attr) + yield (name, c2str(attr.value)) + + def __len__(self) -> int: + return len(self.cdata) + + @staticmethod + def _get_attr_name(cdata) -> str: + if cdata.name.prefix != ffi.NULL: + return f"{c2str(cdata.name.prefix)}:{c2str(cdata.name.name)}" + return c2str(cdata.name.name) + + # ------------------------------------------------------------------------------------- class DNode: """ Data tree node. """ - __slots__ = ("context", "cdata", "free_func", "__dict__") + __slots__ = ("context", "cdata", "attributes", "free_func", "__dict__") def __init__(self, context: "libyang.Context", cdata): """ @@ -207,6 +263,7 @@ def __init__(self, context: "libyang.Context", cdata): """ self.context = context self.cdata = cdata # C type: "struct lyd_node *" + self.attributes = None self.free_func = None # type: Callable[DNode] def meta(self): @@ -254,6 +311,11 @@ def new_meta(self, name: str, value: str, clear_dflt: bool = False): if ret != lib.LY_SUCCESS: raise self.context.error("cannot create meta") + def attrs(self) -> DNodeAttrs: + if not self.attributes: + self.attributes = DNodeAttrs(self.context, self) + return self.attributes + def add_defaults( self, no_config: bool = False, diff --git a/tests/test_data.py b/tests/test_data.py index 6a160359..10d9045f 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -16,6 +16,7 @@ DLeaf, DList, DNode, + DNodeAttrs, DNotif, DRpc, IOType, @@ -950,13 +951,86 @@ def test_dnode_insert_sibling(self): self.assertIsInstance(sibling, DLeaf) self.assertEqual(sibling.cdata, dnode2.cdata) - def test_dnode_new_opaq_find_one(self): + def _create_opaq_hostname(self): root = self.ctx.create_data_path(path="/yolo-system:conf") root.new_path( "hostname", None, opt_opaq=True, ) - dnode = root.find_one("/yolo-system:conf/hostname") + return root.find_one("/yolo-system:conf/hostname") + + def test_dnode_new_opaq_find_one(self): + dnode = self._create_opaq_hostname() self.assertIsInstance(dnode, DLeaf) + + def test_dnode_attrs(self): + dnode = self._create_opaq_hostname() + attrs = dnode.attrs() + + self.assertIsInstance(attrs, DNodeAttrs) + + def test_dnode_attrs_set(self): + dnode = self._create_opaq_hostname() + attrs = dnode.attrs() + + self.assertEqual(len(attrs.cdata), 0) + attrs.set("ietf-netconf:operation", "remove") + + self.assertEqual(len(attrs.cdata), 1) + + def test_dnode_attrs_get(self): + dnode = self._create_opaq_hostname() + attrs = dnode.attrs() + + attrs.set("ietf-netconf:operation", "remove") + + value = attrs.get("ietf-netconf:operation") + self.assertEqual(value, "remove") + + def test_dnode_attrs__len(self): + dnode = self._create_opaq_hostname() + attrs = dnode.attrs() + + self.assertEqual(len(attrs), 0) + attrs.set("ietf-netconf:operation", "remove") + + self.assertEqual(len(attrs), 1) + + def test_dnode_attrs__contains(self): + dnode = self._create_opaq_hostname() + attrs = dnode.attrs() + + attrs.set("ietf-netconf:operation", "remove") + + self.assertTrue("ietf-netconf:operation" in attrs) + + def test_dnode_attrs_remove(self): + dnode = self._create_opaq_hostname() + attrs = dnode.attrs() + + attrs.set("ietf-netconf:operation", "remove") + attrs.remove("ietf-netconf:operation") + + self.assertEqual(len(attrs), 0) + + def test_dnode_attrs_set_and_remove_multiple(self): + dnode = self._create_opaq_hostname() + attrs = dnode.attrs() + + attrs.set("ietf-netconf:operation", "remove") + attrs.set("something:else", "test") + attrs.set("no_prefix", "test") + self.assertEqual(len(attrs), 3) + + attrs.remove("something:else") + self.assertEqual(len(attrs), 2) + self.assertIn("no_prefix", attrs) + self.assertIn("ietf-netconf:operation", attrs) + + attrs.remove("no_prefix") + self.assertEqual(len(attrs), 1) + + attrs.remove("ietf-netconf:operation") + self.assertEqual(len(attrs), 0) From 6e94f1bd744af16bfc4dd12218e82f98cd356f08 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Fri, 26 Jan 2024 11:13:49 +0100 Subject: [PATCH 24/71] schema: add default function to SChoice class This patches introduces a default function to SChoice class, including unit test. Fixes: #97 Signed-off-by: Stefan Gula Acked-by: Samuel Gauthier --- cffi/cdefs.h | 13 +++++++++++++ libyang/__init__.py | 4 ++++ libyang/schema.py | 11 +++++++++++ tests/test_schema.py | 19 +++++++++++++++++++ tests/yang/yolo/yolo-system.yang | 1 + 5 files changed, 48 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index b9d3b773..304ae55d 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1023,6 +1023,19 @@ struct lysc_when { struct lysc_when** lysc_node_when(const struct lysc_node *); +struct lysc_node_case { + struct lysc_node *child; + struct lysc_when **when; + ...; +}; + +struct lysc_node_choice { + struct lysc_node_case *cases; + struct lysc_when **when; + struct lysc_node_case *dflt; + ...; +}; + #define LYD_DEFAULT ... #define LYD_WHEN_TRUE ... #define LYD_NEW ... diff --git a/libyang/__init__.py b/libyang/__init__.py index 5e3854ad..bc79e2f0 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -78,6 +78,8 @@ Must, Pattern, Revision, + SCase, + SChoice, SContainer, SLeaf, SLeafList, @@ -156,6 +158,8 @@ "RangeAdded", "RangeRemoved", "Revision", + "SCase", + "SChoice", "SContainer", "SLeaf", "SLeafList", diff --git a/libyang/schema.py b/libyang/schema.py index b257c8f6..d49c39a7 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1416,6 +1416,12 @@ def children( # ------------------------------------------------------------------------------------- @SNode.register(SNode.CHOICE) class SChoice(SNode): + __slots__ = ("cdata_choice",) + + def __init__(self, context: "libyang.Context", cdata): + super().__init__(context, cdata) + self.cdata_choice = ffi.cast("struct lysc_node_choice *", cdata) + def __iter__(self) -> Iterator[SNode]: return self.children() @@ -1424,6 +1430,11 @@ def children( ) -> Iterator[SNode]: return iter_children(self.context, self.cdata, types=types, with_case=with_case) + def default(self) -> Optional[SNode]: + if self.cdata_choice.dflt == ffi.NULL: + return None + return SNode.new(self.context, self.cdata_choice.dflt) + # ------------------------------------------------------------------------------------- @SNode.register(SNode.CASE) diff --git a/tests/test_schema.py b/tests/test_schema.py index 5b947cd8..64a8e30e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -15,6 +15,8 @@ Must, Pattern, Revision, + SCase, + SChoice, SContainer, SLeaf, SLeafList, @@ -584,3 +586,20 @@ def test_leaf_list_min_max(self): self.assertIsInstance(leaflist2, SLeafList) self.assertEqual(leaflist2.min_elements(), 0) self.assertEqual(leaflist2.max_elements(), None) + + +# ------------------------------------------------------------------------------------- +class ChoiceTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.ctx.load_module("yolo-system") + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_choice_default(self): + conf = next(self.ctx.find_path("/yolo-system:conf")) + choice = next(conf.children((SNode.CHOICE,), with_choice=True)) + self.assertIsInstance(choice, SChoice) + self.assertIsInstance(choice.default(), SCase) diff --git a/tests/yang/yolo/yolo-system.yang b/tests/yang/yolo/yolo-system.yang index 5aa633ad..ef612546 100644 --- a/tests/yang/yolo/yolo-system.yang +++ b/tests/yang/yolo/yolo-system.yang @@ -72,6 +72,7 @@ module yolo-system { type boolean; } } + default red; } list url { From 52da1c4ee4e0bb6c6e9865466f40effff1e61deb Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Wed, 7 Feb 2024 13:59:19 +0100 Subject: [PATCH 25/71] Revert "schema: use Pattern class for patterns method" This reverts commit a6c7164b743d83041cb0db5363b0863f9a311be1. It actually breaks compatibility, remove this. Signed-off-by: Samuel Gauthier --- libyang/schema.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index d49c39a7..deec466d 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -681,8 +681,20 @@ def all_lengths(self) -> Iterator[str]: yield length def patterns(self) -> Iterator[Tuple[str, bool]]: - for pattern in self.pattern_details(): - yield pattern.expression(), pattern.inverted() + if not self.cdata_parsed or self.cdata.basetype != self.STRING: + return + if self.cdata_parsed.patterns == ffi.NULL: + return + for p in ly_array_iter(self.cdata_parsed.patterns): + if not p: + continue + # in case of pattern restriction, the first byte has a special meaning: + # 0x06 (ACK) for regular match and 0x15 (NACK) for invert-match + invert_match = p.arg.str[0] == b"\x15" + # yield tuples like: + # ('[a-zA-Z_][a-zA-Z0-9\-_.]*', False) + # ('[xX][mM][lL].*', True) + yield c2str(p.arg.str + 1), invert_match def all_patterns(self) -> Iterator[Tuple[str, bool]]: if self.cdata.basetype == lib.LY_TYPE_UNION: From f09ed1164d860fd9f04c008e42bbac13ccdb2ff2 Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Thu, 1 Feb 2024 16:15:30 +0800 Subject: [PATCH 26/71] Fix typos, Optionnal -> Optional Found via `codespell -H` --- libyang/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libyang/diff.py b/libyang/diff.py index b2a15118..37441f14 100644 --- a/libyang/diff.py +++ b/libyang/diff.py @@ -23,7 +23,7 @@ def schema_diff( :arg ctx_new: The second context. :arg exclude_node_cb: - Optionnal user callback that will be called with each node that is found in each + Optional user callback that will be called with each node that is found in each context. If the callback returns a "trueish" value, the node will be excluded from the diff (as well as all its children). :arg use_data_path: From 83e76f93bfac3b8132f54cc40ab5ab49443b43e3 Mon Sep 17 00:00:00 2001 From: Robin Jarry Date: Wed, 27 Mar 2024 23:41:32 +0100 Subject: [PATCH 27/71] ci: check commit messages in pull requests Unfortunately Github does not allow commenting on commit messages directly. At least perform basic checks to enforce our rules: * titles should be less than 72 characters * titles should start with a short lower case prefix to mention the "topic" of the commit. * no capitalization nor punctuation in the commit title * all commits should have English prose describing the changes. * all commits should be signed-off-by their author (git commit -s). * the list of sanctioned commit trailers is enforced. * referencing github issues/pull_requests must be done via full urls. * referenced commit ids must be valid. Signed-off-by: Robin Jarry --- .github/workflows/ci.yml | 13 ++++ Makefile | 8 ++- README.rst | 71 ++++++++++++++++----- check-commits.sh | 130 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 205 insertions(+), 17 deletions(-) create mode 100755 check-commits.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c308738..fd6ad584 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,19 @@ jobs: - run: python -m pip install --upgrade tox - run: python -m tox -e lint + check-commits: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + env: + LYPY_START_COMMIT: "${{ github.event.pull_request.base.sha }}" + LYPY_END_COMMIT: "${{ github.event.pull_request.head.sha }}" + steps: + - run: sudo apt-get install git make + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - run: make check-commits + test: runs-on: ubuntu-20.04 strategy: diff --git a/Makefile b/Makefile index 90147aaa..6e9c436b 100644 --- a/Makefile +++ b/Makefile @@ -12,4 +12,10 @@ tests: format: tox -e format -.PHONY: lint tests format +LYPY_START_COMMIT ?= origin/master +LYPY_END_COMMIT ?= HEAD + +check-commits: + ./check-commits.sh $(LYPY_START_COMMIT)..$(LYPY_END_COMMIT) + +.PHONY: lint tests format check-commits diff --git a/README.rst b/README.rst index 0387cf57..4ea8977f 100644 --- a/README.rst +++ b/README.rst @@ -232,7 +232,7 @@ Here are the steps for submitting a change in the code base: #. Create a new branch named after what your are working on:: - git checkout -b my-topic + git checkout -b my-topic -t origin/master #. Edit the code and call ``make format`` to ensure your modifications comply with the `coding style`__. @@ -251,21 +251,60 @@ Here are the steps for submitting a change in the code base: your changes do not break anything. You can also run ``make`` which will run both. -#. Create commits by following these simple guidelines: - - - Solve only one problem per commit. - - Use a short (less than 72 characters) title on the first line followed by - an blank line and a more thorough description body. - - Wrap the body of the commit message should be wrapped at 72 characters too - unless it breaks long URLs or code examples. - - If the commit fixes a Github issue, include the following line:: - - Fixes: #NNNN - - Inspirations: - - https://chris.beams.io/posts/git-commit/ - https://wiki.openstack.org/wiki/GitCommitMessages +#. Once you are happy with your work, you can create a commit (or several + commits). Follow these general rules: + + - Address only one issue/topic per commit. + - Describe your changes in imperative mood, e.g. *"make xyzzy do frotz"* + instead of *"[This patch] makes xyzzy do frotz"* or *"[I] changed xyzzy to + do frotz"*, as if you are giving orders to the codebase to change its + behaviour. + - Limit the first line (title) of the commit message to 60 characters. + - Use a short prefix for the commit title for readability with ``git log + --oneline``. Do not use the `fix:` nor `feature:` prefixes. See recent + commits for inspiration. + - Only use lower case letters for the commit title except when quoting + symbols or known acronyms. + - Use the body of the commit message to actually explain what your patch + does and why it is useful. Even if your patch is a one line fix, the + description is not limited in length and may span over multiple + paragraphs. Use proper English syntax, grammar and punctuation. + - If you are fixing an issue, use appropriate ``Closes: `` or + ``Fixes: `` trailers. + - If you are fixing a regression introduced by another commit, add a + ``Fixes: ("")`` trailer. + - When in doubt, follow the format and layout of the recent existing + commits. + - The following trailers are accepted in commits. If you are using multiple + trailers in a commit, it's preferred to also order them according to this + list. + + * ``Closes: <URL>``: close the referenced issue or pull request. + * ``Fixes: <SHA> ("<TITLE>")``: reference the commit that introduced + a regression. + * ``Link: <URL>``: any useful link to provide context for your commit. + * ``Suggested-by`` + * ``Requested-by`` + * ``Reported-by`` + * ``Co-authored-by`` + * ``Tested-by`` + * ``Reviewed-by`` + * ``Acked-by`` + * ``Signed-off-by``: Compulsory! + + There is a great reference for commit messages in the `Linux kernel + documentation`__. + + __ https://www.kernel.org/doc/html/latest/process/submitting-patches.html#describe-your-changes + + IMPORTANT: you must sign-off your work using ``git commit --signoff``. Follow + the `Linux kernel developer's certificate of origin`__ for more details. All + contributions are made under the MIT license. If you do not want to disclose + your real name, you may sign-off using a pseudonym. Here is an example:: + + Signed-off-by: Robin Jarry <robin@jarry.cc> + + __ https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin #. Push your topic branch in your forked repository:: diff --git a/check-commits.sh b/check-commits.sh new file mode 100755 index 00000000..acf322d9 --- /dev/null +++ b/check-commits.sh @@ -0,0 +1,130 @@ +#!/bin/sh + +set -e + +revision_range="${1?revision range}" + +valid=0 +revisions=$(git rev-list --reverse "$revision_range") +total=$(echo $revisions | wc -w) +if [ "$total" -eq 0 ]; then + exit 0 +fi +tmp=$(mktemp) +trap "rm -f $tmp" EXIT + +allowed_trailers=" +Closes +Fixes +Link +Suggested-by +Requested-by +Reported-by +Co-authored-by +Signed-off-by +Tested-by +Reviewed-by +Acked-by +" + +n=0 +title= +shortrev= +fail=false +repo=CESNET/libyang-python +repo_url=https://github.com/$repo +api_url=https://api.github.com/repos/$repo + +err() { + + echo "error: commit $shortrev (\"$title\") $*" >&2 + fail=true +} + +check_issue() { + json=$(curl -f -X GET -L --no-progress-meter \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "$api_url/issues/${1##*/}") || return 1 + test $(echo "$json" | jq -r .state) = open +} + +for rev in $revisions; do + n=$((n + 1)) + title=$(git log --format='%s' -1 "$rev") + fail=false + shortrev=$(printf '%-12.12s' $rev) + + if [ "$(echo "$title" | wc -m)" -gt 72 ]; then + err "title is longer than 72 characters, please make it shorter" + fi + if ! echo "$title" | grep -qE '^[a-z0-9,{}/_-]+: '; then + err "title lacks a lowercase topic prefix (e.g. 'data: ')" + fi + if echo "$title" | grep -qE '^[a-z0-9,{}/_-]+: [A-Z][a-z]'; then + err "title starts with an capital letter, please use lower case" + fi + if ! echo "$title" | grep -qE '[A-Za-z0-9]$'; then + err "title ends with punctuation, please remove it" + fi + + author=$(git log --format='%an <%ae>' -1 "$rev") + if ! git log --format="%(trailers:key=Signed-off-by,only,valueonly,unfold)" -1 "$rev" | + grep -qFx "$author"; then + err "'Signed-off-by: $author' trailer is missing" + fi + + for trailer in $(git log --format="%(trailers:only,keyonly)" -1 "$rev"); do + if ! echo "$allowed_trailers" | grep -qFx "$trailer"; then + err "trailer '$trailer' is misspelled or not in the sanctioned list" + fi + done + + git log --format="%(trailers:key=Closes,only,valueonly,unfold)" -1 "$rev" > $tmp + while read -r value; do + if [ -z "$value" ]; then + continue + fi + case "$value" in + $repo_url/*/[0-9]*) + if ! check_issue "$value"; then + err "'$value' does not reference a valid open issue" + fi + ;; + \#[0-9]*) + err "please use the full issue URL: 'Closes: $repo_url/issues/$value'" + ;; + *) + err "invalid trailer value '$value'. The 'Closes:' trailer must only be used to reference issue URLs" + ;; + esac + done < "$tmp" + + git log --format="%(trailers:key=Fixes,only,valueonly,unfold)" -1 "$rev" > $tmp + while read -r value; do + if [ -z "$value" ]; then + continue + fi + fixes_rev=$(echo "$value" | sed -En 's/([A-Fa-f0-9]{7,}[[:space:]]\(".*"\))/\1/p') + if ! git cat-file commit "$fixes_rev" >/dev/null; then + err "trailer '$value' does not refer to a known commit" + fi + done < "$tmp" + + body=$(git log --format='%b' -1 "$rev") + body=${body%$(git log --format='%(trailers)' -1 "$rev")} + if [ "$(echo "$body" | wc -w)" -lt 3 ]; then + err "body has less than three words, please describe your changes" + fi + + if [ "$fail" = true ]; then + continue + fi + echo "ok commit $shortrev (\"$title\")" + valid=$((valid + 1)) +done + +echo "$valid/$total valid commit messages" +if [ "$valid" -ne "$total" ]; then + exit 1 +fi From 2e6261aab6c530d50c32b8f6090db25851e48980 Mon Sep 17 00:00:00 2001 From: Robin Jarry <robin@jarry.cc> Date: Thu, 28 Mar 2024 00:30:45 +0100 Subject: [PATCH 28/71] editorconfig: fix bad syntax The proper indent_style value is "tab" not "tabs". Signed-off-by: Robin Jarry <robin@jarry.cc> --- .editorconfig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.editorconfig b/.editorconfig index b5343938..37f96c4f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -21,13 +21,13 @@ indent_style = space indent_size = 3 [Makefile] -indent_style = tabs +indent_style = tab indent_size = tab [*.sh] -indent_style = tabs +indent_style = tab indent_size = tab [*.{h,c}] -indent_style = tabs +indent_style = tab indent_size = tab From 04b1c81c8057120f2075ab82baae2437fa141cac Mon Sep 17 00:00:00 2001 From: Robin Jarry <robin@jarry.cc> Date: Thu, 28 Mar 2024 12:43:56 +0100 Subject: [PATCH 29/71] ci: disable libyang devel check As libyang 3.x is under development, the devel branch is currently not compatible with the master. Let's disable this check for the meantime until we revert back to a more stable state. Signed-off-by: Robin Jarry <robin@jarry.cc> --- .github/workflows/ci.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd6ad584..144e73b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,22 +70,6 @@ jobs: - run: python -m pip install --upgrade tox - run: python -m tox -e ${{ matrix.toxenv }} - libyang_devel: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: 3.x - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: pip - restore-keys: pip - - run: python -m pip install --upgrade pip setuptools wheel - - run: python -m pip install --upgrade tox - - run: python -m tox -e lydevel - coverage: runs-on: ubuntu-latest steps: From 92b525085b9ce3d1e9d57795624a23f8f1c5ce97 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Sat, 6 Apr 2024 10:42:24 +0200 Subject: [PATCH 30/71] data: remove no_parent_ret flag This flag should have been removed when we ported to libyang 2. It can safely be removed. Although it is an API change, it seems acceptable as the flag was doing nothing. Fixes: 806be4c3bc4f ("Port to libyang 2") Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/context.py | 5 +---- libyang/data.py | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/libyang/context.py b/libyang/context.py index eefc4e05..f598a225 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -234,7 +234,6 @@ def create_data_path( parent: Optional[DNode] = None, value: Any = None, update: bool = True, - no_parent_ret: bool = True, rpc_output: bool = False, force_return_value: bool = True, ) -> Optional[DNode]: @@ -245,9 +244,7 @@ def create_data_path( value = str(value).lower() elif not isinstance(value, str): value = str(value) - flags = path_flags( - update=update, no_parent_ret=no_parent_ret, rpc_output=rpc_output - ) + flags = path_flags(update=update, rpc_output=rpc_output) dnode = ffi.new("struct lyd_node **") ret = lib.lyd_new_path( parent.cdata if parent else ffi.NULL, diff --git a/libyang/data.py b/libyang/data.py index f0caf240..1c1dfe35 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -77,9 +77,7 @@ def data_format(fmt_string: str) -> int: # ------------------------------------------------------------------------------------- -def path_flags( - update: bool = False, rpc_output: bool = False, no_parent_ret: bool = False -) -> int: +def path_flags(update: bool = False, rpc_output: bool = False) -> int: flags = 0 if update: flags |= lib.LYD_NEW_PATH_UPDATE From ae31525dedc750394f2bd8bb70d9218d191f32f0 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Mon, 12 Feb 2024 09:31:05 +0100 Subject: [PATCH 31/71] Port to libyang 3 Refactor the code to switch to libyang 3: - the cdefs.h file is updated to match the new definitions - the flags used in lyd_new_* are regrouped in a newvaloptions bitmap - the log callback management is reworked - the system-ordered lists / leaf-lists are now ordered by key (hence the unit test change) Fixes: https://github.com/CESNET/libyang-python/pull/105 Link: https://github.com/CESNET/libyang/blob/master/doc/transition_2_3.dox Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 43 +++++++++++++++++++------------ cffi/source.c | 7 ++--- libyang/context.py | 15 +++++++---- libyang/data.py | 64 +++++++++++++++++++++++++++++++--------------- libyang/log.py | 21 +++++++++------ tests/test_data.py | 16 +++++++----- 6 files changed, 105 insertions(+), 61 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 304ae55d..e5bcc935 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -179,10 +179,10 @@ int ly_log_options(int); LY_LOG_LEVEL ly_log_level(LY_LOG_LEVEL); extern "Python" void lypy_log_cb(LY_LOG_LEVEL, const char *, const char *); -void ly_set_log_clb(void (*)(LY_LOG_LEVEL, const char *, const char *), int); -struct ly_err_item *ly_err_first(const struct ly_ctx *); +void ly_set_log_clb(void (*)(LY_LOG_LEVEL, const char *, const char *, const char *, uint64_t)); +const struct ly_err_item *ly_err_first(const struct ly_ctx *); +const struct ly_err_item *ly_err_last(const struct ly_ctx *); void ly_err_clean(struct ly_ctx *, struct ly_err_item *); -LY_VECODE ly_vecode(const struct ly_ctx *); #define LYS_UNKNOWN ... #define LYS_CONTAINER ... @@ -238,14 +238,15 @@ struct lysc_node { struct ly_err_item { LY_LOG_LEVEL level; - LY_ERR no; + LY_ERR err; LY_VECODE vecode; char *msg; - char *path; + char *data_path; + char *schema_path; + uint64_t line; char *apptag; struct ly_err_item *next; struct ly_err_item *prev; - ...; }; struct lyd_node { @@ -261,11 +262,12 @@ struct lyd_node { LY_ERR lys_set_implemented(struct lys_module *, const char **); +#define LYD_NEW_VAL_OUTPUT ... +#define LYD_NEW_VAL_BIN ... +#define LYD_NEW_VAL_CANON ... +#define LYD_NEW_META_CLEAR_DFLT ... #define LYD_NEW_PATH_UPDATE ... -#define LYD_NEW_PATH_OUTPUT ... -#define LYD_NEW_PATH_OPAQ ... -#define LYD_NEW_PATH_BIN_VALUE ... -#define LYD_NEW_PATH_CANON_VALUE ... +#define LYD_NEW_PATH_OPAQ ... LY_ERR lyd_new_path(struct lyd_node *, const struct ly_ctx *, const char *, const char *, uint32_t, struct lyd_node **); LY_ERR lyd_find_xpath(const struct lyd_node *, const char *, struct ly_set **); void lyd_unlink_siblings(struct lyd_node *node); @@ -614,6 +616,7 @@ struct lysp_node_list { }; struct lysc_type { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -641,6 +644,7 @@ struct lysp_type { struct lysp_qname { const char *str; const struct lysp_module *mod; + ...; }; struct lysp_node { @@ -682,7 +686,6 @@ struct lysc_ext { struct lysc_ext_instance *exts; struct lyplg_ext *plugin; struct lys_module *module; - uint32_t refcount; uint16_t flags; }; @@ -703,11 +706,10 @@ typedef enum { LYD_PATH_STD_NO_LAST_PRED } LYD_PATH_TYPE; -LY_ERR lyd_new_term(struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_node **); +LY_ERR lyd_new_term(struct lyd_node *, const struct lys_module *, const char *, const char *, uint32_t, struct lyd_node **); char* lyd_path(const struct lyd_node *, LYD_PATH_TYPE, char *, size_t); LY_ERR lyd_new_inner(struct lyd_node *, const struct lys_module *, const char *, ly_bool, struct lyd_node **); -LY_ERR lyd_new_list(struct lyd_node *, const struct lys_module *, const char *, ly_bool, struct lyd_node **, ...); -LY_ERR lyd_new_list2(struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_node **); +LY_ERR lyd_new_list(struct lyd_node *, const struct lys_module *, const char *, uint32_t, struct lyd_node **node, ...); struct lyd_node_inner { union { @@ -821,6 +823,7 @@ struct lysp_restr { }; struct lysc_type_num { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -829,6 +832,7 @@ struct lysc_type_num { }; struct lysc_type_dec { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -838,6 +842,7 @@ struct lysc_type_dec { }; struct lysc_type_str { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -859,6 +864,7 @@ struct lysc_type_bitenum_item { }; struct lysc_type_enum { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -867,6 +873,7 @@ struct lysc_type_enum { }; struct lysc_type_bits { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -875,18 +882,19 @@ struct lysc_type_bits { }; struct lysc_type_leafref { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; uint32_t refcount; struct lyxp_expr *path; struct lysc_prefix *prefixes; - const struct lys_module *cur_mod; struct lysc_type *realtype; uint8_t require_instance; }; struct lysc_type_identityref { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -895,6 +903,7 @@ struct lysc_type_identityref { }; struct lysc_type_instanceid { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -903,6 +912,7 @@ struct lysc_type_instanceid { }; struct lysc_type_union { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -911,6 +921,7 @@ struct lysc_type_union { }; struct lysc_type_bin { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -1053,7 +1064,7 @@ LY_ERR lyd_merge_module(struct lyd_node **, const struct lyd_node *, const struc LY_ERR lyd_new_implicit_tree(struct lyd_node *, uint32_t, struct lyd_node **); LY_ERR lyd_new_implicit_all(struct lyd_node **, const struct ly_ctx *, uint32_t, struct lyd_node **); -LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_meta **); +LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, uint32_t, struct lyd_meta **); struct ly_opaq_name { const char *name; diff --git a/cffi/source.c b/cffi/source.c index 2682dd88..b54ba0de 100644 --- a/cffi/source.c +++ b/cffi/source.c @@ -6,9 +6,6 @@ #include <libyang/libyang.h> #include <libyang/version.h> -#if (LY_VERSION_MAJOR != 2) -#error "This version of libyang bindings only works with libyang 2.x" -#endif -#if (LY_VERSION_MINOR < 37) -#error "Need at least libyang 2.37" +#if (LY_VERSION_MAJOR != 3) +#error "This version of libyang bindings only works with libyang 3.x" #endif diff --git a/libyang/context.py b/libyang/context.py index f598a225..fa5eb5bf 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -11,8 +11,8 @@ DNode, data_format, data_type, + newval_flags, parser_flags, - path_flags, validation_flags, ) from .schema import Module, SNode, schema_in_format @@ -117,8 +117,12 @@ def error(self, msg: str, *args) -> LibyangError: while err: if err.msg: msg += ": %s" % c2str(err.msg) - if err.path: - msg += ": %s" % c2str(err.path) + if err.data_path: + msg += ": Data path: %s" % c2str(err.data_path) + if err.schema_path: + msg += ": Schema path: %s" % c2str(err.schema_path) + if err.line != 0: + msg += " (line %u)" % err.line err = err.next lib.ly_err_clean(self.cdata, ffi.NULL) @@ -244,7 +248,7 @@ def create_data_path( value = str(value).lower() elif not isinstance(value, str): value = str(value) - flags = path_flags(update=update, rpc_output=rpc_output) + flags = newval_flags(update=update, rpc_output=rpc_output) dnode = ffi.new("struct lyd_node **") ret = lib.lyd_new_path( parent.cdata if parent else ffi.NULL, @@ -256,7 +260,8 @@ def create_data_path( ) dnode = dnode[0] if ret != lib.LY_SUCCESS: - if lib.ly_vecode(self.cdata) != lib.LYVE_SUCCESS: + err = lib.ly_err_last(self.cdata) + if err != ffi.NULL and err.vecode != lib.LYVE_SUCCESS: raise self.error("cannot create data path: %s", path) lib.ly_err_clean(self.cdata, ffi.NULL) if not dnode and not force_return_value: diff --git a/libyang/data.py b/libyang/data.py index 1c1dfe35..2895daaf 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -77,12 +77,30 @@ def data_format(fmt_string: str) -> int: # ------------------------------------------------------------------------------------- -def path_flags(update: bool = False, rpc_output: bool = False) -> int: +def newval_flags( + rpc_output: bool = False, + bin_value: bool = False, + canon_value: bool = False, + meta_clear_default: bool = False, + update: bool = False, + opaq: bool = False, +) -> int: + """ + Translate from booleans to newvaloptions flags. + """ flags = 0 + if rpc_output: + flags |= lib.LYD_NEW_VAL_OUTPUT + if bin_value: + flags |= lib.LYD_NEW_VAL_BIN + if canon_value: + flags |= lib.LYD_NEW_VAL_CANON + if meta_clear_default: + flags |= lib.LYD_NEW_META_CLEAR_DFLT if update: flags |= lib.LYD_NEW_PATH_UPDATE - if rpc_output: - flags |= lib.LYD_NEW_PATH_OUTPUT + if opaq: + flags |= lib.LYD_NEW_PATH_OPAQ return flags @@ -297,13 +315,14 @@ def meta_free(self, name): item = item.next def new_meta(self, name: str, value: str, clear_dflt: bool = False): + flags = newval_flags(meta_clear_default=clear_dflt) ret = lib.lyd_new_meta( ffi.NULL, self.cdata, ffi.NULL, str2c(name), str2c(value), - clear_dflt, + flags, ffi.NULL, ) if ret != lib.LY_SUCCESS: @@ -364,20 +383,15 @@ def new_path( opt_bin_value: bool = False, opt_canon_value: bool = False, ): - opt = 0 - if opt_update: - opt |= lib.LYD_NEW_PATH_UPDATE - if opt_output: - opt |= lib.LYD_NEW_PATH_OUTPUT - if opt_opaq: - opt |= lib.LYD_NEW_PATH_OPAQ - if opt_bin_value: - opt |= lib.LYD_NEW_PATH_BIN_VALUE - if opt_canon_value: - opt |= lib.LYD_NEW_PATH_CANON_VALUE - + flags = newval_flags( + update=opt_update, + rpc_output=opt_output, + opaq=opt_opaq, + bin_value=opt_bin_value, + canon_value=opt_canon_value, + ) ret = lib.lyd_new_path( - self.cdata, ffi.NULL, str2c(path), str2c(value), opt, ffi.NULL + self.cdata, ffi.NULL, str2c(path), str2c(value), flags, ffi.NULL ) if ret != lib.LY_SUCCESS: raise self.context.error("cannot get module") @@ -1003,7 +1017,10 @@ def _get_path(cdata) -> str: @DNode.register(SNode.CONTAINER) class DContainer(DNode): def create_path( - self, path: str, value: Any = None, rpc_output: bool = False + self, + path: str, + value: Any = None, + rpc_output: bool = False, ) -> Optional[DNode]: return self.context.create_data_path( path, parent=self, value=value, rpc_output=rpc_output @@ -1177,8 +1194,14 @@ def _create_leaf(_parent, module, name, value, in_rpc_output=False): value = str(value) n = ffi.new("struct lyd_node **") + flags = newval_flags(rpc_output=in_rpc_output) ret = lib.lyd_new_term( - _parent, module.cdata, str2c(name), str2c(value), in_rpc_output, n + _parent, + module.cdata, + str2c(name), + str2c(value), + flags, + n, ) if ret != lib.LY_SUCCESS: @@ -1209,11 +1232,12 @@ def _create_container(_parent, module, name, in_rpc_output=False): def _create_list(_parent, module, name, key_values, in_rpc_output=False): n = ffi.new("struct lyd_node **") + flags = newval_flags(rpc_output=in_rpc_output) ret = lib.lyd_new_list( _parent, module.cdata, str2c(name), - in_rpc_output, + flags, n, *[str2c(str(i)) for i in key_values], ) diff --git a/libyang/log.py b/libyang/log.py index 2b241157..b033ccaa 100644 --- a/libyang/log.py +++ b/libyang/log.py @@ -20,13 +20,18 @@ @ffi.def_extern(name="lypy_log_cb") -def libyang_c_logging_callback(level, msg, path): +def libyang_c_logging_callback(level, msg, data_path, schema_path, line): args = [c2str(msg)] - if path: - fmt = "%s: %s" - args.append(c2str(path)) - else: - fmt = "%s" + fmt = "%s" + if data_path: + fmt += ": %s" + args.append(c2str(data_path)) + if schema_path: + fmt += ": %s" + args.append(c2str(schema_path)) + if line != 0: + fmt += " line %u" + args.append(str(line)) LOG.log(LOG_LEVELS.get(level, logging.NOTSET), fmt, *args) @@ -51,10 +56,10 @@ def configure_logging(enable_py_logger: bool, level: int = logging.ERROR) -> Non break if enable_py_logger: lib.ly_log_options(lib.LY_LOLOG | lib.LY_LOSTORE) - lib.ly_set_log_clb(lib.lypy_log_cb, True) + lib.ly_set_log_clb(lib.lypy_log_cb) else: lib.ly_log_options(lib.LY_LOSTORE) - lib.ly_set_log_clb(ffi.NULL, False) + lib.ly_set_log_clb(ffi.NULL) configure_logging(False, logging.ERROR) diff --git a/tests/test_data.py b/tests/test_data.py index 10d9045f..a40056c0 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -132,17 +132,17 @@ def test_data_parse_config_json_without_yang_lib(self): "path": "/CESNET/libyang-python", "enabled": false }, + { + "proto": "http", + "host": "barfoo.com", + "path": "/barfoo/index.html" + }, { "proto": "http", "host": "foobar.com", "port": 8080, "path": "/index.html", "enabled": true - }, - { - "proto": "http", - "host": "barfoo.com", - "path": "/barfoo/index.html" } ], "number": [ @@ -282,7 +282,9 @@ def test_data_parse_config_xml_multi_error(self): self.assertEqual( str(cm.exception), 'failed to parse data tree: Invalid boolean value "abcd".: ' - 'List instance is missing its key "host".', + "Data path: /yolo-system:conf/url[proto='https']/enabled (line 6): " + 'List instance is missing its key "host".: ' + "Data path: /yolo-system:conf/url[proto='https'] (line 7)", ) XML_STATE = """<state xmlns="urn:yang:yolo:system"> @@ -808,7 +810,7 @@ def test_data_to_dict_keyless_list(self): <host>foobar.com</host> <enabled yang:operation="replace" yang:orig-default="false" yang:orig-value="true">false</enabled> </url> - <url yang:operation="create"> + <url yang:operation="create" yang:key="[proto='http'][host='foobar.com']"> <proto>ftp</proto> <host>github.com</host> <path>/CESNET/libyang-python</path> From e602016200ff3db1c8cde4886dfc1fa8a828ecf8 Mon Sep 17 00:00:00 2001 From: Jeremie Leska <jeremie.leska@6wind.com> Date: Tue, 16 Apr 2024 13:49:17 +0200 Subject: [PATCH 32/71] cdefs: fix lypy_log_cb prototype The following error is raised when logging: > File "/usr/lib/python3/dist-packages/libyang/log.py", line 59, in configure_logging > TypeError: initializer for ctype 'void(*)(LY_LOG_LEVEL, char *, char *, char *, > uint64_t)' must be a pointer to same type, not cdata 'void(*)(LY_LOG_LEVEL, char *, char > *)' Fix lypy_log_cb's prototype. Fixes: 3849dd523d30 ("Port to libyang3") Signed-off-by: Jeremie Leska <jeremie.leska@6wind.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index e5bcc935..921dd8f6 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -178,7 +178,7 @@ enum ly_stmt { int ly_log_options(int); LY_LOG_LEVEL ly_log_level(LY_LOG_LEVEL); -extern "Python" void lypy_log_cb(LY_LOG_LEVEL, const char *, const char *); +extern "Python" void lypy_log_cb(LY_LOG_LEVEL, const char *, const char *, const char *, uint64_t); void ly_set_log_clb(void (*)(LY_LOG_LEVEL, const char *, const char *, const char *, uint64_t)); const struct ly_err_item *ly_err_first(const struct ly_ctx *); const struct ly_err_item *ly_err_last(const struct ly_ctx *); From c3305ab946c951aaa01b59244e9a69e3894a9e9e Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Mon, 10 Jun 2024 12:24:20 +0200 Subject: [PATCH 33/71] schema: fix empty compiled shema in iter_children A module schema can have no compiled schema. In this case, lys_getnext returns an error (module and parent being NULL), leading to this log "Invalid argument parent || module || ext (lys_getnext_()).". Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/schema.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libyang/schema.py b/libyang/schema.py index deec466d..fb0787ff 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1647,6 +1647,8 @@ def _skip(node) -> bool: return False if ffi.typeof(parent) == ffi.typeof("struct lys_module *"): + if parent.compiled == ffi.NULL: + return module = parent.compiled parent = ffi.NULL else: From 4a9b85e18ef759469179d573248e3767d998554c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-S=C3=A9bastien=20Bevilacqua?= <jean-sebastien.bevilacqua@6wind.com> Date: Tue, 19 Mar 2024 10:00:28 +0100 Subject: [PATCH 34/71] xpath: add support for indexed xpath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch adds support for indexed xpath to xpath_set. In the case of a leaf-list, it is now possible to apply xpath_set in this way: xpath_set(d, "/lstnum[5]", 33, after="4") This will replace the 5th element of the leaf-list with the new value 33. This feature is important because libyang can return this type of xpath when a callback is called with sr_module_change_subscribe from sysrepo. The tests are updated accordingly. Signed-off-by: Jean-Sébastien Bevilacqua <jean-sebastien.bevilacqua@6wind.com> --- libyang/xpath.py | 105 +++++++++++++++++++++++++++++--------------- tests/test_xpath.py | 7 ++- 2 files changed, 75 insertions(+), 37 deletions(-) diff --git a/libyang/xpath.py b/libyang/xpath.py index facaa8b7..ad5f2a58 100644 --- a/libyang/xpath.py +++ b/libyang/xpath.py @@ -1,6 +1,7 @@ # Copyright (c) 2020 6WIND S.A. # SPDX-License-Identifier: MIT +import contextlib import fnmatch import re from typing import Any, Dict, Iterator, List, Optional, Tuple, Union @@ -56,17 +57,28 @@ def xpath_split(xpath: str) -> Iterator[Tuple[str, str, List[Tuple[str, str]]]]: while i < len(xpath) and xpath[i] == "[": i += 1 # skip opening '[' j = xpath.find("=", i) # find key name end - key_name = xpath[i:j] - quote = xpath[j + 1] # record opening quote character - j = i = j + 2 # skip '=' and opening quote - while True: - if xpath[j] == quote and xpath[j - 1] != "\\": - break - j += 1 - # replace escaped chars by their non-escape version - key_value = xpath[i:j].replace(f"\\{quote}", f"{quote}") - keys.append((key_name, key_value)) - i = j + 2 # skip closing quote and ']' + + if j != -1: # keyed specifier + key_name = xpath[i:j] + quote = xpath[j + 1] # record opening quote character + j = i = j + 2 # skip '=' and opening quote + while True: + if xpath[j] == quote and xpath[j - 1] != "\\": + break + j += 1 + # replace escaped chars by their non-escape version + key_value = xpath[i:j].replace(f"\\{quote}", f"{quote}") + keys.append((key_name, key_value)) + i = j + 2 # skip closing quote and ']' + else: # index specifier + j = i + while True: + if xpath[j] == "]": + break + j += 1 + key_value = xpath[i:j] + keys.append(("", key_value)) + i = j + 2 yield prefix, name, keys @@ -134,6 +146,12 @@ def _list_find_key_index(keys: List[Tuple[str, str]], lst: List) -> int: if py_to_yang(elem) == keys[0][1]: return i + elif keys[0][0] == "": + # keys[0][1] is directly the index + index = int(keys[0][1]) - 1 + if len(lst) > index: + return index + else: for i, elem in enumerate(lst): if not isinstance(elem, dict): @@ -410,32 +428,47 @@ def xpath_set( lst.append(value) return lst[key_val] - if isinstance(lst, list): - # regular python list, need to iterate over it - try: - i = _list_find_key_index(keys, lst) - # found - if force: - lst[i] = value - return lst[i] - except ValueError: - # not found - if after is None: - lst.append(value) - elif after == "": - lst.insert(0, value) - else: - if after[0] != "[": - after = "[.=%r]" % str(after) - _, _, after_keys = next(xpath_split("/*" + after)) - insert_index = _list_find_key_index(after_keys, lst) + 1 - if insert_index == len(lst): - lst.append(value) - else: - lst.insert(insert_index, value) - return value + # regular python list from now + if not isinstance(lst, list): + raise TypeError("expected a list") + + with contextlib.suppress(ValueError): + i = _list_find_key_index(keys, lst) + # found + if force: + lst[i] = value + return lst[i] + + # value not found; handle insertion based on 'after' + if after is None: + lst.append(value) + return value + + if after == "": + lst.insert(0, value) + return value + + # first try to find the value in the leaf list + try: + _, _, after_keys = next( + xpath_split(f"/*{after}" if after[0] == "[" else f"/*[.={after!r}]") + ) + insert_index = _list_find_key_index(after_keys, lst) + 1 + except ValueError: + # handle 'after' as numeric index + if not after.isnumeric(): + raise + + insert_index = int(after) + if insert_index > len(lst): + raise + + if insert_index == len(lst): + lst.append(value) + else: + lst.insert(insert_index, value) - raise TypeError("expected a list") + return value # ------------------------------------------------------------------------------------- diff --git a/tests/test_xpath.py b/tests/test_xpath.py index bf7c7bc7..0901a7ec 100644 --- a/tests/test_xpath.py +++ b/tests/test_xpath.py @@ -41,6 +41,11 @@ def test_xpath_set(self): ) ly.xpath_set(d, "/lstnum[.='100']", 100) ly.xpath_set(d, "/lstnum[.='1']", 1, after="") + ly.xpath_set(d, "/lstnum[5]", 33, after="4") + ly.xpath_set(d, "/lstnum[5]", 34, after="4") + ly.xpath_set(d, "/lstnum[5]", 35, after="4") + ly.xpath_set(d, "/lstnum[7]", 101, after="6") + ly.xpath_set(d, "/lstnum[8]", 102, after="7") with self.assertRaises(ValueError): ly.xpath_set(d, "/lstnum[.='1000']", 1000, after="1000000") with self.assertRaises(ValueError): @@ -101,7 +106,7 @@ def test_xpath_set(self): {"name": "eth3", "mtu": 1000}, ], "lst2": ["a", "b", "c"], - "lstnum": [1, 10, 20, 30, 40, 100], + "lstnum": [1, 10, 20, 30, 35, 100, 101, 102], "val": 43, }, ) From 766ef3c24534290207f701ae8e0109657370cfbe Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 13:35:34 +0200 Subject: [PATCH 35/71] libyang: export SAnydata / SAnyxml Those two classes are missing, export them. Closes: https://github.com/CESNET/libyang-python/pull/102 Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libyang/__init__.py b/libyang/__init__.py index bc79e2f0..02bd26af 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -78,6 +78,8 @@ Must, Pattern, Revision, + SAnydata, + SAnyxml, SCase, SChoice, SContainer, @@ -158,6 +160,8 @@ "RangeAdded", "RangeRemoved", "Revision", + "SAnydata", + "SAnyxml", "SCase", "SChoice", "SContainer", From 3fc55b05cec8c32bc658289a0e34859313248f41 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 13:50:57 +0200 Subject: [PATCH 36/71] schema: fix type for max_elements Those functions can return None, fix their expected type. Closes: https://github.com/CESNET/libyang-python/pull/102 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index fb0787ff..c0985a4b 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1382,7 +1382,7 @@ def defaults(self) -> Iterator[Union[None, bool, int, str, float]]: else: yield val - def max_elements(self) -> int: + def max_elements(self) -> Optional[int]: return ( self.cdata_leaflist.max if self.cdata_leaflist.max != (2**32 - 1) @@ -1506,7 +1506,7 @@ def uniques(self) -> Iterator[List[SNode]]: nodes.append(SNode.new(self.context, node)) yield nodes - def max_elements(self) -> int: + def max_elements(self) -> Optional[int]: return self.cdata_list.max if self.cdata_list.max != (2**32 - 1) else None def min_elements(self) -> int: From fe0b3d0528296390b335d9d01b83f49b9a096008 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 13:51:21 +0200 Subject: [PATCH 37/71] schema: fix comment in Import class The cdata type that is actually lysp_import, fix it. Closes: https://github.com/CESNET/libyang-python/pull/102 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libyang/schema.py b/libyang/schema.py index c0985a4b..75206bd3 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -300,7 +300,7 @@ class Import: def __init__(self, context: "libyang.Context", cdata, module): self.context = context - self.cdata = cdata # C type: "struct lysp_revision *" + self.cdata = cdata # C type: "struct lysp_import *" self.module = module def name(self) -> str: From 5f05203e9c0c005cc4ae243df891ac46509a44b3 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 14:28:33 +0200 Subject: [PATCH 38/71] util: add ly_list_iter Add a new function to go through a libyang list, equivalent to LY_LIST_FOR. Closes: https://github.com/CESNET/libyang-python/pull/102 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/util.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libyang/util.py b/libyang/util.py index 9554356e..d640a511 100644 --- a/libyang/util.py +++ b/libyang/util.py @@ -59,6 +59,14 @@ def ly_array_iter(cdata): yield cdata[i] +# ------------------------------------------------------------------------------------- +def ly_list_iter(cdata): + item = cdata + while item != ffi.NULL: + yield item + item = item.next + + # ------------------------------------------------------------------------------------- class IOType(enum.Enum): FD = enum.auto() From b7adea400520477f005a5b8c9857b9daf574de4b Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Fri, 1 Mar 2024 13:42:03 +0100 Subject: [PATCH 39/71] schema: adds PNode and its exact variants This patch introduces PNode alternative to SNode, which allows to traverse parsed module tree including groupings, uses and other special nodes. Closes: https://github.com/CESNET/libyang-python/pull/102 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 124 +++++ libyang/__init__.py | 34 ++ libyang/schema.py | 746 +++++++++++++++++++++++++++- tests/test_schema.py | 252 ++++++++++ tests/yang/yolo/yolo-nodetypes.yang | 31 ++ 5 files changed, 1181 insertions(+), 6 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 921dd8f6..cbe92969 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -528,6 +528,25 @@ typedef enum { char* lysc_path(const struct lysc_node *, LYSC_PATH_TYPE, char *, size_t); +struct lysp_when { + const char *cond; + ...; +}; + +struct lysp_refine { + const char *nodeid; + const char *dsc; + const char *ref; + struct lysp_qname *iffeatures; + struct lysp_restr *musts; + const char *presence; + struct lysp_qname *dflts; + uint32_t min; + uint32_t max; + struct lysp_ext_instance *exts; + uint16_t flags; +}; + struct lysp_node_container { struct lysp_restr *musts; struct lysp_when *when; @@ -615,6 +634,101 @@ struct lysp_node_list { ...; }; +struct lysp_node_choice { + struct lysp_node *child; + struct lysp_when *when; + struct lysp_qname dflt; + ...; +}; + +struct lysp_node_case { + struct lysp_node *child; + struct lysp_when *when; + ...; +}; + +struct lysp_node_anydata { + struct lysp_restr *musts; + struct lysp_when *when; + ...; +}; + +struct lysp_node_uses { + struct lysp_refine *refines; + struct lysp_node_augment *augments; + struct lysp_when *when; + ...; +}; + +struct lysp_node_action_inout { + struct lysp_restr *musts; + struct lysp_tpdf *typedefs; + struct lysp_node_grp *groupings; + struct lysp_node *child; + ...; +}; + +struct lysp_node_action { + union { + struct lysp_node node; + struct { + struct lysp_node_action *next; + ...; + }; + }; + struct lysp_tpdf *typedefs; + struct lysp_node_grp *groupings; + struct lysp_node_action_inout input; + struct lysp_node_action_inout output; + ...; +}; + +struct lysp_node_notif { + union { + struct lysp_node node; + struct { + struct lysp_node_notif *next; + ...; + }; + }; + struct lysp_restr *musts; + struct lysp_tpdf *typedefs; + struct lysp_node_grp *groupings; + struct lysp_node *child; + ...; +}; + +struct lysp_node_grp { + union { + struct lysp_node node; + struct { + struct lysp_node_grp *next; + ...; + }; + }; + struct lysp_tpdf *typedefs; + struct lysp_node_grp *groupings; + struct lysp_node *child; + struct lysp_node_action *actions; + struct lysp_node_notif *notifs; + ...; +}; + +struct lysp_node_augment { + union { + struct lysp_node node; + struct { + struct lysp_node_augment *next; + ...; + }; + }; + struct lysp_node *child; + struct lysp_when *when; + struct lysp_node_action *actions; + struct lysp_node_notif *notifs; + ...; +}; + struct lysc_type { const char *name; struct lysc_ext_instance *exts; @@ -623,6 +737,16 @@ struct lysc_type { uint32_t refcount; }; +struct lysp_type_enum { + const char *name; + const char *dsc; + const char *ref; + int64_t value; + struct lysp_qname *iffeatures; + struct lysp_ext_instance *exts; + uint16_t flags; +}; + struct lysp_type { const char *name; struct lysp_restr *range; diff --git a/libyang/__init__.py b/libyang/__init__.py index 02bd26af..5c931de4 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -76,6 +76,23 @@ IfOrFeatures, Module, Must, + PAction, + PActionInOut, + PAnydata, + PAugment, + PCase, + PChoice, + PContainer, + PEnum, + PGrouping, + PLeaf, + PLeafList, + PList, + PNode, + PNotif, + PRefine, + PType, + PUses, Pattern, Revision, SAnydata, @@ -152,6 +169,23 @@ "NodeTypeRemoved", "OrderedByUserAdded", "OrderedByUserRemoved", + "PAction", + "PActionInOut", + "PAnydata", + "PAugment", + "PCase", + "PChoice", + "PContainer", + "PEnum", + "PGrouping", + "PLeaf", + "PLeafList", + "PList", + "PNode", + "PNotif", + "PRefine", + "PType", + "PUses", "Pattern", "PatternAdded", "PatternRemoved", diff --git a/libyang/schema.py b/libyang/schema.py index 75206bd3..7cd2289b 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -6,8 +6,15 @@ from typing import IO, Any, Dict, Iterator, List, Optional, Tuple, Union from _libyang import ffi, lib -from .util import IOType, LibyangError, c2str, init_output, ly_array_iter, str2c - +from .util import ( + IOType, + LibyangError, + c2str, + init_output, + ly_array_iter, + ly_list_iter, + str2c, +) # ------------------------------------------------------------------------------------- def schema_in_format(fmt_string: str) -> int: @@ -144,6 +151,26 @@ def children( self.context, self.cdata, types=types, with_choice=with_choice ) + def parsed_children(self) -> Iterator["PNode"]: + for c in ly_list_iter(self.cdata.parsed.data): + yield PNode.new(self.context, c, self) + + def groupings(self) -> Iterator["PGrouping"]: + for g in ly_list_iter(self.cdata.parsed.groupings): + yield PGrouping(self.context, g, self) + + def augments(self) -> Iterator["PAugment"]: + for a in ly_array_iter(self.cdata.parsed.augments): + yield PAugment(self.context, a, self) + + def actions(self) -> Iterator["PAction"]: + for a in ly_list_iter(self.cdata.parsed.rpcs): + yield PAction(self.context, a, self) + + def notifications(self) -> Iterator["PNotif"]: + for n in ly_list_iter(self.cdata.parsed.notifs): + yield PNotif(self.context, n, self) + def __str__(self) -> str: return self.name() @@ -454,19 +481,26 @@ class Bit(_EnumBit): # ------------------------------------------------------------------------------------- class Pattern: - __slots__ = ("context", "cdata") + __slots__ = ("context", "cdata", "cdata_parsed") - def __init__(self, context: "libyang.Context", cdata): + def __init__(self, context: "libyang.Context", cdata, cdata_parsed=None): self.context = context self.cdata = cdata # C type: "struct lysc_pattern *" + self.cdata_parsed = cdata_parsed # C type: "struct lysp_restr *" def expression(self) -> str: + if self.cdata is None and self.cdata_parsed: + return c2str(self.cdata_parsed.arg.str + 1) return c2str(self.cdata.expr) def inverted(self) -> bool: + if self.cdata is None and self.cdata_parsed: + return self.cdata_parsed.arg.str[0] == b"\x15" return self.cdata.inverted def error_message(self) -> Optional[str]: + if self.cdata is None and self.cdata_parsed: + return c2str(self.cdata_parsed.emsg) return c2str(self.cdata.emsg) if self.cdata.emsg != ffi.NULL else None @@ -756,6 +790,11 @@ def __repr__(self): def __str__(self): return self.name() + def parsed(self) -> Optional["PType"]: + if self.cdata_parsed is None or self.cdata_parsed == ffi.NULL: + return None + return PType(self.context, self.cdata_parsed, self.module()) + # ------------------------------------------------------------------------------------- class Typedef: @@ -1078,16 +1117,21 @@ def __str__(self): # ------------------------------------------------------------------------------------- class Must: - __slots__ = ("context", "cdata") + __slots__ = ("context", "cdata", "cdata_parsed") - def __init__(self, context: "libyang.Context", cdata): + def __init__(self, context: "libyang.Context", cdata, cdata_parsed=None): self.context = context self.cdata = cdata # C type: "struct lysc_must *" + self.cdata_parsed = cdata_parsed # C type: "struct lysp_must *" def condition(self) -> str: + if self.cdata is None and self.cdata_parsed: + return c2str(self.cdata_parsed.arg.str + 1) return c2str(lib.lyxp_get_expr(self.cdata.cond)) def error_message(self) -> Optional[str]: + if self.cdata is None and self.cdata_parsed: + return c2str(self.cdata_parsed.emsg) return c2str(self.cdata.emsg) if self.cdata.emsg != ffi.NULL else None @@ -1240,6 +1284,11 @@ def when_conditions(self): for cond in ly_array_iter(wh): yield c2str(lib.lyxp_get_expr(cond.cond)) + def parsed(self) -> Optional["PNode"]: + if self.cdata_parsed is None or self.cdata_parsed == ffi.NULL: + return None + return PNode.new(self.context, self.cdata_parsed, self.module()) + def iter_tree(self, full: bool = False) -> Iterator["SNode"]: """ Do a DFS walk of the schema node. @@ -1679,3 +1728,688 @@ def _skip(node) -> bool: Rpc = SRpc RpcInOut = SRpcInOut Anyxml = SAnyxml + + +# ------------------------------------------------------------------------------------- +class PEnum: + __slots__ = ("context", "cdata", "module") + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + self.context = context + self.cdata = cdata # C type of "struct lysp_type_enum *" + self.module = module + + def name(self) -> str: + return c2str(self.cdata.name) + + def description(self) -> Optional[str]: + return c2str(self.cdata.dsc) + + def reference(self) -> Optional[str]: + return c2str(self.cdata.ref) + + def value(self) -> int: + return self.cdata.value + + def if_features(self) -> Iterator[IfFeatureExpr]: + for f in ly_array_iter(self.cdata.iffeatures): + yield IfFeatureExpr(self.context, f, list(self.module.features())) + + def extensions(self) -> Iterator["ExtensionParsed"]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionParsed(self.context, ext, self.module) + + +# ------------------------------------------------------------------------------------- +class PType: + __slots__ = ("context", "cdata", "module") + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + self.context = context + self.cdata = cdata # C type of "struct lysp_type *" + self.module = module + + def name(self) -> str: + return c2str(self.cdata.name) + + def range(self) -> Optional[str]: + if self.cdata.range == ffi.NULL: + return None + return c2str(self.cdata.range.arg.str) + + def length(self) -> Optional[str]: + if self.cdata.length == ffi.NULL: + return None + return c2str(self.cdata.length.arg.str) + + def patterns(self) -> Iterator[Pattern]: + for p in ly_array_iter(self.cdata.patterns): + yield Pattern(self.context, None, p) + + def enums(self) -> Iterator[PEnum]: + for e in ly_array_iter(self.cdata.enums): + yield PEnum(self.context, e, self.module) + + def bits(self) -> Iterator[PEnum]: + for b in ly_array_iter(self.cdata.bits): + yield PEnum(self.context, b, self.module) + + def path(self) -> Optional[str]: + if self.cdata.path == ffi.NULL: + return None + return c2str(lib.lyxp_get_expr(self.cdata.path)) + + def bases(self) -> Iterator[str]: + for b in ly_array_iter(self.cdata.bases): + yield c2str(b) + + def types(self) -> Iterator["PType"]: + for t in ly_array_iter(self.cdata.types): + yield PType(self.context, t, self.module) + + def extensions(self) -> Iterator["ExtensionParsed"]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionParsed(self.context, ext, self.module) + + def pmod(self) -> Optional[Module]: + if self.cdata.pmod == ffi.NULL: + return None + return Module(self.context, self.cdata.pmod.mod) + + def compiled(self) -> Optional[Type]: + if self.cdata.compiled == ffi.NULL: + return None + return Type(self.context, self.cdata.compiled, self.cdata) + + def fraction_digits(self) -> int: + return self.cdata.fraction_digits + + def require_instance(self) -> bool: + return self.cdata.require_instance + + +# ------------------------------------------------------------------------------------- +class PRefine: + __slots__ = ("context", "cdata", "module") + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + self.context = context + self.cdata = cdata # C type of "struct lysp_refine *" + self.module = module + + def nodeid(self) -> str: + return c2str(self.cdata.nodeid) + + def description(self) -> Optional[str]: + return c2str(self.cdata.dsc) + + def reference(self) -> Optional[str]: + return c2str(self.cdata.ref) + + def if_features(self) -> Iterator[IfFeatureExpr]: + for f in ly_array_iter(self.cdata.iffeatures): + yield IfFeatureExpr(self.context, f, list(self.module.features())) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata.musts): + yield Must(self.context, None, m) + + def presence(self) -> Optional[str]: + return c2str(self.cdata.presence) + + def defaults(self) -> Iterator[str]: + for d in ly_array_iter(self.cdata.dflts): + yield c2str(d.str) + + def min_elements(self) -> int: + return self.cdata.min + + def max_elements(self) -> Optional[int]: + return self.cdata.max if self.cdata.max != 0 else None + + def extensions(self) -> Iterator["ExtensionParsed"]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionParsed(self.context, ext, self.module) + + +# ------------------------------------------------------------------------------------- +class PNode: + CONTAINER = lib.LYS_CONTAINER + CHOICE = lib.LYS_CHOICE + CASE = lib.LYS_CASE + LEAF = lib.LYS_LEAF + LEAFLIST = lib.LYS_LEAFLIST + LIST = lib.LYS_LIST + RPC = lib.LYS_RPC + ACTION = lib.LYS_ACTION + INPUT = lib.LYS_INPUT + OUTPUT = lib.LYS_OUTPUT + NOTIF = lib.LYS_NOTIF + ANYXML = lib.LYS_ANYXML + ANYDATA = lib.LYS_ANYDATA + AUGMENT = lib.LYS_AUGMENT + USES = lib.LYS_USES + GROUPING = lib.LYS_GROUPING + KEYWORDS = { + CONTAINER: "container", + LEAF: "leaf", + LEAFLIST: "leaf-list", + LIST: "list", + RPC: "rpc", + ACTION: "action", + INPUT: "input", + OUTPUT: "output", + NOTIF: "notification", + ANYXML: "anyxml", + ANYDATA: "anydata", + AUGMENT: "augment", + USES: "uses", + GROUPING: "grouping", + } + + __slots__ = ("context", "cdata", "module", "__dict__") + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + self.context = context + self.cdata = ffi.cast("struct lysp_node *", cdata) + self.module = module + + def parent(self) -> Optional["PNode"]: + if self.cdata.parent == ffi.NULL: + return None + return PNode.new(self.context, self.cdata.parent, self.module) + + def nodetype(self) -> int: + return self.cdata.nodetype + + def siblings(self) -> Iterator["PNode"]: + for s in ly_list_iter(self.cdata.next): + yield PNode.new(self.context, s, self.module) + + def name(self) -> str: + return c2str(self.cdata.name) + + def description(self) -> Optional[str]: + return c2str(self.cdata.dsc) + + def reference(self) -> Optional[str]: + return c2str(self.cdata.ref) + + def if_features(self) -> Iterator[IfFeatureExpr]: + for f in ly_array_iter(self.cdata.iffeatures): + yield IfFeatureExpr(self.context, f, list(self.module.features())) + + def extensions(self) -> Iterator["ExtensionParsed"]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionParsed(self.context, ext, self.module) + + def get_extension( + self, name: str, prefix: Optional[str] = None, arg_value: Optional[str] = None + ) -> Optional["ExtensionParsed"]: + for ext in self.extensions(): + if ext.name() != name: + continue + if prefix is not None and ext.module().name() != prefix: + continue + if arg_value is not None and ext.argument() != arg_value: + continue + return ext + return None + + def config_set(self) -> bool: + return bool(self.cdata.flags & lib.LYS_SET_CONFIG) + + def config_false(self) -> bool: + return bool(self.cdata.flags & lib.LYS_CONFIG_R) + + def mandatory(self) -> bool: + return bool(self.cdata.flags & lib.LYS_MAND_TRUE) + + def deprecated(self) -> bool: + return bool(self.cdata.flags & lib.LYS_STATUS_DEPRC) + + def obsolete(self) -> bool: + return bool(self.cdata.flags & lib.LYS_STATUS_OBSLT) + + def status(self) -> str: + if self.cdata.flags & lib.LYS_STATUS_OBSLT: + return "obsolete" + if self.cdata.flags & lib.LYS_STATUS_DEPRC: + return "deprecated" + return "current" + + def __repr__(self): + cls = self.__class__ + return "<%s.%s: %s>" % (cls.__module__, cls.__name__, str(self)) + + def __str__(self): + return self.name() + + NODETYPE_CLASS = {} + + @staticmethod + def register(nodetype): + def _decorator(nodeclass): + PNode.NODETYPE_CLASS[nodetype] = nodeclass + return nodeclass + + return _decorator + + @staticmethod + def new(context: "libyang.Context", cdata, module: Module) -> "PNode": + cdata = ffi.cast("struct lysp_node *", cdata) + nodecls = PNode.NODETYPE_CLASS.get(cdata.nodetype, None) + if nodecls is None: + raise TypeError("node type %s not implemented" % cdata.nodetype) + return nodecls(context, cdata, module) + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.CONTAINER) +class PContainer(PNode): + __slots__ = ("cdata_container",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_container = ffi.cast("struct lysp_node_container *", cdata) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata_container.musts): + yield Must(self.context, None, m) + + def when_condition(self) -> Optional[str]: + if self.cdata_container.when == ffi.NULL: + return None + return c2str(self.cdata_container.when.cond) + + def presence(self) -> Optional[str]: + return c2str(self.cdata_container.presence) + + def typedefs(self) -> Iterator[Typedef]: + for t in ly_array_iter(self.cdata_container.typedefs): + yield Typedef(self.context, t) + + def groupings(self) -> Iterator["PGrouping"]: + for g in ly_list_iter(self.cdata_container.groupings): + yield PGrouping(self.context, g, self.module) + + def children(self) -> Iterator[PNode]: + for c in ly_list_iter(self.cdata_container.child): + yield PNode.new(self.context, c, self.module) + + def actions(self) -> Iterator["PAction"]: + for a in ly_list_iter(self.cdata_container.actions): + yield PAction(self.context, a, self.module) + + def notifications(self) -> Iterator["PNotif"]: + for n in ly_list_iter(self.cdata_container.notifs): + yield PNotif(self.context, n, self.module) + + def __iter__(self) -> Iterator[PNode]: + return self.children() + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.LEAF) +class PLeaf(PNode): + __slots__ = ("cdata_leaf",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_leaf = ffi.cast("struct lysp_node_leaf *", cdata) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata_leaf.musts): + yield Must(self.context, None, m) + + def when_condition(self) -> Optional[str]: + if self.cdata_leaf.when == ffi.NULL: + return None + return c2str(self.cdata_leaf.when.cond) + + def type(self) -> PType: + return PType(self.context, self.cdata_leaf.type, self.module) + + def units(self) -> Optional[str]: + return c2str(self.cdata_leaf.units) + + def default(self) -> Optional[str]: + return c2str(self.cdata_leaf.dflt.str) + + def is_key(self) -> bool: + if self.cdata.flags & lib.LYS_KEY: + return True + return False + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.LEAFLIST) +class PLeafList(PNode): + __slots__ = ("cdata_leaflist",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_leaflist = ffi.cast("struct lysp_node_leaflist *", cdata) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata_leaflist.musts): + yield Must(self.context, None, m) + + def when_condition(self) -> Optional[str]: + if self.cdata_leaflist.when == ffi.NULL: + return None + return c2str(self.cdata_leaflist.when.cond) + + def type(self) -> PType: + return PType(self.context, self.cdata_leaflist.type, self.module) + + def units(self) -> Optional[str]: + return c2str(self.cdata_leaflist.units) + + def defaults(self) -> Iterator[str]: + for d in ly_array_iter(self.cdata_leaflist.dflts): + yield c2str(d.str) + + def min_elements(self) -> int: + return self.cdata_leaflist.min + + def max_elements(self) -> Optional[int]: + return self.cdata_leaflist.max if self.cdata_leaflist.max != 0 else None + + def ordered(self) -> bool: + return bool(self.cdata.flags & lib.LYS_ORDBY_USER) + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.LIST) +class PList(PNode): + __slots__ = ("cdata_list",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_list = ffi.cast("struct lysp_node_list *", cdata) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata_list.musts): + yield Must(self.context, None, m) + + def when_condition(self) -> Optional[str]: + if self.cdata_list.when == ffi.NULL: + return None + return c2str(self.cdata_list.when.cond) + + def key(self) -> Optional[str]: + return c2str(self.cdata_list.key) + + def typedefs(self) -> Iterator[Typedef]: + for t in ly_array_iter(self.cdata_list.typedefs): + yield Typedef(self.context, t) + + def groupings(self) -> Iterator["PGrouping"]: + for g in ly_list_iter(self.cdata_list.groupings): + yield PGrouping(self.context, g, self.module) + + def children(self) -> Iterator[PNode]: + for c in ly_list_iter(self.cdata_list.child): + yield PNode.new(self.context, c, self.module) + + def actions(self) -> Iterator["PAction"]: + for a in ly_list_iter(self.cdata_list.actions): + yield PAction(self.context, a, self.module) + + def notifications(self) -> Iterator["PNotif"]: + for n in ly_list_iter(self.cdata_list.notifs): + yield PNotif(self.context, n, self.module) + + def uniques(self) -> Iterator[str]: + for u in ly_array_iter(self.cdata_list.uniques): + yield c2str(u.str) + + def min_elements(self) -> int: + return self.cdata_list.min + + def max_elements(self) -> Optional[int]: + return self.cdata_list.max if self.cdata_list.max != 0 else None + + def ordered(self) -> bool: + return bool(self.cdata.flags & lib.LYS_ORDBY_USER) + + def __iter__(self) -> Iterator[PNode]: + return self.children() + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.CASE) +class PCase(PNode): + __slots__ = ("cdata_case",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_case = ffi.cast("struct lysp_node_case *", cdata) + + def children(self) -> Iterator[PNode]: + for c in ly_list_iter(self.cdata_case.child): + yield PNode.new(self.context, c, self.module) + + def when_condition(self) -> Optional[str]: + if self.cdata_case.when == ffi.NULL: + return None + return c2str(self.cdata_case.when.cond) + + def __iter__(self) -> Iterator[PNode]: + return self.children() + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.CHOICE) +class PChoice(PNode): + __slots__ = ("cdata_choice",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_choice = ffi.cast("struct lysp_node_choice *", cdata) + + def children(self) -> Iterator[PCase]: + for c in ly_list_iter(self.cdata_choice.child): + yield PCase(self.context, c, self.module) + + def when_condition(self) -> Optional[str]: + if self.cdata_choice.when == ffi.NULL: + return None + return c2str(self.cdata_choice.when.cond) + + def default(self) -> Optional[str]: + return c2str(self.cdata_choice.dflt.str) + + def __iter__(self) -> Iterator[PCase]: + return self.children() + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.ANYXML) +@PNode.register(PNode.ANYDATA) +class PAnydata(PNode): + __slots__ = ("cdata_anydata",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_anydata = ffi.cast("struct lysp_node_anydata *", cdata) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata_anydata.musts): + yield Must(self.context, None, m) + + def when_condition(self) -> Optional[str]: + if self.cdata_anydata.when == ffi.NULL: + return None + return c2str(self.cdata_anydata.when.cond) + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.AUGMENT) +class PAugment(PNode): + __slots__ = ("cdata_augment",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_augment = ffi.cast("struct lysp_node_augment *", cdata) + + def children(self) -> Iterator["PNode"]: + for c in ly_list_iter(self.cdata_augment.child): + yield PNode.new(self.context, c, self.module) + + def when_condition(self) -> Optional[str]: + if self.cdata_augment.when == ffi.NULL: + return None + return c2str(self.cdata_augment.when.cond) + + def actions(self) -> Iterator["PAction"]: + for a in ly_list_iter(self.cdata_augment.actions): + yield PAction(self.context, a, self.module) + + def notifications(self) -> Iterator["PNotif"]: + for n in ly_list_iter(self.cdata_augment.notifs): + yield PNotif(self.context, n, self.module) + + def __iter__(self) -> Iterator[PNode]: + return self.children() + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.USES) +class PUses(PNode): + __slots__ = ("cdata_uses",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_uses = ffi.cast("struct lysp_node_uses *", cdata) + + def refines(self) -> Iterator[PRefine]: + for r in ly_array_iter(self.cdata_uses.refines): + yield PRefine(self.context, r, self.module) + + def augments(self) -> Iterator[PAugment]: + for a in ly_list_iter(self.cdata_uses.augments): + yield PAugment(self.context, a, self.module) + + def when_condition(self) -> Optional[str]: + if self.cdata_uses.when == ffi.NULL: + return None + return c2str(self.cdata_uses.when.cond) + + +# ------------------------------------------------------------------------------------- +class PActionInOut(PNode): + __slots__ = ("cdata_action_inout",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_action_inout = ffi.cast("struct lysp_node_action_inout *", cdata) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata_action_inout.musts): + yield Must(self.context, None, m) + + def typedefs(self) -> Iterator[Typedef]: + for t in ly_array_iter(self.cdata_action_inout.typedefs): + yield Typedef(self.context, t) + + def groupings(self) -> Iterator["PGrouping"]: + for g in ly_list_iter(self.cdata_action_inout.groupings): + yield PGrouping(self.context, g, self.module) + + def children(self) -> Iterator[PNode]: + for c in ly_list_iter(self.cdata_action_inout.child): + yield PNode.new(self.context, c, self.module) + + def __iter__(self) -> Iterator[PNode]: + return self.children() + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.RPC) +@PNode.register(PNode.ACTION) +class PAction(PNode): + __slots__ = ("cdata_action",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_action = ffi.cast("struct lysp_node_action *", cdata) + + def typedefs(self) -> Iterator[Typedef]: + for t in ly_array_iter(self.cdata_action.typedefs): + yield Typedef(self.context, t) + + def groupings(self) -> Iterator["PGrouping"]: + for g in ly_list_iter(self.cdata_action.groupings): + yield PGrouping(self.context, g, self.module) + + def input(self) -> PActionInOut: + ptr = ffi.addressof(self.cdata_action.input) + return PActionInOut(self.context, ptr, self.module) + + def output(self) -> PActionInOut: + ptr = ffi.addressof(self.cdata_action.output) + return PActionInOut(self.context, ptr, self.module) + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.NOTIF) +class PNotif(PNode): + __slots__ = ("cdata_notif",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_notif = ffi.cast("struct lysp_node_notif *", cdata) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata_notif.musts): + yield Must(self.context, None, m) + + def typedefs(self) -> Iterator[Typedef]: + for t in ly_array_iter(self.cdata_notif.typedefs): + yield Typedef(self.context, t) + + def groupings(self) -> Iterator["PGrouping"]: + for g in ly_list_iter(self.cdata_notif.groupings): + yield PGrouping(self.context, g, self.module) + + def children(self) -> Iterator[PNode]: + for c in ly_list_iter(self.cdata_notif.child): + yield PNode.new(self.context, c, self.module) + + def __iter__(self) -> Iterator[PNode]: + return self.children() + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.GROUPING) +class PGrouping(PNode): + __slots__ = ("cdata_grouping",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_grouping = ffi.cast("struct lysp_node_grp *", cdata) + + def typedefs(self) -> Iterator[Typedef]: + for t in ly_array_iter(self.cdata_grouping.typedefs): + yield Typedef(self.context, t) + + def groupings(self) -> Iterator["PGrouping"]: + for g in ly_list_iter(self.cdata_grouping.groupings): + yield PGrouping(self.context, g, self.module) + + def children(self) -> Iterator[PNode]: + for c in ly_list_iter(self.cdata_grouping.child): + yield PNode.new(self.context, c, self.module) + + def actions(self) -> Iterator[PAction]: + for a in ly_list_iter(self.cdata_grouping.actions): + yield PAction(self.context, a, self.module) + + def notifications(self) -> Iterator[PNotif]: + for n in ly_list_iter(self.cdata_grouping.notifs): + yield PNotif(self.context, n, self.module) + + def __iter__(self) -> Iterator[PNode]: + return self.children() diff --git a/tests/test_schema.py b/tests/test_schema.py index 64a8e30e..87409f90 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -13,8 +13,25 @@ LibyangError, Module, Must, + PAction, + PActionInOut, + PAnydata, Pattern, + PAugment, + PCase, + PChoice, + PContainer, + PGrouping, + PLeaf, + PLeafList, + PList, + PNode, + PNotif, + PRefine, + PType, + PUses, Revision, + SAnydata, SCase, SChoice, SContainer, @@ -287,6 +304,79 @@ def test_iter_tree(self): tree = list(self.container.iter_tree(full=True)) self.assertEqual(len(tree), 25) + def test_container_parsed(self): + pnode = self.container.parsed() + self.assertIsInstance(pnode, PContainer) + self.assertIsNone(next(pnode.musts(), None)) + self.assertIsNone(pnode.when_condition()) + self.assertIsNone(pnode.presence()) + self.assertIsNone(next(pnode.typedefs(), None)) + self.assertIsNone(next(pnode.groupings(), None)) + self.assertIsNotNone(next(iter(pnode))) + self.assertIsNone(next(pnode.actions(), None)) + self.assertIsNone(next(pnode.notifications(), None)) + + +# ------------------------------------------------------------------------------------- +class UsesTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + mod = self.ctx.load_module("yolo-nodetypes") + mod.feature_enable_all() + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_uses_parsed(self): + snode = next(self.ctx.find_path("/yolo-nodetypes:cont2")) + self.assertIsInstance(snode, SContainer) + pnode = snode.parsed() + self.assertIsInstance(pnode, PContainer) + pnode = next(iter(pnode)) + self.assertIsInstance(pnode, PUses) + + ref_pnode = next(pnode.refines()) + self.assertIsInstance(ref_pnode, PRefine) + self.assertEqual("cont3/leaf1", ref_pnode.nodeid()) + self.assertIsNone(ref_pnode.description()) + self.assertIsNone(ref_pnode.reference()) + self.assertIsNone(next(ref_pnode.if_features(), None)) + self.assertIsNone(next(ref_pnode.musts(), None)) + self.assertIsNone(ref_pnode.presence()) + self.assertIsNone(next(ref_pnode.defaults(), None)) + self.assertEqual(0, ref_pnode.min_elements()) + self.assertIsNone(ref_pnode.max_elements()) + self.assertIsNone(next(ref_pnode.extensions(), None)) + + aug_pnode = next(pnode.augments()) + self.assertIsInstance(aug_pnode, PAugment) + self.assertIsNotNone(next(iter(aug_pnode))) + self.assertIsNone(aug_pnode.when_condition()) + self.assertIsNone(next(aug_pnode.actions(), None)) + self.assertIsNone(next(aug_pnode.notifications(), None)) + + +# ------------------------------------------------------------------------------------- +class GroupingTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_grouping_parsed(self): + mod = self.ctx.load_module("yolo-nodetypes") + pnode = next(mod.groupings()) + self.assertIsInstance(pnode, PGrouping) + self.assertIsNone(next(pnode.typedefs(), None)) + self.assertIsNone(next(pnode.groupings(), None)) + child = next(iter(pnode)) + self.assertIsNotNone(child) + self.assertIsNone(next(pnode.actions(), None)) + self.assertIsNone(next(pnode.notifications(), None)) + # ------------------------------------------------------------------------------------- class ListTest(unittest.TestCase): @@ -363,6 +453,25 @@ def test_list_min_max(self): self.assertEqual(list2.min_elements(), 0) self.assertEqual(list2.max_elements(), None) + def test_list_parsed(self): + list1 = next(self.ctx.find_path("/yolo-nodetypes:conf/list1")) + self.assertIsInstance(list1, SList) + pnode = list1.parsed() + self.assertIsInstance(pnode, PList) + self.assertIsNone(next(pnode.musts(), None)) + self.assertIsNone(pnode.when_condition()) + self.assertEqual("leaf1", pnode.key()) + self.assertIsNone(next(pnode.typedefs(), None)) + self.assertIsNone(next(pnode.groupings(), None)) + child = next(iter(pnode)) + self.assertIsInstance(child, PLeaf) + self.assertIsNone(next(pnode.actions(), None)) + self.assertIsNone(next(pnode.notifications(), None)) + self.assertEqual("leaf2 leaf3", next(pnode.uniques())) + self.assertEqual(2, pnode.min_elements()) + self.assertEqual(10, pnode.max_elements()) + self.assertFalse(pnode.ordered()) + # ------------------------------------------------------------------------------------- class RpcTest(unittest.TestCase): @@ -398,6 +507,21 @@ def test_rpc_params(self): def test_rpc_no_parent(self): self.assertIsNone(self.rpc.parent()) + def test_rpc_parsed(self): + self.assertIsInstance(self.rpc, SRpc) + pnode = self.rpc.parsed() + self.assertIsInstance(pnode, PAction) + self.assertIsNone(next(pnode.typedefs(), None)) + self.assertIsNone(next(pnode.groupings(), None)) + pnode2 = pnode.input() + self.assertIsInstance(pnode2, PActionInOut) + self.assertIsInstance(pnode.output(), PActionInOut) + self.assertIsNone(next(pnode2.musts(), None)) + self.assertIsNone(next(pnode2.typedefs(), None)) + self.assertIsNone(next(pnode2.groupings(), None)) + pnode3 = next(iter(pnode2)) + self.assertIsInstance(pnode3, PLeaf) + # ------------------------------------------------------------------------------------- class LeafTypeTest(unittest.TestCase): @@ -536,6 +660,28 @@ def test_leaf_type_require_instance(self): self.assertIsInstance(t, Type) self.assertFalse(t.require_instance()) + def test_leaf_type_parsed(self): + leaf = next(self.ctx.find_path("/yolo-system:conf/yolo-system:hostname")) + self.assertIsInstance(leaf, SLeaf) + t = leaf.type() + self.assertIsInstance(t, Type) + pnode = t.parsed() + self.assertIsInstance(pnode, PType) + self.assertEqual("types:host", pnode.name()) + self.assertIsNone(pnode.range()) + self.assertIsNone(pnode.length()) + self.assertIsNone(next(pnode.patterns(), None)) + self.assertIsNone(next(pnode.enums(), None)) + self.assertIsNone(next(pnode.bits(), None)) + self.assertIsNone(pnode.path()) + self.assertIsNone(next(pnode.bases(), None)) + self.assertIsNone(next(pnode.types(), None)) + self.assertIsNone(next(pnode.extensions(), None)) + self.assertIsNotNone(pnode.pmod()) + self.assertIsNone(pnode.compiled()) + self.assertEqual(0, pnode.fraction_digits()) + self.assertFalse(pnode.require_instance()) + # ------------------------------------------------------------------------------------- class LeafTest(unittest.TestCase): @@ -560,6 +706,41 @@ def test_leaf_default(self): leaf = next(self.ctx.find_path("/yolo-nodetypes:conf/percentage")) self.assertIsInstance(leaf.default(), float) + def test_leaf_parsed(self): + leaf = next(self.ctx.find_path("/yolo-nodetypes:conf/percentage")) + self.assertIsInstance(leaf, SLeaf) + pnode = leaf.parsed() + self.assertIsInstance(pnode, PLeaf) + must = next(pnode.musts()) + self.assertIsInstance(must, Must) + self.assertEqual(must.error_message(), "ERROR1") + must = next(leaf.must_conditions()) + self.assertIsInstance(must, str) + self.assertIsNone(pnode.when_condition()) + self.assertIsInstance(pnode.type(), PType) + self.assertIsNone(pnode.units()) + self.assertEqual("10.2", pnode.default()) + self.assertFalse(pnode.is_key()) + + # test basic PNode settings + self.assertIsNotNone(pnode.parent()) + self.assertEqual(PNode.LEAF, pnode.nodetype()) + self.assertIsNotNone(next(pnode.siblings())) + self.assertEqual("<libyang.schema.PLeaf: percentage>", repr(pnode)) + self.assertIsNone(pnode.description()) + self.assertIsNone(pnode.reference()) + self.assertIsNone(next(pnode.if_features(), None)) + self.assertIsNone(next(pnode.extensions(), None)) + self.assertIsNone(pnode.get_extension("test", prefix="test")) + self.assertFalse(pnode.config_set()) + self.assertFalse(pnode.config_false()) + self.assertFalse(pnode.mandatory()) + self.assertFalse(pnode.deprecated()) + self.assertFalse(pnode.obsolete()) + self.assertEqual("current", pnode.status()) + + NODETYPE_CLASS = {} + # ------------------------------------------------------------------------------------- class LeafListTest(unittest.TestCase): @@ -587,6 +768,20 @@ def test_leaf_list_min_max(self): self.assertEqual(leaflist2.min_elements(), 0) self.assertEqual(leaflist2.max_elements(), None) + def test_leaf_list_parsed(self): + leaflist = next(self.ctx.find_path("/yolo-nodetypes:conf/ratios")) + self.assertIsInstance(leaflist, SLeafList) + pnode = leaflist.parsed() + self.assertIsInstance(pnode, PLeafList) + self.assertIsNone(next(pnode.musts(), None)) + self.assertIsNone(pnode.when_condition()) + self.assertIsInstance(pnode.type(), PType) + self.assertIsNone(pnode.units()) + self.assertEqual("2.5", next(pnode.defaults())) + self.assertEqual(0, pnode.min_elements()) + self.assertIsNone(pnode.max_elements()) + self.assertFalse(pnode.ordered()) + # ------------------------------------------------------------------------------------- class ChoiceTest(unittest.TestCase): @@ -603,3 +798,60 @@ def test_choice_default(self): choice = next(conf.children((SNode.CHOICE,), with_choice=True)) self.assertIsInstance(choice, SChoice) self.assertIsInstance(choice.default(), SCase) + + def test_choice_parsed(self): + conf = next(self.ctx.find_path("/yolo-system:conf")) + choice = next(conf.children((SNode.CHOICE,), with_choice=True)) + self.assertIsInstance(choice, SChoice) + pnode = choice.parsed() + self.assertIsInstance(pnode, PChoice) + + case_pnode = next(iter(pnode)) + self.assertIsInstance(case_pnode, PCase) + self.assertIsNotNone(next(iter(case_pnode))) + self.assertIsNone(case_pnode.when_condition()) + + self.assertIsNone(pnode.when_condition()) + self.assertEqual("red", pnode.default()) + + +# ------------------------------------------------------------------------------------- +class AnydataTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.ctx.load_module("yolo-nodetypes") + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_anydata_parsed(self): + snode = next(self.ctx.find_path("/yolo-nodetypes:any1")) + self.assertIsInstance(snode, SAnydata) + pnode = snode.parsed() + self.assertIsInstance(pnode, PAnydata) + self.assertIsNone(next(pnode.musts(), None)) + self.assertEqual("../cont2", pnode.when_condition()) + + +# ------------------------------------------------------------------------------------- +class NotificationTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.ctx.load_module("yolo-nodetypes") + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_notification_parsed(self): + snode = next(self.ctx.find_path("/yolo-nodetypes:cont2")) + self.assertIsInstance(snode, SContainer) + pnode = snode.parsed() + self.assertIsInstance(pnode, PContainer) + pnode = next(pnode.notifications()) + self.assertIsInstance(pnode, PNotif) + self.assertIsNone(next(pnode.musts(), None)) + self.assertIsNone(next(pnode.typedefs(), None)) + self.assertIsNone(next(pnode.groupings(), None)) + self.assertIsNotNone(next(iter(pnode))) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index a456ae1d..c2690dc9 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -81,4 +81,35 @@ module yolo-nodetypes { leaf test1 { type uint8; } + + grouping grp1 { + container cont3 { + leaf leaf1 { + type string; + } + } + } + + container cont2 { + presence "special container enabled"; + uses grp1 { + refine cont3/leaf1 { + mandatory true; + } + augment cont3 { + leaf leaf2 { + type int8; + } + } + } + notification interface-enabled { + leaf by-user { + type string; + } + } + } + + anydata any1 { + when "../cont2"; + } } From 41a4f74737dba100bbfe5e42b069d58d91a70f8c Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Fri, 1 Mar 2024 14:37:04 +0100 Subject: [PATCH 40/71] context: add add_to_dict/remove_from_dict APIs This patch adds add_to_dict and remove_from_dict API, which allows to create internal string reference within the context. Closes: https://github.com/CESNET/libyang-python/pull/102 Signed-off-by: Stefan Gula <steweg@gmail.com> --- cffi/cdefs.h | 3 +++ libyang/context.py | 10 ++++++++++ tests/test_context.py | 8 ++++++++ 3 files changed, 21 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index cbe92969..8eaaa0a9 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1071,6 +1071,9 @@ LY_ERR lys_parse(struct ly_ctx *, struct ly_in *, LYS_INFORMAT, const char **, s LY_ERR ly_ctx_new_ylpath(const char *, const char *, LYD_FORMAT, int, struct ly_ctx **); LY_ERR ly_ctx_get_yanglib_data(const struct ly_ctx *, struct lyd_node **, const char *, ...); +LY_ERR lydict_insert(const struct ly_ctx *, const char *, size_t, const char **); +LY_ERR lydict_remove(const struct ly_ctx *, const char *); + struct lyd_meta { struct lyd_node *parent; struct lyd_meta *next; diff --git a/libyang/context.py b/libyang/context.py index fa5eb5bf..9e3f89be 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -465,3 +465,13 @@ def __iter__(self) -> Iterator[Module]: while mod: yield Module(self, mod) mod = lib.ly_ctx_get_module_iter(self.cdata, idx) + + def add_to_dict(self, orig_str: str) -> Any: + cstr = ffi.new("char **") + ret = lib.lydict_insert(self.cdata, str2c(orig_str), 0, cstr) + if ret != lib.LY_SUCCESS: + raise LibyangError("Unable to insert string into context dictionary") + return cstr[0] + + def remove_from_dict(self, orig_str: str) -> None: + lib.lydict_remove(self.cdata, str2c(orig_str)) diff --git a/tests/test_context.py b/tests/test_context.py index 59839284..f43bf1cf 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -5,6 +5,7 @@ import unittest from libyang import Context, LibyangError, Module, SLeaf, SLeafList +from libyang.util import c2str YANG_DIR = os.path.join(os.path.dirname(__file__), "yang") @@ -111,3 +112,10 @@ def test_ctx_leafref_extended(self): with Context(YANG_DIR, leafref_extended=True) as ctx: mod = ctx.load_module("yolo-leafref-extended") self.assertIsInstance(mod, Module) + + def test_context_dict(self): + with Context(YANG_DIR) as ctx: + orig_str = "teststring" + handle = ctx.add_to_dict(orig_str) + self.assertEqual(orig_str, c2str(handle)) + ctx.remove_from_dict(orig_str) From 487cec571b3732d87ee68eab7c19227c004b674f Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 14:49:08 +0200 Subject: [PATCH 41/71] schema: fix missing line For some reason, my tox env did not detect it. Fix it. Fixes: b7adea400520 ("schema: adds PNode and its exact variants") Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libyang/schema.py b/libyang/schema.py index 7cd2289b..7c79b866 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -16,6 +16,7 @@ str2c, ) + # ------------------------------------------------------------------------------------- def schema_in_format(fmt_string: str) -> int: if fmt_string == "yang": From dc5a72666f63bd939db151d927c7e0673f15721c Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 14:56:06 +0200 Subject: [PATCH 42/71] test_schema: fix DATA_PATTERN in list test Fix the key order. I guess it was changed on libyang side, it does not seem to have an impact. Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- tests/test_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index 87409f90..6b171af2 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -383,7 +383,7 @@ class ListTest(unittest.TestCase): PATH = { "LOG": "/yolo-system:conf/url", "DATA": "/yolo-system:conf/url", - "DATA_PATTERN": "/yolo-system:conf/url[host='%s'][proto='%s']", + "DATA_PATTERN": "/yolo-system:conf/url[proto='%s'][host='%s']", } def setUp(self): From a3132f808bb0e78bc5255dde8acf68c2c549316d Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 15:00:08 +0200 Subject: [PATCH 43/71] libyang: fix import order I prefer as it was, but isort doesn't, let's make him happy. Fixes: b7adea400520 ("schema: adds PNode and its exact variants") Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libyang/__init__.py b/libyang/__init__.py index 5c931de4..26235de1 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -79,6 +79,7 @@ PAction, PActionInOut, PAnydata, + Pattern, PAugment, PCase, PChoice, @@ -93,7 +94,6 @@ PRefine, PType, PUses, - Pattern, Revision, SAnydata, SAnyxml, From 25d7de885d7f82030204cb5b73dbcf37c382e833 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Fri, 8 Mar 2024 09:52:42 +0100 Subject: [PATCH 44/71] data: add add_defaults module option This patch adds ability for user to restrict adding off implicit default values based on specified module. Closes: https://github.com/CESNET/libyang-python/pull/110 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 1 + libyang/data.py | 11 ++++++++++- tests/test_data.py | 17 +++++++++++++++-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 8eaaa0a9..7cd37574 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1189,6 +1189,7 @@ LY_ERR lyd_merge_module(struct lyd_node **, const struct lyd_node *, const struc #define LYD_IMPLICIT_NO_DEFAULTS ... LY_ERR lyd_new_implicit_tree(struct lyd_node *, uint32_t, struct lyd_node **); +LY_ERR lyd_new_implicit_module(struct lyd_node **, const struct lys_module *, uint32_t, struct lyd_node **); LY_ERR lyd_new_implicit_all(struct lyd_node **, const struct ly_ctx *, uint32_t, struct lyd_node **); LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, uint32_t, struct lyd_meta **); diff --git a/libyang/data.py b/libyang/data.py index 2895daaf..d36ffe1b 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -340,6 +340,7 @@ def add_defaults( no_state: bool = False, output: bool = False, only_node: bool = False, + only_module: Optional[Module] = None, ): flags = implicit_flags( no_config=no_config, @@ -353,7 +354,15 @@ def add_defaults( else: node_p = ffi.new("struct lyd_node **") node_p[0] = self.cdata - ret = lib.lyd_new_implicit_all(node_p, self.context.cdata, flags, ffi.NULL) + if only_module is not None: + ret = lib.lyd_new_implicit_module( + node_p, only_module.cdata, flags, ffi.NULL + ) + else: + ret = lib.lyd_new_implicit_all( + node_p, self.context.cdata, flags, ffi.NULL + ) + if ret != lib.LY_SUCCESS: raise self.context.error("cannot get module") diff --git a/tests/test_data.py b/tests/test_data.py index a40056c0..a612646f 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -897,7 +897,7 @@ def test_find_all(self): dnode.free() def test_add_defaults(self): - JSON = '{"yolo-nodetypes:records": [{"id": "rec1"}]}' + JSON = '{"yolo-nodetypes:records": [{"id": "rec1"}], "yolo-nodetypes:conf": {}}' dnode = self.ctx.parse_data_mem( JSON, "json", validate_present=True, parse_only=True ) @@ -906,12 +906,25 @@ def test_add_defaults(self): self.assertIsInstance(node, DLeaf) node = dnode.find_one("name") self.assertIsNone(node) + node = dnode.find_one("/yolo-system:conf/speed") + self.assertIsNone(node) + dnode.add_defaults(only_node=True) node = dnode.find_one("name") self.assertIsInstance(node, DLeaf) self.assertEqual(node.value(), "ASD") - node = dnode.find_path("/yolo-nodetypes:conf/speed") + node = dnode.find_one("/yolo-nodetypes:conf/percentage") self.assertIsNone(node) + node = dnode.find_one("/yolo-system:conf/speed") + self.assertIsNone(node) + + dnode.add_defaults(only_module=dnode.module()) + node = dnode.find_one("/yolo-nodetypes:conf/percentage") + self.assertIsInstance(node, DLeaf) + self.assertEqual(node.value(), 10.2) + node = dnode.find_one("/yolo-system:conf/speed") + self.assertIsNone(node) + dnode.add_defaults(only_node=False) node = dnode.find_path("/yolo-system:conf/speed") self.assertIsInstance(node, DLeaf) From 2fc312b93e3db09d15b78c8cec175f10f421bdcf Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Thu, 9 May 2024 12:38:07 +0200 Subject: [PATCH 45/71] data: add insert_[before,after] API to lists This patch adds insert_[before,after] functions, which allows user to specify where to insert the sibling in case of ordered (leaf-)lists. Closes: https://github.com/CESNET/libyang-python/pull/120 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 2 ++ libyang/data.py | 10 ++++++++++ tests/test_data.py | 17 +++++++++++++++++ tests/yang/yolo/yolo-nodetypes.yang | 1 + 4 files changed, 30 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 7cd37574..39e44357 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1126,6 +1126,8 @@ LY_ERR lyd_merge_tree(struct lyd_node **, const struct lyd_node *, uint16_t); LY_ERR lyd_merge_siblings(struct lyd_node **, const struct lyd_node *, uint16_t); LY_ERR lyd_insert_child(struct lyd_node *, struct lyd_node *); LY_ERR lyd_insert_sibling(struct lyd_node *, struct lyd_node *, struct lyd_node **); +LY_ERR lyd_insert_after(struct lyd_node *, struct lyd_node *); +LY_ERR lyd_insert_before(struct lyd_node *, struct lyd_node *); LY_ERR lyd_diff_apply_all(struct lyd_node **, const struct lyd_node *); #define LYD_DUP_NO_META ... diff --git a/libyang/data.py b/libyang/data.py index d36ffe1b..96b792ba 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -415,6 +415,16 @@ def insert_sibling(self, node): if ret != lib.LY_SUCCESS: raise self.context.error("cannot insert sibling") + def insert_after(self, node): + ret = lib.lyd_insert_after(self.cdata, node.cdata) + if ret != lib.LY_SUCCESS: + raise self.context.error("cannot insert sibling after") + + def insert_before(self, node): + ret = lib.lyd_insert_before(self.cdata, node.cdata) + if ret != lib.LY_SUCCESS: + raise self.context.error("cannot insert sibling before") + def name(self) -> str: return c2str(self.cdata.schema.name) diff --git a/tests/test_data.py b/tests/test_data.py index a612646f..d43f212b 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -966,6 +966,23 @@ def test_dnode_insert_sibling(self): self.assertIsInstance(sibling, DLeaf) self.assertEqual(sibling.cdata, dnode2.cdata) + def test_dnode_insert_sibling_before_after(self): + R1 = {"yolo-nodetypes:records": [{"id": "id1", "name": "name1"}]} + R2 = {"yolo-nodetypes:records": [{"id": "id2", "name": "name2"}]} + R3 = {"yolo-nodetypes:records": [{"id": "id3", "name": "name3"}]} + module = self.ctx.get_module("yolo-nodetypes") + dnode1 = dict_to_dnode(R1, module, None, validate=False) + dnode2 = dict_to_dnode(R2, module, None, validate=False) + dnode3 = dict_to_dnode(R3, module, None, validate=False) + self.assertEqual(dnode1.first_sibling().cdata, dnode1.cdata) + dnode1.insert_before(dnode2) + dnode1.insert_after(dnode3) + self.assertEqual( + [dnode2.cdata, dnode1.cdata, dnode3.cdata], + [s.cdata for s in dnode1.first_sibling().siblings()], + ) + self.assertEqual(dnode1.first_sibling().cdata, dnode2.cdata) + def _create_opaq_hostname(self): root = self.ctx.create_data_path(path="/yolo-system:conf") root.new_path( diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index c2690dc9..d9ef77de 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -20,6 +20,7 @@ module yolo-nodetypes { type string; default "ASD"; } + ordered-by user; } container conf { From e1cefcf64d5374f4c0bcc769e46bfe8758b317bb Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 15:42:13 +0200 Subject: [PATCH 46/71] schema: use ly_array_iter for SLeafList defaults API It is way clearer this way. Closes: https://github.com/CESNET/libyang-python/pull/119 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/schema.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index 7c79b866..539af886 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1414,15 +1414,12 @@ def type(self) -> Type: def defaults(self) -> Iterator[Union[None, bool, int, str, float]]: if self.cdata_leaflist.dflts == ffi.NULL: return - arr_length = ffi.cast("uint64_t *", self.cdata_leaflist.dflts)[-1] - for i in range(arr_length): - val = lib.lyd_value_get_canonical( - self.context.cdata, self.cdata_leaflist.dflts[i] - ) + for dflt in ly_array_iter(self.cdata_leaflist.dflts): + val = lib.lyd_value_get_canonical(self.context.cdata, dflt) if not val: yield None val = c2str(val) - val_type = Type(self.context, self.cdata_leaflist.dflts[i].realtype, None) + val_type = Type(self.context, dflt.realtype, None) if val_type == Type.BOOL: yield val == "true" elif val_type in Type.NUM_TYPES: From 32165ef8989afb1bb2384ab87f49973127fe5d86 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Thu, 18 Apr 2024 09:32:25 +0200 Subject: [PATCH 47/71] schema: fix of SLeafList defaults API This patch fixes defaults API of SLeafList. The returned values are now correctly converted to python types. Closes: https://github.com/CESNET/libyang-python/pull/119 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/schema.py | 4 ++-- tests/test_schema.py | 6 ++++++ tests/yang/yolo/yolo-nodetypes.yang | 11 +++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index 539af886..f35a5e9a 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1420,9 +1420,9 @@ def defaults(self) -> Iterator[Union[None, bool, int, str, float]]: yield None val = c2str(val) val_type = Type(self.context, dflt.realtype, None) - if val_type == Type.BOOL: + if val_type.base() == Type.BOOL: yield val == "true" - elif val_type in Type.NUM_TYPES: + elif val_type.base() in Type.NUM_TYPES: yield int(val) elif val_type.base() == Type.DEC64: yield float(val) diff --git a/tests/test_schema.py b/tests/test_schema.py index 6b171af2..e2a833ce 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -756,6 +756,12 @@ def test_leaflist_defaults(self): leaflist = next(self.ctx.find_path("/yolo-nodetypes:conf/ratios")) for d in leaflist.defaults(): self.assertIsInstance(d, float) + leaflist = next(self.ctx.find_path("/yolo-nodetypes:conf/bools")) + for d in leaflist.defaults(): + self.assertIsInstance(d, bool) + leaflist = next(self.ctx.find_path("/yolo-nodetypes:conf/integers")) + for d in leaflist.defaults(): + self.assertIsInstance(d, int) def test_leaf_list_min_max(self): leaflist1 = next(self.ctx.find_path("/yolo-nodetypes:conf/leaf-list1")) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index d9ef77de..0732ee72 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -45,6 +45,17 @@ module yolo-nodetypes { default 2.6; } + leaf-list bools { + type boolean; + default true; + } + + leaf-list integers { + type uint32; + default 10; + default 20; + } + list list1 { key leaf1; unique "leaf2 leaf3"; From f437bfb8a3ca4a705c76d8aabb70e815de354960 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Wed, 28 Feb 2024 13:09:20 +0100 Subject: [PATCH 48/71] context: add disable_searchdirs options This patch adds support to disable searching of local directories during loading of module. Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/context.py | 3 +++ tests/test_context.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/libyang/context.py b/libyang/context.py index 9e3f89be..f0471cb6 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -26,6 +26,7 @@ class Context: def __init__( self, search_path: Optional[str] = None, + disable_searchdirs: bool = False, disable_searchdir_cwd: bool = True, explicit_compile: Optional[bool] = False, leafref_extended: bool = False, @@ -38,6 +39,8 @@ def __init__( return # already initialized options = 0 + if disable_searchdirs: + options |= lib.LY_CTX_DISABLE_SEARCHDIRS if disable_searchdir_cwd: options |= lib.LY_CTX_DISABLE_SEARCHDIR_CWD if explicit_compile: diff --git a/tests/test_context.py b/tests/test_context.py index f43bf1cf..02d20a45 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -119,3 +119,8 @@ def test_context_dict(self): handle = ctx.add_to_dict(orig_str) self.assertEqual(orig_str, c2str(handle)) ctx.remove_from_dict(orig_str) + + def test_ctx_disable_searchdirs(self): + with Context(YANG_DIR, disable_searchdirs=True) as ctx: + with self.assertRaises(LibyangError): + ctx.load_module("yolo-nodetypes") From 47b0f0953031aedbd08ba3960c23f97dfecec4f1 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Sun, 3 Mar 2024 11:39:40 +0100 Subject: [PATCH 49/71] context: add leafref_linking options This patch adds leafref_linking context option, which allows the usage of lyd_leafref_get_links and lyd_leafref_link_node_tree functions, whose support will be added in the next commit. Closes: https://github.com/CESNET/libyang-python/pull/108 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 1 + libyang/context.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 39e44357..24688fca 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -15,6 +15,7 @@ struct ly_ctx; #define LY_CTX_REF_IMPLEMENTED ... #define LY_CTX_SET_PRIV_PARSED ... #define LY_CTX_LEAFREF_EXTENDED ... +#define LY_CTX_LEAFREF_LINKING ... typedef enum { diff --git a/libyang/context.py b/libyang/context.py index f0471cb6..1300a09c 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -30,6 +30,7 @@ def __init__( disable_searchdir_cwd: bool = True, explicit_compile: Optional[bool] = False, leafref_extended: bool = False, + leafref_linking: bool = False, yanglib_path: Optional[str] = None, yanglib_fmt: str = "json", cdata=None, # C type: "struct ly_ctx *" @@ -47,6 +48,8 @@ def __init__( options |= lib.LY_CTX_EXPLICIT_COMPILE if leafref_extended: options |= lib.LY_CTX_LEAFREF_EXTENDED + if leafref_linking: + options |= lib.LY_CTX_LEAFREF_LINKING # force priv parsed options |= lib.LY_CTX_SET_PRIV_PARSED From f14116c9216a448cdc2877fcd1e934bd9af1fbd2 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Sun, 3 Mar 2024 11:40:00 +0100 Subject: [PATCH 50/71] data: add leafref_nodes and leafref_link_node_tree This patch adds API, which allows user to: - determine if DNode is being referenced by some other leafref DNodes. - trigger process for creating cross-references between leafref DNodes and target DNodes. Closes: https://github.com/CESNET/libyang-python/pull/108 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 9 +++++++++ libyang/data.py | 24 +++++++++++++++++++++++- tests/test_data.py | 26 ++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 24688fca..6079ecff 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1248,5 +1248,14 @@ struct lyd_attr { LY_ERR lyd_new_attr(struct lyd_node *, const char *, const char *, const char *, struct lyd_attr **); void lyd_free_attr_single(const struct ly_ctx *ctx, struct lyd_attr *attr); +struct lyd_leafref_links_rec { + const struct lyd_node_term *node; + const struct lyd_node_term **leafref_nodes; + const struct lyd_node_term **target_nodes; +}; + +LY_ERR lyd_leafref_get_links(const struct lyd_node_term *e, const struct lyd_leafref_links_rec **); +LY_ERR lyd_leafref_link_node_tree(struct lyd_node *); + /* from libc, needed to free allocated strings */ void free(void *); diff --git a/libyang/data.py b/libyang/data.py index 96b792ba..00490db6 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -18,7 +18,7 @@ SRpc, Type, ) -from .util import DataType, IOType, LibyangError, c2str, str2c +from .util import DataType, IOType, LibyangError, c2str, ly_array_iter, str2c LOG = logging.getLogger(__name__) @@ -991,6 +991,28 @@ def free(self, with_siblings: bool = True) -> None: finally: self.cdata = ffi.NULL + def leafref_link_node_tree(self) -> None: + """ + Traverse through data tree including root node siblings and adds + leafrefs links to the given nodes. + + Requires leafref_linking to be set on the libyang context. + """ + lib.lyd_leafref_link_node_tree(self.cdata) + + def leafref_nodes(self) -> Iterator["DNode"]: + """ + Gets the leafref links record for given node. + + Requires leafref_linking to be set on the libyang context. + """ + term_node = ffi.cast("struct lyd_node_term *", self.cdata) + out = ffi.new("const struct lyd_leafref_links_rec **") + if lib.lyd_leafref_get_links(term_node, out) != lib.LY_SUCCESS: + return + for n in ly_array_iter(out[0].leafref_nodes): + yield DNode.new(self.context, n) + def __repr__(self): cls = self.__class__ return "<%s.%s: %s>" % (cls.__module__, cls.__name__, str(self)) diff --git a/tests/test_data.py b/tests/test_data.py index d43f212b..9ed35dc3 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -21,6 +21,7 @@ DRpc, IOType, LibyangError, + Module, ) from libyang.data import dict_to_dnode @@ -1066,3 +1067,28 @@ def test_dnode_attrs_set_and_remove_multiple(self): attrs.remove("ietf-netconf:operation") self.assertEqual(len(attrs), 0) + + def test_dnode_leafref_linking(self): + MAIN = """{ + "yolo-leafref-extended:list1": [{ + "leaf1": "val1", + "leaflist2": ["val2", "val3"] + }], + "yolo-leafref-extended:ref1": "val1" + }""" + self.ctx.destroy() + self.ctx = Context(YANG_DIR, leafref_extended=True, leafref_linking=True) + mod = self.ctx.load_module("yolo-leafref-extended") + self.assertIsInstance(mod, Module) + dnode1 = self.ctx.parse_data_mem(MAIN, "json", parse_only=True) + self.assertIsInstance(dnode1, DList) + dnode2 = next(dnode1.siblings(include_self=False)) + self.assertIsInstance(dnode2, DLeaf) + dnode3 = next(dnode1.children()) + self.assertIsInstance(dnode3, DLeaf) + self.assertIsNone(next(dnode3.leafref_nodes(), None)) + dnode2.leafref_link_node_tree() + dnode4 = next(dnode3.leafref_nodes()) + self.assertIsInstance(dnode4, DLeaf) + self.assertEqual(dnode4.cdata, dnode2.cdata) + dnode1.free() From 4831b70cadaa7238284f00b123cfe0cd9330c913 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Wed, 28 Feb 2024 13:09:20 +0100 Subject: [PATCH 51/71] context: adding ContextExternalModuleLoader class This patch adds class ContextExternalModuleLoader, which adds ability to add custom module load callback, which allows user to load modules from remote source etc. Closes: https://github.com/CESNET/libyang-python/pull/103 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 5 ++ libyang/context.py | 170 +++++++++++++++++++++++++++++++++++++++++- tests/test_context.py | 23 ++++++ 3 files changed, 196 insertions(+), 2 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 6079ecff..474b4c6a 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1071,6 +1071,11 @@ typedef enum { LY_ERR lys_parse(struct ly_ctx *, struct ly_in *, LYS_INFORMAT, const char **, struct lys_module **); LY_ERR ly_ctx_new_ylpath(const char *, const char *, LYD_FORMAT, int, struct ly_ctx **); LY_ERR ly_ctx_get_yanglib_data(const struct ly_ctx *, struct lyd_node **, const char *, ...); +typedef void (*ly_module_imp_data_free_clb)(void *, void *); +typedef LY_ERR (*ly_module_imp_clb)(const char *, const char *, const char *, const char *, void *, LYS_INFORMAT *, const char **, ly_module_imp_data_free_clb *); +void ly_ctx_set_module_imp_clb(struct ly_ctx *, ly_module_imp_clb, void *); +extern "Python" void lypy_module_imp_data_free_clb(void *, void *); +extern "Python" LY_ERR lypy_module_imp_clb(const char *, const char *, const char *, const char *, void *, LYS_INFORMAT *, const char **, ly_module_imp_data_free_clb *); LY_ERR lydict_insert(const struct ly_ctx *, const char *, size_t, const char **); LY_ERR lydict_remove(const struct ly_ctx *, const char *); diff --git a/libyang/context.py b/libyang/context.py index 1300a09c..94bba597 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: MIT import os -from typing import IO, Any, Iterator, Optional, Union +from typing import IO, Any, Callable, Iterator, Optional, Tuple, Union from _libyang import ffi, lib from .data import ( @@ -19,9 +19,173 @@ from .util import DataType, IOType, LibyangError, c2str, data_load, str2c +# ------------------------------------------------------------------------------------- +@ffi.def_extern(name="lypy_module_imp_data_free_clb") +def libyang_c_module_imp_data_free_clb(cdata, user_data): + instance = ffi.from_handle(user_data) + instance.free_module_data(cdata) + + +# ------------------------------------------------------------------------------------- +@ffi.def_extern(name="lypy_module_imp_clb") +def libyang_c_module_imp_clb( + mod_name, + mod_rev, + submod_name, + submod_rev, + user_data, + fmt, + module_data, + free_module_data, +): + """ + Implement the C callback function for loading modules from any location. + + :arg c_str mod_name: + The YANG module name + :arg c_str mod_rev: + The YANG module revision + :arg c_str submod_name: + The YANG submodule name + :arg c_str submod_rev: + The YANG submodule revision + :arg user_data: + The user data provided by user during registration. In this implementation + it is always considered to be handle of Python object + :arg fmt: + The output pointer where to set the format of schema + :arg module_data: + The output pointer where to set the schema data itself + :arg free_module_data: + The output pointer of callback function which will be called when the schema + data are no longer needed + + :returns: + The LY_SUCCESS in case the needed YANG (sub)module schema was found + The LY_ENOT in case the needed YANG (sub)module schema was not found + """ + fmt[0] = lib.LYS_IN_UNKNOWN + module_data[0] = ffi.NULL + free_module_data[0] = lib.lypy_module_imp_data_free_clb + instance = ffi.from_handle(user_data) + ret = instance.get_module_data( + c2str(mod_name), c2str(mod_rev), c2str(submod_name), c2str(submod_rev) + ) + if ret is None: + return lib.LY_ENOT + in_fmt, content = ret + fmt[0] = schema_in_format(in_fmt) + module_data[0] = content + return lib.LY_SUCCESS + + +# ------------------------------------------------------------------------------------- +class ContextExternalModuleLoader: + __slots__ = ( + "_cdata", + "_module_data_clb", + "_cffi_handle", + "_cdata_modules", + ) + + def __init__(self, cdata) -> None: + self._cdata = cdata # C type: "struct ly_ctx *" + self._module_data_clb = None + self._cffi_handle = ffi.new_handle(self) + self._cdata_modules = [] + + def free_module_data(self, cdata) -> None: + """ + Free previously stored data, obtained after a get_module_data. + + :arg cdata: + The pointer to YANG modelu schema (c_str), which shall be released from memory + """ + self._cdata_modules.remove(cdata) + + def get_module_data( + self, + mod_name: Optional[str], + mod_rev: Optional[str], + submod_name: Optional[str], + submod_rev: Optional[str], + ) -> Optional[Tuple[str, str]]: + """ + Get the YANG module schema data based requirements from libyang_c_module_imp_clb + function and forward that request to user Python based callback function. + + The returned data from callback function are stored within the context to make sure + of no memory access issues. These data a stored until the free_module_data function + is called directly by libyang. + + :arg self + This instance on context + :arg mod_name: + The optional YANG module name + :arg mod_rev: + The optional YANG module revision + :arg submod_name: + The optional YANG submodule name + :arg submod_rev: + The optional YANG submodule revision + + :returns: + Tuple of format string and YANG (sub)module schema + """ + if self._module_data_clb is None: + return "", None + fmt_str, module_data = self._module_data_clb( + mod_name, mod_rev, submod_name, submod_rev + ) + if module_data is None: + return fmt_str, None + module_data_c = str2c(module_data) + self._cdata_modules.append(module_data_c) + return fmt_str, module_data_c + + def set_module_data_clb( + self, + clb: Optional[ + Callable[ + [Optional[str], Optional[str], Optional[str], Optional[str]], + Optional[Tuple[str, str]], + ] + ] = None, + ) -> None: + """ + Set the callback function, which will be called if libyang context would like to + load module or submodule, which is not locally available in context path(s). + + :arg self + This instance on context + :arg clb: + The callback function. The expected arguments are: + mod_name: Module name + mod_rev: Module revision + submod_name: Submodule name + submod_rev: Submodule revision + The expeted return value is either: + tuple of: + format: The string format of the loaded data + data: The YANG (sub)module data as string + or None in case of error + """ + self._module_data_clb = clb + if clb is None: + lib.ly_ctx_set_module_imp_clb(self._cdata, ffi.NULL, ffi.NULL) + else: + lib.ly_ctx_set_module_imp_clb( + self._cdata, lib.lypy_module_imp_clb, self._cffi_handle + ) + + # ------------------------------------------------------------------------------------- class Context: - __slots__ = ("cdata", "__dict__") + __slots__ = ( + "cdata", + "external_module_loader", + "__dict__", + ) def __init__( self, @@ -37,6 +201,7 @@ def __init__( ): if cdata is not None: self.cdata = ffi.cast("struct ly_ctx *", cdata) + self.external_module_loader = ContextExternalModuleLoader(self.cdata) return # already initialized options = 0 @@ -90,6 +255,7 @@ def __init__( ) if not self.cdata: raise self.error("cannot create context") + self.external_module_loader = ContextExternalModuleLoader(self.cdata) def compile_schema(self): ret = lib.ly_ctx_compile(self.cdata) diff --git a/tests/test_context.py b/tests/test_context.py index 02d20a45..8a0412e2 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -124,3 +124,26 @@ def test_ctx_disable_searchdirs(self): with Context(YANG_DIR, disable_searchdirs=True) as ctx: with self.assertRaises(LibyangError): ctx.load_module("yolo-nodetypes") + + def test_ctx_using_clb(self): + def get_module_valid_clb(mod_name, *_): + YOLO_NODETYPES_MOD_PATH = os.path.join(YANG_DIR, "yolo/yolo-nodetypes.yang") + self.assertEqual(mod_name, "yolo-nodetypes") + with open(YOLO_NODETYPES_MOD_PATH, encoding="utf-8") as f: + mod_str = f.read() + return "yang", mod_str + + def get_module_invalid_clb(mod_name, *_): + return None + + with Context(YANG_DIR, disable_searchdirs=True) as ctx: + with self.assertRaises(LibyangError): + ctx.load_module("yolo-nodetypes") + + ctx.external_module_loader.set_module_data_clb(get_module_invalid_clb) + with self.assertRaises(LibyangError): + mod = ctx.load_module("yolo-nodetypes") + + ctx.external_module_loader.set_module_data_clb(get_module_valid_clb) + mod = ctx.load_module("yolo-nodetypes") + self.assertIsInstance(mod, Module) From 012d1448a1da862f5259d8f50d75b53a3ba84640 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Wed, 28 Feb 2024 13:09:20 +0100 Subject: [PATCH 52/71] context: fix of get_module_data This patch fixes broken test related to get_module_data function. Closes: https://github.com/CESNET/libyang-python/pull/103 Signed-off-by: Stefan Gula <steweg@gmail.com> --- libyang/context.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/libyang/context.py b/libyang/context.py index 94bba597..b457e7b8 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -133,12 +133,11 @@ def get_module_data( Tuple of format string and YANG (sub)module schema """ if self._module_data_clb is None: - return "", None - fmt_str, module_data = self._module_data_clb( - mod_name, mod_rev, submod_name, submod_rev - ) - if module_data is None: - return fmt_str, None + return None + ret = self._module_data_clb(mod_name, mod_rev, submod_name, submod_rev) + if ret is None: + return None + fmt_str, module_data = ret module_data_c = str2c(module_data) self._cdata_modules.append(module_data_c) return fmt_str, module_data_c From 0c43437c3ead3615a0b3f3a5c23b2bc5603b7f27 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Wed, 25 Sep 2024 15:18:09 +0200 Subject: [PATCH 53/71] cdefs: update with libyang master The libyang master branch has changed the lysp_ext_instance structure, update it. Link: https://github.com/CESNET/libyang/commit/35274cfbd Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 1 + 1 file changed, 1 insertion(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 474b4c6a..499a1b32 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -437,6 +437,7 @@ struct lysp_ext_instance { struct lysp_ext_substmt *substmts; void *parsed; struct lysp_stmt *child; + struct lysp_ext_instance *exts; }; struct lysp_import { From 5330a3ff0a01ed4118982f9bbd0c80b62bf07a99 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Mon, 12 Feb 2024 09:31:05 +0100 Subject: [PATCH 54/71] data: introduce store_only flags This patch introduces store_only flag, which allows user to only store value without performing any sort of additional type checks as length, range, pattern, etc. Closes: https://github.com/CESNET/libyang-python/pull/107 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 2 ++ libyang/context.py | 11 ++++++++++- libyang/data.py | 24 +++++++++++++++++++----- tests/test_data.py | 8 ++++++++ tests/yang/yolo/yolo-nodetypes.yang | 4 +++- 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 499a1b32..6f201878 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -264,6 +264,7 @@ struct lyd_node { LY_ERR lys_set_implemented(struct lys_module *, const char **); #define LYD_NEW_VAL_OUTPUT ... +#define LYD_NEW_VAL_STORE_ONLY ... #define LYD_NEW_VAL_BIN ... #define LYD_NEW_VAL_CANON ... #define LYD_NEW_META_CLEAR_DFLT ... @@ -308,6 +309,7 @@ LY_ERR lyd_print_all(struct ly_out *, const struct lyd_node *, LYD_FORMAT, uint3 #define LYD_PARSE_LYB_MOD_UPDATE ... #define LYD_PARSE_NO_STATE ... +#define LYD_PARSE_STORE_ONLY ... #define LYD_PARSE_ONLY ... #define LYD_PARSE_OPAQ ... #define LYD_PARSE_OPTS_MASK ... diff --git a/libyang/context.py b/libyang/context.py index b457e7b8..b08df0ca 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -409,6 +409,7 @@ def create_data_path( parent: Optional[DNode] = None, value: Any = None, update: bool = True, + store_only: bool = False, rpc_output: bool = False, force_return_value: bool = True, ) -> Optional[DNode]: @@ -419,7 +420,9 @@ def create_data_path( value = str(value).lower() elif not isinstance(value, str): value = str(value) - flags = newval_flags(update=update, rpc_output=rpc_output) + flags = newval_flags( + update=update, store_only=store_only, rpc_output=rpc_output + ) dnode = ffi.new("struct lyd_node **") ret = lib.lyd_new_path( parent.cdata if parent else ffi.NULL, @@ -513,6 +516,7 @@ def parse_data( strict: bool = False, validate_present: bool = False, validate_multi_error: bool = False, + store_only: bool = False, ) -> Optional[DNode]: if self.cdata is None: raise RuntimeError("context already destroyed") @@ -523,6 +527,7 @@ def parse_data( opaq=opaq, ordered=ordered, strict=strict, + store_only=store_only, ) validation_flgs = validation_flags( no_state=no_state, @@ -580,6 +585,7 @@ def parse_data_mem( strict: bool = False, validate_present: bool = False, validate_multi_error: bool = False, + store_only: bool = False, ) -> Optional[DNode]: return self.parse_data( fmt, @@ -594,6 +600,7 @@ def parse_data_mem( strict=strict, validate_present=validate_present, validate_multi_error=validate_multi_error, + store_only=store_only, ) def parse_data_file( @@ -609,6 +616,7 @@ def parse_data_file( strict: bool = False, validate_present: bool = False, validate_multi_error: bool = False, + store_only: bool = False, ) -> Optional[DNode]: return self.parse_data( fmt, @@ -623,6 +631,7 @@ def parse_data_file( strict=strict, validate_present=validate_present, validate_multi_error=validate_multi_error, + store_only=store_only, ) def __iter__(self) -> Iterator[Module]: diff --git a/libyang/data.py b/libyang/data.py index 00490db6..288303d6 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -79,6 +79,7 @@ def data_format(fmt_string: str) -> int: # ------------------------------------------------------------------------------------- def newval_flags( rpc_output: bool = False, + store_only: bool = False, bin_value: bool = False, canon_value: bool = False, meta_clear_default: bool = False, @@ -91,6 +92,8 @@ def newval_flags( flags = 0 if rpc_output: flags |= lib.LYD_NEW_VAL_OUTPUT + if store_only: + flags |= lib.LYD_NEW_VAL_STORE_ONLY if bin_value: flags |= lib.LYD_NEW_VAL_BIN if canon_value: @@ -112,6 +115,7 @@ def parser_flags( opaq: bool = False, ordered: bool = False, strict: bool = False, + store_only: bool = False, ) -> int: flags = 0 if lyb_mod_update: @@ -126,6 +130,8 @@ def parser_flags( flags |= lib.LYD_PARSE_ORDERED if strict: flags |= lib.LYD_PARSE_STRICT + if store_only: + flags |= lib.LYD_PARSE_STORE_ONLY return flags @@ -314,8 +320,10 @@ def meta_free(self, name): break item = item.next - def new_meta(self, name: str, value: str, clear_dflt: bool = False): - flags = newval_flags(meta_clear_default=clear_dflt) + def new_meta( + self, name: str, value: str, clear_dflt: bool = False, store_only: bool = False + ): + flags = newval_flags(meta_clear_default=clear_dflt, store_only=store_only) ret = lib.lyd_new_meta( ffi.NULL, self.cdata, @@ -391,6 +399,7 @@ def new_path( opt_opaq: bool = False, opt_bin_value: bool = False, opt_canon_value: bool = False, + opt_store_only: bool = False, ): flags = newval_flags( update=opt_update, @@ -398,6 +407,7 @@ def new_path( opaq=opt_opaq, bin_value=opt_bin_value, canon_value=opt_canon_value, + store_only=opt_store_only, ) ret = lib.lyd_new_path( self.cdata, ffi.NULL, str2c(path), str2c(value), flags, ffi.NULL @@ -1062,9 +1072,10 @@ def create_path( path: str, value: Any = None, rpc_output: bool = False, + store_only: bool = False, ) -> Optional[DNode]: return self.context.create_data_path( - path, parent=self, value=value, rpc_output=rpc_output + path, parent=self, value=value, rpc_output=rpc_output, store_only=store_only ) def children(self, no_keys=False) -> Iterator[DNode]: @@ -1188,6 +1199,7 @@ def dict_to_dnode( rpc: bool = False, rpcreply: bool = False, notification: bool = False, + store_only: bool = False, ) -> Optional[DNode]: """ Convert a python dictionary to a DNode object given a YANG module object. The return @@ -1214,6 +1226,8 @@ def dict_to_dnode( Data represents RPC or action output parameters. :arg notification: Data represents notification parameters. + :arg store_only: + Data are being stored regardless of type validation (length, range, pattern, etc.) """ if not dic: return None @@ -1235,7 +1249,7 @@ def _create_leaf(_parent, module, name, value, in_rpc_output=False): value = str(value) n = ffi.new("struct lyd_node **") - flags = newval_flags(rpc_output=in_rpc_output) + flags = newval_flags(rpc_output=in_rpc_output, store_only=store_only) ret = lib.lyd_new_term( _parent, module.cdata, @@ -1273,7 +1287,7 @@ def _create_container(_parent, module, name, in_rpc_output=False): def _create_list(_parent, module, name, key_values, in_rpc_output=False): n = ffi.new("struct lyd_node **") - flags = newval_flags(rpc_output=in_rpc_output) + flags = newval_flags(rpc_output=in_rpc_output, store_only=store_only) ret = lib.lyd_new_list( _parent, module.cdata, diff --git a/tests/test_data.py b/tests/test_data.py index 9ed35dc3..9515a58f 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1092,3 +1092,11 @@ def test_dnode_leafref_linking(self): self.assertIsInstance(dnode4, DLeaf) self.assertEqual(dnode4.cdata, dnode2.cdata) dnode1.free() + + def test_dnode_store_only(self): + MAIN = {"yolo-nodetypes:test1": 50} + module = self.ctx.load_module("yolo-nodetypes") + dnode = dict_to_dnode(MAIN, module, None, validate=False, store_only=True) + self.assertIsInstance(dnode, DLeaf) + self.assertEqual(dnode.value(), 50) + dnode.free() diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index 0732ee72..de4656b1 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -91,7 +91,9 @@ module yolo-nodetypes { } leaf test1 { - type uint8; + type uint8 { + range "2..20"; + } } grouping grp1 { From 551f44b5db419cfb02a1e34acb0ecc35d85615a0 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Sun, 3 Mar 2024 15:56:15 +0100 Subject: [PATCH 55/71] context: add builtin_plugins_only options This patch adds builtin_plugins_only option, which allows user to use only basic YANG type plugins, instead of using advanced plugins as ipv4-address etc. Signed-off-by: Stefan Gula <steweg@gmail.com> --- cffi/cdefs.h | 1 + libyang/context.py | 3 +++ tests/test_data.py | 11 +++++++++++ tests/yang/yolo/yolo-nodetypes.yang | 9 +++++++++ 4 files changed, 24 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 6f201878..93f68c79 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -16,6 +16,7 @@ struct ly_ctx; #define LY_CTX_SET_PRIV_PARSED ... #define LY_CTX_LEAFREF_EXTENDED ... #define LY_CTX_LEAFREF_LINKING ... +#define LY_CTX_BUILTIN_PLUGINS_ONLY ... typedef enum { diff --git a/libyang/context.py b/libyang/context.py index b08df0ca..1b1d4cda 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -194,6 +194,7 @@ def __init__( explicit_compile: Optional[bool] = False, leafref_extended: bool = False, leafref_linking: bool = False, + builtin_plugins_only: bool = False, yanglib_path: Optional[str] = None, yanglib_fmt: str = "json", cdata=None, # C type: "struct ly_ctx *" @@ -214,6 +215,8 @@ def __init__( options |= lib.LY_CTX_LEAFREF_EXTENDED if leafref_linking: options |= lib.LY_CTX_LEAFREF_LINKING + if builtin_plugins_only: + options |= lib.LY_CTX_BUILTIN_PLUGINS_ONLY # force priv parsed options |= lib.LY_CTX_SET_PRIV_PARSED diff --git a/tests/test_data.py b/tests/test_data.py index 9515a58f..790b5cd7 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1,6 +1,7 @@ # Copyright (c) 2020 6WIND S.A. # SPDX-License-Identifier: MIT +import gc import json import os import unittest @@ -1100,3 +1101,13 @@ def test_dnode_store_only(self): self.assertIsInstance(dnode, DLeaf) self.assertEqual(dnode.value(), 50) dnode.free() + + def test_dnode_builtin_plugins_only(self): + MAIN = {"yolo-nodetypes:ip-address": "test"} + self.tearDown() + gc.collect() + self.ctx = Context(YANG_DIR, builtin_plugins_only=True) + module = self.ctx.load_module("yolo-nodetypes") + dnode = dict_to_dnode(MAIN, module, None, validate=False, store_only=True) + self.assertIsInstance(dnode, DLeaf) + dnode.free() diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index de4656b1..0ec00a6f 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -3,6 +3,11 @@ module yolo-nodetypes { namespace "urn:yang:yolo:nodetypes"; prefix sys; + import ietf-inet-types { + prefix inet; + revision-date 2013-07-15; + } + description "YOLO Nodetypes."; @@ -126,4 +131,8 @@ module yolo-nodetypes { anydata any1 { when "../cont2"; } + + leaf ip-address { + type inet:ipv4-address; + } } From 8d64c70a21bdb7972c39d645cbb2c1717a0c0f45 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Mon, 4 Mar 2024 15:16:44 +0100 Subject: [PATCH 56/71] data: fixing broken DLeaf value and print_dict API This patch fixes broken APIs, which were using no longer valid structs and performing validation step instead of just checking the realtype of data. Closes: https://github.com/CESNET/libyang-python/pull/107 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 6 +++++- libyang/data.py | 52 ++++++++++++++++++++-------------------------- tests/test_data.py | 1 + 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 93f68c79..40d36692 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -902,9 +902,13 @@ struct lyd_value { ...; }; +struct lyd_value_union { + struct lyd_value value; + ...; +}; + const char * lyd_get_value(const struct lyd_node *); struct lyd_node* lyd_child(const struct lyd_node *); -LY_ERR lyd_value_validate(const struct ly_ctx *, const struct lysc_node *, const char *, size_t, const struct lyd_node *, const struct lysc_type **, const char **); LY_ERR lyd_find_path(const struct lyd_node *, const char *, ly_bool, struct lyd_node **); void lyd_free_siblings(struct lyd_node *); struct lyd_node* lyd_first_sibling(const struct lyd_node *); diff --git a/libyang/data.py b/libyang/data.py index 288303d6..19ef0ca7 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -1119,38 +1119,30 @@ def cdata_leaf_value(cdata, context: "libyang.Context" = None) -> Any: return None val = c2str(val) - term_node = ffi.cast("struct lyd_node_term *", cdata) - val_type = ffi.new("const struct lysc_type **", ffi.NULL) - - # get real value type - ctx = context.cdata if context else ffi.NULL - ret = lib.lyd_value_validate( - ctx, - term_node.schema, - str2c(val), - len(val), - ffi.NULL, - val_type, - ffi.NULL, - ) - - if ret in (lib.LY_SUCCESS, lib.LY_EINCOMPLETE): - val_type = val_type[0].basetype - if val_type in Type.STR_TYPES: - return val - if val_type in Type.NUM_TYPES: - return int(val) - if val_type == Type.BOOL: - return val == "true" - if val_type == Type.DEC64: - return float(val) - if val_type == Type.LEAFREF: - return DLeaf.cdata_leaf_value(cdata.value.leafref, context) - if val_type == Type.EMPTY: - return None + if cdata.schema == ffi.NULL: + # opaq node return val - raise TypeError("value type validation error") + node_term = ffi.cast("struct lyd_node_term *", cdata) + + # inspired from libyang lyd_value_validate + val_type = Type(context, node_term.value.realtype, None).base() + if val_type == Type.UNION: + val_type = Type( + context, node_term.value.subvalue.value.realtype, None + ).base() + + if val_type in Type.STR_TYPES: + return val + if val_type in Type.NUM_TYPES: + return int(val) + if val_type == Type.BOOL: + return val == "true" + if val_type == Type.DEC64: + return float(val) + if val_type == Type.EMPTY: + return None + return val # ------------------------------------------------------------------------------------- diff --git a/tests/test_data.py b/tests/test_data.py index 790b5cd7..1728042b 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1110,4 +1110,5 @@ def test_dnode_builtin_plugins_only(self): module = self.ctx.load_module("yolo-nodetypes") dnode = dict_to_dnode(MAIN, module, None, validate=False, store_only=True) self.assertIsInstance(dnode, DLeaf) + self.assertEqual(dnode.value(), "test") dnode.free() From adab39c83947c9c08d3a39d212f687f002b8c13e Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Mon, 5 Aug 2024 12:25:42 +0200 Subject: [PATCH 57/71] schema: adding parent_node API to extensions This patch adds parent_node API, which allows caller to get parent PNode from PExtension or SNode from Extension instance. Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 2 ++ libyang/__init__.py | 1 + libyang/schema.py | 16 ++++++++++++++++ tests/test_schema.py | 7 +++++++ 4 files changed, 26 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 40d36692..1c1d8f3b 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -174,6 +174,8 @@ enum ly_stmt { LY_STMT_ARG_VALUE }; +#define LY_STMT_NODE_MASK ... + #define LY_LOLOG ... #define LY_LOSTORE ... #define LY_LOSTORE_LAST ... diff --git a/libyang/__init__.py b/libyang/__init__.py index 26235de1..cab99712 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -67,6 +67,7 @@ from .log import configure_logging from .schema import ( Extension, + ExtensionParsed, Feature, IfAndFeatures, IfFeature, diff --git a/libyang/schema.py b/libyang/schema.py index f35a5e9a..a47974ee 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -411,6 +411,14 @@ def name(self) -> str: def module(self) -> Module: return self._module_from_parsed() + def parent_node(self) -> Optional["PNode"]: + if not bool(self.cdata.parent_stmt & lib.LY_STMT_NODE_MASK): + return None + try: + return PNode.new(self.context, self.cdata.parent, self.module()) + except LibyangError: + return None + # ------------------------------------------------------------------------------------- class ExtensionCompiled(Extension): @@ -428,6 +436,14 @@ def module(self) -> Module: raise self.context.error("cannot get module") return Module(self.context, self.cdata_def.module) + def parent_node(self) -> Optional["SNode"]: + if not bool(self.cdata.parent_stmt & lib.LY_STMT_NODE_MASK): + return None + try: + return SNode.new(self.context, self.cdata.parent) + except LibyangError: + return None + # ------------------------------------------------------------------------------------- class _EnumBit: diff --git a/tests/test_schema.py b/tests/test_schema.py index e2a833ce..1ae9fdfc 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -7,6 +7,7 @@ from libyang import ( Context, Extension, + ExtensionParsed, IfFeature, IfOrFeatures, IOType, @@ -496,6 +497,11 @@ def test_rpc_extensions(self): self.assertEqual(len(ext), 1) ext = self.rpc.get_extension("require-admin", prefix="omg-extensions") self.assertIsInstance(ext, Extension) + self.assertIsInstance(ext.parent_node(), SRpc) + parsed = self.rpc.parsed() + ext = parsed.get_extension("require-admin", prefix="omg-extensions") + self.assertIsInstance(ext, ExtensionParsed) + self.assertIsInstance(ext.parent_node(), PAction) def test_rpc_params(self): leaf = next(self.rpc.children()) @@ -609,6 +615,7 @@ def test_leaf_type_extensions(self): "type-desc", prefix="omg-extensions", arg_value="<protocol>" ) self.assertIsInstance(ext, Extension) + self.assertIsNone(ext.parent_node()) def test_leaf_type_enum(self): leaf = next( From 74e08c41e9b6f1f66a3874f1f1d462beda1b440d Mon Sep 17 00:00:00 2001 From: Ariel Otilibili <otilibil@eurecom.fr> Date: Fri, 25 Oct 2024 13:03:25 +0200 Subject: [PATCH 58/71] tox: bumped Python versions * `envlist` looks for Python 3.9 to 3.13; these are the ones being supported [1] * dropped 3.6 to 3.8 from CI/CD tests * removed `py3`: it is never used, since first found wins [2]. ``` $ tox -l format lint py39 py310 py311 py312 py313 pypy3 lydevel coverage ``` [1] https://devguide.python.org/versions/ [2] https://tox.wiki/en/stable/config.html#python-options Signed-off-by: Ariel Otilibili <otilibil@eurecom.fr> --- .github/workflows/ci.yml | 8 ++------ tox.ini | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 144e73b9..48be6a11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,12 +40,6 @@ jobs: strategy: matrix: include: - - python: 3.6 - toxenv: py36 - - python: 3.7 - toxenv: py37 - - python: 3.8 - toxenv: py38 - python: 3.9 toxenv: py39 - python: "3.10" @@ -54,6 +48,8 @@ jobs: toxenv: py311 - python: "3.12" toxenv: py312 + - python: "3.13" + toxenv: py313 - python: pypy3.9 toxenv: pypy3 steps: diff --git a/tox.ini b/tox.ini index 9fd15d15..b6b12c4c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = format,lint,py{36,37,38,39,310,311,312,py3,3},lydevel,coverage +envlist = format,lint,py{39,310,311,312,313,py3},lydevel,coverage skip_missing_interpreters = true isolated_build = true distdir = {toxinidir}/dist From 1afc2a63108e15431b4903d4f7c909bb8c6c1884 Mon Sep 17 00:00:00 2001 From: Jeremie Leska <jeremie.leska@6wind.com> Date: Wed, 6 Nov 2024 17:29:08 +0100 Subject: [PATCH 59/71] schema: add store_only parameter to parse_data_dict Sometimes it is necessary to store only the data. Signed-off-by: Jeremie Leska <jeremie.leska@6wind.com> Acked-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/schema.py | 2 ++ tests/test_data.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/libyang/schema.py b/libyang/schema.py index a47974ee..db0514d8 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -241,6 +241,7 @@ def parse_data_dict( rpc: bool = False, rpcreply: bool = False, notification: bool = False, + store_only: bool = False, ) -> "libyang.data.DNode": """ Convert a python dictionary to a DNode object following the schema of this @@ -276,6 +277,7 @@ def parse_data_dict( rpc=rpc, rpcreply=rpcreply, notification=notification, + store_only=store_only, ) diff --git a/tests/test_data.py b/tests/test_data.py index 1728042b..4b7914e9 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1112,3 +1112,22 @@ def test_dnode_builtin_plugins_only(self): self.assertIsInstance(dnode, DLeaf) self.assertEqual(dnode.value(), "test") dnode.free() + + def test_merge_store_only(self): + MAIN = {"yolo-nodetypes:test1": 50} + module = self.ctx.load_module("yolo-nodetypes") + dnode = module.parse_data_dict(MAIN, validate=False, store_only=True) + self.assertIsInstance(dnode, DLeaf) + self.assertEqual(dnode.value(), 50) + dnode.free() + + def test_merge_builtin_plugins_only(self): + MAIN = {"yolo-nodetypes:ip-address": "test"} + self.tearDown() + gc.collect() + self.ctx = Context(YANG_DIR, builtin_plugins_only=True) + module = self.ctx.load_module("yolo-nodetypes") + dnode = module.parse_data_dict(MAIN, validate=False, store_only=True) + self.assertIsInstance(dnode, DLeaf) + self.assertEqual(dnode.value(), "test") + dnode.free() From a19cab64fa2ea2143d85c3feb903fedbafe16160 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Mon, 5 Aug 2024 12:29:34 +0200 Subject: [PATCH 60/71] extension: add ExtensionPlugin class This patch introduces new class ExtensionPlugin, which is wrapper around libyang extension plugin, which allows user to define custom action for parsing, compiling, and freeing parsed or compiled extensions. Custom actions can also raise a new type of exception LibyangExtensionError, which allows proper translation of exception to libyang error codes and logging of error message Closes: https://github.com/CESNET/libyang-python/pull/116 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 55 ++++++++ libyang/__init__.py | 5 + libyang/extension.py | 216 +++++++++++++++++++++++++++++ libyang/log.py | 14 +- libyang/schema.py | 6 +- tests/test_diff.py | 3 + tests/test_extension.py | 193 ++++++++++++++++++++++++++ tests/yang/omg/omg-extensions.yang | 10 ++ tests/yang/yolo/yolo-system.yang | 2 + 9 files changed, 498 insertions(+), 6 deletions(-) create mode 100644 libyang/extension.py create mode 100644 tests/test_extension.py diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 1c1d8f3b..7210adbd 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -174,6 +174,8 @@ enum ly_stmt { LY_STMT_ARG_VALUE }; +#define LY_STMT_OP_MASK ... +#define LY_STMT_DATA_NODE_MASK ... #define LY_STMT_NODE_MASK ... #define LY_LOLOG ... @@ -359,6 +361,7 @@ LY_ERR lys_print_module(struct ly_out *, const struct lys_module *, LYS_OUTFORMA #define LYS_PRINT_SHRINK ... struct lys_module { + struct ly_ctx *ctx; const char *name; const char *revision; const char *ns; @@ -428,6 +431,22 @@ struct lysc_node_container { struct lysc_node_notif *notifs; }; +struct lysp_stmt { + const char *stmt; + const char *arg; + LY_VALUE_FORMAT format; + void *prefix_data; + struct lysp_stmt *next; + struct lysp_stmt *child; + uint16_t flags; + enum ly_stmt kw; +}; + +struct lysp_ext_substmt { + enum ly_stmt stmt; + ...; +}; + struct lysp_ext_instance { const char *name; const char *argument; @@ -1271,6 +1290,42 @@ struct lyd_leafref_links_rec { LY_ERR lyd_leafref_get_links(const struct lyd_node_term *e, const struct lyd_leafref_links_rec **); LY_ERR lyd_leafref_link_node_tree(struct lyd_node *); +const char *lyplg_ext_stmt2str(enum ly_stmt stmt); +const struct lysp_module *lyplg_ext_parse_get_cur_pmod(const struct lysp_ctx *); +struct ly_ctx *lyplg_ext_compile_get_ctx(const struct lysc_ctx *); +void lyplg_ext_parse_log(const struct lysp_ctx *, const struct lysp_ext_instance *, LY_LOG_LEVEL, LY_ERR, const char *, ...); +void lyplg_ext_compile_log(const struct lysc_ctx *, const struct lysc_ext_instance *, LY_LOG_LEVEL, LY_ERR, const char *, ...); +LY_ERR lyplg_ext_parse_extension_instance(struct lysp_ctx *, struct lysp_ext_instance *); +LY_ERR lyplg_ext_compile_extension_instance(struct lysc_ctx *, const struct lysp_ext_instance *, struct lysc_ext_instance *); +void lyplg_ext_pfree_instance_substatements(const struct ly_ctx *ctx, struct lysp_ext_substmt *substmts); +void lyplg_ext_cfree_instance_substatements(const struct ly_ctx *ctx, struct lysc_ext_substmt *substmts); +typedef LY_ERR (*lyplg_ext_parse_clb)(struct lysp_ctx *, struct lysp_ext_instance *); +typedef LY_ERR (*lyplg_ext_compile_clb)(struct lysc_ctx *, const struct lysp_ext_instance *, struct lysc_ext_instance *); +typedef void (*lyplg_ext_parse_free_clb)(const struct ly_ctx *, struct lysp_ext_instance *); +typedef void (*lyplg_ext_compile_free_clb)(const struct ly_ctx *, struct lysc_ext_instance *); +struct lyplg_ext { + const char *id; + lyplg_ext_parse_clb parse; + lyplg_ext_compile_clb compile; + lyplg_ext_parse_free_clb pfree; + lyplg_ext_compile_free_clb cfree; + ...; +}; + +struct lyplg_ext_record { + const char *module; + const char *revision; + const char *name; + struct lyplg_ext plugin; + ...; +}; + +#define LYPLG_EXT_API_VERSION ... +LY_ERR lyplg_add_extension_plugin(struct ly_ctx *, uint32_t, const struct lyplg_ext_record *); +extern "Python" LY_ERR lypy_lyplg_ext_parse_clb(struct lysp_ctx *, struct lysp_ext_instance *); +extern "Python" LY_ERR lypy_lyplg_ext_compile_clb(struct lysc_ctx *, const struct lysp_ext_instance *, struct lysc_ext_instance *); +extern "Python" void lypy_lyplg_ext_parse_free_clb(const struct ly_ctx *, struct lysp_ext_instance *); +extern "Python" void lypy_lyplg_ext_compile_free_clb(const struct ly_ctx *, struct lysc_ext_instance *); /* from libc, needed to free allocated strings */ void free(void *); diff --git a/libyang/__init__.py b/libyang/__init__.py index cab99712..7af2794c 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -63,10 +63,12 @@ UnitsRemoved, schema_diff, ) +from .extension import ExtensionPlugin, LibyangExtensionError from .keyed_list import KeyedList from .log import configure_logging from .schema import ( Extension, + ExtensionCompiled, ExtensionParsed, Feature, IfAndFeatures, @@ -144,6 +146,9 @@ "EnumRemoved", "Extension", "ExtensionAdded", + "ExtensionCompiled", + "ExtensionParsed", + "ExtensionPlugin", "ExtensionRemoved", "Feature", "IfAndFeatures", diff --git a/libyang/extension.py b/libyang/extension.py new file mode 100644 index 00000000..57f7cb2d --- /dev/null +++ b/libyang/extension.py @@ -0,0 +1,216 @@ +# Copyright (c) 2018-2019 Robin Jarry +# Copyright (c) 2020 6WIND S.A. +# Copyright (c) 2021 RACOM s.r.o. +# SPDX-License-Identifier: MIT + +from typing import Callable, Optional + +from _libyang import ffi, lib +from .context import Context +from .log import get_libyang_level +from .schema import ExtensionCompiled, ExtensionParsed, Module +from .util import LibyangError, c2str, str2c + + +# ------------------------------------------------------------------------------------- +extensions_plugins = {} + + +class LibyangExtensionError(LibyangError): + def __init__(self, message: str, ret: int, log_level: int) -> None: + super().__init__(message) + self.ret = ret + self.log_level = log_level + + +@ffi.def_extern(name="lypy_lyplg_ext_parse_clb") +def libyang_c_lyplg_ext_parse_clb(pctx, pext): + plugin = extensions_plugins[pext.record.plugin] + module_cdata = lib.lyplg_ext_parse_get_cur_pmod(pctx).mod + context = Context(cdata=module_cdata.ctx) + module = Module(context, module_cdata) + parsed_ext = ExtensionParsed(context, pext, module) + plugin.set_parse_ctx(pctx) + try: + plugin.parse_clb(module, parsed_ext) + return lib.LY_SUCCESS + except LibyangExtensionError as e: + ly_level = get_libyang_level(e.log_level) + if ly_level is None: + ly_level = lib.LY_EPLUGIN + e_str = str(e) + plugin.add_error_message(e_str) + lib.lyplg_ext_parse_log(pctx, pext, ly_level, e.ret, str2c(e_str)) + return e.ret + + +@ffi.def_extern(name="lypy_lyplg_ext_compile_clb") +def libyang_c_lyplg_ext_compile_clb(cctx, pext, cext): + plugin = extensions_plugins[pext.record.plugin] + context = Context(cdata=lib.lyplg_ext_compile_get_ctx(cctx)) + module = Module(context, cext.module) + parsed_ext = ExtensionParsed(context, pext, module) + compiled_ext = ExtensionCompiled(context, cext) + plugin.set_compile_ctx(cctx) + try: + plugin.compile_clb(parsed_ext, compiled_ext) + return lib.LY_SUCCESS + except LibyangExtensionError as e: + ly_level = get_libyang_level(e.log_level) + if ly_level is None: + ly_level = lib.LY_EPLUGIN + e_str = str(e) + plugin.add_error_message(e_str) + lib.lyplg_ext_compile_log(cctx, cext, ly_level, e.ret, str2c(e_str)) + return e.ret + + +@ffi.def_extern(name="lypy_lyplg_ext_parse_free_clb") +def libyang_c_lyplg_ext_parse_free_clb(ctx, pext): + plugin = extensions_plugins[pext.record.plugin] + context = Context(cdata=ctx) + parsed_ext = ExtensionParsed(context, pext, None) + plugin.parse_free_clb(parsed_ext) + + +@ffi.def_extern(name="lypy_lyplg_ext_compile_free_clb") +def libyang_c_lyplg_ext_compile_free_clb(ctx, cext): + plugin = extensions_plugins[getattr(cext, "def").plugin] + context = Context(cdata=ctx) + compiled_ext = ExtensionCompiled(context, cext) + plugin.compile_free_clb(compiled_ext) + + +class ExtensionPlugin: + ERROR_SUCCESS = lib.LY_SUCCESS + ERROR_MEM = lib.LY_EMEM + ERROR_INVALID_INPUT = lib.LY_EINVAL + ERROR_NOT_VALID = lib.LY_EVALID + ERROR_DENIED = lib.LY_EDENIED + ERROR_NOT = lib.LY_ENOT + + def __init__( + self, + module_name: str, + name: str, + id_str: str, + context: Optional[Context] = None, + parse_clb: Optional[Callable[[Module, ExtensionParsed], None]] = None, + compile_clb: Optional[ + Callable[[ExtensionParsed, ExtensionCompiled], None] + ] = None, + parse_free_clb: Optional[Callable[[ExtensionParsed], None]] = None, + compile_free_clb: Optional[Callable[[ExtensionCompiled], None]] = None, + ) -> None: + """ + Set the callback functions, which will be called if libyang will be processing + given extension defined by name from module defined by module_name. + + :arg self: + This instance of extension plugin + :arg module_name: + The name of module in which the extension is defined + :arg name: + The name of extension itself + :arg id_str: + The unique ID of extension plugin within the libyang context + :arg context: + The context in which the extension plugin will be used. If set to None, + the plugin will be used for all existing and even future contexts + :arg parse_clb: + The optional callback function of which will be called during extension parsing + Expected arguments are: + module: The module which is being parsed + extension: The exact extension instance + Expected raises: + LibyangExtensionError in case of processing error + :arg compile_clb: + The optional callback function of which will be called during extension compiling + Expected arguments are: + extension_parsed: The parsed extension instance + extension_compiled: The compiled extension instance + Expected raises: + LibyangExtensionError in case of processing error + :arg parse_free_clb + The optional callback function of which will be called during freeing of parsed extension + Expected arguments are: + extension: The parsed extension instance to be freed + :arg compile_free_clb + The optional callback function of which will be called during freeing of compiled extension + Expected arguments are: + extension: The compiled extension instance to be freed + """ + self.context = context + self.module_name = module_name + self.module_name_cstr = str2c(self.module_name) + self.name = name + self.name_cstr = str2c(self.name) + self.id_str = id_str + self.id_cstr = str2c(self.id_str) + self.parse_clb = parse_clb + self.compile_clb = compile_clb + self.parse_free_clb = parse_free_clb + self.compile_free_clb = compile_free_clb + self._error_messages = [] + self._pctx = ffi.NULL + self._cctx = ffi.NULL + + self.cdata = ffi.new("struct lyplg_ext_record[2]") + self.cdata[0].module = self.module_name_cstr + self.cdata[0].name = self.name_cstr + self.cdata[0].plugin.id = self.id_cstr + if self.parse_clb is not None: + self.cdata[0].plugin.parse = lib.lypy_lyplg_ext_parse_clb + if self.compile_clb is not None: + self.cdata[0].plugin.compile = lib.lypy_lyplg_ext_compile_clb + if self.parse_free_clb is not None: + self.cdata[0].plugin.pfree = lib.lypy_lyplg_ext_parse_free_clb + if self.compile_free_clb is not None: + self.cdata[0].plugin.cfree = lib.lypy_lyplg_ext_compile_free_clb + ret = lib.lyplg_add_extension_plugin( + context.cdata if context is not None else ffi.NULL, + lib.LYPLG_EXT_API_VERSION, + ffi.cast("const void *", self.cdata), + ) + if ret != lib.LY_SUCCESS: + raise LibyangError("Unable to add extension plugin") + if self.cdata[0].plugin not in extensions_plugins: + extensions_plugins[self.cdata[0].plugin] = self + + def __del__(self) -> None: + if self.cdata[0].plugin in extensions_plugins: + del extensions_plugins[self.cdata[0].plugin] + + @staticmethod + def stmt2str(stmt: int) -> str: + return c2str(lib.lyplg_ext_stmt2str(stmt)) + + def add_error_message(self, err_msg: str) -> None: + self._error_messages.append(err_msg) + + def clear_error_messages(self) -> None: + self._error_messages.clear() + + def set_parse_ctx(self, pctx) -> None: + self._pctx = pctx + + def set_compile_ctx(self, cctx) -> None: + self._cctx = cctx + + def parse_substmts(self, ext: ExtensionParsed) -> int: + return lib.lyplg_ext_parse_extension_instance(self._pctx, ext.cdata) + + def compile_substmts(self, pext: ExtensionParsed, cext: ExtensionCompiled) -> int: + return lib.lyplg_ext_compile_extension_instance( + self._cctx, pext.cdata, cext.cdata + ) + + def free_parse_substmts(self, ext: ExtensionParsed) -> None: + lib.lyplg_ext_pfree_instance_substatements( + self.context.cdata, ext.cdata.substmts + ) + + def free_compile_substmts(self, ext: ExtensionCompiled) -> None: + lib.lyplg_ext_cfree_instance_substatements( + self.context.cdata, ext.cdata.substmts + ) diff --git a/libyang/log.py b/libyang/log.py index b033ccaa..f92c70fd 100644 --- a/libyang/log.py +++ b/libyang/log.py @@ -19,6 +19,13 @@ } +def get_libyang_level(py_level): + for ly_lvl, py_lvl in LOG_LEVELS.items(): + if py_lvl == py_level: + return ly_lvl + return None + + @ffi.def_extern(name="lypy_log_cb") def libyang_c_logging_callback(level, msg, data_path, schema_path, line): args = [c2str(msg)] @@ -50,10 +57,9 @@ def configure_logging(enable_py_logger: bool, level: int = logging.ERROR) -> Non :arg level: Python logging level. By default only ERROR messages are stored/logged. """ - for ly_lvl, py_lvl in LOG_LEVELS.items(): - if py_lvl == level: - lib.ly_log_level(ly_lvl) - break + ly_level = get_libyang_level(level) + if ly_level is not None: + lib.ly_log_level(ly_level) if enable_py_logger: lib.ly_log_options(lib.LY_LOLOG | lib.LY_LOSTORE) lib.ly_set_log_clb(lib.lypy_log_cb) diff --git a/libyang/schema.py b/libyang/schema.py index db0514d8..eb076e23 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -374,7 +374,7 @@ def __str__(self): class Extension: __slots__ = ("context", "cdata", "__dict__") - def __init__(self, context: "libyang.Context", cdata, module_parent: Module = None): + def __init__(self, context: "libyang.Context", cdata): self.context = context self.cdata = cdata @@ -402,6 +402,8 @@ def __init__(self, context: "libyang.Context", cdata, module_parent: Module = No def _module_from_parsed(self) -> Module: prefix = c2str(self.cdata.name).split(":")[0] + if self.module_parent is None: + raise self.context.error("cannot get module") for cdata_imp_mod in ly_array_iter(self.module_parent.cdata.parsed.imports): if ffi.string(cdata_imp_mod.prefix).decode() == prefix: return Module(self.context, cdata_imp_mod.module) @@ -417,7 +419,7 @@ def parent_node(self) -> Optional["PNode"]: if not bool(self.cdata.parent_stmt & lib.LY_STMT_NODE_MASK): return None try: - return PNode.new(self.context, self.cdata.parent, self.module()) + return PNode.new(self.context, self.cdata.parent, self.module_parent) except LibyangError: return None diff --git a/tests/test_diff.py b/tests/test_diff.py index 49bf77a2..d4b7e87e 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -12,6 +12,7 @@ EnumRemoved, EnumStatusAdded, EnumStatusRemoved, + ExtensionAdded, NodeTypeAdded, NodeTypeRemoved, SNodeAdded, @@ -82,6 +83,8 @@ class DiffTest(unittest.TestCase): (EnumRemoved, "/yolo-system:state/url/proto"), (EnumStatusAdded, "/yolo-system:conf/url/proto"), (EnumStatusAdded, "/yolo-system:state/url/proto"), + (ExtensionAdded, "/yolo-system:conf/url/proto"), + (ExtensionAdded, "/yolo-system:state/url/proto"), (EnumStatusRemoved, "/yolo-system:conf/url/proto"), (EnumStatusRemoved, "/yolo-system:state/url/proto"), (SNodeAdded, "/yolo-system:conf/pill/red/out"), diff --git a/tests/test_extension.py b/tests/test_extension.py new file mode 100644 index 00000000..b932788c --- /dev/null +++ b/tests/test_extension.py @@ -0,0 +1,193 @@ +# Copyright (c) 2018-2019 Robin Jarry +# SPDX-License-Identifier: MIT + +import logging +import os +from typing import Any, Optional +import unittest + +from libyang import ( + Context, + ExtensionCompiled, + ExtensionParsed, + ExtensionPlugin, + LibyangError, + LibyangExtensionError, + Module, + PLeaf, + SLeaf, +) + + +YANG_DIR = os.path.join(os.path.dirname(__file__), "yang") + + +# ------------------------------------------------------------------------------------- +class TestExtensionPlugin(ExtensionPlugin): + def __init__(self, context: Context) -> None: + super().__init__( + "omg-extensions", + "type-desc", + "omg-extensions-type-desc-plugin-v1", + context, + parse_clb=self._parse_clb, + compile_clb=self._compile_clb, + parse_free_clb=self._parse_free_clb, + compile_free_clb=self._compile_free_clb, + ) + self.parse_clb_called = 0 + self.compile_clb_called = 0 + self.parse_free_clb_called = 0 + self.compile_free_clb_called = 0 + self.parse_clb_exception: Optional[LibyangExtensionError] = None + self.compile_clb_exception: Optional[LibyangExtensionError] = None + self.parse_parent_stmt = None + + def reset(self) -> None: + self.parse_clb_called = 0 + self.compile_clb_called = 0 + self.parse_free_clb_called = 0 + self.compile_free_clb_called = 0 + self.parse_clb_exception = None + self.compile_clb_exception = None + + def _parse_clb(self, module: Module, ext: ExtensionParsed) -> None: + self.parse_clb_called += 1 + if self.parse_clb_exception is not None: + raise self.parse_clb_exception + self.parse_substmts(ext) + self.parse_parent_stmt = self.stmt2str(ext.cdata.parent_stmt) + + def _compile_clb(self, pext: ExtensionParsed, cext: ExtensionCompiled) -> None: + self.compile_clb_called += 1 + if self.compile_clb_exception is not None: + raise self.compile_clb_exception + self.compile_substmts(pext, cext) + + def _parse_free_clb(self, ext: ExtensionParsed) -> None: + self.parse_free_clb_called += 1 + self.free_parse_substmts(ext) + + def _compile_free_clb(self, ext: ExtensionCompiled) -> None: + self.compile_free_clb_called += 1 + self.free_compile_substmts(ext) + + +# ------------------------------------------------------------------------------------- +class ExtensionTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.plugin = TestExtensionPlugin(self.ctx) + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_extension_basic(self): + self.ctx.load_module("yolo-system") + self.assertEqual(5, self.plugin.parse_clb_called) + self.assertEqual(6, self.plugin.compile_clb_called) + self.assertEqual(0, self.plugin.parse_free_clb_called) + self.assertEqual(0, self.plugin.compile_free_clb_called) + self.assertEqual("type", self.plugin.parse_parent_stmt) + self.ctx.destroy() + self.assertEqual(5, self.plugin.parse_clb_called) + self.assertEqual(6, self.plugin.compile_clb_called) + self.assertEqual(5, self.plugin.parse_free_clb_called) + self.assertEqual(6, self.plugin.compile_free_clb_called) + + def test_extension_invalid_parse(self): + self.plugin.parse_clb_exception = LibyangExtensionError( + "this extension cannot be parsed", + self.plugin.ERROR_NOT_VALID, + logging.ERROR, + ) + with self.assertRaises(LibyangError): + self.ctx.load_module("yolo-system") + + def test_extension_invalid_compile(self): + self.plugin.compile_clb_exception = LibyangExtensionError( + "this extension cannot be compiled", + self.plugin.ERROR_NOT_VALID, + logging.ERROR, + ) + with self.assertRaises(LibyangError): + self.ctx.load_module("yolo-system") + + +# ------------------------------------------------------------------------------------- +class ExampleParseExtensionPlugin(ExtensionPlugin): + def __init__(self, context: Context) -> None: + super().__init__( + "omg-extensions", + "parse-validation", + "omg-extensions-parse-validation-plugin-v1", + context, + parse_clb=self._parse_clb, + ) + + def _verify_single(self, parent: Any) -> None: + count = 0 + for e in parent.extensions(): + if e.name() == self.name and e.module().name() == self.module_name: + count += 1 + if count > 1: + raise LibyangExtensionError( + f"Extension {self.name} is allowed to be defined just once per given " + "parent node context.", + self.ERROR_NOT_VALID, + logging.ERROR, + ) + + def _parse_clb(self, _, ext: ExtensionParsed) -> None: + parent = ext.parent_node() + if not isinstance(parent, PLeaf): + raise LibyangExtensionError( + f"Extension {ext.name()} is allowed only in leaf nodes", + self.ERROR_NOT_VALID, + logging.ERROR, + ) + self._verify_single(parent) + # here you put code to perform something reasonable actions you need for your extension + + +class ExampleCompileExtensionPlugin(ExtensionPlugin): + def __init__(self, context: Context) -> None: + super().__init__( + "omg-extensions", + "compile-validation", + "omg-extensions-compile-validation-plugin-v1", + context, + compile_clb=self._compile_clb, + ) + + def _compile_clb(self, pext: ExtensionParsed, cext: ExtensionCompiled) -> None: + parent = cext.parent_node() + if not isinstance(parent, SLeaf): + raise LibyangExtensionError( + f"Extension {cext.name()} is allowed only in leaf nodes", + self.ERROR_NOT_VALID, + logging.ERROR, + ) + # here you put code to perform something reasonable actions you need for your extension + + +class ExtensionExampleTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.plugins = [] + + def tearDown(self): + self.plugins.clear() + self.ctx.destroy() + self.ctx = None + + def test_parse_validation_example(self): + self.plugins.append(ExampleParseExtensionPlugin(self.ctx)) + self.ctx.load_module("yolo-system") + + def test_compile_validation_example(self): + self.plugins.append(ExampleParseExtensionPlugin(self.ctx)) + self.plugins.append(ExampleCompileExtensionPlugin(self.ctx)) + with self.assertRaises(LibyangError): + self.ctx.load_module("yolo-system") diff --git a/tests/yang/omg/omg-extensions.yang b/tests/yang/omg/omg-extensions.yang index fe20e7e5..926bf3db 100644 --- a/tests/yang/omg/omg-extensions.yang +++ b/tests/yang/omg/omg-extensions.yang @@ -18,4 +18,14 @@ module omg-extensions { "Extend a type to add a desc."; argument name; } + + extension parse-validation { + description + "Example of parse-validation extension which should be put only under leaf nodes."; + } + + extension compile-validation { + description + "Example of compile-validation extension which should be put only under leaf nodes."; + } } diff --git a/tests/yang/yolo/yolo-system.yang b/tests/yang/yolo/yolo-system.yang index ef612546..36c76416 100644 --- a/tests/yang/yolo/yolo-system.yang +++ b/tests/yang/yolo/yolo-system.yang @@ -83,6 +83,7 @@ module yolo-system { type types:protocol { ext:type-desc "<protocol>"; } + ext:parse-validation; } leaf host { type string { @@ -114,6 +115,7 @@ module yolo-system { type string; } } + ext:compile-validation; } } From 76504c474a4e48762674245717d60dc24ea2d3d5 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Fri, 5 Apr 2024 12:08:31 +0200 Subject: [PATCH 61/71] schema: adds Identity and PIdentity classes This patch introduces Identity and PIdentity classes. It also adds identities() API to get list of identities from Module Closes: https://github.com/CESNET/libyang-python/pull/118 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 20 ++++ libyang/__init__.py | 4 + libyang/schema.py | 162 +++++++++++++++++++++++++--- tests/test_schema.py | 51 +++++++++ tests/yang/yolo/yolo-nodetypes.yang | 33 ++++++ 5 files changed, 256 insertions(+), 14 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 7210adbd..e48a8173 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -475,6 +475,16 @@ struct lysp_ext_instance { char rev[LY_REV_SIZE]; }; +struct lysp_ident { + const char *name; + struct lysp_qname *iffeatures; + const char **bases; + const char *dsc; + const char *ref; + struct lysp_ext_instance *exts; + uint16_t flags; +}; + struct lysp_feature { const char *name; struct lysp_qname *iffeatures; @@ -976,6 +986,16 @@ struct lysp_restr { struct lysp_ext_instance *exts; }; +struct lysc_ident { + const char *name; + const char *dsc; + const char *ref; + struct lys_module *module; + struct lysc_ident **derived; + struct lysc_ext_instance *exts; + uint16_t flags; +}; + struct lysc_type_num { const char *name; struct lysc_ext_instance *exts; diff --git a/libyang/__init__.py b/libyang/__init__.py index 7af2794c..ff15755c 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -71,6 +71,7 @@ ExtensionCompiled, ExtensionParsed, Feature, + Identity, IfAndFeatures, IfFeature, IfFeatureExpr, @@ -89,6 +90,7 @@ PContainer, PEnum, PGrouping, + PIdentity, PLeaf, PLeafList, PList, @@ -151,6 +153,7 @@ "ExtensionPlugin", "ExtensionRemoved", "Feature", + "Identity", "IfAndFeatures", "IfFeature", "IfFeatureExpr", @@ -184,6 +187,7 @@ "PContainer", "PEnum", "PGrouping", + "PIdentity", "PLeaf", "PLeafList", "PList", diff --git a/libyang/schema.py b/libyang/schema.py index eb076e23..992db0eb 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -172,6 +172,14 @@ def notifications(self) -> Iterator["PNotif"]: for n in ly_list_iter(self.cdata.parsed.notifs): yield PNotif(self.context, n, self) + def identities(self) -> Iterator["Identity"]: + for i in ly_array_iter(self.cdata.identities): + yield Identity(self.context, i) + + def parsed_identities(self) -> Iterator["PIdentity"]: + for i in ly_array_iter(self.cdata.parsed.identities): + yield PIdentity(self.context, i, self) + def __str__(self) -> str: return self.name() @@ -415,13 +423,16 @@ def name(self) -> str: def module(self) -> Module: return self._module_from_parsed() - def parent_node(self) -> Optional["PNode"]: - if not bool(self.cdata.parent_stmt & lib.LY_STMT_NODE_MASK): - return None - try: - return PNode.new(self.context, self.cdata.parent, self.module_parent) - except LibyangError: - return None + def parent_node(self) -> Optional[Union["PNode", "PIdentity"]]: + if self.cdata.parent_stmt == lib.LY_STMT_IDENTITY: + cdata = ffi.cast("struct lysp_ident *", self.cdata.parent) + return PIdentity(self.context, cdata, self.module_parent) + if bool(self.cdata.parent_stmt & lib.LY_STMT_NODE_MASK): + try: + return PNode.new(self.context, self.cdata.parent, self.module_parent) + except LibyangError: + return None + return None # ------------------------------------------------------------------------------------- @@ -440,13 +451,16 @@ def module(self) -> Module: raise self.context.error("cannot get module") return Module(self.context, self.cdata_def.module) - def parent_node(self) -> Optional["SNode"]: - if not bool(self.cdata.parent_stmt & lib.LY_STMT_NODE_MASK): - return None - try: - return SNode.new(self.context, self.cdata.parent) - except LibyangError: - return None + def parent_node(self) -> Optional[Union["SNode", "Identity"]]: + if self.cdata.parent_stmt == lib.LY_STMT_IDENTITY: + cdata = ffi.cast("struct lysc_ident *", self.cdata.parent) + return Identity(self.context, cdata) + if bool(self.cdata.parent_stmt & lib.LY_STMT_NODE_MASK): + try: + return SNode.new(self.context, self.cdata.parent) + except LibyangError: + return None + return None # ------------------------------------------------------------------------------------- @@ -624,6 +638,13 @@ def leafref_path(self) -> Optional["str"]: lr = ffi.cast("struct lysc_type_leafref *", self.cdata) return c2str(lib.lyxp_get_expr(lr.path)) + def identity_bases(self) -> Iterator["Identity"]: + if self.cdata.basetype != lib.LY_TYPE_IDENT: + return + ident = ffi.cast("struct lysc_type_identityref *", self.cdata) + for b in ly_array_iter(ident.bases): + yield Identity(self.context, b) + def typedef(self) -> "Typedef": if ":" in self.name(): module_prefix, type_name = self.name().split(":") @@ -870,6 +891,68 @@ def __str__(self): return self.name() +# ------------------------------------------------------------------------------------- +class Identity: + __slots__ = ("context", "cdata") + + def __init__(self, context: "libyang.Context", cdata): + self.context = context + self.cdata = cdata # C type: "struct lysc_ident *" + + def name(self) -> str: + return c2str(self.cdata.name) + + def description(self) -> Optional[str]: + return c2str(self.cdata.dsc) + + def reference(self) -> Optional[str]: + return c2str(self.cdata.ref) + + def module(self) -> Module: + return Module(self.context, self.cdata.module) + + def derived(self) -> Iterator["Identity"]: + for i in ly_array_iter(self.cdata.derived): + yield Identity(self.context, i) + + def extensions(self) -> Iterator[ExtensionCompiled]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionCompiled(self.context, ext) + + def get_extension( + self, name: str, prefix: Optional[str] = None, arg_value: Optional[str] = None + ) -> Optional[ExtensionCompiled]: + for ext in self.extensions(): + if ext.name() != name: + continue + if prefix is not None and ext.module().name() != prefix: + continue + if arg_value is not None and ext.argument() != arg_value: + continue + return ext + return None + + def deprecated(self) -> bool: + return bool(self.cdata.flags & lib.LYS_STATUS_DEPRC) + + def obsolete(self) -> bool: + return bool(self.cdata.flags & lib.LYS_STATUS_OBSLT) + + def status(self) -> str: + if self.cdata.flags & lib.LYS_STATUS_OBSLT: + return "obsolete" + if self.cdata.flags & lib.LYS_STATUS_DEPRC: + return "deprecated" + return "current" + + def __repr__(self): + cls = self.__class__ + return "<%s.%s: %s>" % (cls.__module__, cls.__name__, str(self)) + + def __str__(self): + return self.name() + + # ------------------------------------------------------------------------------------- class Feature: __slots__ = ("context", "cdata", "__dict__") @@ -1890,6 +1973,57 @@ def extensions(self) -> Iterator["ExtensionParsed"]: yield ExtensionParsed(self.context, ext, self.module) +# ------------------------------------------------------------------------------------- +class PIdentity: + __slots__ = ("context", "cdata", "module") + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + self.context = context + self.cdata = cdata # C type: "struct lysp_ident *" + self.module = module + + def name(self) -> str: + return c2str(self.cdata.name) + + def if_features(self) -> Iterator[IfFeatureExpr]: + for f in ly_array_iter(self.cdata.iffeatures): + yield IfFeatureExpr(self.context, f, list(self.module.features())) + + def bases(self) -> Iterator[str]: + for b in ly_array_iter(self.cdata.bases): + yield c2str(b) + + def description(self) -> Optional[str]: + return c2str(self.cdata.dsc) + + def reference(self) -> Optional[str]: + return c2str(self.cdata.ref) + + def extensions(self) -> Iterator[ExtensionParsed]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionParsed(self.context, ext, self.module) + + def deprecated(self) -> bool: + return bool(self.cdata.flags & lib.LYS_STATUS_DEPRC) + + def obsolete(self) -> bool: + return bool(self.cdata.flags & lib.LYS_STATUS_OBSLT) + + def status(self) -> str: + if self.cdata.flags & lib.LYS_STATUS_OBSLT: + return "obsolete" + if self.cdata.flags & lib.LYS_STATUS_DEPRC: + return "deprecated" + return "current" + + def __repr__(self): + cls = self.__class__ + return "<%s.%s: %s>" % (cls.__module__, cls.__name__, str(self)) + + def __str__(self): + return self.name() + + # ------------------------------------------------------------------------------------- class PNode: CONTAINER = lib.LYS_CONTAINER diff --git a/tests/test_schema.py b/tests/test_schema.py index 1ae9fdfc..923f5f07 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -8,6 +8,7 @@ Context, Extension, ExtensionParsed, + Identity, IfFeature, IfOrFeatures, IOType, @@ -23,6 +24,7 @@ PChoice, PContainer, PGrouping, + PIdentity, PLeaf, PLeafList, PList, @@ -868,3 +870,52 @@ def test_notification_parsed(self): self.assertIsNone(next(pnode.typedefs(), None)) self.assertIsNone(next(pnode.groupings(), None)) self.assertIsNotNone(next(iter(pnode))) + + +# ------------------------------------------------------------------------------------- +class IdentityTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.module = self.ctx.load_module("yolo-nodetypes") + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_identity_compiled(self): + sidentity = next(self.module.identities()) + self.assertIsInstance(sidentity, Identity) + self.assertEqual(sidentity.name(), "base1") + self.assertEqual(sidentity.description(), "Base 1.") + self.assertEqual(sidentity.reference(), "Some reference.") + self.assertIsInstance(sidentity.module(), Module) + derived = list(sidentity.derived()) + self.assertEqual(2, len(derived)) + for i in derived: + self.assertIsInstance(i, Identity) + self.assertEqual(derived[0].name(), "derived1") + self.assertEqual(derived[1].name(), "derived2") + self.assertEqual(next(derived[1].extensions()).name(), "identity-name") + self.assertIsNone(next(sidentity.extensions(), None)) + self.assertIsNone(sidentity.get_extension("ext1")) + self.assertFalse(sidentity.deprecated()) + self.assertFalse(sidentity.obsolete()) + self.assertEqual("current", sidentity.status()) + + snode = next(self.ctx.find_path("/yolo-nodetypes:identity_ref")) + identities = list(snode.type().identity_bases()) + self.assertEqual(identities[0].name(), sidentity.name()) + self.assertEqual(identities[1].name(), "base2") + + def test_identity_parsed(self): + pidentity = next(self.module.parsed_identities()) + self.assertIsInstance(pidentity, PIdentity) + self.assertEqual(pidentity.name(), "base1") + self.assertIsNone(next(pidentity.if_features(), None)) + self.assertIsNone(next(pidentity.bases(), None)) + self.assertEqual(pidentity.description(), "Base 1.") + self.assertEqual(pidentity.reference(), "Some reference.") + self.assertIsNone(next(pidentity.extensions(), None)) + self.assertFalse(pidentity.deprecated()) + self.assertFalse(pidentity.obsolete()) + self.assertEqual("current", pidentity.status()) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index 0ec00a6f..5b994752 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -132,6 +132,39 @@ module yolo-nodetypes { when "../cont2"; } + extension identity-name { + description + "Extend an identity to provide an alternative name."; + argument name; + } + + identity base1 { + description + "Base 1."; + reference "Some reference."; + } + identity base2; + + identity derived1 { + base base1; + } + + identity derived2 { + base base1; + sys:identity-name "Derived2"; + } + + identity derived3 { + base derived1; + } + + leaf identity_ref { + type identityref { + base base1; + base base2; + } + } + leaf ip-address { type inet:ipv4-address; } From 421d3211040087dc5b881c7a5930e37c18ca2864 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Thu, 19 Dec 2024 10:26:56 +0100 Subject: [PATCH 62/71] cdefs: removal of unnecessary variable name defintion This patch aligns the cdefs definition with the rest of the file structure Signed-off-by: Stefan Gula <steweg@gmail.com> --- cffi/cdefs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index e48a8173..fad8d549 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1308,7 +1308,7 @@ struct lyd_leafref_links_rec { const struct lyd_node_term **target_nodes; }; -LY_ERR lyd_leafref_get_links(const struct lyd_node_term *e, const struct lyd_leafref_links_rec **); +LY_ERR lyd_leafref_get_links(const struct lyd_node_term *, const struct lyd_leafref_links_rec **); LY_ERR lyd_leafref_link_node_tree(struct lyd_node *); const char *lyplg_ext_stmt2str(enum ly_stmt stmt); const struct lysp_module *lyplg_ext_parse_get_cur_pmod(const struct lysp_ctx *); From 407fab3727084e57105af9152f3122aba4d099e5 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Thu, 19 Dec 2024 10:25:23 +0100 Subject: [PATCH 63/71] schema: adds nested extensions access This patch introduces access to nested extensions Signed-off-by: Stefan Gula <steweg@gmail.com> --- libyang/schema.py | 8 ++++++++ tests/test_schema.py | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/libyang/schema.py b/libyang/schema.py index 992db0eb..2e3a1154 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -434,6 +434,10 @@ def parent_node(self) -> Optional[Union["PNode", "PIdentity"]]: return None return None + def extensions(self) -> Iterator["ExtensionParsed"]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionParsed(self.context, ext, self.module_parent) + # ------------------------------------------------------------------------------------- class ExtensionCompiled(Extension): @@ -462,6 +466,10 @@ def parent_node(self) -> Optional[Union["SNode", "Identity"]]: return None return None + def extensions(self) -> Iterator["ExtensionCompiled"]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionCompiled(self.context, ext) + # ------------------------------------------------------------------------------------- class _EnumBit: diff --git a/tests/test_schema.py b/tests/test_schema.py index 923f5f07..a310aadc 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -7,6 +7,7 @@ from libyang import ( Context, Extension, + ExtensionCompiled, ExtensionParsed, Identity, IfFeature, @@ -498,12 +499,14 @@ def test_rpc_extensions(self): ext = list(self.rpc.extensions()) self.assertEqual(len(ext), 1) ext = self.rpc.get_extension("require-admin", prefix="omg-extensions") - self.assertIsInstance(ext, Extension) + self.assertIsInstance(ext, ExtensionCompiled) self.assertIsInstance(ext.parent_node(), SRpc) + self.assertIsNone(next(ext.extensions(), None)) parsed = self.rpc.parsed() ext = parsed.get_extension("require-admin", prefix="omg-extensions") self.assertIsInstance(ext, ExtensionParsed) self.assertIsInstance(ext.parent_node(), PAction) + self.assertIsNone(next(ext.extensions(), None)) def test_rpc_params(self): leaf = next(self.rpc.children()) From 727927bb90d117baf0598b67e4670b5f5d937f34 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Fri, 5 Apr 2024 11:36:52 +0200 Subject: [PATCH 64/71] data/context: adds json_null parsing option This patch adds options to support JSON 'null' values Signed-off-by: Stefan Gula <steweg@gmail.com> --- cffi/cdefs.h | 1 + libyang/context.py | 6 ++++++ libyang/data.py | 3 +++ tests/test_data.py | 6 ++++++ 4 files changed, 16 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index fad8d549..aa750042 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -315,6 +315,7 @@ LY_ERR lyd_print_all(struct ly_out *, const struct lyd_node *, LYD_FORMAT, uint3 #define LYD_PARSE_LYB_MOD_UPDATE ... #define LYD_PARSE_NO_STATE ... #define LYD_PARSE_STORE_ONLY ... +#define LYD_PARSE_JSON_NULL ... #define LYD_PARSE_ONLY ... #define LYD_PARSE_OPAQ ... #define LYD_PARSE_OPTS_MASK ... diff --git a/libyang/context.py b/libyang/context.py index 1b1d4cda..ba3332a3 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -520,6 +520,7 @@ def parse_data( validate_present: bool = False, validate_multi_error: bool = False, store_only: bool = False, + json_null: bool = False, ) -> Optional[DNode]: if self.cdata is None: raise RuntimeError("context already destroyed") @@ -531,6 +532,7 @@ def parse_data( ordered=ordered, strict=strict, store_only=store_only, + json_null=json_null, ) validation_flgs = validation_flags( no_state=no_state, @@ -589,6 +591,7 @@ def parse_data_mem( validate_present: bool = False, validate_multi_error: bool = False, store_only: bool = False, + json_null: bool = False, ) -> Optional[DNode]: return self.parse_data( fmt, @@ -604,6 +607,7 @@ def parse_data_mem( validate_present=validate_present, validate_multi_error=validate_multi_error, store_only=store_only, + json_null=json_null, ) def parse_data_file( @@ -620,6 +624,7 @@ def parse_data_file( validate_present: bool = False, validate_multi_error: bool = False, store_only: bool = False, + json_null: bool = False, ) -> Optional[DNode]: return self.parse_data( fmt, @@ -635,6 +640,7 @@ def parse_data_file( validate_present=validate_present, validate_multi_error=validate_multi_error, store_only=store_only, + json_null=json_null, ) def __iter__(self) -> Iterator[Module]: diff --git a/libyang/data.py b/libyang/data.py index 19ef0ca7..0d63d3c9 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -116,6 +116,7 @@ def parser_flags( ordered: bool = False, strict: bool = False, store_only: bool = False, + json_null: bool = False, ) -> int: flags = 0 if lyb_mod_update: @@ -132,6 +133,8 @@ def parser_flags( flags |= lib.LYD_PARSE_STRICT if store_only: flags |= lib.LYD_PARSE_STORE_ONLY + if json_null: + flags |= lib.LYD_PARSE_JSON_NULL return flags diff --git a/tests/test_data.py b/tests/test_data.py index 4b7914e9..1479eb9a 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1131,3 +1131,9 @@ def test_merge_builtin_plugins_only(self): self.assertIsInstance(dnode, DLeaf) self.assertEqual(dnode.value(), "test") dnode.free() + + def test_dnode_parse_json_null(self): + JSON = """{"yolo-nodetypes:ip-address": null}""" + dnode = self.ctx.parse_data_mem(JSON, "json", json_null=True) + dnode_names = [d.name() for d in dnode.siblings()] + self.assertFalse("ip-address" in dnode_names) From aea34d1e787163248c7b5de38e22d5e0a77c83f4 Mon Sep 17 00:00:00 2001 From: Robin Jarry <robin@jarry.cc> Date: Thu, 6 Feb 2025 09:20:10 +0100 Subject: [PATCH 65/71] tox: fix lint target on python 3.13 Update all packages used in the lint target to their latest release. Explicitly bump the ubuntu image version in all GitHub action jobs. Fix errors reported by new pylint/black versions. Signed-off-by: Robin Jarry <robin@jarry.cc> --- .github/workflows/ci.yml | 24 +++++++++++++++--------- Makefile | 5 ++--- libyang/schema.py | 4 +--- libyang/util.py | 2 ++ libyang/xpath.py | 2 +- pylintrc | 1 + tox.ini | 18 +++++++++--------- 7 files changed, 31 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48be6a11..a7797c88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,11 +3,17 @@ --- name: CI -on: [push, pull_request] +on: + pull_request: + branches: + - master + push: + branches: + - master jobs: lint: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -23,24 +29,24 @@ jobs: - run: python -m tox -e lint check-commits: - if: github.event_name == 'pull_request' + if: ${{ github.event.pull_request.commits }} runs-on: ubuntu-latest env: - LYPY_START_COMMIT: "${{ github.event.pull_request.base.sha }}" - LYPY_END_COMMIT: "${{ github.event.pull_request.head.sha }}" + LYPY_COMMIT_RANGE: "HEAD~${{ github.event.pull_request.commits }}.." steps: - run: sudo apt-get install git make - uses: actions/checkout@v4 with: fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.ref }} - run: make check-commits test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: matrix: include: - - python: 3.9 + - python: "3.9" toxenv: py39 - python: "3.10" toxenv: py310 @@ -56,7 +62,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python }} + python-version: "${{ matrix.python }}" - uses: actions/cache@v3 with: path: ~/.cache/pip @@ -86,7 +92,7 @@ jobs: deploy: needs: [lint, test] if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-latest + runs-on: ubuntu-24.03 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 diff --git a/Makefile b/Makefile index 6e9c436b..fc9a6ea3 100644 --- a/Makefile +++ b/Makefile @@ -12,10 +12,9 @@ tests: format: tox -e format -LYPY_START_COMMIT ?= origin/master -LYPY_END_COMMIT ?= HEAD +LYPY_COMMIT_RANGE ?= origin/master.. check-commits: - ./check-commits.sh $(LYPY_START_COMMIT)..$(LYPY_END_COMMIT) + ./check-commits.sh $(LYPY_COMMIT_RANGE) .PHONY: lint tests format check-commits diff --git a/libyang/schema.py b/libyang/schema.py index 2e3a1154..3f3bd4d0 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1542,9 +1542,7 @@ def defaults(self) -> Iterator[Union[None, bool, int, str, float]]: def max_elements(self) -> Optional[int]: return ( - self.cdata_leaflist.max - if self.cdata_leaflist.max != (2**32 - 1) - else None + self.cdata_leaflist.max if self.cdata_leaflist.max != (2**32 - 1) else None ) def min_elements(self) -> int: diff --git a/libyang/util.py b/libyang/util.py index d640a511..c380ae5e 100644 --- a/libyang/util.py +++ b/libyang/util.py @@ -121,4 +121,6 @@ def data_load(in_type, in_data, data, data_keepalive, encode=True): c_str = str2c(in_data, encode=encode) data_keepalive.append(c_str) ret = lib.ly_in_new_memory(c_str, data) + else: + raise ValueError("invalid input") return ret diff --git a/libyang/xpath.py b/libyang/xpath.py index ad5f2a58..e2a33196 100644 --- a/libyang/xpath.py +++ b/libyang/xpath.py @@ -85,7 +85,7 @@ def xpath_split(xpath: str) -> Iterator[Tuple[str, str, List[Tuple[str, str]]]]: # ------------------------------------------------------------------------------------- def _xpath_keys_to_key_name( - keys: List[Tuple[str, str]] + keys: List[Tuple[str, str]], ) -> Optional[Union[str, Tuple[str, ...]]]: """ Extract key name from parsed xpath keys returned by xpath_split. The return value diff --git a/pylintrc b/pylintrc index 16a9f0ae..acf27338 100644 --- a/pylintrc +++ b/pylintrc @@ -74,6 +74,7 @@ disable= too-many-branches, too-many-lines, too-many-locals, + too-many-positional-arguments, too-many-return-statements, too-many-statements, unused-argument, diff --git a/tox.ini b/tox.ini index b6b12c4c..524ad49c 100644 --- a/tox.ini +++ b/tox.ini @@ -36,8 +36,8 @@ basepython = python3 description = Format python code using isort and black. changedir = . deps = - black~=23.12.1 - isort~=5.13.2 + black~=25.1.0 + isort~=6.0.0 skip_install = true install_command = python3 -m pip install {opts} {packages} allowlist_externals = @@ -52,14 +52,14 @@ basepython = python3 description = Run coding style checks. changedir = . deps = - astroid~=3.0.2 - black~=23.12.1 - flake8~=7.0.0 - isort~=5.13.2 - pycodestyle~=2.11.1 + astroid~=3.3.8 + black~=25.1.0 + flake8~=7.1.1 + isort~=6.0.0 + pycodestyle~=2.12.1 pyflakes~=3.2.0 - pylint~=3.0.3 - setuptools~=69.0.3 + pylint~=3.3.4 + setuptools~=75.8.0 allowlist_externals = /bin/sh /usr/bin/sh From 8534053f8a8a542127e8f74acbcc095591b411e9 Mon Sep 17 00:00:00 2001 From: Brad House <brad@brad-house.com> Date: Wed, 5 Feb 2025 19:38:43 -0500 Subject: [PATCH 66/71] context: correct memory leak of `struct ly_in *` objects `data_load()` returns a reference to `struct ly_in *` which must be free'd by `lys_in_free()`. This is done in one of three locations. This corrects this oversight. This was found during unit tests of SONiC during porting to libyang3 from libyang 1.0.73. Signed-off-by: Brad House <brad@brad-house.com> --- libyang/context.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libyang/context.py b/libyang/context.py index ba3332a3..fb4a330d 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -325,7 +325,9 @@ def parse_module( mod = ffi.new("struct lys_module **") fmt = schema_in_format(fmt) - if lib.lys_parse(self.cdata, data[0], fmt, feat, mod) != lib.LY_SUCCESS: + ret = lib.lys_parse(self.cdata, data[0], fmt, feat, mod) + lib.ly_in_free(data[0], 0) + if ret != lib.LY_SUCCESS: raise self.error("failed to parse module") return Module(self, mod[0]) @@ -489,6 +491,7 @@ def parse_op( par[0] = parent.cdata ret = lib.lyd_parse_op(self.cdata, par[0], data[0], fmt, dtype, tree, op) + lib.ly_in_free(data[0], 0) if ret != lib.LY_SUCCESS: raise self.error("failed to parse input data") From bdd4cea20a05aa490cff6728b11fa3ba83242e52 Mon Sep 17 00:00:00 2001 From: Robin Jarry <robin@jarry.cc> Date: Mon, 17 Feb 2025 14:19:41 +0100 Subject: [PATCH 67/71] ci: fix issue status check For some reason, storing the output of curl into a bash variable breaks UTF-8 encoding. Use a shell pipeline to prevent any encoding from happening. Signed-off-by: Robin Jarry <robin@jarry.cc> --- .github/workflows/ci.yml | 4 ++-- check-commits.sh | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7797c88..27efc7dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,11 +30,11 @@ jobs: check-commits: if: ${{ github.event.pull_request.commits }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: LYPY_COMMIT_RANGE: "HEAD~${{ github.event.pull_request.commits }}.." steps: - - run: sudo apt-get install git make + - run: sudo apt-get install git make jq curl - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/check-commits.sh b/check-commits.sh index acf322d9..936236cf 100755 --- a/check-commits.sh +++ b/check-commits.sh @@ -42,11 +42,10 @@ err() { } check_issue() { - json=$(curl -f -X GET -L --no-progress-meter \ + curl -f -X GET -L --no-progress-meter \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "$api_url/issues/${1##*/}") || return 1 - test $(echo "$json" | jq -r .state) = open + "$api_url/issues/${1##*/}" | jq -r .state | grep -Fx open } for rev in $revisions; do From c5af6814f2388e78f4d7f48a322d73cf0b96ee40 Mon Sep 17 00:00:00 2001 From: Matthias Breuninger <matthias.breuninger@etas.com> Date: Mon, 17 Feb 2025 11:57:15 +0100 Subject: [PATCH 68/71] load_module: add missing parameters for ly_ctx_load_module Add parameters to define Yang model revision and features that shall be enabled. Closes: https://github.com/CESNET/libyang-python/issues/101 Signed-off-by: Matthias Breuninger <matthias.breuninger@etas.com> --- libyang/context.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/libyang/context.py b/libyang/context.py index fb4a330d..f9bd5a57 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: MIT import os -from typing import IO, Any, Callable, Iterator, Optional, Tuple, Union +from typing import IO, Any, Callable, Iterator, Optional, Sequence, Tuple, Union from _libyang import ffi, lib from .data import ( @@ -340,10 +340,19 @@ def parse_module_file( def parse_module_str(self, s: str, fmt: str = "yang", features=None) -> Module: return self.parse_module(s, IOType.MEMORY, fmt, features) - def load_module(self, name: str) -> Module: + def load_module( + self, + name: str, + revision: Optional[str] = None, + enabled_features: Sequence[str] = (), + ) -> Module: if self.cdata is None: raise RuntimeError("context already destroyed") - mod = lib.ly_ctx_load_module(self.cdata, str2c(name), ffi.NULL, ffi.NULL) + if enabled_features: + features = tuple([str2c(f) for f in enabled_features] + [ffi.NULL]) + else: + features = ffi.NULL + mod = lib.ly_ctx_load_module(self.cdata, str2c(name), str2c(revision), features) if mod == ffi.NULL: raise self.error("cannot load module") From 3f4ef9f2dea81f82bc3771ef04921e5148a1e158 Mon Sep 17 00:00:00 2001 From: Matthias Breuninger <matthias.breuninger@etas.com> Date: Mon, 17 Feb 2025 12:04:55 +0100 Subject: [PATCH 69/71] tests: add test to enable all features Expects that all features are enabled due to the "*" input. Closes: https://github.com/CESNET/libyang-python/issues/101 Signed-off-by: Matthias Breuninger <matthias.breuninger@etas.com> --- tests/test_context.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_context.py b/tests/test_context.py index 8a0412e2..db03c329 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -63,6 +63,13 @@ def test_ctx_load_module(self): mod = ctx.load_module("yolo-system") self.assertIsInstance(mod, Module) + def test_ctx_load_module_with_features(self): + with Context(YANG_DIR) as ctx: + mod = ctx.load_module("yolo-system", None, ["*"]) + self.assertIsInstance(mod, Module) + for f in list(mod.features()): + self.assertTrue(f.state()) + def test_ctx_get_module(self): with Context(YANG_DIR) as ctx: ctx.load_module("yolo-system") From d8acc345aeb8d5b9644f7f2197ba6d8733e86378 Mon Sep 17 00:00:00 2001 From: Robin Jarry <robin@jarry.cc> Date: Fri, 7 Mar 2025 17:38:47 +0100 Subject: [PATCH 70/71] github: fix publish on pypi.org when only pushing a tag Also trigger the CI on tag push. Fix a typo in the distro image name. Fixes: aea34d1e7871 ("tox: fix lint target on python 3.13") Signed-off-by: Robin Jarry <robin@jarry.cc> --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27efc7dd..2177a27b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,8 @@ on: push: branches: - master + tags: + - v* jobs: lint: @@ -92,7 +94,7 @@ jobs: deploy: needs: [lint, test] if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-24.03 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 From 3072af036386a6bb0fde285eba751e09a4948261 Mon Sep 17 00:00:00 2001 From: Christian Hopps <chopps@labn.net> Date: Sun, 29 Jun 2025 08:38:32 -0400 Subject: [PATCH 71/71] data: add type hinting to meta functions of DNode The other functions in DNode are properly typed, but the meta functions were missed. Add the missing typing hints. Signed-off-by: Christian Hopps <chopps@labn.net> --- libyang/data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libyang/data.py b/libyang/data.py index 0d63d3c9..9595ea16 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -291,7 +291,7 @@ def __init__(self, context: "libyang.Context", cdata): self.attributes = None self.free_func = None # type: Callable[DNode] - def meta(self): + def meta(self) -> Dict[str, str]: ret = {} item = self.cdata.meta while item != ffi.NULL: @@ -303,7 +303,7 @@ def meta(self): item = item.next return ret - def get_meta(self, name): + def get_meta(self, name: str) -> Optional[str]: item = self.cdata.meta while item != ffi.NULL: if c2str(item.name) == name: @@ -315,7 +315,7 @@ def get_meta(self, name): item = item.next return None - def meta_free(self, name): + def meta_free(self, name: str): item = self.cdata.meta while item != ffi.NULL: if c2str(item.name) == name: