From 6eb6adff2f1a2fe988e79c53e622b18f9ff7ab26 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Fri, 2 Feb 2024 17:22:54 +0100 Subject: [PATCH 1/3] update requirements --- azure-pipelines.yml | 4 ++-- requirements-dev.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 907bb9f..61587f4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -4,8 +4,8 @@ jobs: vmImage: 'ubuntu-latest' strategy: matrix: - Python311-Linux: - python.version: '3.11' + Python312-Linux: + python.version: '3.12' maxParallel: 3 steps: diff --git a/requirements-dev.txt b/requirements-dev.txt index 5804529..5e262e3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,7 +11,7 @@ lightgbm matplotlib ml-dtypes git+https://github.com/onnx/onnxmltools.git -onnxruntime>=1.16.1 +onnxruntime>=1.17.0 openpyxl packaging pandas From 2d6727152a9f6fd9056eb311adc677377181e2e6 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 6 Feb 2024 12:15:44 +0100 Subject: [PATCH 2/3] add line numbers --- .../ut_reference/test_evaluator_yield.py | 15 ++--- onnx_array_api/reference/evaluator_yield.py | 61 ++++++++++++------- 2 files changed, 48 insertions(+), 28 deletions(-) diff --git a/_unittests/ut_reference/test_evaluator_yield.py b/_unittests/ut_reference/test_evaluator_yield.py index 7181456..467e6f9 100644 --- a/_unittests/ut_reference/test_evaluator_yield.py +++ b/_unittests/ut_reference/test_evaluator_yield.py @@ -422,18 +422,19 @@ def test_distance_sequence_str(self): text = dc.to_str(s1, s2, align) self.assertIn("OUTPUT", text) expected = """ - =|INPUTfloat322x2ABCDA|INPUTfloat322x2ABCDA - =|INPUTfloat322x2ABCDB|INPUTfloat322x2ABCDB - ~|INPUTfloat322x3ABCDX|INPUTfloat322x2ABCDX - -|RESULTfloat322x2CEIOExpH| - =|RESULTfloat322x2CEIOLinearRegrY1|RESULTfloat322x2CEIOLinearRegrY1 - ~|RESULTfloat322x2CEIOAbsY|RESULTfloat322x3CEIPAbsZ - ~|OUTPUTfloat322x2CEIOY|OUTPUTfloat322x2CEIPY + 1=|INPUTfloat322x2ABCDA|INPUTfloat322x2ABCDA + 2=|INPUTfloat322x2ABCDB|INPUTfloat322x2ABCDB + 3~|INPUTfloat322x3ABCDX|INPUTfloat322x2ABCDX + 4-|RESULTfloat322x2CEIOExpH| + 5=|RESULTfloat322x2CEIOLinearRegrY1|RESULTfloat322x2CEIOLinearRegrY1 + 6~|RESULTfloat322x2CEIOAbsY|RESULTfloat322x3CEIPAbsZ + 7~|OUTPUTfloat322x2CEIOY|OUTPUTfloat322x2CEIPY """.replace( " ", "" ).strip( "\n " ) + self.maxDiff = None self.assertEqual(expected, text.replace(" ", "").strip("\n")) def test_compare_execution(self): diff --git a/onnx_array_api/reference/evaluator_yield.py b/onnx_array_api/reference/evaluator_yield.py index 3935913..05add79 100644 --- a/onnx_array_api/reference/evaluator_yield.py +++ b/onnx_array_api/reference/evaluator_yield.py @@ -118,6 +118,7 @@ def enumerate_results( self, output_names: Optional[List[str]] = None, feed_inputs: Optional[Dict[str, Any]] = None, + raise_exc: bool = True, ) -> Iterator[Tuple[ResultType, str, Any]]: """ Executes the onnx model and enumerate all the intermediate results. @@ -148,6 +149,7 @@ def enumerate_results( yield ResultType.INPUT, k, v, None # step 2: execute nodes + yield_output = True for node in self.evaluator.rt_nodes_: for i in node.input: if i not in results: @@ -160,26 +162,36 @@ def enumerate_results( linked_attributes = {} if node.has_linked_attribute and attributes: linked_attributes["linked_attributes"] = attributes - if node.need_context(): - outputs = node.run(*inputs, context=results, **linked_attributes) - else: - outputs = node.run(*inputs, **linked_attributes) + + try: + if node.need_context(): + outputs = node.run(*inputs, context=results, **linked_attributes) + else: + outputs = node.run(*inputs, **linked_attributes) + except Exception: + if raise_exc: + raise + yield_output = False + break + for name, value in zip(node.output, outputs): yield ResultType.RESULT, name, value, node.op_type results[name] = value # step 3: outputs - for name in output_names: - if name not in results: - raise RuntimeError( - f"Unable to find output name {name!r} in {sorted(results)}, proto is\n{self.proto_}" - ) - yield ResultType.OUTPUT, name, results[name], None + if yield_output: + for name in output_names: + if name not in results: + raise RuntimeError( + f"Unable to find output name {name!r} in {sorted(results)}, proto is\n{self.proto_}" + ) + yield ResultType.OUTPUT, name, results[name], None def enumerate_summarized( self, output_names: Optional[List[str]] = None, feed_inputs: Optional[Dict[str, Any]] = None, + raise_exc: bool = True, ) -> Iterator[ResultExecution]: """ Executes the onnx model and enumerate intermediate results without their names. @@ -187,12 +199,14 @@ def enumerate_summarized( Args: output_names: requested outputs by names, None for all feed_inputs: dictionary `{ input name: input value }` + raise_exc: raises an exception if the execution fails or stop + where it is Returns: - iterator on tuple(result kind, node.type, dtype, shape, value, result name) + iterator on ResultExecution """ for kind, name, value, op_type in self.enumerate_results( - output_names, feed_inputs + output_names, feed_inputs, raise_exc=raise_exc ): summary = make_summary(value) yield ResultExecution( @@ -328,6 +342,7 @@ def to_str( """ rows = [] last = -1, -1 + row_index = 1 for i, j in alignment: assert i < len(s1), f"Unexpected value i={i} >= len(s1)={len(s1)}" assert j < len(s2), f"Unexpected value i={j} >= len(s2)={len(s2)}" @@ -338,20 +353,18 @@ def to_str( d2 = s2[j] d = self.distance_pair(d1, d2) symbol = "=" if d == 0 else "~" - rows.append( - f"{symbol} | {_align(str(d1), column_size)} | {_align(str(d2), column_size)}" - ) + line = f"{symbol} | {_align(str(d1), column_size)} | {_align(str(d2), column_size)}" elif i == last[0]: d2 = s2[j] - rows.append( + line = ( f"+ | {_align('', column_size)} | {_align(str(d2), column_size)} " ) else: d1 = s1[i] - rows.append( - f"- | {_align(str(d1), column_size)} | {_align('', column_size)}" - ) + line = f"- | {_align(str(d1), column_size)} | {_align('', column_size)}" + rows.append(f"{row_index: 3d} {line}") last = i, j + row_index += 1 return "\n".join(rows) @@ -410,6 +423,7 @@ def compare_onnx_execution( model2: ModelProto, inputs: Optional[List[Any]] = None, verbose: int = 0, + raise_exc: bool = True, ) -> Tuple[List[ResultExecution], List[ResultExecution], List[Tuple[int, int]]]: """ Compares the execution of two onnx models. @@ -421,6 +435,7 @@ def compare_onnx_execution( :param model2: second model :param inputs: inputs to use :param verbose: verbosity + :param raise_exc: raise exception if the execution fails or stop at the error :return: four results, a sequence of results for the first model and the second model, the alignment between the two, DistanceExecution """ @@ -433,11 +448,15 @@ def compare_onnx_execution( if verbose: print(f"[compare_onnx_execution] got {len(inputs)} inputs") print("[compare_onnx_execution] execute first model") - res1 = list(YieldEvaluator(model1).enumerate_summarized(None, feeds1)) + res1 = list( + YieldEvaluator(model1).enumerate_summarized(None, feeds1, raise_exc=raise_exc) + ) if verbose: print(f"[compare_onnx_execution] got {len(res1)} results") print("[compare_onnx_execution] execute second model") - res2 = list(YieldEvaluator(model2).enumerate_summarized(None, feeds2)) + res2 = list( + YieldEvaluator(model2).enumerate_summarized(None, feeds2, raise_exc=raise_exc) + ) if verbose: print(f"[compare_onnx_execution] got {len(res2)} results") print("[compare_onnx_execution] compute edit distance") From 7ee11afed279134c6fe91b2fa16570512c2b2058 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 6 Feb 2024 12:24:58 +0100 Subject: [PATCH 3/3] doc --- onnx_array_api/reference/evaluator_yield.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/onnx_array_api/reference/evaluator_yield.py b/onnx_array_api/reference/evaluator_yield.py index 05add79..a5e4d4f 100644 --- a/onnx_array_api/reference/evaluator_yield.py +++ b/onnx_array_api/reference/evaluator_yield.py @@ -196,14 +196,11 @@ def enumerate_summarized( """ Executes the onnx model and enumerate intermediate results without their names. - Args: - output_names: requested outputs by names, None for all - feed_inputs: dictionary `{ input name: input value }` - raise_exc: raises an exception if the execution fails or stop - where it is - - Returns: - iterator on ResultExecution + :param output_names: requested outputs by names, None for all + :param feed_inputs: dictionary `{ input name: input value }` + :param raise_exc: raises an exception if the execution fails or stop + where it is + :return: iterator on ResultExecution """ for kind, name, value, op_type in self.enumerate_results( output_names, feed_inputs, raise_exc=raise_exc