Disable view for some torch api tests. (#1567)

Xreki · web-flow · commit 575e009840d8 · 2023-02-10T13:23:23.000+08:00
* Disable view for some torch api tests.

* Optimize the parsing of gpu_time and add the compute of flop, gbs of stack.

* Add the compute of flop and gbs for unstack.

* Add swish.

* Add the compute of flop and gbs for concat and split.

* Disable swish for torch.

* Fix the setting and use of run_torch.
diff --git a/api/common/api_param.py b/api/common/api_param.py
@@ -166,6 +166,7 @@ def __init__(self, op_type, params=None):
         self.__name = op_type
         self.__framework = "paddle"
         self.__run_tf = True
+        self.__run_torch = True
         self.api_name = self.name
         self.paddle_api = None
         self.torch_api = None
@@ -174,7 +175,6 @@ def __init__(self, op_type, params=None):
         self.params_list = None
         self.backward = False
         self.feed_spec = None
-        self.run_torch = True
         self.alias_name = None
 
     @classmethod
@@ -213,6 +213,14 @@ def run_tf(self):
     def run_tf(self, value):
         self.__run_tf = value
 
+    @property
+    def run_torch(self):
+        return self.__run_torch
+
+    @run_torch.setter
+    def run_torch(self, value):
+        self.__run_torch = value
+
     def compute_dtype(self):
         dtype = None
         for name, value in vars(self).items():
@@ -336,7 +344,7 @@ def __str__(self):
             'feed_spec', 'alias_name'
         ]
         if self.framework != "paddle":
-            exclude_attrs.append("run_torch")
+            exclude_attrs.append("_APIConfig__run_torch")
             exclude_attrs.append("_APIConfig__run_tf")
         prefix = ""
         debug_str = ('[%s][%s] %s {\n') % (self.framework, self.name,
diff --git a/api/common/benchmark.py b/api/common/benchmark.py
@@ -26,6 +26,9 @@ def __init__(self, framework, testing_mode):
         self.reset()
 
     def reset(self):
+        if hasattr(self, "extra_tensors") and self.extra_tensors is not None:
+            del self.extra_tensors
+        self.extra_tensors = None
         self.feed_list = None
         self.fetch_list = None
         self._backward = False
diff --git a/api/common/launch.py b/api/common/launch.py
@@ -96,11 +96,11 @@ def _parse_gpu_time(self, line):
 class NsightRunner(object):
     def run(self, cmd, profile_from_start=False):
         stdout, exit_code = self._nsight(cmd, profile_from_start)
+        parse_status, gpu_time = self._parse_logs(stdout.split("\n"))
+        if parse_status:
+            return gpu_time
         if exit_code == 0:
-            parse_status, gpu_time = self._parse_logs(stdout.split("\n"))
-            if parse_status:
-                return gpu_time
-        print("Running Error:\n {}".format(stdout))
+            print("Running Error:\n {}".format(stdout))
         return 0.0
 
     def _nsight(self, cmd, profile_from_start):
@@ -179,16 +179,20 @@ def _print_logs_in_range(self, logs, line_from, line_to):
             print("")
 
     def _parse_gpu_time(self, line):
-        infos = line.strip().split()
-        percent = float(infos[0].replace("%", "")) * 0.01
-        gpu_time = float(infos[1].replace(",", "")) * 1E-6
-        calls = int(infos[2].replace(",", ""))
-        function = infos[7]
-        for i in range(8, len(infos)):
-            function = function + " " + infos[i]
-        #print("percent: %.2f; gpu_time: %.4f ms; calls: %d; function: %s" %
-        #      (percent, gpu_time, calls, function))
-        return gpu_time / percent
+        try:
+            infos = line.strip().split()
+            percent = float(infos[0].replace("%", "")) * 0.01
+            gpu_time = float(infos[1].replace(",", "")) * 1E-6
+            calls = int(infos[2].replace(",", ""))
+            function = infos[7]
+            for i in range(8, len(infos)):
+                function = function + " " + infos[i]
+            #print("percent: %.2f; gpu_time: %.4f ms; calls: %d; function: %s" %
+            #      (percent, gpu_time, calls, function))
+            return gpu_time / percent
+        except Exception as e:
+            sys.stderr.write("Error: parsing \"{}\". {}\n".format(line, e))
+            return 0.0
 
 
 class NsightRunnerForDynamicScheduling(object):
diff --git a/api/common/paddle_op_benchmark.py b/api/common/paddle_op_benchmark.py
@@ -316,8 +316,8 @@ def __init__(self, testing_mode):
         else:
             paddle.disable_static()
         flags = paddle.get_flags(['FLAGS_use_autotune'])
-        self.use_autotune = flags['FLAGS_use_autotune']
-        if self.use_autotune:
+        self._use_autotune = flags['FLAGS_use_autotune']
+        if self._use_autotune:
             config = {
                 "kernel": {
                     "enable": True,
@@ -448,7 +448,7 @@ def _run_main_iter(step=1):
                         outputs.append(var)
                     else:
                         outputs.append(var.numpy())
-            if self.use_autotune and not self.backward:
+            if self._use_autotune and not self.backward:
                 paddle.fluid.core.update_autotune_status()
             return outputs
 
@@ -661,8 +661,3 @@ def build_graph(self, config=None):
         self.build_program(config)
         self.feed_list = self.feed_vars
         self.fetch_list = self.fetch_vars
-
-
-class PaddleDynamicAPIBenchmarkBase(PaddleOpBenchmarkBase):
-    def __init__(self):
-        super(PaddleDynamicAPIBenchmarkBase, self).__init__("dynamic")
diff --git a/api/common/special_op_list.py b/api/common/special_op_list.py
@@ -85,6 +85,7 @@
     "less",
     "linspace",
     "remainder",
+    "set_value",
     "shard_index",
     "size",
     "truncated_gaussian_random",
diff --git a/api/tests/activation.py b/api/tests/activation.py
@@ -23,6 +23,7 @@ def __init__(self):
         self.api_list = {
             'sigmoid': 'sigmoid',
             'hardsigmoid': 'hardsigmoid',
+            'swish': 'swish',
             'hardswish': 'hardswish',
             'softplus': 'softplus',
             'tanhshrink': 'tanhshrink',
@@ -39,6 +40,13 @@ def run_tf(self):
             return False
         return True
 
+    @property
+    def run_torch(self):
+        if self.api_name in ["swish"]:
+            print("-- %s is not supported in torch." % self.api_name)
+            return False
+        return True
+
 
 @benchmark_registry.register("activation")
 class PaddleActivation(PaddleOpBenchmarkBase):
diff --git a/api/tests/concat.py b/api/tests/concat.py
@@ -32,6 +32,17 @@ def build_graph(self, config):
         if config.backward:
             self.append_gradients(result, xs)
 
+    def compute_flop_and_byte(self, config):
+        forward_flop = 0
+        forward_byte = 0
+        for shape in config.x_shape:
+            forward_byte += numel(shape) * sizeof(config.x_dtype[0]) * 2
+        if not config.backward:
+            return forward_flop, forward_byte
+        else:
+            # To be implemented.
+            return None, None
+
 
 @benchmark_registry.register("concat")
 class TorchConcat(PytorchOpBenchmarkBase):
diff --git a/api/tests/expand_as.py b/api/tests/expand_as.py
@@ -36,7 +36,7 @@ def build_graph(self, config):
         x = self.variable(name='x', shape=config.x_shape, dtype=config.x_dtype)
         y = self.variable(name='y', shape=config.y_shape, dtype=config.y_dtype)
         y.requires_grad = False
-        result = x.expand_as(y)
+        result = x.expand_as(y).contiguous()
 
         self.feed_list = [x]
         self.fetch_list = [result]
diff --git a/api/tests/set_value.py b/api/tests/set_value.py
@@ -16,35 +16,52 @@
 
 
 @benchmark_registry.register("set_value")
-class SetValueConfig(APIConfig):
-    def __init__(self):
-        super(SetValueConfig, self).__init__("set_value")
-        self.run_torch = False
+class PaddleSetValue(PaddleOpBenchmarkBase):
+    def build_graph(self, config):
+        x = self.variable(
+            name='x',
+            shape=config.x_shape,
+            dtype=config.x_dtype,
+            stop_gradient=True)
 
-    def init_from_json(self, filename, config_id=0, unknown_dim=16):
-        super(SetValueConfig, self).init_from_json(filename, config_id,
-                                                   unknown_dim)
+        if config.is_tensor_value:
+            value = self.variable(
+                name='value',
+                shape=config.value_shape,
+                dtype=config.value_dtype,
+                stop_gradient=True)
+            x[:, 10:500:2] = value
+
+            self.feed_list = [x, value]
+            self.fetch_list = [x]
+        else:
+            x[:, 0:20, ::2] = 10000
+
+            self.feed_list = [x]
+            self.fetch_list = [x]
 
 
 @benchmark_registry.register("set_value")
-class PaddleSetValue(PaddleOpBenchmarkBase):
+class TorchSetValue(PytorchOpBenchmarkBase):
     def build_graph(self, config):
-        input = self.variable(
-            name='input', shape=config.input_shape, dtype=config.input_dtype)
-        input.stop_gradient = True
+        x = self.variable(
+            name='x',
+            shape=config.x_shape,
+            dtype=config.x_dtype,
+            stop_gradient=True)
 
         if config.is_tensor_value:
             value = self.variable(
                 name='value',
                 shape=config.value_shape,
-                dtype=config.value_dtype)
-            input[:, 10:500:2] = value
-
-            self.feed_list = [input, value]
-            self.fetch_list = [input]
+                dtype=config.value_dtype,
+                stop_gradient=True)
+            x[:, 10:500:2] = value
 
+            self.feed_list = [x, value]
+            self.fetch_list = [x.contiguous()]
         else:
-            input[:, 0:20, ::2] = 10000
+            x[:, 0:20, ::2] = 10000
 
-            self.feed_list = [input]
-            self.fetch_list = [input]
+            self.feed_list = [x]
+            self.fetch_list = [x.contiguous()]
diff --git a/api/tests/size.py b/api/tests/size.py
@@ -18,20 +18,18 @@
 @benchmark_registry.register("size")
 class PaddleSize(PaddleOpBenchmarkBase):
     def build_graph(self, config):
-        input = self.variable(
-            name='input', shape=config.input_shape, dtype=config.input_dtype)
-        result = paddle.fluid.layers.size(input)
+        x = self.variable(name='x', shape=config.x_shape, dtype=config.x_dtype)
+        result = paddle.numel(x)
 
-        self.feed_list = [input]
+        self.feed_list = [x]
         self.fetch_list = [result]
 
 
 @benchmark_registry.register("size")
 class TorchSize(PytorchOpBenchmarkBase):
     def build_graph(self, config):
-        input = self.variable(
-            name='input', shape=config.input_shape, dtype=config.input_dtype)
-        result = torch.numel(input=input)
+        x = self.variable(name='x', shape=config.x_shape, dtype=config.x_dtype)
+        result = torch.numel(x=x)
 
-        self.feed_list = [input]
+        self.feed_list = [x]
         self.fetch_list = [result]
diff --git a/api/tests/slice.py b/api/tests/slice.py
@@ -90,7 +90,7 @@ def build_graph(self, config):
             input=x,
             dim=config.axes[0],
             start=config.starts[0],
-            length=config.length)
+            length=config.length).contiguous()
 
         self.feed_list = [x]
         self.fetch_list = [result]
diff --git a/api/tests/split.py b/api/tests/split.py
@@ -45,6 +45,15 @@ def build_graph(self, config):
         if config.backward:
             self.append_gradients(result, [x])
 
+    def compute_flop_and_byte(self, config):
+        forward_flop = 0
+        forward_byte = numel(config.x_shape) * sizeof(config.x_dtype) * 2
+        if not config.backward:
+            return forward_flop, forward_byte
+        else:
+            # To be implemented.
+            return None, None
+
 
 @benchmark_registry.register("split")
 class TorchSplit(PytorchOpBenchmarkBase):
@@ -62,6 +71,7 @@ def build_graph(self, config):
             config.backward = False
         else:
             self.fetch_list = [result]
+
         self.feed_list = [x]
         if config.backward:
             self.append_gradients(result, [x])
diff --git a/api/tests/stack.py b/api/tests/stack.py
@@ -32,6 +32,17 @@ def build_graph(self, config):
         if config.backward:
             self.append_gradients(result, xs)
 
+    def compute_flop_and_byte(self, config):
+        num_xs = len(config.x_shape)
+        forward_flop = 0
+        forward_byte = num_xs * numel(config.x_shape[0]) * sizeof(
+            config.x_dtype[0]) * 2
+        if not config.backward:
+            return forward_flop, forward_byte
+        else:
+            # To be implemented.
+            return None, None
+
 
 @benchmark_registry.register("stack")
 class TorchStack(PytorchOpBenchmarkBase):
diff --git a/api/tests/unstack.py b/api/tests/unstack.py
@@ -38,3 +38,12 @@ def build_graph(self, config):
             self.fetch_list = [result]
         if config.backward:
             self.append_gradients(result, [x])
+
+    def compute_flop_and_byte(self, config):
+        forward_flop = 0
+        forward_byte = numel(config.x_shape) * sizeof(config.x_dtype) * 2
+        if not config.backward:
+            return forward_flop, forward_byte
+        else:
+            # To be implemented.
+            return None, None
diff --git a/api/tests_v2/configs/set_value.json b/api/tests_v2/configs/set_value.json
@@ -1,7 +1,7 @@
 [{
     "op": "set_value",
     "param_info": {
-        "input": {
+        "x": {
             "dtype": "float32",
             "shape": "[-1L, 1000000L]",
             "type": "Variable"
@@ -19,7 +19,7 @@
 }, {
     "op": "set_value",
     "param_info": {
-        "input": {
+        "x": {
             "dtype": "float32",
             "shape": "[-1L, 60, 20000L]",
             "type": "Variable"
diff --git a/api/tests_v2/configs/size.json b/api/tests_v2/configs/size.json
@@ -1,7 +1,7 @@
 [{
     "op": "size",
     "param_info": {
-        "input": {
+        "x": {
             "dtype": "float32",
             "shape": "[-1L, 102400L]",
             "type": "Variable"
@@ -10,7 +10,7 @@
 }, {
     "op": "size",
     "param_info": {
-        "input": {
+        "x": {
             "dtype": "int64",
             "shape": "[512000L]",
             "type": "Variable"
