diff --git a/source/api/ut.pkb b/source/api/ut.pkb index 5c91e54d4..cff35b771 100644 --- a/source/api/ut.pkb +++ b/source/api/ut.pkb @@ -222,10 +222,15 @@ create or replace package body ut is raise_if_packages_invalidated(); raise no_data_found; end if; - g_result_lines := ut_utils.clob_to_table(l_clob, ut_utils.gc_max_storage_varchar2_len); - g_result_line_no := g_result_lines.first; + if l_clob is not null and l_clob != empty_clob() then + if length(l_clob) > ut_utils.gc_max_storage_varchar2_len then + g_result_lines := ut_utils.clob_to_table(l_clob, ut_utils.gc_max_storage_varchar2_len); + else + g_result_lines := ut_varchar2_list(l_clob); + end if; + g_result_line_no := g_result_lines.first; + end if; end if; - if g_result_line_no is not null then l_result := g_result_lines(g_result_line_no); g_result_line_no := g_result_lines.next(g_result_line_no); @@ -273,6 +278,7 @@ create or replace package body ut is ); if l_reporter is of (ut_output_reporter_base) then l_results := treat(l_reporter as ut_output_reporter_base).get_lines_cursor(); + g_result_lines := ut_varchar2_list(); loop pipe row( get_report_outputs( l_results ) ); end loop; @@ -320,6 +326,7 @@ create or replace package body ut is ); if l_reporter is of (ut_output_reporter_base) then l_results := treat(l_reporter as ut_output_reporter_base).get_lines_cursor(); + g_result_lines := ut_varchar2_list(); loop pipe row( get_report_outputs( l_results ) ); end loop; @@ -368,6 +375,7 @@ create or replace package body ut is ); if l_reporter is of (ut_output_reporter_base) then l_results := treat(l_reporter as ut_output_reporter_base).get_lines_cursor(); + g_result_lines := ut_varchar2_list(); loop pipe row( get_report_outputs( l_results ) ); end loop; @@ -416,6 +424,7 @@ create or replace package body ut is ); if l_reporter is of (ut_output_reporter_base) then l_results := treat(l_reporter as ut_output_reporter_base).get_lines_cursor(); + g_result_lines := ut_varchar2_list(); loop pipe row( get_report_outputs( l_results ) ); end loop; @@ -464,6 +473,7 @@ create or replace package body ut is ); if l_reporter is of (ut_output_reporter_base) then l_results := treat(l_reporter as ut_output_reporter_base).get_lines_cursor(); + g_result_lines := ut_varchar2_list(); loop pipe row( get_report_outputs( l_results ) ); end loop; @@ -512,6 +522,7 @@ create or replace package body ut is ); if l_reporter is of (ut_output_reporter_base) then l_results := treat(l_reporter as ut_output_reporter_base).get_lines_cursor(); + g_result_lines := ut_varchar2_list(); loop pipe row( get_report_outputs( l_results ) ); end loop; diff --git a/source/core/output_buffers/ut_output_bulk_buffer.tpb b/source/core/output_buffers/ut_output_bulk_buffer.tpb new file mode 100644 index 000000000..ceae07862 --- /dev/null +++ b/source/core/output_buffers/ut_output_bulk_buffer.tpb @@ -0,0 +1,163 @@ +create or replace type body ut_output_bulk_buffer is + /* + utPLSQL - Version 3 + Copyright 2016 - 2023 utPLSQL Project + + Licensed under the Apache License, Version 2.0 (the "License"): + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + + constructor function ut_output_bulk_buffer(self in out nocopy ut_output_bulk_buffer, a_output_id raw := null) return self as result is + begin + self.init(a_output_id, $$plsql_unit); + return; + end; + + overriding member procedure send_line(self in out nocopy ut_output_bulk_buffer, a_text varchar2, a_item_type varchar2 := null) is + pragma autonomous_transaction; + begin + if a_text is not null or a_item_type is not null then + if length(a_text) > ut_utils.gc_max_storage_varchar2_len then + self.send_lines( + ut_utils.convert_collection( + ut_utils.clob_to_table(a_text, ut_utils.gc_max_storage_varchar2_len) + ), + a_item_type + ); + else + self.last_write_message_id := self.last_write_message_id + 1; + insert /*+ no_parallel */ into ut_output_buffer_tmp(output_id, message_id, text, item_type) + values (self.output_id, self.last_write_message_id, a_text, a_item_type); + end if; + commit; + end if; + end; + + overriding member procedure send_lines(self in out nocopy ut_output_bulk_buffer, a_text_list ut_varchar2_rows, a_item_type varchar2 := null) is + pragma autonomous_transaction; + begin + insert /*+ no_parallel */ into ut_output_buffer_tmp(output_id, message_id, text, item_type) + select /*+ no_parallel */ self.output_id, self.last_write_message_id + rownum, t.column_value, a_item_type + from table(a_text_list) t + where t.column_value is not null or a_item_type is not null; + self.last_write_message_id := self.last_write_message_id + sql%rowcount; + commit; + end; + + overriding member procedure send_clob(self in out nocopy ut_output_bulk_buffer, a_text clob, a_item_type varchar2 := null) is + pragma autonomous_transaction; + begin + if a_text is not null and a_text != empty_clob() or a_item_type is not null then + if length(a_text) > ut_utils.gc_max_storage_varchar2_len then + self.send_lines( + ut_utils.convert_collection( + ut_utils.clob_to_table(a_text, ut_utils.gc_max_storage_varchar2_len) + ), + a_item_type + ); + else + self.last_write_message_id := self.last_write_message_id + 1; + insert /*+ no_parallel */ into ut_output_buffer_tmp(output_id, message_id, text, item_type) + values (self.output_id, self.last_write_message_id, a_text, a_item_type); + end if; + commit; + end if; + end; + + overriding member procedure lines_to_dbms_output(self in ut_output_bulk_buffer, a_initial_timeout number := null, a_timeout_sec number := null) is + l_data sys_refcursor; + l_text ut_varchar2_rows; + l_item_type ut_varchar2_rows; + begin + l_data := self.get_lines_cursor(a_initial_timeout, a_timeout_sec); + loop + fetch l_data bulk collect into l_text, l_item_type limit 10000; + for idx in 1 .. l_text.count loop + dbms_output.put_line(l_text(idx)); + end loop; + exit when l_data%notfound; + end loop; + close l_data; + self.remove_buffer_info(); + end; + + overriding member function get_lines_cursor(a_initial_timeout number := null, a_timeout_sec number := null) return sys_refcursor is + lc_init_wait_sec constant number := coalesce(a_initial_timeout, 10 ); + l_already_waited_sec number(10,2) := 0; + l_sleep_time number(2,1); + l_exists integer; + l_finished boolean := false; + l_data_produced boolean := false; + l_producer_active boolean := false; + l_producer_started boolean := false; + l_producer_finished boolean := false; + l_results sys_refcursor; + begin + + while not l_finished loop + + if not l_data_produced then + select /*+ no_parallel */ count(1) into l_exists + from ut_output_buffer_tmp o + where o.output_id = self.output_id and rownum = 1; + l_data_produced := (l_exists = 1); + end if; + + l_sleep_time := case when l_already_waited_sec >= 1 then 0.5 else 0.1 end; + l_producer_active := (self.get_lock_status() <> 0); + l_producer_started := (l_producer_active or l_data_produced ) or l_producer_started; + l_producer_finished := (l_producer_started and not l_producer_active) or l_producer_finished; + l_finished := + self.timeout_producer_not_finished(l_producer_finished, l_already_waited_sec, a_timeout_sec) + or self.timeout_producer_not_started(l_producer_started, l_already_waited_sec, lc_init_wait_sec) + or l_producer_finished; + + dbms_lock.sleep(l_sleep_time); + l_already_waited_sec := l_already_waited_sec + l_sleep_time; + end loop; + + open l_results for + select /*+ no_parallel */ o.text, o.item_type + from ut_output_buffer_tmp o + where o.output_id = self.output_id + and o.text is not null + order by o.output_id, o.message_id; + + return l_results; + + end; + + /* Important note. + This function code is almost duplicated between two types for performance reasons. + The pipe row clause is much faster on VARCHAR2 then it is on clob. + That is the key reason for two implementations. + */ + overriding member function get_lines(a_initial_timeout number := null, a_timeout_sec number := null) return ut_output_data_rows pipelined is + l_data sys_refcursor; + l_text ut_varchar2_rows; + l_item_type ut_varchar2_rows; + begin + l_data := self.get_lines_cursor(a_initial_timeout, a_timeout_sec); + loop + fetch l_data bulk collect into l_text, l_item_type limit 10000; + for idx in 1 .. l_text.count loop + pipe row( ut_output_data_row(l_text(idx), l_item_type(idx)) ); + end loop; + exit when l_data%notfound; + end loop; + close l_data; + self.remove_buffer_info(); + return; + end; + +end; +/ diff --git a/source/core/output_buffers/ut_output_bulk_buffer.tps b/source/core/output_buffers/ut_output_bulk_buffer.tps new file mode 100644 index 000000000..d74d4ee14 --- /dev/null +++ b/source/core/output_buffers/ut_output_bulk_buffer.tps @@ -0,0 +1,27 @@ +create or replace type ut_output_bulk_buffer under ut_output_buffer_base ( + /* + utPLSQL - Version 3 + Copyright 2016 - 2023 utPLSQL Project + + Licensed under the Apache License, Version 2.0 (the "License"): + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + + constructor function ut_output_bulk_buffer(self in out nocopy ut_output_bulk_buffer, a_output_id raw := null) return self as result, + overriding member procedure send_line(self in out nocopy ut_output_bulk_buffer, a_text varchar2, a_item_type varchar2 := null), + overriding member procedure send_lines(self in out nocopy ut_output_bulk_buffer, a_text_list ut_varchar2_rows, a_item_type varchar2 := null), + overriding member procedure send_clob(self in out nocopy ut_output_bulk_buffer, a_text clob, a_item_type varchar2 := null), + overriding member procedure lines_to_dbms_output(self in ut_output_bulk_buffer, a_initial_timeout number := null, a_timeout_sec number := null), + overriding member function get_lines_cursor(a_initial_timeout number := null, a_timeout_sec number := null) return sys_refcursor, + overriding member function get_lines(a_initial_timeout number := null, a_timeout_sec number := null) return ut_output_data_rows pipelined +) not final +/ diff --git a/source/core/output_buffers/ut_output_table_buffer.tpb b/source/core/output_buffers/ut_output_table_buffer.tpb index f38363e49..480ae9144 100644 --- a/source/core/output_buffers/ut_output_table_buffer.tpb +++ b/source/core/output_buffers/ut_output_table_buffer.tpb @@ -75,14 +75,16 @@ create or replace type body ut_output_table_buffer is overriding member procedure lines_to_dbms_output(self in ut_output_table_buffer, a_initial_timeout number := null, a_timeout_sec number := null) is l_data sys_refcursor; - l_text varchar2(32767); - l_item_type varchar2(32767); + l_text ut_varchar2_rows; + l_item_type ut_varchar2_rows; begin l_data := self.get_lines_cursor(a_initial_timeout, a_timeout_sec); loop - fetch l_data into l_text, l_item_type; + fetch l_data bulk collect into l_text, l_item_type limit 10000; + for idx in 1 .. l_text.count loop + dbms_output.put_line(l_text(idx)); + end loop; exit when l_data%notfound; - dbms_output.put_line(l_text); end loop; close l_data; end; diff --git a/source/core/ut_utils.pkb b/source/core/ut_utils.pkb index 162e50f8f..5114cecad 100644 --- a/source/core/ut_utils.pkb +++ b/source/core/ut_utils.pkb @@ -570,6 +570,7 @@ create or replace package body ut_utils is end loop; exit when l_lines_data%notfound; end loop; + close l_lines_data; execute immediate 'truncate table ut_dbms_output_cache'; commit; end; diff --git a/source/install.sql b/source/install.sql index 6f3c16801..0873d6a1e 100644 --- a/source/install.sql +++ b/source/install.sql @@ -120,6 +120,8 @@ create or replace context &&ut3_owner._info using &&ut3_owner..ut_session_contex @@install_component.sql 'core/output_buffers/ut_output_table_buffer.tpb' @@install_component.sql 'core/output_buffers/ut_output_clob_table_buffer.tps' @@install_component.sql 'core/output_buffers/ut_output_clob_table_buffer.tpb' +@@install_component.sql 'core/output_buffers/ut_output_bulk_buffer.tps' +@@install_component.sql 'core/output_buffers/ut_output_bulk_buffer.tpb' @@install_component.sql 'core/types/ut_output_reporter_base.tps' diff --git a/source/reporters/ut_coverage_cobertura_reporter.tpb b/source/reporters/ut_coverage_cobertura_reporter.tpb index 48082a6d4..50eb34631 100644 --- a/source/reporters/ut_coverage_cobertura_reporter.tpb +++ b/source/reporters/ut_coverage_cobertura_reporter.tpb @@ -20,7 +20,7 @@ create or replace type body ut_coverage_cobertura_reporter is self in out nocopy ut_coverage_cobertura_reporter ) return self as result is begin - self.init($$plsql_unit); + self.init($$plsql_unit,ut_output_bulk_buffer()); return; end; diff --git a/source/reporters/ut_coverage_html_reporter.tpb b/source/reporters/ut_coverage_html_reporter.tpb index c88c91a80..b1f9f6651 100644 --- a/source/reporters/ut_coverage_html_reporter.tpb +++ b/source/reporters/ut_coverage_html_reporter.tpb @@ -22,7 +22,7 @@ create or replace type body ut_coverage_html_reporter is a_html_report_assets_path varchar2 := null ) return self as result is begin - self.init($$plsql_unit); + self.init($$plsql_unit,ut_output_bulk_buffer()); self.project_name := a_project_name; assets_path := nvl(a_html_report_assets_path, ut_coverage_report_html_helper.get_default_html_assets_path()); return; diff --git a/source/reporters/ut_coverage_sonar_reporter.tpb b/source/reporters/ut_coverage_sonar_reporter.tpb index 8ee78dc24..1861a9be3 100644 --- a/source/reporters/ut_coverage_sonar_reporter.tpb +++ b/source/reporters/ut_coverage_sonar_reporter.tpb @@ -20,7 +20,7 @@ create or replace type body ut_coverage_sonar_reporter is self in out nocopy ut_coverage_sonar_reporter ) return self as result is begin - self.init($$plsql_unit); + self.init($$plsql_unit,ut_output_bulk_buffer()); return; end; @@ -84,6 +84,12 @@ create or replace type body ut_coverage_sonar_reporter is end; begin +-- execute immediate 'alter session set statistics_level=all'; +-- dbms_hprof.start_profiling( +-- location => 'PLSHPROF_DIR' +-- , filename => 'profiler_utPLSQL_run_on_'||$$plsql_unit||'_'||rawtohex(self.id)||'.txt' +-- ); +-- ut_coverage.coverage_stop(); self.print_text_lines( @@ -92,6 +98,7 @@ create or replace type body ut_coverage_sonar_reporter is a_run ) ); +-- dbms_hprof.stop_profiling; end; overriding member function get_description return varchar2 as diff --git a/source/reporters/ut_coveralls_reporter.tpb b/source/reporters/ut_coveralls_reporter.tpb index 3a54aa7f7..14672303e 100644 --- a/source/reporters/ut_coveralls_reporter.tpb +++ b/source/reporters/ut_coveralls_reporter.tpb @@ -20,7 +20,7 @@ create or replace type body ut_coveralls_reporter is self in out nocopy ut_coveralls_reporter ) return self as result is begin - self.init($$plsql_unit); + self.init($$plsql_unit,ut_output_bulk_buffer()); return; end; diff --git a/source/reporters/ut_junit_reporter.tpb b/source/reporters/ut_junit_reporter.tpb index 44affa030..3bceb52a4 100644 --- a/source/reporters/ut_junit_reporter.tpb +++ b/source/reporters/ut_junit_reporter.tpb @@ -18,7 +18,7 @@ create or replace type body ut_junit_reporter is constructor function ut_junit_reporter(self in out nocopy ut_junit_reporter) return self as result is begin - self.init($$plsql_unit); + self.init($$plsql_unit,ut_output_bulk_buffer()); return; end; diff --git a/source/reporters/ut_sonar_test_reporter.tpb b/source/reporters/ut_sonar_test_reporter.tpb index a3ec72241..7c46879d2 100644 --- a/source/reporters/ut_sonar_test_reporter.tpb +++ b/source/reporters/ut_sonar_test_reporter.tpb @@ -20,7 +20,7 @@ create or replace type body ut_sonar_test_reporter is self in out nocopy ut_sonar_test_reporter ) return self as result is begin - self.init($$plsql_unit); + self.init($$plsql_unit,ut_output_bulk_buffer()); return; end; diff --git a/source/reporters/ut_tfs_junit_reporter.tpb b/source/reporters/ut_tfs_junit_reporter.tpb index 710c3bc7f..e2d45a369 100644 --- a/source/reporters/ut_tfs_junit_reporter.tpb +++ b/source/reporters/ut_tfs_junit_reporter.tpb @@ -18,7 +18,7 @@ create or replace type body ut_tfs_junit_reporter is constructor function ut_tfs_junit_reporter(self in out nocopy ut_tfs_junit_reporter) return self as result is begin - self.init($$plsql_unit); + self.init($$plsql_unit,ut_output_bulk_buffer()); return; end; diff --git a/source/reporters/ut_xunit_reporter.tpb b/source/reporters/ut_xunit_reporter.tpb index eab0c8a3c..4612fb640 100644 --- a/source/reporters/ut_xunit_reporter.tpb +++ b/source/reporters/ut_xunit_reporter.tpb @@ -18,7 +18,7 @@ create or replace type body ut_xunit_reporter is constructor function ut_xunit_reporter(self in out nocopy ut_xunit_reporter) return self as result is begin - self.init($$plsql_unit); + self.init($$plsql_unit,ut_output_bulk_buffer()); return; end; diff --git a/source/uninstall_objects.sql b/source/uninstall_objects.sql index 571d187a2..9531736b9 100644 --- a/source/uninstall_objects.sql +++ b/source/uninstall_objects.sql @@ -323,6 +323,8 @@ drop type ut_output_table_buffer force; drop type ut_output_clob_table_buffer force; +drop type ut_output_bulk_buffer force; + drop type ut_output_buffer_base force; drop table ut_output_buffer_tmp purge; diff --git a/test/ut3_tester_helper/coverage_helper.pkb b/test/ut3_tester_helper/coverage_helper.pkb index 2a508ca6a..ef14c586f 100644 --- a/test/ut3_tester_helper/coverage_helper.pkb +++ b/test/ut3_tester_helper/coverage_helper.pkb @@ -489,16 +489,30 @@ create or replace package body coverage_helper is l_coverage_id raw(32) := sys_guid(); begin l_plsql_block := q'[ + declare + x dbms_output.chararr; + i integer := 100000; begin +/* + execute immediate 'alter session set statistics_level=all'; + dbms_hprof.start_profiling( + location => 'PLSHPROF_DIR' + , filename => 'profiler_utPLSQL_run_]'||rawtohex(l_coverage_id)||q'[.txt' + ); +*/ ut3_develop.ut_runner.coverage_start(']'||rawtohex(l_coverage_id)||q'['); ut3_develop.ut_coverage.set_develop_mode(a_develop_mode => true); --gather coverage on the command executed begin {a_run_command}; end; + dbms_output.get_lines(x,i); ut3_develop.ut_coverage.set_develop_mode(a_develop_mode => false); ut3_develop.ut_runner.coverage_stop(); --get the actual results of the command gathering the coverage insert into test_results select rownum as id, x.* from table( {a_run_command} ) x; commit; +/* + dbms_hprof.stop_profiling; +*/ end;]'; l_plsql_block := replace(l_plsql_block,'{a_run_command}',a_run_command); l_result_clob := run_code_as_job( l_plsql_block );