From b05dbd7427959e8e5a5b30475b740a2e13da15da Mon Sep 17 00:00:00 2001 From: lwasylow Date: Sat, 27 Oct 2018 15:13:59 +0100 Subject: [PATCH 1/7] Fix to issue #770 when the merge statement is run via current user which can result in error due to missing privs to internal function --- .../data_values/ut_compound_data_helper.pkb | 53 +++++++++++++++++++ .../data_values/ut_compound_data_helper.pks | 3 ++ .../data_values/ut_compound_data_value.tpb | 47 ++-------------- 3 files changed, 59 insertions(+), 44 deletions(-) diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index c6a38e35c..0fba5b0e7 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -541,5 +541,58 @@ create or replace package body ut_compound_data_helper is return l_no_missing_keys; end; + procedure update_row_and_pk_hash(a_self_data_id in raw, a_other_data_id in raw, a_exclude_xpath varchar2, + a_include_xpath varchar2, a_join_by_xpath varchar2) is + l_ut_owner varchar2(250) := ut_utils.ut_owner; + l_column_filter varchar2(32767); + l_pk_hash_sql varchar2(32767); + + function get_column_pk_hash(a_join_by_xpath varchar2) return varchar2 is + l_column varchar2(32767); + begin + /* due to possibility of key being to columns we cannot use xmlextractvalue + usage of xmlagg is possible however it greatly complicates code and performance is impacted. + xpath to be looked at or regex + */ + if a_join_by_xpath is not null then + l_column := l_ut_owner ||'.ut_compound_data_helper.get_hash(extract(ucd.item_data,:join_by_xpath).GetClobVal()) pk_hash'; + else + l_column := ':join_by_xpath pk_hash'; + end if; + return l_column; + end; + + begin + l_column_filter := ut_compound_data_helper.get_columns_filter(a_exclude_xpath, a_include_xpath); + l_pk_hash_sql := get_column_pk_hash(a_join_by_xpath); + + execute immediate 'merge into ' || l_ut_owner || '.ut_compound_data_tmp tgt + using ( + select ucd_out.item_hash, + ucd_out.pk_hash, + ucd_out.item_no, + ucd_out.data_id, + row_number() over (partition by ucd_out.pk_hash,ucd_out.item_hash,ucd_out.data_id order by 1,2) duplicate_no + from + ( + select '||l_ut_owner ||'.ut_compound_data_helper.get_hash(ucd.item_data.getclobval()) item_hash, + pk_hash, ucd.item_no, ucd.data_id + from + ( + select '||l_column_filter||','||l_pk_hash_sql||', item_no, data_id + from ' || l_ut_owner || q'[.ut_compound_data_tmp ucd + where data_id = :self_guid or data_id = :other_guid + ) ucd + )ucd_out + ) src + on (tgt.item_no = src.item_no and tgt.data_id = src.data_id) + when matched then update + set tgt.item_hash = src.item_hash, + tgt.pk_hash = src.pk_hash, + tgt.duplicate_no = src.duplicate_no]' + using a_exclude_xpath, a_include_xpath,a_join_by_xpath,a_self_data_id, a_other_data_id; + + end; + end; / diff --git a/source/expectations/data_values/ut_compound_data_helper.pks b/source/expectations/data_values/ut_compound_data_helper.pks index ab3693920..a18c20952 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pks +++ b/source/expectations/data_values/ut_compound_data_helper.pks @@ -81,5 +81,8 @@ create or replace package ut_compound_data_helper authid definer is function is_pk_exists(a_expected_cursor xmltype, a_actual_cursor xmltype, a_exclude_xpath varchar2, a_include_xpath varchar2,a_join_by_xpath varchar2) return tt_missing_pk; + procedure update_row_and_pk_hash(a_self_data_id in raw, a_other_data_id in raw, a_exclude_xpath varchar2, + a_include_xpath varchar2, a_join_by_xpath varchar2); + end; / diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb index de81830dd..e5a766890 100644 --- a/source/expectations/data_values/ut_compound_data_value.tpb +++ b/source/expectations/data_values/ut_compound_data_value.tpb @@ -197,27 +197,11 @@ create or replace type body ut_compound_data_value as member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean ) return integer is l_other ut_compound_data_value; l_ut_owner varchar2(250) := ut_utils.ut_owner; - l_column_filter varchar2(32767); l_diff_id ut_compound_data_helper.t_hash; l_result integer; l_row_diffs ut_compound_data_helper.tt_row_diffs; c_max_rows constant integer := 20; - function get_column_pk_hash(a_join_by_xpath varchar2) return varchar2 is - l_column varchar2(32767); - begin - /* due to possibility of key being to columns we cannot use xmlextractvalue - usage of xmlagg is possible however it greatly complicates code and performance is impacted. - xpath to be looked at or regex - */ - if a_join_by_xpath is not null then - l_column := l_ut_owner ||'.ut_compound_data_helper.get_hash(extract(ucd.item_data,:join_by_xpath).GetClobVal()) pk_hash'; - else - l_column := ':join_by_xpath pk_hash'; - end if; - return l_column; - end; - begin if not a_other is of (ut_compound_data_value) then raise value_error; @@ -226,37 +210,12 @@ create or replace type body ut_compound_data_value as l_other := treat(a_other as ut_compound_data_value); l_diff_id := ut_compound_data_helper.get_hash(self.data_id||l_other.data_id); - l_column_filter := ut_compound_data_helper.get_columns_filter(a_exclude_xpath, a_include_xpath); - + /** * Due to incompatibility issues in XML between 11 and 12.2 and 12.1 versions we will prepopulate pk_hash upfront to * avoid optimizer incorrectly rewrite and causing NULL error or ORA-600 - **/ - execute immediate 'merge into ' || l_ut_owner || '.ut_compound_data_tmp tgt - using ( - select ucd_out.item_hash, - ucd_out.pk_hash, - ucd_out.item_no, - ucd_out.data_id, - row_number() over (partition by ucd_out.pk_hash,ucd_out.item_hash,ucd_out.data_id order by 1,2) duplicate_no - from - ( - select '||l_ut_owner ||'.ut_compound_data_helper.get_hash(ucd.item_data.getclobval()) item_hash, - pk_hash, ucd.item_no, ucd.data_id - from - ( - select '||l_column_filter||','||get_column_pk_hash(a_join_by_xpath)||', item_no, data_id - from ' || l_ut_owner || q'[.ut_compound_data_tmp ucd - where data_id = :self_guid or data_id = :other_guid - ) ucd - )ucd_out - ) src - on (tgt.item_no = src.item_no and tgt.data_id = src.data_id) - when matched then update - set tgt.item_hash = src.item_hash, - tgt.pk_hash = src.pk_hash, - tgt.duplicate_no = src.duplicate_no]' - using a_exclude_xpath, a_include_xpath,a_join_by_xpath,self.data_id, l_other.data_id; + **/ + ut_compound_data_helper.update_row_and_pk_hash(self.data_id, l_other.data_id, a_exclude_xpath,a_include_xpath,a_join_by_xpath); /* Peform minus on two sets two get diffrences that will be used later on to print results */ execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no) From 94234573bad50053c8e486acd36f8d084f4d5c63 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Sun, 28 Oct 2018 07:48:16 +0000 Subject: [PATCH 2/7] Added test for min user test --- test/install__min_usr_tests.sql | 39 ++++++++++++++++++++++ test/install_and_run_tests.sh | 30 ++++++++++++++++- test/min_grant_user/min_grant_user_exp.pkb | 19 +++++++++++ test/min_grant_user/min_grant_user_exp.pks | 9 +++++ 4 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 test/install__min_usr_tests.sql create mode 100644 test/min_grant_user/min_grant_user_exp.pkb create mode 100644 test/min_grant_user/min_grant_user_exp.pks diff --git a/test/install__min_usr_tests.sql b/test/install__min_usr_tests.sql new file mode 100644 index 000000000..fa788f8f9 --- /dev/null +++ b/test/install__min_usr_tests.sql @@ -0,0 +1,39 @@ +set define off +whenever sqlerror exit failure rollback +whenever oserror exit failure rollback + +alter session set plsql_optimize_level=0; + +--Install tests +@@min_grant_user/min_grant_user_exp.pks +@@min_grant_user/min_grant_user_exp.pkb + +set linesize 200 +set define on +set verify off +column text format a100 +column error_count noprint new_value error_count + +prompt Validating installation + +set heading on +select type, name, sequence, line, position, text, count(1) over() error_count + from all_errors + where owner = USER + and name not like 'BIN$%' --not recycled + and name != 'UT_WITH_INVALID_BODY' + -- errors only. ignore warnings + and attribute = 'ERROR' + order by name, type, sequence +/ + +begin + if to_number('&&error_count') > 0 then + raise_application_error(-20000, 'Not all sources were successfully installed.'); + else + dbms_output.put_line('Installation completed successfully'); + end if; +end; +/ + +exit; diff --git a/test/install_and_run_tests.sh b/test/install_and_run_tests.sh index a1e5e0849..04d70fb39 100644 --- a/test/install_and_run_tests.sh +++ b/test/install_and_run_tests.sh @@ -1,10 +1,35 @@ #!/bin/bash set -ev +check_result() +{ + RC1=$1 + RC2=$2 + + if [ "$RC1" != "0" ] || [ "$RC2" != "0" ]; then + return 1 + fi + return 0 +} #goto git root directory git rev-parse && cd "$(git rev-parse --show-cdup)" +cd test + +time "$SQLCLI" ${UT3_USER}/${UT3_USER_PASSWORD}@//${CONNECTION_STR} @install__min_usr_tests.sql + +cd .. + +time utPLSQL-cli/bin/utplsql run ${UT3_USER}/${UT3_USER_PASSWORD}@${CONNECTION_STR} \ +-source_path=source -owner=ut3 \ +-test_path=test -c \ +-f=ut_documentation_reporter -o=min_test_results.log -s \ +-scc + +status_line_regex="^[0-9]+ tests, ([0-9]+) failed, ([0-9]+) errored.*" + +RC1=$(cat min_test_results.log | grep -E "${status_line_regex}" | sed -re "s/${status_line_regex}/\1\2/") cd test @@ -29,6 +54,9 @@ status_line_regex="^[0-9]+ tests, ([0-9]+) failed, ([0-9]+) errored.*" #cat coverage.xml #cat test_results.xml -RC=$(cat test_results.log | grep -E "${status_line_regex}" | sed -re "s/${status_line_regex}/\1\2/") +RC2=$(cat test_results.log | grep -E "${status_line_regex}" | sed -re "s/${status_line_regex}/\1\2/") + +check_result $RC1 $RC2 +RC=$? exit $RC diff --git a/test/min_grant_user/min_grant_user_exp.pkb b/test/min_grant_user/min_grant_user_exp.pkb new file mode 100644 index 000000000..1a8089c0b --- /dev/null +++ b/test/min_grant_user/min_grant_user_exp.pkb @@ -0,0 +1,19 @@ +create or replace package body min_grant_user_exp is + + procedure test_join_by_cursor is + l_actual SYS_REFCURSOR; + l_expected SYS_REFCURSOR; + begin + --Arrange + open l_actual for select owner, object_name,object_type from all_objects where owner = user + order by 1,2,3 asc; + open l_expected for select owner, object_name,object_type from all_objects where owner = user + order by 1,2,3 desc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by('OWNER'); + + end; + +end; +/ diff --git a/test/min_grant_user/min_grant_user_exp.pks b/test/min_grant_user/min_grant_user_exp.pks new file mode 100644 index 000000000..467d62ce0 --- /dev/null +++ b/test/min_grant_user/min_grant_user_exp.pks @@ -0,0 +1,9 @@ +create or replace package min_grant_user_exp is + + --%suite(minimum grant user tests) + + --%test(execute join by test) + procedure test_join_by_cursor; + +end; +/ From 4b3fe8afa42febfce52769e4bb5b6d9eb20180f0 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Sun, 28 Oct 2018 07:57:51 +0000 Subject: [PATCH 3/7] Update sqlcl download --- .travis/download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/download.sh b/.travis/download.sh index d7f0b7d0f..a071fafeb 100644 --- a/.travis/download.sh +++ b/.travis/download.sh @@ -60,7 +60,7 @@ fi if [ "$PRODUCT" == "sqlcl" ]; then agreementUrl="http://www.oracle.com/technetwork/developer-tools/sqlcl/downloads/index.html" - downloadUrl="http://download.oracle.com/otn/java/sqldeveloper/sqlcl-18.2.0.zip" + downloadUrl="http://download.oracle.com/otn/java/sqldeveloper/sqlcl-18.3.0.259.2029.zip" outputFile=sqlcl-18.2.0.zip downloadFile $agreementUrl $downloadUrl $outputFile exit 0 From 40314b6045acbd3e5f71c83b502796b3c15da018 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Sun, 28 Oct 2018 08:03:23 +0000 Subject: [PATCH 4/7] Revert "Update sqlcl download" This reverts commit 4b3fe8afa42febfce52769e4bb5b6d9eb20180f0. --- .travis/download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/download.sh b/.travis/download.sh index a071fafeb..d7f0b7d0f 100644 --- a/.travis/download.sh +++ b/.travis/download.sh @@ -60,7 +60,7 @@ fi if [ "$PRODUCT" == "sqlcl" ]; then agreementUrl="http://www.oracle.com/technetwork/developer-tools/sqlcl/downloads/index.html" - downloadUrl="http://download.oracle.com/otn/java/sqldeveloper/sqlcl-18.3.0.259.2029.zip" + downloadUrl="http://download.oracle.com/otn/java/sqldeveloper/sqlcl-18.2.0.zip" outputFile=sqlcl-18.2.0.zip downloadFile $agreementUrl $downloadUrl $outputFile exit 0 From 72851ff5699583671415a890d17317fe9e258ab8 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Thu, 1 Nov 2018 18:46:01 +0000 Subject: [PATCH 5/7] Squashed commit of the following: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit 899ddcffd57b5bb66448ea8306933dafae543cc2 Author: Travis CI Date: Thu Nov 1 01:18:58 2018 +0000 Updated project version after build [skip ci] commit df9dac7b2f4b5323db7a54865bc8f0be42ca34a8 Merge: 013cba3b 2a6c82c5 Author: Jacek Gębal Date: Thu Nov 1 01:05:42 2018 +0000 Merge pull request #772 from utPLSQL/bugfix/fix_travis_sqlcl_download_vini Fix sqlcl download commit 2a6c82c513dfe08681d401873762af8d40582976 Author: Jacek Gębal Date: Sun Oct 28 08:01:14 2018 +0000 Fix sqlcl download --- .travis/download.js | 4 +++- .travis/download.sh | 7 ++++--- .travis/install_sqlcl.sh | 2 +- source/core/ut_utils.pks | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.travis/download.js b/.travis/download.js index bc981cdf4..339001c57 100644 --- a/.travis/download.js +++ b/.travis/download.js @@ -17,6 +17,7 @@ var paramUsername = casper.cli.get(0); var paramPassword = casper.cli.get(1); var agreementUrl = casper.cli.get(2); var downloadUrl = casper.cli.get(3); +var downloaded = false; casper.start(); // TODO: Error handling. @@ -41,10 +42,11 @@ casper.thenOpen(downloadUrl).waitForUrl(/signon\.jsp$/, function (re) { }); casper.on("resource.received", function (resource) { - if (resource.url.indexOf("AuthParam") !== -1) { + if (resource.url.indexOf("AuthParam") !== -1 && !downloaded) { // this.echo("DownloadUrl:"); // Print the download url. this.echo(resource.url); + downloaded = true; // TODO: Try to download file from here. this.download is not working because of cross site request. } }); diff --git a/.travis/download.sh b/.travis/download.sh index d7f0b7d0f..fdbcdef7c 100644 --- a/.travis/download.sh +++ b/.travis/download.sh @@ -12,8 +12,9 @@ PRODUCT="" # Then download the file using curl. downloadFile() { downloadUrl=$(exec casperjs download.js $ORACLE_OTN_USER $ORACLE_OTN_PASSWORD $1 $2) + downloadUrl=${downloadUrl%$'\r'} echo "DownloadURL: $downloadUrl" - curl $downloadUrl -o $3 + curl -o $3 -L "$downloadUrl" } ############################# @@ -60,8 +61,8 @@ fi if [ "$PRODUCT" == "sqlcl" ]; then agreementUrl="http://www.oracle.com/technetwork/developer-tools/sqlcl/downloads/index.html" - downloadUrl="http://download.oracle.com/otn/java/sqldeveloper/sqlcl-18.2.0.zip" - outputFile=sqlcl-18.2.0.zip + downloadUrl="https://download.oracle.com/otn/java/sqldeveloper/sqlcl-18.3.0.259.2029.zip" + outputFile=sqlcl-18.3.0.259.2029.zip downloadFile $agreementUrl $downloadUrl $outputFile exit 0 fi diff --git a/.travis/install_sqlcl.sh b/.travis/install_sqlcl.sh index 9db262c5d..9743ddac2 100644 --- a/.travis/install_sqlcl.sh +++ b/.travis/install_sqlcl.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -SQLCL_FILE=sqlcl-18.2.0.zip +SQLCL_FILE=sqlcl-18.3.0.259.2029.zip cd .travis # Download if not present on cache dir. diff --git a/source/core/ut_utils.pks b/source/core/ut_utils.pks index 5623efa9b..60f189a66 100644 --- a/source/core/ut_utils.pks +++ b/source/core/ut_utils.pks @@ -21,7 +21,7 @@ create or replace package ut_utils authid definer is * */ - gc_version constant varchar2(50) := 'v3.1.3.2297-develop'; + gc_version constant varchar2(50) := 'v3.1.3.2321-develop'; /* Constants: Event names */ subtype t_event_name is varchar2(30); From 5f25daf8241da361a2f34e9adf1362ed2d9c1aad Mon Sep 17 00:00:00 2001 From: lwasylow Date: Thu, 1 Nov 2018 19:10:40 +0000 Subject: [PATCH 6/7] Update run test shell. --- test/install_and_run_tests.sh | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/test/install_and_run_tests.sh b/test/install_and_run_tests.sh index 04d70fb39..79dfda65e 100644 --- a/test/install_and_run_tests.sh +++ b/test/install_and_run_tests.sh @@ -1,17 +1,6 @@ #!/bin/bash set -ev -check_result() -{ - RC1=$1 - RC2=$2 - - if [ "$RC1" != "0" ] || [ "$RC2" != "0" ]; then - return 1 - fi - return 0 -} - #goto git root directory git rev-parse && cd "$(git rev-parse --show-cdup)" @@ -29,7 +18,11 @@ time utPLSQL-cli/bin/utplsql run ${UT3_USER}/${UT3_USER_PASSWORD}@${CONNECTION_S status_line_regex="^[0-9]+ tests, ([0-9]+) failed, ([0-9]+) errored.*" -RC1=$(cat min_test_results.log | grep -E "${status_line_regex}" | sed -re "s/${status_line_regex}/\1\2/") +RC=$(cat min_test_results.log | grep -E "${status_line_regex}" | sed -re "s/${status_line_regex}/\1\2/") + +if [ "$RC" == "1" ] then + exit 1 +fi cd test @@ -54,9 +47,7 @@ status_line_regex="^[0-9]+ tests, ([0-9]+) failed, ([0-9]+) errored.*" #cat coverage.xml #cat test_results.xml -RC2=$(cat test_results.log | grep -E "${status_line_regex}" | sed -re "s/${status_line_regex}/\1\2/") - -check_result $RC1 $RC2 -RC=$? +RC=$(cat test_results.log | grep -E "${status_line_regex}" | sed -re "s/${status_line_regex}/\1\2/") exit $RC + From 2df635485077cb5e26af37a2b1e5f812348781d3 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Thu, 1 Nov 2018 19:18:25 +0000 Subject: [PATCH 7/7] Update shell --- test/install_and_run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/install_and_run_tests.sh b/test/install_and_run_tests.sh index 79dfda65e..b8aba77e2 100644 --- a/test/install_and_run_tests.sh +++ b/test/install_and_run_tests.sh @@ -20,7 +20,7 @@ status_line_regex="^[0-9]+ tests, ([0-9]+) failed, ([0-9]+) errored.*" RC=$(cat min_test_results.log | grep -E "${status_line_regex}" | sed -re "s/${status_line_regex}/\1\2/") -if [ "$RC" == "1" ] then +if [ "$RC" == "1" ]; then exit 1 fi