From 5c8d56d769e3c4669c6c187b86c19305992bc2aa Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 10:59:19 -0500 Subject: [PATCH 01/14] chore(python): update dependencies in /.kokoro (#287) Source-Link: https://github.com/googleapis/synthtool/commit/db94845da69ccdfefd7ce55c84e6cfa74829747e Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:a8a80fc6456e433df53fc2a0d72ca0345db0ddefb409f1b75b118dfd1babd952 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 +- .kokoro/build.sh | 7 -- .kokoro/docker/docs/Dockerfile | 4 + .kokoro/docker/docs/requirements.in | 1 + .kokoro/docker/docs/requirements.txt | 38 +++++++++ .kokoro/requirements.in | 3 +- .kokoro/requirements.txt | 114 ++++++++++++--------------- 7 files changed, 99 insertions(+), 72 deletions(-) create mode 100644 .kokoro/docker/docs/requirements.in create mode 100644 .kokoro/docker/docs/requirements.txt diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index e4e943e0..4bdeef39 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:98f3afd11308259de6e828e37376d18867fd321aba07826e29e4f8d9cab56bad -# created: 2024-02-27T15:56:18.442440378Z + digest: sha256:a8a80fc6456e433df53fc2a0d72ca0345db0ddefb409f1b75b118dfd1babd952 +# created: 2024-03-15T16:25:47.905264637Z diff --git a/.kokoro/build.sh b/.kokoro/build.sh index c44a3ba9..e5247962 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -33,13 +33,6 @@ export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json # Setup project id. export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") -# Remove old nox -python3 -m pip uninstall --yes --quiet nox-automation - -# Install nox -python3 -m pip install --upgrade --quiet nox -python3 -m nox --version - # If this is a continuous build, send the test log to the FlakyBot. # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile index 8e39a2cc..bdaf39fe 100644 --- a/.kokoro/docker/docs/Dockerfile +++ b/.kokoro/docker/docs/Dockerfile @@ -80,4 +80,8 @@ RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ # Test pip RUN python3 -m pip +# Install build requirements +COPY requirements.txt /requirements.txt +RUN python3 -m pip install --require-hashes -r requirements.txt + CMD ["python3.8"] diff --git a/.kokoro/docker/docs/requirements.in b/.kokoro/docker/docs/requirements.in new file mode 100644 index 00000000..816817c6 --- /dev/null +++ b/.kokoro/docker/docs/requirements.in @@ -0,0 +1 @@ +nox diff --git a/.kokoro/docker/docs/requirements.txt b/.kokoro/docker/docs/requirements.txt new file mode 100644 index 00000000..0e5d70f2 --- /dev/null +++ b/.kokoro/docker/docs/requirements.txt @@ -0,0 +1,38 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes requirements.in +# +argcomplete==3.2.3 \ + --hash=sha256:bf7900329262e481be5a15f56f19736b376df6f82ed27576fa893652c5de6c23 \ + --hash=sha256:c12355e0494c76a2a7b73e3a59b09024ca0ba1e279fb9ed6c1b82d5b74b6a70c + # via nox +colorlog==6.8.2 \ + --hash=sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44 \ + --hash=sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33 + # via nox +distlib==0.3.8 \ + --hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \ + --hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64 + # via virtualenv +filelock==3.13.1 \ + --hash=sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e \ + --hash=sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c + # via virtualenv +nox==2024.3.2 \ + --hash=sha256:e53514173ac0b98dd47585096a55572fe504fecede58ced708979184d05440be \ + --hash=sha256:f521ae08a15adbf5e11f16cb34e8d0e6ea521e0b92868f684e91677deb974553 + # via -r requirements.in +packaging==24.0 \ + --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \ + --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9 + # via nox +platformdirs==4.2.0 \ + --hash=sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068 \ + --hash=sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768 + # via virtualenv +virtualenv==20.25.1 \ + --hash=sha256:961c026ac520bac5f69acb8ea063e8a4f071bcc9457b9c1f28f6b085c511583a \ + --hash=sha256:e08e13ecdca7a0bd53798f356d5831434afa5b07b93f0abdf0797b7a06ffe197 + # via nox diff --git a/.kokoro/requirements.in b/.kokoro/requirements.in index ec867d9f..fff4d9ce 100644 --- a/.kokoro/requirements.in +++ b/.kokoro/requirements.in @@ -1,5 +1,5 @@ gcp-docuploader -gcp-releasetool>=1.10.5 # required for compatibility with cryptography>=39.x +gcp-releasetool>=2 # required for compatibility with cryptography>=42.x importlib-metadata typing-extensions twine @@ -8,3 +8,4 @@ setuptools nox>=2022.11.21 # required to remove dependency on py charset-normalizer<3 click<8.1.0 +cryptography>=42.0.5 diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index bda8e38c..dd61f5f3 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -93,40 +93,41 @@ colorlog==6.7.0 \ # via # gcp-docuploader # nox -cryptography==42.0.4 \ - --hash=sha256:01911714117642a3f1792c7f376db572aadadbafcd8d75bb527166009c9f1d1b \ - --hash=sha256:0e89f7b84f421c56e7ff69f11c441ebda73b8a8e6488d322ef71746224c20fce \ - --hash=sha256:12d341bd42cdb7d4937b0cabbdf2a94f949413ac4504904d0cdbdce4a22cbf88 \ - --hash=sha256:15a1fb843c48b4a604663fa30af60818cd28f895572386e5f9b8a665874c26e7 \ - --hash=sha256:1cdcdbd117681c88d717437ada72bdd5be9de117f96e3f4d50dab3f59fd9ab20 \ - --hash=sha256:1df6fcbf60560d2113b5ed90f072dc0b108d64750d4cbd46a21ec882c7aefce9 \ - --hash=sha256:3c6048f217533d89f2f8f4f0fe3044bf0b2090453b7b73d0b77db47b80af8dff \ - --hash=sha256:3e970a2119507d0b104f0a8e281521ad28fc26f2820687b3436b8c9a5fcf20d1 \ - --hash=sha256:44a64043f743485925d3bcac548d05df0f9bb445c5fcca6681889c7c3ab12764 \ - --hash=sha256:4e36685cb634af55e0677d435d425043967ac2f3790ec652b2b88ad03b85c27b \ - --hash=sha256:5f8907fcf57392cd917892ae83708761c6ff3c37a8e835d7246ff0ad251d9298 \ - --hash=sha256:69b22ab6506a3fe483d67d1ed878e1602bdd5912a134e6202c1ec672233241c1 \ - --hash=sha256:6bfadd884e7280df24d26f2186e4e07556a05d37393b0f220a840b083dc6a824 \ - --hash=sha256:6d0fbe73728c44ca3a241eff9aefe6496ab2656d6e7a4ea2459865f2e8613257 \ - --hash=sha256:6ffb03d419edcab93b4b19c22ee80c007fb2d708429cecebf1dd3258956a563a \ - --hash=sha256:810bcf151caefc03e51a3d61e53335cd5c7316c0a105cc695f0959f2c638b129 \ - --hash=sha256:831a4b37accef30cccd34fcb916a5d7b5be3cbbe27268a02832c3e450aea39cb \ - --hash=sha256:887623fe0d70f48ab3f5e4dbf234986b1329a64c066d719432d0698522749929 \ - --hash=sha256:a0298bdc6e98ca21382afe914c642620370ce0470a01e1bef6dd9b5354c36854 \ - --hash=sha256:a1327f280c824ff7885bdeef8578f74690e9079267c1c8bd7dc5cc5aa065ae52 \ - --hash=sha256:c1f25b252d2c87088abc8bbc4f1ecbf7c919e05508a7e8628e6875c40bc70923 \ - --hash=sha256:c3a5cbc620e1e17009f30dd34cb0d85c987afd21c41a74352d1719be33380885 \ - --hash=sha256:ce8613beaffc7c14f091497346ef117c1798c202b01153a8cc7b8e2ebaaf41c0 \ - --hash=sha256:d2a27aca5597c8a71abbe10209184e1a8e91c1fd470b5070a2ea60cafec35bcd \ - --hash=sha256:dad9c385ba8ee025bb0d856714f71d7840020fe176ae0229de618f14dae7a6e2 \ - --hash=sha256:db4b65b02f59035037fde0998974d84244a64c3265bdef32a827ab9b63d61b18 \ - --hash=sha256:e09469a2cec88fb7b078e16d4adec594414397e8879a4341c6ace96013463d5b \ - --hash=sha256:e53dc41cda40b248ebc40b83b31516487f7db95ab8ceac1f042626bc43a2f992 \ - --hash=sha256:f1e85a178384bf19e36779d91ff35c7617c885da487d689b05c1366f9933ad74 \ - --hash=sha256:f47be41843200f7faec0683ad751e5ef11b9a56a220d57f300376cd8aba81660 \ - --hash=sha256:fb0cef872d8193e487fc6bdb08559c3aa41b659a7d9be48b2e10747f47863925 \ - --hash=sha256:ffc73996c4fca3d2b6c1c8c12bfd3ad00def8621da24f547626bf06441400449 +cryptography==42.0.5 \ + --hash=sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee \ + --hash=sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576 \ + --hash=sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d \ + --hash=sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30 \ + --hash=sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413 \ + --hash=sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb \ + --hash=sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da \ + --hash=sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4 \ + --hash=sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd \ + --hash=sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc \ + --hash=sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8 \ + --hash=sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1 \ + --hash=sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc \ + --hash=sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e \ + --hash=sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8 \ + --hash=sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940 \ + --hash=sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400 \ + --hash=sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7 \ + --hash=sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16 \ + --hash=sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278 \ + --hash=sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74 \ + --hash=sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec \ + --hash=sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1 \ + --hash=sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2 \ + --hash=sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c \ + --hash=sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922 \ + --hash=sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a \ + --hash=sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6 \ + --hash=sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1 \ + --hash=sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e \ + --hash=sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac \ + --hash=sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7 # via + # -r requirements.in # gcp-releasetool # secretstorage distlib==0.3.7 \ @@ -145,9 +146,9 @@ gcp-docuploader==0.6.5 \ --hash=sha256:30221d4ac3e5a2b9c69aa52fdbef68cc3f27d0e6d0d90e220fc024584b8d2318 \ --hash=sha256:b7458ef93f605b9d46a4bf3a8dc1755dad1f31d030c8679edf304e343b347eea # via -r requirements.in -gcp-releasetool==1.16.0 \ - --hash=sha256:27bf19d2e87aaa884096ff941aa3c592c482be3d6a2bfe6f06afafa6af2353e3 \ - --hash=sha256:a316b197a543fd036209d0caba7a8eb4d236d8e65381c80cbc6d7efaa7606d63 +gcp-releasetool==2.0.0 \ + --hash=sha256:3d73480b50ba243f22d7c7ec08b115a30e1c7817c4899781840c26f9c55b8277 \ + --hash=sha256:7aa9fd935ec61e581eb8458ad00823786d91756c25e492f372b2b30962f3c28f # via -r requirements.in google-api-core==2.12.0 \ --hash=sha256:c22e01b1e3c4dcd90998494879612c38d0a3411d1f7b679eb89e2abe3ce1f553 \ @@ -392,29 +393,18 @@ platformdirs==3.11.0 \ --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e # via virtualenv -protobuf==3.20.3 \ - --hash=sha256:03038ac1cfbc41aa21f6afcbcd357281d7521b4157926f30ebecc8d4ea59dcb7 \ - --hash=sha256:28545383d61f55b57cf4df63eebd9827754fd2dc25f80c5253f9184235db242c \ - --hash=sha256:2e3427429c9cffebf259491be0af70189607f365c2f41c7c3764af6f337105f2 \ - --hash=sha256:398a9e0c3eaceb34ec1aee71894ca3299605fa8e761544934378bbc6c97de23b \ - --hash=sha256:44246bab5dd4b7fbd3c0c80b6f16686808fab0e4aca819ade6e8d294a29c7050 \ - --hash=sha256:447d43819997825d4e71bf5769d869b968ce96848b6479397e29fc24c4a5dfe9 \ - --hash=sha256:67a3598f0a2dcbc58d02dd1928544e7d88f764b47d4a286202913f0b2801c2e7 \ - --hash=sha256:74480f79a023f90dc6e18febbf7b8bac7508420f2006fabd512013c0c238f454 \ - --hash=sha256:819559cafa1a373b7096a482b504ae8a857c89593cf3a25af743ac9ecbd23480 \ - --hash=sha256:899dc660cd599d7352d6f10d83c95df430a38b410c1b66b407a6b29265d66469 \ - --hash=sha256:8c0c984a1b8fef4086329ff8dd19ac77576b384079247c770f29cc8ce3afa06c \ - --hash=sha256:9aae4406ea63d825636cc11ffb34ad3379335803216ee3a856787bcf5ccc751e \ - --hash=sha256:a7ca6d488aa8ff7f329d4c545b2dbad8ac31464f1d8b1c87ad1346717731e4db \ - --hash=sha256:b6cc7ba72a8850621bfec987cb72623e703b7fe2b9127a161ce61e61558ad905 \ - --hash=sha256:bf01b5720be110540be4286e791db73f84a2b721072a3711efff6c324cdf074b \ - --hash=sha256:c02ce36ec760252242a33967d51c289fd0e1c0e6e5cc9397e2279177716add86 \ - --hash=sha256:d9e4432ff660d67d775c66ac42a67cf2453c27cb4d738fc22cb53b5d84c135d4 \ - --hash=sha256:daa564862dd0d39c00f8086f88700fdbe8bc717e993a21e90711acfed02f2402 \ - --hash=sha256:de78575669dddf6099a8a0f46a27e82a1783c557ccc38ee620ed8cc96d3be7d7 \ - --hash=sha256:e64857f395505ebf3d2569935506ae0dfc4a15cb80dc25261176c784662cdcc4 \ - --hash=sha256:f4bd856d702e5b0d96a00ec6b307b0f51c1982c2bf9c0052cf9019e9a544ba99 \ - --hash=sha256:f4c42102bc82a51108e449cbb32b19b180022941c727bac0cfd50170341f16ee +protobuf==4.25.3 \ + --hash=sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4 \ + --hash=sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8 \ + --hash=sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c \ + --hash=sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d \ + --hash=sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4 \ + --hash=sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa \ + --hash=sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c \ + --hash=sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019 \ + --hash=sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9 \ + --hash=sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c \ + --hash=sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2 # via # gcp-docuploader # gcp-releasetool @@ -518,7 +508,7 @@ zipp==3.17.0 \ # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==68.2.2 \ - --hash=sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87 \ - --hash=sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a +setuptools==69.2.0 \ + --hash=sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e \ + --hash=sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c # via -r requirements.in From cf838dc3675a93bed7931f38e29cbcc0f059c389 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 09:25:04 -0700 Subject: [PATCH 02/14] chore(python): add requirements for docs build (#288) Source-Link: https://github.com/googleapis/synthtool/commit/85c23b6bc4352c1b0674848eaeb4e48645aeda6b Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:3741fd1f5f5150378563c76afa06bcc12777b5fe54c5ee01115218f83872134f Co-authored-by: Owl Bot Co-authored-by: Holt Skinner <13262395+holtskinner@users.noreply.github.com> --- .github/.OwlBot.lock.yaml | 4 ++-- .github/auto-label.yaml | 5 +++++ .github/blunderbuss.yml | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 .github/blunderbuss.yml diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 4bdeef39..59b9e7c0 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:a8a80fc6456e433df53fc2a0d72ca0345db0ddefb409f1b75b118dfd1babd952 -# created: 2024-03-15T16:25:47.905264637Z + digest: sha256:d0e70b7cbce0f340c4755bafee5c1ef754c4e83927ecce8988f4f4c530572f61 +# created: 2024-03-18T16:26:15.743347415Z diff --git a/.github/auto-label.yaml b/.github/auto-label.yaml index b2016d11..8b37ee89 100644 --- a/.github/auto-label.yaml +++ b/.github/auto-label.yaml @@ -13,3 +13,8 @@ # limitations under the License. requestsize: enabled: true + +path: + pullrequest: true + paths: + samples: "samples" diff --git a/.github/blunderbuss.yml b/.github/blunderbuss.yml new file mode 100644 index 00000000..d3d1a9e5 --- /dev/null +++ b/.github/blunderbuss.yml @@ -0,0 +1,17 @@ +# Blunderbuss config +# +# This file controls who is assigned for pull requests and issues. +# Note: This file is autogenerated. To make changes to the assignee +# team, please update `codeowner_team` in `.repo-metadata.json`. +assign_issues: + - googleapis/cdpe-cloudai + +assign_issues_by: + - labels: + - "samples" + to: + - googleapis/python-samples-reviewers + - googleapis/cdpe-cloudai + +assign_prs: + - googleapis/cdpe-cloudai From 0dd5d5c73931b86ddf16290d8b5480fb578cf2ff Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 19 Mar 2024 15:49:53 +0100 Subject: [PATCH 03/14] chore(deps): update dependency google-cloud-storage to v2.16.0 (#289) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index a249eb5f..f308c49c 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ google-cloud-bigquery==3.19.0 google-cloud-documentai==2.24.1 -google-cloud-storage==2.15.0 +google-cloud-storage==2.16.0 google-cloud-documentai-toolbox==0.13.1a0 From e75019c3b9e7ff402722cc4ed441680e3da360c5 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Fri, 22 Mar 2024 16:44:28 +0100 Subject: [PATCH 04/14] chore(deps): update dependency google-cloud-documentai to v2.24.2 (#290) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index f308c49c..df5d7edc 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ google-cloud-bigquery==3.19.0 -google-cloud-documentai==2.24.1 +google-cloud-documentai==2.24.2 google-cloud-storage==2.16.0 google-cloud-documentai-toolbox==0.13.1a0 From 6ccf3a29ffac0c728b63dc6b8caaa4fd2757ec70 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 25 Mar 2024 16:41:54 +0100 Subject: [PATCH 05/14] chore(deps): update dependency filelock to v3.13.2 (#291) From b6edc3d7886c12a9d2c69583df25948b3de8337d Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 28 Mar 2024 14:23:48 +0100 Subject: [PATCH 06/14] chore(deps): update all dependencies (#294) * chore(deps): update all dependencies --- samples/snippets/requirements-test.txt | 2 +- samples/snippets/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index 4abbc9d1..eee44e7e 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,3 @@ pytest==8.1.1 mock==5.1.0 -google-cloud-bigquery==3.19.0 +google-cloud-bigquery==3.20.0 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index df5d7edc..093b9572 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-bigquery==3.19.0 +google-cloud-bigquery==3.20.0 google-cloud-documentai==2.24.2 google-cloud-storage==2.16.0 google-cloud-documentai-toolbox==0.13.1a0 From bbf4c1adebf0283a125975d0fb9da99919381a06 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 2 Apr 2024 19:32:06 +0200 Subject: [PATCH 07/14] chore(deps): update all dependencies (#297) --------- Co-authored-by: Owl Bot Co-authored-by: Holt Skinner --- docs/conf.py | 2 +- .../converters/config/bbox_conversion.py | 5 ++--- google/cloud/documentai_toolbox/wrappers/document.py | 1 - google/cloud/documentai_toolbox/wrappers/page.py | 3 +-- owlbot.py | 3 +-- samples/snippets/create_batches_sample.py | 1 + samples/snippets/noxfile.py | 4 +--- samples/snippets/quickstart_sample.py | 3 +-- samples/snippets/requirements-test.txt | 2 +- samples/snippets/requirements.txt | 2 +- samples/snippets/test_convert_document_to_vision_sample.py | 1 + .../snippets/test_convert_external_annotations_sample.py | 1 + samples/snippets/test_create_batches_sample.py | 1 + samples/snippets/test_entities_to_bigquery_sample.py | 4 ++-- samples/snippets/test_export_images_sample.py | 1 + samples/snippets/test_merge_document_shards_sample.py | 1 + samples/snippets/test_quickstart_sample.py | 7 ++++--- samples/snippets/test_split_pdf_sample.py | 1 + samples/snippets/test_table_sample.py | 1 + setup.py | 1 + 20 files changed, 24 insertions(+), 21 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index eab40935..e1af8af8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,9 +24,9 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os import shlex +import sys # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the diff --git a/google/cloud/documentai_toolbox/converters/config/bbox_conversion.py b/google/cloud/documentai_toolbox/converters/config/bbox_conversion.py index dce55550..b256fea7 100644 --- a/google/cloud/documentai_toolbox/converters/config/bbox_conversion.py +++ b/google/cloud/documentai_toolbox/converters/config/bbox_conversion.py @@ -16,11 +16,10 @@ from typing import Callable, List, Optional -from google.cloud import documentai -from google.cloud.documentai_toolbox.converters.config.block import Block - from intervaltree import intervaltree +from google.cloud import documentai +from google.cloud.documentai_toolbox.converters.config.block import Block PIXEL_CONVERSION_RATES = { "pxl": 1, diff --git a/google/cloud/documentai_toolbox/wrappers/document.py b/google/cloud/documentai_toolbox/wrappers/document.py index bface9af..57fc0dd9 100644 --- a/google/cloud/documentai_toolbox/wrappers/document.py +++ b/google/cloud/documentai_toolbox/wrappers/document.py @@ -26,7 +26,6 @@ from google.api_core.operation import from_gapic as operation_from_gapic from google.cloud.vision import AnnotateFileResponse from google.longrunning.operations_pb2 import GetOperationRequest - from jinja2 import Environment, PackageLoader from pikepdf import Pdf diff --git a/google/cloud/documentai_toolbox/wrappers/page.py b/google/cloud/documentai_toolbox/wrappers/page.py index 1d314c27..83287ee3 100644 --- a/google/cloud/documentai_toolbox/wrappers/page.py +++ b/google/cloud/documentai_toolbox/wrappers/page.py @@ -16,9 +16,8 @@ """Wrappers for Document AI Page type.""" from abc import ABC - import dataclasses -from typing import cast, List, Optional, Type +from typing import List, Optional, Type, cast import pandas as pd diff --git a/owlbot.py b/owlbot.py index 4c6e8171..f5483bfc 100644 --- a/owlbot.py +++ b/owlbot.py @@ -18,7 +18,6 @@ import synthtool as s from synthtool import gcp -from synthtool.languages import python REPO_ROOT = pathlib.Path(__file__).parent.absolute() @@ -41,4 +40,4 @@ # Run blacken session # ---------------------------------------------------------------------------- -s.shell.run(["nox", "-s", "blacken"], hide_output=False) +s.shell.run(["nox", "-s", "format"], hide_output=False) diff --git a/samples/snippets/create_batches_sample.py b/samples/snippets/create_batches_sample.py index 6700f569..783a0648 100644 --- a/samples/snippets/create_batches_sample.py +++ b/samples/snippets/create_batches_sample.py @@ -47,6 +47,7 @@ def create_batches_sample( request = documentai.BatchProcessRequest( name="processor_name", input_documents=batch ) + print(request) # [END documentai_toolbox_create_batches] diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 8050fe14..af478289 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import annotations -from __future__ import print_function +from __future__ import annotations, print_function import glob import os @@ -23,7 +22,6 @@ import nox - # WARNING - WARNING - WARNING - WARNING - WARNING # WARNING - WARNING - WARNING - WARNING - WARNING # DO NOT EDIT THIS FILE EVER! diff --git a/samples/snippets/quickstart_sample.py b/samples/snippets/quickstart_sample.py index a387c438..e08dc4f3 100644 --- a/samples/snippets/quickstart_sample.py +++ b/samples/snippets/quickstart_sample.py @@ -18,8 +18,7 @@ from typing import Optional from google.cloud import documentai -from google.cloud.documentai_toolbox import document -from google.cloud.documentai_toolbox import gcs_utilities +from google.cloud.documentai_toolbox import document, gcs_utilities # TODO(developer): Uncomment these variables before running the sample. # Given a Document JSON or sharded Document JSON in path gs://bucket/path/to/folder diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index eee44e7e..5c53022a 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,3 @@ pytest==8.1.1 mock==5.1.0 -google-cloud-bigquery==3.20.0 +google-cloud-bigquery==3.20.1 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 093b9572..9cafc473 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-bigquery==3.20.0 +google-cloud-bigquery==3.20.1 google-cloud-documentai==2.24.2 google-cloud-storage==2.16.0 google-cloud-documentai-toolbox==0.13.1a0 diff --git a/samples/snippets/test_convert_document_to_vision_sample.py b/samples/snippets/test_convert_document_to_vision_sample.py index 668e1acd..d3bb94b8 100644 --- a/samples/snippets/test_convert_document_to_vision_sample.py +++ b/samples/snippets/test_convert_document_to_vision_sample.py @@ -16,6 +16,7 @@ import os import pytest + from samples.snippets import convert_document_to_vision_sample location = "us" diff --git a/samples/snippets/test_convert_external_annotations_sample.py b/samples/snippets/test_convert_external_annotations_sample.py index c29a645a..23b7440d 100644 --- a/samples/snippets/test_convert_external_annotations_sample.py +++ b/samples/snippets/test_convert_external_annotations_sample.py @@ -16,6 +16,7 @@ import os import pytest + from samples.snippets import convert_external_annotations_sample location = "us" diff --git a/samples/snippets/test_create_batches_sample.py b/samples/snippets/test_create_batches_sample.py index b13c6279..5ad8a6fd 100644 --- a/samples/snippets/test_create_batches_sample.py +++ b/samples/snippets/test_create_batches_sample.py @@ -15,6 +15,7 @@ import pytest + from samples.snippets import create_batches_sample gcs_bucket_name = "cloud-samples-data" diff --git a/samples/snippets/test_entities_to_bigquery_sample.py b/samples/snippets/test_entities_to_bigquery_sample.py index 04c74a74..1687a7c1 100644 --- a/samples/snippets/test_entities_to_bigquery_sample.py +++ b/samples/snippets/test_entities_to_bigquery_sample.py @@ -16,10 +16,10 @@ import os import uuid +from google.cloud import bigquery import pytest -from samples.snippets import entities_to_bigquery_sample -from google.cloud import bigquery +from samples.snippets import entities_to_bigquery_sample location = "us" project_id = os.environ["GOOGLE_CLOUD_PROJECT"] diff --git a/samples/snippets/test_export_images_sample.py b/samples/snippets/test_export_images_sample.py index 4c2f92c4..9aa5ab43 100644 --- a/samples/snippets/test_export_images_sample.py +++ b/samples/snippets/test_export_images_sample.py @@ -17,6 +17,7 @@ import shutil import pytest + from samples.snippets import export_images_sample document_path = "../../tests/unit/resources/images/dl3-0.json" diff --git a/samples/snippets/test_merge_document_shards_sample.py b/samples/snippets/test_merge_document_shards_sample.py index 58a99467..d0405e3e 100644 --- a/samples/snippets/test_merge_document_shards_sample.py +++ b/samples/snippets/test_merge_document_shards_sample.py @@ -17,6 +17,7 @@ import shutil import pytest + from samples.snippets import merge_document_shards_sample gcs_bucket_name = "documentai_toolbox_samples" diff --git a/samples/snippets/test_quickstart_sample.py b/samples/snippets/test_quickstart_sample.py index e1dd1370..6c2244ed 100644 --- a/samples/snippets/test_quickstart_sample.py +++ b/samples/snippets/test_quickstart_sample.py @@ -15,11 +15,12 @@ import os +from google.cloud import documentai +from google.longrunning.operations_pb2 import \ + ListOperationsRequest # type: ignore import pytest -from samples.snippets import quickstart_sample -from google.cloud import documentai -from google.longrunning.operations_pb2 import ListOperationsRequest # type: ignore +from samples.snippets import quickstart_sample location = "us" project_id = os.environ["GOOGLE_CLOUD_PROJECT"] diff --git a/samples/snippets/test_split_pdf_sample.py b/samples/snippets/test_split_pdf_sample.py index 5adac3c9..31aaeead 100644 --- a/samples/snippets/test_split_pdf_sample.py +++ b/samples/snippets/test_split_pdf_sample.py @@ -17,6 +17,7 @@ import shutil import pytest + from samples.snippets import split_pdf_sample document_path = "../../tests/unit/resources/splitter/procurement_splitter_output.json" diff --git a/samples/snippets/test_table_sample.py b/samples/snippets/test_table_sample.py index 45935ff5..d347b47e 100644 --- a/samples/snippets/test_table_sample.py +++ b/samples/snippets/test_table_sample.py @@ -16,6 +16,7 @@ import os import pytest + from samples.snippets import table_sample document_path = "resources/form_with_tables.json" diff --git a/setup.py b/setup.py index 23265516..6a6f6c98 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ # import io import os + import setuptools # type: ignore package_root = os.path.abspath(os.path.dirname(__file__)) From 5e1bb1a394af47522fa35861840509b17caba4cd Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 3 Apr 2024 18:02:06 +0200 Subject: [PATCH 08/14] chore(deps): update all dependencies (#299) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 9cafc473..3e71f647 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ google-cloud-bigquery==3.20.1 -google-cloud-documentai==2.24.2 +google-cloud-documentai==2.25.0 google-cloud-storage==2.16.0 google-cloud-documentai-toolbox==0.13.1a0 From 09f38f32f3287f56365f8ed6b98a527bce296c95 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 17:39:05 -0400 Subject: [PATCH 09/14] chore(python): bump idna from 3.4 to 3.7 in .kokoro (#304) * chore(python): bump idna from 3.4 to 3.7 in .kokoro Source-Link: https://github.com/googleapis/synthtool/commit/d50980e704793a2d3310bfb3664f3a82f24b5796 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:5a4c19d17e597b92d786e569be101e636c9c2817731f80a5adec56b2aa8fe070 * Apply changes from googleapis/synthtool#1950 --------- Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- .github/.OwlBot.lock.yaml | 4 ++-- .kokoro/requirements.txt | 6 +++--- docs/index.rst | 5 +++++ docs/summary_overview.md | 22 ++++++++++++++++++++++ 4 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 docs/summary_overview.md diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 59b9e7c0..81f87c56 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:d0e70b7cbce0f340c4755bafee5c1ef754c4e83927ecce8988f4f4c530572f61 -# created: 2024-03-18T16:26:15.743347415Z + digest: sha256:5a4c19d17e597b92d786e569be101e636c9c2817731f80a5adec56b2aa8fe070 +# created: 2024-04-12T11:35:58.922854369Z diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index dd61f5f3..51f92b8e 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -252,9 +252,9 @@ googleapis-common-protos==1.61.0 \ --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b # via google-api-core -idna==3.4 \ - --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ - --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 +idna==3.7 \ + --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ + --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via requests importlib-metadata==6.8.0 \ --hash=sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb \ diff --git a/docs/index.rst b/docs/index.rst index 222fbe7a..34bc5efa 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,3 +14,8 @@ Changelog :maxdepth: 2 changelog + +.. toctree:: + :hidden: + + summary_overview.md diff --git a/docs/summary_overview.md b/docs/summary_overview.md new file mode 100644 index 00000000..adc78936 --- /dev/null +++ b/docs/summary_overview.md @@ -0,0 +1,22 @@ +[ +This is a templated file. Adding content to this file may result in it being +reverted. Instead, if you want to place additional content, create an +"overview_content.md" file in `docs/` directory. The Sphinx tool will +pick up on the content and merge the content. +]: # + +# Document AI Toolbox API + +Overview of the APIs available for Document AI Toolbox API. + +## All entries + +Classes, methods and properties & attributes for +Document AI Toolbox API. + +[classes](https://cloud.google.com/python/docs/reference/documentai-toolbox/latest/summary_class.html) + +[methods](https://cloud.google.com/python/docs/reference/documentai-toolbox/latest/summary_method.html) + +[properties and +attributes](https://cloud.google.com/python/docs/reference/documentai-toolbox/latest/summary_property.html) From 355684d6197c5e11aa629af6d16f0ff5bd5f6903 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 16:31:00 -0500 Subject: [PATCH 10/14] chore(deps): bump jinja2 from 3.1.3 to 3.1.4 in /.kokoro (#307) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): bump jinja2 from 3.1.3 to 3.1.4 in /.kokoro Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.3 to 3.1.4. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.3...3.1.4) --- updated-dependencies: - dependency-name: jinja2 dependency-type: indirect ... Signed-off-by: dependabot[bot] * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Owl Bot From 67be44a43b04e3a72fc1dfc26978679255825e4c Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 21 May 2024 17:14:54 +0200 Subject: [PATCH 11/14] chore(deps): update all dependencies (#309) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- samples/snippets/requirements-test.txt | 4 ++-- samples/snippets/requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index 5c53022a..12c2b12b 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==8.1.1 +pytest==8.2.1 mock==5.1.0 -google-cloud-bigquery==3.20.1 +google-cloud-bigquery==3.23.0 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 3e71f647..231a6522 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-bigquery==3.20.1 -google-cloud-documentai==2.25.0 +google-cloud-bigquery==3.23.0 +google-cloud-documentai==2.27.1 google-cloud-storage==2.16.0 google-cloud-documentai-toolbox==0.13.1a0 From 7773ffe4a5d3f57f12bb607932367c157e34f993 Mon Sep 17 00:00:00 2001 From: Holt Skinner <13262395+holtskinner@users.noreply.github.com> Date: Wed, 12 Jun 2024 12:56:17 -0500 Subject: [PATCH 12/14] refactor: Change read-only properties to `@cached_property` for readability and speed improvements (#210) * refactor: Change read-only properties to `@cached_property` for readability --------- Co-authored-by: Anthonios Partheniou Co-authored-by: Owl Bot --- .../documentai_toolbox/wrappers/document.py | 25 +- .../cloud/documentai_toolbox/wrappers/page.py | 270 ++++++------------ 2 files changed, 91 insertions(+), 204 deletions(-) diff --git a/google/cloud/documentai_toolbox/wrappers/document.py b/google/cloud/documentai_toolbox/wrappers/document.py index 57fc0dd9..92c0e8c6 100644 --- a/google/cloud/documentai_toolbox/wrappers/document.py +++ b/google/cloud/documentai_toolbox/wrappers/document.py @@ -17,6 +17,7 @@ import copy import dataclasses +from functools import cached_property import glob import os import re @@ -389,29 +390,17 @@ class Document: gcs_uri: Optional[str] = dataclasses.field(default=None, repr=False) gcs_input_uri: Optional[str] = dataclasses.field(default=None, repr=False) - _pages: Optional[List[Page]] = dataclasses.field( - init=False, repr=False, default=None - ) - _entities: List[Entity] = dataclasses.field(init=False, repr=False, default=None) - _text: str = dataclasses.field(init=False, repr=False, default=None) - - @property + @cached_property def pages(self): - if self._pages is None: - self._pages = _pages_from_shards(shards=self.shards) - return self._pages + return _pages_from_shards(shards=self.shards) - @property + @cached_property def entities(self): - if self._entities is None: - self._entities = _entities_from_shards(shards=self.shards) - return self._entities + return _entities_from_shards(shards=self.shards) - @property + @cached_property def text(self): - if self._text is None: - self._text = "".join(shard.text for shard in self.shards) - return self._text + return "".join(shard.text for shard in self.shards) @classmethod def from_document_path( diff --git a/google/cloud/documentai_toolbox/wrappers/page.py b/google/cloud/documentai_toolbox/wrappers/page.py index 83287ee3..1ca84e14 100644 --- a/google/cloud/documentai_toolbox/wrappers/page.py +++ b/google/cloud/documentai_toolbox/wrappers/page.py @@ -17,6 +17,7 @@ from abc import ABC import dataclasses +from functools import cached_property from typing import List, Optional, Type, cast import pandas as pd @@ -42,30 +43,19 @@ class Table: documentai_object: documentai.Document.Page.Table = dataclasses.field(repr=False) _page: "Page" = dataclasses.field(repr=False) - _body_rows: Optional[List[List[str]]] = dataclasses.field( - init=False, repr=False, default=None - ) - _header_rows: Optional[List[List[str]]] = dataclasses.field( - init=False, repr=False, default=None - ) - - @property + @cached_property def body_rows(self): - if self._body_rows is None: - self._body_rows = _table_rows_from_documentai_table_rows( - table_rows=list(self.documentai_object.body_rows), - text=self._page._document_text, - ) - return self._body_rows + return _table_rows_from_documentai_table_rows( + table_rows=list(self.documentai_object.body_rows), + text=self._page._document_text, + ) - @property + @cached_property def header_rows(self): - if self._header_rows is None: - self._header_rows = _table_rows_from_documentai_table_rows( - table_rows=list(self.documentai_object.header_rows), - text=self._page._document_text, - ) - return self._header_rows + return _table_rows_from_documentai_table_rows( + table_rows=list(self.documentai_object.header_rows), + text=self._page._document_text, + ) def to_dataframe(self) -> pd.DataFrame: r"""Returns pd.DataFrame from documentai.table @@ -104,30 +94,21 @@ class FormField: ) _page: "Page" = dataclasses.field(repr=False) - _field_name: Optional[str] = dataclasses.field(init=False, repr=False, default=None) - _field_value: Optional[str] = dataclasses.field( - init=False, repr=False, default=None - ) - - @property + @cached_property def field_name(self): - if self._field_name is None: - self._field_name = _trim_text( - _text_from_layout( - self.documentai_object.field_name, self._page._document_text - ) + return _trim_text( + _text_from_layout( + self.documentai_object.field_name, self._page._document_text ) - return self._field_name + ) - @property + @cached_property def field_value(self): - if self._field_value is None: - self._field_value = _trim_text( - _text_from_layout( - self.documentai_object.field_value, self._page._document_text - ) + return _trim_text( + _text_from_layout( + self.documentai_object.field_value, self._page._document_text ) - return self._field_value + ) @dataclasses.dataclass @@ -137,33 +118,24 @@ class _BasePageElement(ABC): documentai_object: ElementWithLayout = dataclasses.field(repr=False) _page: "Page" = dataclasses.field(repr=False) - _text: Optional[str] = dataclasses.field(init=False, repr=False, default=None) - _hocr_bounding_box: Optional[str] = dataclasses.field( - init=False, repr=False, default=None - ) - - @property + @cached_property def text(self): """ Text of the page element. """ - if self._text is None: - self._text = _text_from_layout( - layout=self.documentai_object.layout, text=self._page._document_text - ) - return self._text + return _text_from_layout( + layout=self.documentai_object.layout, text=self._page._document_text + ) - @property + @cached_property def hocr_bounding_box(self): """ hOCR bounding box of the page element. """ - if self._hocr_bounding_box is None: - self._hocr_bounding_box = _get_hocr_bounding_box( - element_with_layout=self.documentai_object, - page_dimension=self._page.documentai_object.dimension, - ) - return self._hocr_bounding_box + return _get_hocr_bounding_box( + element_with_layout=self.documentai_object, + page_dimension=self._page.documentai_object.dimension, + ) @dataclasses.dataclass @@ -178,7 +150,7 @@ class Symbol(_BasePageElement): Required. The text of the Symbol. """ - @property + @cached_property def hocr_bounding_box(self): # Symbols are not represented in hOCR return None @@ -197,18 +169,12 @@ class Token(_BasePageElement): Optional. The Symbols contained within the Token. """ - _symbols: Optional[List[Symbol]] = dataclasses.field( - init=False, repr=False, default=None - ) - - @property + @cached_property def symbols(self): - if self._symbols is None: - self._symbols = cast( - List[Symbol], - _get_children_of_element(self.documentai_object, self._page.symbols), - ) - return self._symbols + return cast( + List[Symbol], + _get_children_of_element(self.documentai_object, self._page.symbols), + ) @dataclasses.dataclass @@ -224,18 +190,12 @@ class Line(_BasePageElement): Optional. The Tokens contained within the Line. """ - _tokens: Optional[List[Token]] = dataclasses.field( - init=False, repr=False, default=None - ) - - @property + @cached_property def tokens(self): - if self._tokens is None: - self._tokens = cast( - List[Token], - _get_children_of_element(self.documentai_object, self._page.tokens), - ) - return self._tokens + return cast( + List[Token], + _get_children_of_element(self.documentai_object, self._page.tokens), + ) @dataclasses.dataclass @@ -255,14 +215,12 @@ class Paragraph(_BasePageElement): init=False, repr=False, default=None ) - @property + @cached_property def lines(self): - if self._lines is None: - self._lines = cast( - List[Line], - _get_children_of_element(self.documentai_object, self._page.lines), - ) - return self._lines + return cast( + List[Line], + _get_children_of_element(self.documentai_object, self._page.lines), + ) @dataclasses.dataclass @@ -278,18 +236,12 @@ class Block(_BasePageElement): Optional. The Paragraphs contained within the Block. """ - _paragraphs: Optional[List[Paragraph]] = dataclasses.field( - init=False, repr=False, default=None - ) - - @property + @cached_property def paragraphs(self): - if self._paragraphs is None: - self._paragraphs = cast( - List[Paragraph], - _get_children_of_element(self.documentai_object, self._page.paragraphs), - ) - return self._paragraphs + return cast( + List[Paragraph], + _get_children_of_element(self.documentai_object, self._page.paragraphs), + ) @dataclasses.dataclass @@ -304,7 +256,7 @@ class MathFormula(_BasePageElement): Required. The text of the MathFormula. """ - @property + @cached_property def hocr_bounding_box(self): # Math Formulas are not represented in hOCR return None @@ -473,38 +425,6 @@ class Page: documentai_object: documentai.Document.Page = dataclasses.field(repr=False) _document_text: str = dataclasses.field(repr=False) - _text: Optional[str] = dataclasses.field(init=False, repr=False, default=None) - _page_number: Optional[int] = dataclasses.field( - init=False, repr=False, default=None - ) - _form_fields: Optional[List[FormField]] = dataclasses.field( - init=False, repr=False, default=None - ) - _symbols: Optional[List[Symbol]] = dataclasses.field( - init=False, repr=False, default=None - ) - _tokens: Optional[List[Token]] = dataclasses.field( - init=False, repr=False, default=None - ) - _lines: Optional[List[Line]] = dataclasses.field( - init=False, repr=False, default=None - ) - _paragraphs: Optional[List[Paragraph]] = dataclasses.field( - init=False, repr=False, default=None - ) - _blocks: Optional[List[Block]] = dataclasses.field( - init=False, repr=False, default=None - ) - _tables: Optional[List[Table]] = dataclasses.field( - init=False, repr=False, default=None - ) - _math_formulas: Optional[List[MathFormula]] = dataclasses.field( - init=False, repr=False, default=None - ) - _hocr_bounding_box: Optional[str] = dataclasses.field( - init=False, repr=False, default=None - ) - def _get_elements(self, element_type: Type, attribute_name: str) -> List: """ Helper method to create elements based on specified type. @@ -514,77 +434,55 @@ def _get_elements(self, element_type: Type, attribute_name: str) -> List: element_type(documentai_object=element, _page=self) for element in elements ] - @property + @cached_property def text(self): - if self._text is None: - self._text = _text_from_layout( - self.documentai_object.layout, text=self._document_text - ) - return self._text + return _text_from_layout( + self.documentai_object.layout, text=self._document_text + ) - @property + @cached_property def page_number(self): - if self._page_number is None: - self._page_number = self.documentai_object.page_number - return self._page_number + return self.documentai_object.page_number - @property + @cached_property def tables(self): - if self._tables is None: - self._tables = self._get_elements(Table, "tables") - return self._tables + return self._get_elements(Table, "tables") - @property + @cached_property def form_fields(self): - if self._form_fields is None: - self._form_fields = self._get_elements(FormField, "form_fields") - return self._form_fields + return self._get_elements(FormField, "form_fields") - @property + @cached_property def math_formulas(self): - if self._math_formulas is None: - self._math_formulas = [ - MathFormula(documentai_object=visual_element, _page=self) - for visual_element in self.documentai_object.visual_elements - if visual_element.type_ == "math_formula" - ] - return self._math_formulas - - @property + return [ + MathFormula(documentai_object=visual_element, _page=self) + for visual_element in self.documentai_object.visual_elements + if visual_element.type_ == "math_formula" + ] + + @cached_property def symbols(self): - if self._symbols is None: - self._symbols = self._get_elements(Symbol, "symbols") - return self._symbols + return self._get_elements(Symbol, "symbols") - @property + @cached_property def tokens(self): - if self._tokens is None: - self._tokens = self._get_elements(Token, "tokens") - return self._tokens + return self._get_elements(Token, "tokens") - @property + @cached_property def lines(self): - if self._lines is None: - self._lines = self._get_elements(Line, "lines") - return self._lines + return self._get_elements(Line, "lines") - @property + @cached_property def paragraphs(self): - if self._paragraphs is None: - self._paragraphs = self._get_elements(Paragraph, "paragraphs") - return self._paragraphs + return self._get_elements(Paragraph, "paragraphs") - @property + @cached_property def blocks(self): - if self._blocks is None: - self._blocks = self._get_elements(Block, "blocks") - return self._blocks + return self._get_elements(Block, "blocks") - @property + @cached_property def hocr_bounding_box(self): - if self._hocr_bounding_box is None: - self._hocr_bounding_box = _get_hocr_bounding_box( - element_with_layout=self.documentai_object, - page_dimension=self.documentai_object.dimension, - ) - return self._hocr_bounding_box + return _get_hocr_bounding_box( + element_with_layout=self.documentai_object, + page_dimension=self.documentai_object.dimension, + ) From 7495e0ed73635bebee92141529974f261bd3a8dd Mon Sep 17 00:00:00 2001 From: Holt Skinner <13262395+holtskinner@users.noreply.github.com> Date: Thu, 13 Jun 2024 09:14:11 -0500 Subject: [PATCH 13/14] fix: Change `if` condition typo in `_get_children_of_element()` (#313) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous `if` condition was inefficient and should have been written with an `and` This resulted in a performance gap. cProfile timing of `export_hocr_string()` on the same document Before ``` 143598720 function calls (129111346 primitive calls) in 44.487 seconds ``` After ``` 97883150 function calls (88084552 primitive calls) in 30.235 seconds ``` Fixes #312 🦕 --- google/cloud/documentai_toolbox/wrappers/page.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/google/cloud/documentai_toolbox/wrappers/page.py b/google/cloud/documentai_toolbox/wrappers/page.py index 1ca84e14..d224d1de 100644 --- a/google/cloud/documentai_toolbox/wrappers/page.py +++ b/google/cloud/documentai_toolbox/wrappers/page.py @@ -359,9 +359,11 @@ def _get_children_of_element( return [ child for child in children - if child.documentai_object.layout.text_anchor.text_segments[0].start_index - >= start_index - if child.documentai_object.layout.text_anchor.text_segments[0].end_index + if start_index + <= child.documentai_object.layout.text_anchor.text_segments[0].start_index + < end_index + and start_index + < child.documentai_object.layout.text_anchor.text_segments[0].end_index <= end_index ] From 369719b1c9c3313da8e5af4b4511ed8c0dc58b16 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 09:45:50 -0500 Subject: [PATCH 14/14] chore(main): release 0.13.4-alpha (#314) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/cloud/documentai_toolbox/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84089243..64e5ba41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.13.4-alpha](https://github.com/googleapis/python-documentai-toolbox/compare/v0.13.3-alpha...v0.13.4-alpha) (2024-06-13) + + +### Bug Fixes + +* Change `if` condition typo in `_get_children_of_element()` ([#313](https://github.com/googleapis/python-documentai-toolbox/issues/313)) ([7495e0e](https://github.com/googleapis/python-documentai-toolbox/commit/7495e0ed73635bebee92141529974f261bd3a8dd)) + ## [0.13.3-alpha](https://github.com/googleapis/python-documentai-toolbox/compare/v0.13.2-alpha...v0.13.3-alpha) (2024-03-11) diff --git a/google/cloud/documentai_toolbox/version.py b/google/cloud/documentai_toolbox/version.py index b2dd4331..7e2ef9e9 100644 --- a/google/cloud/documentai_toolbox/version.py +++ b/google/cloud/documentai_toolbox/version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "0.13.3-alpha" +__version__ = "0.13.4-alpha"